From 91681c8393ba5ac74b49885d7f2c6ad4b33e97ca Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 25 Jul 2023 10:54:40 +0200 Subject: [PATCH 01/90] Adopt the new HashDB API --- Cargo.toml | 2 - hash-db/src/lib.rs | 143 +---- memory-db/benches/bench.rs | 2 +- memory-db/src/lib.rs | 169 +---- test-support/reference-trie/Cargo.toml | 3 +- test-support/reference-trie/src/lib.rs | 167 ++--- test-support/reference-trie/src/substrate.rs | 26 +- .../reference-trie/src/substrate_like.rs | 26 +- test-support/trie-bench/src/lib.rs | 15 +- trie-db/Cargo.toml | 1 + trie-db/src/fatdb.rs | 187 ------ trie-db/src/fatdbmut.rs | 116 ---- trie-db/src/iter_build.rs | 46 +- trie-db/src/iterator.rs | 81 ++- trie-db/src/lib.rs | 233 +------ trie-db/src/lookup.rs | 118 ++-- trie-db/src/nibble/nibblevec.rs | 6 + trie-db/src/node.rs | 227 ++++--- trie-db/src/node_codec.rs | 22 +- trie-db/src/proof/generate.rs | 155 +++-- trie-db/src/proof/verify.rs | 26 +- trie-db/src/recorder.rs | 4 +- trie-db/src/sectriedb.rs | 97 --- trie-db/src/sectriedbmut.rs | 96 --- trie-db/src/trie_codec.rs | 60 +- trie-db/src/triedb.rs | 63 +- trie-db/src/triedbmut.rs | 581 ++++++++++-------- trie-db/test/benches/bench.rs | 4 +- trie-db/test/src/fatdb.rs | 37 -- trie-db/test/src/fatdbmut.rs | 47 -- trie-db/test/src/iter_build.rs | 37 +- trie-db/test/src/iterator.rs | 19 +- trie-db/test/src/lib.rs | 8 - trie-db/test/src/proof.rs | 15 +- trie-db/test/src/recorder.rs | 26 +- trie-db/test/src/sectriedb.rs | 30 - trie-db/test/src/sectriedbmut.rs | 30 - trie-db/test/src/trie_codec.rs | 35 +- trie-db/test/src/triedb.rs | 181 +++--- trie-db/test/src/triedbmut.rs | 407 ++++++------ trie-eip1186/CHANGELOG.md | 8 - trie-eip1186/Cargo.toml | 19 - trie-eip1186/src/eip1186.rs | 311 ---------- trie-eip1186/src/lib.rs | 30 - trie-eip1186/test/Cargo.toml | 15 - trie-eip1186/test/src/eip1186.rs | 181 ------ trie-eip1186/test/src/lib.rs | 18 - 47 files changed, 1291 insertions(+), 2839 deletions(-) delete mode 100644 trie-db/src/fatdb.rs delete mode 100644 trie-db/src/fatdbmut.rs delete mode 100644 trie-db/src/sectriedb.rs delete mode 100644 trie-db/src/sectriedbmut.rs delete mode 100644 trie-db/test/src/fatdb.rs delete mode 100644 trie-db/test/src/fatdbmut.rs delete mode 100644 trie-db/test/src/sectriedb.rs delete mode 100644 trie-db/test/src/sectriedbmut.rs delete mode 100644 trie-eip1186/CHANGELOG.md delete mode 100644 trie-eip1186/Cargo.toml delete mode 100644 trie-eip1186/src/eip1186.rs delete mode 100644 trie-eip1186/src/lib.rs delete mode 100644 trie-eip1186/test/Cargo.toml delete mode 100644 trie-eip1186/test/src/eip1186.rs delete mode 100644 trie-eip1186/test/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 607558fa..c2a1e083 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,8 +9,6 @@ members = [ "test-support/trie-bench", "trie-db", "trie-db/test", - "trie-eip1186", - "trie-eip1186/test", "trie-root", "trie-root/test" ] diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 4825aada..19f0ac86 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -16,6 +16,11 @@ #![cfg_attr(not(feature = "std"), no_std)] +#[cfg(not(feature = "std"))] +extern crate alloc; + +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; #[cfg(not(feature = "std"))] use core::hash; #[cfg(feature = "std")] @@ -73,144 +78,18 @@ pub trait Hasher: Sync + Send { fn hash(x: &[u8]) -> Self::Out; } -/// Trait modelling a plain datastore whose key is a fixed type. -/// The caller should ensure that a key only corresponds to -/// one value. -pub trait PlainDB: Send + Sync + AsPlainDB { - /// Look up a given hash into the bytes that hash to it, returning None if the - /// hash is not known. - fn get(&self, key: &K) -> Option; - - /// Check for the existence of a hash-key. - fn contains(&self, key: &K) -> bool; - - /// Insert a datum item into the DB. Insertions are counted and the equivalent - /// number of `remove()`s must be performed before the data is considered dead. - /// The caller should ensure that a key only corresponds to one value. - fn emplace(&mut self, key: K, value: V); - - /// Remove a datum previously inserted. Insertions can be "owed" such that the - /// same number of `insert()`s may happen without the data being eventually - /// being inserted into the DB. It can be "owed" more than once. - /// The caller should ensure that a key only corresponds to one value. - fn remove(&mut self, key: &K); -} - -/// Trait for immutable reference of PlainDB. -pub trait PlainDBRef { - /// Look up a given hash into the bytes that hash to it, returning None if the - /// hash is not known. - fn get(&self, key: &K) -> Option; - - /// Check for the existance of a hash-key. - fn contains(&self, key: &K) -> bool; -} - -impl<'a, K, V> PlainDBRef for &'a dyn PlainDB { - fn get(&self, key: &K) -> Option { - PlainDB::get(*self, key) - } - fn contains(&self, key: &K) -> bool { - PlainDB::contains(*self, key) - } -} - -impl<'a, K, V> PlainDBRef for &'a mut dyn PlainDB { - fn get(&self, key: &K) -> Option { - PlainDB::get(*self, key) - } - fn contains(&self, key: &K) -> bool { - PlainDB::contains(*self, key) - } -} - /// Trait modelling datastore keyed by a hash defined by the `Hasher`. -pub trait HashDB: Send + Sync + AsHashDB { +pub trait HashDB: Send + Sync { /// Look up a given hash into the bytes that hash to it, returning None if the /// hash is not known. - fn get(&self, key: &H::Out, prefix: Prefix) -> Option; + fn get(&self, key: &H::Out, prefix: Prefix, location: C) -> Option<(T, Vec)>; /// Check for the existence of a hash-key. - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool; - - /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions - /// are counted and the equivalent number of `remove()`s must be performed before the data - /// is considered dead. - fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out; - - /// Like `insert()`, except you provide the key and the data is all moved. - fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T); - - /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of - /// `insert()`s may happen without the data being eventually being inserted into the DB. - /// It can be "owed" more than once. - fn remove(&mut self, key: &H::Out, prefix: Prefix); -} - -/// Trait for immutable reference of HashDB. -pub trait HashDBRef { - /// Look up a given hash into the bytes that hash to it, returning None if the - /// hash is not known. - fn get(&self, key: &H::Out, prefix: Prefix) -> Option; - - /// Check for the existance of a hash-key. - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool; -} - -impl<'a, H: Hasher, T> HashDBRef for &'a dyn HashDB { - fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - HashDB::get(*self, key, prefix) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - HashDB::contains(*self, key, prefix) + fn contains(&self, key: &H::Out, prefix: Prefix, location: C) -> bool { + self.get(key, prefix, location).is_some() } -} -impl<'a, H: Hasher, T> HashDBRef for &'a mut dyn HashDB { - fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - HashDB::get(*self, key, prefix) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - HashDB::contains(*self, key, prefix) - } -} - -/// Upcast trait for HashDB. -pub trait AsHashDB { - /// Perform upcast to HashDB for anything that derives from HashDB. - fn as_hash_db(&self) -> &dyn HashDB; - /// Perform mutable upcast to HashDB for anything that derives from HashDB. - fn as_hash_db_mut<'a>(&'a mut self) -> &'a mut (dyn HashDB + 'a); -} - -/// Upcast trait for PlainDB. -pub trait AsPlainDB { - /// Perform upcast to PlainDB for anything that derives from PlainDB. - fn as_plain_db(&self) -> &dyn PlainDB; - /// Perform mutable upcast to PlainDB for anything that derives from PlainDB. - fn as_plain_db_mut<'a>(&'a mut self) -> &'a mut (dyn PlainDB + 'a); -} - -// NOTE: There used to be a `impl AsHashDB for T` but that does not work with generics. -// See https://stackoverflow.com/questions/48432842/ -// implementing-a-trait-for-reference-and-non-reference-types-causes-conflicting-im -// This means we need concrete impls of AsHashDB in several places, which somewhat defeats -// the point of the trait. -impl<'a, H: Hasher, T> AsHashDB for &'a mut dyn HashDB { - fn as_hash_db(&self) -> &dyn HashDB { - &**self - } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { - &mut **self - } -} - -#[cfg(feature = "std")] -impl<'a, K, V> AsPlainDB for &'a mut dyn PlainDB { - fn as_plain_db(&self) -> &dyn PlainDB { - &**self - } - fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn PlainDB + 'b) { - &mut **self + fn hash(&self, value: &[u8]) -> H::Out { + H::hash(value) } } diff --git a/memory-db/benches/bench.rs b/memory-db/benches/bench.rs index b3e0fd9a..455d0a30 100644 --- a/memory-db/benches/bench.rs +++ b/memory-db/benches/bench.rs @@ -13,7 +13,7 @@ // limitations under the License. use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; +use hash_db::{Hasher, EMPTY_PREFIX}; use keccak_hasher::KeccakHasher; use memory_db::{HashKey, MemoryDB}; diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index 9aff2c54..3bcd93d7 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -20,12 +20,12 @@ extern crate alloc; use hash_db::{ - AsHashDB, AsPlainDB, HashDB, HashDBRef, Hasher as KeyHasher, MaybeDebug, PlainDB, PlainDBRef, + HashDB, Hasher as KeyHasher, MaybeDebug, Prefix, }; #[cfg(feature = "std")] use std::{ - borrow::Borrow, cmp::Eq, collections::hash_map::Entry, collections::HashMap as Map, hash, + cmp::Eq, collections::hash_map::Entry, collections::HashMap as Map, hash, marker::PhantomData, mem, }; @@ -33,7 +33,7 @@ use std::{ use alloc::collections::btree_map::{BTreeMap as Map, Entry}; #[cfg(not(feature = "std"))] -use core::{borrow::Borrow, cmp::Eq, hash, marker::PhantomData, mem}; +use core::{cmp::Eq, hash, marker::PhantomData, mem}; #[cfg(not(feature = "std"))] use alloc::vec::Vec; @@ -47,7 +47,7 @@ use alloc::vec::Vec; /// /// # Example /// ```rust -/// use hash_db::{Hasher, HashDB, EMPTY_PREFIX}; +/// use hash_db::{Hasher, EMPTY_PREFIX}; /// use keccak_hasher::KeccakHasher; /// use memory_db::{MemoryDB, HashKey}; /// @@ -114,7 +114,7 @@ where T: Eq + MaybeDebug, { fn eq(&self, other: &MemoryDB) -> bool { - for a in self.data.iter() { + for a in self.data.iter().filter(|(_, (_, rc))| *rc > 0) { match other.data.get(a.0) { Some(v) if v != a.1 => return false, None => return false, @@ -201,43 +201,6 @@ pub fn prefixed_key(key: &H::Out, prefix: Prefix) -> Vec { prefixed_key } -/// Key function that concatenates prefix and hash. -/// This is doing useless computation and should only be -/// used for legacy purpose. -/// It shall be remove in the future. -#[derive(Clone, Debug)] -#[deprecated(since = "0.22.0")] -pub struct LegacyPrefixedKey(PhantomData); - -#[allow(deprecated)] -impl KeyFunction for LegacyPrefixedKey { - type Key = Vec; - - fn key(hash: &H::Out, prefix: Prefix) -> Vec { - legacy_prefixed_key::(hash, prefix) - } -} - -/// Legacy method for db using previous version of prefix encoding. -/// Only for trie radix 16 trie. -#[deprecated(since = "0.22.0")] -pub fn legacy_prefixed_key(key: &H::Out, prefix: Prefix) -> Vec { - let mut prefixed_key = Vec::with_capacity(key.as_ref().len() + prefix.0.len() + 1); - if let Some(last) = prefix.1 { - let mut prev = 0x01u8; - for i in prefix.0.iter() { - prefixed_key.push((prev << 4) + (*i >> 4)); - prev = *i; - } - prefixed_key.push((prev << 4) + (last >> 4)); - } else { - prefixed_key.push(0); - prefixed_key.extend_from_slice(prefix.0); - } - prefixed_key.extend_from_slice(key.as_ref()); - prefixed_key -} - impl Default for MemoryDB where H: KeyHasher, @@ -396,78 +359,28 @@ where } } -impl PlainDB for MemoryDB +impl HashDB for MemoryDB where H: KeyHasher, - T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, - KF: Send + Sync + KeyFunction, - KF::Key: Borrow<[u8]> + for<'a> From<&'a [u8]>, + T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, + KF: KeyFunction + Send + Sync, { - fn get(&self, key: &H::Out) -> Option { - match self.data.get(key.as_ref()) { - Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), - _ => None, - } - } - - fn contains(&self, key: &H::Out) -> bool { - match self.data.get(key.as_ref()) { - Some(&(_, x)) if x > 0 => true, - _ => false, - } - } - - fn emplace(&mut self, key: H::Out, value: T) { - match self.data.entry(key.as_ref().into()) { - Entry::Occupied(mut entry) => { - let &mut (ref mut old_value, ref mut rc) = entry.get_mut(); - if *rc <= 0 { - *old_value = value; - } - *rc += 1; - }, - Entry::Vacant(entry) => { - entry.insert((value, 1)); - }, - } - } - - fn remove(&mut self, key: &H::Out) { - match self.data.entry(key.as_ref().into()) { - Entry::Occupied(mut entry) => { - let &mut (_, ref mut rc) = entry.get_mut(); - *rc -= 1; - }, - Entry::Vacant(entry) => { - let value = T::default(); - entry.insert((value, -1)); - }, - } + fn get(&self, key: &H::Out, prefix: Prefix, _location: L) -> Option<(T, Vec)> { + MemoryDB::get(self, key, prefix).map(|d| (d, Default::default())) } -} -impl PlainDBRef for MemoryDB -where - H: KeyHasher, - T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, - KF: Send + Sync + KeyFunction, - KF::Key: Borrow<[u8]> + for<'a> From<&'a [u8]>, -{ - fn get(&self, key: &H::Out) -> Option { - PlainDB::get(self, key) - } - fn contains(&self, key: &H::Out) -> bool { - PlainDB::contains(self, key) + fn contains(&self, key: &H::Out, prefix: Prefix, _location: L) -> bool { + MemoryDB::contains(self, key, prefix) } } -impl HashDB for MemoryDB +impl MemoryDB where H: KeyHasher, T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, KF: KeyFunction + Send + Sync, { - fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + pub fn get(&self, key: &H::Out, prefix: Prefix) -> Option { if key == &self.hashed_null_node { return Some(self.null_node_data.clone()) } @@ -479,7 +392,7 @@ where } } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + pub fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { if key == &self.hashed_null_node { return true } @@ -491,7 +404,7 @@ where } } - fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T) { + pub fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T) { if value == self.null_node_data { return } @@ -511,17 +424,17 @@ where } } - fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { + pub fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { if T::from(value) == self.null_node_data { return self.hashed_null_node } let key = H::hash(value); - HashDB::emplace(self, key, prefix, value.into()); + self.emplace(key, prefix, value.into()); key } - fn remove(&mut self, key: &H::Out, prefix: Prefix) { + pub fn remove(&mut self, key: &H::Out, prefix: Prefix) { if key == &self.hashed_null_node { return } @@ -538,54 +451,12 @@ where }, } } -} - -impl HashDBRef for MemoryDB -where - H: KeyHasher, - T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, - KF: KeyFunction + Send + Sync, -{ - fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - HashDB::get(self, key, prefix) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - HashDB::contains(self, key, prefix) - } -} - -impl AsPlainDB for MemoryDB -where - H: KeyHasher, - T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, - KF: KeyFunction + Send + Sync, - KF::Key: Borrow<[u8]> + for<'a> From<&'a [u8]>, -{ - fn as_plain_db(&self) -> &dyn PlainDB { - self - } - fn as_plain_db_mut(&mut self) -> &mut dyn PlainDB { - self - } -} -impl AsHashDB for MemoryDB -where - H: KeyHasher, - T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, - KF: KeyFunction + Send + Sync, -{ - fn as_hash_db(&self) -> &dyn HashDB { - self - } - fn as_hash_db_mut(&mut self) -> &mut dyn HashDB { - self - } } #[cfg(test)] mod tests { - use super::{HashDB, HashKey, KeyHasher, MemoryDB}; + use super::{HashKey, KeyHasher, MemoryDB}; use hash_db::EMPTY_PREFIX; use keccak_hasher::KeccakHasher; diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index 90fe577a..3e58492b 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -8,7 +8,8 @@ license = "Apache-2.0" edition = "2018" [dependencies] -hash-db = { path = "../../hash-db" , version = "0.16.0"} +hash-db = { path = "../../hash-db", version = "0.16.0"} +memory-db = { path = "../../memory-db", default-features = false, version = "0.32.0"} keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } trie-db = { path = "../../trie-db", default-features = false, version = "0.27.0" } trie-root = { path = "../../trie-root", default-features = false, version = "0.18.0" } diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index be626801..3b048898 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -15,6 +15,7 @@ //! Reference implementation of a streamer. use hashbrown::{hash_map::Entry, HashMap}; +use memory_db::MemoryDB; use parity_scale_codec::{Compact, Decode, Encode, Error as CodecError, Input, Output}; use std::{borrow::Borrow, fmt, iter::once, marker::PhantomData, ops::Range}; use trie_db::{ @@ -23,7 +24,7 @@ use trie_db::{ trie_visit, triedbmut::ChildReference, DBValue, NodeCodec, Trie, TrieBuilder, TrieConfiguration, TrieDBBuilder, TrieDBMutBuilder, - TrieHash, TrieLayout, TrieMut, TrieRoot, + TrieHash, TrieLayout, TrieRoot, }; pub use trie_root::TrieStream; use trie_root::{Hasher, Value as TrieStreamValue}; @@ -101,6 +102,7 @@ impl TrieLayout for ExtensionLayout { const MAX_INLINE_VALUE: Option = None; type Hash = RefHasher; type Codec = ReferenceNodeCodec; + type Location = (); } impl TrieConfiguration for ExtensionLayout {} @@ -127,6 +129,7 @@ impl TrieLayout for GenericNoExtensionLayout { const MAX_INLINE_VALUE: Option = None; type Hash = H; type Codec = ReferenceNodeCodecNoExt; + type Location = (); } /// Trie that allows empty values. @@ -139,6 +142,7 @@ impl TrieLayout for AllowEmptyLayout { const MAX_INLINE_VALUE: Option = None; type Hash = RefHasher; type Codec = ReferenceNodeCodec; + type Location = (); } impl TrieConfiguration for GenericNoExtensionLayout {} @@ -184,10 +188,6 @@ pub type RefTrieDBMutAllowEmpty<'a> = trie_db::TrieDBMut<'a, AllowEmptyLayout>; pub type RefTrieDBMutAllowEmptyBuilder<'a> = trie_db::TrieDBMutBuilder<'a, AllowEmptyLayout>; pub type RefTestTrieDBCache = TestTrieCache; pub type RefTestTrieDBCacheNoExt = TestTrieCache; -pub type RefFatDB<'a, 'cache> = trie_db::FatDB<'a, 'cache, ExtensionLayout>; -pub type RefFatDBMut<'a> = trie_db::FatDBMut<'a, ExtensionLayout>; -pub type RefSecTrieDB<'a, 'cache> = trie_db::SecTrieDB<'a, 'cache, ExtensionLayout>; -pub type RefSecTrieDBMut<'a> = trie_db::SecTrieDBMut<'a, ExtensionLayout>; pub type RefLookup<'a, 'cache, Q> = trie_db::Lookup<'a, 'cache, ExtensionLayout, Q>; pub type RefLookupNoExt<'a, 'cache, Q> = trie_db::Lookup<'a, 'cache, NoExtensionLayout, Q>; @@ -679,7 +679,7 @@ impl NodeCodec for ReferenceNodeCodec { &[EMPTY_TRIE] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { let mut output = partial_from_iterator_to_key(partial, number_nibble, LEAF_NODE_OFFSET, LEAF_NODE_OVER); match value { @@ -692,10 +692,10 @@ impl NodeCodec for ReferenceNodeCodec { output } - fn extension_node( + fn extension_node( partial: impl Iterator, number_nibble: usize, - child: ChildReference, + child: ChildReference, ) -> Vec { let mut output = partial_from_iterator_to_key( partial, @@ -704,16 +704,16 @@ impl NodeCodec for ReferenceNodeCodec { EXTENSION_NODE_OVER, ); match child { - ChildReference::Hash(h) => h.as_ref().encode_to(&mut output), + ChildReference::Hash(h, _) => h.as_ref().encode_to(&mut output), ChildReference::Inline(inline_data, len) => (&AsRef::<[u8]>::as_ref(&inline_data)[..len]).encode_to(&mut output), }; output } - fn branch_node( - children: impl Iterator>>>, - maybe_value: Option, + fn branch_node( + children: impl Iterator>>>, + maybe_value: Option>, ) -> Vec { let mut output = vec![0; BITMAP_LENGTH + 1]; let mut prefix: [u8; 3] = [0; 3]; @@ -727,7 +727,7 @@ impl NodeCodec for ReferenceNodeCodec { _ => unimplemented!("unsupported"), }; let has_children = children.map(|maybe_child| match maybe_child.borrow() { - Some(ChildReference::Hash(h)) => { + Some(ChildReference::Hash(h, _)) => { h.as_ref().encode_to(&mut output); true }, @@ -742,11 +742,11 @@ impl NodeCodec for ReferenceNodeCodec { output } - fn branch_node_nibbled( + fn branch_node_nibbled( _partial: impl Iterator, _number_nibble: usize, - _children: impl Iterator>>>, - _maybe_value: Option, + _children: impl Iterator>>>, + _maybe_value: Option>, ) -> Vec { unreachable!("codec with extension branch") } @@ -835,7 +835,7 @@ impl NodeCodec for ReferenceNodeCodecNoExt { &[EMPTY_TRIE_NO_EXT] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { let mut output = partial_from_iterator_encode(partial, number_nibble, NodeKindNoExt::Leaf); match value { Value::Inline(value) => { @@ -847,26 +847,26 @@ impl NodeCodec for ReferenceNodeCodecNoExt { output } - fn extension_node( + fn extension_node( _partial: impl Iterator, _nbnibble: usize, - _child: ChildReference<::Out>, + _child: ChildReference<::Out, L>, ) -> Vec { unreachable!("no extension codec") } - fn branch_node( - _children: impl Iterator::Out>>>>, - _maybe_value: Option, + fn branch_node( + _children: impl Iterator::Out, L>>>>, + _maybe_value: Option>, ) -> Vec { unreachable!("no extension codec") } - fn branch_node_nibbled( + fn branch_node_nibbled( partial: impl Iterator, number_nibble: usize, - children: impl Iterator>>>, - maybe_value: Option, + children: impl Iterator>>>, + maybe_value: Option>, ) -> Vec { let mut output = if maybe_value.is_none() { partial_from_iterator_encode(partial, number_nibble, NodeKindNoExt::BranchNoValue) @@ -887,7 +887,7 @@ impl NodeCodec for ReferenceNodeCodecNoExt { Bitmap::encode( children.map(|maybe_child| match maybe_child.borrow() { - Some(ChildReference::Hash(h)) => { + Some(ChildReference::Hash(h, _)) => { h.as_ref().encode_to(&mut output); true }, @@ -906,33 +906,34 @@ impl NodeCodec for ReferenceNodeCodecNoExt { } /// Compare trie builder and in memory trie. -pub fn compare_implementations(data: Vec<(Vec, Vec)>, mut memdb: DB, mut hashdb: DB) +pub fn compare_implementations(data: Vec<(Vec, Vec)>) where T: TrieLayout, - DB: hash_db::HashDB + Eq, + T::Location: std::fmt::Debug, + K: memory_db::KeyFunction + Send + Sync, { - let root_new = calc_root_build::(data.clone(), &mut hashdb); + let (mut mem_db1, _) = MemoryDB::::default_with_root(); + let (mut mem_db2, _) = MemoryDB::::default_with_root(); + let root_new = calc_root_build::(data.clone(), &mut mem_db1); let root = { - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut mem_db2).build(); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); } - t.commit(); - *t.root() + *t.commit().root.hash() }; if root_new != root { { - let db: &dyn hash_db::HashDB<_, _> = &hashdb; - let t = TrieDBBuilder::::new(&db, &root_new).build(); + let db: &dyn hash_db::HashDB<_, _, _> = &mem_db1; + let t = TrieDBBuilder::::new(db, &root_new).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:x?}", a); } } { - let db: &dyn hash_db::HashDB<_, _> = &memdb; - let t = TrieDBBuilder::::new(&db, &root).build(); + let db: &dyn hash_db::HashDB<_, _, _> = &mem_db2; + let t = TrieDBBuilder::::new(db, &root).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:x?}", a); @@ -942,22 +943,21 @@ where assert_eq!(root, root_new); // compare db content for key fuzzing - assert!(memdb == hashdb); + assert!(mem_db2 == mem_db2); } /// Compare trie builder and trie root implementations. -pub fn compare_root>( +pub fn compare_root>( data: Vec<(Vec, Vec)>, - mut memdb: DB, + memdb: DB, ) { let root_new = reference_trie_root_iter_build::(data.clone()); let root = { - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); } - *t.root() + *t.commit().root.hash() }; assert_eq!(root, root_new); @@ -1002,57 +1002,57 @@ where } /// Trie builder trie building utility. -pub fn calc_root_build(data: I, hashdb: &mut DB) -> ::Out +pub fn calc_root_build(data: I, memdb: &mut memory_db::MemoryDB) -> TrieHash where T: TrieLayout, I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB, + K: memory_db::KeyFunction + Send + Sync, { - let mut cb = TrieBuilder::::new(hashdb); + let mut cb = TrieBuilder::::new(memdb); trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or_default() } /// `compare_implementations_no_extension` for unordered input (trie_root does /// ordering before running when trie_build expect correct ordering). -pub fn compare_implementations_unordered( - data: Vec<(Vec, Vec)>, - mut memdb: DB, - mut hashdb: DB, -) where + +pub fn compare_implementations_unordered(data: Vec<(Vec, Vec)>) +where T: TrieLayout, - DB: hash_db::HashDB + Eq, + T::Location: std::fmt::Debug, + K: memory_db::KeyFunction + Send + Sync, { + let mut mem_db1 = MemoryDB::::default(); + let mut mem_db2 = MemoryDB::::default(); let mut b_map = std::collections::btree_map::BTreeMap::new(); let root = { - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut mem_db1).build(); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); b_map.insert(data[i].0.clone(), data[i].1.clone()); } - *t.root() + *t.commit().root.hash() }; let root_new = { - let mut cb = TrieBuilder::::new(&mut hashdb); + let mut cb = TrieBuilder::::new(&mut mem_db2); trie_visit::(b_map.into_iter(), &mut cb); cb.root.unwrap_or_default() }; if root != root_new { { - let db: &dyn hash_db::HashDB<_, _> = &memdb; - let t = TrieDBBuilder::::new(&db, &root).build(); + let db: &dyn hash_db::HashDB<_, _, _> = &mem_db1; + let t = TrieDBBuilder::::new(db, &root).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:?}", a); } } { - let db: &dyn hash_db::HashDB<_, _> = &hashdb; - let t = TrieDBBuilder::::new(&db, &root_new).build(); + let db: &dyn hash_db::HashDB<_, _, _> = &mem_db2; + let t = TrieDBBuilder::::new(db, &root_new).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:?}", a); @@ -1065,24 +1065,23 @@ pub fn compare_implementations_unordered( /// Testing utility that uses some periodic removal over /// its input test data. -pub fn compare_insert_remove>( +pub fn compare_insert_remove( data: Vec<(bool, Vec, Vec)>, - mut memdb: DB, ) where T: TrieLayout, - DB: hash_db::HashDB + Eq, + K: memory_db::KeyFunction + Send + Sync, { let mut data2 = std::collections::BTreeMap::new(); - let mut root = Default::default(); let mut a = 0; - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.commit(); - } + let mut memdb = MemoryDB::::default(); + let mut root = { + let t = TrieDBMutBuilder::::new(&memdb).build(); + *t.commit().root.hash() + }; while a < data.len() { // new triemut every 3 element root = { - let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::from_existing(&memdb, root).build(); for _ in 0..3 { if data[a].0 { // remove @@ -1099,14 +1098,12 @@ pub fn compare_insert_remove>( break } } - t.commit(); - *t.root() + t.commit().apply_to(&mut memdb) }; } - let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); // we are testing the RefTrie code here so we do not sort or check uniqueness // before. - assert_eq!(*t.root(), calc_root::(data2)); + assert_eq!(root, calc_root::(data2)); } /// Example trie cache implementation. @@ -1114,8 +1111,8 @@ pub fn compare_insert_remove>( /// Should not be used for anything in production. pub struct TestTrieCache { /// In a real implementation we need to make sure that this is unique per trie root. - value_cache: HashMap, trie_db::CachedValue>>, - node_cache: HashMap, NodeOwned>>, + value_cache: HashMap, trie_db::CachedValue, L::Location>>, + node_cache: HashMap, NodeOwned, L::Location>>, } impl TestTrieCache { @@ -1136,24 +1133,25 @@ impl Default for TestTrieCache { } } -impl trie_db::TrieCache for TestTrieCache { - fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&trie_db::CachedValue>> { +impl trie_db::TrieCache for TestTrieCache { + fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&trie_db::CachedValue, L::Location>> { self.value_cache.get(key) } - fn cache_value_for_key(&mut self, key: &[u8], value: trie_db::CachedValue>) { + fn cache_value_for_key(&mut self, key: &[u8], value: trie_db::CachedValue, L::Location>) { self.value_cache.insert(key.to_vec(), value); } fn get_or_insert_node( &mut self, hash: TrieHash, + _location: L::Location, fetch_node: &mut dyn FnMut() -> trie_db::Result< - NodeOwned>, + NodeOwned, L::Location>, TrieHash, trie_db::CError, >, - ) -> trie_db::Result<&NodeOwned>, TrieHash, trie_db::CError> { + ) -> trie_db::Result<&NodeOwned, L::Location>, TrieHash, trie_db::CError> { match self.node_cache.entry(hash) { Entry::Occupied(e) => Ok(e.into_mut()), Entry::Vacant(e) => { @@ -1163,9 +1161,12 @@ impl trie_db::TrieCache for TestTrieCache { } } - fn get_node(&mut self, hash: &TrieHash) -> Option<&NodeOwned>> { + fn get_node(&mut self, hash: &TrieHash, _location: L::Location) -> Option<&NodeOwned, L::Location>> { self.node_cache.get(hash) } + + fn insert_new_node(&mut self, _hash: &TrieHash) { + } } #[cfg(test)] @@ -1203,9 +1204,9 @@ mod tests { let enc = as NodeCodec>::leaf_node( input.iter().cloned(), input.len() * NIBBLE_PER_BYTE, - Value::Inline(&[1]), + Value::<()>::Inline(&[1]), ); - let dec = as NodeCodec>::decode(&enc).unwrap(); + let dec = as NodeCodec>::decode(&enc, &[] as &[()]).unwrap(); let o_sl = if let Node::Leaf(sl, _) = dec { Some(sl) } else { None }; assert!(o_sl.is_some()); } diff --git a/test-support/reference-trie/src/substrate.rs b/test-support/reference-trie/src/substrate.rs index 9b1573f1..7e5c5d04 100644 --- a/test-support/reference-trie/src/substrate.rs +++ b/test-support/reference-trie/src/substrate.rs @@ -310,7 +310,7 @@ where &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { let contains_hash = matches!(&value, Value::Node(..)); let mut output = if contains_hash { partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueLeaf) @@ -322,7 +322,7 @@ where Compact(value.len() as u32).encode_to(&mut output); output.extend_from_slice(value); }, - Value::Node(hash) => { + Value::Node(hash, _) => { debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, @@ -330,26 +330,26 @@ where output } - fn extension_node( + fn extension_node( _partial: impl Iterator, _nbnibble: usize, - _child: ChildReference<::Out>, + _child: ChildReference<::Out, L>, ) -> Vec { unreachable!("No extension codec.") } - fn branch_node( - _children: impl Iterator::Out>>>>, - _maybe_value: Option, + fn branch_node( + _children: impl Iterator::Out, L>>>>, + _maybe_value: Option>, ) -> Vec { unreachable!("No extension codec.") } - fn branch_node_nibbled( + fn branch_node_nibbled( partial: impl Iterator, number_nibble: usize, - children: impl Iterator::Out>>>>, - value: Option, + children: impl Iterator::Out, L>>>>, + value: Option>, ) -> Vec { let contains_hash = matches!(&value, Some(Value::Node(..))); let mut output = match (&value, contains_hash) { @@ -369,7 +369,7 @@ where Compact(value.len() as u32).encode_to(&mut output); output.extend_from_slice(value); }, - Some(Value::Node(hash)) => { + Some(Value::Node(hash, _)) => { debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, @@ -377,7 +377,7 @@ where } Bitmap::encode( children.map(|maybe_child| match maybe_child.borrow() { - Some(ChildReference::Hash(h)) => { + Some(ChildReference::Hash(h, _)) => { h.as_ref().encode_to(&mut output); true }, @@ -470,6 +470,7 @@ where type Hash = H; type Codec = NodeCodec; + type Location = (); } impl TrieConfiguration for LayoutV0 @@ -512,6 +513,7 @@ where type Hash = H; type Codec = NodeCodec; + type Location = (); } impl TrieConfiguration for LayoutV1 diff --git a/test-support/reference-trie/src/substrate_like.rs b/test-support/reference-trie/src/substrate_like.rs index 37cccefc..46bd23da 100644 --- a/test-support/reference-trie/src/substrate_like.rs +++ b/test-support/reference-trie/src/substrate_like.rs @@ -31,6 +31,7 @@ impl TrieLayout for HashedValueNoExt { type Hash = RefHasher; type Codec = ReferenceNodeCodecNoExtMeta; + type Location = (); } impl TrieLayout for HashedValueNoExtThreshold { @@ -40,6 +41,7 @@ impl TrieLayout for HashedValueNoExtThreshold { type Hash = RefHasher; type Codec = ReferenceNodeCodecNoExtMeta; + type Location = (); } /// Constants specific to encoding with external value node support. @@ -169,7 +171,7 @@ where &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { let contains_hash = matches!(&value, Value::Node(..)); let mut output = if contains_hash { partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueLeaf) @@ -181,7 +183,7 @@ where Compact(value.len() as u32).encode_to(&mut output); output.extend_from_slice(value); }, - Value::Node(hash) => { + Value::Node(hash, _) => { debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, @@ -189,26 +191,26 @@ where output } - fn extension_node( + fn extension_node( _partial: impl Iterator, _nbnibble: usize, - _child: ChildReference<::Out>, + _child: ChildReference<::Out, L>, ) -> Vec { unreachable!("Codec without extension.") } - fn branch_node( - _children: impl Iterator::Out>>>>, - _maybe_value: Option, + fn branch_node( + _children: impl Iterator::Out, L>>>>, + _maybe_value: Option>, ) -> Vec { unreachable!("Codec without extension.") } - fn branch_node_nibbled( + fn branch_node_nibbled( partial: impl Iterator, number_nibble: usize, - children: impl Iterator::Out>>>>, - value: Option, + children: impl Iterator::Out, L>>>>, + value: Option>, ) -> Vec { let contains_hash = matches!(&value, Some(Value::Node(..))); let mut output = match (&value, contains_hash) { @@ -228,7 +230,7 @@ where Compact(value.len() as u32).encode_to(&mut output); output.extend_from_slice(value); }, - Some(Value::Node(hash)) => { + Some(Value::Node(hash, _)) => { debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, @@ -236,7 +238,7 @@ where } Bitmap::encode( children.map(|maybe_child| match maybe_child.borrow() { - Some(ChildReference::Hash(h)) => { + Some(ChildReference::Hash(h, _)) => { h.as_ref().encode_to(&mut output); true }, diff --git a/test-support/trie-bench/src/lib.rs b/test-support/trie-bench/src/lib.rs index afa45849..2ab7c578 100644 --- a/test-support/trie-bench/src/lib.rs +++ b/test-support/trie-bench/src/lib.rs @@ -20,7 +20,7 @@ use keccak_hasher::KeccakHasher; use memory_db::{HashKey, MemoryDB}; use parity_scale_codec::{Compact, Encode}; use std::default::Default; -use trie_db::{NodeCodec, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieHash, TrieLayout, TrieMut}; +use trie_db::{NodeCodec, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout}; use trie_root::{trie_root, TrieStream}; use trie_standardmap::*; @@ -60,8 +60,7 @@ fn benchmark( |b, d: &TrieInsertionList| { b.iter(&mut || { let mut memdb = MemoryDB::<_, HashKey, _>::new(L::Codec::empty_node()); - let mut root = >::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for i in d.0.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -73,13 +72,15 @@ fn benchmark( bench_list, |b, d: &TrieInsertionList| { let mut memdb = MemoryDB::<_, HashKey<_>, _>::new(L::Codec::empty_node()); - let mut root = >::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let commit = { + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for i in d.0.iter() { t.insert(&i.0, &i.1).unwrap(); } - } + t.commit() + }; + let root = commit.root.hash(); + commit.apply_to(&mut memdb); b.iter(&mut || { let t = TrieDBBuilder::::new(&memdb, &root).build(); for n in t.iter().unwrap() { diff --git a/trie-db/Cargo.toml b/trie-db/Cargo.toml index f8e6b00c..a8ff1afb 100644 --- a/trie-db/Cargo.toml +++ b/trie-db/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" log = "0.4" smallvec = { version = "1.0.0", features = ["union", "const_new"] } hash-db = { path = "../hash-db", default-features = false, version = "0.16.0"} +memory-db = { path = "../memory-db", default-features = false, version = "0.32.0"} hashbrown = { version = "0.13.2", default-features = false, features = ["ahash"] } rustc-hex = { version = "2.1.0", default-features = false, optional = true } diff --git a/trie-db/src/fatdb.rs b/trie-db/src/fatdb.rs deleted file mode 100644 index 16883394..00000000 --- a/trie-db/src/fatdb.rs +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2017, 2021 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::{ - CError, DBValue, Query, Result, Trie, TrieDB, TrieDBIterator, TrieDBKeyIterator, TrieHash, - TrieItem, TrieIterator, TrieKeyItem, TrieLayout, -}; -use hash_db::{HashDBRef, Hasher}; - -use crate::{rstd::boxed::Box, TrieDBBuilder}; - -/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. -/// Additionaly it stores inserted hash-key mappings for later retrieval. -/// -/// Use it as a `Trie` or `TrieMut` trait object. -pub struct FatDB<'db, 'cache, L> -where - L: TrieLayout, -{ - raw: TrieDB<'db, 'cache, L>, -} - -impl<'db, 'cache, L> FatDB<'db, 'cache, L> -where - L: TrieLayout, -{ - /// Create a new trie with the backing database `db` and empty `root` - /// Initialise to the state entailed by the genesis block. - /// This guarantees the trie is built correctly. - pub fn new(db: &'db dyn HashDBRef, root: &'db TrieHash) -> Self { - FatDB { raw: TrieDBBuilder::new(db, root).build() } - } - - /// Get the backing database. - pub fn db(&self) -> &dyn HashDBRef { - self.raw.db() - } -} - -impl<'db, 'cache, L> Trie for FatDB<'db, 'cache, L> -where - L: TrieLayout, -{ - fn root(&self) -> &TrieHash { - self.raw.root() - } - - fn contains(&self, key: &[u8]) -> Result, CError> { - self.raw.contains(L::Hash::hash(key).as_ref()) - } - - fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { - self.raw.get_hash(key) - } - - fn get_with>( - &self, - key: &[u8], - query: Q, - ) -> Result, TrieHash, CError> { - self.raw.get_with(L::Hash::hash(key).as_ref(), query) - } - - fn iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - FatDBIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) - } - - fn key_iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - FatDBKeyIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) - } -} - -/// Iterator over inserted pairs of key values. -pub struct FatDBIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - trie_iterator: TrieDBIterator<'db, 'cache, L>, - trie: &'db TrieDB<'db, 'cache, L>, -} - -impl<'db, 'cache, L> FatDBIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - /// Creates new iterator. - pub fn new(trie: &'db TrieDB<'db, 'cache, L>) -> Result, CError> { - Ok(FatDBIterator { trie_iterator: TrieDBIterator::new(trie)?, trie }) - } -} - -impl<'db, 'cache, L> TrieIterator for FatDBIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { - let hashed_key = L::Hash::hash(key); - self.trie_iterator.seek(hashed_key.as_ref()) - } -} - -impl<'db, 'cache, L> Iterator for FatDBIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - type Item = TrieItem, CError>; - - fn next(&mut self) -> Option { - self.trie_iterator.next().map(|res| { - res.map(|(hash, value)| { - let aux_hash = L::Hash::hash(&hash); - ( - self.trie.db().get(&aux_hash, Default::default()).expect("Missing fatdb hash"), - value, - ) - }) - }) - } -} - -/// Iterator over inserted keys. -pub struct FatDBKeyIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - trie_iterator: TrieDBKeyIterator<'db, 'cache, L>, - trie: &'db TrieDB<'db, 'cache, L>, -} - -impl<'db, 'cache, L> FatDBKeyIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - /// Creates new iterator. - pub fn new(trie: &'db TrieDB<'db, 'cache, L>) -> Result, CError> { - Ok(FatDBKeyIterator { trie_iterator: TrieDBKeyIterator::new(trie)?, trie }) - } -} - -impl<'db, 'cache, L> TrieIterator for FatDBKeyIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { - let hashed_key = L::Hash::hash(key); - self.trie_iterator.seek(hashed_key.as_ref()) - } -} - -impl<'db, 'cache, L> Iterator for FatDBKeyIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - type Item = TrieKeyItem, CError>; - - fn next(&mut self) -> Option { - self.trie_iterator.next().map(|res| { - res.map(|hash| { - let aux_hash = L::Hash::hash(&hash); - self.trie.db().get(&aux_hash, Default::default()).expect("Missing fatdb hash") - }) - }) - } -} diff --git a/trie-db/src/fatdbmut.rs b/trie-db/src/fatdbmut.rs deleted file mode 100644 index fa7a8f07..00000000 --- a/trie-db/src/fatdbmut.rs +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{ - triedbmut::{TrieDBMutBuilder, Value}, - CError, DBValue, Result, TrieDBMut, TrieHash, TrieLayout, TrieMut, -}; -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; - -/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. -/// Additionaly it stores inserted hash-key mappings for later retrieval. -/// -/// Use it as a `Trie` or `TrieMut` trait object. -pub struct FatDBMut<'db, L> -where - L: TrieLayout, -{ - raw: TrieDBMut<'db, L>, -} - -impl<'db, L> FatDBMut<'db, L> -where - L: TrieLayout, -{ - /// Create a new trie with the backing database `db` and empty `root` - /// Initialise to the state entailed by the genesis block. - /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut dyn HashDB, root: &'db mut TrieHash) -> Self { - FatDBMut { raw: TrieDBMutBuilder::new(db, root).build() } - } - - /// Create a new trie with the backing database `db` and `root`. - /// - /// Returns an error if root does not exist. - pub fn from_existing( - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, - ) -> Self { - FatDBMut { raw: TrieDBMutBuilder::from_existing(db, root).build() } - } - - /// Get the backing database. - pub fn db(&self) -> &dyn HashDB { - self.raw.db() - } - - /// Get the backing database. - pub fn db_mut(&mut self) -> &mut dyn HashDB { - self.raw.db_mut() - } -} - -impl<'db, L> TrieMut for FatDBMut<'db, L> -where - L: TrieLayout, -{ - fn root(&mut self) -> &TrieHash { - self.raw.root() - } - - fn is_empty(&self) -> bool { - self.raw.is_empty() - } - - fn contains(&self, key: &[u8]) -> Result, CError> { - self.raw.contains(L::Hash::hash(key).as_ref()) - } - - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, TrieHash, CError> - where - 'a: 'key, - { - self.raw.get(L::Hash::hash(key).as_ref()) - } - - fn insert( - &mut self, - key: &[u8], - value: &[u8], - ) -> Result>, TrieHash, CError> { - let hash = L::Hash::hash(key); - let out = self.raw.insert(hash.as_ref(), value)?; - let db = self.raw.db_mut(); - - // insert if it doesn't exist. - if out.is_none() { - let aux_hash = L::Hash::hash(hash.as_ref()); - db.emplace(aux_hash, EMPTY_PREFIX, key.to_vec()); - } - Ok(out) - } - - fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { - let hash = L::Hash::hash(key); - let out = self.raw.remove(hash.as_ref())?; - - // remove if it already exists. - if out.is_some() { - let aux_hash = L::Hash::hash(hash.as_ref()); - self.raw.db_mut().remove(&aux_hash, EMPTY_PREFIX); - } - - Ok(out) - } -} diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index c843c3f1..73fbc4aa 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -25,7 +25,8 @@ use crate::{ triedbmut::ChildReference, DBValue, TrieHash, TrieLayout, }; -use hash_db::{HashDB, Hasher, Prefix}; +use hash_db::{Hasher, Prefix}; +use memory_db::MemoryDB; macro_rules! exponential_out { (@3, [$($inpp:expr),*]) => { exponential_out!(@2, [$($inpp,)* $($inpp),*]) }; @@ -33,7 +34,7 @@ macro_rules! exponential_out { (@1, [$($inpp:expr),*]) => { [$($inpp,)* $($inpp),*] }; } -type CacheNode = Option>; +type CacheNode = Option>; #[inline(always)] fn new_vec_slice_buffer() -> [CacheNode; 16] { @@ -134,7 +135,7 @@ where value } else { hashed = callback.process_inner_hashed_value((k2.as_ref(), None), v2.as_ref()); - Value::Node(hashed.as_ref()) + Value::Node(hashed.as_ref(), ()) }; let encoded = T::Codec::leaf_node(nkey.right_iter(), nkey.len(), value); let hash = callback.process(pr.left(), encoded, false); @@ -184,7 +185,7 @@ where branch_d: usize, is_root: bool, nkey: Option<(usize, usize)>, - ) -> ChildReference> { + ) -> ChildReference, ()> { let last = self.0.len() - 1; assert_eq!(self.0[last].2, branch_d); @@ -201,7 +202,7 @@ where let mut prefix = NibbleSlice::new_offset(&key_branch, 0); prefix.advance(branch_d); hashed = callback.process_inner_hashed_value(prefix.left(), v.as_ref()); - Value::Node(hashed.as_ref()) + Value::Node(hashed.as_ref(), ()) }) } else { None @@ -229,7 +230,7 @@ where branch_d: usize, is_root: bool, nkey: Option<(usize, usize)>, - ) -> ChildReference> { + ) -> ChildReference, ()> { let (children, v, depth) = self.0.pop().expect("checked"); debug_assert!(branch_d == depth); @@ -244,7 +245,7 @@ where let mut prefix = NibbleSlice::new_offset(&key_branch, 0); prefix.advance(branch_d); hashed = callback.process_inner_hashed_value(prefix.left(), v.as_ref()); - Value::Node(hashed.as_ref()) + Value::Node(hashed.as_ref(), ()) }) } else { None @@ -318,7 +319,7 @@ where value } else { hashed = callback.process_inner_hashed_value((k2.as_ref(), None), v2.as_ref()); - Value::Node(hashed.as_ref()) + Value::Node(hashed.as_ref(), ()) }; let encoded = T::Codec::leaf_node(nkey.right_iter(), nkey.len(), value); @@ -348,7 +349,7 @@ pub trait ProcessEncodedNode { prefix: Prefix, encoded_node: Vec, is_root: bool, - ) -> ChildReference; + ) -> ChildReference; /// Callback for hashed value in encoded node. fn process_inner_hashed_value(&mut self, prefix: Prefix, value: &[u8]) -> HO; @@ -357,28 +358,27 @@ pub trait ProcessEncodedNode { /// Get trie root and insert visited node in a hash_db. /// As for all `ProcessEncodedNode` implementation, it /// is only for full trie parsing (not existing trie). -pub struct TrieBuilder<'a, T: TrieLayout, DB> { - db: &'a mut DB, +pub struct TrieBuilder<'a, T: TrieLayout, K: memory_db::KeyFunction + Send + Sync> { + db: &'a mut MemoryDB, pub root: Option>, } -impl<'a, T: TrieLayout, DB> TrieBuilder<'a, T, DB> { - pub fn new(db: &'a mut DB) -> Self { +impl<'a, T: TrieLayout, K: memory_db::KeyFunction + Send + Sync> TrieBuilder<'a, T, K> { + pub fn new(db: &'a mut MemoryDB) -> Self { TrieBuilder { db, root: None } } } -impl<'a, T, DB> ProcessEncodedNode> for TrieBuilder<'a, T, DB> +impl<'a, T, K: memory_db::KeyFunction + Send + Sync> ProcessEncodedNode> for TrieBuilder<'a, T, K> where T: TrieLayout, - DB: HashDB, { fn process( &mut self, prefix: Prefix, encoded_node: Vec, is_root: bool, - ) -> ChildReference> { + ) -> ChildReference, ()> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { let mut h = <::Out as Default>::default(); @@ -390,7 +390,7 @@ where if is_root { self.root = Some(hash); }; - ChildReference::Hash(hash) + ChildReference::Hash(hash, ()) } fn process_inner_hashed_value(&mut self, prefix: Prefix, value: &[u8]) -> TrieHash { @@ -416,7 +416,7 @@ impl ProcessEncodedNode> for TrieRoot { _: Prefix, encoded_node: Vec, is_root: bool, - ) -> ChildReference> { + ) -> ChildReference, ()> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { let mut h = <::Out as Default>::default(); @@ -428,7 +428,7 @@ impl ProcessEncodedNode> for TrieRoot { if is_root { self.root = Some(hash); }; - ChildReference::Hash(hash) + ChildReference::Hash(hash, ()) } fn process_inner_hashed_value(&mut self, _prefix: Prefix, value: &[u8]) -> TrieHash { @@ -472,7 +472,7 @@ impl ProcessEncodedNode> for TrieRootPrint { p: Prefix, encoded_node: Vec, is_root: bool, - ) -> ChildReference> { + ) -> ChildReference, ()> { println!("Encoded node: {:x?}", &encoded_node); println!(" with prefix: {:x?}", &p); let len = encoded_node.len(); @@ -488,7 +488,7 @@ impl ProcessEncodedNode> for TrieRootPrint { self.root = Some(hash); }; println!(" hashed to {:x?}", hash.as_ref()); - ChildReference::Hash(hash) + ChildReference::Hash(hash, ()) } fn process_inner_hashed_value(&mut self, _prefix: Prefix, value: &[u8]) -> TrieHash { @@ -503,7 +503,7 @@ impl ProcessEncodedNode> for TrieRootUnhashed { _: Prefix, encoded_node: Vec, is_root: bool, - ) -> ChildReference<::Out> { + ) -> ChildReference<::Out, ()> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { let mut h = <::Out as Default>::default(); @@ -516,7 +516,7 @@ impl ProcessEncodedNode> for TrieRootUnhashed { if is_root { self.root = Some(encoded_node); }; - ChildReference::Hash(hash) + ChildReference::Hash(hash, ()) } fn process_inner_hashed_value(&mut self, _prefix: Prefix, value: &[u8]) -> TrieHash { diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index ca382b70..5539c8cf 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -34,13 +34,13 @@ enum Status { #[cfg_attr(feature = "std", derive(Debug))] #[derive(Eq, PartialEq)] -struct Crumb { +struct Crumb { hash: Option, - node: Arc>, + node: Arc>, status: Status, } -impl Crumb { +impl Crumb { /// Move on to next status in the node's sequence. fn increment(&mut self) { self.status = match (self.status, self.node.node_plan()) { @@ -60,7 +60,7 @@ impl Crumb { /// Iterator for going through all nodes in the trie in pre-order traversal order. pub struct TrieDBRawIterator { - trail: Vec>, + trail: Vec>, key_nibbles: NibbleVec, } @@ -76,7 +76,7 @@ impl TrieDBRawIterator { TrieDBRawIterator { trail: Vec::with_capacity(8), key_nibbles: NibbleVec::new() }; let (root_node, root_hash) = db.get_raw_or_lookup( *db.root(), - NodeHandle::Hash(db.root().as_ref()), + NodeHandle::Hash(db.root().as_ref(), Default::default()), EMPTY_PREFIX, true, )?; @@ -106,7 +106,7 @@ impl TrieDBRawIterator { } /// Descend into a payload. - fn descend(&mut self, node: OwnedNode, node_hash: Option>) { + fn descend(&mut self, node: OwnedNode, node_hash: Option>) { self.trail .push(Crumb { hash: node_hash, status: Status::Entering, node: Arc::new(node) }); } @@ -116,10 +116,11 @@ impl TrieDBRawIterator { db: &TrieDB, key: &[u8], prefix: Prefix, + location: L::Location, ) -> Result, CError> { let mut res = TrieHash::::default(); res.as_mut().copy_from_slice(key); - db.fetch_value(res, prefix) + db.fetch_value(res, prefix, location) } /// Seek a node position at 'key' for iterator. @@ -139,7 +140,7 @@ impl TrieDBRawIterator { let (mut node, mut node_hash) = db.get_raw_or_lookup( >::default(), - NodeHandle::Hash(db.root().as_ref()), + NodeHandle::Hash(db.root().as_ref(), Default::default()), EMPTY_PREFIX, true, )?; @@ -152,19 +153,17 @@ impl TrieDBRawIterator { "descend_into_node pushes a crumb onto the trial; \ thus the trail is non-empty; qed", ); - let node_data = crumb.node.data(); + let node = crumb.node.node(); - match crumb.node.node_plan() { - NodePlan::Leaf { partial: partial_plan, .. } => { - let slice = partial_plan.build(node_data); + match node { + Node::Leaf(slice, _) => { if slice < partial { crumb.status = Status::Exiting; return Ok(false) } return Ok(slice.starts_with(&partial)) }, - NodePlan::Extension { partial: partial_plan, child } => { - let slice = partial_plan.build(node_data); + Node::Extension(slice, child) => { if !partial.starts_with(&slice) { if slice < partial { crumb.status = Status::Exiting; @@ -182,12 +181,12 @@ impl TrieDBRawIterator { let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - child.build(node_data), + child, prefix.left(), true, )? }, - NodePlan::Branch { value: _, children } => { + Node::Branch(children, _value) => { if partial.is_empty() { return Ok(true) } @@ -203,7 +202,7 @@ impl TrieDBRawIterator { let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - child.build(node_data), + *child, prefix.left(), true, )? @@ -211,8 +210,7 @@ impl TrieDBRawIterator { return Ok(false) } }, - NodePlan::NibbledBranch { partial: partial_plan, value: _, children } => { - let slice = partial_plan.build(node_data); + Node::NibbledBranch(slice, children, _value) => { if !partial.starts_with(&slice) { if slice < partial { crumb.status = Status::Exiting; @@ -242,7 +240,7 @@ impl TrieDBRawIterator { let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - child.build(node_data), + *child, prefix.left(), true, )? @@ -250,7 +248,7 @@ impl TrieDBRawIterator { return Ok(false) } }, - NodePlan::Empty => { + Node::Empty => { if !partial.is_empty() { crumb.status = Status::Exiting; return Ok(false) @@ -354,16 +352,16 @@ impl TrieDBRawIterator { db: &TrieDB, ) -> Option< Result< - (&NibbleVec, Option<&TrieHash>, &Arc>), + (&NibbleVec, Option<&TrieHash>, &Arc>), TrieHash, CError, >, > { loop { let crumb = self.trail.last_mut()?; - let node_data = crumb.node.data(); + let node = crumb.node.node(); - match (crumb.status, crumb.node.node_plan()) { + match (crumb.status, node) { (Status::Entering, _) => { // This is only necessary due to current borrow checker's limitation. let crumb = self.trail.last_mut().expect("we've just fetched the last element using `last_mut` so this cannot fail; qed"); @@ -372,27 +370,26 @@ impl TrieDBRawIterator { }, (Status::Exiting, node) => { match node { - NodePlan::Empty | NodePlan::Leaf { .. } => {}, - NodePlan::Extension { partial, .. } => { + Node::Empty | Node::Leaf { .. } => {}, + Node::Extension(partial, ..) => { self.key_nibbles.drop_lasts(partial.len()); }, - NodePlan::Branch { .. } => { + Node::Branch { .. } => { self.key_nibbles.pop(); }, - NodePlan::NibbledBranch { partial, .. } => { + Node::NibbledBranch(partial, ..) => { self.key_nibbles.drop_lasts(partial.len() + 1); }, } self.trail.pop().expect("we've just fetched the last element using `last_mut` so this cannot fail; qed"); self.trail.last_mut()?.increment(); }, - (Status::At, NodePlan::Extension { partial: partial_plan, child }) => { - let partial = partial_plan.build(node_data); + (Status::At, Node::Extension(partial, child)) => { self.key_nibbles.append_partial(partial.right()); match db.get_raw_or_lookup( crumb.hash.unwrap_or_default(), - child.build(node_data), + child, self.key_nibbles.as_prefix(), true, ) { @@ -405,25 +402,24 @@ impl TrieDBRawIterator { }, } }, - (Status::At, NodePlan::Branch { .. }) => { + (Status::At, Node::Branch { .. }) => { self.key_nibbles.push(0); crumb.increment(); }, - (Status::At, NodePlan::NibbledBranch { partial: partial_plan, .. }) => { - let partial = partial_plan.build(node_data); + (Status::At, Node::NibbledBranch(partial, ..)) => { self.key_nibbles.append_partial(partial.right()); self.key_nibbles.push(0); crumb.increment(); }, - (Status::AtChild(i), NodePlan::Branch { children, .. }) | - (Status::AtChild(i), NodePlan::NibbledBranch { children, .. }) => { + (Status::AtChild(i), Node::Branch(children, ..)) | + (Status::AtChild(i), Node::NibbledBranch(_, children, ..)) => { if let Some(child) = &children[i] { self.key_nibbles.pop(); self.key_nibbles.push(i as u8); match db.get_raw_or_lookup( crumb.hash.unwrap_or_default(), - child.build(node_data), + *child, self.key_nibbles.as_prefix(), true, ) { @@ -484,7 +480,7 @@ impl TrieDBRawIterator { } let value = match value { - Value::Node(hash) => match Self::fetch_value(db, &hash, (key_slice, None)) { + Value::Node(hash, location) => match Self::fetch_value(db, &hash, (key_slice, None), location) { Ok(value) => value, Err(err) => return Some(Err(err)), }, @@ -508,7 +504,7 @@ impl TrieDBRawIterator { let mut prefix = prefix.clone(); match node.node() { - Node::Leaf(partial, _) => { + Node::Leaf(partial, _) => { prefix.append_partial(partial.right()); }, Node::Branch(_, value) => @@ -563,8 +559,9 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeIterator<'a, 'cache, L> { &self, key: &[u8], prefix: Prefix, + location: L::Location, ) -> Result, CError> { - TrieDBRawIterator::fetch_value(self.db, key, prefix) + TrieDBRawIterator::fetch_value(self.db, key, prefix, location) } /// Advance the iterator into a prefix, no value out of the prefix will be accessed @@ -584,7 +581,7 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeIterator<'a, 'cache, L> { } /// Access inner hash db. - pub fn db(&self) -> &dyn hash_db::HashDBRef { + pub fn db(&self) -> &dyn hash_db::HashDB { self.db.db() } } @@ -597,7 +594,7 @@ impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBNodeIterator<'a, 'cach impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBNodeIterator<'a, 'cache, L> { type Item = - Result<(NibbleVec, Option>, Arc>), TrieHash, CError>; + Result<(NibbleVec, Option>, Arc>), TrieHash, CError>; fn next(&mut self) -> Option { self.raw_iter.next_raw_item(self.db).map(|result| { diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index b09372b2..d149e5b0 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -44,13 +44,9 @@ use node::NodeOwned; pub mod node; pub mod proof; pub mod recorder; -pub mod sectriedb; -pub mod sectriedbmut; pub mod triedb; pub mod triedbmut; -mod fatdb; -mod fatdbmut; mod iter_build; mod iterator; mod lookup; @@ -59,15 +55,14 @@ mod node_codec; mod trie_codec; pub use self::{ - fatdb::{FatDB, FatDBIterator}, - fatdbmut::FatDBMut, lookup::Lookup, nibble::{nibble_ops, NibbleSlice, NibbleVec}, recorder::Recorder, - sectriedb::SecTrieDB, - sectriedbmut::SecTrieDBMut, triedb::{TrieDB, TrieDBBuilder, TrieDBIterator, TrieDBKeyIterator}, - triedbmut::{ChildReference, TrieDBMut, TrieDBMutBuilder, Value}, + triedbmut::{ + ChildReference, TrieDBMut, TrieDBMutBuilder, Value, + Changeset, ChangesetNodeRef, NewChangesetNode, ExistingChangesetNode + }, }; pub use crate::{ iter_build::{trie_visit, ProcessEncodedNode, TrieBuilder, TrieRoot, TrieRootUnhashed}, @@ -75,7 +70,7 @@ pub use crate::{ node_codec::{NodeCodec, Partial}, trie_codec::{decode_compact, decode_compact_from_iter, encode_compact}, }; -pub use hash_db::{HashDB, HashDBRef, Hasher}; +pub use hash_db::{HashDB, Hasher}; #[cfg(feature = "std")] pub use crate::iter_build::TrieRootPrint; @@ -162,9 +157,9 @@ pub trait Query { /// If a cache is used, [`Self::Key`] and [`Self::NodeOwned`] are possible /// values. Otherwise only [`Self::EncodedNode`] is a possible value. #[cfg_attr(feature = "std", derive(Debug))] -pub enum TrieAccess<'a, H> { +pub enum TrieAccess<'a, H, L> { /// The given [`NodeOwned`] was accessed using its `hash`. - NodeOwned { hash: H, node_owned: &'a NodeOwned }, + NodeOwned { hash: H, node_owned: &'a NodeOwned }, /// The given `encoded_node` was accessed using its `hash`. EncodedNode { hash: H, encoded_node: rstd::borrow::Cow<'a, [u8]> }, /// The given `value` was accessed using its `hash`. @@ -223,12 +218,12 @@ impl RecordedForKey { /// /// To build a trie proof a recorder is required that records all trie accesses. These recorded trie /// accesses can then be used to create the proof. -pub trait TrieRecorder { +pub trait TrieRecorder { /// Record the given [`TrieAccess`]. /// /// Depending on the [`TrieAccess`] a call of [`Self::trie_nodes_recorded_for_key`] afterwards /// must return the correct recorded state. - fn record<'a>(&mut self, access: TrieAccess<'a, H>); + fn record<'a>(&mut self, access: TrieAccess<'a, H, L>); /// Check if we have recorded any trie nodes for the given `key`. /// @@ -296,37 +291,6 @@ pub trait Trie { >; } -/// A key-value datastore implemented as a database-backed modified Merkle tree. -pub trait TrieMut { - /// Return the root of the trie. - fn root(&mut self) -> &TrieHash; - - /// Is the trie empty? - fn is_empty(&self) -> bool; - - /// Does the trie contain a given key? - fn contains(&self, key: &[u8]) -> Result, CError> { - self.get(key).map(|x| x.is_some()) - } - - /// What is the value of the given key in this trie? - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, TrieHash, CError> - where - 'a: 'key; - - /// Insert a `key`/`value` pair into the trie. An empty value is equivalent to removing - /// `key` from the trie. Returns the old value associated with this key, if it existed. - fn insert( - &mut self, - key: &[u8], - value: &[u8], - ) -> Result>, TrieHash, CError>; - - /// Remove a `key` from the trie. Equivalent to making it equal to the empty - /// value. Returns the old value associated with this key, if it existed. - fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError>; -} - /// A trie iterator that also supports random access (`seek()`). pub trait TrieIterator: Iterator { /// Position the iterator on the first element with key >= `key` @@ -339,140 +303,11 @@ pub trait TrieIterator: Iterator { pub enum TrieSpec { /// Generic trie. Generic, - /// Secure trie. - Secure, - /// Secure trie with fat database. - Fat, } impl Default for TrieSpec { fn default() -> TrieSpec { - TrieSpec::Secure - } -} - -/// Trie factory. -#[derive(Default, Clone)] -pub struct TrieFactory { - spec: TrieSpec, -} - -/// All different kinds of tries. -/// This is used to prevent a heap allocation for every created trie. -pub enum TrieKinds<'db, 'cache, L: TrieLayout> { - /// A generic trie db. - Generic(TrieDB<'db, 'cache, L>), - /// A secure trie db. - Secure(SecTrieDB<'db, 'cache, L>), - /// A fat trie db. - Fat(FatDB<'db, 'cache, L>), -} - -// wrapper macro for making the match easier to deal with. -macro_rules! wrapper { - ($me: ident, $f_name: ident, $($param: ident),*) => { - match *$me { - TrieKinds::Generic(ref t) => t.$f_name($($param),*), - TrieKinds::Secure(ref t) => t.$f_name($($param),*), - TrieKinds::Fat(ref t) => t.$f_name($($param),*), - } - } -} - -impl<'db, 'cache, L: TrieLayout> Trie for TrieKinds<'db, 'cache, L> { - fn root(&self) -> &TrieHash { - wrapper!(self, root,) - } - - fn is_empty(&self) -> bool { - wrapper!(self, is_empty,) - } - - fn contains(&self, key: &[u8]) -> Result, CError> { - wrapper!(self, contains, key) - } - - fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { - wrapper!(self, get_hash, key) - } - - fn get_with>( - &self, - key: &[u8], - query: Q, - ) -> Result, TrieHash, CError> { - wrapper!(self, get_with, key, query) - } - - fn iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - wrapper!(self, iter,) - } - - fn key_iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - wrapper!(self, key_iter,) - } -} - -impl TrieFactory { - /// Creates new factory. - pub fn new(spec: TrieSpec) -> Self { - TrieFactory { spec } - } - - /// Create new immutable instance of Trie. - pub fn readonly<'db, 'cache, L: TrieLayout>( - &self, - db: &'db dyn HashDBRef, - root: &'db TrieHash, - ) -> TrieKinds<'db, 'cache, L> { - match self.spec { - TrieSpec::Generic => TrieKinds::Generic(TrieDBBuilder::new(db, root).build()), - TrieSpec::Secure => TrieKinds::Secure(SecTrieDB::new(db, root)), - TrieSpec::Fat => TrieKinds::Fat(FatDB::new(db, root)), - } - } - - /// Create new mutable instance of Trie. - pub fn create<'db, L: TrieLayout + 'db>( - &self, - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, - ) -> Box + 'db> { - match self.spec { - TrieSpec::Generic => Box::new(TrieDBMutBuilder::::new(db, root).build()), - TrieSpec::Secure => Box::new(SecTrieDBMut::::new(db, root)), - TrieSpec::Fat => Box::new(FatDBMut::::new(db, root)), - } - } - - /// Create new mutable instance of trie and check for errors. - pub fn from_existing<'db, L: TrieLayout + 'db>( - &self, - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, - ) -> Box + 'db> { - match self.spec { - TrieSpec::Generic => Box::new(TrieDBMutBuilder::::from_existing(db, root).build()), - TrieSpec::Secure => Box::new(SecTrieDBMut::::from_existing(db, root)), - TrieSpec::Fat => Box::new(FatDBMut::::from_existing(db, root)), - } - } - - /// Returns true iff the trie DB is a fat DB (allows enumeration of keys). - pub fn is_fat(&self) -> bool { - self.spec == TrieSpec::Fat + TrieSpec::Generic } } @@ -494,6 +329,7 @@ pub trait TrieLayout { type Hash: Hasher; /// Codec to use (needs to match hasher and nibble ops). type Codec: NodeCodec::Out>; + type Location: Copy + Default + Eq + PartialEq; } /// This trait associates a trie definition with preferred methods. @@ -501,14 +337,13 @@ pub trait TrieLayout { /// used to allow switching implementation. pub trait TrieConfiguration: Sized + TrieLayout { /// Operation to build a trie db from its ordered iterator over its key/values. - fn trie_build(db: &mut DB, input: I) -> ::Out + fn trie_build(db: &mut memory_db::MemoryDB, DBValue>, input: I) -> ::Out where - DB: HashDB, I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - let mut cb = TrieBuilder::::new(db); + let mut cb = TrieBuilder::>::new(db); trie_visit::(input.into_iter(), &mut cb); cb.root.unwrap_or_default() } @@ -560,11 +395,11 @@ pub type CError = <::Codec as NodeCodec>::Error; /// A value as cached by the [`TrieCache`]. #[derive(Clone, Debug)] -pub enum CachedValue { +pub enum CachedValue { /// The value doesn't exist in the trie. NonExisting, - /// We cached the hash, because we did not yet accessed the data. - ExistingHash(H), + /// We cached the hash and location, because we did not yet accessed the data. + ExistingHash(H, L), /// The value exists in the trie. Existing { /// The hash of the value. @@ -578,7 +413,7 @@ pub enum CachedValue { }, } -impl CachedValue { +impl CachedValue { /// Returns the data of the value. /// /// If a value doesn't exist in the trie or only the value hash is cached, this function returns @@ -596,36 +431,24 @@ impl CachedValue { /// Returns only `None` when the value doesn't exist. pub fn hash(&self) -> Option { match self { - Self::ExistingHash(hash) | Self::Existing { hash, .. } => Some(*hash), + Self::ExistingHash(hash, _) | Self::Existing { hash, .. } => Some(*hash), Self::NonExisting => None, } } } -impl From<(Bytes, H)> for CachedValue { +impl From<(Bytes, H)> for CachedValue { fn from(value: (Bytes, H)) -> Self { Self::Existing { hash: value.1, data: value.0.into() } } } -impl From for CachedValue { - fn from(value: H) -> Self { - Self::ExistingHash(value) - } -} - -impl From> for CachedValue { +impl From> for CachedValue { fn from(value: Option<(Bytes, H)>) -> Self { value.map_or(Self::NonExisting, |v| Self::Existing { hash: v.1, data: v.0.into() }) } } -impl From> for CachedValue { - fn from(value: Option) -> Self { - value.map_or(Self::NonExisting, |v| Self::ExistingHash(v)) - } -} - /// A cache that can be used to speed-up certain operations when accessing the trie. /// /// The [`TrieDB`]/[`TrieDBMut`] by default are working with the internal hash-db in a non-owning @@ -642,7 +465,7 @@ impl From> for CachedValue { /// different values under the same key, it up to the cache implementation to ensure that the /// correct value is returned. As each trie has a different root, this root can be used to /// differentiate values under the same key. -pub trait TrieCache { +pub trait TrieCache { /// Lookup value for the given `key`. /// /// Returns the `None` if the `key` is unknown or otherwise `Some(_)` with the associated @@ -656,7 +479,7 @@ pub trait TrieCache { /// The cache can be used for different tries, aka with different roots. This means /// that the cache implementation needs to take care of always returning the correct value /// for the current trie root. - fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&CachedValue>; + fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&CachedValue>; /// Cache the given `value` for the given `key`. /// @@ -665,7 +488,7 @@ pub trait TrieCache { /// The cache can be used for different tries, aka with different roots. This means /// that the cache implementation needs to take care of caching `value` for the current /// trie root. - fn cache_value_for_key(&mut self, key: &[u8], value: CachedValue); + fn cache_value_for_key(&mut self, key: &[u8], value: CachedValue); /// Get or insert a [`NodeOwned`]. /// @@ -677,11 +500,15 @@ pub trait TrieCache { fn get_or_insert_node( &mut self, hash: NC::HashOut, - fetch_node: &mut dyn FnMut() -> Result, NC::HashOut, NC::Error>, - ) -> Result<&NodeOwned, NC::HashOut, NC::Error>; + location: L, + fetch_node: &mut dyn FnMut() -> Result, NC::HashOut, NC::Error>, + ) -> Result<&NodeOwned, NC::HashOut, NC::Error>; /// Get the [`NodeOwned`] that corresponds to the given `hash`. - fn get_node(&mut self, hash: &NC::HashOut) -> Option<&NodeOwned>; + fn get_node(&mut self, hash: &NC::HashOut, location: L) -> Option<&NodeOwned>; + + /// Put a new node. This is used to clear location info for existing nodes with the same hash. + fn insert_new_node(&mut self, hash: &NC::HashOut); } /// A container for storing bytes. diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs index 5e34055f..73cba979 100644 --- a/trie-db/src/lookup.rs +++ b/trie-db/src/lookup.rs @@ -22,20 +22,20 @@ use crate::{ Bytes, CError, CachedValue, DBValue, Query, RecordedForKey, Result, TrieAccess, TrieCache, TrieError, TrieHash, TrieLayout, TrieRecorder, }; -use hash_db::{HashDBRef, Hasher, Prefix}; +use hash_db::{HashDB, Hasher, Prefix}; /// Trie lookup helper object. pub struct Lookup<'a, 'cache, L: TrieLayout, Q: Query> { /// database to query from. - pub db: &'a dyn HashDBRef, + pub db: &'a dyn HashDB, /// Query object to record nodes and transform data. pub query: Q, /// Hash to start at pub hash: TrieHash, /// Optional cache that should be used to speed up the lookup. - pub cache: Option<&'cache mut dyn TrieCache>, + pub cache: Option<&'cache mut dyn TrieCache>, /// Optional recorder that will be called to record all trie accesses. - pub recorder: Option<&'cache mut dyn TrieRecorder>>, + pub recorder: Option<&'cache mut dyn TrieRecorder, L::Location>>, } impl<'a, 'cache, L, Q> Lookup<'a, 'cache, L, Q> @@ -50,19 +50,19 @@ where /// /// Returns the bytes representing the value. fn load_value( - v: Value, + v: Value, prefix: Prefix, full_key: &[u8], - db: &dyn HashDBRef, - recorder: &mut Option<&mut dyn TrieRecorder>>, + db: &dyn HashDB, + recorder: &mut Option<&mut dyn TrieRecorder, L::Location>>, query: Q, ) -> Result, CError> { match v { Value::Inline(value) => Ok(query.decode(&value)), - Value::Node(hash) => { + Value::Node(hash, location) => { let mut res = TrieHash::::default(); res.as_mut().copy_from_slice(hash); - if let Some(value) = db.get(&res, prefix) { + if let Some((value, _)) = db.get(&res, prefix, location) { if let Some(recorder) = recorder { recorder.record(TrieAccess::Value { hash: res, @@ -86,19 +86,19 @@ where /// /// Returns the bytes representing the value and its hash. fn load_owned_value( - v: ValueOwned>, + v: ValueOwned, L::Location>, prefix: Prefix, full_key: &[u8], - cache: &mut dyn crate::TrieCache, - db: &dyn HashDBRef, - recorder: &mut Option<&mut dyn TrieRecorder>>, + cache: &mut dyn crate::TrieCache, + db: &dyn HashDB, + recorder: &mut Option<&mut dyn TrieRecorder, L::Location>>, ) -> Result<(Bytes, TrieHash), TrieHash, CError> { match v { ValueOwned::Inline(value, hash) => Ok((value.clone(), hash)), - ValueOwned::Node(hash) => { - let node = cache.get_or_insert_node(hash, &mut || { - let value = db - .get(&hash, prefix) + ValueOwned::Node(hash, location) => { + let node = cache.get_or_insert_node(hash, location, &mut || { + let (value, _) = db + .get(&hash, prefix, location) .ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; Ok(NodeOwned::Value(value.into(), hash)) @@ -125,9 +125,10 @@ where } } - fn record<'b>(&mut self, get_access: impl FnOnce() -> TrieAccess<'b, TrieHash>) + fn record<'b>(&mut self, get_access: impl FnOnce() -> TrieAccess<'b, TrieHash, L::Location>) where TrieHash: 'b, + L::Location: 'b, { if let Some(recorder) = self.recorder.as_mut() { recorder.record(get_access()); @@ -144,10 +145,11 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, + location: L::Location, ) -> Result, TrieHash, CError> { match self.cache.take() { - Some(cache) => self.look_up_with_cache(full_key, nibble_key, cache), - None => self.look_up_without_cache(nibble_key, full_key, Self::load_value), + Some(cache) => self.look_up_with_cache(full_key, nibble_key, location, cache), + None => self.look_up_without_cache(nibble_key, location, full_key, Self::load_value), } } @@ -159,16 +161,18 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, + location: L::Location, ) -> Result>, TrieHash, CError> { match self.cache.take() { - Some(cache) => self.look_up_hash_with_cache(full_key, nibble_key, cache), + Some(cache) => self.look_up_hash_with_cache(full_key, nibble_key, location, cache), None => self.look_up_without_cache( nibble_key, + location, full_key, |v, _, full_key, _, recorder, _| { Ok(match v { Value::Inline(v) => L::Hash::hash(&v), - Value::Node(hash_bytes) => { + Value::Node(hash_bytes, _location) => { if let Some(recoder) = recorder.as_mut() { recoder.record(TrieAccess::Hash { full_key }); } @@ -190,7 +194,8 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, - cache: &mut dyn crate::TrieCache, + location: L::Location, + cache: &mut dyn crate::TrieCache, ) -> Result>, TrieHash, CError> { let value_cache_allowed = self .recorder @@ -208,24 +213,25 @@ where } else { let hash_and_value = self.look_up_with_cache_internal( nibble_key, + location, full_key, cache, |value, _, full_key, _, _, recorder| match value { - ValueOwned::Inline(value, hash) => Ok((hash, Some(value.clone()))), - ValueOwned::Node(hash) => { + ValueOwned::Inline(value, hash) => Ok((hash, Some(value.clone()), Default::default())), + ValueOwned::Node(hash, location) => { if let Some(recoder) = recorder.as_mut() { recoder.record(TrieAccess::Hash { full_key }); } - Ok((hash, None)) + Ok((hash, None, location)) }, }, )?; match &hash_and_value { - Some((hash, Some(value))) => + Some((hash, Some(value), _location)) => cache.cache_value_for_key(full_key, (value.clone(), *hash).into()), - Some((hash, None)) => cache.cache_value_for_key(full_key, (*hash).into()), + Some((hash, None, location)) => cache.cache_value_for_key(full_key, CachedValue::ExistingHash(*hash, *location)), None => cache.cache_value_for_key(full_key, CachedValue::NonExisting), } @@ -243,7 +249,8 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, - cache: &mut dyn crate::TrieCache, + location: L::Location, + cache: &mut dyn crate::TrieCache, ) -> Result, TrieHash, CError> { let trie_nodes_recorded = self.recorder.as_ref().map(|r| r.trie_nodes_recorded_for_key(full_key)); @@ -260,10 +267,11 @@ where }; let lookup_data = |lookup: &mut Self, - cache: &mut dyn crate::TrieCache| + cache: &mut dyn crate::TrieCache| -> Result, TrieHash, CError> { let data = lookup.look_up_with_cache_internal( nibble_key, + location, full_key, cache, Self::load_owned_value, @@ -277,11 +285,11 @@ where let res = match value_cache_allowed.then(|| cache.lookup_value_for_key(full_key)).flatten() { Some(CachedValue::NonExisting) => None, - Some(CachedValue::ExistingHash(hash)) => { + Some(CachedValue::ExistingHash(hash, location)) => { let data = Self::load_owned_value( // If we only have the hash cached, this can only be a value node. // For inline nodes we cache them directly as `CachedValue::Existing`. - ValueOwned::Node(*hash), + ValueOwned::Node(*hash, *location), nibble_key.original_data_as_prefix(), full_key, cache, @@ -323,25 +331,28 @@ where fn look_up_with_cache_internal( &mut self, nibble_key: NibbleSlice, + location: L::Location, full_key: &[u8], - cache: &mut dyn crate::TrieCache, + cache: &mut dyn crate::TrieCache, load_value_owned: impl Fn( - ValueOwned>, + ValueOwned, L::Location>, Prefix, &[u8], - &mut dyn crate::TrieCache, - &dyn HashDBRef, - &mut Option<&mut dyn TrieRecorder>>, + &mut dyn crate::TrieCache, + &dyn HashDB, + &mut Option<&mut dyn TrieRecorder, L::Location>>, ) -> Result, CError>, ) -> Result, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; let mut key_nibbles = 0; + let mut location = location; + // this loop iterates through non-inline nodes. for depth in 0.. { - let mut node = cache.get_or_insert_node(hash, &mut || { - let node_data = match self.db.get(&hash, nibble_key.mid(key_nibbles).left()) { + let mut node = cache.get_or_insert_node(hash, location, &mut || { + let (node_data, locations) = match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { Some(value) => value, None => return Err(Box::new(match depth { @@ -350,7 +361,7 @@ where })), }; - let decoded = match L::Codec::decode(&node_data[..]) { + let decoded = match L::Codec::decode(&node_data[..], &locations) { Ok(node) => node, Err(e) => return Err(Box::new(TrieError::DecoderError(hash, e))), }; @@ -367,7 +378,6 @@ where NodeOwned::Leaf(slice, value) => return if partial == *slice { let value = (*value).clone(); - drop(node); load_value_owned( value, nibble_key.original_data_as_prefix(), @@ -395,7 +405,6 @@ where NodeOwned::Branch(children, value) => if partial.is_empty() { return if let Some(value) = value.clone() { - drop(node); load_value_owned( value, nibble_key.original_data_as_prefix(), @@ -433,7 +442,6 @@ where if partial.len() == slice.len() { return if let Some(value) = value.clone() { - drop(node); load_value_owned( value, nibble_key.original_data_as_prefix(), @@ -479,8 +487,9 @@ where // check if new node data is inline or hash. match next_node { - NodeHandleOwned::Hash(new_hash) => { + NodeHandleOwned::Hash(new_hash, new_location) => { hash = *new_hash; + location = *new_location; break }, NodeHandleOwned::Inline(inline_node) => { @@ -501,23 +510,25 @@ where fn look_up_without_cache( mut self, nibble_key: NibbleSlice, + location: L::Location, full_key: &[u8], load_value: impl Fn( - Value, + Value, Prefix, &[u8], - &dyn HashDBRef, - &mut Option<&mut dyn TrieRecorder>>, + &dyn HashDB, + &mut Option<&mut dyn TrieRecorder, L::Location>>, Q, ) -> Result, CError>, ) -> Result, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; let mut key_nibbles = 0; + let mut location = location; // this loop iterates through non-inline nodes. for depth in 0.. { - let node_data = match self.db.get(&hash, nibble_key.mid(key_nibbles).left()) { + let (node_data, locations) = match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { Some(value) => value, None => return Err(Box::new(match depth { @@ -535,7 +546,7 @@ where // without incrementing the depth. let mut node_data = &node_data[..]; loop { - let decoded = match L::Codec::decode(node_data) { + let decoded = match L::Codec::decode(node_data, &locations) { Ok(node) => node, Err(e) => return Err(Box::new(TrieError::DecoderError(hash, e))), }; @@ -585,7 +596,8 @@ where Ok(None) } } else { - match children[partial.at(0) as usize] { + let i = partial.at(0) as usize; + match children[i] { Some(x) => { partial = partial.mid(1); key_nibbles += 1; @@ -622,7 +634,8 @@ where Ok(None) } } else { - match children[partial.at(slice.len()) as usize] { + let i = partial.at(slice.len()) as usize; + match children[i] { Some(x) => { partial = partial.mid(slice.len() + 1); key_nibbles += slice.len() + 1; @@ -645,9 +658,10 @@ where // check if new node data is inline or hash. match next_node { - NodeHandle::Hash(data) => { + NodeHandle::Hash(data, l) => { hash = decode_hash::(data) .ok_or_else(|| Box::new(TrieError::InvalidHash(hash, data.to_vec())))?; + location = l; break }, NodeHandle::Inline(data) => { diff --git a/trie-db/src/nibble/nibblevec.rs b/trie-db/src/nibble/nibblevec.rs index f612585a..5ec8a3a5 100644 --- a/trie-db/src/nibble/nibblevec.rs +++ b/trie-db/src/nibble/nibblevec.rs @@ -117,6 +117,12 @@ impl NibbleVec { } } + /// Get `Prefix` representation of this `NibbleVec`. + pub fn as_owned_prefix(&self) -> (BackingByteVec, Option) { + let (inner, pad) = self.as_prefix(); + (inner.into(), pad) + } + /// Append another `NibbleVec`. Can be slow (alignement of second vec). pub fn append(&mut self, v: &NibbleVec) { if v.len == 0 { diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index 19ed9162..c2e78c58 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -29,22 +29,24 @@ pub type NodeKey = (usize, nibble::BackingByteVec); /// A reference to a trie node which may be stored within another trie node. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum NodeHandle<'a> { - Hash(&'a [u8]), +pub enum NodeHandle<'a, L> { + Hash(&'a [u8], L), Inline(&'a [u8]), } -impl NodeHandle<'_> { +impl<'a, L: Copy + Default> NodeHandle<'a, L> { /// Converts this node handle into a [`NodeHandleOwned`]. - pub fn to_owned_handle( + pub fn to_owned_handle( &self, - ) -> Result>, TrieHash, CError> { + ) -> Result, TL::Location>, TrieHash, CError> + where TL::Location: From, + { match self { - Self::Hash(h) => decode_hash::(h) + Self::Hash(h, l) => decode_hash::(h) .ok_or_else(|| Box::new(TrieError::InvalidHash(Default::default(), h.to_vec()))) - .map(NodeHandleOwned::Hash), - Self::Inline(i) => match L::Codec::decode(i) { - Ok(node) => Ok(NodeHandleOwned::Inline(Box::new(node.to_owned_node::()?))), + .map(|h| NodeHandleOwned::Hash(h, (*l).into())), + Self::Inline(i) => match TL::Codec::decode(i, &[] as &[L]) { + Ok(node) => Ok(NodeHandleOwned::Inline(Box::new(node.to_owned_node::()?))), Err(e) => Err(Box::new(TrieError::DecoderError(Default::default(), e))), }, } @@ -54,14 +56,15 @@ impl NodeHandle<'_> { /// Owned version of [`NodeHandleOwned`]. #[derive(Clone, PartialEq, Eq)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum NodeHandleOwned { - Hash(H), - Inline(Box>), +pub enum NodeHandleOwned { + Hash(H, L), + Inline(Box>), } -impl NodeHandleOwned +impl NodeHandleOwned where H: Default + AsRef<[u8]> + AsMut<[u8]> + Copy, + L: Default + Copy, { /// Returns `self` as a [`ChildReference`]. /// @@ -69,9 +72,9 @@ where /// /// This function panics if `self == Self::Inline(_)` and the inline node encoded length is /// greater then the length of the hash. - fn as_child_reference>(&self) -> ChildReference { + fn as_child_reference>(&self) -> ChildReference { match self { - NodeHandleOwned::Hash(h) => ChildReference::Hash(*h), + NodeHandleOwned::Hash(h, l) => ChildReference::Hash(*h, *l), NodeHandleOwned::Inline(n) => { let encoded = n.to_encoded::(); let mut store = H::default(); @@ -84,11 +87,21 @@ where } } -impl NodeHandleOwned { +impl NodeHandleOwned { + /// Check if location hint is missing + fn missing_location(&self) -> bool { + match self { + NodeHandleOwned::Hash(_, l) => *l == Default::default(), + NodeHandleOwned::Inline(_) => false, + } + } +} + +impl NodeHandleOwned { /// Returns `self` as inline node. - pub fn as_inline(&self) -> Option<&NodeOwned> { + pub fn as_inline(&self) -> Option<&NodeOwned> { match self { - Self::Hash(_) => None, + Self::Hash(_, _) => None, Self::Inline(node) => Some(&*node), } } @@ -107,14 +120,14 @@ pub fn decode_hash(data: &[u8]) -> Option { /// Value representation in `Node`. #[derive(Eq, PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum Value<'a> { +pub enum Value<'a, L> { /// Value byte slice as stored in a trie node. Inline(&'a [u8]), /// Hash byte slice as stored in a trie node. - Node(&'a [u8]), + Node(&'a [u8], L), } -impl<'a> Value<'a> { +impl<'a, L: Copy + Default> Value<'a, L> { pub(crate) fn new_inline(value: &'a [u8], threshold: Option) -> Option { if let Some(threshold) = threshold { if value.len() >= threshold as usize { @@ -127,14 +140,16 @@ impl<'a> Value<'a> { } } - pub fn to_owned_value(&self) -> ValueOwned> { + pub fn to_owned_value(&self) -> ValueOwned, TL::Location> + where TL::Location: From, + { match self { - Self::Inline(data) => ValueOwned::Inline(Bytes::from(*data), L::Hash::hash(data)), - Self::Node(hash) => { - let mut res = TrieHash::::default(); + Self::Inline(data) => ValueOwned::Inline(Bytes::from(*data), TL::Hash::hash(data)), + Self::Node(hash, l) => { + let mut res = TrieHash::::default(); res.as_mut().copy_from_slice(hash); - ValueOwned::Node(res) + ValueOwned::Node(res, (*l).into()) }, } } @@ -143,19 +158,19 @@ impl<'a> Value<'a> { /// Owned value representation in `Node`. #[derive(Eq, PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum ValueOwned { +pub enum ValueOwned { /// Value bytes as stored in a trie node and its hash. Inline(Bytes, H), /// Hash byte slice as stored in a trie node. - Node(H), + Node(H, L), } -impl + Copy> ValueOwned { +impl + Copy, L: Copy + Default> ValueOwned { /// Returns self as [`Value`]. - pub fn as_value(&self) -> Value { + pub fn as_value(&self) -> Value { match self { Self::Inline(data, _) => Value::Inline(&data), - Self::Node(hash) => Value::Node(hash.as_ref()), + Self::Node(hash, location) => Value::Node(hash.as_ref(), *location), } } @@ -163,17 +178,17 @@ impl + Copy> ValueOwned { pub fn data_hash(&self) -> Option { match self { Self::Inline(_, hash) => Some(*hash), - Self::Node(hash) => Some(*hash), + Self::Node(hash, _) => Some(*hash), } } } -impl ValueOwned { +impl ValueOwned { /// Returns the data stored in self. pub fn data(&self) -> Option<&Bytes> { match self { Self::Inline(data, _) => Some(data), - Self::Node(_) => None, + Self::Node(_, _) => None, } } } @@ -181,29 +196,32 @@ impl ValueOwned { /// Type of node in the trie and essential information thereof. #[derive(Eq, PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum Node<'a> { +pub enum Node<'a, L> { /// Null trie node; could be an empty root or an empty branch entry. Empty, /// Leaf node; has key slice and value. Value may not be empty. - Leaf(NibbleSlice<'a>, Value<'a>), + Leaf(NibbleSlice<'a>, Value<'a, L>), /// Extension node; has key slice and node data. Data may not be null. - Extension(NibbleSlice<'a>, NodeHandle<'a>), + Extension(NibbleSlice<'a>, NodeHandle<'a, L>), /// Branch node; has slice of child nodes (each possibly null) /// and an optional immediate node data. - Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), + Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), /// Branch node with support for a nibble (when extension nodes are not used). NibbledBranch( NibbleSlice<'a>, - [Option>; nibble_ops::NIBBLE_LENGTH], - Option>, + [Option>; nibble_ops::NIBBLE_LENGTH], + Option>, ), } -impl Node<'_> { +impl Node<'_, Location> { /// Converts this node into a [`NodeOwned`]. pub fn to_owned_node( &self, - ) -> Result>, TrieHash, CError> { + ) -> Result, L::Location>, TrieHash, CError> + where L::Location: From, + Location: Copy + Default, + { match self { Self::Empty => Ok(NodeOwned::Empty), Self::Leaf(n, d) => Ok(NodeOwned::Leaf((*n).into(), d.to_owned_value::())), @@ -248,21 +266,21 @@ impl Node<'_> { /// Owned version of [`Node`]. #[derive(Eq, PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum NodeOwned { +pub enum NodeOwned { /// Null trie node; could be an empty root or an empty branch entry. Empty, /// Leaf node; has key slice and value. Value may not be empty. - Leaf(NibbleVec, ValueOwned), + Leaf(NibbleVec, ValueOwned), /// Extension node; has key slice and node data. Data may not be null. - Extension(NibbleVec, NodeHandleOwned), + Extension(NibbleVec, NodeHandleOwned), /// Branch node; has slice of child nodes (each possibly null) /// and an optional immediate node data. - Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), + Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), /// Branch node with support for a nibble (when extension nodes are not used). NibbledBranch( NibbleVec, - [Option>; nibble_ops::NIBBLE_LENGTH], - Option>, + [Option>; nibble_ops::NIBBLE_LENGTH], + Option>, ), /// Node that represents a value. /// @@ -271,7 +289,7 @@ pub enum NodeOwned { Value(Bytes, H), } -impl NodeOwned +impl NodeOwned where H: Default + AsRef<[u8]> + AsMut<[u8]> + Copy, { @@ -304,15 +322,15 @@ where } /// Returns an iterator over all existing children with their optional nibble. - pub fn child_iter(&self) -> impl Iterator, &NodeHandleOwned)> { - enum ChildIter<'a, H> { + pub fn child_iter(&self) -> impl Iterator, &NodeHandleOwned)> { + enum ChildIter<'a, H, L> { Empty, - Single(&'a NodeHandleOwned, bool), - Array(&'a [Option>; nibble_ops::NIBBLE_LENGTH], usize), + Single(&'a NodeHandleOwned, bool), + Array(&'a [Option>; nibble_ops::NIBBLE_LENGTH], usize), } - impl<'a, H> Iterator for ChildIter<'a, H> { - type Item = (Option, &'a NodeHandleOwned); + impl<'a, H, L> Iterator for ChildIter<'a, H, L> { + type Item = (Option, &'a NodeHandleOwned); fn next(&mut self) -> Option { loop { @@ -362,7 +380,19 @@ where } } -impl NodeOwned { +impl NodeOwned { + /// Return true if the node may have at least one child node. + pub fn has_missing_children_locations(&self) -> bool { + match self { + Self::Leaf(_, _) | Self::Empty | Self::Value(_, _) => false, + Self::Extension(_, h) => h.missing_location(), + Self::Branch(c, ..) | Self::NibbledBranch(_, c, ..) => c.iter().any(|c| c.as_ref().map_or(false, |c| c.missing_location())), + } + } + +} + +impl NodeOwned { /// Returns the data attached to this node. pub fn data(&self) -> Option<&Bytes> { match &self { @@ -391,8 +421,8 @@ impl NodeOwned { pub fn size_in_bytes(&self) -> usize { let self_size = mem::size_of::(); - fn childs_size<'a, H: 'a>( - childs: impl Iterator>>, + fn childs_size<'a, H: 'a, L: 'a>( + childs: impl Iterator>>, ) -> usize { // If a `child` isn't an inline node, its size is already taken account for by // `self_size`. @@ -439,12 +469,20 @@ impl NodeHandlePlan { /// Build a node handle by decoding a byte slice according to the node handle plan. It is the /// responsibility of the caller to ensure that the node plan was created for the argument /// data, otherwise the call may decode incorrectly or panic. - pub fn build<'a, 'b>(&'a self, data: &'b [u8]) -> NodeHandle<'b> { + pub fn build<'a, 'b, L>(&'a self, data: &'b [u8], location: L) -> NodeHandle<'b, L> { match self { - NodeHandlePlan::Hash(range) => NodeHandle::Hash(&data[range.clone()]), + NodeHandlePlan::Hash(range) => NodeHandle::Hash(&data[range.clone()], location), NodeHandlePlan::Inline(range) => NodeHandle::Inline(&data[range.clone()]), } } + + /// Check if the node is innline. + pub fn is_inline(&self) -> bool { + match self { + NodeHandlePlan::Hash(_) => false, + NodeHandlePlan::Inline(_) => true, + } + } } /// A `NibbleSlicePlan` is a blueprint for decoding a nibble slice from a byte slice. The @@ -488,10 +526,18 @@ pub enum ValuePlan { impl ValuePlan { /// Build a value slice by decoding a byte slice according to the plan. - pub fn build<'a, 'b>(&'a self, data: &'b [u8]) -> Value<'b> { + pub fn build<'a, 'b, L>(&'a self, data: &'b [u8], location: L) -> Value<'b, L> { match self { ValuePlan::Inline(range) => Value::Inline(&data[range.clone()]), - ValuePlan::Node(range) => Value::Node(&data[range.clone()]), + ValuePlan::Node(range) => Value::Node(&data[range.clone()], location), + } + } + + /// Check if the value is inline. + pub fn is_inline(&self) -> bool { + match self { + ValuePlan::Inline(_) => true, + ValuePlan::Node(_) => false, } } } @@ -526,31 +572,51 @@ pub enum NodePlan { } impl NodePlan { - /// Build a node by decoding a byte slice according to the node plan. It is the responsibility - /// of the caller to ensure that the node plan was created for the argument data, otherwise the - /// call may decode incorrectly or panic. - pub fn build<'a, 'b>(&'a self, data: &'b [u8]) -> Node<'b> { + /// Build a node by decoding a byte slice according to the node plan and attaching location dats. + /// It is the responsibility of the caller to ensure that the node plan was created for the + /// argument data, otherwise the call may decode incorrectly or panic. + pub fn build<'a, 'b, L: Copy + Default>(&'a self, data: &'b [u8], locations: &[L]) -> Node<'b, L> { match self { NodePlan::Empty => Node::Empty, - NodePlan::Leaf { partial, value } => Node::Leaf(partial.build(data), value.build(data)), + NodePlan::Leaf { partial, value } => Node::Leaf(partial.build(data), value.build(data, locations.first().copied().unwrap_or_default())), NodePlan::Extension { partial, child } => - Node::Extension(partial.build(data), child.build(data)), + Node::Extension(partial.build(data), child.build(data, locations.first().copied().unwrap_or_default())), NodePlan::Branch { value, children } => { let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; + let mut nc = 0; for i in 0..nibble_ops::NIBBLE_LENGTH { - child_slices[i] = children[i].as_ref().map(|child| child.build(data)); + if let Some(child) = &children[i] { + let location = if child.is_inline() { + Default::default() + } else { + let l = locations.get(nc).copied().unwrap_or_default(); + nc += 1; + l + }; + child_slices[i] = Some(child.build(data, location)); + } } - Node::Branch(child_slices, value.as_ref().map(|v| v.build(data))) + Node::Branch(child_slices, value.as_ref().map(|v| v.build(data, locations.last().copied().unwrap_or_default()))) }, NodePlan::NibbledBranch { partial, value, children } => { let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; + let mut nc = 0; for i in 0..nibble_ops::NIBBLE_LENGTH { - child_slices[i] = children[i].as_ref().map(|child| child.build(data)); + if let Some(child) = &children[i] { + let location = if child.is_inline() { + Default::default() + } else { + let l = locations.get(nc).copied().unwrap_or_default(); + nc += 1; + l + }; + child_slices[i] = Some(child.build(data, location)); + } } Node::NibbledBranch( partial.build(data), child_slices, - value.as_ref().map(|v| v.build(data)), + value.as_ref().map(|v| v.build(data, locations.last().copied().unwrap_or_default())), ) }, } @@ -583,16 +649,17 @@ impl NodePlan { /// the `OwnedNode`. This is useful for trie iterators. #[cfg_attr(feature = "std", derive(Debug))] #[derive(PartialEq, Eq)] -pub struct OwnedNode> { +pub struct OwnedNode, L> { data: D, plan: NodePlan, + locations: Vec, } -impl> OwnedNode { +impl, L: Default + Copy> OwnedNode { /// Construct an `OwnedNode` by decoding an owned data source according to some codec. - pub fn new(data: D) -> core::result::Result { + pub fn new(data: D, locations: Vec) -> core::result::Result { let plan = C::decode_plan(data.borrow())?; - Ok(OwnedNode { data, plan }) + Ok(OwnedNode { data, plan, locations }) } /// Returns a reference to the backing data. @@ -600,6 +667,10 @@ impl> OwnedNode { self.data.borrow() } + /// Returns a reference to children locations. + pub fn locations(&self) -> &[L] { + &self.locations + } /// Returns a reference to the node decode plan. pub fn node_plan(&self) -> &NodePlan { &self.plan @@ -611,7 +682,7 @@ impl> OwnedNode { } /// Construct a `Node` by borrowing data from this struct. - pub fn node(&self) -> Node { - self.plan.build(self.data.borrow()) + pub fn node(&self) -> Node { + self.plan.build(self.data.borrow(), &self.locations) } } diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index eb9b1f67..c0cfede6 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -59,8 +59,8 @@ pub trait NodeCodec: Sized { fn decode_plan(data: &[u8]) -> Result; /// Decode bytes to a `Node`. Returns `Self::E` on failure. - fn decode<'a>(data: &'a [u8]) -> Result, Self::Error> { - Ok(Self::decode_plan(data)?.build(data)) + fn decode<'a, L: Copy + Default>(data: &'a [u8], locations: &[L]) -> Result, Self::Error> { + Ok(Self::decode_plan(data)?.build(data, locations)) } /// Check if the provided bytes correspond to the codecs "empty" node. @@ -74,32 +74,32 @@ pub trait NodeCodec: Sized { /// Note that number_nibble is the number of element of the iterator /// it can possibly be obtain by `Iterator` `size_hint`, but /// for simplicity it is used directly as a parameter. - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec; + fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec; /// Returns an encoded extension node /// /// Note that number_nibble is the number of element of the iterator /// it can possibly be obtain by `Iterator` `size_hint`, but /// for simplicity it is used directly as a parameter. - fn extension_node( + fn extension_node( partial: impl Iterator, number_nibble: usize, - child_ref: ChildReference, + child_ref: ChildReference, ) -> Vec; /// Returns an encoded branch node. /// Takes an iterator yielding `ChildReference` and an optional value. - fn branch_node( - children: impl Iterator>>>, - value: Option, + fn branch_node( + children: impl Iterator>>>, + value: Option>, ) -> Vec; /// Returns an encoded branch node with a possible partial path. /// `number_nibble` is the partial path length as in `extension_node`. - fn branch_node_nibbled( + fn branch_node_nibbled( partial: impl Iterator, number_nibble: usize, - children: impl Iterator>>>, - value: Option, + children: impl Iterator>>>, + value: Option>, ) -> Vec; } diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index 2db1eafa..1a91df91 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -16,22 +16,22 @@ use crate::rstd::{boxed::Box, convert::TryInto, marker::PhantomData, vec, vec::Vec}; -use hash_db::{HashDBRef, Hasher}; +use hash_db::{HashDB, Hasher}; use crate::{ nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, - node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, Value, ValuePlan}, + node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, Node, Value, ValuePlan}, recorder::Record, CError, ChildReference, DBValue, NibbleSlice, NodeCodec, Recorder, Result as TrieResult, Trie, TrieDBBuilder, TrieError, TrieHash, TrieLayout, }; -struct StackEntry<'a, C: NodeCodec> { +struct StackEntry<'a, C: NodeCodec, L> { /// The prefix is the nibble path to the node in the trie. prefix: LeftNibbleSlice<'a>, /// Stacked node. - node: OwnedNode>, + node: OwnedNode, L>, /// The hash of the node or None if it is referenced inline. node_hash: Option, /// Whether the value should be omitted in the generated proof. @@ -40,20 +40,21 @@ struct StackEntry<'a, C: NodeCodec> { /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, /// The child references to use in constructing the proof nodes. - children: Vec>>, + children: Vec>>, /// The index into the proof vector that the encoding of this entry should be placed at. output_index: Option, _marker: PhantomData, } -impl<'a, C: NodeCodec> StackEntry<'a, C> { +impl<'a, C: NodeCodec, L: Copy + Default> StackEntry<'a, C, L> { fn new( prefix: LeftNibbleSlice<'a>, node_data: Vec, + locations: Vec, node_hash: Option, output_index: Option, ) -> TrieResult { - let node = OwnedNode::new::(node_data) + let node = OwnedNode::new::(node_data, locations) .map_err(|err| Box::new(TrieError::DecoderError(node_hash.unwrap_or_default(), err)))?; let children_len = match node.node_plan() { NodePlan::Empty | NodePlan::Leaf { .. } => 0, @@ -76,9 +77,9 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { fn encode_node(mut self) -> TrieResult, C::HashOut, C::Error> { let omit_value = self.omit_value; let node_data = self.node.data(); - let value_with_omission = |value_range: ValuePlan| -> Option { + let value_with_omission = |value_range: ValuePlan| -> Option> { if !omit_value { - Some(value_range.build(&node_data)) + Some(value_range.build(&node_data, Default::default())) } else { None } @@ -87,12 +88,12 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { NodePlan::Empty => node_data.to_vec(), NodePlan::Leaf { .. } if !omit_value => node_data.to_vec(), NodePlan::Leaf { partial, value: _ } => { - let partial = partial.build(node_data); - C::leaf_node(partial.right_iter(), partial.len(), Value::Inline(&[])) + let partial = partial.build(&node_data); + C::leaf_node::(partial.right_iter(), partial.len(), Value::Inline(&[])) }, NodePlan::Extension { .. } if self.child_index == 0 => node_data.to_vec(), NodePlan::Extension { partial: partial_plan, child: _ } => { - let partial = partial_plan.build(node_data); + let partial = partial_plan.build(&node_data); let child = self.children[0].expect( "for extension nodes, children[0] is guaranteed to be Some when \ child_index > 0; \ @@ -102,8 +103,8 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { }, NodePlan::Branch { value, children } => { Self::complete_branch_children( - node_data, - children, + &node_data, + &children, self.child_index, &mut self.children, )?; @@ -113,10 +114,10 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { ) }, NodePlan::NibbledBranch { partial: partial_plan, value, children } => { - let partial = partial_plan.build(node_data); + let partial = partial_plan.build(&node_data); Self::complete_branch_children( - node_data, - children, + &node_data, + &children, self.child_index, &mut self.children, )?; @@ -140,13 +141,13 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { node_data: &[u8], child_handles: &[Option; NIBBLE_LENGTH], child_index: usize, - children: &mut [Option>], + children: &mut [Option>], ) -> TrieResult<(), C::HashOut, C::Error> { for i in child_index..NIBBLE_LENGTH { children[i] = child_handles[i] .as_ref() .map(|child_plan| { - child_plan.build(node_data).try_into().map_err(|hash| { + child_plan.build(node_data, Default::default()).try_into().map_err(|hash| { Box::new(TrieError::InvalidHash(C::HashOut::default(), hash)) }) }) @@ -172,7 +173,7 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { set_child is called when the only child is popped from the stack; \ child_index is 0 before child is pushed to the stack; qed" ); - Some(Self::replacement_child_ref(encoded_child, child)) + Some(Self::replacement_child_ref(encoded_child, &child)) }, NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { assert!( @@ -196,7 +197,7 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { fn replacement_child_ref( encoded_child: &[u8], child: &NodeHandlePlan, - ) -> ChildReference { + ) -> ChildReference { match child { NodeHandlePlan::Hash(_) => ChildReference::Inline(C::HashOut::default(), 0), NodeHandlePlan::Inline(_) => { @@ -224,7 +225,7 @@ pub fn generate_proof<'a, D, L, I, K>( keys: I, ) -> TrieResult>, TrieHash, CError> where - D: HashDBRef, + D: HashDB, L: TrieLayout, I: IntoIterator, K: 'a + AsRef<[u8]>, @@ -236,7 +237,7 @@ where // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. - let mut stack = >>::new(); + let mut stack = >>::new(); // The mutated trie nodes comprising the final proof. let mut proof_nodes = Vec::new(); @@ -274,9 +275,8 @@ where loop { let step = match stack.last_mut() { - Some(entry) => match_key_to_node::( - entry.node.data(), - entry.node.node_plan(), + Some(entry) => match_key_to_node::( + &entry.node, &mut entry.omit_value, &mut entry.child_index, &mut entry.children, @@ -286,14 +286,14 @@ where )?, // If stack is empty, descend into the root node. None => - Step::Descend { child_prefix_len: 0, child: NodeHandle::Hash(root.as_ref()) }, + Step::Descend { child_prefix_len: 0, child: NodeHandle::Hash(root.as_ref(), Default::default()) }, }; match step { Step::Descend { child_prefix_len, child } => { let child_prefix = key.truncate(child_prefix_len); let child_entry = match child { - NodeHandle::Hash(hash) => { + NodeHandle::Hash(hash, _) => { let child_record = recorded_nodes.next().expect( "this function's trie traversal logic mirrors that of Lookup; \ thus the sequence of traversed nodes must be the same; \ @@ -310,6 +310,7 @@ where StackEntry::new( child_prefix, child_record.data, + Vec::new(), Some(child_record.hash), Some(output_index), )? @@ -321,7 +322,7 @@ where data.to_vec(), ))) } - StackEntry::new(child_prefix, data.to_vec(), None, None)? + StackEntry::new(child_prefix, data.to_vec(), Vec::new(), None, None)? }, }; stack.push(child_entry); @@ -368,19 +369,19 @@ where } } - unwind_stack::(&mut stack, &mut proof_nodes, None)?; + unwind_stack::(&mut stack, &mut proof_nodes, None)?; Ok(proof_nodes) } -enum Step<'a> { - Descend { child_prefix_len: usize, child: NodeHandle<'a> }, +enum Step<'a, L> { + Descend { child_prefix_len: usize, child: NodeHandle<'a, L> }, FoundValue(Option<&'a [u8]>), FoundHashedValue(Vec), } -fn resolve_value( +fn resolve_value( recorded_nodes: &mut dyn Iterator>, -) -> TrieResult, C::HashOut, C::Error> { +) -> TrieResult, C::HashOut, C::Error> { if let Some(resolve_value) = recorded_nodes.next() { Ok(Step::FoundHashedValue(resolve_value.data)) } else { @@ -390,49 +391,45 @@ fn resolve_value( /// Determine the next algorithmic step to take by matching the current key against the current top /// entry on the stack. -fn match_key_to_node<'a, C: NodeCodec>( - node_data: &'a [u8], - node_plan: &NodePlan, +fn match_key_to_node<'a, C: NodeCodec, L: Copy + Default>( + node: &'a OwnedNode, L>, omit_value: &mut bool, child_index: &mut usize, - children: &mut [Option>], - key: &LeftNibbleSlice, + children: &mut [Option>], + key: &'a LeftNibbleSlice, prefix_len: usize, recorded_nodes: &mut dyn Iterator>, -) -> TrieResult, C::HashOut, C::Error> { - Ok(match node_plan { - NodePlan::Empty => Step::FoundValue(None), - NodePlan::Leaf { partial: partial_plan, value: value_range } => { - let partial = partial_plan.build(node_data); +) -> TrieResult, C::HashOut, C::Error> { + let node = node.node(); + Ok(match node { + Node::Empty => Step::FoundValue(None), + Node::Leaf(partial, value) => { if key.contains(&partial, prefix_len) && key.len() == prefix_len + partial.len() { - match value_range { - ValuePlan::Inline(value_range) => { + match value{ + Value::Inline(data) => { *omit_value = true; - Step::FoundValue(Some(&node_data[value_range.clone()])) + Step::FoundValue(Some(data)) }, - ValuePlan::Node(..) => { + Value::Node(..) => { *omit_value = true; - resolve_value::(recorded_nodes)? + resolve_value::(recorded_nodes)? }, } } else { Step::FoundValue(None) } }, - NodePlan::Extension { partial: partial_plan, child: child_plan } => { - let partial = partial_plan.build(node_data); + Node::Extension(partial, child) => { if key.contains(&partial, prefix_len) { assert_eq!(*child_index, 0); let child_prefix_len = prefix_len + partial.len(); - let child = child_plan.build(&node_data); Step::Descend { child_prefix_len, child } } else { Step::FoundValue(None) } }, - NodePlan::Branch { value, children: child_handles } => match_key_to_branch_node::( - node_data, - value.as_ref(), + Node::Branch(child_handles, value) => match_key_to_branch_node::( + value.clone(), &child_handles, omit_value, child_index, @@ -442,47 +439,45 @@ fn match_key_to_node<'a, C: NodeCodec>( NibbleSlice::new(&[]), recorded_nodes, )?, - NodePlan::NibbledBranch { partial: partial_plan, value, children: child_handles } => - match_key_to_branch_node::( - node_data, - value.as_ref(), + Node::NibbledBranch(partial, child_handles, value) => + match_key_to_branch_node::( + value.clone(), &child_handles, omit_value, child_index, children, key, prefix_len, - partial_plan.build(node_data), + partial, recorded_nodes, )?, }) } -fn match_key_to_branch_node<'a, 'b, C: NodeCodec>( - node_data: &'a [u8], - value_range: Option<&'b ValuePlan>, - child_handles: &'b [Option; NIBBLE_LENGTH], +fn match_key_to_branch_node<'a, 'b, C: NodeCodec, L: Copy + Default>( + value: Option>, + child_handles: &'b [Option>; NIBBLE_LENGTH], omit_value: &mut bool, child_index: &mut usize, - children: &mut [Option>], - key: &'b LeftNibbleSlice<'b>, + children: &mut [Option>], + key: &'b LeftNibbleSlice<'a>, prefix_len: usize, - partial: NibbleSlice<'b>, + partial: NibbleSlice<'a>, recorded_nodes: &mut dyn Iterator>, -) -> TrieResult, C::HashOut, C::Error> { +) -> TrieResult, C::HashOut, C::Error> { if !key.contains(&partial, prefix_len) { return Ok(Step::FoundValue(None)) } if key.len() == prefix_len + partial.len() { - let value = match value_range { - Some(ValuePlan::Inline(range)) => { + let value = match value { + Some(Value::Inline(data)) => { *omit_value = true; - Some(&node_data[range.clone()]) + Some(data.clone()) }, - Some(ValuePlan::Node(..)) => { + Some(Value::Node(_, _)) => { *omit_value = true; - return resolve_value::(recorded_nodes) + return resolve_value::(recorded_nodes) }, None => None, }; @@ -499,19 +494,19 @@ fn match_key_to_branch_node<'a, 'b, C: NodeCodec>( while *child_index < new_index { children[*child_index] = child_handles[*child_index] .as_ref() - .map(|child_plan| { - child_plan - .build(node_data) + .map(|child| { + child + .clone() .try_into() .map_err(|hash| Box::new(TrieError::InvalidHash(C::HashOut::default(), hash))) }) .transpose()?; *child_index += 1; } - if let Some(child_plan) = &child_handles[*child_index] { + if let Some(child) = &child_handles[*child_index] { Ok(Step::Descend { child_prefix_len: prefix_len + partial.len() + 1, - child: child_plan.build(node_data), + child: child.clone(), }) } else { Ok(Step::FoundValue(None)) @@ -521,8 +516,8 @@ fn match_key_to_branch_node<'a, 'b, C: NodeCodec>( /// Unwind the stack until the given key is prefixed by the entry at the top of the stack. If the /// key is None, unwind the stack completely. As entries are popped from the stack, they are /// encoded into proof nodes and added to the finalized proof. -fn unwind_stack( - stack: &mut Vec>, +fn unwind_stack( + stack: &mut Vec>, proof_nodes: &mut Vec>, maybe_key: Option<&LeftNibbleSlice>, ) -> TrieResult<(), C::HashOut, C::Error> { diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index fedd0579..df76fe43 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -89,15 +89,15 @@ impl std::error::Error for struct StackEntry<'a, L: TrieLayout> { /// The prefix is the nibble path to the node in the trie. prefix: LeftNibbleSlice<'a>, - node: Node<'a>, + node: Node<'a, ()>, is_inline: bool, /// The value associated with this trie node. - value: Option>, + value: Option>, /// The next entry in the stack is a child of the preceding entry at this index. For branch /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, /// The child references to use in reconstructing the trie nodes. - children: Vec>>>, + children: Vec, ()>>>, /// Technical to attach lifetime to entry. next_value_hash: Option>, _marker: PhantomData, @@ -109,7 +109,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { prefix: LeftNibbleSlice<'a>, is_inline: bool, ) -> Result, CError>> { - let node = L::Codec::decode(&node_data[..]).map_err(Error::DecodeError)?; + let node = L::Codec::decode(&node_data[..], &[]).map_err(Error::DecodeError)?; let children_len = match &node { Node::Empty | Node::Leaf(..) => 0, Node::Extension(..) => 1, @@ -132,9 +132,9 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { }) } - fn value(&self) -> Option { + fn value(&self) -> Option> { if let Some(hash) = self.next_value_hash.as_ref() { - Some(Value::Node(hash.as_ref())) + Some(Value::Node(hash.as_ref(), ())) } else { self.value.clone() } @@ -226,7 +226,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { fn make_child_entry( proof_iter: &mut I, - child: NodeHandle<'a>, + child: NodeHandle<'a, ()>, prefix: LeftNibbleSlice<'a>, ) -> Result, CError>> where @@ -240,7 +240,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { } else { StackEntry::new(data, prefix, true) }, - NodeHandle::Hash(data) => { + NodeHandle::Hash(data, _) => { let mut hash = TrieHash::::default(); if data.len() != hash.as_ref().len() { return Err(Error::InvalidChildReference(data.to_vec())) @@ -323,7 +323,7 @@ enum ValueMatch<'a> { fn match_key_to_node<'a>( key: &LeftNibbleSlice<'a>, prefix_len: usize, - node: &Node, + node: &Node<()>, ) -> ValueMatch<'a> { match node { Node::Empty => ValueMatch::NotFound, @@ -365,8 +365,8 @@ fn match_key_to_node<'a>( fn match_key_to_branch_node<'a>( key: &LeftNibbleSlice<'a>, prefix_plus_partial_len: usize, - children: &[Option; NIBBLE_LENGTH], - value: Option<&Value>, + children: &[Option>; NIBBLE_LENGTH], + value: Option<&Value<()>>, ) -> ValueMatch<'a> { if key.len() == prefix_plus_partial_len { if value.is_none() { @@ -455,7 +455,7 @@ where ChildReference::Inline(hash, node_data.len()) } else { let hash = L::Hash::hash(&node_data); - ChildReference::Hash(hash) + ChildReference::Hash(hash, ()) }; if let Some(entry) = stack.pop() { @@ -467,7 +467,7 @@ where return Err(Error::ExtraneousNode) } let computed_root = match child_ref { - ChildReference::Hash(hash) => hash, + ChildReference::Hash(hash, _) => hash, ChildReference::Inline(_, _) => panic!("the bottom item on the stack has is_inline = false; qed"), }; diff --git a/trie-db/src/recorder.rs b/trie-db/src/recorder.rs index ba4c77e5..e54aa738 100644 --- a/trie-db/src/recorder.rs +++ b/trie-db/src/recorder.rs @@ -53,8 +53,8 @@ impl Recorder { } } -impl TrieRecorder> for Recorder { - fn record<'a>(&mut self, access: TrieAccess<'a, TrieHash>) { +impl TrieRecorder, L::Location> for Recorder { + fn record<'a>(&mut self, access: TrieAccess<'a, TrieHash, L::Location>) { match access { TrieAccess::EncodedNode { hash, encoded_node, .. } => { self.nodes.push(Record { hash, data: encoded_node.to_vec() }); diff --git a/trie-db/src/sectriedb.rs b/trie-db/src/sectriedb.rs deleted file mode 100644 index ccfbd971..00000000 --- a/trie-db/src/sectriedb.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{ - rstd::boxed::Box, triedb::TrieDB, CError, DBValue, Query, Result, Trie, TrieDBBuilder, - TrieHash, TrieItem, TrieIterator, TrieKeyItem, TrieLayout, -}; -use hash_db::{HashDBRef, Hasher}; - -/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. -/// -/// Use it as a `Trie` trait object. You can use `raw()` to get the backing `TrieDB` object. -pub struct SecTrieDB<'db, 'cache, L> -where - L: TrieLayout, -{ - raw: TrieDB<'db, 'cache, L>, -} - -impl<'db, 'cache, L> SecTrieDB<'db, 'cache, L> -where - L: TrieLayout, -{ - /// Create a new trie with the backing database `db` and `root`. - /// - /// Initialise to the state entailed by the genesis block. - /// This guarantees the trie is built correctly. - pub fn new(db: &'db dyn HashDBRef, root: &'db TrieHash) -> Self { - SecTrieDB { raw: TrieDBBuilder::new(db, root).build() } - } - - /// Get a reference to the underlying raw `TrieDB` struct. - pub fn raw(&self) -> &TrieDB<'db, 'cache, L> { - &self.raw - } - - /// Get a mutable reference to the underlying raw `TrieDB` struct. - pub fn raw_mut(&mut self) -> &mut TrieDB<'db, 'cache, L> { - &mut self.raw - } -} - -impl<'db, 'cache, L> Trie for SecTrieDB<'db, 'cache, L> -where - L: TrieLayout, -{ - fn root(&self) -> &TrieHash { - self.raw.root() - } - - fn contains(&self, key: &[u8]) -> Result, CError> { - self.raw.contains(L::Hash::hash(key).as_ref()) - } - - fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { - self.raw.get_hash(key) - } - - fn get_with>( - &self, - key: &[u8], - query: Q, - ) -> Result, TrieHash, CError> { - self.raw.get_with(L::Hash::hash(key).as_ref(), query) - } - - fn iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - TrieDB::iter(&self.raw) - } - - fn key_iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - TrieDB::key_iter(&self.raw) - } -} diff --git a/trie-db/src/sectriedbmut.rs b/trie-db/src/sectriedbmut.rs deleted file mode 100644 index b56dc55a..00000000 --- a/trie-db/src/sectriedbmut.rs +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{ - triedbmut::TrieDBMutBuilder, CError, DBValue, Result, TrieDBMut, TrieHash, TrieLayout, TrieMut, - Value, -}; -use hash_db::{HashDB, Hasher}; - -/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. -/// -/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` -/// object. -pub struct SecTrieDBMut<'db, L> -where - L: TrieLayout, -{ - raw: TrieDBMut<'db, L>, -} - -impl<'db, L> SecTrieDBMut<'db, L> -where - L: TrieLayout, -{ - /// Create a new trie with the backing database `db` and empty `root` - /// Initialize to the state entailed by the genesis block. - /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut dyn HashDB, root: &'db mut TrieHash) -> Self { - SecTrieDBMut { raw: TrieDBMutBuilder::new(db, root).build() } - } - - /// Create a new trie with the backing database `db` and `root`. - pub fn from_existing( - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, - ) -> Self { - SecTrieDBMut { raw: TrieDBMutBuilder::from_existing(db, root).build() } - } - - /// Get the backing database. - pub fn db(&self) -> &dyn HashDB { - self.raw.db() - } - - /// Get the backing database. - pub fn db_mut(&mut self) -> &mut dyn HashDB { - self.raw.db_mut() - } -} - -impl<'db, L> TrieMut for SecTrieDBMut<'db, L> -where - L: TrieLayout, -{ - fn root(&mut self) -> &TrieHash { - self.raw.root() - } - - fn is_empty(&self) -> bool { - self.raw.is_empty() - } - - fn contains(&self, key: &[u8]) -> Result, CError> { - self.raw.contains(&L::Hash::hash(key).as_ref()) - } - - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, TrieHash, CError> - where - 'a: 'key, - { - self.raw.get(&L::Hash::hash(key).as_ref()) - } - - fn insert( - &mut self, - key: &[u8], - value: &[u8], - ) -> Result>, TrieHash, CError> { - self.raw.insert(&L::Hash::hash(key).as_ref(), value) - } - - fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { - self.raw.remove(&L::Hash::hash(key).as_ref()) - } -} diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 9a1f51b3..ac0771dd 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -32,15 +32,16 @@ use crate::{ CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, TrieDB, TrieDBRawIterator, TrieError, TrieHash, TrieLayout, }; -use hash_db::{HashDB, Prefix}; +use hash_db::Prefix; +use memory_db::MemoryDB; -const OMIT_VALUE_HASH: crate::node::Value<'static> = crate::node::Value::Inline(&[]); +const OMIT_VALUE_HASH: crate::node::Value<'static, ()> = crate::node::Value::Inline(&[]); -struct EncoderStackEntry { +struct EncoderStackEntry { /// The prefix is the nibble path to the node in the trie. prefix: NibbleVec, /// Node in memory content. - node: Arc>, + node: Arc>, /// The next entry in the stack is a child of the preceding entry at this index. For branch /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, @@ -54,7 +55,7 @@ struct EncoderStackEntry { _marker: PhantomData, } -impl EncoderStackEntry { +impl EncoderStackEntry { /// Given the prefix of the next child node, identify its index and advance `child_index` to /// that. For a given entry, this must be called sequentially only with strictly increasing /// child prefixes. Returns an error if the child prefix is not a child of this entry or if @@ -116,14 +117,14 @@ impl EncoderStackEntry { node_data.to_vec() } else { let partial = partial.build(node_data); - let empty_child = ChildReference::Inline(C::HashOut::default(), 0); + let empty_child = ChildReference::<_, ()>::Inline(C::HashOut::default(), 0); C::extension_node(partial.right_iter(), partial.len(), empty_child) }, NodePlan::Branch { value, children } => { let value = if self.omit_value { value.is_some().then_some(OMIT_VALUE_HASH) } else { - value.as_ref().map(|v| v.build(node_data)) + value.as_ref().map(|v| v.build(node_data, ())) }; C::branch_node( Self::branch_children(node_data, &children, &self.omit_children)?.iter(), @@ -135,7 +136,7 @@ impl EncoderStackEntry { let value = if self.omit_value { value.is_some().then_some(OMIT_VALUE_HASH) } else { - value.as_ref().map(|v| v.build(node_data)) + value.as_ref().map(|v| v.build(node_data, ())) }; C::branch_node_nibbled( partial.right_iter(), @@ -165,14 +166,14 @@ impl EncoderStackEntry { node_data: &[u8], child_handles: &[Option; NIBBLE_LENGTH], omit_children: &[bool], - ) -> Result<[Option>; NIBBLE_LENGTH], C::HashOut, C::Error> { + ) -> Result<[Option>; NIBBLE_LENGTH], C::HashOut, C::Error> { let empty_child = ChildReference::Inline(C::HashOut::default(), 0); let mut children = [None; NIBBLE_LENGTH]; for i in 0..NIBBLE_LENGTH { children[i] = if omit_children[i] { Some(empty_child) } else if let Some(child_plan) = &child_handles[i] { - let child_ref = child_plan.build(node_data).try_into().map_err(|hash| { + let child_ref = child_plan.build(node_data, ()).try_into().map_err(|hash| { Box::new(TrieError::InvalidHash(C::HashOut::default(), hash)) })?; Some(child_ref) @@ -192,12 +193,13 @@ fn detached_value( value: &ValuePlan, node_data: &[u8], node_prefix: Prefix, + location: L::Location, ) -> Option> { let fetched; match value { ValuePlan::Node(hash_plan) => { if let Ok(value) = - TrieDBRawIterator::fetch_value(db, &node_data[hash_plan.clone()], node_prefix) + TrieDBRawIterator::fetch_value(db, &node_data[hash_plan.clone()], node_prefix, location) { fetched = value; } else { @@ -224,7 +226,7 @@ where // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. - let mut stack: Vec> = Vec::new(); + let mut stack: Vec> = Vec::new(); // TrieDBRawIterator guarantees that: // - It yields at least one node. @@ -269,11 +271,11 @@ where let (children_len, detached_value) = match node.node_plan() { NodePlan::Empty => (0, None), NodePlan::Leaf { value, .. } => - (0, detached_value(db, value, node.data(), prefix.as_prefix())), + (0, detached_value(db, &value, node.data(), prefix.as_prefix(), node.locations().first().copied().unwrap_or_default())), NodePlan::Extension { .. } => (1, None), NodePlan::NibbledBranch { value: Some(value), .. } | NodePlan::Branch { value: Some(value), .. } => - (NIBBLE_LENGTH, detached_value(db, value, node.data(), prefix.as_prefix())), + (NIBBLE_LENGTH, detached_value(db, &value, node.data(), prefix.as_prefix(), node.locations().last().copied().unwrap_or_default())), NodePlan::NibbledBranch { value: None, .. } | NodePlan::Branch { value: None, .. } => (NIBBLE_LENGTH, None), }; @@ -311,19 +313,19 @@ where Ok(output) } -struct DecoderStackEntry<'a, C: NodeCodec> { - node: Node<'a>, +struct DecoderStackEntry<'a, C: NodeCodec, L> { + node: Node<'a, L>, /// The next entry in the stack is a child of the preceding entry at this index. For branch /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, /// The reconstructed child references. - children: Vec>>, + children: Vec>>, /// A value attached as a node. The node will need to use its hash as value. attached_value: Option<&'a [u8]>, _marker: PhantomData, } -impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { +impl<'a, C: NodeCodec, L: Copy + Default> DecoderStackEntry<'a, C, L> { /// Advance the child index until either it exceeds the number of children or the child is /// marked as omitted. Omitted children are indicated by an empty inline reference. For each /// child that is passed over and not omitted, copy over the child reference from the node to @@ -407,7 +409,7 @@ impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { /// Preconditions: /// - if node is an extension node, then `children[0]` is Some. fn encode_node(self, attached_hash: Option<&[u8]>) -> Vec { - let attached_hash = attached_hash.map(|h| crate::node::Value::Node(h)); + let attached_hash = attached_hash.map(|h| crate::node::Value::Node(h, Default::default())); match self.node { Node::Empty => C::empty_node().to_vec(), Node::Leaf(partial, value) => @@ -433,7 +435,7 @@ impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { /// Reconstructs a partial trie DB from a compact representation. The encoding is a vector of /// mutated trie nodes with those child references omitted. The decode function reads them in order -/// from the given slice, reconstructing the full nodes and inserting them into the given `HashDB`. +/// from the given slice, reconstructing the full nodes and inserting them into the given `MemoryDB`. /// It stops after fully constructing one partial trie and returns the root hash and the number of /// nodes read. If an error occurs during decoding, there are no guarantees about which entries /// were or were not added to the DB. @@ -443,30 +445,28 @@ impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { // /// This function makes the assumption that all child references in an inline trie node are inline /// references. -pub fn decode_compact( - db: &mut DB, +pub fn decode_compact( + db: &mut MemoryDB, DBValue>, encoded: &[Vec], ) -> Result<(TrieHash, usize), TrieHash, CError> where L: TrieLayout, - DB: HashDB, { - decode_compact_from_iter::(db, encoded.iter().map(Vec::as_slice)) + decode_compact_from_iter::(db, encoded.iter().map(Vec::as_slice)) } /// Variant of 'decode_compact' that accept an iterator of encoded nodes as input. -pub fn decode_compact_from_iter<'a, L, DB, I>( - db: &mut DB, +pub fn decode_compact_from_iter<'a, L, I>( + db: &mut MemoryDB, DBValue>, encoded: I, ) -> Result<(TrieHash, usize), TrieHash, CError> where L: TrieLayout, - DB: HashDB, I: IntoIterator, { // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. - let mut stack: Vec> = Vec::new(); + let mut stack: Vec> = Vec::new(); // The prefix of the next item to be read from the slice of encoded items. let mut prefix = NibbleVec::new(); @@ -479,7 +479,7 @@ where attached_node = 1; } } - let node = L::Codec::decode(&encoded_node[attached_node..]) + let node = L::Codec::decode(&encoded_node[attached_node..], &[]) .map_err(|err| Box::new(TrieError::DecoderError(>::default(), err)))?; let children_len = match node { @@ -523,7 +523,7 @@ where if let Some(entry) = stack.pop() { last_entry = entry; last_entry.pop_from_prefix(&mut prefix); - last_entry.children[last_entry.child_index] = Some(ChildReference::Hash(node_hash)); + last_entry.children[last_entry.child_index] = Some(ChildReference::Hash(node_hash, Default::default())); last_entry.child_index += 1; } else { return Ok((node_hash, i + 1)) diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index ad6166eb..94045788 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -27,14 +27,14 @@ use crate::{ node::Node, rstd::{fmt, vec::Vec}, }; -use hash_db::{HashDBRef, Prefix, EMPTY_PREFIX}; +use hash_db::{HashDB, Prefix, EMPTY_PREFIX}; /// A builder for creating a [`TrieDB`]. pub struct TrieDBBuilder<'db, 'cache, L: TrieLayout> { - db: &'db dyn HashDBRef, + db: &'db dyn HashDB, root: &'db TrieHash, - cache: Option<&'cache mut dyn TrieCache>, - recorder: Option<&'cache mut dyn TrieRecorder>>, + cache: Option<&'cache mut dyn TrieCache>, + recorder: Option<&'cache mut dyn TrieRecorder, L::Location>>, } impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { @@ -43,13 +43,13 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// This doesn't check if `root` exists in the given `db`. If `root` doesn't exist it will fail /// when trying to lookup any key. #[inline] - pub fn new(db: &'db dyn HashDBRef, root: &'db TrieHash) -> Self { + pub fn new(db: &'db dyn HashDB, root: &'db TrieHash) -> Self { Self { db, root, cache: None, recorder: None } } /// Use the given `cache` for the db. #[inline] - pub fn with_cache(mut self, cache: &'cache mut dyn TrieCache) -> Self { + pub fn with_cache(mut self, cache: &'cache mut dyn TrieCache) -> Self { self.cache = Some(cache); self } @@ -58,7 +58,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { #[inline] pub fn with_optional_cache<'ocache: 'cache>( mut self, - cache: Option<&'ocache mut dyn TrieCache>, + cache: Option<&'ocache mut dyn TrieCache>, ) -> Self { // Make the compiler happy by "converting" the lifetime self.cache = cache.map(|c| c as _); @@ -67,7 +67,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// Use the given `recorder` to record trie accesses. #[inline] - pub fn with_recorder(mut self, recorder: &'cache mut dyn TrieRecorder>) -> Self { + pub fn with_recorder(mut self, recorder: &'cache mut dyn TrieRecorder, L::Location>) -> Self { self.recorder = Some(recorder); self } @@ -76,7 +76,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { #[inline] pub fn with_optional_recorder<'recorder: 'cache>( mut self, - recorder: Option<&'recorder mut dyn TrieRecorder>>, + recorder: Option<&'recorder mut dyn TrieRecorder, L::Location>>, ) -> Self { // Make the compiler happy by "converting" the lifetime self.recorder = recorder.map(|r| r as _); @@ -121,10 +121,10 @@ pub struct TrieDB<'db, 'cache, L> where L: TrieLayout, { - db: &'db dyn HashDBRef, + db: &'db dyn HashDB, root: &'db TrieHash, - cache: Option>>, - recorder: Option>>>, + cache: Option>>, + recorder: Option, L::Location>>>, } impl<'db, 'cache, L> TrieDB<'db, 'cache, L> @@ -132,7 +132,7 @@ where L: TrieLayout, { /// Get the backing database. - pub fn db(&'db self) -> &'db dyn HashDBRef { + pub fn db(&'db self) -> &'db dyn HashDB { self.db } @@ -150,15 +150,15 @@ where pub(crate) fn get_raw_or_lookup( &self, parent_hash: TrieHash, - node_handle: NodeHandle, + node_handle: NodeHandle, partial_key: Prefix, record_access: bool, - ) -> Result<(OwnedNode, Option>), TrieHash, CError> { - let (node_hash, node_data) = match node_handle { - NodeHandle::Hash(data) => { + ) -> Result<(OwnedNode, Option>), TrieHash, CError> { + let (node_hash, (node_data, locations)) = match node_handle { + NodeHandle::Hash(data, location) => { let node_hash = decode_hash::(data) .ok_or_else(|| Box::new(TrieError::InvalidHash(parent_hash, data.to_vec())))?; - let node_data = self.db.get(&node_hash, partial_key).ok_or_else(|| { + let node_data = self.db.get(&node_hash, partial_key, location).ok_or_else(|| { if partial_key == EMPTY_PREFIX { Box::new(TrieError::InvalidStateRoot(node_hash)) } else { @@ -168,9 +168,9 @@ where (Some(node_hash), node_data) }, - NodeHandle::Inline(data) => (None, data.to_vec()), + NodeHandle::Inline(data) => (None, (data.to_vec(), Default::default())), }; - let owned_node = OwnedNode::new::(node_data) + let owned_node = OwnedNode::new::(node_data, locations) .map_err(|e| Box::new(TrieError::DecoderError(node_hash.unwrap_or(parent_hash), e)))?; if record_access { @@ -192,10 +192,11 @@ where &self, hash: TrieHash, prefix: Prefix, + location: L::Location, ) -> Result, CError> { - let value = self + let (value, _) = self .db - .get(&hash, prefix) + .get(&hash, prefix, location) .ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; if let Some(recorder) = self.recorder.as_ref() { @@ -228,10 +229,10 @@ where db: self.db, query: |_: &[u8]| (), hash: *self.root, - cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), - recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder>), + cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), + recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } - .look_up_hash(key, NibbleSlice::new(key)) + .look_up_hash(key, NibbleSlice::new(key), Default::default()) } fn get_with>( @@ -246,10 +247,10 @@ where db: self.db, query, hash: *self.root, - cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), - recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder>), + cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), + recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } - .look_up(key, NibbleSlice::new(key)) + .look_up(key, NibbleSlice::new(key), Default::default()) } fn iter<'a>( @@ -280,7 +281,7 @@ where L: TrieLayout, { trie: &'db TrieDB<'db, 'cache, L>, - node_key: NodeHandle<'a>, + node_key: NodeHandle<'a, L::Location>, partial_key: NibbleVec, index: Option, } @@ -289,6 +290,7 @@ where impl<'db, 'cache, 'a, L> fmt::Debug for TrieAwareDebugNode<'db, 'cache, 'a, L> where L: TrieLayout, + L::Location: fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self.trie.get_raw_or_lookup( @@ -386,6 +388,7 @@ where impl<'db, 'cache, L> fmt::Debug for TrieDB<'db, 'cache, L> where L: TrieLayout, + L::Location: fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("TrieDB") @@ -393,7 +396,7 @@ where "root", &TrieAwareDebugNode { trie: self, - node_key: NodeHandle::Hash(self.root().as_ref()), + node_key: NodeHandle::Hash(self.root().as_ref(), Default::default()), partial_key: NibbleVec::new(), index: None, }, diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index bf3edbcb..19455b67 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -23,15 +23,19 @@ use crate::{ }, node_codec::NodeCodec, rstd::{boxed::Box, convert::TryFrom, mem, ops::Index, result, vec::Vec, VecDeque}, - Bytes, CError, CachedValue, DBValue, Result, TrieAccess, TrieCache, TrieError, TrieHash, - TrieLayout, TrieMut, TrieRecorder, + Bytes, CError, DBValue, Result, TrieAccess, TrieCache, TrieError, TrieHash, + TrieLayout, TrieRecorder, }; -use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; +use hash_db::{HashDB, Hasher, Prefix}; use hashbrown::HashSet; +#[cfg(not(feature = "std"))] +use alloc::vec; + #[cfg(feature = "std")] use log::trace; +use memory_db::MemoryDB; #[cfg(feature = "std")] use crate::rstd::fmt::{self, Debug}; @@ -43,20 +47,20 @@ struct StorageHandle(usize); // Handles to nodes in the trie. #[cfg_attr(feature = "std", derive(Debug))] -enum NodeHandle { +enum NodeHandle { /// Loaded into memory. InMemory(StorageHandle), /// Either a hash or an inline node - Hash(H), + Hash(H, L), } -impl From for NodeHandle { +impl From for NodeHandle { fn from(handle: StorageHandle) -> Self { NodeHandle::InMemory(handle) } } -fn empty_children() -> Box<[Option>; nibble_ops::NIBBLE_LENGTH]> { +fn empty_children() -> Box<[Option>; nibble_ops::NIBBLE_LENGTH]> { Box::new([ None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, @@ -73,7 +77,7 @@ pub enum Value { /// Value bytes inlined in a trie node. Inline(Bytes), /// Hash of the value. - Node(TrieHash), + Node(TrieHash, L::Location), /// Hash of value bytes if calculated and value bytes. /// The hash may be undefined until it node is added /// to the db. @@ -84,7 +88,7 @@ impl PartialEq for Value { fn eq(&self, other: &Self) -> bool { match (self, other) { (Value::Inline(v), Value::Inline(ov)) => v == ov, - (Value::Node(h), Value::Node(oh)) => h == oh, + (Value::Node(h, _), Value::Node(oh, _)) => h == oh, (Value::NewNode(Some(h), _), Value::NewNode(Some(oh), _)) => h == oh, (Value::NewNode(_, v), Value::NewNode(_, ov)) => v == ov, // Note that for uncalculated hash we do not calculate it and default to true. @@ -94,24 +98,24 @@ impl PartialEq for Value { } } -impl<'a, L: TrieLayout> From> for Value { - fn from(v: EncodedValue<'a>) -> Self { +impl<'a, L: TrieLayout> From> for Value { + fn from(v: EncodedValue<'a, L::Location>) -> Self { match v { EncodedValue::Inline(value) => Value::Inline(value.into()), - EncodedValue::Node(hash) => { + EncodedValue::Node(hash, l) => { let mut h = TrieHash::::default(); h.as_mut().copy_from_slice(hash); - Value::Node(h) + Value::Node(h, l) }, } } } -impl From<&ValueOwned>> for Value { - fn from(val: &ValueOwned>) -> Self { +impl From<&ValueOwned, L::Location>> for Value { + fn from(val: &ValueOwned, L::Location>) -> Self { match val { ValueOwned::Inline(data, _) => Self::Inline(data.clone()), - ValueOwned::Node(hash) => Self::Node(*hash), + ValueOwned::Node(hash, location) => Self::Node(*hash, *location), } } } @@ -129,9 +133,9 @@ impl From<(Bytes, Option)> for Value { } } -enum NodeToEncode<'a, H> { +enum NodeToEncode<'a, H, L> { Node(&'a [u8]), - TrieNode(NodeHandle), + TrieNode(NodeHandle), } impl Value { @@ -143,17 +147,17 @@ impl Value { &'a mut self, partial: Option<&NibbleSlice>, f: &mut F, - ) -> EncodedValue<'a> + ) -> EncodedValue<'a, L::Location> where F: FnMut( - NodeToEncode>, + NodeToEncode, L::Location>, Option<&NibbleSlice>, Option, - ) -> ChildReference>, + ) -> ChildReference, L::Location>, { if let Value::NewNode(hash, value) = self { let new_hash = - if let ChildReference::Hash(hash) = f(NodeToEncode::Node(&value), partial, None) { + if let ChildReference::Hash(hash, _) = f(NodeToEncode::Node(&value), partial, None) { hash } else { unreachable!("Value node can never be inlined; qed") @@ -166,8 +170,8 @@ impl Value { } let value = match &*self { Value::Inline(value) => EncodedValue::Inline(&value), - Value::Node(hash) => EncodedValue::Node(hash.as_ref()), - Value::NewNode(Some(hash), _value) => EncodedValue::Node(hash.as_ref()), + Value::Node(hash, location) => EncodedValue::Node(hash.as_ref(), *location), + Value::NewNode(Some(hash), _value) => EncodedValue::Node(hash.as_ref(), Default::default()), Value::NewNode(None, _value) => unreachable!("New external value are always added before encoding anode"), }; @@ -177,15 +181,15 @@ impl Value { fn in_memory_fetched_value( &self, prefix: Prefix, - db: &dyn HashDB, - recorder: &Option>>>, + db: &dyn HashDB, + recorder: &Option, L::Location>>>, full_key: &[u8], ) -> Result, TrieHash, CError> { Ok(Some(match self { Value::Inline(value) => value.to_vec(), Value::NewNode(_, value) => value.to_vec(), - Value::Node(hash) => - if let Some(value) = db.get(hash, prefix) { + Value::Node(hash, location) => + if let Some((value, _)) = db.get(hash, prefix, *location) { recorder.as_ref().map(|r| { r.borrow_mut().record(TrieAccess::Value { hash: *hash, @@ -214,13 +218,13 @@ enum Node { /// The shared portion is encoded from a `NibbleSlice` meaning it contains /// a flag indicating it is an extension. /// The child node is always a branch. - Extension(NodeKey, NodeHandle>), + Extension(NodeKey, NodeHandle, L::Location>), /// A branch has up to 16 children and an optional value. - Branch(Box<[Option>>; nibble_ops::NIBBLE_LENGTH]>, Option>), + Branch(Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, Option>), /// Branch node with support for a nibble (to avoid extension node). NibbledBranch( NodeKey, - Box<[Option>>; nibble_ops::NIBBLE_LENGTH]>, + Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, Option>, ), } @@ -243,7 +247,7 @@ impl Debug for Value { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { match self { Self::Inline(value) => write!(fmt, "Some({:?})", ToHex(value)), - Self::Node(hash) => write!(fmt, "Hash({:?})", ToHex(hash.as_ref())), + Self::Node(hash, _l) => write!(fmt, "Hash({:?})", ToHex(hash.as_ref())), Self::NewNode(Some(hash), _) => write!(fmt, "Hash({:?})", ToHex(hash.as_ref())), Self::NewNode(_hash, value) => write!(fmt, "Some({:?})", ToHex(value)), } @@ -254,6 +258,7 @@ impl Debug for Value { impl Debug for Node where L::Hash: Debug, + L::Location: Debug, { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { match *self { @@ -273,17 +278,17 @@ impl Node { // load an inline node into memory or get the hash to do the lookup later. fn inline_or_hash( parent_hash: TrieHash, - child: EncodedNodeHandle, + child: EncodedNodeHandle, storage: &mut NodeStorage, - ) -> Result>, TrieHash, CError> { + ) -> Result, L::Location>, TrieHash, CError> { let handle = match child { - EncodedNodeHandle::Hash(data) => { + EncodedNodeHandle::Hash(data, location) => { let hash = decode_hash::(data) .ok_or_else(|| Box::new(TrieError::InvalidHash(parent_hash, data.to_vec())))?; - NodeHandle::Hash(hash) + NodeHandle::Hash(hash, location) }, EncodedNodeHandle::Inline(data) => { - let child = Node::from_encoded(parent_hash, data, storage)?; + let child = Node::from_encoded(parent_hash, data, &[], storage)?; NodeHandle::InMemory(storage.alloc(Stored::New(child))) }, }; @@ -292,11 +297,11 @@ impl Node { // load an inline node into memory or get the hash to do the lookup later. fn inline_or_hash_owned( - child: &NodeHandleOwned>, + child: &NodeHandleOwned, L::Location>, storage: &mut NodeStorage, - ) -> NodeHandle> { + ) -> NodeHandle, L::Location> { match child { - NodeHandleOwned::Hash(hash) => NodeHandle::Hash(*hash), + NodeHandleOwned::Hash(hash, location) => NodeHandle::Hash(*hash, *location), NodeHandleOwned::Inline(node) => { let child = Node::from_node_owned(&**node, storage); NodeHandle::InMemory(storage.alloc(Stored::New(child))) @@ -308,10 +313,11 @@ impl Node { fn from_encoded<'a, 'b>( node_hash: TrieHash, data: &'a [u8], + locations: &[L::Location], storage: &'b mut NodeStorage, ) -> Result, CError> { let encoded_node = - L::Codec::decode(data).map_err(|e| Box::new(TrieError::DecoderError(node_hash, e)))?; + L::Codec::decode(data, locations).map_err(|e| Box::new(TrieError::DecoderError(node_hash, e)))?; let node = match encoded_node { EncodedNode::Empty => Node::Empty, EncodedNode::Leaf(k, v) => Node::Leaf(k.into(), v.into()), @@ -376,7 +382,7 @@ impl Node { } /// Decode a node from a [`NodeOwned`]. - fn from_node_owned(node_owned: &NodeOwned>, storage: &mut NodeStorage) -> Self { + fn from_node_owned(node_owned: &NodeOwned, L::Location>, storage: &mut NodeStorage) -> Self { match node_owned { NodeOwned::Empty => Node::Empty, NodeOwned::Leaf(k, v) => Node::Leaf(k.into(), v.into()), @@ -449,10 +455,10 @@ impl Node { fn into_encoded(self, mut child_cb: F) -> Vec where F: FnMut( - NodeToEncode>, + NodeToEncode, L::Location>, Option<&NibbleSlice>, Option, - ) -> ChildReference>, + ) -> ChildReference, L::Location>, { match self { Node::Empty => L::Codec::empty_node().to_vec(), @@ -499,17 +505,6 @@ impl Node { }, } } - - /// Returns the key partial key of this node. - fn partial_key(&self) -> Option<&NodeKey> { - match &self { - Self::Empty => None, - Self::Leaf(key, _) => Some(key), - Self::Branch(_, _) => None, - Self::NibbledBranch(key, _, _) => Some(key), - Self::Extension(key, _) => Some(key), - } - } } // post-inspect action. @@ -551,33 +546,33 @@ enum Stored { // A new node. New(Node), // A cached node, loaded from the DB. - Cached(Node, TrieHash), + Cached(Node, TrieHash, L::Location), } /// Used to build a collection of child nodes from a collection of `NodeHandle`s #[derive(Clone, Copy)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum ChildReference { +pub enum ChildReference { // `HO` is e.g. `H256`, i.e. the output of a `Hasher` - Hash(HO), + Hash(HO, L), Inline(HO, usize), // usize is the length of the node data we store in the `H::Out` } -impl<'a, HO> TryFrom> for ChildReference +impl<'a, HO, L> TryFrom> for ChildReference where HO: AsRef<[u8]> + AsMut<[u8]> + Default + Clone + Copy, { type Error = Vec; - fn try_from(handle: EncodedNodeHandle<'a>) -> result::Result> { + fn try_from(handle: EncodedNodeHandle<'a, L>) -> result::Result> { match handle { - EncodedNodeHandle::Hash(data) => { + EncodedNodeHandle::Hash(data, location) => { let mut hash = HO::default(); if data.len() != hash.as_ref().len() { return Err(data.to_vec()) } hash.as_mut().copy_from_slice(data); - Ok(ChildReference::Hash(hash)) + Ok(ChildReference::Hash(hash, location)) }, EncodedNodeHandle::Inline(data) => { let mut hash = HO::default(); @@ -629,25 +624,24 @@ impl<'a, L: TrieLayout> Index<&'a StorageHandle> for NodeStorage { fn index(&self, handle: &'a StorageHandle) -> &Node { match self.nodes[handle.0] { Stored::New(ref node) => node, - Stored::Cached(ref node, _) => node, + Stored::Cached(ref node, _, _) => node, } } } /// A builder for creating a [`TrieDBMut`]. pub struct TrieDBMutBuilder<'db, L: TrieLayout> { - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, - cache: Option<&'db mut dyn TrieCache>, - recorder: Option<&'db mut dyn TrieRecorder>>, + db: &'db dyn HashDB, + root: TrieHash, + cache: Option<&'db mut dyn TrieCache>, + recorder: Option<&'db mut dyn TrieRecorder, L::Location>>, } impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Create a builder for constructing a new trie with the backing database `db` and empty /// `root`. - pub fn new(db: &'db mut dyn HashDB, root: &'db mut TrieHash) -> Self { - *root = L::Codec::hashed_null_node(); - + pub fn new(db: &'db dyn HashDB) -> Self { + let root = L::Codec::hashed_null_node(); Self { root, db, cache: None, recorder: None } } @@ -656,14 +650,14 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// This doesn't check if `root` exists in the given `db`. If `root` doesn't exist it will fail /// when trying to lookup any key. pub fn from_existing( - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, + db: &'db dyn HashDB, + root: TrieHash, ) -> Self { Self { db, root, cache: None, recorder: None } } /// Use the given `cache` for the db. - pub fn with_cache(mut self, cache: &'db mut dyn TrieCache) -> Self { + pub fn with_cache(mut self, cache: &'db mut dyn TrieCache) -> Self { self.cache = Some(cache); self } @@ -671,7 +665,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Use the given optional `cache` for the db. pub fn with_optional_cache<'cache: 'db>( mut self, - cache: Option<&'cache mut dyn TrieCache>, + cache: Option<&'cache mut dyn TrieCache>, ) -> Self { // Make the compiler happy by "converting" the lifetime self.cache = cache.map(|c| c as _); @@ -679,7 +673,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { } /// Use the given `recorder` to record trie accesses. - pub fn with_recorder(mut self, recorder: &'db mut dyn TrieRecorder>) -> Self { + pub fn with_recorder(mut self, recorder: &'db mut dyn TrieRecorder, L::Location>) -> Self { self.recorder = Some(recorder); self } @@ -687,7 +681,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Use the given optional `recorder` to record trie accesses. pub fn with_optional_recorder<'recorder: 'db>( mut self, - recorder: Option<&'recorder mut dyn TrieRecorder>>, + recorder: Option<&'recorder mut dyn TrieRecorder, L::Location>>, ) -> Self { // Make the compiler happy by "converting" the lifetime self.recorder = recorder.map(|r| r as _); @@ -696,7 +690,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Build the [`TrieDBMut`]. pub fn build(self) -> TrieDBMut<'db, L> { - let root_handle = NodeHandle::Hash(*self.root); + let root_handle = NodeHandle::Hash(self.root, Default::default()); TrieDBMut { db: self.db, @@ -711,13 +705,134 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { } } +#[derive(Debug)] +pub struct NewChangesetNode { + pub hash: H, + pub prefix: OwnedPrefix, + pub data: Vec, + pub children: Vec>, +} + +#[derive(Debug)] +pub struct ExistingChangesetNode { + pub hash: H, + pub prefix: OwnedPrefix, + pub location: DL, +} + +#[derive(Debug)] +pub enum ChangesetNodeRef { + New (NewChangesetNode), + Existing (ExistingChangesetNode), +} + +impl ChangesetNodeRef { + pub fn hash(&self) -> &H { + match self { + ChangesetNodeRef::New(node) => &node.hash, + ChangesetNodeRef::Existing(node) => &node.hash, + } + } +} + +#[derive(Debug)] +pub struct Changeset { + pub root: ChangesetNodeRef, + pub removed: Vec<(H, OwnedPrefix)>, +} + +impl Changeset { + pub fn apply_to(&self, mem_db: &mut MemoryDB) -> H + where + K: memory_db::KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { + for (hash, prefix) in &self.removed { + mem_db.remove(hash, (prefix.0.as_slice(), prefix.1)); + } + fn apply_node(node: &ChangesetNodeRef, mem_db: &mut MemoryDB) + where + K: memory_db::KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { + match node { + ChangesetNodeRef::New(node) => { + for child in &node.children { + apply_node(child, mem_db); + } + mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); + }, + ChangesetNodeRef::Existing(_) => {} + } + } + apply_node::(&self.root, mem_db); + self.root_hash() + } + + pub fn apply_with_prefix(&self, mem_db: &mut MemoryDB, ks: &[u8]) -> H + where + K: memory_db::KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { + fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { + let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); + result.extend_from_slice(ks); + result.extend_from_slice(prefix.0); + (result, prefix.1) + } + + fn apply_node(node: &ChangesetNodeRef, ks: &[u8], mem_db: &mut MemoryDB) + where + K: memory_db::KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { + match node { + ChangesetNodeRef::New(node) => { + for child in &node.children { + apply_node(child, ks, mem_db); + } + let prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); + mem_db.insert((prefixed.0.as_slice(), prefixed.1), &node.data); + }, + ChangesetNodeRef::Existing(_) => {} + } + } + + for (hash, p) in &self.removed { + let prefixed = prefix_prefix(ks, (p.0.as_slice(), p.1)); + mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); + } + apply_node(&self.root, ks, mem_db); + self.root_hash() + } + + + pub fn root_hash(&self) -> H { + match &self.root { + ChangesetNodeRef::New(node) => node.hash, + ChangesetNodeRef::Existing(node) => node.hash, + } + } + + pub fn empty(root: H) -> Self { + Self { + root: ChangesetNodeRef::Existing(ExistingChangesetNode { + hash: root, + prefix: (BackingByteVec::new(), None), + location: Default::default(), + }), + removed: Default::default(), + } + } +} + +type OwnedPrefix = (BackingByteVec, Option); + /// A `Trie` implementation using a generic `HashDB` backing database. /// /// Use it as a `TrieMut` trait object. You can use `db()` to get the backing database object. /// Note that changes are not committed to the database until `commit` is called. /// -/// Querying the root or dropping the trie will commit automatically. -/// /// /// # Example /// ```ignore @@ -743,17 +858,17 @@ where L: TrieLayout, { storage: NodeStorage, - db: &'a mut dyn HashDB, - root: &'a mut TrieHash, - root_handle: NodeHandle>, - death_row: HashSet<(TrieHash, (BackingByteVec, Option))>, + db: &'a dyn HashDB, + root: TrieHash, + root_handle: NodeHandle, L::Location>, + death_row: HashSet<(TrieHash, OwnedPrefix)>, /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. hash_count: usize, /// Optional cache for speeding up the lookup of nodes. - cache: Option<&'a mut dyn TrieCache>, + cache: Option<&'a mut dyn TrieCache>, /// Optional trie recorder for recording trie accesses. - recorder: Option>>>, + recorder: Option, L::Location>>>, } impl<'a, L> TrieDBMut<'a, L> @@ -761,12 +876,7 @@ where L: TrieLayout, { /// Get the backing database. - pub fn db(&self) -> &dyn HashDB { - self.db - } - - /// Get the backing database mutably. - pub fn db_mut(&mut self) -> &mut dyn HashDB { + pub fn db(&self) -> &dyn HashDB { self.db } @@ -775,12 +885,13 @@ where &mut self, hash: TrieHash, key: Prefix, + location: L::Location, ) -> Result, CError> { // We only check the `cache` for a node with `get_node` and don't insert // the node if it wasn't there, because in substrate we only access the node while computing // a new trie (aka some branch). We assume that this node isn't that important // to have it being cached. - let node = match self.cache.as_mut().and_then(|c| c.get_node(&hash)) { + let node = match self.cache.as_mut().and_then(|c| c.get_node(&hash, location)) { Some(node) => { if let Some(recorder) = self.recorder.as_mut() { recorder.borrow_mut().record(TrieAccess::NodeOwned { hash, node_owned: &node }); @@ -789,9 +900,9 @@ where Node::from_node_owned(&node, &mut self.storage) }, None => { - let node_encoded = self + let (node_encoded, locations) = self .db - .get(&hash, key) + .get(&hash, key, location) .ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; if let Some(recorder) = self.recorder.as_mut() { @@ -801,11 +912,11 @@ where }); } - Node::from_encoded(hash, &node_encoded, &mut self.storage)? + Node::from_encoded(hash, &node_encoded, &locations, &mut self.storage)? }, }; - Ok(self.storage.alloc(Stored::Cached(node, hash))) + Ok(self.storage.alloc(Stored::Cached(node, hash, location))) } // Inspect a node, choosing either to replace, restore, or delete it. @@ -830,8 +941,8 @@ where Action::Replace(node) => Some((Stored::New(node), true)), Action::Delete => None, }, - Stored::Cached(node, hash) => match inspector(self, node, key)? { - Action::Restore(node) => Some((Stored::Cached(node, hash), false)), + Stored::Cached(node, hash, location) => match inspector(self, node, key)? { + Action::Restore(node) => Some((Stored::Cached(node, hash, location), false)), Action::Replace(node) => { self.death_row.insert((hash, current_key.left_owned())); Some((Stored::New(node), true)) @@ -849,25 +960,25 @@ where &self, full_key: &[u8], mut partial: NibbleSlice, - handle: &NodeHandle>, + handle: &NodeHandle, L::Location>, ) -> Result, TrieHash, CError> { let mut handle = handle; let prefix = (full_key, None); loop { let (mid, child) = match handle { - NodeHandle::Hash(hash) => { + NodeHandle::Hash(hash, location) => { let mut recorder = self.recorder.as_ref().map(|r| r.borrow_mut()); return Lookup:: { - db: &self.db, + db: self.db, query: |v: &[u8]| v.to_vec(), hash: *hash, cache: None, recorder: recorder .as_mut() - .map(|r| &mut ***r as &mut dyn TrieRecorder>), + .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } - .look_up(full_key, partial) + .look_up(full_key, partial, *location) }, NodeHandle::InMemory(handle) => match &self.storage[handle] { Node::Empty => return Ok(None), @@ -943,14 +1054,14 @@ where /// Insert a key-value pair into the trie, creating new nodes if necessary. fn insert_at( &mut self, - handle: NodeHandle>, + handle: NodeHandle, L::Location>, key: &mut NibbleFullKey, value: Bytes, old_val: &mut Option>, ) -> Result<(StorageHandle, bool), TrieHash, CError> { let h = match handle { NodeHandle::InMemory(h) => h, - NodeHandle::Hash(h) => self.cache(h, key.left())?, + NodeHandle::Hash(h, l) => self.cache(h, key.left(), l)?, }; // cache then destroy for hash handle (handle being root in most case) let stored = self.storage.destroy(h); @@ -971,7 +1082,7 @@ where ) { match &stored_value { Some(Value::NewNode(Some(hash), _)) // also removing new node in case commit is called multiple times - | Some(Value::Node(hash)) => { + | Some(Value::Node(hash, _)) => { self.death_row.insert(( hash.clone(), (prefix.0.into(), prefix.1), @@ -1334,14 +1445,14 @@ where /// Removes a node from the trie based on key. fn remove_at( &mut self, - handle: NodeHandle>, + handle: NodeHandle, L::Location>, key: &mut NibbleFullKey, old_val: &mut Option>, ) -> Result, TrieHash, CError> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), - NodeHandle::Hash(h) => { - let handle = self.cache(h, key.left())?; + NodeHandle::Hash(h, l) => { + let handle = self.cache(h, key.left(), l)?; self.storage.destroy(handle) }, }; @@ -1645,14 +1756,14 @@ where ); let stored = match child { NodeHandle::InMemory(h) => self.storage.destroy(h), - NodeHandle::Hash(h) => { - let handle = self.cache(h, child_prefix)?; + NodeHandle::Hash(h, l) => { + let handle = self.cache(h, child_prefix, l)?; self.storage.destroy(handle) }, }; let child_node = match stored { Stored::New(node) => node, - Stored::Cached(node, hash) => { + Stored::Cached(node, hash, _location) => { self.death_row .insert((hash, (child_prefix.0[..].into(), child_prefix.1))); node @@ -1730,15 +1841,15 @@ where let stored = match child { NodeHandle::InMemory(h) => self.storage.destroy(h), - NodeHandle::Hash(h) => { - let handle = self.cache(h, child_prefix)?; + NodeHandle::Hash(h, l) => { + let handle = self.cache(h, child_prefix, l)?; self.storage.destroy(handle) }, }; - let (child_node, maybe_hash) = match stored { - Stored::New(node) => (node, None), - Stored::Cached(node, hash) => (node, Some(hash)), + let (child_node, maybe_hash, child_location) = match stored { + Stored::New(node) => (node, None, Default::default()), + Stored::Cached(node, hash, location) => (node, Some(hash), location), }; match child_node { @@ -1785,7 +1896,7 @@ where // reallocate the child node. let stored = if let Some(hash) = maybe_hash { - Stored::Cached(child_node, hash) + Stored::Cached(child_node, hash, child_location) } else { Stored::New(child_node) }; @@ -1798,44 +1909,54 @@ where } } - /// Commit the in-memory changes to disk, freeing their storage and - /// updating the state root. - pub fn commit(&mut self) { + /// Calculate the changeset for the trie. + pub fn commit(mut self) -> Changeset, L::Location> { #[cfg(feature = "std")] trace!(target: "trie", "Committing trie changes to db."); // always kill all the nodes on death row. #[cfg(feature = "std")] trace!(target: "trie", "{:?} nodes to remove from db", self.death_row.len()); + let mut removed = Vec::with_capacity(self.death_row.len()); for (hash, prefix) in self.death_row.drain() { - self.db.remove(&hash, (&prefix.0[..], prefix.1)); + removed.push((hash, prefix)); } let handle = match self.root_handle() { - NodeHandle::Hash(_) => return, // no changes necessary. + NodeHandle::Hash(hash, location) => return Changeset { + root: ChangesetNodeRef::Existing(ExistingChangesetNode { + hash, + prefix: Default::default(), + location, + }), + removed, + }, // no changes necessary. NodeHandle::InMemory(h) => h, }; match self.storage.destroy(handle) { Stored::New(node) => { - // Reconstructs the full key for root node. - let full_key = self.cache.as_ref().and_then(|_| { - node.partial_key().and_then(|k| Some(NibbleSlice::from_stored(k).into())) - }); - let mut k = NibbleVec::new(); + let mut children = Vec::new(); let encoded_root = node.into_encoded(|node, o_slice, o_index| { let mov = k.append_optional_slice_and_nibble(o_slice, o_index); match node { NodeToEncode::Node(value) => { - let value_hash = self.db.insert(k.as_prefix(), value); + let value_hash = self.db.hash(value); self.cache_value(k.inner(), value, value_hash); + children.push(ChangesetNodeRef::New(NewChangesetNode { + hash: value_hash, + prefix: k.as_owned_prefix(), + data: value.to_vec(), //TODO: avoid allocation + children: Default::default(), + })); + k.drop_lasts(mov); - ChildReference::Hash(value_hash) + ChildReference::Hash(value_hash, Default::default()) }, NodeToEncode::TrieNode(child) => { - let result = self.commit_child(child, &mut k); + let result = self.commit_child(child, &mut k, &mut children); k.drop_lasts(mov); result }, @@ -1844,99 +1965,51 @@ where #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:?}", ToHex(&encoded_root[..])); - *self.root = self.db.insert(EMPTY_PREFIX, &encoded_root); + self.root = self.db.hash(&encoded_root); self.hash_count += 1; - self.cache_node(*self.root, &encoded_root, full_key); + self.cache_node(self.root); - self.root_handle = NodeHandle::Hash(*self.root); + self.root_handle = NodeHandle::Hash(self.root, Default::default()); + Changeset { + root: ChangesetNodeRef::New(NewChangesetNode { + hash: self.root.clone(), + prefix: Default::default(), + data: encoded_root, + children, + }), + removed, + } }, - Stored::Cached(node, hash) => { + Stored::Cached(node, hash, location) => { // probably won't happen, but update the root and move on. - *self.root = hash; + self.root = hash; self.root_handle = - NodeHandle::InMemory(self.storage.alloc(Stored::Cached(node, hash))); + NodeHandle::InMemory(self.storage.alloc(Stored::Cached(node, hash, Default::default()))); + Changeset { + root: ChangesetNodeRef::Existing(ExistingChangesetNode { + hash, + prefix: Default::default(), + location, + }), + removed, + } }, } } /// Cache the given `encoded` node. - fn cache_node(&mut self, hash: TrieHash, encoded: &[u8], full_key: Option) { + fn cache_node(&mut self, hash: TrieHash) { // If we have a cache, cache our node directly. if let Some(cache) = self.cache.as_mut() { - let node = cache.get_or_insert_node(hash, &mut || { - Ok(L::Codec::decode(&encoded) - .ok() - .and_then(|n| n.to_owned_node::().ok()) - .expect("Just encoded the node, so it should decode without any errors; qed")) - }); - - // `node` should always be `OK`, but let's play it safe. - let node = if let Ok(node) = node { node } else { return }; - - let mut values_to_cache = Vec::new(); - - // If the given node has data attached, the `full_key` is the full key to this node. - if let Some(full_key) = full_key { - node.data().and_then(|v| node.data_hash().map(|h| (&full_key, v, h))).map( - |(k, v, h)| { - values_to_cache.push((k.inner().to_vec(), (v.clone(), h).into())); - }, - ); - - fn cache_child_values( - node: &NodeOwned>, - values_to_cache: &mut Vec<(Vec, CachedValue>)>, - full_key: NibbleVec, - ) { - node.child_iter().flat_map(|(n, c)| c.as_inline().map(|c| (n, c))).for_each( - |(n, c)| { - let mut key = full_key.clone(); - n.map(|n| key.push(n)); - c.partial_key().map(|p| key.append(p)); - - if let Some((hash, data)) = - c.data().and_then(|d| c.data_hash().map(|h| (h, d))) - { - values_to_cache - .push((key.inner().to_vec(), (data.clone(), hash).into())); - } - - cache_child_values::(c, values_to_cache, key); - }, - ); - } - - // Also cache values of inline nodes. - cache_child_values::(&node, &mut values_to_cache, full_key.clone()); - } - - drop(node); - values_to_cache.into_iter().for_each(|(k, v)| cache.cache_value_for_key(&k, v)); + cache.insert_new_node(&hash); } } /// Cache the given `value`. /// /// `hash` is the hash of `value`. - fn cache_value(&mut self, full_key: &[u8], value: impl Into, hash: TrieHash) { - if let Some(cache) = self.cache.as_mut() { - let value = value.into(); - - // `get_or_insert` should always return `Ok`, but be safe. - let value = if let Ok(value) = cache - .get_or_insert_node(hash, &mut || Ok(NodeOwned::Value(value.clone(), hash))) - .map(|n| n.data().cloned()) - { - value - } else { - None - }; - - if let Some(value) = value { - cache.cache_value_for_key(full_key, (value, hash).into()) - } - } + fn cache_value(&mut self, _full_key: &[u8], _value: impl Into, _hash: TrieHash) { } /// Commit a node by hashing it and writing it to the db. Returns a @@ -1946,40 +2019,53 @@ where /// `into_encoded` method of `Node`. fn commit_child( &mut self, - handle: NodeHandle>, + handle: NodeHandle, L::Location>, prefix: &mut NibbleVec, - ) -> ChildReference> { + children: &mut Vec, L::Location>> + ) -> ChildReference, L::Location> { match handle { - NodeHandle::Hash(hash) => ChildReference::Hash(hash), + NodeHandle::Hash(hash, location) => { + children.push(ChangesetNodeRef::Existing(ExistingChangesetNode { + hash, + prefix: prefix.as_owned_prefix(), + location, + })); + ChildReference::Hash(hash, location) + } NodeHandle::InMemory(storage_handle) => { match self.storage.destroy(storage_handle) { - Stored::Cached(_, hash) => ChildReference::Hash(hash), + Stored::Cached(_, hash, location) => { + children.push(ChangesetNodeRef::Existing(ExistingChangesetNode { + hash, + prefix: prefix.as_owned_prefix(), + location, + })); + ChildReference::Hash(hash, location) + }, Stored::New(node) => { - // Reconstructs the full key - let full_key = self.cache.as_ref().and_then(|_| { - let mut prefix = prefix.clone(); - if let Some(partial) = node.partial_key() { - prefix.append_partial(NibbleSlice::from_stored(partial).right()); - } - Some(prefix) - }); - + let mut sub_children = Vec::new(); let encoded = { - let commit_child = |node: NodeToEncode>, + let commit_child = |node: NodeToEncode, L::Location>, o_slice: Option<&NibbleSlice>, o_index: Option| { let mov = prefix.append_optional_slice_and_nibble(o_slice, o_index); match node { NodeToEncode::Node(value) => { - let value_hash = self.db.insert(prefix.as_prefix(), value); + let value_hash = self.db.hash(value); + sub_children.push(ChangesetNodeRef::New(NewChangesetNode { + hash: value_hash, + prefix: prefix.as_owned_prefix(), + data: value.to_vec(), //TODO: avoid allocation + children: Default::default(), + })); self.cache_value(prefix.inner(), value, value_hash); prefix.drop_lasts(mov); - ChildReference::Hash(value_hash) + ChildReference::Hash(value_hash, Default::default()) }, NodeToEncode::TrieNode(node_handle) => { - let result = self.commit_child(node_handle, prefix); + let result = self.commit_child(node_handle, prefix, &mut sub_children); prefix.drop_lasts(mov); result }, @@ -1988,12 +2074,16 @@ where node.into_encoded(commit_child) }; if encoded.len() >= L::Hash::LENGTH { - let hash = self.db.insert(prefix.as_prefix(), &encoded); + let hash = self.db.hash(&encoded); + self.cache_node(hash); + children.push(ChangesetNodeRef::New(NewChangesetNode { + hash, + prefix: prefix.as_owned_prefix(), + data: encoded, + children: sub_children, + })); self.hash_count += 1; - - self.cache_node(hash, &encoded, full_key); - - ChildReference::Hash(hash) + ChildReference::Hash(hash, Default::default()) } else { // it's a small value, so we cram it into a `TrieHash` // and tag with length @@ -2010,26 +2100,16 @@ where } // a hack to get the root node's handle - fn root_handle(&self) -> NodeHandle> { + fn root_handle(&self) -> NodeHandle, L::Location> { match self.root_handle { - NodeHandle::Hash(h) => NodeHandle::Hash(h), + NodeHandle::Hash(h, l) => NodeHandle::Hash(h, l), NodeHandle::InMemory(StorageHandle(x)) => NodeHandle::InMemory(StorageHandle(x)), } } -} - -impl<'a, L> TrieMut for TrieDBMut<'a, L> -where - L: TrieLayout, -{ - fn root(&mut self) -> &TrieHash { - self.commit(); - self.root - } - fn is_empty(&self) -> bool { + pub fn is_empty(&self) -> bool { match self.root_handle { - NodeHandle::Hash(h) => h == L::Codec::hashed_null_node(), + NodeHandle::Hash(h, _) => h == L::Codec::hashed_null_node(), NodeHandle::InMemory(ref h) => match self.storage[h] { Node::Empty => true, _ => false, @@ -2037,14 +2117,14 @@ where } } - fn get<'x, 'key>(&'x self, key: &'key [u8]) -> Result, TrieHash, CError> + pub fn get<'x, 'key>(&'x self, key: &'key [u8]) -> Result, TrieHash, CError> where 'x: 'key, { self.lookup(key, NibbleSlice::new(key), &self.root_handle) } - fn insert( + pub fn insert( &mut self, key: &[u8], value: &[u8], @@ -2070,7 +2150,7 @@ where Ok(old_val) } - fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { + pub fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { #[cfg(feature = "std")] trace!(target: "trie", "remove: key={:?}", ToHex(key)); @@ -2087,8 +2167,8 @@ where None => { #[cfg(feature = "std")] trace!(target: "trie", "remove: obliterated trie"); - self.root_handle = NodeHandle::Hash(L::Codec::hashed_null_node()); - *self.root = L::Codec::hashed_null_node(); + self.root_handle = NodeHandle::Hash(L::Codec::hashed_null_node(), Default::default()); + self.root = L::Codec::hashed_null_node(); }, } @@ -2096,15 +2176,6 @@ where } } -impl<'a, L> Drop for TrieDBMut<'a, L> -where - L: TrieLayout, -{ - fn drop(&mut self) { - self.commit(); - } -} - /// combine two NodeKeys fn combine_key(start: &mut NodeKey, end: (usize, &[u8])) { debug_assert!(start.0 < nibble_ops::NIBBLE_PER_BYTE); diff --git a/trie-db/test/benches/bench.rs b/trie-db/test/benches/bench.rs index bd2611d5..50d18686 100644 --- a/trie-db/test/benches/bench.rs +++ b/trie-db/test/benches/bench.rs @@ -254,7 +254,6 @@ fn trie_mut_ref_root(c: &mut Criterion) { fn trie_mut(c: &mut Criterion) { use memory_db::HashKey; - use trie_db::TrieMut; let params = vec![(29, 204800 / 2, 512 * 2), (29, 204800, 32)]; @@ -265,10 +264,9 @@ fn trie_mut(c: &mut Criterion) { let param = format!("seed({}), len({}), value_length({})", seed, len, value_length); group.bench_with_input(BenchmarkId::new("trie_mut", param), &input, |b, i| { b.iter(|| { - let mut root = Default::default(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); let mut trie = - trie_db::TrieDBMutBuilder::::new(&mut mdb, &mut root).build(); + trie_db::TrieDBMutBuilder::::new(&mut mdb).build(); for (key, value) in i { trie.insert(&key, &value).expect( "changes trie: insertion to trie is not allowed to fail within runtime", diff --git a/trie-db/test/src/fatdb.rs b/trie-db/test/src/fatdb.rs deleted file mode 100644 index fb8e7350..00000000 --- a/trie-db/test/src/fatdb.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefFatDB, RefFatDBMut, RefHasher}; -use trie_db::{DBValue, Trie, TrieMut}; - -#[test] -fn fatdb_to_trie() { - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = RefFatDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - let t = RefFatDB::new(&memdb, &root); - assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); - assert_eq!( - t.iter().unwrap().map(Result::unwrap).collect::>(), - vec![(vec![0x01u8, 0x23], vec![0x01u8, 0x23])] - ); - assert_eq!( - t.key_iter().unwrap().map(Result::unwrap).collect::>(), - vec![vec![0x01u8, 0x23]] - ); -} diff --git a/trie-db/test/src/fatdbmut.rs b/trie-db/test/src/fatdbmut.rs deleted file mode 100644 index 519f5f0f..00000000 --- a/trie-db/test/src/fatdbmut.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use hash_db::{Hasher, EMPTY_PREFIX}; -use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefFatDBMut, RefHasher, RefTrieDBBuilder}; -use trie_db::{Trie, TrieMut}; - -#[test] -fn fatdbmut_to_trie() { - let mut memdb = MemoryDB::, _>::default(); - let mut root = Default::default(); - { - let mut t = RefFatDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - - let t = RefTrieDBBuilder::new(&memdb, &root).build(); - assert_eq!(t.get(&RefHasher::hash(&[0x01u8, 0x23])), Ok(Some(vec![0x01u8, 0x23])),); -} - -#[test] -fn fatdbmut_insert_remove_key_mapping() { - let mut memdb = MemoryDB::, _>::default(); - let mut root = Default::default(); - let key = [0x01u8, 0x23]; - let val = [0x01u8, 0x24]; - let key_hash = RefHasher::hash(&key); - let aux_hash = RefHasher::hash(&key_hash); - let mut t = RefFatDBMut::new(&mut memdb, &mut root); - t.insert(&key, &val).unwrap(); - assert_eq!(t.get(&key), Ok(Some(val.to_vec()))); - assert_eq!(t.db().get(&aux_hash, EMPTY_PREFIX), Some(key.to_vec())); - t.remove(&key).unwrap(); - assert_eq!(t.db().get(&aux_hash, EMPTY_PREFIX), None); -} diff --git a/trie-db/test/src/iter_build.rs b/trie-db/test/src/iter_build.rs index bb4c0d85..76806d36 100644 --- a/trie-db/test/src/iter_build.rs +++ b/trie-db/test/src/iter_build.rs @@ -15,9 +15,8 @@ use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ test_layouts, ExtensionLayout, HashedValueNoExt, HashedValueNoExtThreshold, NoExtensionLayout, - RefHasher, }; -use trie_db::{DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieMut}; +use trie_db::{DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout}; #[test] fn trie_root_empty() { @@ -39,16 +38,17 @@ fn root_extension_one() { fn test_iter(data: Vec<(Vec, Vec)>) { let mut db = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); + let changeset = { + let mut t = TrieDBMutBuilder::::new(&mut db).build(); for i in 0..data.len() { let key: &[u8] = &data[i].0; let value: &[u8] = &data[i].1; t.insert(key, value).unwrap(); } - } - let t = TrieDBBuilder::::new(&db, &root).build(); + t.commit() + }; + changeset.apply_to(&mut db); + let t = TrieDBBuilder::::new(&db, changeset.root.hash()).build(); for (i, kv) in t.iter().unwrap().enumerate() { let (k, v) = kv.unwrap(); let key: &[u8] = &data[i].0; @@ -76,10 +76,10 @@ fn compare_implementations_prefixed(data: Vec<(Vec, Vec)>) { compare_implementations_prefixed_internal::(data.clone()); compare_implementations_prefixed_internal::(data.clone()); } -fn compare_implementations_prefixed_internal(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations::(data, memdb, hashdb); +fn compare_implementations_prefixed_internal(data: Vec<(Vec, Vec)>) + where T::Location: std::fmt::Debug, +{ + reference_trie::compare_implementations::>(data); } fn compare_implementations_h(data: Vec<(Vec, Vec)>) { compare_implementations_h_internal::>(data.clone()); @@ -87,19 +87,16 @@ fn compare_implementations_h(data: Vec<(Vec, Vec)>) { compare_implementations_h_internal::(data.clone()); compare_implementations_h_internal::(data.clone()); } -fn compare_implementations_h_internal(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations::(data.clone(), memdb, hashdb); +fn compare_implementations_h_internal(data: Vec<(Vec, Vec)>) + where T::Location: std::fmt::Debug, +{ + reference_trie::compare_implementations::>(data.clone()); } fn compare_implementations_no_extension_unordered(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations_unordered::(data, memdb, hashdb); + reference_trie::compare_implementations_unordered::>(data); } fn compare_insert_remove(data: Vec<(bool, Vec, Vec)>) { - let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - reference_trie::compare_insert_remove::(data, memdb); + reference_trie::compare_insert_remove::>(data); } fn compare_root(data: Vec<(Vec, Vec)>) { let memdb = MemoryDB::, _>::default(); diff --git a/trie-db/test/src/iterator.rs b/trie-db/test/src/iterator.rs index e36ef6d0..d9909cae 100644 --- a/trie-db/test/src/iterator.rs +++ b/trie-db/test/src/iterator.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::{HashDB, Hasher}; +use hash_db::Hasher; use hex_literal::hex; use reference_trie::test_layouts; use trie_db::{ node::{Node, Value}, DBValue, NibbleSlice, NibbleVec, TrieDBBuilder, TrieDBNodeIterator, TrieError, TrieIterator, - TrieLayout, TrieMut, + TrieLayout, }; type MemoryDB = memory_db::MemoryDB< @@ -31,14 +31,15 @@ fn build_trie_db( pairs: &[(Vec, Vec)], ) -> (MemoryDB, ::Out) { let mut memdb = MemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = trie_db::TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let changeset = { + let mut t = trie_db::TrieDBMutBuilder::::new(&mut memdb).build(); for (x, y) in pairs.iter() { t.insert(x, y).unwrap(); } - } - (memdb, root) + t.commit() + }; + changeset.apply_to(&mut memdb); + (memdb, *changeset.root.hash()) } fn nibble_vec>(bytes: T, len: usize) -> NibbleVec { @@ -266,7 +267,9 @@ fn seek_over_empty_works_internal() { } test_layouts!(iterate_over_incomplete_db, iterate_over_incomplete_db_internal); -fn iterate_over_incomplete_db_internal() { +fn iterate_over_incomplete_db_internal() + where T::Location: std::fmt::Debug, +{ let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), diff --git a/trie-db/test/src/lib.rs b/trie-db/test/src/lib.rs index 7802247e..2c5b7c34 100644 --- a/trie-db/test/src/lib.rs +++ b/trie-db/test/src/lib.rs @@ -14,10 +14,6 @@ //! Tests for trie-db crate. -#[cfg(test)] -mod fatdb; -#[cfg(test)] -mod fatdbmut; #[cfg(test)] mod iter_build; #[cfg(test)] @@ -27,10 +23,6 @@ mod proof; #[cfg(test)] mod recorder; #[cfg(test)] -mod sectriedb; -#[cfg(test)] -mod sectriedbmut; -#[cfg(test)] mod trie_codec; #[cfg(test)] mod triedb; diff --git a/trie-db/test/src/proof.rs b/trie-db/test/src/proof.rs index cca2c70e..365ebf59 100644 --- a/trie-db/test/src/proof.rs +++ b/trie-db/test/src/proof.rs @@ -17,7 +17,7 @@ use reference_trie::{test_layouts, NoExtensionLayout}; use trie_db::{ proof::{generate_proof, verify_proof, VerifyError}, - DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieMut, + DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, }; type MemoryDB = memory_db::MemoryDB< @@ -51,14 +51,13 @@ fn test_generate_proof( // Populate DB with full trie from entries. let (db, root) = { let mut db = >::default(); - let mut root = Default::default(); - { - let mut trie = >::new(&mut db, &mut root).build(); - for (key, value) in entries.iter() { - trie.insert(key, value).unwrap(); - } + let mut trie = >::new(&db).build(); + for (key, value) in entries.iter() { + trie.insert(key, value).unwrap(); } - (db, root) + let commit = trie.commit(); + commit.apply_to(&mut db); + (db, *commit.root.hash()) }; // Generate proof for the given keys.. diff --git a/trie-db/test/src/recorder.rs b/trie-db/test/src/recorder.rs index 485d0153..3ee92375 100644 --- a/trie-db/test/src/recorder.rs +++ b/trie-db/test/src/recorder.rs @@ -16,24 +16,24 @@ use memory_db::{HashKey, MemoryDB}; use reference_trie::{NoExtensionLayout, RefHasher, RefTrieDBBuilder, RefTrieDBMutBuilder}; -use trie_db::{Recorder, Trie, TrieMut}; +use trie_db::{Recorder, Trie}; #[test] fn trie_record() { let mut db = MemoryDB::, _>::default(); - let mut root = Default::default(); - { - let mut x = RefTrieDBMutBuilder::new(&mut db, &mut root).build(); + let mut x = RefTrieDBMutBuilder::new(&mut db).build(); - x.insert(b"dog", b"cat").unwrap(); - x.insert(b"lunch", b"time").unwrap(); - x.insert(b"notdog", b"notcat").unwrap(); - x.insert(b"hotdog", b"hotcat").unwrap(); - x.insert(b"letter", b"confusion").unwrap(); - x.insert(b"insert", b"remove").unwrap(); - x.insert(b"pirate", b"aargh!").unwrap(); - x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); - } + x.insert(b"dog", b"cat").unwrap(); + x.insert(b"lunch", b"time").unwrap(); + x.insert(b"notdog", b"notcat").unwrap(); + x.insert(b"hotdog", b"hotcat").unwrap(); + x.insert(b"letter", b"confusion").unwrap(); + x.insert(b"insert", b"remove").unwrap(); + x.insert(b"pirate", b"aargh!").unwrap(); + x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); + let commit = x.commit(); + let root = *commit.root.hash(); + commit.apply_to(&mut db); { let mut recorder = Recorder::::new(); diff --git a/trie-db/test/src/sectriedb.rs b/trie-db/test/src/sectriedb.rs deleted file mode 100644 index bc04a5c3..00000000 --- a/trie-db/test/src/sectriedb.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use hash_db::Hasher; -use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefHasher, RefSecTrieDB, RefTrieDBMutBuilder}; -use trie_db::{DBValue, Trie, TrieMut}; - -#[test] -fn trie_to_sectrie() { - let mut db = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = RefTrieDBMutBuilder::new(&mut db, &mut root).build(); - t.insert(&RefHasher::hash(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); - } - let t = RefSecTrieDB::new(&db, &root); - assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); -} diff --git a/trie-db/test/src/sectriedbmut.rs b/trie-db/test/src/sectriedbmut.rs deleted file mode 100644 index e984d2e2..00000000 --- a/trie-db/test/src/sectriedbmut.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use hash_db::Hasher; -use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefHasher, RefSecTrieDBMut, RefTrieDBBuilder}; -use trie_db::{DBValue, Trie, TrieMut}; - -#[test] -fn sectrie_to_trie() { - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = RefSecTrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - let t = RefTrieDBBuilder::new(&memdb, &root).build(); - assert_eq!(t.get(&RefHasher::hash(&[0x01u8, 0x23])).unwrap().unwrap(), vec![0x01u8, 0x23],); -} diff --git a/trie-db/test/src/trie_codec.rs b/trie-db/test/src/trie_codec.rs index 17b52cb5..ac5d5389 100644 --- a/trie-db/test/src/trie_codec.rs +++ b/trie-db/test/src/trie_codec.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; +use hash_db::{Hasher, EMPTY_PREFIX}; use reference_trie::{test_layouts, ExtensionLayout}; use trie_db::{ decode_compact, encode_compact, DBValue, NodeCodec, Recorder, Trie, TrieDBBuilder, - TrieDBMutBuilder, TrieError, TrieLayout, TrieMut, + TrieDBMutBuilder, TrieError, TrieLayout, }; type MemoryDB = memory_db::MemoryDB< @@ -32,14 +32,13 @@ fn test_encode_compact( // Populate DB with full trie from entries. let (db, root) = { let mut db = >::default(); - let mut root = Default::default(); - { - let mut trie = >::new(&mut db, &mut root).build(); - for (key, value) in entries.iter() { - trie.insert(key, value).unwrap(); - } + let mut trie = >::new(&mut db).build(); + for (key, value) in entries.iter() { + trie.insert(key, value).unwrap(); } - (db, root) + let commit = trie.commit(); + commit.apply_to(&mut db); + (db, commit.root_hash()) }; // Lookup items in trie while recording traversed nodes. @@ -77,7 +76,7 @@ fn test_decode_compact( ) { // Reconstruct the partial DB from the compact encoding. let mut db = MemoryDB::::default(); - let (root, used) = decode_compact::(&mut db, encoded).unwrap(); + let (root, used) = decode_compact::(&mut db, encoded).unwrap(); assert_eq!(root, expected_root); assert_eq!(used, expected_used); @@ -130,7 +129,7 @@ fn trie_decoding_fails_with_incomplete_database_internal() { // Reconstruct the partial DB from the compact encoding. let mut db = MemoryDB::::default(); - match decode_compact::(&mut db, &encoded[..encoded.len() - 1]) { + match decode_compact::(&mut db, &encoded[..encoded.len() - 1]) { Err(err) => match *err { TrieError::IncompleteDatabase(_) => {}, _ => panic!("got unexpected TrieError"), @@ -160,14 +159,14 @@ fn encoding_node_owned_and_decoding_node_works() { // Populate DB with full trie from entries. let mut recorder = { let mut db = >::default(); - let mut root = Default::default(); let mut recorder = Recorder::::new(); - { - let mut trie = >::new(&mut db, &mut root).build(); - for (key, value) in entries.iter() { - trie.insert(key, value).unwrap(); - } + let mut trie = >::new(&mut db).build(); + for (key, value) in entries.iter() { + trie.insert(key, value).unwrap(); } + let commit = trie.commit(); + commit.apply_to(&mut db); + let root = commit.root_hash(); let trie = TrieDBBuilder::::new(&db, &root) .with_recorder(&mut recorder) @@ -181,7 +180,7 @@ fn encoding_node_owned_and_decoding_node_works() { for record in recorder.drain() { let node = - <::Codec as NodeCodec>::decode(&record.data).unwrap(); + <::Codec as NodeCodec>::decode(&record.data, &[(); 0]).unwrap(); let node_owned = node.to_owned_node::().unwrap(); assert_eq!(record.data, node_owned.to_encoded::<::Codec>()); diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index 9825ab50..c988b89d 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -14,7 +14,7 @@ use std::ops::Deref; -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; +use hash_db::{Hasher, EMPTY_PREFIX}; use hex_literal::hex; use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ @@ -22,7 +22,7 @@ use reference_trie::{ }; use trie_db::{ encode_compact, CachedValue, DBValue, Lookup, NibbleSlice, Recorder, Trie, TrieCache, - TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieMut, + TrieDBBuilder, TrieDBMutBuilder, TrieLayout, }; type PrefixedMemoryDB = @@ -38,13 +38,13 @@ fn iterator_works_internal() { ]; let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (x, y) in &pairs { - t.insert(x, y).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (x, y) in &pairs { + t.insert(x, y).unwrap(); } + let commit = t.commit(); + commit.apply_to(&mut memdb); + let root = commit.root_hash(); let trie = TrieDBBuilder::::new(&memdb, &root).build(); @@ -66,13 +66,13 @@ fn iterator_seek_works_internal() { ]; let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (x, y) in &pairs { - t.insert(x, y).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (x, y) in &pairs { + t.insert(x, y).unwrap(); } + let commit = t.commit(); + commit.apply_to(&mut memdb); + let root = commit.root_hash(); let t = TrieDBBuilder::::new(&memdb, &root).build(); @@ -99,13 +99,11 @@ fn iterator_internal() { let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"B".to_vec()]; let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for x in &d { - t.insert(x, x).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for x in &d { + t.insert(x, x).unwrap(); } + let root = t.commit().apply_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); assert_eq!( @@ -121,13 +119,11 @@ fn iterator_seek_internal() { let vals = vec![vec![0; 32], vec![1; 32], vec![2; 32], vec![4; 32], vec![3; 32]]; let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (k, val) in d.iter().zip(vals.iter()) { - t.insert(k, val.as_slice()).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (k, val) in d.iter().zip(vals.iter()) { + t.insert(k, val.as_slice()).unwrap(); } + let root = t.commit().apply_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = t.iter().unwrap(); @@ -176,13 +172,11 @@ where T: TrieLayout, { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (index, key) in keys.iter().enumerate() { - t.insert(&array_bytes::hex2bytes(key).unwrap(), &[index as u8]).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (index, key) in keys.iter().enumerate() { + t.insert(&array_bytes::hex2bytes(key).unwrap(), &[index as u8]).unwrap(); } + let root = t.commit().apply_to(&mut memdb); let mut t = TrieDBBuilder::::new(&memdb, &root).build(); callback(&mut t); @@ -293,12 +287,10 @@ fn iterator_prefixed_then_seek_testcase_5() { test_layouts!(get_length_with_extension, get_length_with_extension_internal); fn get_length_with_extension_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(b"A", b"ABC").unwrap(); - t.insert(b"B", b"ABCBAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(b"A", b"ABC").unwrap(); + t.insert(b"B", b"ABCBAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(); + let root = t.commit().apply_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()).unwrap(), Some(3)); @@ -307,18 +299,18 @@ fn get_length_with_extension_internal() { } test_layouts!(debug_output_supports_pretty_print, debug_output_supports_pretty_print_internal); -fn debug_output_supports_pretty_print_internal() { +fn debug_output_supports_pretty_print_internal() + where T::Location: std::fmt::Debug, +{ let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"B".to_vec()]; let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let root = { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for x in &d { - t.insert(x, x).unwrap(); - } - t.root().clone() - }; + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for x in &d { + t.insert(x, x).unwrap(); + } + let root = t.commit().apply_to(&mut memdb); + let t = TrieDBBuilder::::new(&memdb, &root).build(); if T::USE_EXTENSION { @@ -393,19 +385,17 @@ test_layouts!( ); fn test_lookup_with_corrupt_data_returns_decoder_error_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(b"A", b"ABC").unwrap(); - t.insert(b"B", b"ABCBA").unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(b"A", b"ABC").unwrap(); + t.insert(b"B", b"ABCBA").unwrap(); + let root = t.commit().apply_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); // query for an invalid data type to trigger an error let q = |x: &[u8]| x.len() < 64; let lookup = Lookup:: { db: t.db(), query: q, hash: root, cache: None, recorder: None }; - let query_result = lookup.look_up(&b"A"[..], NibbleSlice::new(b"A")); + let query_result = lookup.look_up(&b"A"[..], NibbleSlice::new(b"A"), Default::default()); assert_eq!(query_result.unwrap().unwrap(), true); } @@ -419,13 +409,11 @@ fn test_recorder_internal() { ]; let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in &key_value { - t.insert(key, value).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); } + let root = t.commit().apply_to(&mut memdb); let mut recorder = Recorder::::new(); { @@ -461,14 +449,12 @@ fn test_recorder_with_cache_internal() { ]; let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in &key_value { - t.insert(key, value).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); } + let root = t.commit().apply_to(&mut memdb); let mut cache = TestTrieCache::::default(); @@ -480,7 +466,7 @@ fn test_recorder_with_cache_internal() { } // Root should now be cached. - assert!(cache.get_node(&root).is_some()); + assert!(cache.get_node(&root, Default::default()).is_some()); // Also the data should be cached. let value = cache.lookup_value_for_key(&key_value[1].0).unwrap(); @@ -548,14 +534,13 @@ fn test_recorder_with_cache_get_hash_internal() { ]; let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in &key_value { - t.insert(key, value).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); } + let root = t.commit().apply_to(&mut memdb); + let mut cache = TestTrieCache::::default(); @@ -570,7 +555,7 @@ fn test_recorder_with_cache_get_hash_internal() { } // Root should now be cached. - assert!(cache.get_node(&root).is_some()); + assert!(cache.get_node(&root, Default::default()).is_some()); // Also the data should be cached. if T::MAX_INLINE_VALUE.map_or(true, |l| l as usize > key_value[1].1.len()) { @@ -581,7 +566,7 @@ fn test_recorder_with_cache_get_hash_internal() { } else { assert!(matches!( cache.lookup_value_for_key(&key_value[1].0).unwrap(), - CachedValue::ExistingHash(hash) if *hash == T::Hash::hash(&key_value[1].1) + CachedValue::ExistingHash(hash, _) if *hash == T::Hash::hash(&key_value[1].1) )); } @@ -650,13 +635,11 @@ fn iterator_seek_with_recorder_internal() { let vals = vec![vec![0; 64], vec![1; 64], vec![2; 64], vec![3; 64]]; let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (k, val) in d.iter().zip(vals.iter()) { - t.insert(k, val.as_slice()).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (k, val) in d.iter().zip(vals.iter()) { + t.insert(k, val.as_slice()).unwrap(); } + let root = t.commit().apply_to(&mut memdb); let mut recorder = Recorder::::new(); { @@ -692,15 +675,20 @@ fn test_cache_internal() { ]; let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); let mut cache = TestTrieCache::::default(); - { + let changeset = { let mut t = - TrieDBMutBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); + TrieDBMutBuilder::::new(&memdb).with_cache(&mut cache).build(); for (key, value) in &key_value { t.insert(key, value).unwrap(); } + t.commit() + }; + let root = changeset.apply_to(&mut memdb); + let t = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); + for (key, _) in &key_value { + t.get(key).unwrap(); } // Ensure that when we cache the same value multiple times under different keys, @@ -719,13 +707,12 @@ fn test_cache_internal() { let cached_value = cache.lookup_value_for_key(&b"AB"[..]).unwrap().clone(); assert_eq!(cached_value.data().flatten().unwrap(), vec![3u8; 4]); - { - let mut t = - TrieDBMutBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); - for (key, value) in &key_value { - t.insert(key, value).unwrap(); - } + let mut t = + TrieDBMutBuilder::::new(&memdb).with_cache(&mut cache).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); } + let root = t.commit().apply_to(&mut memdb); assert_eq!( cache.lookup_value_for_key(&b"AB"[..]).unwrap().data().flatten().unwrap(), @@ -737,7 +724,7 @@ fn test_cache_internal() { cache.clear_node_cache(); { - let t = TrieDBBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); + let t = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); for (key, value) in &key_value { assert_eq!(*value, t.get(key).unwrap().unwrap()); } @@ -747,7 +734,7 @@ fn test_cache_internal() { cache.clear_node_cache(); { - let t = TrieDBBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); + let t = TrieDBBuilder::::new(&mut memdb, &root).with_cache(&mut cache).build(); for (key, value) in &key_value { assert_eq!(T::Hash::hash(value), t.get_hash(key).unwrap().unwrap()); } @@ -762,13 +749,11 @@ fn test_record_value() { // Add some initial data to the trie let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in key_value.iter() { - t.insert(key, value).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (key, value) in key_value.iter() { + t.insert(key, value).unwrap(); } + let root = t.commit().apply_to(&mut memdb); // Value access would record a two nodes (branch and leaf with value 32 len inline). let mut recorder = Recorder::::new(); diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 02316892..410631bc 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -25,7 +25,7 @@ use reference_trie::{ }; use trie_db::{ DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, - TrieError, TrieLayout, TrieMut, Value, + TrieError, TrieLayout, Value, CachedValue, }; use trie_standardmap::*; @@ -35,12 +35,10 @@ type MemoryDBProof = MemoryDB<::Hash, HashKey<::Hash>, DBValue>; fn populate_trie<'db, T: TrieLayout>( - db: &'db mut dyn HashDB, - root: &'db mut ::Out, + db: &'db dyn HashDB, v: &[(Vec, Vec)], ) -> TrieDBMut<'db, T> { - let mut t = TrieDBMutBuilder::::new(db, root).build(); - + let mut t = TrieDBMutBuilder::::new(db).build(); for i in 0..v.len() { let key: &[u8] = &v[i].0; let val: &[u8] = &v[i].1; @@ -96,51 +94,49 @@ fn playpen_internal() { let real = reference_trie_root::(x.clone()); let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); - - memtrie.commit(); - if *memtrie.root() != real { + let memtrie = populate_trie::(&memdb, &x); + let root = memtrie.commit().apply_to(&mut memdb); + if root != real { println!("TRIE MISMATCH"); println!(); - println!("{:?} vs {:?}", memtrie.root(), real); + println!("{:?} vs {:?}", root, real); for i in &x { println!("{:#x?} -> {:#x?}", i.0, i.1); } } - assert_eq!(*memtrie.root(), real); + assert_eq!(root, real); + + let mut memtrie = TrieDBMutBuilder::::from_existing(&memdb, root).build(); assert!(unpopulate_trie(&mut memtrie, &x), "{:?}", (test_i, initial_seed)); - memtrie.commit(); + let root = memtrie.commit().apply_to(&mut memdb); let hashed_null_node = reference_hashed_null_node::(); - if *memtrie.root() != hashed_null_node { + if root != hashed_null_node { println!("- TRIE MISMATCH"); println!(); - println!("{:#x?} vs {:#x?}", memtrie.root(), hashed_null_node); + println!("{:#x?} vs {:#x?}", root, hashed_null_node); for i in &x { println!("{:#x?} -> {:#x?}", i.0, i.1); } } - assert_eq!(*memtrie.root(), hashed_null_node); + assert_eq!(root, hashed_null_node); } } test_layouts!(init, init_internal); fn init_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let memdb = PrefixedMemoryDB::::default(); + let t = TrieDBMutBuilder::::new(&memdb).build(); let hashed_null_node = reference_hashed_null_node::(); - assert_eq!(*t.root(), hashed_null_node); + assert_eq!(t.commit().root_hash(), hashed_null_node); } test_layouts!(insert_on_empty, insert_on_empty_internal); fn insert_on_empty_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let memdb = PrefixedMemoryDB::::default(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!( - *t.root(), + t.commit().root_hash(), reference_trie_root::(vec![(vec![0x01u8, 0x23], vec![0x01u8, 0x23])]), ); } @@ -150,17 +146,15 @@ fn remove_to_empty_internal() { let big_value = b"00000000000000000000000000000000"; let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - - t.insert(&[0x01], big_value).unwrap(); - t.insert(&[0x01, 0x23], big_value).unwrap(); - t.insert(&[0x01, 0x34], big_value).unwrap(); - t.remove(&[0x01]).unwrap(); - t.remove(&[0x01, 0x23]).unwrap(); - t.remove(&[0x01, 0x34]).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + + t.insert(&[0x01], big_value).unwrap(); + t.insert(&[0x01, 0x23], big_value).unwrap(); + t.insert(&[0x01, 0x34], big_value).unwrap(); + t.remove(&[0x01]).unwrap(); + t.remove(&[0x01, 0x23]).unwrap(); + t.remove(&[0x01, 0x34]).unwrap(); + t.commit().apply_to(&mut memdb); assert_eq!(memdb.keys().len(), 0); } @@ -169,21 +163,18 @@ fn remove_to_empty_checked_internal() { let big_value = b"00000000000000000000000000000000"; let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - - t.insert(&[0x01], big_value).unwrap(); - t.insert(&[0x01, 0x23], big_value).unwrap(); - t.insert(&[0x01, 0x34], big_value).unwrap(); - t.commit(); - assert_eq!(t.get(&[0x01]).unwrap(), Some(big_value.to_vec()),); - assert_eq!(t.get(&[0x01, 0x34]).unwrap(), Some(big_value.to_vec()),); - t.commit(); - t.remove(&[0x01]).unwrap(); - t.remove(&[0x01, 0x23]).unwrap(); - t.remove(&[0x01, 0x34]).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01], big_value).unwrap(); + t.insert(&[0x01, 0x23], big_value).unwrap(); + t.insert(&[0x01, 0x34], big_value).unwrap(); + let root = t.commit().apply_to(&mut memdb); + let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); + assert_eq!(t.get(&[0x01]).unwrap(), Some(big_value.to_vec()),); + assert_eq!(t.get(&[0x01, 0x34]).unwrap(), Some(big_value.to_vec()),); + t.remove(&[0x01]).unwrap(); + t.remove(&[0x01, 0x23]).unwrap(); + t.remove(&[0x01, 0x34]).unwrap(); + t.commit().apply_to(&mut memdb); assert_eq!(memdb.keys().len(), 0); } @@ -194,16 +185,14 @@ fn remove_to_empty_no_extension_internal() { let big_value3 = b"00000000000000000000000000000004"; let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - - t.insert(&[0x01, 0x23], big_value3).unwrap(); - t.insert(&[0x01], big_value2).unwrap(); - t.insert(&[0x01, 0x34], big_value).unwrap(); - t.remove(&[0x01]).unwrap(); - // commit on drop - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + + t.insert(&[0x01, 0x23], big_value3).unwrap(); + t.insert(&[0x01], big_value2).unwrap(); + t.insert(&[0x01, 0x34], big_value).unwrap(); + t.remove(&[0x01]).unwrap(); + + let root = t.commit().apply_to(&mut memdb); assert_eq!( &root, &reference_trie::calc_root::(vec![ @@ -216,12 +205,12 @@ fn remove_to_empty_no_extension_internal() { test_layouts!(insert_replace_root, insert_replace_root_internal); fn insert_replace_root_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); + let root = t.commit().apply_to(&mut memdb); assert_eq!( - *t.root(), + root, reference_trie_root::(vec![(vec![0x01u8, 0x23], vec![0x23u8, 0x45])]), ); } @@ -229,12 +218,12 @@ fn insert_replace_root_internal() { test_layouts!(insert_make_branch_root, insert_make_branch_root_internal); fn insert_make_branch_root_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); + let root = t.commit().apply_to(&mut memdb); assert_eq!( - *t.root(), + root, reference_trie_root::(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) @@ -245,13 +234,13 @@ fn insert_make_branch_root_internal() { test_layouts!(insert_into_branch_root, insert_into_branch_root_internal); fn insert_into_branch_root_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); + let root = t.commit().apply_to(&mut memdb); assert_eq!( - *t.root(), + root, reference_trie_root::(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), @@ -263,12 +252,12 @@ fn insert_into_branch_root_internal() { test_layouts!(insert_value_into_branch_root, insert_value_into_branch_root_internal); fn insert_value_into_branch_root_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[], &[0x0]).unwrap(); + let root = t.commit().apply_to(&mut memdb); assert_eq!( - *t.root(), + root, reference_trie_root::(vec![ (vec![], vec![0x0]), (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), @@ -279,12 +268,12 @@ fn insert_value_into_branch_root_internal() { test_layouts!(insert_split_leaf, insert_split_leaf_internal); fn insert_split_leaf_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); + let root = t.commit().apply_to(&mut memdb); assert_eq!( - *t.root(), + root, reference_trie_root::(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), @@ -295,13 +284,13 @@ fn insert_split_leaf_internal() { test_layouts!(insert_split_extenstion, insert_split_extenstion_internal); fn insert_split_extenstion_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); + let root = t.commit().apply_to(&mut memdb); assert_eq!( - *t.root(), + root, reference_trie_root::(vec![ (vec![0x01, 0x23, 0x45], vec![0x01]), (vec![0x01, 0xf3, 0x45], vec![0x02]), @@ -316,12 +305,12 @@ fn insert_big_value_internal() { let big_value1 = b"11111111111111111111111111111111"; let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], big_value0).unwrap(); t.insert(&[0x11u8, 0x23], big_value1).unwrap(); + let root = t.commit().apply_to(&mut memdb); assert_eq!( - *t.root(), + root, reference_trie_root::(vec![ (vec![0x01u8, 0x23], big_value0.to_vec()), (vec![0x11u8, 0x23], big_value1.to_vec()) @@ -334,12 +323,12 @@ fn insert_duplicate_value_internal() { let big_value = b"00000000000000000000000000000000"; let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], big_value).unwrap(); t.insert(&[0x11u8, 0x23], big_value).unwrap(); + let root = t.commit().apply_to(&mut memdb); assert_eq!( - *t.root(), + root, reference_trie_root::(vec![ (vec![0x01u8, 0x23], big_value.to_vec()), (vec![0x11u8, 0x23], big_value.to_vec()) @@ -350,24 +339,22 @@ fn insert_duplicate_value_internal() { test_layouts!(test_at_empty, test_at_empty_internal); fn test_at_empty_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let t = TrieDBMutBuilder::::new(&mut memdb).build(); assert_eq!(t.get(&[0x5]).unwrap(), None); } test_layouts!(test_at_one_and_two, test_at_one_and_two_internal); fn test_at_one_and_two_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); - t.commit(); - assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); - t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x24]).unwrap(); - } - let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); + let root = t.commit().apply_to(&mut memdb); + let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); + assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); + t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x24]).unwrap(); + let root = t.commit().apply_to(&mut memdb); + let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x25]).unwrap(); // This test that middle node get resolved correctly (modified // triedbmut node due to change of child node). @@ -377,8 +364,7 @@ fn test_at_one_and_two_internal() { test_layouts!(test_at_three, test_at_three_internal); fn test_at_three_internal() { let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); @@ -386,7 +372,8 @@ fn test_at_three_internal() { assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), vec![0xf1u8, 0x23]); assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), vec![0x81u8, 0x23]); assert_eq!(t.get(&[0x82, 0x23]).unwrap(), None); - t.commit(); + let root = t.commit().apply_to(&mut memdb); + let t = TrieDBMutBuilder::::from_existing(&memdb, root).build(); assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), vec![0xf1u8, 0x23]); assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), vec![0x81u8, 0x23]); @@ -395,9 +382,8 @@ fn test_at_three_internal() { #[test] fn test_nibbled_branch_changed_value() { - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - let mut t = reference_trie::RefTrieDBMutNoExtBuilder::new(&mut memdb, &mut root).build(); + let memdb = MemoryDB::, DBValue>::default(); + let mut t = reference_trie::RefTrieDBMutNoExtBuilder::new(&memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x23, 0x11], &[0xf1u8, 0x23]).unwrap(); assert_eq!(t.get(&[0x01u8, 0x23]).unwrap(), Some(vec![0x01u8, 0x23])); @@ -418,42 +404,37 @@ fn stress_internal() { let real = reference_trie_root::(x.clone()); let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); + let memtrie = populate_trie::(&mut memdb, &x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); let mut memdb2 = PrefixedMemoryDB::::default(); - let mut root2 = Default::default(); - let mut memtrie_sorted = populate_trie::(&mut memdb2, &mut root2, &y); - if *memtrie.root() != real || *memtrie_sorted.root() != real { + let memtrie_sorted = populate_trie::(&mut memdb2, &y); + let root = memtrie.commit().apply_to(&mut memdb); + let root2 = memtrie_sorted.commit().apply_to(&mut memdb2); + if root != real || root2 != real { println!("TRIE MISMATCH"); println!(); - println!("ORIGINAL... {:#x?}", memtrie.root()); + println!("ORIGINAL... {:#x?}", root); for i in &x { println!("{:#x?} -> {:#x?}", i.0, i.1); } - println!("SORTED... {:#x?}", memtrie_sorted.root()); + println!("SORTED... {:#x?}", root2); for i in &y { println!("{:#x?} -> {:#x?}", i.0, i.1); } } - assert_eq!(*memtrie.root(), real); - assert_eq!(*memtrie_sorted.root(), real); + assert_eq!(root, real); + assert_eq!(root2, real); } } test_layouts!(test_trie_existing, test_trie_existing_internal); fn test_trie_existing_internal() { let mut db = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - - { - let _ = TrieDBMutBuilder::::from_existing(&mut db, &mut root); - } + let mut t = TrieDBMutBuilder::::new(&mut db).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + let root = t.commit().apply_to(&mut db); + let _ = TrieDBMutBuilder::::from_existing(&db, root); } test_layouts!(insert_empty, insert_empty_internal); @@ -469,21 +450,23 @@ fn insert_empty_internal() { .make_with(&mut seed); let mut db = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&db).build(); for &(ref key, ref value) in &x { t.insert(key, value).unwrap(); } + let root = t.commit().apply_to(&mut db); - assert_eq!(*t.root(), reference_trie_root::(x.clone())); + assert_eq!(root, reference_trie_root::(x.clone())); + let mut t = TrieDBMutBuilder::::from_existing(&db, root).build(); for &(ref key, _) in &x { t.insert(key, &[]).unwrap(); } - assert!(t.is_empty()); + let root = t.commit().apply_to(&mut db); + let hashed_null_node = reference_hashed_null_node::(); - assert_eq!(*t.root(), hashed_null_node); + assert_eq!(root, hashed_null_node); } test_layouts!(return_old_values, return_old_values_internal); @@ -500,8 +483,7 @@ fn return_old_values_internal() { .make_with(&mut seed); let mut db = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut db).build(); for &(ref key, ref value) in &x { assert!(t.insert(key, value).unwrap() == None); if threshold.map(|t| value.len() < t as usize).unwrap_or(true) { @@ -523,23 +505,24 @@ fn return_old_values_internal() { #[test] fn insert_empty_allowed() { let mut db = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - let mut t = reference_trie::RefTrieDBMutAllowEmptyBuilder::new(&mut db, &mut root).build(); + let mut t = reference_trie::RefTrieDBMutAllowEmptyBuilder::new(&db).build(); t.insert(b"test", &[]).unwrap(); + let root = t.commit().apply_to(&mut db); assert_eq!( - *t.root(), + root, reference_trie_root::(vec![( b"test".to_vec(), Vec::new() )],) ); + let t = reference_trie::RefTrieDBMutAllowEmptyBuilder::from_existing(&db, root).build(); assert_eq!(t.get(b"test").unwrap(), Some(Vec::new())); } #[test] fn register_proof_without_value() { - use hash_db::{AsHashDB, Prefix}; + use hash_db::Prefix; use reference_trie::HashedValueNoExtThreshold; use std::{cell::RefCell, collections::HashMap}; @@ -552,8 +535,8 @@ fn register_proof_without_value() { ]; let mut memdb = MemoryDB::default(); - let mut root = Default::default(); - let _ = populate_trie::(&mut memdb, &mut root, &x); + let t = populate_trie::(&mut memdb, &x); + let root = t.commit().apply_to(&mut memdb); { let trie = TrieDBBuilder::::new(&memdb, &root).build(); println!("{:?}", trie); @@ -567,56 +550,31 @@ fn register_proof_without_value() { unsafe impl Send for ProofRecorder {} unsafe impl Sync for ProofRecorder {} - impl HashDB for ProofRecorder { - fn get(&self, key: &::Out, prefix: Prefix) -> Option { - let v = self.db.get(key, prefix); - if let Some(v) = v.as_ref() { + impl HashDB for ProofRecorder { + fn get(&self, key: &::Out, prefix: Prefix, _location: ()) -> Option<(DBValue, Vec<()>)> { + let v = HashDB::get(&self.db, key, prefix, ()); + if let Some((v, _)) = v.as_ref() { self.record.borrow_mut().entry(key[..].to_vec()).or_insert_with(|| v.clone()); } v } - - fn contains(&self, key: &::Out, prefix: Prefix) -> bool { - self.get(key, prefix).is_some() - } - - fn emplace(&mut self, key: ::Out, prefix: Prefix, value: DBValue) { - self.db.emplace(key, prefix, value) - } - - fn insert(&mut self, prefix: Prefix, value: &[u8]) -> ::Out { - self.db.insert(prefix, value) - } - - fn remove(&mut self, key: &::Out, prefix: Prefix) { - self.db.remove(key, prefix) - } - } - - impl AsHashDB for ProofRecorder { - fn as_hash_db(&self) -> &dyn HashDB { - self - } - fn as_hash_db_mut<'a>(&'a mut self) -> &'a mut (dyn HashDB + 'a) { - self + fn contains(&self, key: &::Out, prefix: Prefix, _locatoin: ()) -> bool { + self.get(key, prefix, ()).is_some() } } let mut memdb = ProofRecorder { db: memdb, record: Default::default() }; let root_proof = root.clone(); - { - let mut trie = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); - // touch te value (test1 remains untouch). - trie.get(b"te").unwrap(); - // cut test_1234 prefix - trie.insert(b"test12", &[2u8; 36][..]).unwrap(); - // remove 1234 - trie.remove(b"test1234").unwrap(); - - // proof should contain value for 'te' only. - } - + let mut trie = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); + // touch te value (test1 remains untouch). + trie.get(b"te").unwrap(); + // cut test_1234 prefix + trie.insert(b"test12", &[2u8; 36][..]).unwrap(); + // remove 1234 + trie.remove(b"test1234").unwrap(); + + // proof should contain value for 'te' only. type MemoryDBProof = memory_db::MemoryDB, DBValue>; let mut memdb_from_proof = MemoryDBProof::default(); for (_key, value) in memdb.record.into_inner().into_iter() { @@ -627,10 +585,10 @@ fn register_proof_without_value() { let root_unpacked = root_proof.clone(); let mut memdb_from_proof = db_unpacked.clone(); - let mut root_proof = root_unpacked.clone(); + let root_proof = root_unpacked.clone(); { let mut trie = - TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, &mut root_proof) + TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, root_proof) .build(); trie.get(b"te").unwrap(); trie.insert(b"test12", &[2u8; 36][..]).unwrap(); @@ -638,7 +596,7 @@ fn register_proof_without_value() { } let mut memdb_from_proof = db_unpacked.clone(); - let mut root_proof = root_unpacked.clone(); + let root_proof = root_unpacked.clone(); { use trie_db::Trie; let trie = TrieDBBuilder::::new(&memdb_from_proof, &root_proof).build(); @@ -651,7 +609,7 @@ fn register_proof_without_value() { { let trie = - TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, &mut root_proof) + TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, root_proof) .build(); assert!(trie.get(b"te").unwrap().is_some()); assert!(matches!( @@ -672,21 +630,19 @@ fn test_recorder_internal() { // Add some initial data to the trie let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in key_value.iter().take(1) { - t.insert(key, value).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (key, value) in key_value.iter().take(1) { + t.insert(key, value).unwrap(); } - + let root = t.commit().apply_to(&mut memdb); + // Add more data, but this time only to the overlay. // While doing that we record all trie accesses to replay this operation. let mut recorder = Recorder::::new(); let mut overlay = memdb.clone(); - let mut new_root = root; + let new_root = root; { - let mut trie = TrieDBMutBuilder::::from_existing(&mut overlay, &mut new_root) + let mut trie = TrieDBMutBuilder::::from_existing(&mut overlay, new_root) .with_recorder(&mut recorder) .build(); @@ -701,10 +657,10 @@ fn test_recorder_internal() { } // Replay the it, but this time we use the proof. - let mut validated_root = root; + let validated_root = root; { let mut trie = - TrieDBMutBuilder::::from_existing(&mut partial_db, &mut validated_root).build(); + TrieDBMutBuilder::::from_existing(&mut partial_db, validated_root).build(); for (key, value) in key_value.iter().skip(1) { trie.insert(key, value).unwrap(); @@ -725,14 +681,13 @@ fn test_recorder_with_cache_internal() { // Add some initial data to the trie let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in key_value.iter().take(1) { - t.insert(key, value).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (key, value) in key_value.iter().take(1) { + t.insert(key, value).unwrap(); } - + let root = t.commit().apply_to(&mut memdb); + let mut validated_root = root; + let mut cache = TestTrieCache::::default(); { @@ -743,22 +698,25 @@ fn test_recorder_with_cache_internal() { } // Root should now be cached. - assert!(cache.get_node(&root).is_some()); + assert!(cache.get_node(&root, Default::default()).is_some()); // Add more data, but this time only to the overlay. // While doing that we record all trie accesses to replay this operation. let mut recorder = Recorder::::new(); let mut overlay = memdb.clone(); - let mut new_root = root; - { - let mut trie = TrieDBMutBuilder::::from_existing(&mut overlay, &mut new_root) - .with_recorder(&mut recorder) - .with_cache(&mut cache) - .build(); + let mut trie = TrieDBMutBuilder::::from_existing(&mut overlay, root) + .with_recorder(&mut recorder) + .with_cache(&mut cache) + .build(); - for (key, value) in key_value.iter().skip(1) { - trie.insert(key, value).unwrap(); - } + for (key, value) in key_value.iter().skip(1) { + trie.insert(key, value).unwrap(); + } + let new_root = trie.commit().apply_to(&mut overlay); + + let t = TrieDBBuilder::::new(&overlay, &new_root).with_cache(&mut cache).build(); + for (key, _) in key_value.iter().skip(1) { + t.get(key).unwrap(); } for (key, value) in key_value.iter().skip(1) { @@ -774,14 +732,14 @@ fn test_recorder_with_cache_internal() { } // Replay the it, but this time we use the proof. - let mut validated_root = root; { let mut trie = - TrieDBMutBuilder::::from_existing(&mut partial_db, &mut validated_root).build(); + TrieDBMutBuilder::::from_existing(&partial_db, validated_root).build(); for (key, value) in key_value.iter().skip(1) { trie.insert(key, value).unwrap(); } + validated_root = trie.commit().apply_to(&mut partial_db); } assert_eq!(new_root, validated_root); @@ -801,26 +759,29 @@ fn test_insert_remove_data_with_cache_internal() { let mut cache = TestTrieCache::::default(); let mut recorder = Recorder::::new(); let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut trie = TrieDBMutBuilder::::new(&mut memdb, &mut root) - .with_recorder(&mut recorder) - .with_cache(&mut cache) - .build(); + let mut trie = TrieDBMutBuilder::::new(&mut memdb) + .with_recorder(&mut recorder) + .with_cache(&mut cache) + .build(); - // Add all values - for (key, value) in key_value.iter() { - trie.insert(key, value).unwrap(); - } + // Add all values + for (key, value) in key_value.iter() { + trie.insert(key, value).unwrap(); + } - // Remove only the last 2 elements - for (key, _) in key_value.iter().skip(3) { - let _ = trie.remove(key); - } + // Remove only the last 2 elements + for (key, _) in key_value.iter().skip(3) { + let _ = trie.remove(key); + } + let mut memdb = MemoryDB::, DBValue>::default(); + let root = trie.commit().apply_to(&mut memdb); + let t = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); + for (key, _) in &key_value { + t.get(key).unwrap(); } // Then only the first 3 elements should be in the cache and the last - // two ones should not be there. + // two ones should be added as non-existent. for (key, value) in key_value.iter().take(3) { let key_str = String::from_utf8_lossy(key); @@ -833,6 +794,6 @@ fn test_insert_remove_data_with_cache_internal() { } for (key, _) in key_value.iter().skip(3) { - assert!(cache.lookup_value_for_key(key).is_none()); + assert!(matches!(cache.lookup_value_for_key(key).unwrap(), CachedValue::NonExisting)); } } diff --git a/trie-eip1186/CHANGELOG.md b/trie-eip1186/CHANGELOG.md deleted file mode 100644 index 38b4e85f..00000000 --- a/trie-eip1186/CHANGELOG.md +++ /dev/null @@ -1,8 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [Unreleased] -Support eip 1186 trie proofs. [#146](https://github.com/paritytech/trie/pull/146) diff --git a/trie-eip1186/Cargo.toml b/trie-eip1186/Cargo.toml deleted file mode 100644 index 5254d261..00000000 --- a/trie-eip1186/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "trie-eip1186" -version = "0.5.0" -authors = ["Parity Technologies "] -description = "EIP-1186 compliant proof generation and verification" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -trie-db = { path = "../trie-db", default-features = false, version = "0.27.0"} -hash-db = { path = "../hash-db", default-features = false, version = "0.16.0"} - -[features] -default = ["std"] -std = [ - "trie-db/std", - "hash-db/std", -] diff --git a/trie-eip1186/src/eip1186.rs b/trie-eip1186/src/eip1186.rs deleted file mode 100644 index 37ad106d..00000000 --- a/trie-eip1186/src/eip1186.rs +++ /dev/null @@ -1,311 +0,0 @@ -use crate::rstd::{result::Result, vec::Vec}; -use hash_db::{HashDBRef, Hasher}; -use trie_db::{ - node::{decode_hash, Node, NodeHandle, Value}, - recorder::Recorder, - CError, DBValue, NibbleSlice, NodeCodec, Result as TrieResult, Trie, TrieDBBuilder, TrieHash, - TrieLayout, -}; - -/// Generate an eip-1186 compatible proof for key-value pairs in a trie given a key. -pub fn generate_proof( - db: &dyn HashDBRef, - root: &TrieHash, - key: &[u8], -) -> TrieResult<(Vec>, Option>), TrieHash, CError> -where - L: TrieLayout, -{ - let mut recorder = Recorder::::new(); - - let item = { - let trie = TrieDBBuilder::::new(db, root).with_recorder(&mut recorder).build(); - trie.get(key)? - }; - - let proof: Vec> = recorder.drain().into_iter().map(|r| r.data).collect(); - Ok((proof, item)) -} - -/// Errors that may occur during proof verification. Most of the errors types simply indicate that -/// the proof is invalid with respect to the statement being verified, and the exact error type can -/// be used for debugging. -#[derive(PartialEq, Eq)] -#[cfg_attr(feature = "std", derive(Debug))] -pub enum VerifyError<'a, HO, CE> { - /// The proof does not contain any value for the given key - /// the error carries the nibbles left after traversing the trie - NonExistingValue(NibbleSlice<'a>), - /// The proof contains a value for the given key - /// while we were expecting to find a non-existence proof - ExistingValue(Vec), - /// The proof indicates that the trie contains a different value. - /// the error carries the value contained in the trie - ValueMismatch(Vec), - /// The proof is missing trie nodes required to verify. - IncompleteProof, - /// The node hash computed from the proof is not matching. - HashMismatch(HO), - /// One of the proof nodes could not be decoded. - DecodeError(CE), - /// Error in converting a plain hash into a HO - HashDecodeError(&'a [u8]), -} - -#[cfg(feature = "std")] -impl<'a, HO: std::fmt::Debug, CE: std::error::Error> std::fmt::Display for VerifyError<'a, HO, CE> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - match self { - VerifyError::NonExistingValue(key) => { - write!(f, "Key does not exist in trie: reaming key={:?}", key) - }, - VerifyError::ExistingValue(value) => { - write!(f, "trie contains a value for given key value={:?}", value) - }, - VerifyError::ValueMismatch(key) => { - write!(f, "Expected value was not found in the trie: key={:?}", key) - }, - VerifyError::IncompleteProof => write!(f, "Proof is incomplete -- expected more nodes"), - VerifyError::HashMismatch(hash) => write!(f, "hash mismatch found: hash={:?}", hash), - VerifyError::DecodeError(err) => write!(f, "Unable to decode proof node: {}", err), - VerifyError::HashDecodeError(plain_hash) => { - write!(f, "Unable to decode hash value plain_hash: {:?}", plain_hash) - }, - } - } -} - -#[cfg(feature = "std")] -impl<'a, HO: std::fmt::Debug, CE: std::error::Error + 'static> std::error::Error - for VerifyError<'a, HO, CE> -{ - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - VerifyError::DecodeError(err) => Some(err), - _ => None, - } - } -} - -/// Verify a compact proof for key-value pairs in a trie given a root hash. -pub fn verify_proof<'a, L>( - root: &::Out, - proof: &'a [Vec], - raw_key: &'a [u8], - expected_value: Option<&[u8]>, -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if proof.is_empty() { - return Err(VerifyError::IncompleteProof) - } - let key = NibbleSlice::new(raw_key); - process_node::(Some(root), &proof[0], key, expected_value, &proof[1..]) -} - -fn process_node<'a, L>( - expected_node_hash: Option<&::Out>, - encoded_node: &'a [u8], - key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if let Some(value) = expected_value { - if encoded_node == value { - return Ok(()) - } - } - if let Some(expected) = expected_node_hash { - let calculated_node_hash = ::hash(encoded_node); - if calculated_node_hash != *expected { - return Err(VerifyError::HashMismatch(calculated_node_hash)) - } - } - let node = ::decode(encoded_node).map_err(VerifyError::DecodeError)?; - match node { - Node::Empty => process_empty::(key, expected_value, proof), - Node::Leaf(nib, data) => process_leaf::(nib, data, key, expected_value, proof), - Node::Extension(nib, handle) => - process_extension::(&nib, handle, key, expected_value, proof), - Node::Branch(children, maybe_data) => - process_branch::(children, maybe_data, key, expected_value, proof), - Node::NibbledBranch(nib, children, maybe_data) => - process_nibbledbranch::(nib, children, maybe_data, key, expected_value, proof), - } -} - -fn process_empty<'a, L>( - key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - _: &[Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if expected_value.is_none() { - Ok(()) - } else { - Err(VerifyError::NonExistingValue(key)) - } -} - -fn process_leaf<'a, L>( - nib: NibbleSlice, - data: Value<'a>, - key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if key != nib && expected_value.is_none() { - return Ok(()) - } else if key != nib { - return Err(VerifyError::NonExistingValue(key)) - } - match_value::(Some(data), key, expected_value, proof) -} -fn process_extension<'a, L>( - nib: &NibbleSlice, - handle: NodeHandle<'a>, - mut key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if !key.starts_with(nib) && expected_value.is_none() { - return Ok(()) - } else if !key.starts_with(nib) { - return Err(VerifyError::NonExistingValue(key)) - } - key.advance(nib.len()); - - match handle { - NodeHandle::Inline(encoded_node) => - process_node::(None, encoded_node, key, expected_value, proof), - NodeHandle::Hash(plain_hash) => { - let new_root = decode_hash::(plain_hash) - .ok_or(VerifyError::HashDecodeError(plain_hash))?; - process_node::(Some(&new_root), &proof[0], key, expected_value, &proof[1..]) - }, - } -} - -fn process_nibbledbranch<'a, L>( - nib: NibbleSlice, - children: [Option>; 16], - maybe_data: Option>, - mut key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if !key.starts_with(&nib) && expected_value.is_none() { - return Ok(()) - } else if !key.starts_with(&nib) && expected_value.is_some() { - return Err(VerifyError::NonExistingValue(key)) - } - key.advance(nib.len()); - - if key.is_empty() { - match_value::(maybe_data, key, expected_value, proof) - } else { - match_children::(children, key, expected_value, proof) - } -} - -fn process_branch<'a, L>( - children: [Option>; 16], - maybe_data: Option>, - key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if key.is_empty() { - match_value::(maybe_data, key, expected_value, proof) - } else { - match_children::(children, key, expected_value, proof) - } -} -fn match_children<'a, L>( - children: [Option>; 16], - mut key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - match children.get(key.at(0) as usize) { - Some(Some(NodeHandle::Hash(hash))) => - if proof.is_empty() { - Err(VerifyError::IncompleteProof) - } else { - key.advance(1); - let new_root = - decode_hash::(hash).ok_or(VerifyError::HashDecodeError(hash))?; - process_node::(Some(&new_root), &proof[0], key, expected_value, &proof[1..]) - }, - Some(Some(NodeHandle::Inline(encoded_node))) => { - key.advance(1); - process_node::(None, encoded_node, key, expected_value, proof) - }, - Some(None) => - if expected_value.is_none() { - Ok(()) - } else { - Err(VerifyError::NonExistingValue(key)) - }, - None => panic!("key index is out of range in children array"), - } -} - -fn match_value<'a, L>( - maybe_data: Option>, - key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - match (maybe_data, proof.first(), expected_value) { - (None, _, None) => Ok(()), - (None, _, Some(_)) => Err(VerifyError::NonExistingValue(key)), - (Some(Value::Inline(inline_data)), _, Some(value)) => - if inline_data == value { - Ok(()) - } else { - Err(VerifyError::ValueMismatch(inline_data.to_vec())) - }, - (Some(Value::Inline(inline_data)), _, None) => - Err(VerifyError::ExistingValue(inline_data.to_vec())), - (Some(Value::Node(plain_hash)), Some(next_proof_item), Some(value)) => { - let value_hash = L::Hash::hash(value); - let node_hash = decode_hash::(plain_hash) - .ok_or(VerifyError::HashDecodeError(plain_hash))?; - if node_hash != value_hash { - Err(VerifyError::HashMismatch(node_hash)) - } else if next_proof_item != value { - Err(VerifyError::ValueMismatch(next_proof_item.to_vec())) - } else { - Ok(()) - } - }, - (Some(Value::Node(_)), None, _) => Err(VerifyError::IncompleteProof), - (Some(Value::Node(_)), Some(proof_item), None) => - Err(VerifyError::ExistingValue(proof_item.to_vec())), - } -} diff --git a/trie-eip1186/src/lib.rs b/trie-eip1186/src/lib.rs deleted file mode 100644 index 1483b857..00000000 --- a/trie-eip1186/src/lib.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2021, 2021 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#![cfg_attr(not(feature = "std"), no_std)] - -#[cfg(feature = "std")] -mod rstd { - pub use std::{result, vec}; -} - -#[cfg(not(feature = "std"))] -mod rstd { - pub use alloc::vec; - pub use core::result; - pub trait Error {} - impl Error for T {} -} - -mod eip1186; -pub use eip1186::{generate_proof, verify_proof, VerifyError}; diff --git a/trie-eip1186/test/Cargo.toml b/trie-eip1186/test/Cargo.toml deleted file mode 100644 index 4ee7c0fc..00000000 --- a/trie-eip1186/test/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "trie-eip1186-test" -version = "0.5.0" -authors = ["Parity Technologies "] -description = "Tests for trie-eip1186 crate" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -trie-eip1186 = { path = "..", version = "0.5.0"} -trie-db = { path = "../../trie-db", version = "0.27.0"} -hash-db = { path = "../../hash-db", version = "0.16.0"} -reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } -memory-db = { path = "../../memory-db", version = "0.32.0" } diff --git a/trie-eip1186/test/src/eip1186.rs b/trie-eip1186/test/src/eip1186.rs deleted file mode 100644 index 05d89edc..00000000 --- a/trie-eip1186/test/src/eip1186.rs +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2019, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use hash_db::Hasher; -use reference_trie::test_layouts; -use trie_db::{DBValue, TrieDBMutBuilder, TrieLayout, TrieMut}; -use trie_eip1186::{generate_proof, verify_proof, VerifyError}; - -type MemoryDB = memory_db::MemoryDB< - ::Hash, - memory_db::HashKey<::Hash>, - DBValue, ->; - -fn test_entries() -> Vec<(&'static [u8], &'static [u8])> { - vec![ - // "alfa" is at a hash-referenced leaf node. - (b"alfa", &[0; 32]), - // "bravo" is at an inline leaf node. - (b"bravo", b"bravo"), - // "do" is at a hash-referenced branch node. - (b"do", b"verb"), - // "dog" is at a hash-referenced branch node. - (b"dog", b"puppy"), - // "doge" is at a hash-referenced leaf node. - (b"doge", &[0; 32]), - // extension node "o" (plus nibble) to next branch. - (b"horse", b"stallion"), - (b"house", b"building"), - ] -} - -fn test_generate_proof( - entries: Vec<(&'static [u8], &'static [u8])>, - key: &[u8], -) -> (::Out, Vec>, Option>) { - // Populate DB with full trie from entries. - let (db, root) = { - let mut db = >::default(); - let mut root = Default::default(); - { - let mut trie = >::new(&mut db, &mut root).build(); - for (key, value) in entries.iter() { - trie.insert(key, value).unwrap(); - } - } - (db, root) - }; - // Generate proof for the given keys.. - let proof = generate_proof::(&db, &root, key).unwrap(); - (root, proof.0, proof.1) -} - -test_layouts!(trie_proof_works2, trie_proof_works_internal2); -fn trie_proof_works_internal2() { - let (root, proof, item) = test_generate_proof::( - vec![ - // "do" is at a hash-referenced branch node. - (b"do", b"verb"), - // "dog" is at a hash-referenced branch node. - (b"dog", b"puppy"), - ], - b"do", - ); - assert_eq!(Some(b"verb".as_ref()), item.as_deref(), "verb is the item"); - assert!(verify_proof::(&root, &proof, b"do", Some(b"verb")).is_ok(), "verifying do"); - - let (root, proof, item) = test_generate_proof::( - vec![ - // "do" is at a hash-referenced branch node. - (b"do", b"verb"), - // "dog" is at a hash-referenced branch node. - (b"dog", b"puppy"), - ], - b"dog", - ); - assert_eq!(Some(b"puppy".as_ref()), item.as_deref(), "puppy is the item"); - assert!(verify_proof::(&root, &proof, b"dog", Some(b"puppy")).is_ok(), "verifying dog"); -} - -test_layouts!(trie_proof_works, trie_proof_works_internal); -fn trie_proof_works_internal() { - let (root, proof, item) = test_generate_proof::(test_entries(), b"do"); - assert_eq!(Some(b"verb".as_ref()), item.as_deref(), "verb is the item"); - assert!(verify_proof::(&root, &proof, b"do", Some(b"verb")).is_ok(), "verifying do"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"dog"); - assert_eq!(Some(b"puppy".as_ref()), item.as_deref(), "puppy is the item"); - assert!(verify_proof::(&root, &proof, b"dog", Some(b"puppy")).is_ok(), "verifying dog"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"doge"); - assert_eq!(Some([0; 32].as_ref()), item.as_deref(), "[0;32] is the item"); - assert!(verify_proof::(&root, &proof, b"doge", Some(&[0; 32])).is_ok(), "verifying doge"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"bravo"); - assert_eq!(Some(b"bravo".as_ref()), item.as_deref(), "bravo is the item"); - assert!(verify_proof::(&root, &proof, b"bravo", Some(b"bravo")).is_ok(), "verifying bravo"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"alfabet"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"alfabet", None).is_ok(), "verifying alfabet"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"d"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"d", None).is_ok(), "verifying d"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"do\x10"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"do\x10", None).is_ok(), "verifying do\x10"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"halp"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"halp", None).is_ok(), "verifying halp"); -} - -test_layouts!(trie_proof_works_for_empty_trie, trie_proof_works_for_empty_trie_internal); -fn trie_proof_works_for_empty_trie_internal() { - let (root, proof, item) = test_generate_proof::(vec![], b"alpha"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"alpha", None).is_ok(), "verifying alpha"); - let (root, proof, item) = test_generate_proof::(vec![], b"bravo"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"bravo", None).is_ok(), "verifying bravo"); - let (root, proof, item) = test_generate_proof::(vec![], b"\x42\x42"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"\x42\x42", None).is_ok(), "verifying \x42\x42"); -} - -test_layouts!( - test_verify_value_mismatch_some_to_none, - test_verify_value_mismatch_some_to_none_internal -); -fn test_verify_value_mismatch_some_to_none_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), b"horse"); - let res = verify_proof::(&root, &proof, b"horse", Some(b"stallion")); - assert!(res.is_ok(), "verifying horse"); - - let res = verify_proof::(&root, &proof, b"halp", Some(b"plz")); - assert!(res.is_err(), "verifying halp"); - assert!(matches!(res.err().unwrap(), VerifyError::NonExistingValue(_))); - - let res = verify_proof::(&root, &proof, b"horse", Some(b"rocinante")); - assert!(res.is_err(), "verifying horse"); - //checking for two variants as it depends on the TrieLayout which one occurs - let is_ok = match res { - Err(VerifyError::HashMismatch(_)) | Err(VerifyError::ValueMismatch(_)) => true, - _ => false, - }; - assert!(is_ok); -} - -test_layouts!(test_verify_incomplete_proof, test_verify_incomplete_proof_internal); -fn test_verify_incomplete_proof_internal() { - let (root, mut proof, item) = test_generate_proof::(test_entries(), b"alfa"); - - proof.pop(); - let res = verify_proof::(&root, &proof, b"alfa", item.as_deref()); - assert!(matches!(res, Err(VerifyError::IncompleteProof))); -} - -test_layouts!(test_verify_decode_error, test_verify_decode_error_internal); -fn test_verify_decode_error_internal() { - let (_, mut proof, item) = test_generate_proof::(test_entries(), b"bravo"); - - let fake_node = b"this is not a trie node"; - proof.insert(0, fake_node.to_vec()); - let fake_root = T::Hash::hash(fake_node); - let res = verify_proof::(&fake_root, &proof, b"bravo", item.as_deref()); - assert!(matches!(res, Err(VerifyError::DecodeError(_)))); -} diff --git a/trie-eip1186/test/src/lib.rs b/trie-eip1186/test/src/lib.rs deleted file mode 100644 index 55910011..00000000 --- a/trie-eip1186/test/src/lib.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2021 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Tests for trie-eip1186 crate. - -#[cfg(test)] -mod eip1186; From 6f097f6e7da4e710374a8d10e76858a30f78a384 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 10 Aug 2023 18:39:44 +0200 Subject: [PATCH 02/90] Fixed a test --- test-support/reference-trie/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 3b048898..d950afd9 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -920,7 +920,7 @@ where for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); } - *t.commit().root.hash() + t.commit().apply_to(&mut mem_db2) }; if root_new != root { { @@ -943,7 +943,7 @@ where assert_eq!(root, root_new); // compare db content for key fuzzing - assert!(mem_db2 == mem_db2); + assert!(mem_db1 == mem_db2); } /// Compare trie builder and trie root implementations. From d9ac5aaa474ec7670c8420f50e8d72ccf30bcc45 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 12 Aug 2023 13:48:17 +0200 Subject: [PATCH 03/90] Fixes --- trie-db/fuzz/src/lib.rs | 28 ++++++++++++---------------- trie-db/src/triedbmut.rs | 17 +++++++---------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index f9de8c07..6c980870 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -18,10 +18,10 @@ use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ calc_root, compare_insert_remove, reference_trie_root_iter_build as reference_trie_root, }; -use std::convert::TryInto; +use std::{convert::TryInto, fmt::Debug}; use trie_db::{ proof::{generate_proof, verify_proof}, - DBValue, Trie, TrieDBBuilder, TrieDBIterator, TrieDBMutBuilder, TrieLayout, TrieMut, + DBValue, Trie, TrieDBBuilder, TrieDBIterator, TrieDBMutBuilder, TrieLayout, }; fn fuzz_to_data(input: &[u8]) -> Vec<(Vec, Vec)> { @@ -92,23 +92,21 @@ fn fuzz_removal(data: Vec<(Vec, Vec)>) -> Vec<(bool, Vec, Vec)> pub fn fuzz_that_reference_trie_root(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data(input)); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } - assert_eq!(*t.root(), reference_trie_root::(data)); + assert_eq!(t.commit().root_hash(), reference_trie_root::(data)); } pub fn fuzz_that_reference_trie_root_fix_length(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data_fix_length(input)); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } - assert_eq!(*t.root(), reference_trie_root::(data)); + assert_eq!(t.commit().root_hash(), reference_trie_root::(data)); } fn fuzz_to_data_fix_length(input: &[u8]) -> Vec<(Vec, Vec)> { @@ -132,20 +130,18 @@ fn data_sorted_unique(input: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec)> m.into_iter().collect() } -pub fn fuzz_that_compare_implementations(input: &[u8]) { +pub fn fuzz_that_compare_implementations(input: &[u8]) + where T::Location: Debug, +{ let data = data_sorted_unique(fuzz_to_data(input)); - //println!("data:{:?}", &data); - let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations::(data, memdb, hashdb); + reference_trie::compare_implementations::>(data); } pub fn fuzz_that_no_extension_insert(input: &[u8]) { let data = fuzz_to_data(input); //println!("data{:?}", data); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } @@ -153,7 +149,7 @@ pub fn fuzz_that_no_extension_insert(input: &[u8]) { // before. let data = data_sorted_unique(fuzz_to_data(input)); //println!("data{:?}", data); - assert_eq!(*t.root(), calc_root::(data)); + assert_eq!(t.commit().root_hash(), calc_root::(data)); } pub fn fuzz_that_no_extension_insert_remove(input: &[u8]) { diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 19455b67..1e38295f 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -30,9 +30,6 @@ use crate::{ use hash_db::{HashDB, Hasher, Prefix}; use hashbrown::HashSet; -#[cfg(not(feature = "std"))] -use alloc::vec; - #[cfg(feature = "std")] use log::trace; use memory_db::MemoryDB; @@ -750,7 +747,7 @@ impl Changeset { for (hash, prefix) in &self.removed { mem_db.remove(hash, (prefix.0.as_slice(), prefix.1)); } - fn apply_node(node: &ChangesetNodeRef, mem_db: &mut MemoryDB) + fn apply_node(node: &ChangesetNodeRef, mem_db: &mut MemoryDB) where K: memory_db::KeyFunction + Send + Sync, MH: Hasher + Send + Sync, @@ -1925,9 +1922,9 @@ where let handle = match self.root_handle() { NodeHandle::Hash(hash, location) => return Changeset { root: ChangesetNodeRef::Existing(ExistingChangesetNode { - hash, + hash, prefix: Default::default(), - location, + location, }), removed, }, // no changes necessary. @@ -1973,10 +1970,10 @@ where self.root_handle = NodeHandle::Hash(self.root, Default::default()); Changeset { root: ChangesetNodeRef::New(NewChangesetNode { - hash: self.root.clone(), + hash: self.root.clone(), prefix: Default::default(), data: encoded_root, - children, + children, }), removed, } @@ -1988,9 +1985,9 @@ where NodeHandle::InMemory(self.storage.alloc(Stored::Cached(node, hash, Default::default()))); Changeset { root: ChangesetNodeRef::Existing(ExistingChangesetNode { - hash, + hash, prefix: Default::default(), - location, + location, }), removed, } From 00b4fa91828c399e3364de1d0ab120d1222fe46b Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 12 Aug 2023 13:48:34 +0200 Subject: [PATCH 04/90] fmt --- memory-db/src/lib.rs | 10 +- test-support/reference-trie/src/lib.rs | 45 ++++-- test-support/reference-trie/src/substrate.rs | 6 +- .../reference-trie/src/substrate_like.rs | 6 +- trie-db/src/iter_build.rs | 3 +- trie-db/src/iterator.rs | 18 ++- trie-db/src/lib.rs | 11 +- trie-db/src/lookup.rs | 46 +++--- trie-db/src/node.rs | 53 ++++--- trie-db/src/node_codec.rs | 11 +- trie-db/src/proof/generate.rs | 43 +++--- trie-db/src/trie_codec.rs | 44 ++++-- trie-db/src/triedb.rs | 28 ++-- trie-db/src/triedbmut.rs | 138 +++++++++++------- trie-db/test/benches/bench.rs | 3 +- trie-db/test/src/iter_build.rs | 10 +- trie-db/test/src/iterator.rs | 5 +- trie-db/test/src/trie_codec.rs | 3 +- trie-db/test/src/triedb.rs | 12 +- trie-db/test/src/triedbmut.rs | 35 +++-- 20 files changed, 327 insertions(+), 203 deletions(-) diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index 3bcd93d7..65d42528 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -19,14 +19,11 @@ #[cfg(not(feature = "std"))] extern crate alloc; -use hash_db::{ - HashDB, Hasher as KeyHasher, MaybeDebug, - Prefix, -}; +use hash_db::{HashDB, Hasher as KeyHasher, MaybeDebug, Prefix}; #[cfg(feature = "std")] use std::{ - cmp::Eq, collections::hash_map::Entry, collections::HashMap as Map, hash, - marker::PhantomData, mem, + cmp::Eq, collections::hash_map::Entry, collections::HashMap as Map, hash, marker::PhantomData, + mem, }; #[cfg(not(feature = "std"))] @@ -451,7 +448,6 @@ where }, } } - } #[cfg(test)] diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index d950afd9..ed483f20 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -679,7 +679,11 @@ impl NodeCodec for ReferenceNodeCodec { &[EMPTY_TRIE] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node( + partial: impl Iterator, + number_nibble: usize, + value: Value, + ) -> Vec { let mut output = partial_from_iterator_to_key(partial, number_nibble, LEAF_NODE_OFFSET, LEAF_NODE_OVER); match value { @@ -835,7 +839,11 @@ impl NodeCodec for ReferenceNodeCodecNoExt { &[EMPTY_TRIE_NO_EXT] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node( + partial: impl Iterator, + number_nibble: usize, + value: Value, + ) -> Vec { let mut output = partial_from_iterator_encode(partial, number_nibble, NodeKindNoExt::Leaf); match value { Value::Inline(value) => { @@ -1002,7 +1010,10 @@ where } /// Trie builder trie building utility. -pub fn calc_root_build(data: I, memdb: &mut memory_db::MemoryDB) -> TrieHash +pub fn calc_root_build( + data: I, + memdb: &mut memory_db::MemoryDB, +) -> TrieHash where T: TrieLayout, I: IntoIterator, @@ -1065,9 +1076,8 @@ where /// Testing utility that uses some periodic removal over /// its input test data. -pub fn compare_insert_remove( - data: Vec<(bool, Vec, Vec)>, -) where +pub fn compare_insert_remove(data: Vec<(bool, Vec, Vec)>) +where T: TrieLayout, K: memory_db::KeyFunction + Send + Sync, { @@ -1134,11 +1144,18 @@ impl Default for TestTrieCache { } impl trie_db::TrieCache for TestTrieCache { - fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&trie_db::CachedValue, L::Location>> { + fn lookup_value_for_key( + &mut self, + key: &[u8], + ) -> Option<&trie_db::CachedValue, L::Location>> { self.value_cache.get(key) } - fn cache_value_for_key(&mut self, key: &[u8], value: trie_db::CachedValue, L::Location>) { + fn cache_value_for_key( + &mut self, + key: &[u8], + value: trie_db::CachedValue, L::Location>, + ) { self.value_cache.insert(key.to_vec(), value); } @@ -1161,12 +1178,15 @@ impl trie_db::TrieCache for TestTrieCache< } } - fn get_node(&mut self, hash: &TrieHash, _location: L::Location) -> Option<&NodeOwned, L::Location>> { + fn get_node( + &mut self, + hash: &TrieHash, + _location: L::Location, + ) -> Option<&NodeOwned, L::Location>> { self.node_cache.get(hash) } - fn insert_new_node(&mut self, _hash: &TrieHash) { - } + fn insert_new_node(&mut self, _hash: &TrieHash) {} } #[cfg(test)] @@ -1206,7 +1226,8 @@ mod tests { input.len() * NIBBLE_PER_BYTE, Value::<()>::Inline(&[1]), ); - let dec = as NodeCodec>::decode(&enc, &[] as &[()]).unwrap(); + let dec = + as NodeCodec>::decode(&enc, &[] as &[()]).unwrap(); let o_sl = if let Node::Leaf(sl, _) = dec { Some(sl) } else { None }; assert!(o_sl.is_some()); } diff --git a/test-support/reference-trie/src/substrate.rs b/test-support/reference-trie/src/substrate.rs index 7e5c5d04..7e5d1d60 100644 --- a/test-support/reference-trie/src/substrate.rs +++ b/test-support/reference-trie/src/substrate.rs @@ -310,7 +310,11 @@ where &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node( + partial: impl Iterator, + number_nibble: usize, + value: Value, + ) -> Vec { let contains_hash = matches!(&value, Value::Node(..)); let mut output = if contains_hash { partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueLeaf) diff --git a/test-support/reference-trie/src/substrate_like.rs b/test-support/reference-trie/src/substrate_like.rs index 46bd23da..e4fd2631 100644 --- a/test-support/reference-trie/src/substrate_like.rs +++ b/test-support/reference-trie/src/substrate_like.rs @@ -171,7 +171,11 @@ where &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node( + partial: impl Iterator, + number_nibble: usize, + value: Value, + ) -> Vec { let contains_hash = matches!(&value, Value::Node(..)); let mut output = if contains_hash { partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueLeaf) diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 73fbc4aa..683165a9 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -369,7 +369,8 @@ impl<'a, T: TrieLayout, K: memory_db::KeyFunction + Send + Sync> TrieBu } } -impl<'a, T, K: memory_db::KeyFunction + Send + Sync> ProcessEncodedNode> for TrieBuilder<'a, T, K> +impl<'a, T, K: memory_db::KeyFunction + Send + Sync> ProcessEncodedNode> + for TrieBuilder<'a, T, K> where T: TrieLayout, { diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index 5539c8cf..c4bd11b2 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -480,10 +480,11 @@ impl TrieDBRawIterator { } let value = match value { - Value::Node(hash, location) => match Self::fetch_value(db, &hash, (key_slice, None), location) { - Ok(value) => value, - Err(err) => return Some(Err(err)), - }, + Value::Node(hash, location) => + match Self::fetch_value(db, &hash, (key_slice, None), location) { + Ok(value) => value, + Err(err) => return Some(Err(err)), + }, Value::Inline(value) => value.to_vec(), }; @@ -504,7 +505,7 @@ impl TrieDBRawIterator { let mut prefix = prefix.clone(); match node.node() { - Node::Leaf(partial, _) => { + Node::Leaf(partial, _) => { prefix.append_partial(partial.right()); }, Node::Branch(_, value) => @@ -593,8 +594,11 @@ impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBNodeIterator<'a, 'cach } impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBNodeIterator<'a, 'cache, L> { - type Item = - Result<(NibbleVec, Option>, Arc>), TrieHash, CError>; + type Item = Result< + (NibbleVec, Option>, Arc>), + TrieHash, + CError, + >; fn next(&mut self) -> Option { self.raw_iter.next_raw_item(self.db).map(|result| { diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index d149e5b0..67cb2a10 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -60,8 +60,8 @@ pub use self::{ recorder::Recorder, triedb::{TrieDB, TrieDBBuilder, TrieDBIterator, TrieDBKeyIterator}, triedbmut::{ - ChildReference, TrieDBMut, TrieDBMutBuilder, Value, - Changeset, ChangesetNodeRef, NewChangesetNode, ExistingChangesetNode + Changeset, ChangesetNodeRef, ChildReference, ExistingChangesetNode, NewChangesetNode, + TrieDBMut, TrieDBMutBuilder, Value, }, }; pub use crate::{ @@ -329,7 +329,7 @@ pub trait TrieLayout { type Hash: Hasher; /// Codec to use (needs to match hasher and nibble ops). type Codec: NodeCodec::Out>; - type Location: Copy + Default + Eq + PartialEq; + type Location: Copy + Default + Eq + PartialEq; } /// This trait associates a trie definition with preferred methods. @@ -337,7 +337,10 @@ pub trait TrieLayout { /// used to allow switching implementation. pub trait TrieConfiguration: Sized + TrieLayout { /// Operation to build a trie db from its ordered iterator over its key/values. - fn trie_build(db: &mut memory_db::MemoryDB, DBValue>, input: I) -> ::Out + fn trie_build( + db: &mut memory_db::MemoryDB, DBValue>, + input: I, + ) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord, diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs index 73cba979..98d20a8d 100644 --- a/trie-db/src/lookup.rs +++ b/trie-db/src/lookup.rs @@ -217,7 +217,8 @@ where full_key, cache, |value, _, full_key, _, _, recorder| match value { - ValueOwned::Inline(value, hash) => Ok((hash, Some(value.clone()), Default::default())), + ValueOwned::Inline(value, hash) => + Ok((hash, Some(value.clone()), Default::default())), ValueOwned::Node(hash, location) => { if let Some(recoder) = recorder.as_mut() { recoder.record(TrieAccess::Hash { full_key }); @@ -231,7 +232,8 @@ where match &hash_and_value { Some((hash, Some(value), _location)) => cache.cache_value_for_key(full_key, (value.clone(), *hash).into()), - Some((hash, None, location)) => cache.cache_value_for_key(full_key, CachedValue::ExistingHash(*hash, *location)), + Some((hash, None, location)) => + cache.cache_value_for_key(full_key, CachedValue::ExistingHash(*hash, *location)), None => cache.cache_value_for_key(full_key, CachedValue::NonExisting), } @@ -285,7 +287,7 @@ where let res = match value_cache_allowed.then(|| cache.lookup_value_for_key(full_key)).flatten() { Some(CachedValue::NonExisting) => None, - Some(CachedValue::ExistingHash(hash, location)) => { + Some(CachedValue::ExistingHash(hash, location)) => { let data = Self::load_owned_value( // If we only have the hash cached, this can only be a value node. // For inline nodes we cache them directly as `CachedValue::Existing`. @@ -352,14 +354,15 @@ where // this loop iterates through non-inline nodes. for depth in 0.. { let mut node = cache.get_or_insert_node(hash, location, &mut || { - let (node_data, locations) = match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { - Some(value) => value, - None => - return Err(Box::new(match depth { - 0 => TrieError::InvalidStateRoot(hash), - _ => TrieError::IncompleteDatabase(hash), - })), - }; + let (node_data, locations) = + match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { + Some(value) => value, + None => + return Err(Box::new(match depth { + 0 => TrieError::InvalidStateRoot(hash), + _ => TrieError::IncompleteDatabase(hash), + })), + }; let decoded = match L::Codec::decode(&node_data[..], &locations) { Ok(node) => node, @@ -528,14 +531,15 @@ where // this loop iterates through non-inline nodes. for depth in 0.. { - let (node_data, locations) = match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { - Some(value) => value, - None => - return Err(Box::new(match depth { - 0 => TrieError::InvalidStateRoot(hash), - _ => TrieError::IncompleteDatabase(hash), - })), - }; + let (node_data, locations) = + match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { + Some(value) => value, + None => + return Err(Box::new(match depth { + 0 => TrieError::InvalidStateRoot(hash), + _ => TrieError::IncompleteDatabase(hash), + })), + }; self.record(|| TrieAccess::EncodedNode { hash, @@ -596,7 +600,7 @@ where Ok(None) } } else { - let i = partial.at(0) as usize; + let i = partial.at(0) as usize; match children[i] { Some(x) => { partial = partial.mid(1); @@ -634,7 +638,7 @@ where Ok(None) } } else { - let i = partial.at(slice.len()) as usize; + let i = partial.at(slice.len()) as usize; match children[i] { Some(x) => { partial = partial.mid(slice.len() + 1); diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index c2e78c58..2b9d57d5 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -38,8 +38,9 @@ impl<'a, L: Copy + Default> NodeHandle<'a, L> { /// Converts this node handle into a [`NodeHandleOwned`]. pub fn to_owned_handle( &self, - ) -> Result, TL::Location>, TrieHash, CError> - where TL::Location: From, + ) -> Result, TL::Location>, TrieHash, CError> + where + TL::Location: From, { match self { Self::Hash(h, l) => decode_hash::(h) @@ -140,8 +141,9 @@ impl<'a, L: Copy + Default> Value<'a, L> { } } - pub fn to_owned_value(&self) -> ValueOwned, TL::Location> - where TL::Location: From, + pub fn to_owned_value(&self) -> ValueOwned, TL::Location> + where + TL::Location: From, { match self { Self::Inline(data) => ValueOwned::Inline(Bytes::from(*data), TL::Hash::hash(data)), @@ -219,8 +221,9 @@ impl Node<'_, Location> { pub fn to_owned_node( &self, ) -> Result, L::Location>, TrieHash, CError> - where L::Location: From, - Location: Copy + Default, + where + L::Location: From, + Location: Copy + Default, { match self { Self::Empty => Ok(NodeOwned::Empty), @@ -386,10 +389,10 @@ impl NodeOwned { match self { Self::Leaf(_, _) | Self::Empty | Self::Value(_, _) => false, Self::Extension(_, h) => h.missing_location(), - Self::Branch(c, ..) | Self::NibbledBranch(_, c, ..) => c.iter().any(|c| c.as_ref().map_or(false, |c| c.missing_location())), + Self::Branch(c, ..) | Self::NibbledBranch(_, c, ..) => + c.iter().any(|c| c.as_ref().map_or(false, |c| c.missing_location())), } } - } impl NodeOwned { @@ -572,15 +575,24 @@ pub enum NodePlan { } impl NodePlan { - /// Build a node by decoding a byte slice according to the node plan and attaching location dats. - /// It is the responsibility of the caller to ensure that the node plan was created for the - /// argument data, otherwise the call may decode incorrectly or panic. - pub fn build<'a, 'b, L: Copy + Default>(&'a self, data: &'b [u8], locations: &[L]) -> Node<'b, L> { + /// Build a node by decoding a byte slice according to the node plan and attaching location + /// dats. It is the responsibility of the caller to ensure that the node plan was created for + /// the argument data, otherwise the call may decode incorrectly or panic. + pub fn build<'a, 'b, L: Copy + Default>( + &'a self, + data: &'b [u8], + locations: &[L], + ) -> Node<'b, L> { match self { NodePlan::Empty => Node::Empty, - NodePlan::Leaf { partial, value } => Node::Leaf(partial.build(data), value.build(data, locations.first().copied().unwrap_or_default())), - NodePlan::Extension { partial, child } => - Node::Extension(partial.build(data), child.build(data, locations.first().copied().unwrap_or_default())), + NodePlan::Leaf { partial, value } => Node::Leaf( + partial.build(data), + value.build(data, locations.first().copied().unwrap_or_default()), + ), + NodePlan::Extension { partial, child } => Node::Extension( + partial.build(data), + child.build(data, locations.first().copied().unwrap_or_default()), + ), NodePlan::Branch { value, children } => { let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; let mut nc = 0; @@ -596,7 +608,12 @@ impl NodePlan { child_slices[i] = Some(child.build(data, location)); } } - Node::Branch(child_slices, value.as_ref().map(|v| v.build(data, locations.last().copied().unwrap_or_default()))) + Node::Branch( + child_slices, + value + .as_ref() + .map(|v| v.build(data, locations.last().copied().unwrap_or_default())), + ) }, NodePlan::NibbledBranch { partial, value, children } => { let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; @@ -616,7 +633,9 @@ impl NodePlan { Node::NibbledBranch( partial.build(data), child_slices, - value.as_ref().map(|v| v.build(data, locations.last().copied().unwrap_or_default())), + value + .as_ref() + .map(|v| v.build(data, locations.last().copied().unwrap_or_default())), ) }, } diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index c0cfede6..fd7ec536 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -59,7 +59,10 @@ pub trait NodeCodec: Sized { fn decode_plan(data: &[u8]) -> Result; /// Decode bytes to a `Node`. Returns `Self::E` on failure. - fn decode<'a, L: Copy + Default>(data: &'a [u8], locations: &[L]) -> Result, Self::Error> { + fn decode<'a, L: Copy + Default>( + data: &'a [u8], + locations: &[L], + ) -> Result, Self::Error> { Ok(Self::decode_plan(data)?.build(data, locations)) } @@ -74,7 +77,11 @@ pub trait NodeCodec: Sized { /// Note that number_nibble is the number of element of the iterator /// it can possibly be obtain by `Iterator` `size_hint`, but /// for simplicity it is used directly as a parameter. - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec; + fn leaf_node( + partial: impl Iterator, + number_nibble: usize, + value: Value, + ) -> Vec; /// Returns an encoded extension node /// diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index 1a91df91..ab24cd9b 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -21,7 +21,7 @@ use hash_db::{HashDB, Hasher}; use crate::{ nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, - node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, Node, Value, ValuePlan}, + node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, Value, ValuePlan}, recorder::Record, CError, ChildReference, DBValue, NibbleSlice, NodeCodec, Recorder, Result as TrieResult, Trie, TrieDBBuilder, TrieError, TrieHash, TrieLayout, @@ -285,8 +285,10 @@ where &mut recorded_nodes, )?, // If stack is empty, descend into the root node. - None => - Step::Descend { child_prefix_len: 0, child: NodeHandle::Hash(root.as_ref(), Default::default()) }, + None => Step::Descend { + child_prefix_len: 0, + child: NodeHandle::Hash(root.as_ref(), Default::default()), + }, }; match step { @@ -405,7 +407,7 @@ fn match_key_to_node<'a, C: NodeCodec, L: Copy + Default>( Node::Empty => Step::FoundValue(None), Node::Leaf(partial, value) => { if key.contains(&partial, prefix_len) && key.len() == prefix_len + partial.len() { - match value{ + match value { Value::Inline(data) => { *omit_value = true; Step::FoundValue(Some(data)) @@ -419,15 +421,14 @@ fn match_key_to_node<'a, C: NodeCodec, L: Copy + Default>( Step::FoundValue(None) } }, - Node::Extension(partial, child) => { + Node::Extension(partial, child) => if key.contains(&partial, prefix_len) { assert_eq!(*child_index, 0); let child_prefix_len = prefix_len + partial.len(); Step::Descend { child_prefix_len, child } } else { Step::FoundValue(None) - } - }, + }, Node::Branch(child_handles, value) => match_key_to_branch_node::( value.clone(), &child_handles, @@ -439,18 +440,17 @@ fn match_key_to_node<'a, C: NodeCodec, L: Copy + Default>( NibbleSlice::new(&[]), recorded_nodes, )?, - Node::NibbledBranch(partial, child_handles, value) => - match_key_to_branch_node::( - value.clone(), - &child_handles, - omit_value, - child_index, - children, - key, - prefix_len, - partial, - recorded_nodes, - )?, + Node::NibbledBranch(partial, child_handles, value) => match_key_to_branch_node::( + value.clone(), + &child_handles, + omit_value, + child_index, + children, + key, + prefix_len, + partial, + recorded_nodes, + )?, }) } @@ -504,10 +504,7 @@ fn match_key_to_branch_node<'a, 'b, C: NodeCodec, L: Copy + Default>( *child_index += 1; } if let Some(child) = &child_handles[*child_index] { - Ok(Step::Descend { - child_prefix_len: prefix_len + partial.len() + 1, - child: child.clone(), - }) + Ok(Step::Descend { child_prefix_len: prefix_len + partial.len() + 1, child: child.clone() }) } else { Ok(Step::FoundValue(None)) } diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index ac0771dd..f3acd8df 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -198,9 +198,12 @@ fn detached_value( let fetched; match value { ValuePlan::Node(hash_plan) => { - if let Ok(value) = - TrieDBRawIterator::fetch_value(db, &node_data[hash_plan.clone()], node_prefix, location) - { + if let Ok(value) = TrieDBRawIterator::fetch_value( + db, + &node_data[hash_plan.clone()], + node_prefix, + location, + ) { fetched = value; } else { return None @@ -270,12 +273,28 @@ where let (children_len, detached_value) = match node.node_plan() { NodePlan::Empty => (0, None), - NodePlan::Leaf { value, .. } => - (0, detached_value(db, &value, node.data(), prefix.as_prefix(), node.locations().first().copied().unwrap_or_default())), + NodePlan::Leaf { value, .. } => ( + 0, + detached_value( + db, + &value, + node.data(), + prefix.as_prefix(), + node.locations().first().copied().unwrap_or_default(), + ), + ), NodePlan::Extension { .. } => (1, None), NodePlan::NibbledBranch { value: Some(value), .. } | - NodePlan::Branch { value: Some(value), .. } => - (NIBBLE_LENGTH, detached_value(db, &value, node.data(), prefix.as_prefix(), node.locations().last().copied().unwrap_or_default())), + NodePlan::Branch { value: Some(value), .. } => ( + NIBBLE_LENGTH, + detached_value( + db, + &value, + node.data(), + prefix.as_prefix(), + node.locations().last().copied().unwrap_or_default(), + ), + ), NodePlan::NibbledBranch { value: None, .. } | NodePlan::Branch { value: None, .. } => (NIBBLE_LENGTH, None), }; @@ -435,10 +454,10 @@ impl<'a, C: NodeCodec, L: Copy + Default> DecoderStackEntry<'a, C, L> { /// Reconstructs a partial trie DB from a compact representation. The encoding is a vector of /// mutated trie nodes with those child references omitted. The decode function reads them in order -/// from the given slice, reconstructing the full nodes and inserting them into the given `MemoryDB`. -/// It stops after fully constructing one partial trie and returns the root hash and the number of -/// nodes read. If an error occurs during decoding, there are no guarantees about which entries -/// were or were not added to the DB. +/// from the given slice, reconstructing the full nodes and inserting them into the given +/// `MemoryDB`. It stops after fully constructing one partial trie and returns the root hash and the +/// number of nodes read. If an error occurs during decoding, there are no guarantees about which +/// entries were or were not added to the DB. /// /// The number of nodes read may be fewer than the total number of items in `encoded`. This allows /// one to concatenate multiple compact encodings together and still reconstruct them all. @@ -523,7 +542,8 @@ where if let Some(entry) = stack.pop() { last_entry = entry; last_entry.pop_from_prefix(&mut prefix); - last_entry.children[last_entry.child_index] = Some(ChildReference::Hash(node_hash, Default::default())); + last_entry.children[last_entry.child_index] = + Some(ChildReference::Hash(node_hash, Default::default())); last_entry.child_index += 1; } else { return Ok((node_hash, i + 1)) diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index 94045788..d7f56a6c 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -67,7 +67,10 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// Use the given `recorder` to record trie accesses. #[inline] - pub fn with_recorder(mut self, recorder: &'cache mut dyn TrieRecorder, L::Location>) -> Self { + pub fn with_recorder( + mut self, + recorder: &'cache mut dyn TrieRecorder, L::Location>, + ) -> Self { self.recorder = Some(recorder); self } @@ -158,13 +161,14 @@ where NodeHandle::Hash(data, location) => { let node_hash = decode_hash::(data) .ok_or_else(|| Box::new(TrieError::InvalidHash(parent_hash, data.to_vec())))?; - let node_data = self.db.get(&node_hash, partial_key, location).ok_or_else(|| { - if partial_key == EMPTY_PREFIX { - Box::new(TrieError::InvalidStateRoot(node_hash)) - } else { - Box::new(TrieError::IncompleteDatabase(node_hash)) - } - })?; + let node_data = + self.db.get(&node_hash, partial_key, location).ok_or_else(|| { + if partial_key == EMPTY_PREFIX { + Box::new(TrieError::InvalidStateRoot(node_hash)) + } else { + Box::new(TrieError::IncompleteDatabase(node_hash)) + } + })?; (Some(node_hash), node_data) }, @@ -230,7 +234,9 @@ where query: |_: &[u8]| (), hash: *self.root, cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), - recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), + recorder: recorder + .as_mut() + .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } .look_up_hash(key, NibbleSlice::new(key), Default::default()) } @@ -248,7 +254,9 @@ where query, hash: *self.root, cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), - recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), + recorder: recorder + .as_mut() + .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } .look_up(key, NibbleSlice::new(key), Default::default()) } diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 1e38295f..d4eb934a 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -23,8 +23,8 @@ use crate::{ }, node_codec::NodeCodec, rstd::{boxed::Box, convert::TryFrom, mem, ops::Index, result, vec::Vec, VecDeque}, - Bytes, CError, DBValue, Result, TrieAccess, TrieCache, TrieError, TrieHash, - TrieLayout, TrieRecorder, + Bytes, CError, DBValue, Result, TrieAccess, TrieCache, TrieError, TrieHash, TrieLayout, + TrieRecorder, }; use hash_db::{HashDB, Hasher, Prefix}; @@ -153,12 +153,13 @@ impl Value { ) -> ChildReference, L::Location>, { if let Value::NewNode(hash, value) = self { - let new_hash = - if let ChildReference::Hash(hash, _) = f(NodeToEncode::Node(&value), partial, None) { - hash - } else { - unreachable!("Value node can never be inlined; qed") - }; + let new_hash = if let ChildReference::Hash(hash, _) = + f(NodeToEncode::Node(&value), partial, None) + { + hash + } else { + unreachable!("Value node can never be inlined; qed") + }; if let Some(h) = hash.as_ref() { debug_assert!(h == &new_hash); } else { @@ -168,7 +169,8 @@ impl Value { let value = match &*self { Value::Inline(value) => EncodedValue::Inline(&value), Value::Node(hash, location) => EncodedValue::Node(hash.as_ref(), *location), - Value::NewNode(Some(hash), _value) => EncodedValue::Node(hash.as_ref(), Default::default()), + Value::NewNode(Some(hash), _value) => + EncodedValue::Node(hash.as_ref(), Default::default()), Value::NewNode(None, _value) => unreachable!("New external value are always added before encoding anode"), }; @@ -217,7 +219,10 @@ enum Node { /// The child node is always a branch. Extension(NodeKey, NodeHandle, L::Location>), /// A branch has up to 16 children and an optional value. - Branch(Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, Option>), + Branch( + Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, + Option>, + ), /// Branch node with support for a nibble (to avoid extension node). NibbledBranch( NodeKey, @@ -313,8 +318,8 @@ impl Node { locations: &[L::Location], storage: &'b mut NodeStorage, ) -> Result, CError> { - let encoded_node = - L::Codec::decode(data, locations).map_err(|e| Box::new(TrieError::DecoderError(node_hash, e)))?; + let encoded_node = L::Codec::decode(data, locations) + .map_err(|e| Box::new(TrieError::DecoderError(node_hash, e)))?; let node = match encoded_node { EncodedNode::Empty => Node::Empty, EncodedNode::Leaf(k, v) => Node::Leaf(k.into(), v.into()), @@ -379,7 +384,10 @@ impl Node { } /// Decode a node from a [`NodeOwned`]. - fn from_node_owned(node_owned: &NodeOwned, L::Location>, storage: &mut NodeStorage) -> Self { + fn from_node_owned( + node_owned: &NodeOwned, L::Location>, + storage: &mut NodeStorage, + ) -> Self { match node_owned { NodeOwned::Empty => Node::Empty, NodeOwned::Leaf(k, v) => Node::Leaf(k.into(), v.into()), @@ -670,7 +678,10 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { } /// Use the given `recorder` to record trie accesses. - pub fn with_recorder(mut self, recorder: &'db mut dyn TrieRecorder, L::Location>) -> Self { + pub fn with_recorder( + mut self, + recorder: &'db mut dyn TrieRecorder, L::Location>, + ) -> Self { self.recorder = Some(recorder); self } @@ -719,8 +730,8 @@ pub struct ExistingChangesetNode { #[derive(Debug)] pub enum ChangesetNodeRef { - New (NewChangesetNode), - Existing (ExistingChangesetNode), + New(NewChangesetNode), + Existing(ExistingChangesetNode), } impl ChangesetNodeRef { @@ -740,17 +751,19 @@ pub struct Changeset { impl Changeset { pub fn apply_to(&self, mem_db: &mut MemoryDB) -> H - where - K: memory_db::KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, + where + K: memory_db::KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, { for (hash, prefix) in &self.removed { mem_db.remove(hash, (prefix.0.as_slice(), prefix.1)); } - fn apply_node(node: &ChangesetNodeRef, mem_db: &mut MemoryDB) - where + fn apply_node( + node: &ChangesetNodeRef, + mem_db: &mut MemoryDB, + ) where K: memory_db::KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, + MH: Hasher + Send + Sync, { match node { ChangesetNodeRef::New(node) => { @@ -759,7 +772,7 @@ impl Changeset { } mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); }, - ChangesetNodeRef::Existing(_) => {} + ChangesetNodeRef::Existing(_) => {}, } } apply_node::(&self.root, mem_db); @@ -767,9 +780,9 @@ impl Changeset { } pub fn apply_with_prefix(&self, mem_db: &mut MemoryDB, ks: &[u8]) -> H - where - K: memory_db::KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, + where + K: memory_db::KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, { fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); @@ -778,10 +791,13 @@ impl Changeset { (result, prefix.1) } - fn apply_node(node: &ChangesetNodeRef, ks: &[u8], mem_db: &mut MemoryDB) - where + fn apply_node( + node: &ChangesetNodeRef, + ks: &[u8], + mem_db: &mut MemoryDB, + ) where K: memory_db::KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, + MH: Hasher + Send + Sync, { match node { ChangesetNodeRef::New(node) => { @@ -791,7 +807,7 @@ impl Changeset { let prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); mem_db.insert((prefixed.0.as_slice(), prefixed.1), &node.data); }, - ChangesetNodeRef::Existing(_) => {} + ChangesetNodeRef::Existing(_) => {}, } } @@ -803,7 +819,6 @@ impl Changeset { self.root_hash() } - pub fn root_hash(&self) -> H { match &self.root { ChangesetNodeRef::New(node) => node.hash, @@ -1907,7 +1922,7 @@ where } /// Calculate the changeset for the trie. - pub fn commit(mut self) -> Changeset, L::Location> { + pub fn commit(mut self) -> Changeset, L::Location> { #[cfg(feature = "std")] trace!(target: "trie", "Committing trie changes to db."); @@ -1920,14 +1935,15 @@ where } let handle = match self.root_handle() { - NodeHandle::Hash(hash, location) => return Changeset { - root: ChangesetNodeRef::Existing(ExistingChangesetNode { - hash, - prefix: Default::default(), - location, - }), - removed, - }, // no changes necessary. + NodeHandle::Hash(hash, location) => + return Changeset { + root: ChangesetNodeRef::Existing(ExistingChangesetNode { + hash, + prefix: Default::default(), + location, + }), + removed, + }, // no changes necessary. NodeHandle::InMemory(h) => h, }; @@ -1981,8 +1997,11 @@ where Stored::Cached(node, hash, location) => { // probably won't happen, but update the root and move on. self.root = hash; - self.root_handle = - NodeHandle::InMemory(self.storage.alloc(Stored::Cached(node, hash, Default::default()))); + self.root_handle = NodeHandle::InMemory(self.storage.alloc(Stored::Cached( + node, + hash, + Default::default(), + ))); Changeset { root: ChangesetNodeRef::Existing(ExistingChangesetNode { hash, @@ -2006,8 +2025,7 @@ where /// Cache the given `value`. /// /// `hash` is the hash of `value`. - fn cache_value(&mut self, _full_key: &[u8], _value: impl Into, _hash: TrieHash) { - } + fn cache_value(&mut self, _full_key: &[u8], _value: impl Into, _hash: TrieHash) {} /// Commit a node by hashing it and writing it to the db. Returns a /// `ChildReference` which in most cases carries a normal hash but for the @@ -2018,7 +2036,7 @@ where &mut self, handle: NodeHandle, L::Location>, prefix: &mut NibbleVec, - children: &mut Vec, L::Location>> + children: &mut Vec, L::Location>>, ) -> ChildReference, L::Location> { match handle { NodeHandle::Hash(hash, location) => { @@ -2028,7 +2046,7 @@ where location, })); ChildReference::Hash(hash, location) - } + }, NodeHandle::InMemory(storage_handle) => { match self.storage.destroy(storage_handle) { Stored::Cached(_, hash, location) => { @@ -2049,12 +2067,14 @@ where match node { NodeToEncode::Node(value) => { let value_hash = self.db.hash(value); - sub_children.push(ChangesetNodeRef::New(NewChangesetNode { - hash: value_hash, - prefix: prefix.as_owned_prefix(), - data: value.to_vec(), //TODO: avoid allocation - children: Default::default(), - })); + sub_children.push(ChangesetNodeRef::New( + NewChangesetNode { + hash: value_hash, + prefix: prefix.as_owned_prefix(), + data: value.to_vec(), //TODO: avoid allocation + children: Default::default(), + }, + )); self.cache_value(prefix.inner(), value, value_hash); @@ -2062,7 +2082,11 @@ where ChildReference::Hash(value_hash, Default::default()) }, NodeToEncode::TrieNode(node_handle) => { - let result = self.commit_child(node_handle, prefix, &mut sub_children); + let result = self.commit_child( + node_handle, + prefix, + &mut sub_children, + ); prefix.drop_lasts(mov); result }, @@ -2114,7 +2138,10 @@ where } } - pub fn get<'x, 'key>(&'x self, key: &'key [u8]) -> Result, TrieHash, CError> + pub fn get<'x, 'key>( + &'x self, + key: &'key [u8], + ) -> Result, TrieHash, CError> where 'x: 'key, { @@ -2164,7 +2191,8 @@ where None => { #[cfg(feature = "std")] trace!(target: "trie", "remove: obliterated trie"); - self.root_handle = NodeHandle::Hash(L::Codec::hashed_null_node(), Default::default()); + self.root_handle = + NodeHandle::Hash(L::Codec::hashed_null_node(), Default::default()); self.root = L::Codec::hashed_null_node(); }, } diff --git a/trie-db/test/benches/bench.rs b/trie-db/test/benches/bench.rs index 50d18686..8688730c 100644 --- a/trie-db/test/benches/bench.rs +++ b/trie-db/test/benches/bench.rs @@ -265,8 +265,7 @@ fn trie_mut(c: &mut Criterion) { group.bench_with_input(BenchmarkId::new("trie_mut", param), &input, |b, i| { b.iter(|| { let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - let mut trie = - trie_db::TrieDBMutBuilder::::new(&mut mdb).build(); + let mut trie = trie_db::TrieDBMutBuilder::::new(&mut mdb).build(); for (key, value) in i { trie.insert(&key, &value).expect( "changes trie: insertion to trie is not allowed to fail within runtime", diff --git a/trie-db/test/src/iter_build.rs b/trie-db/test/src/iter_build.rs index 76806d36..20d37b5d 100644 --- a/trie-db/test/src/iter_build.rs +++ b/trie-db/test/src/iter_build.rs @@ -76,8 +76,9 @@ fn compare_implementations_prefixed(data: Vec<(Vec, Vec)>) { compare_implementations_prefixed_internal::(data.clone()); compare_implementations_prefixed_internal::(data.clone()); } -fn compare_implementations_prefixed_internal(data: Vec<(Vec, Vec)>) - where T::Location: std::fmt::Debug, +fn compare_implementations_prefixed_internal(data: Vec<(Vec, Vec)>) +where + T::Location: std::fmt::Debug, { reference_trie::compare_implementations::>(data); } @@ -87,8 +88,9 @@ fn compare_implementations_h(data: Vec<(Vec, Vec)>) { compare_implementations_h_internal::(data.clone()); compare_implementations_h_internal::(data.clone()); } -fn compare_implementations_h_internal(data: Vec<(Vec, Vec)>) - where T::Location: std::fmt::Debug, +fn compare_implementations_h_internal(data: Vec<(Vec, Vec)>) +where + T::Location: std::fmt::Debug, { reference_trie::compare_implementations::>(data.clone()); } diff --git a/trie-db/test/src/iterator.rs b/trie-db/test/src/iterator.rs index d9909cae..b5616c2d 100644 --- a/trie-db/test/src/iterator.rs +++ b/trie-db/test/src/iterator.rs @@ -267,8 +267,9 @@ fn seek_over_empty_works_internal() { } test_layouts!(iterate_over_incomplete_db, iterate_over_incomplete_db_internal); -fn iterate_over_incomplete_db_internal() - where T::Location: std::fmt::Debug, +fn iterate_over_incomplete_db_internal() +where + T::Location: std::fmt::Debug, { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), diff --git a/trie-db/test/src/trie_codec.rs b/trie-db/test/src/trie_codec.rs index ac5d5389..dcd050ee 100644 --- a/trie-db/test/src/trie_codec.rs +++ b/trie-db/test/src/trie_codec.rs @@ -180,7 +180,8 @@ fn encoding_node_owned_and_decoding_node_works() { for record in recorder.drain() { let node = - <::Codec as NodeCodec>::decode(&record.data, &[(); 0]).unwrap(); + <::Codec as NodeCodec>::decode(&record.data, &[(); 0]) + .unwrap(); let node_owned = node.to_owned_node::().unwrap(); assert_eq!(record.data, node_owned.to_encoded::<::Codec>()); diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index c988b89d..c3226094 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -299,8 +299,9 @@ fn get_length_with_extension_internal() { } test_layouts!(debug_output_supports_pretty_print, debug_output_supports_pretty_print_internal); -fn debug_output_supports_pretty_print_internal() - where T::Location: std::fmt::Debug, +fn debug_output_supports_pretty_print_internal() +where + T::Location: std::fmt::Debug, { let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"B".to_vec()]; @@ -540,7 +541,6 @@ fn test_recorder_with_cache_get_hash_internal() { t.insert(key, value).unwrap(); } let root = t.commit().apply_to(&mut memdb); - let mut cache = TestTrieCache::::default(); @@ -678,8 +678,7 @@ fn test_cache_internal() { let mut cache = TestTrieCache::::default(); let changeset = { - let mut t = - TrieDBMutBuilder::::new(&memdb).with_cache(&mut cache).build(); + let mut t = TrieDBMutBuilder::::new(&memdb).with_cache(&mut cache).build(); for (key, value) in &key_value { t.insert(key, value).unwrap(); } @@ -707,8 +706,7 @@ fn test_cache_internal() { let cached_value = cache.lookup_value_for_key(&b"AB"[..]).unwrap().clone(); assert_eq!(cached_value.data().flatten().unwrap(), vec![3u8; 4]); - let mut t = - TrieDBMutBuilder::::new(&memdb).with_cache(&mut cache).build(); + let mut t = TrieDBMutBuilder::::new(&memdb).with_cache(&mut cache).build(); for (key, value) in &key_value { t.insert(key, value).unwrap(); } diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 410631bc..2a9bdd59 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -24,8 +24,8 @@ use reference_trie::{ ReferenceNodeCodecNoExt, TestTrieCache, }; use trie_db::{ - DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, - TrieError, TrieLayout, Value, CachedValue, + CachedValue, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, + TrieDBMutBuilder, TrieError, TrieLayout, Value, }; use trie_standardmap::*; @@ -105,7 +105,7 @@ fn playpen_internal() { } } assert_eq!(root, real); - + let mut memtrie = TrieDBMutBuilder::::from_existing(&memdb, root).build(); assert!(unpopulate_trie(&mut memtrie, &x), "{:?}", (test_i, initial_seed)); let root = memtrie.commit().apply_to(&mut memdb); @@ -191,7 +191,7 @@ fn remove_to_empty_no_extension_internal() { t.insert(&[0x01], big_value2).unwrap(); t.insert(&[0x01, 0x34], big_value).unwrap(); t.remove(&[0x01]).unwrap(); - + let root = t.commit().apply_to(&mut memdb); assert_eq!( &root, @@ -551,14 +551,24 @@ fn register_proof_without_value() { unsafe impl Sync for ProofRecorder {} impl HashDB for ProofRecorder { - fn get(&self, key: &::Out, prefix: Prefix, _location: ()) -> Option<(DBValue, Vec<()>)> { + fn get( + &self, + key: &::Out, + prefix: Prefix, + _location: (), + ) -> Option<(DBValue, Vec<()>)> { let v = HashDB::get(&self.db, key, prefix, ()); if let Some((v, _)) = v.as_ref() { self.record.borrow_mut().entry(key[..].to_vec()).or_insert_with(|| v.clone()); } v } - fn contains(&self, key: &::Out, prefix: Prefix, _locatoin: ()) -> bool { + fn contains( + &self, + key: &::Out, + prefix: Prefix, + _locatoin: (), + ) -> bool { self.get(key, prefix, ()).is_some() } } @@ -588,8 +598,7 @@ fn register_proof_without_value() { let root_proof = root_unpacked.clone(); { let mut trie = - TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, root_proof) - .build(); + TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, root_proof).build(); trie.get(b"te").unwrap(); trie.insert(b"test12", &[2u8; 36][..]).unwrap(); trie.remove(b"test1234").unwrap(); @@ -609,8 +618,7 @@ fn register_proof_without_value() { { let trie = - TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, root_proof) - .build(); + TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, root_proof).build(); assert!(trie.get(b"te").unwrap().is_some()); assert!(matches!( trie.get(b"test1").map_err(|e| *e), @@ -635,7 +643,7 @@ fn test_recorder_internal() { t.insert(key, value).unwrap(); } let root = t.commit().apply_to(&mut memdb); - + // Add more data, but this time only to the overlay. // While doing that we record all trie accesses to replay this operation. let mut recorder = Recorder::::new(); @@ -687,7 +695,7 @@ fn test_recorder_with_cache_internal() { } let root = t.commit().apply_to(&mut memdb); let mut validated_root = root; - + let mut cache = TestTrieCache::::default(); { @@ -733,8 +741,7 @@ fn test_recorder_with_cache_internal() { // Replay the it, but this time we use the proof. { - let mut trie = - TrieDBMutBuilder::::from_existing(&partial_db, validated_root).build(); + let mut trie = TrieDBMutBuilder::::from_existing(&partial_db, validated_root).build(); for (key, value) in key_value.iter().skip(1) { trie.insert(key, value).unwrap(); From 719aa79140b889d5ca7cc7e72f5d42a065f8032d Mon Sep 17 00:00:00 2001 From: cheme Date: Sun, 13 Aug 2023 13:28:12 +0200 Subject: [PATCH 05/90] fixing triedbmut lookup, added some testing in test. (#198) --- trie-db/src/triedbmut.rs | 9 +++--- trie-db/test/src/triedbmut.rs | 53 ++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index d4eb934a..bacc7f51 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -975,6 +975,7 @@ where handle: &NodeHandle, L::Location>, ) -> Result, TrieHash, CError> { let mut handle = handle; + // prefix only use for value node access, so this is always correct. let prefix = (full_key, None); loop { let (mid, child) = match handle { @@ -994,8 +995,8 @@ where }, NodeHandle::InMemory(handle) => match &self.storage[handle] { Node::Empty => return Ok(None), - Node::Leaf(key, value) => - if NibbleSlice::from_stored(key) == partial { + Node::Leaf(slice, value) => + if NibbleSlice::from_stored(slice) == partial { return Ok(value.in_memory_fetched_value( prefix, self.db, @@ -1046,7 +1047,7 @@ where None }) } else if partial.starts_with(&slice) { - let idx = partial.at(0); + let idx = partial.at(slice.len()); match children[idx as usize].as_ref() { Some(child) => (1 + slice.len(), child), None => return Ok(None), @@ -2016,7 +2017,7 @@ where /// Cache the given `encoded` node. fn cache_node(&mut self, hash: TrieHash) { - // If we have a cache, cache our node directly. + // Mark the node as new so that it is removed from the shared cache. if let Some(cache) = self.cache.as_mut() { cache.insert_new_node(&hash); } diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 2a9bdd59..371236dc 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -95,7 +95,15 @@ fn playpen_internal() { let real = reference_trie_root::(x.clone()); let mut memdb = PrefixedMemoryDB::::default(); let memtrie = populate_trie::(&memdb, &x); - let root = memtrie.commit().apply_to(&mut memdb); + // avoid duplicate + let value_set: std::collections::BTreeMap<&[u8], &[u8]> = + x.iter().map(|(k, v)| (k.as_slice(), v.as_slice())).collect(); + for (k, v) in value_set { + assert_eq!(memtrie.get(k).unwrap().unwrap(), v); + } + let commit = memtrie.commit(); + let root = commit.apply_to(&mut memdb); + if root != real { println!("TRIE MISMATCH"); println!(); @@ -804,3 +812,46 @@ fn test_insert_remove_data_with_cache_internal() { assert!(matches!(cache.lookup_value_for_key(key).unwrap(), CachedValue::NonExisting)); } } + +#[test] +fn test_two_assets_memory_db() { + test_two_assets_memory_db_inner_1::>(); + test_two_assets_memory_db_inner_2::>(); +} +fn test_two_assets_memory_db_inner_1() { + let memdb = PrefixedMemoryDB::::new(&[0u8]); + let mut state = TrieDBMutBuilder::::new(&memdb).build(); + + let key1 = [1u8; 3]; + let data1 = [1u8; 2]; + state.insert(key1.as_ref(), &data1).unwrap(); + assert_eq!(state.get(key1.as_ref()).unwrap().unwrap(), data1); //PASSING + let key2 = [2u8; 3]; + let data2 = [2u8; 2]; + state.insert(key2.as_ref(), &data2).unwrap(); + assert_eq!(state.get(key1.as_ref()).unwrap().unwrap(), data1); + + state.commit(); +} + +fn test_two_assets_memory_db_inner_2() { + let memdb = PrefixedMemoryDB::::new(&[0u8]); + let mut state = TrieDBMutBuilder::::new(&memdb).build(); + + let key1 = [1u8]; + let data1 = [1u8; 2]; + state.insert(key1.as_ref(), &data1).unwrap(); + assert_eq!(state.get(key1.as_ref()).unwrap().unwrap(), data1); + let key2 = [1u8, 2]; + let data2 = [2u8; 2]; + state.insert(key2.as_ref(), &data2).unwrap(); + assert_eq!(state.get(key1.as_ref()).unwrap().unwrap(), data1); + assert_eq!(state.get(key2.as_ref()).unwrap().unwrap(), data2); + + let key3 = [1u8, 3]; + let data3 = [3u8; 2]; + state.insert(key3.as_ref(), &data3).unwrap(); + assert_eq!(state.get(key1.as_ref()).unwrap().unwrap(), data1); + assert_eq!(state.get(key2.as_ref()).unwrap().unwrap(), data2); + assert_eq!(state.get(key3.as_ref()).unwrap().unwrap(), data3); +} From bd9e3d11d4500ee73ce0565cff1753e3a99dd892 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 13 Aug 2023 13:39:59 +0200 Subject: [PATCH 06/90] Fixed fuzzing tests --- trie-db/fuzz/src/lib.rs | 63 +++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index 6c980870..69dbeb28 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -91,7 +91,7 @@ fn fuzz_removal(data: Vec<(Vec, Vec)>) -> Vec<(bool, Vec, Vec)> pub fn fuzz_that_reference_trie_root(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data(input)); - let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); + let memdb = MemoryDB::<_, HashKey<_>, _>::default(); let mut t = TrieDBMutBuilder::::new(&memdb).build(); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); @@ -101,7 +101,7 @@ pub fn fuzz_that_reference_trie_root(input: &[u8]) { pub fn fuzz_that_reference_trie_root_fix_length(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data_fix_length(input)); - let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); + let memdb = MemoryDB::<_, HashKey<_>, _>::default(); let mut t = TrieDBMutBuilder::::new(&memdb).build(); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); @@ -140,7 +140,7 @@ pub fn fuzz_that_compare_implementations(input: &[u8]) pub fn fuzz_that_no_extension_insert(input: &[u8]) { let data = fuzz_to_data(input); //println!("data{:?}", data); - let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); + let memdb = MemoryDB::<_, HashKey<_>, _>::default(); let mut t = TrieDBMutBuilder::::new(&memdb).build(); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); @@ -156,21 +156,18 @@ pub fn fuzz_that_no_extension_insert_remove(input: &[u8]) { let data = fuzz_to_data(input); let data = fuzz_removal(data); - let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - compare_insert_remove::(data, memdb); + compare_insert_remove::>(data); } pub fn fuzz_seek_iter(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data_fix_length(input)); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for a in 0..data.len() { - t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for a in 0..data.len() { + t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } + let root = t.commit().apply_to(&mut memdb); // fuzzing around a fix prefix of 6 nibble. let prefix = &b"012"[..]; @@ -213,13 +210,11 @@ pub fn fuzz_prefix_iter(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data_fix_length(input)); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for a in 0..data.len() { - t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for a in 0..data.len() { + t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } + let root = t.commit().apply_to(&mut memdb); // fuzzing around a fix prefix of 6 nibble. let prefix = &b"012"[..]; @@ -279,13 +274,11 @@ pub fn fuzz_prefix_seek_iter(mut input: PrefixSeekTestInput) { input.keys.dedup(); let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (index, key) in input.keys.iter().enumerate() { - t.insert(&key, &[index as u8]).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (index, key) in input.keys.iter().enumerate() { + t.insert(&key, &[index as u8]).unwrap(); } + let root = t.commit().apply_to(&mut memdb); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let iter = @@ -389,13 +382,11 @@ fn test_generate_proof( // Populate DB with full trie from entries. let (db, root) = { let mut db = , _>>::default(); - let mut root = Default::default(); - { - let mut trie = TrieDBMutBuilder::::new(&mut db, &mut root).build(); - for (key, value) in entries { - trie.insert(&key, &value).unwrap(); - } + let mut trie = TrieDBMutBuilder::::new(&mut db).build(); + for (key, value) in entries { + trie.insert(&key, &value).unwrap(); } + let root = trie.commit().apply_to(&mut db); (db, root) }; @@ -414,19 +405,17 @@ fn test_generate_proof( } fn test_trie_codec_proof(entries: Vec<(Vec, Vec)>, keys: Vec>) { - use hash_db::{HashDB, EMPTY_PREFIX}; + use hash_db::EMPTY_PREFIX; use trie_db::{decode_compact, encode_compact, Recorder}; // Populate DB with full trie from entries. let (db, root) = { let mut db = , _>>::default(); - let mut root = Default::default(); - { - let mut trie = TrieDBMutBuilder::::new(&mut db, &mut root).build(); - for (key, value) in entries { - trie.insert(&key, &value).unwrap(); - } + let mut trie = TrieDBMutBuilder::::new(&db).build(); + for (key, value) in entries { + trie.insert(&key, &value).unwrap(); } + let root = trie.commit().apply_to(&mut db); (db, root) }; let expected_root = root; @@ -457,7 +446,7 @@ fn test_trie_codec_proof(entries: Vec<(Vec, Vec)>, keys: let expected_used = compact_trie.len(); // Reconstruct the partial DB from the compact encoding. let mut db = , _>>::default(); - let (root, used) = decode_compact::(&mut db, &compact_trie).unwrap(); + let (root, used) = decode_compact::(&mut db, &compact_trie).unwrap(); assert_eq!(root, expected_root); assert_eq!(used, expected_used); From 1cf9d43096bec86752da710691c5776ec4a00e98 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 6 Sep 2023 11:08:20 +0200 Subject: [PATCH 07/90] Removed missing_location --- trie-db/src/node.rs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index 2b9d57d5..865c93db 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -88,16 +88,6 @@ where } } -impl NodeHandleOwned { - /// Check if location hint is missing - fn missing_location(&self) -> bool { - match self { - NodeHandleOwned::Hash(_, l) => *l == Default::default(), - NodeHandleOwned::Inline(_) => false, - } - } -} - impl NodeHandleOwned { /// Returns `self` as inline node. pub fn as_inline(&self) -> Option<&NodeOwned> { @@ -383,18 +373,6 @@ where } } -impl NodeOwned { - /// Return true if the node may have at least one child node. - pub fn has_missing_children_locations(&self) -> bool { - match self { - Self::Leaf(_, _) | Self::Empty | Self::Value(_, _) => false, - Self::Extension(_, h) => h.missing_location(), - Self::Branch(c, ..) | Self::NibbledBranch(_, c, ..) => - c.iter().any(|c| c.as_ref().map_or(false, |c| c.missing_location())), - } - } -} - impl NodeOwned { /// Returns the data attached to this node. pub fn data(&self) -> Option<&Bytes> { From bac0cbcf00b88228d902cfd11bb7b536ec58023d Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 20 Sep 2023 10:11:03 +0200 Subject: [PATCH 08/90] Comments --- Cargo.toml | 1 + hash-db/src/lib.rs | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c2a1e083..e1db486b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "memory-db", "hash256-std-hasher", "test-support/keccak-hasher", + "test-support/mem-tree-db", "test-support/reference-trie", "test-support/trie-standardmap", "test-support/trie-bench", diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index 19f0ac86..ac5ca6c8 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -78,17 +78,18 @@ pub trait Hasher: Sync + Send { fn hash(x: &[u8]) -> Self::Out; } -/// Trait modelling datastore keyed by a hash defined by the `Hasher`. -pub trait HashDB: Send + Sync { - /// Look up a given hash into the bytes that hash to it, returning None if the - /// hash is not known. - fn get(&self, key: &H::Out, prefix: Prefix, location: C) -> Option<(T, Vec)>; +/// Trait modelling datastore keyed by a hash defined by the `Hasher` and optional location tag. +pub trait HashDB: Send + Sync { + /// Look up a trie node by hash and location. + /// Returns the node bytes and the list of children node locations if any. + fn get(&self, key: &H::Out, prefix: Prefix, location: L) -> Option<(T, Vec)>; - /// Check for the existence of a hash-key. - fn contains(&self, key: &H::Out, prefix: Prefix, location: C) -> bool { + /// Check for the existence of a hash-key at the location. + fn contains(&self, key: &H::Out, prefix: Prefix, location: L) -> bool { self.get(key, prefix, location).is_some() } + /// Compute value hash. fn hash(&self, value: &[u8]) -> H::Out { H::hash(value) } From 0ff5025be7ba8b5d549fb4db24bccd8a109bf4b5 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 26 Nov 2023 19:03:54 +0100 Subject: [PATCH 09/90] Tests and fixes --- test-support/mem-tree-db/Cargo.toml | 17 + test-support/mem-tree-db/src/lib.rs | 323 ++++++++++++++++++ test-support/reference-trie/Cargo.toml | 1 + test-support/reference-trie/src/lib.rs | 42 ++- .../reference-trie/src/substrate_like.rs | 8 +- trie-db/src/node.rs | 28 +- trie-db/src/proof/generate.rs | 2 +- trie-db/src/triedbmut.rs | 4 +- trie-db/test/Cargo.toml | 1 + trie-db/test/src/iter_build.rs | 20 +- trie-db/test/src/iterator.rs | 46 ++- trie-db/test/src/lib.rs | 62 ++++ trie-db/test/src/proof.rs | 60 ++-- trie-db/test/src/trie_codec.rs | 18 +- trie-db/test/src/triedb.rs | 149 ++++---- trie-db/test/src/triedbmut.rs | 163 ++++----- 16 files changed, 689 insertions(+), 255 deletions(-) create mode 100644 test-support/mem-tree-db/Cargo.toml create mode 100644 test-support/mem-tree-db/src/lib.rs diff --git a/test-support/mem-tree-db/Cargo.toml b/test-support/mem-tree-db/Cargo.toml new file mode 100644 index 00000000..08f8e5b0 --- /dev/null +++ b/test-support/mem-tree-db/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "mem-tree-db" +version = "0.0.1" +authors = ["Parity Technologies "] +description = "Simple reference trie format" +repository = "https://github.com/paritytech/trie/" +license = "Apache-2.0" +edition = "2018" + +[dependencies] +hash-db = { path = "../../hash-db", version = "0.16.0"} +keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } +trie-db = { path = "../../trie-db", version = "0.28.0" } +trie-root = { path = "../../trie-root", version = "0.18.0" } +parity-scale-codec = { version = "3.0.0", features = ["derive"] } + +[dev-dependencies] \ No newline at end of file diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs new file mode 100644 index 00000000..3221b491 --- /dev/null +++ b/test-support/mem-tree-db/src/lib.rs @@ -0,0 +1,323 @@ +// Copyright 2017-2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Flat memory-based `HashDB` implementation. + +use std::collections::HashMap; + +use hash_db::{HashDB, Hasher, Prefix}; +use trie_db::{Changeset, ChangesetNodeRef}; + +/// Node location which is just an index into the `nodes` vector. +pub type Location = Option; + +/// Tree based `HashDB` implementation. +#[derive(Clone)] +pub struct MemTreeDB +where + H: Hasher, +{ + nodes: Vec>, + roots: HashMap, + hashed_null_node: H::Out, + null_node_data: Vec, +} + +#[derive(Clone)] +enum NodeEntry { + Live { + key: H, + data: Vec, + children: Vec, + rc: u32, + }, + Removed, +} + +impl Default for MemTreeDB +where + H: Hasher, +{ + fn default() -> Self { + Self::from_null_node(&[0u8][..], [0u8][..].into()) + } +} + +impl MemTreeDB +where + H: Hasher, +{ + /// Create a new `MemoryDB` from a given null key/data + pub fn from_null_node(null_key: &[u8], null_node_data: &[u8]) -> Self { + MemTreeDB { + nodes: vec![], + roots: HashMap::default(), + hashed_null_node: H::hash(null_key), + null_node_data: null_node_data.to_vec(), + } + } + + /// Create a new instance of `Self`. + pub fn new(data: &[u8]) -> Self { + Self::from_null_node(data, data.into()) + } + + pub fn clear(&mut self) { + self.nodes.clear(); + self.roots.clear(); + } + + pub fn remove(&mut self, key: &H::Out) { + let Some(location) = self.roots.get(key) else { + return; + }; + + if self.remove_node(*location) { + self.roots.remove(key); + } + } + + fn remove_node(&mut self, location: usize) -> bool{ + let entry = self.nodes.get_mut(location).unwrap(); + match entry { + NodeEntry::Live { rc, children, .. } => { + if *rc == 1 { + let children = std::mem::take(children); + *entry = NodeEntry::Removed; + for c in children { + self.remove_node(c); + } + true + } else { + *rc -= 1; + false + } + } + NodeEntry::Removed => { + panic!("Accessing removed node"); + } + } + } + + pub fn is_empty(&self) -> bool { + self.roots.is_empty() + } + + fn apply(&mut self, c: &ChangesetNodeRef) -> usize { + match c { + ChangesetNodeRef::Existing(e) => { + let location = e.location.unwrap_or_else(|| *self.roots.get(&e.hash).unwrap()); + let entry = self.nodes.get_mut(location).unwrap(); + match entry { + NodeEntry::Live { rc, .. } => { + *rc += 1; + }, + NodeEntry::Removed => { + panic!("Accessing removed node"); + }, + }; + location + }, + ChangesetNodeRef::New(n) => { + let children = n.children.iter().map(|c| self.apply(c)).collect(); + self.nodes.push(NodeEntry::Live { + key: n.hash, + data: n.data.clone(), + children, + rc: 1, + }); + self.nodes.len() - 1 + }, + } + } + + pub fn apply_commit(&mut self, commit: Changeset) { + if commit.root_hash() != self.hashed_null_node { + let root = self.apply(&commit.root); + let key = commit.root.hash(); + self.roots.insert(*key, root); + } + for (k, _) in commit.removed { + self.remove(&k); + } + } +} + +impl HashDB, Location> for MemTreeDB +where + H: Hasher, +{ + fn get(&self, k: &H::Out, _prefix: Prefix, location: Location) -> Option<(Vec, Vec)> { + if k == &self.hashed_null_node { + return Some((self.null_node_data.clone(), Default::default())) + } + + let location = match location { + Some(l) => l, + None => if let Some(l) = self.roots.get(k) { + *l + } else { + return None + } + }; + match self.nodes.get(location) { + Some(NodeEntry::Live { data, children, key, .. }) => { + assert_eq!(k, key); + Some((data.clone(), children.iter().map(|l| Some(*l)).collect())) + }, + _ => None, + } + } + + fn contains(&self, key: &H::Out, _prefix: Prefix, location: Location) -> bool { + if key == &self.hashed_null_node { + return true; + } + if let Some(l) = location { + l < self.nodes.len() && !matches!(self.nodes[l], NodeEntry::Removed) + } else { + self.roots.contains_key(key) + } + } +} + +#[cfg(test)] +mod tests { + +use super::{MemTreeDB, Location, NodeEntry}; +use keccak_hasher::{KeccakHash, KeccakHasher}; +use trie_db::{Changeset, NewChangesetNode, ChangesetNodeRef, ExistingChangesetNode}; +use hash_db::{Hasher, HashDB}; + +fn hash(i: u32) -> KeccakHash { + KeccakHasher::hash(&i.to_le_bytes()) +} + +#[test] +fn test_apply_existing_node() { + let mut db = MemTreeDB::::default(); + + // First, apply a new node + let new_node = ChangesetNodeRef::New(NewChangesetNode { + hash: hash(1), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + }); + let new_location = db.apply(&new_node); + + // Then, apply an existing node that refers to the new node + let existing_node = ChangesetNodeRef::Existing(ExistingChangesetNode { + hash: hash(1), + location: Some(new_location), + prefix: Default::default(), + }); + let existing_location = db.apply(&existing_node); + + assert_eq!(existing_location, new_location); +} + +#[test] +fn test_apply_new_node() { + let mut db = MemTreeDB::::default(); + let node = ChangesetNodeRef::New(NewChangesetNode { + hash: KeccakHash::default(), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + }); + let location = db.apply(&node); + assert_eq!(location, db.nodes.len() - 1); +} + +#[test] +fn test_apply_commit() { + let mut db = MemTreeDB::::default(); + let commit = Changeset:: { + root: ChangesetNodeRef::New(NewChangesetNode { + hash: KeccakHash::default(), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + }), + removed: Default::default(), + }; + db.apply_commit(commit); + assert_eq!(db.roots.len(), 1); +} +#[test] +fn test_commit_changeset_with_children() { + let mut db = MemTreeDB::::default(); + + // Create two child nodes + let child1 = ChangesetNodeRef::New(NewChangesetNode { + hash: hash(1), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + }); + let child2 = ChangesetNodeRef::New(NewChangesetNode { + hash: hash(2), + prefix: Default::default(), + data: vec![4, 5, 6], + children: vec![], + }); + + // Create a root node that refers to the child nodes + let root = ChangesetNodeRef::New(NewChangesetNode { + hash: hash(0), + prefix: Default::default(), + data: vec![7, 8, 9], + children: vec![child1, child2], + }); + + let commit = Changeset:: { + root, + removed: Default::default(), + }; + db.apply_commit(commit); + + // Check that the root node and child nodes are in the database + assert_eq!(db.nodes.len(), 3); + assert_eq!(db.roots.len(), 1); +} + #[test] + fn test_get() { + let mut db = MemTreeDB::::default(); + let key = KeccakHash::default(); + db.nodes.push(NodeEntry::Live { + key: key.clone(), + data: vec![1, 2, 3], + children: vec![], + rc: 1, + }); + db.roots.insert(key.clone(), 0); + let result = db.get(&key, Default::default(), None); + assert_eq!(result, Some((vec![1, 2, 3], vec![]))); + } + + #[test] + fn test_contains() { + let mut db = MemTreeDB::::default(); + let key = KeccakHash::default(); + db.nodes.push(NodeEntry::Live { + key: key.clone(), + data: vec![1, 2, 3], + children: vec![], + rc: 1, + }); + db.roots.insert(key.clone(), 0); + assert!(db.contains(&key, Default::default(), None)); + } +} diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index 4d402724..d38ae152 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -10,6 +10,7 @@ edition = "2018" [dependencies] hash-db = { path = "../../hash-db", version = "0.16.0"} memory-db = { path = "../../memory-db", default-features = false, version = "0.32.0"} +mem-tree-db = { path = "../mem-tree-db", default-features = false, version = "0.0.1"} keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } trie-db = { path = "../../trie-db", default-features = false, version = "0.28.0" } trie-root = { path = "../../trie-root", default-features = false, version = "0.18.0" } diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index ed483f20..c058746c 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -42,24 +42,36 @@ pub use substrate_like::{ pub use paste::paste; pub use substrate::{LayoutV0 as SubstrateV0, LayoutV1 as SubstrateV1}; +pub use mem_tree_db::MemTreeDB; +pub use mem_tree_db::Location as MemLocation; /// Reference hasher is a keccak hasher. pub type RefHasher = keccak_hasher::KeccakHasher; +pub type PrefixedMemoryDB = + MemoryDB<::Hash, memory_db::PrefixedKey<::Hash>, DBValue>; + /// Apply a test method on every test layouts. #[macro_export] macro_rules! test_layouts { ($test:ident, $test_internal:ident) => { #[test] fn $test() { - eprintln!("Running with layout `HashedValueNoExtThreshold`"); - $test_internal::<$crate::HashedValueNoExtThreshold<1>>(); - eprintln!("Running with layout `HashedValueNoExt`"); - $test_internal::<$crate::HashedValueNoExt>(); - eprintln!("Running with layout `NoExtensionLayout`"); - $test_internal::<$crate::NoExtensionLayout>(); - eprintln!("Running with layout `ExtensionLayout`"); - $test_internal::<$crate::ExtensionLayout>(); + //eprintln!("Running with layout `HashedValueNoExtThreshold` and MemoryDB"); + //$test_internal::<$crate::HashedValueNoExtThreshold<1, ()>, $crate::PrefixedMemoryDB<$crate::HashedValueNoExtThreshold<1, ()>>>(); + //eprintln!("Running with layout `HashedValueNoExt` and MemoryDB"); + //$test_internal::<$crate::HashedValueNoExt, $crate::PrefixedMemoryDB<$crate::HashedValueNoExt>>(); + //eprintln!("Running with layout `NoExtensionLayout` and MemoryDB"); + //$test_internal::<$crate::NoExtensionLayout, $crate::PrefixedMemoryDB<$crate::NoExtensionLayout>>(); + //eprintln!("Running with layout `ExtensionLayout` and MemoryDB"); + //$test_internal::<$crate::ExtensionLayout, $crate::PrefixedMemoryDB<$crate::ExtensionLayout>>(); + + //eprintln!("Running with layout `HashedValueNoExtThreshold` and MemTreeDB"); + //$test_internal::<$crate::HashedValueNoExtThreshold<1, $crate::MemLocation>, $crate::MemTreeDB<$crate::RefHasher>>(); + eprintln!("Running with layout `HashedValueNoExt` and MemTreeDB"); + $test_internal::<$crate::HashedValueNoExt, $crate::MemTreeDB<$crate::RefHasher>>(); + eprintln!("Running with layout `NoExtensionLayout` and MemTreeDB"); + $test_internal::<$crate::GenericNoExtensionLayout<$crate::RefHasher, $crate::MemLocation>, $crate::MemTreeDB<$crate::RefHasher>>(); } }; } @@ -109,27 +121,27 @@ impl TrieConfiguration for ExtensionLayout {} /// Trie layout without extension nodes, allowing /// generic hasher. -pub struct GenericNoExtensionLayout(PhantomData); +pub struct GenericNoExtensionLayout(PhantomData<(H, L)>); -impl Default for GenericNoExtensionLayout { +impl Default for GenericNoExtensionLayout { fn default() -> Self { GenericNoExtensionLayout(PhantomData) } } -impl Clone for GenericNoExtensionLayout { +impl Clone for GenericNoExtensionLayout { fn clone(&self) -> Self { GenericNoExtensionLayout(PhantomData) } } -impl TrieLayout for GenericNoExtensionLayout { +impl TrieLayout for GenericNoExtensionLayout { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = false; const MAX_INLINE_VALUE: Option = None; type Hash = H; type Codec = ReferenceNodeCodecNoExt; - type Location = (); + type Location = L; } /// Trie that allows empty values. @@ -145,10 +157,10 @@ impl TrieLayout for AllowEmptyLayout { type Location = (); } -impl TrieConfiguration for GenericNoExtensionLayout {} +impl TrieConfiguration for GenericNoExtensionLayout {} /// Trie layout without extension nodes. -pub type NoExtensionLayout = GenericNoExtensionLayout; +pub type NoExtensionLayout = GenericNoExtensionLayout; /// Children bitmap codec for radix 16 trie. pub struct Bitmap(u16); diff --git a/test-support/reference-trie/src/substrate_like.rs b/test-support/reference-trie/src/substrate_like.rs index e4fd2631..104d1618 100644 --- a/test-support/reference-trie/src/substrate_like.rs +++ b/test-support/reference-trie/src/substrate_like.rs @@ -22,7 +22,7 @@ pub struct HashedValueNoExt; /// No extension trie which stores value above a static size /// as external node. -pub struct HashedValueNoExtThreshold; +pub struct HashedValueNoExtThreshold(PhantomData); impl TrieLayout for HashedValueNoExt { const USE_EXTENSION: bool = false; @@ -31,17 +31,17 @@ impl TrieLayout for HashedValueNoExt { type Hash = RefHasher; type Codec = ReferenceNodeCodecNoExtMeta; - type Location = (); + type Location = mem_tree_db::Location; } -impl TrieLayout for HashedValueNoExtThreshold { +impl TrieLayout for HashedValueNoExtThreshold { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = false; const MAX_INLINE_VALUE: Option = Some(C); type Hash = RefHasher; type Codec = ReferenceNodeCodecNoExtMeta; - type Location = (); + type Location = L; } /// Constants specific to encoding with external value node support. diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index 865c93db..b47d978d 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -574,6 +574,16 @@ impl NodePlan { NodePlan::Branch { value, children } => { let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; let mut nc = 0; + let value = if let Some(v) = value { + if v.is_inline() { + Some(v.build(data, locations.first().copied().unwrap_or_default())) + } else { + nc += 1; + Some(v.build(data, Default::default())) + } + } else { + None + }; for i in 0..nibble_ops::NIBBLE_LENGTH { if let Some(child) = &children[i] { let location = if child.is_inline() { @@ -588,14 +598,22 @@ impl NodePlan { } Node::Branch( child_slices, - value - .as_ref() - .map(|v| v.build(data, locations.last().copied().unwrap_or_default())), + value, ) }, NodePlan::NibbledBranch { partial, value, children } => { let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; let mut nc = 0; + let value = if let Some(v) = value { + if v.is_inline() { + Some(v.build(data, locations.first().copied().unwrap_or_default())) + } else { + nc += 1; + Some(v.build(data, Default::default())) + } + } else { + None + }; for i in 0..nibble_ops::NIBBLE_LENGTH { if let Some(child) = &children[i] { let location = if child.is_inline() { @@ -611,9 +629,7 @@ impl NodePlan { Node::NibbledBranch( partial.build(data), child_slices, - value - .as_ref() - .map(|v| v.build(data, locations.last().copied().unwrap_or_default())), + value, ) }, } diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index ab24cd9b..a4ca9c21 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -473,7 +473,7 @@ fn match_key_to_branch_node<'a, 'b, C: NodeCodec, L: Copy + Default>( let value = match value { Some(Value::Inline(data)) => { *omit_value = true; - Some(data.clone()) + Some(data) }, Some(Value::Node(_, _)) => { *omit_value = true; diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index fa79a3e8..841afccd 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -170,8 +170,10 @@ impl Value { } else { *hash = Some(new_hash); } + } else if let Value::Node(hash, location) = self { + f(NodeToEncode::TrieNode(NodeHandle::Hash(*hash, *location)), partial, None); } - let value = match &*self { + let value = match &*self { Value::Inline(value) => EncodedValue::Inline(&value), Value::Node(hash, location) => EncodedValue::Node(hash.as_ref(), *location), Value::NewNode(Some(hash), _value) => diff --git a/trie-db/test/Cargo.toml b/trie-db/test/Cargo.toml index 6282df38..c8b31a20 100644 --- a/trie-db/test/Cargo.toml +++ b/trie-db/test/Cargo.toml @@ -18,6 +18,7 @@ memory-db = { path = "../../memory-db", version = "0.32.0" } rand = { version = "0.8", default-features = false, features = ["small_rng"] } trie-standardmap = { path = "../../test-support/trie-standardmap", version = "0.16.0" } reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } +mem-tree-db = { path = "../../test-support/mem-tree-db", version = "0.0.1" } hex-literal = "0.4" criterion = "0.4.0" env_logger = { version = "0.10", default-features = false } diff --git a/trie-db/test/src/iter_build.rs b/trie-db/test/src/iter_build.rs index 20d37b5d..55a0b811 100644 --- a/trie-db/test/src/iter_build.rs +++ b/trie-db/test/src/iter_build.rs @@ -18,6 +18,8 @@ use reference_trie::{ }; use trie_db::{DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout}; +use crate::TestDB; + #[test] fn trie_root_empty() { compare_implementations(vec![]) @@ -62,7 +64,7 @@ fn test_iter(data: Vec<(Vec, Vec)>) { } fn compare_implementations(data: Vec<(Vec, Vec)>) { - test_iter::>(data.clone()); + test_iter::>(data.clone()); test_iter::(data.clone()); test_iter::(data.clone()); test_iter::(data.clone()); @@ -71,7 +73,7 @@ fn compare_implementations(data: Vec<(Vec, Vec)>) { } fn compare_implementations_prefixed(data: Vec<(Vec, Vec)>) { - compare_implementations_prefixed_internal::>(data.clone()); + compare_implementations_prefixed_internal::>(data.clone()); compare_implementations_prefixed_internal::(data.clone()); compare_implementations_prefixed_internal::(data.clone()); compare_implementations_prefixed_internal::(data.clone()); @@ -83,7 +85,7 @@ where reference_trie::compare_implementations::>(data); } fn compare_implementations_h(data: Vec<(Vec, Vec)>) { - compare_implementations_h_internal::>(data.clone()); + compare_implementations_h_internal::>(data.clone()); compare_implementations_h_internal::(data.clone()); compare_implementations_h_internal::(data.clone()); compare_implementations_h_internal::(data.clone()); @@ -100,8 +102,8 @@ fn compare_implementations_no_extension_unordered(data: Vec<(Vec, Vec)>) fn compare_insert_remove(data: Vec<(bool, Vec, Vec)>) { reference_trie::compare_insert_remove::>(data); } -fn compare_root(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::, _>::default(); +fn compare_root>(data: Vec<(Vec, Vec)>) { + let memdb = DB::default(); reference_trie::compare_root::(data, memdb); } fn compare_unhashed(data: Vec<(Vec, Vec)>) { @@ -131,8 +133,8 @@ fn trie_middle_node2() { ]); } test_layouts!(root_extension_bis, root_extension_bis_internal); -fn root_extension_bis_internal() { - compare_root::(vec![ +fn root_extension_bis_internal>() { + compare_root::(vec![ (vec![1u8, 2u8, 3u8, 3u8], vec![8u8; 32]), (vec![1u8, 2u8, 3u8, 4u8], vec![7u8; 32]), ]); @@ -238,7 +240,7 @@ fn fuzz_no_extension4() { ]); } test_layouts!(fuzz_no_extension_insert_remove_1, fuzz_no_extension_insert_remove_1_internal); -fn fuzz_no_extension_insert_remove_1_internal() { +fn fuzz_no_extension_insert_remove_1_internal>() { let data = vec![ (false, vec![0], vec![251, 255]), (false, vec![0, 1], vec![251, 255]), @@ -248,7 +250,7 @@ fn fuzz_no_extension_insert_remove_1_internal() { compare_insert_remove::(data); } test_layouts!(fuzz_no_extension_insert_remove_2, fuzz_no_extension_insert_remove_2_internal); -fn fuzz_no_extension_insert_remove_2_internal() { +fn fuzz_no_extension_insert_remove_2_internal>() { let data = vec![ (false, vec![0x00], vec![0xfd, 0xff]), (false, vec![0x10, 0x00], vec![1; 32]), diff --git a/trie-db/test/src/iterator.rs b/trie-db/test/src/iterator.rs index b5616c2d..b0ded21f 100644 --- a/trie-db/test/src/iterator.rs +++ b/trie-db/test/src/iterator.rs @@ -17,20 +17,16 @@ use hex_literal::hex; use reference_trie::test_layouts; use trie_db::{ node::{Node, Value}, - DBValue, NibbleSlice, NibbleVec, TrieDBBuilder, TrieDBNodeIterator, TrieError, TrieIterator, + NibbleSlice, NibbleVec, TrieDBBuilder, TrieDBNodeIterator, TrieError, TrieIterator, TrieLayout, }; -type MemoryDB = memory_db::MemoryDB< - ::Hash, - memory_db::PrefixedKey<::Hash>, - DBValue, ->; +use crate::TestDB; -fn build_trie_db( +fn build_trie_db>( pairs: &[(Vec, Vec)], -) -> (MemoryDB, ::Out) { - let mut memdb = MemoryDB::::default(); +) -> (DB, ::Out) { + let mut memdb = DB::default(); let changeset = { let mut t = trie_db::TrieDBMutBuilder::::new(&mut memdb).build(); for (x, y) in pairs.iter() { @@ -38,8 +34,8 @@ fn build_trie_db( } t.commit() }; - changeset.apply_to(&mut memdb); - (memdb, *changeset.root.hash()) + let root = memdb.commit(changeset); + (memdb, root) } fn nibble_vec>(bytes: T, len: usize) -> NibbleVec { @@ -53,14 +49,14 @@ fn nibble_vec>(bytes: T, len: usize) -> NibbleVec { } test_layouts!(iterator_works, iterator_works_internal); -fn iterator_works_internal() { +fn iterator_works_internal>() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db::(&pairs); + let (memdb, root) = build_trie_db::(&pairs); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); @@ -185,8 +181,8 @@ fn iterator_works_internal() { } test_layouts!(iterator_over_empty_works, iterator_over_empty_works_internal); -fn iterator_over_empty_works_internal() { - let (memdb, root) = build_trie_db::(&[]); +fn iterator_over_empty_works_internal>() { + let (memdb, root) = build_trie_db::(&[]); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); @@ -205,14 +201,14 @@ fn iterator_over_empty_works_internal() { } test_layouts!(seek_works, seek_works_internal); -fn seek_works_internal() { +fn seek_works_internal>() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db::(&pairs); + let (memdb, root) = build_trie_db::(&pairs); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); @@ -245,8 +241,8 @@ fn seek_works_internal() { } test_layouts!(seek_over_empty_works, seek_over_empty_works_internal); -fn seek_over_empty_works_internal() { - let (memdb, root) = build_trie_db::(&[]); +fn seek_over_empty_works_internal>() { + let (memdb, root) = build_trie_db::(&[]); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); @@ -267,7 +263,7 @@ fn seek_over_empty_works_internal() { } test_layouts!(iterate_over_incomplete_db, iterate_over_incomplete_db_internal); -fn iterate_over_incomplete_db_internal() +fn iterate_over_incomplete_db_internal>() where T::Location: std::fmt::Debug, { @@ -278,7 +274,7 @@ where (hex!("03").to_vec(), vec![2; 32]), ]; - let (mut memdb, root) = build_trie_db::(&pairs); + let (mut memdb, root) = build_trie_db::(&pairs); // Look up the leaf node with prefix "02". let leaf_hash = { @@ -342,7 +338,7 @@ where } test_layouts!(prefix_works, prefix_works_internal); -fn prefix_works_internal() { +fn prefix_works_internal>() { let can_expand = T::MAX_INLINE_VALUE.unwrap_or(T::Hash::LENGTH as u32) < T::Hash::LENGTH as u32; let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), @@ -350,7 +346,7 @@ fn prefix_works_internal() { (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db::(&pairs); + let (memdb, root) = build_trie_db::(&pairs); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); @@ -411,8 +407,8 @@ fn prefix_works_internal() { } test_layouts!(prefix_over_empty_works, prefix_over_empty_works_internal); -fn prefix_over_empty_works_internal() { - let (memdb, root) = build_trie_db::(&[]); +fn prefix_over_empty_works_internal>() { + let (memdb, root) = build_trie_db::(&[]); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); iter.prefix(&hex!("")[..]).unwrap(); diff --git a/trie-db/test/src/lib.rs b/trie-db/test/src/lib.rs index 2c5b7c34..d02cdf8b 100644 --- a/trie-db/test/src/lib.rs +++ b/trie-db/test/src/lib.rs @@ -14,6 +14,8 @@ //! Tests for trie-db crate. +use memory_db::MemoryDB; + #[cfg(test)] mod iter_build; #[cfg(test)] @@ -28,3 +30,63 @@ mod trie_codec; mod triedb; #[cfg(test)] mod triedbmut; + +use hash_db::{Hasher, Prefix}; +use trie_db::{DBValue, TrieLayout, TrieHash, Changeset}; +use mem_tree_db::MemTreeDB; + +trait TestDB: hash_db::HashDB + Clone + Default { + fn commit(&mut self, commit: trie_db::Changeset<<::Hash as Hasher>::Out, T::Location>) -> TrieHash; + fn remove(&mut self, hash: &::Out, prefix: Prefix); + fn is_empty(&self) -> bool; +} + +impl, H, KF> TestDB for MemoryDB + where + H: Hasher, + KF: memory_db::KeyFunction + Send + Sync, +{ + fn commit(&mut self, commit: trie_db::Changeset::Location>) -> H::Out { + commit.apply_to(self) + } + + fn remove(&mut self, hash: &::Out, prefix: Prefix) { + MemoryDB::remove(self, hash, prefix); + } + + fn is_empty(&self) -> bool { + self.keys().is_empty() + } +} + +impl, H> TestDB for MemTreeDB + where + H: Hasher + Clone, +{ + fn commit(&mut self, commit: trie_db::Changeset) -> H::Out { + let root = commit.root_hash(); + self.apply_commit(commit); + root + } + + fn remove(&mut self, hash: &H::Out, _prefix: Prefix) { + MemTreeDB::remove(self, hash); + } + + fn is_empty(&self) -> bool { + MemTreeDB::is_empty(self) + } +} + +trait TestCommit { + fn commit_to>(self, db: &mut DB) -> TrieHash; +} + +impl> TestCommit for Changeset +where +T::Hash: Hasher, +{ + fn commit_to>(self, db: &mut DB) -> TrieHash { + db.commit(self) + } +} \ No newline at end of file diff --git a/trie-db/test/src/proof.rs b/trie-db/test/src/proof.rs index 365ebf59..7db308cb 100644 --- a/trie-db/test/src/proof.rs +++ b/trie-db/test/src/proof.rs @@ -20,6 +20,8 @@ use trie_db::{ DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, }; +use crate::TestDB; + type MemoryDB = memory_db::MemoryDB< ::Hash, memory_db::HashKey<::Hash>, @@ -44,20 +46,20 @@ fn test_entries() -> Vec<(&'static [u8], &'static [u8])> { ] } -fn test_generate_proof( +fn test_generate_proof>( entries: Vec<(&'static [u8], &'static [u8])>, keys: Vec<&'static [u8]>, ) -> (::Out, Vec>, Vec<(&'static [u8], Option)>) { // Populate DB with full trie from entries. let (db, root) = { - let mut db = >::default(); + let mut db = DB::default(); let mut trie = >::new(&db).build(); for (key, value) in entries.iter() { trie.insert(key, value).unwrap(); } let commit = trie.commit(); - commit.apply_to(&mut db); - (db, *commit.root.hash()) + let root = db.commit(commit); + (db, root) }; // Generate proof for the given keys.. @@ -69,8 +71,8 @@ fn test_generate_proof( } test_layouts!(trie_proof_works2, trie_proof_works_internal2); -fn trie_proof_works_internal2() { - let (root, proof, items) = test_generate_proof::( +fn trie_proof_works_internal2>() { + let (root, proof, items) = test_generate_proof::( vec![ // "do" is at a hash-referenced branch node. (&b"do"[..], b"verb"), @@ -84,8 +86,8 @@ fn trie_proof_works_internal2() { } test_layouts!(trie_proof_works, trie_proof_works_internal); -fn trie_proof_works_internal() { - let (root, proof, items) = test_generate_proof::( +fn trie_proof_works_internal>() { + let (root, proof, items) = test_generate_proof::( test_entries(), vec![ b"do", b"dog", b"doge", b"bravo", b"alfabet", // None, not found under leaf node @@ -99,16 +101,16 @@ fn trie_proof_works_internal() { } test_layouts!(trie_proof_works_for_empty_trie, trie_proof_works_for_empty_trie_internal); -fn trie_proof_works_for_empty_trie_internal() { +fn trie_proof_works_for_empty_trie_internal>() { let (root, proof, items) = - test_generate_proof::(vec![], vec![b"alpha", b"bravo", b"\x42\x42"]); + test_generate_proof::(vec![], vec![b"alpha", b"bravo", b"\x42\x42"]); verify_proof::(&root, &proof, items.iter()).unwrap(); } test_layouts!(test_verify_duplicate_keys, test_verify_duplicate_keys_internal); -fn test_verify_duplicate_keys_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); +fn test_verify_duplicate_keys_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); let items = vec![(b"bravo", Some(b"bravo")), (b"bravo", Some(b"bravo"))]; assert!(if let Err(VerifyError::DuplicateKey(key)) = @@ -121,8 +123,8 @@ fn test_verify_duplicate_keys_internal() { } test_layouts!(test_verify_extraneaous_node, test_verify_extraneaous_node_internal); -fn test_verify_extraneaous_node_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo", b"do"]); +fn test_verify_extraneaous_node_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo", b"do"]); let items = vec![(b"bravo", Some(b"bravo"))]; assert!(matches!( @@ -132,8 +134,8 @@ fn test_verify_extraneaous_node_internal() { } test_layouts!(test_verify_extraneaous_value, test_verify_extraneaous_value_internal); -fn test_verify_extraneaous_value_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"doge"]); +fn test_verify_extraneaous_value_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"doge"]); let items = vec![(&b"do"[..], Some(&b"verb"[..])), (&b"doge"[..], Some(&[0; 32][..]))]; assert!(if let Err(VerifyError::ExtraneousValue(val)) = @@ -147,7 +149,7 @@ fn test_verify_extraneaous_value_internal() { #[test] fn test_verify_extraneous_hash_reference() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"do"]); + let (root, proof, _) = test_generate_proof::>(test_entries(), vec![b"do"]); let items = vec![(&b"alfa"[..], Some(&[0; 32][..])), (&b"do"[..], Some(&b"verb"[..]))]; match verify_proof::(&root, &proof, items.iter()) { @@ -157,8 +159,8 @@ fn test_verify_extraneous_hash_reference() { } test_layouts!(test_verify_invalid_child_reference, test_verify_invalid_child_reference_internal); -fn test_verify_invalid_child_reference_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); +fn test_verify_invalid_child_reference_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); if T::MAX_INLINE_VALUE.map_or(false, |t| t as usize <= b"bravo".len()) { // node will not be inline: ignore test @@ -177,8 +179,8 @@ test_layouts!( test_verify_value_mismatch_some_to_none, test_verify_value_mismatch_some_to_none_internal ); -fn test_verify_value_mismatch_some_to_none_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"horse"]); +fn test_verify_value_mismatch_some_to_none_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"horse"]); let items = vec![(&b"horse"[..], Some(&b"stallion"[..])), (&b"halp"[..], Some(&b"plz"[..]))]; assert!(if let Err(VerifyError::ValueMismatch(val)) = @@ -194,8 +196,8 @@ test_layouts!( test_verify_value_mismatch_none_to_some, test_verify_value_mismatch_none_to_some_internal ); -fn test_verify_value_mismatch_none_to_some_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"alfa", b"bravo"]); +fn test_verify_value_mismatch_none_to_some_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"alfa", b"bravo"]); let items = vec![(&b"alfa"[..], Some(&[0; 32][..])), (&b"bravo"[..], None)]; assert!(if let Err(VerifyError::ValueMismatch(val)) = @@ -208,8 +210,8 @@ fn test_verify_value_mismatch_none_to_some_internal() { } test_layouts!(test_verify_incomplete_proof, test_verify_incomplete_proof_internal); -fn test_verify_incomplete_proof_internal() { - let (root, mut proof, items) = test_generate_proof::(test_entries(), vec![b"alfa"]); +fn test_verify_incomplete_proof_internal>() { + let (root, mut proof, items) = test_generate_proof::(test_entries(), vec![b"alfa"]); proof.pop(); assert!(matches!( @@ -219,8 +221,8 @@ fn test_verify_incomplete_proof_internal() { } test_layouts!(test_verify_root_mismatch, test_verify_root_mismatch_internal); -fn test_verify_root_mismatch_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); +fn test_verify_root_mismatch_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); let items = vec![(b"bravo", Some("incorrect"))]; match verify_proof::(&root, &proof, items.iter()) { @@ -230,8 +232,8 @@ fn test_verify_root_mismatch_internal() { } test_layouts!(test_verify_decode_error, test_verify_decode_error_internal); -fn test_verify_decode_error_internal() { - let (root, mut proof, items) = test_generate_proof::(test_entries(), vec![b"bravo"]); +fn test_verify_decode_error_internal>() { + let (root, mut proof, items) = test_generate_proof::(test_entries(), vec![b"bravo"]); proof.insert(0, b"this is not a trie node".to_vec()); match verify_proof::(&root, &proof, items.iter()) { diff --git a/trie-db/test/src/trie_codec.rs b/trie-db/test/src/trie_codec.rs index dcd050ee..3c13b40f 100644 --- a/trie-db/test/src/trie_codec.rs +++ b/trie-db/test/src/trie_codec.rs @@ -19,26 +19,28 @@ use trie_db::{ TrieDBMutBuilder, TrieError, TrieLayout, }; +use crate::TestDB; + type MemoryDB = memory_db::MemoryDB< ::Hash, memory_db::HashKey<::Hash>, DBValue, >; -fn test_encode_compact( +fn test_encode_compact>( entries: Vec<(&'static [u8], &'static [u8])>, keys: Vec<&'static [u8]>, ) -> (::Out, Vec>, Vec<(&'static [u8], Option)>) { // Populate DB with full trie from entries. let (db, root) = { - let mut db = >::default(); + let mut db = DB::default(); let mut trie = >::new(&mut db).build(); for (key, value) in entries.iter() { trie.insert(key, value).unwrap(); } let commit = trie.commit(); - commit.apply_to(&mut db); - (db, commit.root_hash()) + let root = db.commit(commit); + (db, root) }; // Lookup items in trie while recording traversed nodes. @@ -88,8 +90,8 @@ fn test_decode_compact( } test_layouts!(trie_compact_encoding_works, trie_compact_encoding_works_internal); -fn trie_compact_encoding_works_internal() { - let (root, mut encoded, items) = test_encode_compact::( +fn trie_compact_encoding_works_internal>() { + let (root, mut encoded, items) = test_encode_compact::( vec![ // "alfa" is at a hash-referenced leaf node. (b"alfa", &[0; 32]), @@ -121,9 +123,9 @@ test_layouts!( trie_decoding_fails_with_incomplete_database, trie_decoding_fails_with_incomplete_database_internal ); -fn trie_decoding_fails_with_incomplete_database_internal() { +fn trie_decoding_fails_with_incomplete_database_internal>() { let (_, encoded, _) = - test_encode_compact::(vec![(b"alfa", &[0; 32]), (b"bravo", b"bravo")], vec![b"alfa"]); + test_encode_compact::(vec![(b"alfa", &[0; 32]), (b"bravo", b"bravo")], vec![b"alfa"]); assert!(encoded.len() > 1); diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index 7c2b9fc1..334f9836 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -16,35 +16,34 @@ use std::ops::Deref; use hash_db::{Hasher, EMPTY_PREFIX}; use hex_literal::hex; -use memory_db::{HashKey, MemoryDB, PrefixedKey}; +use memory_db::{HashKey, MemoryDB}; use reference_trie::{ - test_layouts, test_layouts_substrate, HashedValueNoExtThreshold, TestTrieCache, + test_layouts, test_layouts_substrate, HashedValueNoExtThreshold, TestTrieCache, PrefixedMemoryDB, }; use trie_db::{ encode_compact, CachedValue, DBValue, Lookup, NibbleSlice, RecordedForKey, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieRecorder, }; -type PrefixedMemoryDB = - MemoryDB<::Hash, PrefixedKey<::Hash>, DBValue>; +use crate::{TestDB, TestCommit}; + type MemoryDBProof = MemoryDB<::Hash, HashKey<::Hash>, DBValue>; test_layouts!(iterator_works, iterator_works_internal); -fn iterator_works_internal() { +fn iterator_works_internal>() { let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()), (hex!("0103000000000010000469").to_vec(), hex!("ffffffffff").to_vec()), ]; - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for (x, y) in &pairs { t.insert(x, y).unwrap(); } let commit = t.commit(); - commit.apply_to(&mut memdb); - let root = commit.root_hash(); + let root = memdb.commit(commit); let trie = TrieDBBuilder::::new(&memdb, &root).build(); @@ -59,20 +58,18 @@ fn iterator_works_internal() { } test_layouts!(iterator_seek_works, iterator_seek_works_internal); -fn iterator_seek_works_internal() { +fn iterator_seek_works_internal>() { let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()), (hex!("0103000000000000000469").to_vec(), hex!("ffffffffff").to_vec()), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for (x, y) in &pairs { t.insert(x, y).unwrap(); } - let commit = t.commit(); - commit.apply_to(&mut memdb); - let root = commit.root_hash(); + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); @@ -95,15 +92,15 @@ fn iterator_seek_works_internal() { } test_layouts!(iterator, iterator_internal); -fn iterator_internal() { +fn iterator_internal>() { let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"B".to_vec()]; - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for x in &d { t.insert(x, x).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); assert_eq!( @@ -114,16 +111,16 @@ fn iterator_internal() { } test_layouts!(iterator_seek, iterator_seek_internal); -fn iterator_seek_internal() { +fn iterator_seek_internal>() { let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"AS".to_vec(), b"B".to_vec()]; let vals = vec![vec![0; 32], vec![1; 32], vec![2; 32], vec![4; 32], vec![3; 32]]; - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for (k, val) in d.iter().zip(vals.iter()) { t.insert(k, val.as_slice()).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = t.iter().unwrap(); @@ -167,22 +164,22 @@ fn iterator_seek_internal() { assert_eq!(&vals[5..], &iter.map(|x| x.unwrap().1).collect::>()[..]); } -fn trie_from_hex_keys(keys: &[&str], callback: impl FnOnce(&mut trie_db::TrieDB)) +fn trie_from_hex_keys>(keys: &[&str], callback: impl FnOnce(&mut trie_db::TrieDB)) where T: TrieLayout, { - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for (index, key) in keys.iter().enumerate() { t.insert(&array_bytes::hex2bytes(key).unwrap(), &[index as u8]).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let mut t = TrieDBBuilder::::new(&memdb, &root).build(); callback(&mut t); } -fn test_prefixed_then_seek( +fn test_prefixed_then_seek>( keys: &[&str], prefix_key: &str, seek_key: &str, @@ -191,7 +188,7 @@ fn test_prefixed_then_seek( let prefix_key = array_bytes::hex2bytes(prefix_key).unwrap(); let seek_key = array_bytes::hex2bytes(seek_key).unwrap(); - trie_from_hex_keys::(keys, |trie| { + trie_from_hex_keys::(keys, |trie| { let iter = trie_db::TrieDBIterator::new_prefixed_then_seek(&trie, &prefix_key, &seek_key).unwrap(); let output: Vec<_> = iter.map(|x| array_bytes::bytes2hex("", x.unwrap().0)).collect(); @@ -242,36 +239,36 @@ fn iterator_prefixed_then_seek_real_world() { ]; let target_key = "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8da7dad55cf08ffe8194efa962146801b0503092b1ed6a3fa6aee9107334aefd7965bbe568c3d24c6d"; - test_prefixed_then_seek::(keys, target_key, target_key, &[]); + test_prefixed_then_seek::>(keys, target_key, target_key, &[]); } // This is the real-word test, but simplified. test_layouts_substrate!(iterator_prefixed_then_seek_simple); fn iterator_prefixed_then_seek_simple() { - test_prefixed_then_seek::(&["0100"], "00", "00", &[]); + test_prefixed_then_seek::>(&["0100"], "00", "00", &[]); } // These are just tests that the fuzzer barfed out while working on the fix for the real-world // issue. test_layouts_substrate!(iterator_prefixed_then_seek_testcase_1); fn iterator_prefixed_then_seek_testcase_1() { - test_prefixed_then_seek::(&["00"], "00", "", &["00"]) + test_prefixed_then_seek::>(&["00"], "00", "", &["00"]) } test_layouts_substrate!(iterator_prefixed_then_seek_testcase_2); fn iterator_prefixed_then_seek_testcase_2() { - test_prefixed_then_seek::(&["00", "0003"], "00", "", &["00", "0003"]) + test_prefixed_then_seek::>(&["00", "0003"], "00", "", &["00", "0003"]) } test_layouts_substrate!(iterator_prefixed_then_seek_testcase_3); fn iterator_prefixed_then_seek_testcase_3() { - test_prefixed_then_seek::(&["20"], "20", "0700", &["20"]) + test_prefixed_then_seek::>(&["20"], "20", "0700", &["20"]) } test_layouts_substrate!(iterator_prefixed_then_seek_testcase_4); fn iterator_prefixed_then_seek_testcase_4() { let keys = &["1701", "ffffffffffffffffffffffdfffffffffffffffffffffffffffffffffffffffff"]; - test_prefixed_then_seek::( + test_prefixed_then_seek::>( keys, "1701", "ffff27272727274949494949ce494949494949494949491768687b737373732b", @@ -281,16 +278,16 @@ fn iterator_prefixed_then_seek_testcase_4() { test_layouts_substrate!(iterator_prefixed_then_seek_testcase_5); fn iterator_prefixed_then_seek_testcase_5() { - test_prefixed_then_seek::(&["20"], "20", "20", &["20"]) + test_prefixed_then_seek::>(&["20"], "20", "20", &["20"]) } test_layouts!(get_length_with_extension, get_length_with_extension_internal); -fn get_length_with_extension_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn get_length_with_extension_internal>() { + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(b"A", b"ABC").unwrap(); t.insert(b"B", b"ABCBAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()).unwrap(), Some(3)); @@ -299,18 +296,18 @@ fn get_length_with_extension_internal() { } test_layouts!(debug_output_supports_pretty_print, debug_output_supports_pretty_print_internal); -fn debug_output_supports_pretty_print_internal() +fn debug_output_supports_pretty_print_internal>() where T::Location: std::fmt::Debug, { let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"B".to_vec()]; - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for x in &d { t.insert(x, x).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); @@ -384,12 +381,12 @@ test_layouts!( test_lookup_with_corrupt_data_returns_decoder_error, test_lookup_with_corrupt_data_returns_decoder_error_internal ); -fn test_lookup_with_corrupt_data_returns_decoder_error_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn test_lookup_with_corrupt_data_returns_decoder_error_internal>() { + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(b"A", b"ABC").unwrap(); t.insert(b"B", b"ABCBA").unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); @@ -401,7 +398,7 @@ fn test_lookup_with_corrupt_data_returns_decoder_error_internal() } test_layouts!(test_recorder, test_recorder_internal); -fn test_recorder_internal() { +fn test_recorder_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -409,12 +406,12 @@ fn test_recorder_internal() { (b"B".to_vec(), vec![4; 64]), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&memdb).build(); for (key, value) in &key_value { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = memdb.commit(t.commit()); let mut recorder = Recorder::::new(); { @@ -441,7 +438,7 @@ fn test_recorder_internal() { } test_layouts!(test_recorder_with_cache, test_recorder_with_cache_internal); -fn test_recorder_with_cache_internal() { +fn test_recorder_with_cache_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -449,13 +446,13 @@ fn test_recorder_with_cache_internal() { (b"B".to_vec(), vec![4; 64]), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for (key, value) in &key_value { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let mut cache = TestTrieCache::::default(); @@ -526,7 +523,7 @@ fn test_recorder_with_cache_internal() { } test_layouts!(test_recorder_with_cache_get_hash, test_recorder_with_cache_get_hash_internal); -fn test_recorder_with_cache_get_hash_internal() { +fn test_recorder_with_cache_get_hash_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -534,13 +531,13 @@ fn test_recorder_with_cache_get_hash_internal() { (b"B".to_vec(), vec![4; 64]), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for (key, value) in &key_value { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let mut cache = TestTrieCache::::default(); @@ -630,8 +627,8 @@ fn test_recorder_with_cache_get_hash_internal() { } test_layouts!(test_merkle_value, test_merkle_value_internal); -fn test_merkle_value_internal() { - let mut memdb = MemoryDB::, DBValue>::default(); +fn test_merkle_value_internal>() { + let mut memdb = DB::default(); // Data set. let key_value = vec![ @@ -647,7 +644,7 @@ fn test_merkle_value_internal() { for (key, value) in &key_value { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = memdb.commit(t.commit()); // Ensure we can fetch the merkle values for all present keys. let trie = TrieDBBuilder::::new(&memdb, &root).build(); @@ -695,8 +692,8 @@ fn test_merkle_value_internal() { } test_layouts!(test_merkle_value_single_key, test_merkle_value_single_key_internal); -fn test_merkle_value_single_key_internal() { - let mut memdb = MemoryDB::, DBValue>::default(); +fn test_merkle_value_single_key_internal>() { + let mut memdb = DB::default(); // Data set. let key_value = vec![(b"AAA".to_vec(), vec![1; 64])]; @@ -704,7 +701,7 @@ fn test_merkle_value_single_key_internal() { for (key, value) in &key_value { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = memdb.commit(t.commit()); let trie = TrieDBBuilder::::new(&memdb, &root).build(); @@ -720,8 +717,8 @@ fn test_merkle_value_single_key_internal() { } test_layouts!(test_merkle_value_branches, test_merkle_value_branches_internal); -fn test_merkle_value_branches_internal() { - let mut memdb = MemoryDB::, DBValue>::default(); +fn test_merkle_value_branches_internal>() { + let mut memdb = DB::default(); // Data set. let key_value = vec![(b"AAAA".to_vec(), vec![1; 64]), (b"AABA".to_vec(), vec![2; 64])]; @@ -729,7 +726,7 @@ fn test_merkle_value_branches_internal() { for (key, value) in &key_value { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = memdb.commit(t.commit()); let trie = TrieDBBuilder::::new(&memdb, &root).build(); @@ -743,13 +740,13 @@ fn test_merkle_value_branches_internal() { } test_layouts!(test_merkle_value_empty_trie, test_merkle_value_empty_trie_internal); -fn test_merkle_value_empty_trie_internal() { - let mut memdb = MemoryDB::, DBValue>::default(); +fn test_merkle_value_empty_trie_internal>() { + let mut memdb = DB::default(); // Valid state root. let mut t = TrieDBMutBuilder::::new(&memdb).build(); t.insert(&[], &[]).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = memdb.commit(t.commit()); // Data set is empty. let trie = TrieDBBuilder::::new(&memdb, &root).build(); @@ -771,15 +768,15 @@ fn test_merkle_value_empty_trie_internal() { } test_layouts!(test_merkle_value_modification, test_merkle_value_modification_internal); -fn test_merkle_value_modification_internal() { - let mut memdb = MemoryDB::, DBValue>::default(); +fn test_merkle_value_modification_internal>() { + let mut memdb = DB::default(); let key_value = vec![(b"AAAA".to_vec(), vec![1; 64]), (b"AABA".to_vec(), vec![2; 64])]; let mut t = TrieDBMutBuilder::::new(&memdb).build(); for (key, value) in &key_value { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = memdb.commit(t.commit()); let (a_hash_lhs, aaaa_hash_lhs, aaba_hash_lhs) = { let trie = TrieDBBuilder::::new(&memdb, &root).build(); @@ -799,7 +796,7 @@ fn test_merkle_value_modification_internal() { // Modify AABA and expect AAAA to return the same merkle value. let mut t = TrieDBMutBuilder::::from_existing(&memdb, root).build(); t.insert(b"AABA", &vec![3; 64]).unwrap(); -let root = t.commit().apply_to(&mut memdb); + let root = memdb.commit(t.commit()); let (a_hash_rhs, aaaa_hash_rhs, aaba_hash_rhs) = { let trie = TrieDBBuilder::::new(&memdb, &root).build(); @@ -824,16 +821,16 @@ let root = t.commit().apply_to(&mut memdb); } test_layouts!(iterator_seek_with_recorder, iterator_seek_with_recorder_internal); -fn iterator_seek_with_recorder_internal() { +fn iterator_seek_with_recorder_internal>() { let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"B".to_vec()]; let vals = vec![vec![0; 64], vec![1; 64], vec![2; 64], vec![3; 64]]; - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for (k, val) in d.iter().zip(vals.iter()) { t.insert(k, val.as_slice()).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let mut recorder = Recorder::::new(); { @@ -859,7 +856,7 @@ fn iterator_seek_with_recorder_internal() { } test_layouts!(test_cache, test_cache_internal); -fn test_cache_internal() { +fn test_cache_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -868,7 +865,7 @@ fn test_cache_internal() { (b"BC".to_vec(), vec![4; 64]), ]; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = DB::default(); let mut cache = TestTrieCache::::default(); let changeset = { @@ -878,7 +875,7 @@ fn test_cache_internal() { } t.commit() }; - let root = changeset.apply_to(&mut memdb); + let root = memdb.commit(changeset); let t = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); for (key, _) in &key_value { t.get(key).unwrap(); @@ -904,7 +901,7 @@ fn test_cache_internal() { for (key, value) in &key_value { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = memdb.commit(t.commit()); assert_eq!( cache.lookup_value_for_key(&b"AB"[..]).unwrap().data().flatten().unwrap(), @@ -935,7 +932,7 @@ fn test_cache_internal() { #[test] fn test_record_value() { - type L = HashedValueNoExtThreshold<33>; + type L = HashedValueNoExtThreshold<33, ()>; // one root branch and two leaf, one with inline value, the other with node value. let key_value = vec![(b"A".to_vec(), vec![1; 32]), (b"B".to_vec(), vec![1; 33])]; @@ -1075,7 +1072,7 @@ fn test_record_value() { } test_layouts!(test_trie_nodes_recorded, test_trie_nodes_recorded_internal); -fn test_trie_nodes_recorded_internal() { +fn test_trie_nodes_recorded_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -1085,13 +1082,13 @@ fn test_trie_nodes_recorded_internal() { ]; const NON_EXISTENT_KEY: &[u8] = &*b"NOT"; - let mut memdb = MemoryDB::, DBValue>::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&memdb).build(); for (key, value) in &key_value { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = memdb.commit(t.commit()); for mut cache in [Some(TestTrieCache::::default()), None] { for get_hash in [true, false] { diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 371236dc..d36e7103 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -21,7 +21,7 @@ use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ reference_trie_root, test_layouts, ExtensionLayout, HashedValueNoExt, HashedValueNoExtThreshold, NoExtensionLayout, RefHasher, ReferenceNodeCodec, - ReferenceNodeCodecNoExt, TestTrieCache, + ReferenceNodeCodecNoExt, TestTrieCache, PrefixedMemoryDB, }; use trie_db::{ CachedValue, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, @@ -29,8 +29,8 @@ use trie_db::{ }; use trie_standardmap::*; -type PrefixedMemoryDB = - MemoryDB<::Hash, PrefixedKey<::Hash>, DBValue>; +use crate::{TestDB, TestCommit}; + type MemoryDBProof = MemoryDB<::Hash, HashKey<::Hash>, DBValue>; @@ -71,7 +71,7 @@ fn reference_hashed_null_node() -> ::Out { #[test] fn playpen() { env_logger::init(); - playpen_internal::>(); + playpen_internal::>(); playpen_internal::(); playpen_internal::(); playpen_internal::(); @@ -131,16 +131,16 @@ fn playpen_internal() { } test_layouts!(init, init_internal); -fn init_internal() { - let memdb = PrefixedMemoryDB::::default(); +fn init_internal>() { + let memdb = DB::default(); let t = TrieDBMutBuilder::::new(&memdb).build(); let hashed_null_node = reference_hashed_null_node::(); assert_eq!(t.commit().root_hash(), hashed_null_node); } test_layouts!(insert_on_empty, insert_on_empty_internal); -fn insert_on_empty_internal() { - let memdb = PrefixedMemoryDB::::default(); +fn insert_on_empty_internal>() { + let memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!( @@ -150,10 +150,10 @@ fn insert_on_empty_internal() { } test_layouts!(remove_to_empty, remove_to_empty_internal); -fn remove_to_empty_internal() { +fn remove_to_empty_internal>() { let big_value = b"00000000000000000000000000000000"; - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01], big_value).unwrap(); @@ -162,37 +162,37 @@ fn remove_to_empty_internal() { t.remove(&[0x01]).unwrap(); t.remove(&[0x01, 0x23]).unwrap(); t.remove(&[0x01, 0x34]).unwrap(); - t.commit().apply_to(&mut memdb); - assert_eq!(memdb.keys().len(), 0); + t.commit().commit_to(&mut memdb); + assert!(memdb.is_empty()); } test_layouts!(remove_to_empty_checked, remove_to_empty_checked_internal); -fn remove_to_empty_checked_internal() { +fn remove_to_empty_checked_internal>() { let big_value = b"00000000000000000000000000000000"; - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01], big_value).unwrap(); t.insert(&[0x01, 0x23], big_value).unwrap(); t.insert(&[0x01, 0x34], big_value).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); assert_eq!(t.get(&[0x01]).unwrap(), Some(big_value.to_vec()),); assert_eq!(t.get(&[0x01, 0x34]).unwrap(), Some(big_value.to_vec()),); t.remove(&[0x01]).unwrap(); t.remove(&[0x01, 0x23]).unwrap(); t.remove(&[0x01, 0x34]).unwrap(); - t.commit().apply_to(&mut memdb); - assert_eq!(memdb.keys().len(), 0); + t.commit().commit_to(&mut memdb); + assert!(memdb.is_empty()); } test_layouts!(remove_to_empty_no_extension, remove_to_empty_no_extension_internal); -fn remove_to_empty_no_extension_internal() { +fn remove_to_empty_no_extension_internal>() { let big_value = b"00000000000000000000000000000000"; let big_value2 = b"00000000000000000000000000000002"; let big_value3 = b"00000000000000000000000000000004"; - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01, 0x23], big_value3).unwrap(); @@ -200,7 +200,7 @@ fn remove_to_empty_no_extension_internal() { t.insert(&[0x01, 0x34], big_value).unwrap(); t.remove(&[0x01]).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); assert_eq!( &root, &reference_trie::calc_root::(vec![ @@ -211,12 +211,12 @@ fn remove_to_empty_no_extension_internal() { } test_layouts!(insert_replace_root, insert_replace_root_internal); -fn insert_replace_root_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn insert_replace_root_internal>() { + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); assert_eq!( root, reference_trie_root::(vec![(vec![0x01u8, 0x23], vec![0x23u8, 0x45])]), @@ -224,12 +224,12 @@ fn insert_replace_root_internal() { } test_layouts!(insert_make_branch_root, insert_make_branch_root_internal); -fn insert_make_branch_root_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn insert_make_branch_root_internal>() { + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); assert_eq!( root, reference_trie_root::(vec![ @@ -240,13 +240,13 @@ fn insert_make_branch_root_internal() { } test_layouts!(insert_into_branch_root, insert_into_branch_root_internal); -fn insert_into_branch_root_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn insert_into_branch_root_internal>() { + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); assert_eq!( root, reference_trie_root::(vec![ @@ -258,12 +258,12 @@ fn insert_into_branch_root_internal() { } test_layouts!(insert_value_into_branch_root, insert_value_into_branch_root_internal); -fn insert_value_into_branch_root_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn insert_value_into_branch_root_internal>() { + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[], &[0x0]).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); assert_eq!( root, reference_trie_root::(vec![ @@ -274,12 +274,12 @@ fn insert_value_into_branch_root_internal() { } test_layouts!(insert_split_leaf, insert_split_leaf_internal); -fn insert_split_leaf_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn insert_split_leaf_internal>() { + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); assert_eq!( root, reference_trie_root::(vec![ @@ -290,13 +290,13 @@ fn insert_split_leaf_internal() { } test_layouts!(insert_split_extenstion, insert_split_extenstion_internal); -fn insert_split_extenstion_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn insert_split_extenstion_internal>() { + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); assert_eq!( root, reference_trie_root::(vec![ @@ -308,15 +308,15 @@ fn insert_split_extenstion_internal() { } test_layouts!(insert_big_value, insert_big_value_internal); -fn insert_big_value_internal() { +fn insert_big_value_internal>() { let big_value0 = b"00000000000000000000000000000000"; let big_value1 = b"11111111111111111111111111111111"; - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], big_value0).unwrap(); t.insert(&[0x11u8, 0x23], big_value1).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); assert_eq!( root, reference_trie_root::(vec![ @@ -327,14 +327,14 @@ fn insert_big_value_internal() { } test_layouts!(insert_duplicate_value, insert_duplicate_value_internal); -fn insert_duplicate_value_internal() { +fn insert_duplicate_value_internal>() { let big_value = b"00000000000000000000000000000000"; - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], big_value).unwrap(); t.insert(&[0x11u8, 0x23], big_value).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); assert_eq!( root, reference_trie_root::(vec![ @@ -345,23 +345,23 @@ fn insert_duplicate_value_internal() { } test_layouts!(test_at_empty, test_at_empty_internal); -fn test_at_empty_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn test_at_empty_internal>() { + let mut memdb = DB::default(); let t = TrieDBMutBuilder::::new(&mut memdb).build(); assert_eq!(t.get(&[0x5]).unwrap(), None); } test_layouts!(test_at_one_and_two, test_at_one_and_two_internal); -fn test_at_one_and_two_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn test_at_one_and_two_internal>() { + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x24]).unwrap(); - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x25]).unwrap(); // This test that middle node get resolved correctly (modified @@ -370,8 +370,8 @@ fn test_at_one_and_two_internal() { } test_layouts!(test_at_three, test_at_three_internal); -fn test_at_three_internal() { - let mut memdb = PrefixedMemoryDB::::default(); +fn test_at_three_internal>() { + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); @@ -380,7 +380,7 @@ fn test_at_three_internal() { assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), vec![0xf1u8, 0x23]); assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), vec![0x81u8, 0x23]); assert_eq!(t.get(&[0x82, 0x23]).unwrap(), None); - let root = t.commit().apply_to(&mut memdb); + let root = memdb.commit(t.commit()); let t = TrieDBMutBuilder::::from_existing(&memdb, root).build(); assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), vec![0xf1u8, 0x23]); @@ -398,7 +398,7 @@ fn test_nibbled_branch_changed_value() { } test_layouts!(stress, stress_internal); -fn stress_internal() { +fn stress_internal>() { let mut seed = Default::default(); for _ in 0..1000 { let x = StandardMap { @@ -411,14 +411,14 @@ fn stress_internal() { .make_with(&mut seed); let real = reference_trie_root::(x.clone()); - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let memtrie = populate_trie::(&mut memdb, &x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); - let mut memdb2 = PrefixedMemoryDB::::default(); + let mut memdb2 = DB::default(); let memtrie_sorted = populate_trie::(&mut memdb2, &y); - let root = memtrie.commit().apply_to(&mut memdb); - let root2 = memtrie_sorted.commit().apply_to(&mut memdb2); + let root = memtrie.commit().commit_to(&mut memdb); + let root2 = memtrie_sorted.commit().commit_to(&mut memdb2); if root != real || root2 != real { println!("TRIE MISMATCH"); println!(); @@ -437,16 +437,16 @@ fn stress_internal() { } test_layouts!(test_trie_existing, test_trie_existing_internal); -fn test_trie_existing_internal() { - let mut db = PrefixedMemoryDB::::default(); - let mut t = TrieDBMutBuilder::::new(&mut db).build(); +fn test_trie_existing_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - let root = t.commit().apply_to(&mut db); - let _ = TrieDBMutBuilder::::from_existing(&db, root); + let root = t.commit().commit_to(&mut memdb); + let _ = TrieDBMutBuilder::::from_existing(&memdb, root); } test_layouts!(insert_empty, insert_empty_internal); -fn insert_empty_internal() { +fn insert_empty_internal>() { let mut seed = Default::default(); let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), @@ -457,12 +457,13 @@ fn insert_empty_internal() { } .make_with(&mut seed); - let mut db = PrefixedMemoryDB::::default(); + let mut db = DB::default(); let mut t = TrieDBMutBuilder::::new(&db).build(); for &(ref key, ref value) in &x { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut db); + let root = db.commit(t.commit()); + assert_eq!(root, reference_trie_root::(x.clone())); @@ -471,14 +472,14 @@ fn insert_empty_internal() { t.insert(key, &[]).unwrap(); } assert!(t.is_empty()); - let root = t.commit().apply_to(&mut db); + let root = db.commit(t.commit()); let hashed_null_node = reference_hashed_null_node::(); assert_eq!(root, hashed_null_node); } test_layouts!(return_old_values, return_old_values_internal); -fn return_old_values_internal() { +fn return_old_values_internal>() { let threshold = T::MAX_INLINE_VALUE; let mut seed = Default::default(); let x = StandardMap { @@ -490,7 +491,7 @@ fn return_old_values_internal() { } .make_with(&mut seed); - let mut db = PrefixedMemoryDB::::default(); + let mut db = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut db).build(); for &(ref key, ref value) in &x { assert!(t.insert(key, value).unwrap() == None); @@ -534,7 +535,7 @@ fn register_proof_without_value() { use reference_trie::HashedValueNoExtThreshold; use std::{cell::RefCell, collections::HashMap}; - type Layout = HashedValueNoExtThreshold<1>; + type Layout = HashedValueNoExtThreshold<1, ()>; type MemoryDB = memory_db::MemoryDB, DBValue>; let x = [ (b"test1".to_vec(), vec![1; 32]), // inline @@ -636,7 +637,7 @@ fn register_proof_without_value() { } test_layouts!(test_recorder, test_recorder_internal); -fn test_recorder_internal() { +fn test_recorder_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -645,12 +646,12 @@ fn test_recorder_internal() { ]; // Add some initial data to the trie - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for (key, value) in key_value.iter().take(1) { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); // Add more data, but this time only to the overlay. // While doing that we record all trie accesses to replay this operation. @@ -687,7 +688,7 @@ fn test_recorder_internal() { } test_layouts!(test_recorder_witch_cache, test_recorder_with_cache_internal); -fn test_recorder_with_cache_internal() { +fn test_recorder_with_cache_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -696,12 +697,12 @@ fn test_recorder_with_cache_internal() { ]; // Add some initial data to the trie - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for (key, value) in key_value.iter().take(1) { t.insert(key, value).unwrap(); } - let root = t.commit().apply_to(&mut memdb); + let root = t.commit().commit_to(&mut memdb); let mut validated_root = root; let mut cache = TestTrieCache::::default(); @@ -728,7 +729,7 @@ fn test_recorder_with_cache_internal() { for (key, value) in key_value.iter().skip(1) { trie.insert(key, value).unwrap(); } - let new_root = trie.commit().apply_to(&mut overlay); + let new_root = trie.commit().commit_to(&mut overlay); let t = TrieDBBuilder::::new(&overlay, &new_root).with_cache(&mut cache).build(); for (key, _) in key_value.iter().skip(1) { @@ -761,7 +762,7 @@ fn test_recorder_with_cache_internal() { } test_layouts!(test_insert_remove_data_with_cache, test_insert_remove_data_with_cache_internal); -fn test_insert_remove_data_with_cache_internal() { +fn test_insert_remove_data_with_cache_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -773,7 +774,7 @@ fn test_insert_remove_data_with_cache_internal() { let mut cache = TestTrieCache::::default(); let mut recorder = Recorder::::new(); - let mut memdb = PrefixedMemoryDB::::default(); + let mut memdb = DB::default(); let mut trie = TrieDBMutBuilder::::new(&mut memdb) .with_recorder(&mut recorder) .with_cache(&mut cache) @@ -815,8 +816,8 @@ fn test_insert_remove_data_with_cache_internal() { #[test] fn test_two_assets_memory_db() { - test_two_assets_memory_db_inner_1::>(); - test_two_assets_memory_db_inner_2::>(); + test_two_assets_memory_db_inner_1::>(); + test_two_assets_memory_db_inner_2::>(); } fn test_two_assets_memory_db_inner_1() { let memdb = PrefixedMemoryDB::::new(&[0u8]); From d00240e9ff4893439c8ca91032e0f1df11e9056a Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 26 Nov 2023 19:05:20 +0100 Subject: [PATCH 10/90] fmt --- test-support/mem-tree-db/src/lib.rs | 211 +++++++++--------- test-support/reference-trie/src/lib.rs | 17 +- .../reference-trie/src/substrate_like.rs | 4 +- trie-db/src/lookup.rs | 20 +- trie-db/src/node.rs | 11 +- trie-db/src/triedb.rs | 4 +- trie-db/src/triedbmut.rs | 2 +- trie-db/test/src/iterator.rs | 3 +- trie-db/test/src/lib.rs | 82 +++---- trie-db/test/src/proof.rs | 5 +- trie-db/test/src/trie_codec.rs | 6 +- trie-db/test/src/triedb.rs | 11 +- trie-db/test/src/triedbmut.rs | 7 +- 13 files changed, 199 insertions(+), 184 deletions(-) diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs index 3221b491..8bf3bbf8 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/test-support/mem-tree-db/src/lib.rs @@ -36,12 +36,7 @@ where #[derive(Clone)] enum NodeEntry { - Live { - key: H, - data: Vec, - children: Vec, - rc: u32, - }, + Live { key: H, data: Vec, children: Vec, rc: u32 }, Removed, } @@ -88,10 +83,10 @@ where } } - fn remove_node(&mut self, location: usize) -> bool{ + fn remove_node(&mut self, location: usize) -> bool { let entry = self.nodes.get_mut(location).unwrap(); match entry { - NodeEntry::Live { rc, children, .. } => { + NodeEntry::Live { rc, children, .. } => if *rc == 1 { let children = std::mem::take(children); *entry = NodeEntry::Removed; @@ -102,11 +97,10 @@ where } else { *rc -= 1; false - } - } + }, NodeEntry::Removed => { panic!("Accessing removed node"); - } + }, } } @@ -158,18 +152,24 @@ impl HashDB, Location> for MemTreeDB where H: Hasher, { - fn get(&self, k: &H::Out, _prefix: Prefix, location: Location) -> Option<(Vec, Vec)> { + fn get( + &self, + k: &H::Out, + _prefix: Prefix, + location: Location, + ) -> Option<(Vec, Vec)> { if k == &self.hashed_null_node { return Some((self.null_node_data.clone(), Default::default())) } - + let location = match location { Some(l) => l, - None => if let Some(l) = self.roots.get(k) { - *l - } else { - return None - } + None => + if let Some(l) = self.roots.get(k) { + *l + } else { + return None + }, }; match self.nodes.get(location) { Some(NodeEntry::Live { data, children, key, .. }) => { @@ -195,103 +195,100 @@ where #[cfg(test)] mod tests { -use super::{MemTreeDB, Location, NodeEntry}; -use keccak_hasher::{KeccakHash, KeccakHasher}; -use trie_db::{Changeset, NewChangesetNode, ChangesetNodeRef, ExistingChangesetNode}; -use hash_db::{Hasher, HashDB}; + use super::{Location, MemTreeDB, NodeEntry}; + use hash_db::{HashDB, Hasher}; + use keccak_hasher::{KeccakHash, KeccakHasher}; + use trie_db::{Changeset, ChangesetNodeRef, ExistingChangesetNode, NewChangesetNode}; -fn hash(i: u32) -> KeccakHash { - KeccakHasher::hash(&i.to_le_bytes()) -} + fn hash(i: u32) -> KeccakHash { + KeccakHasher::hash(&i.to_le_bytes()) + } -#[test] -fn test_apply_existing_node() { - let mut db = MemTreeDB::::default(); - - // First, apply a new node - let new_node = ChangesetNodeRef::New(NewChangesetNode { - hash: hash(1), - prefix: Default::default(), - data: vec![1, 2, 3], - children: vec![], - }); - let new_location = db.apply(&new_node); - - // Then, apply an existing node that refers to the new node - let existing_node = ChangesetNodeRef::Existing(ExistingChangesetNode { - hash: hash(1), - location: Some(new_location), - prefix: Default::default(), - }); - let existing_location = db.apply(&existing_node); - - assert_eq!(existing_location, new_location); -} + #[test] + fn test_apply_existing_node() { + let mut db = MemTreeDB::::default(); -#[test] -fn test_apply_new_node() { - let mut db = MemTreeDB::::default(); - let node = ChangesetNodeRef::New(NewChangesetNode { - hash: KeccakHash::default(), - prefix: Default::default(), - data: vec![1, 2, 3], - children: vec![], - }); - let location = db.apply(&node); - assert_eq!(location, db.nodes.len() - 1); -} + // First, apply a new node + let new_node = ChangesetNodeRef::New(NewChangesetNode { + hash: hash(1), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + }); + let new_location = db.apply(&new_node); + + // Then, apply an existing node that refers to the new node + let existing_node = ChangesetNodeRef::Existing(ExistingChangesetNode { + hash: hash(1), + location: Some(new_location), + prefix: Default::default(), + }); + let existing_location = db.apply(&existing_node); + + assert_eq!(existing_location, new_location); + } -#[test] -fn test_apply_commit() { - let mut db = MemTreeDB::::default(); - let commit = Changeset:: { - root: ChangesetNodeRef::New(NewChangesetNode { + #[test] + fn test_apply_new_node() { + let mut db = MemTreeDB::::default(); + let node = ChangesetNodeRef::New(NewChangesetNode { hash: KeccakHash::default(), prefix: Default::default(), data: vec![1, 2, 3], children: vec![], - }), - removed: Default::default(), - }; - db.apply_commit(commit); - assert_eq!(db.roots.len(), 1); -} -#[test] -fn test_commit_changeset_with_children() { - let mut db = MemTreeDB::::default(); - - // Create two child nodes - let child1 = ChangesetNodeRef::New(NewChangesetNode { - hash: hash(1), - prefix: Default::default(), - data: vec![1, 2, 3], - children: vec![], - }); - let child2 = ChangesetNodeRef::New(NewChangesetNode { - hash: hash(2), - prefix: Default::default(), - data: vec![4, 5, 6], - children: vec![], - }); - - // Create a root node that refers to the child nodes - let root = ChangesetNodeRef::New(NewChangesetNode { - hash: hash(0), - prefix: Default::default(), - data: vec![7, 8, 9], - children: vec![child1, child2], - }); - - let commit = Changeset:: { - root, - removed: Default::default(), - }; - db.apply_commit(commit); - - // Check that the root node and child nodes are in the database - assert_eq!(db.nodes.len(), 3); - assert_eq!(db.roots.len(), 1); -} + }); + let location = db.apply(&node); + assert_eq!(location, db.nodes.len() - 1); + } + + #[test] + fn test_apply_commit() { + let mut db = MemTreeDB::::default(); + let commit = Changeset:: { + root: ChangesetNodeRef::New(NewChangesetNode { + hash: KeccakHash::default(), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + }), + removed: Default::default(), + }; + db.apply_commit(commit); + assert_eq!(db.roots.len(), 1); + } + #[test] + fn test_commit_changeset_with_children() { + let mut db = MemTreeDB::::default(); + + // Create two child nodes + let child1 = ChangesetNodeRef::New(NewChangesetNode { + hash: hash(1), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + }); + let child2 = ChangesetNodeRef::New(NewChangesetNode { + hash: hash(2), + prefix: Default::default(), + data: vec![4, 5, 6], + children: vec![], + }); + + // Create a root node that refers to the child nodes + let root = ChangesetNodeRef::New(NewChangesetNode { + hash: hash(0), + prefix: Default::default(), + data: vec![7, 8, 9], + children: vec![child1, child2], + }); + + let commit = Changeset:: { root, removed: Default::default() }; + db.apply_commit(commit); + + // Check that the root node and child nodes are in the database + assert_eq!(db.nodes.len(), 3); + assert_eq!(db.roots.len(), 1); + } #[test] fn test_get() { let mut db = MemTreeDB::::default(); diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index c058746c..613ce216 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -40,10 +40,9 @@ pub use substrate_like::{ NodeCodec as ReferenceNodeCodecNoExtMeta, ReferenceTrieStreamNoExt, }; +pub use mem_tree_db::{Location as MemLocation, MemTreeDB}; pub use paste::paste; pub use substrate::{LayoutV0 as SubstrateV0, LayoutV1 as SubstrateV1}; -pub use mem_tree_db::MemTreeDB; -pub use mem_tree_db::Location as MemLocation; /// Reference hasher is a keccak hasher. pub type RefHasher = keccak_hasher::KeccakHasher; @@ -67,11 +66,16 @@ macro_rules! test_layouts { //$test_internal::<$crate::ExtensionLayout, $crate::PrefixedMemoryDB<$crate::ExtensionLayout>>(); //eprintln!("Running with layout `HashedValueNoExtThreshold` and MemTreeDB"); - //$test_internal::<$crate::HashedValueNoExtThreshold<1, $crate::MemLocation>, $crate::MemTreeDB<$crate::RefHasher>>(); + //$test_internal::<$crate::HashedValueNoExtThreshold<1, $crate::MemLocation>, + //$test_internal::<$crate::HashedValueNoExtThreshold<1, $crate::MemTreeDB<$crate::RefHasher>>(); + //$test_internal::<$crate::HashedValueNoExtThreshold<1, eprintln!("Running with layout `HashedValueNoExt` and MemTreeDB"); $test_internal::<$crate::HashedValueNoExt, $crate::MemTreeDB<$crate::RefHasher>>(); eprintln!("Running with layout `NoExtensionLayout` and MemTreeDB"); - $test_internal::<$crate::GenericNoExtensionLayout<$crate::RefHasher, $crate::MemLocation>, $crate::MemTreeDB<$crate::RefHasher>>(); + $test_internal::< + $crate::GenericNoExtensionLayout<$crate::RefHasher, $crate::MemLocation>, + $crate::MemTreeDB<$crate::RefHasher>, + >(); } }; } @@ -157,7 +161,10 @@ impl TrieLayout for AllowEmptyLayout { type Location = (); } -impl TrieConfiguration for GenericNoExtensionLayout {} +impl TrieConfiguration + for GenericNoExtensionLayout +{ +} /// Trie layout without extension nodes. pub type NoExtensionLayout = GenericNoExtensionLayout; diff --git a/test-support/reference-trie/src/substrate_like.rs b/test-support/reference-trie/src/substrate_like.rs index 104d1618..a6290737 100644 --- a/test-support/reference-trie/src/substrate_like.rs +++ b/test-support/reference-trie/src/substrate_like.rs @@ -34,7 +34,9 @@ impl TrieLayout for HashedValueNoExt { type Location = mem_tree_db::Location; } -impl TrieLayout for HashedValueNoExtThreshold { +impl TrieLayout + for HashedValueNoExtThreshold +{ const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = false; const MAX_INLINE_VALUE: Option = Some(C); diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs index 91d72430..cb6f4869 100644 --- a/trie-db/src/lookup.rs +++ b/trie-db/src/lookup.rs @@ -178,14 +178,15 @@ where // Get the owned node representation from the database. let mut get_owned_node = |depth: i32| { - let (data, locations) = match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { - Some(value) => value, - None => - return Err(Box::new(match depth { - 0 => TrieError::InvalidStateRoot(hash), - _ => TrieError::IncompleteDatabase(hash), - })), - }; + let (data, locations) = + match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { + Some(value) => value, + None => + return Err(Box::new(match depth { + 0 => TrieError::InvalidStateRoot(hash), + _ => TrieError::IncompleteDatabase(hash), + })), + }; let decoded = match L::Codec::decode(&data[..], &locations) { Ok(node) => node, @@ -198,7 +199,8 @@ where }; let mut node = if let Some(cache) = &mut cache { - let node = cache.get_or_insert_node(hash, location, &mut || get_owned_node(depth))?; + let node = + cache.get_or_insert_node(hash, location, &mut || get_owned_node(depth))?; self.record(|| TrieAccess::NodeOwned { hash, node_owned: node }); diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index b47d978d..7c35795f 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -596,10 +596,7 @@ impl NodePlan { child_slices[i] = Some(child.build(data, location)); } } - Node::Branch( - child_slices, - value, - ) + Node::Branch(child_slices, value) }, NodePlan::NibbledBranch { partial, value, children } => { let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; @@ -626,11 +623,7 @@ impl NodePlan { child_slices[i] = Some(child.build(data, location)); } } - Node::NibbledBranch( - partial.build(data), - child_slices, - value, - ) + Node::NibbledBranch(partial.build(data), child_slices, value) }, } } diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index 6f3f8081..d721a586 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -273,7 +273,9 @@ where query: |_: &[u8]| (), hash: *self.root, cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), - recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), + recorder: recorder + .as_mut() + .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } .lookup_first_descendant(key, NibbleSlice::new(key), Default::default()) } diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 841afccd..50797cee 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -173,7 +173,7 @@ impl Value { } else if let Value::Node(hash, location) = self { f(NodeToEncode::TrieNode(NodeHandle::Hash(*hash, *location)), partial, None); } - let value = match &*self { + let value = match &*self { Value::Inline(value) => EncodedValue::Inline(&value), Value::Node(hash, location) => EncodedValue::Node(hash.as_ref(), *location), Value::NewNode(Some(hash), _value) => diff --git a/trie-db/test/src/iterator.rs b/trie-db/test/src/iterator.rs index b0ded21f..7f4c52e4 100644 --- a/trie-db/test/src/iterator.rs +++ b/trie-db/test/src/iterator.rs @@ -17,8 +17,7 @@ use hex_literal::hex; use reference_trie::test_layouts; use trie_db::{ node::{Node, Value}, - NibbleSlice, NibbleVec, TrieDBBuilder, TrieDBNodeIterator, TrieError, TrieIterator, - TrieLayout, + NibbleSlice, NibbleVec, TrieDBBuilder, TrieDBNodeIterator, TrieError, TrieIterator, TrieLayout, }; use crate::TestDB; diff --git a/trie-db/test/src/lib.rs b/trie-db/test/src/lib.rs index d02cdf8b..e3d08c5d 100644 --- a/trie-db/test/src/lib.rs +++ b/trie-db/test/src/lib.rs @@ -32,61 +32,67 @@ mod triedb; mod triedbmut; use hash_db::{Hasher, Prefix}; -use trie_db::{DBValue, TrieLayout, TrieHash, Changeset}; use mem_tree_db::MemTreeDB; +use trie_db::{Changeset, DBValue, TrieHash, TrieLayout}; trait TestDB: hash_db::HashDB + Clone + Default { - fn commit(&mut self, commit: trie_db::Changeset<<::Hash as Hasher>::Out, T::Location>) -> TrieHash; - fn remove(&mut self, hash: &::Out, prefix: Prefix); - fn is_empty(&self) -> bool; + fn commit( + &mut self, + commit: trie_db::Changeset<<::Hash as Hasher>::Out, T::Location>, + ) -> TrieHash; + fn remove(&mut self, hash: &::Out, prefix: Prefix); + fn is_empty(&self) -> bool; } -impl, H, KF> TestDB for MemoryDB - where - H: Hasher, - KF: memory_db::KeyFunction + Send + Sync, +impl, H, KF> TestDB for MemoryDB +where + H: Hasher, + KF: memory_db::KeyFunction + Send + Sync, { - fn commit(&mut self, commit: trie_db::Changeset::Location>) -> H::Out { - commit.apply_to(self) - } + fn commit( + &mut self, + commit: trie_db::Changeset::Location>, + ) -> H::Out { + commit.apply_to(self) + } - fn remove(&mut self, hash: &::Out, prefix: Prefix) { - MemoryDB::remove(self, hash, prefix); - } + fn remove(&mut self, hash: &::Out, prefix: Prefix) { + MemoryDB::remove(self, hash, prefix); + } - fn is_empty(&self) -> bool { - self.keys().is_empty() - } + fn is_empty(&self) -> bool { + self.keys().is_empty() + } } impl, H> TestDB for MemTreeDB - where - H: Hasher + Clone, +where + H: Hasher + Clone, { - fn commit(&mut self, commit: trie_db::Changeset) -> H::Out { - let root = commit.root_hash(); - self.apply_commit(commit); - root - } + fn commit(&mut self, commit: trie_db::Changeset) -> H::Out { + let root = commit.root_hash(); + self.apply_commit(commit); + root + } - fn remove(&mut self, hash: &H::Out, _prefix: Prefix) { - MemTreeDB::remove(self, hash); - } + fn remove(&mut self, hash: &H::Out, _prefix: Prefix) { + MemTreeDB::remove(self, hash); + } - fn is_empty(&self) -> bool { - MemTreeDB::is_empty(self) - } + fn is_empty(&self) -> bool { + MemTreeDB::is_empty(self) + } } trait TestCommit { - fn commit_to>(self, db: &mut DB) -> TrieHash; + fn commit_to>(self, db: &mut DB) -> TrieHash; } -impl> TestCommit for Changeset -where -T::Hash: Hasher, +impl> TestCommit for Changeset +where + T::Hash: Hasher, { - fn commit_to>(self, db: &mut DB) -> TrieHash { - db.commit(self) - } -} \ No newline at end of file + fn commit_to>(self, db: &mut DB) -> TrieHash { + db.commit(self) + } +} diff --git a/trie-db/test/src/proof.rs b/trie-db/test/src/proof.rs index 7db308cb..e1ee342d 100644 --- a/trie-db/test/src/proof.rs +++ b/trie-db/test/src/proof.rs @@ -149,7 +149,10 @@ fn test_verify_extraneaous_value_internal>() { #[test] fn test_verify_extraneous_hash_reference() { - let (root, proof, _) = test_generate_proof::>(test_entries(), vec![b"do"]); + let (root, proof, _) = test_generate_proof::>( + test_entries(), + vec![b"do"], + ); let items = vec![(&b"alfa"[..], Some(&[0; 32][..])), (&b"do"[..], Some(&b"verb"[..]))]; match verify_proof::(&root, &proof, items.iter()) { diff --git a/trie-db/test/src/trie_codec.rs b/trie-db/test/src/trie_codec.rs index 3c13b40f..77ffcaa8 100644 --- a/trie-db/test/src/trie_codec.rs +++ b/trie-db/test/src/trie_codec.rs @@ -124,8 +124,10 @@ test_layouts!( trie_decoding_fails_with_incomplete_database_internal ); fn trie_decoding_fails_with_incomplete_database_internal>() { - let (_, encoded, _) = - test_encode_compact::(vec![(b"alfa", &[0; 32]), (b"bravo", b"bravo")], vec![b"alfa"]); + let (_, encoded, _) = test_encode_compact::( + vec![(b"alfa", &[0; 32]), (b"bravo", b"bravo")], + vec![b"alfa"], + ); assert!(encoded.len() > 1); diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index 334f9836..944332ef 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -18,14 +18,15 @@ use hash_db::{Hasher, EMPTY_PREFIX}; use hex_literal::hex; use memory_db::{HashKey, MemoryDB}; use reference_trie::{ - test_layouts, test_layouts_substrate, HashedValueNoExtThreshold, TestTrieCache, PrefixedMemoryDB, + test_layouts, test_layouts_substrate, HashedValueNoExtThreshold, PrefixedMemoryDB, + TestTrieCache, }; use trie_db::{ encode_compact, CachedValue, DBValue, Lookup, NibbleSlice, RecordedForKey, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieRecorder, }; -use crate::{TestDB, TestCommit}; +use crate::{TestCommit, TestDB}; type MemoryDBProof = MemoryDB<::Hash, HashKey<::Hash>, DBValue>; @@ -164,8 +165,10 @@ fn iterator_seek_internal>() { assert_eq!(&vals[5..], &iter.map(|x| x.unwrap().1).collect::>()[..]); } -fn trie_from_hex_keys>(keys: &[&str], callback: impl FnOnce(&mut trie_db::TrieDB)) -where +fn trie_from_hex_keys>( + keys: &[&str], + callback: impl FnOnce(&mut trie_db::TrieDB), +) where T: TrieLayout, { let mut memdb = DB::default(); diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index d36e7103..99d2c572 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -20,8 +20,8 @@ use log::debug; use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ reference_trie_root, test_layouts, ExtensionLayout, HashedValueNoExt, - HashedValueNoExtThreshold, NoExtensionLayout, RefHasher, ReferenceNodeCodec, - ReferenceNodeCodecNoExt, TestTrieCache, PrefixedMemoryDB, + HashedValueNoExtThreshold, NoExtensionLayout, PrefixedMemoryDB, RefHasher, ReferenceNodeCodec, + ReferenceNodeCodecNoExt, TestTrieCache, }; use trie_db::{ CachedValue, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, @@ -29,7 +29,7 @@ use trie_db::{ }; use trie_standardmap::*; -use crate::{TestDB, TestCommit}; +use crate::{TestCommit, TestDB}; type MemoryDBProof = MemoryDB<::Hash, HashKey<::Hash>, DBValue>; @@ -464,7 +464,6 @@ fn insert_empty_internal>() { } let root = db.commit(t.commit()); - assert_eq!(root, reference_trie_root::(x.clone())); let mut t = TrieDBMutBuilder::::from_existing(&db, root).build(); From 81446abc3470d24376e3d4a585750317d32f8915 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 27 Nov 2023 11:59:39 +0100 Subject: [PATCH 11/90] Test fixes --- test-support/mem-tree-db/src/lib.rs | 19 +++++++++---- test-support/reference-trie/src/lib.rs | 39 +++++++++++++++++--------- trie-db/src/node.rs | 8 +++--- trie-db/test/src/lib.rs | 2 +- 4 files changed, 45 insertions(+), 23 deletions(-) diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs index 8bf3bbf8..c83e3210 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/test-support/mem-tree-db/src/lib.rs @@ -73,17 +73,26 @@ where self.roots.clear(); } - pub fn remove(&mut self, key: &H::Out) { + pub fn remove_root(&mut self, key: &H::Out) { let Some(location) = self.roots.get(key) else { return; }; - if self.remove_node(*location) { + if self.remove_tree(*location) { self.roots.remove(key); } } - fn remove_node(&mut self, location: usize) -> bool { + pub fn remove_node(&mut self, k: &H::Out) { + self.roots.remove(k); + for node in self.nodes.iter_mut() { + if matches!(node, NodeEntry::Live { key, .. } if key == k) { + *node = NodeEntry::Removed; + } + } + } + + fn remove_tree(&mut self, location: usize) -> bool { let entry = self.nodes.get_mut(location).unwrap(); match entry { NodeEntry::Live { rc, children, .. } => @@ -91,7 +100,7 @@ where let children = std::mem::take(children); *entry = NodeEntry::Removed; for c in children { - self.remove_node(c); + self.remove_tree(c); } true } else { @@ -143,7 +152,7 @@ where self.roots.insert(*key, root); } for (k, _) in commit.removed { - self.remove(&k); + self.remove_root(&k); } } } diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 613ce216..ff012378 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -56,19 +56,32 @@ macro_rules! test_layouts { ($test:ident, $test_internal:ident) => { #[test] fn $test() { - //eprintln!("Running with layout `HashedValueNoExtThreshold` and MemoryDB"); - //$test_internal::<$crate::HashedValueNoExtThreshold<1, ()>, $crate::PrefixedMemoryDB<$crate::HashedValueNoExtThreshold<1, ()>>>(); - //eprintln!("Running with layout `HashedValueNoExt` and MemoryDB"); - //$test_internal::<$crate::HashedValueNoExt, $crate::PrefixedMemoryDB<$crate::HashedValueNoExt>>(); - //eprintln!("Running with layout `NoExtensionLayout` and MemoryDB"); - //$test_internal::<$crate::NoExtensionLayout, $crate::PrefixedMemoryDB<$crate::NoExtensionLayout>>(); - //eprintln!("Running with layout `ExtensionLayout` and MemoryDB"); - //$test_internal::<$crate::ExtensionLayout, $crate::PrefixedMemoryDB<$crate::ExtensionLayout>>(); - - //eprintln!("Running with layout `HashedValueNoExtThreshold` and MemTreeDB"); - //$test_internal::<$crate::HashedValueNoExtThreshold<1, $crate::MemLocation>, - //$test_internal::<$crate::HashedValueNoExtThreshold<1, $crate::MemTreeDB<$crate::RefHasher>>(); - //$test_internal::<$crate::HashedValueNoExtThreshold<1, + eprintln!("Running with layout `HashedValueNoExtThreshold` and MemoryDB"); + $test_internal::< + $crate::HashedValueNoExtThreshold<1, ()>, + $crate::PrefixedMemoryDB<$crate::HashedValueNoExtThreshold<1, ()>>, + >(); + eprintln!("Running with layout `HashedValueNoExt` and MemoryDB"); + $test_internal::< + $crate::HashedValueNoExt, + $crate::PrefixedMemoryDB<$crate::HashedValueNoExt>, + >(); + eprintln!("Running with layout `NoExtensionLayout` and MemoryDB"); + $test_internal::< + $crate::NoExtensionLayout, + $crate::PrefixedMemoryDB<$crate::NoExtensionLayout>, + >(); + eprintln!("Running with layout `ExtensionLayout` and MemoryDB"); + $test_internal::< + $crate::ExtensionLayout, + $crate::PrefixedMemoryDB<$crate::ExtensionLayout>, + >(); + + eprintln!("Running with layout `HashedValueNoExtThreshold` and MemTreeDB"); + $test_internal::< + $crate::HashedValueNoExtThreshold<1, $crate::MemLocation>, + $crate::MemTreeDB<$crate::RefHasher>, + >(); eprintln!("Running with layout `HashedValueNoExt` and MemTreeDB"); $test_internal::<$crate::HashedValueNoExt, $crate::MemTreeDB<$crate::RefHasher>>(); eprintln!("Running with layout `NoExtensionLayout` and MemTreeDB"); diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index 7c35795f..27de0a9f 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -576,10 +576,10 @@ impl NodePlan { let mut nc = 0; let value = if let Some(v) = value { if v.is_inline() { - Some(v.build(data, locations.first().copied().unwrap_or_default())) + Some(v.build(data, Default::default())) } else { nc += 1; - Some(v.build(data, Default::default())) + Some(v.build(data, locations.first().copied().unwrap_or_default())) } } else { None @@ -603,10 +603,10 @@ impl NodePlan { let mut nc = 0; let value = if let Some(v) = value { if v.is_inline() { - Some(v.build(data, locations.first().copied().unwrap_or_default())) + Some(v.build(data, Default::default())) } else { nc += 1; - Some(v.build(data, Default::default())) + Some(v.build(data, locations.first().copied().unwrap_or_default())) } } else { None diff --git a/trie-db/test/src/lib.rs b/trie-db/test/src/lib.rs index e3d08c5d..51fa1f94 100644 --- a/trie-db/test/src/lib.rs +++ b/trie-db/test/src/lib.rs @@ -76,7 +76,7 @@ where } fn remove(&mut self, hash: &H::Out, _prefix: Prefix) { - MemTreeDB::remove(self, hash); + MemTreeDB::remove_node(self, hash); } fn is_empty(&self) -> bool { From 599b0f1a03ac07ab0dab4c24f3d766fc6b623a55 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 15 Jan 2024 10:20:43 +0100 Subject: [PATCH 12/90] api for child changeset --- trie-db/src/triedbmut.rs | 89 +++++++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 50797cee..286aeabc 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -32,6 +32,8 @@ use hash_db::{HashDB, Hasher, Prefix}; #[cfg(feature = "std")] use std::collections::HashSet as Set; +#[cfg(not(feature = "std"))] +use alloc::boxed::Box; #[cfg(not(feature = "std"))] use alloc::collections::btree_set::BTreeSet as Set; @@ -69,6 +71,8 @@ fn empty_children() -> Box<[Option>; nibble_ops::NIBBLE_L ]) } +pub type ChildChangeset = Box, ::Location>>; + /// Type alias to indicate the nible covers a full key, /// therefore its left side is a full prefix. type NibbleFullKey<'key> = NibbleSlice<'key>; @@ -557,8 +561,10 @@ impl InsertAction { enum Stored { // A new node. New(Node), + //New(Node, Option>), // A cached node, loaded from the DB. Cached(Node, TrieHash, L::Location), + //Cached(Node, TrieHash, L::Location, Option>), } /// Used to build a collection of child nodes from a collection of `NodeHandle`s @@ -1078,6 +1084,7 @@ where key: &mut NibbleFullKey, value: Bytes, old_val: &mut Option>, + child_changes: Option>, ) -> Result<(StorageHandle, bool), TrieHash, CError> { let h = match handle { NodeHandle::InMemory(h) => h, @@ -1087,7 +1094,8 @@ where let stored = self.storage.destroy(h); let (new_stored, changed) = self .inspect(stored, key, move |trie, stored, key| { - trie.insert_inspector(stored, key, value, old_val).map(|a| a.into_action()) + trie.insert_inspector(stored, key, value, old_val, child_changes) + .map(|a| a.into_action()) })? .expect("Insertion never deletes."); @@ -1120,6 +1128,7 @@ where key: &mut NibbleFullKey, value: Bytes, old_val: &mut Option>, + child_changes: Option>, ) -> Result, TrieHash, CError> { let partial = *key; @@ -1155,7 +1164,8 @@ where key.advance(1); if let Some(child) = children[idx].take() { // Original had something there. recurse down into it. - let (new_child, changed) = self.insert_at(child, key, value, old_val)?; + let (new_child, changed) = + self.insert_at(child, key, value, old_val, child_changes)?; children[idx] = Some(new_child.into()); if !changed { // The new node we composed didn't change. @@ -1241,7 +1251,8 @@ where key.advance(common + 1); if let Some(child) = children[idx].take() { // Original had something there. recurse down into it. - let (new_child, changed) = self.insert_at(child, key, value, old_val)?; + let (new_child, changed) = + self.insert_at(child, key, value, old_val, child_changes)?; children[idx] = Some(new_child.into()); if !changed { // The new node we composed didn't change. @@ -1318,8 +1329,9 @@ where // always replace because whatever we get out here // is not the branch we started with. - let branch_action = - self.insert_inspector(branch, key, value, old_val)?.unwrap_node(); + let branch_action = self + .insert_inspector(branch, key, value, old_val, child_changes)? + .unwrap_node(); InsertAction::Replace(branch_action) } else if !L::USE_EXTENSION { #[cfg(feature = "std")] @@ -1333,7 +1345,9 @@ where Some(stored_value), ); // augment the new branch. - let branch = self.insert_inspector(branch, key, value, old_val)?.unwrap_node(); + let branch = self + .insert_inspector(branch, key, value, old_val, child_changes)? + .unwrap_node(); InsertAction::Replace(branch) } else if common == existing_key.len() { @@ -1346,7 +1360,9 @@ where let branch = Node::Branch(empty_children(), Some(stored_value)); // augment the new branch. key.advance(common); - let branch = self.insert_inspector(branch, key, value, old_val)?.unwrap_node(); + let branch = self + .insert_inspector(branch, key, value, old_val, child_changes)? + .unwrap_node(); // always replace since we took a leaf and made an extension. let leaf = self.storage.alloc(Stored::New(branch)); @@ -1370,8 +1386,9 @@ where // augment it. this will result in the Leaf -> common == 0 routine, // which creates a branch. key.advance(common); - let augmented_low = - self.insert_inspector(low, key, value, old_val)?.unwrap_node(); + let augmented_low = self + .insert_inspector(low, key, value, old_val, child_changes)? + .unwrap_node(); // make an extension using it. this is a replacement. InsertAction::Replace(Node::Extension( existing_key.to_stored_range(common), @@ -1411,7 +1428,13 @@ where // continue inserting. let branch_action = self - .insert_inspector(Node::Branch(children, None), key, value, old_val)? + .insert_inspector( + Node::Branch(children, None), + key, + value, + old_val, + child_changes, + )? .unwrap_node(); InsertAction::Replace(branch_action) } else if common == existing_key.len() { @@ -1422,7 +1445,8 @@ where // insert into the child node. key.advance(common); - let (new_child, changed) = self.insert_at(child_branch, key, value, old_val)?; + let (new_child, changed) = + self.insert_at(child_branch, key, value, old_val, child_changes)?; let new_ext = Node::Extension(existing_key.to_stored(), new_child.into()); @@ -1448,8 +1472,9 @@ where // augment the extension. this will take the common == 0 path, // creating a branch. key.advance(common); - let augmented_low = - self.insert_inspector(low, key, value, old_val)?.unwrap_node(); + let augmented_low = self + .insert_inspector(low, key, value, old_val, child_changes)? + .unwrap_node(); // always replace, since this extension is not the one we started with. // this is known because the partial key is only the common prefix. @@ -1468,6 +1493,7 @@ where handle: NodeHandle, L::Location>, key: &mut NibbleFullKey, old_val: &mut Option>, + child_changes: Option>, ) -> Result, TrieHash, CError> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), @@ -1478,7 +1504,7 @@ where }; let opt = self.inspect(stored, key, move |trie, node, key| { - trie.remove_inspector(node, key, old_val) + trie.remove_inspector(node, key, old_val, child_changes) })?; Ok(opt.map(|(new, changed)| (self.storage.alloc(new), changed))) @@ -1490,6 +1516,7 @@ where node: Node, key: &mut NibbleFullKey, old_val: &mut Option>, + child_changes: Option>, ) -> Result, TrieHash, CError> { let partial = *key; Ok(match (node, partial.is_empty()) { @@ -1518,7 +1545,7 @@ where ); let prefix = *key; key.advance(1); - match self.remove_at(child, key, old_val)? { + match self.remove_at(child, key, old_val, child_changes)? { Some((new, changed)) => { children[idx] = Some(new.into()); let branch = Node::Branch(children, value); @@ -1574,7 +1601,7 @@ where ); let prefix = *key; key.advance(common + 1); - match self.remove_at(child, key, old_val)? { + match self.remove_at(child, key, old_val, child_changes)? { Some((new, changed)) => { children[idx] = Some(new.into()); let branch = Node::NibbledBranch(encoded, children, value); @@ -1639,7 +1666,7 @@ where trace!(target: "trie", "removing from extension child, partial={:?}", partial); let prefix = *key; key.advance(common); - match self.remove_at(child_branch, key, old_val)? { + match self.remove_at(child_branch, key, old_val, child_changes)? { Some((new_child, changed)) => { // if the child branch was unchanged, then the extension is too. // otherwise, this extension may need fixing. @@ -2160,6 +2187,15 @@ where &mut self, key: &[u8], value: &[u8], + ) -> Result>, TrieHash, CError> { + self.insert_with_child_changes(key, value, None) + } + + pub fn insert_with_child_changes( + &mut self, + key: &[u8], + value: &[u8], + child_changes: Option>, ) -> Result>, TrieHash, CError> { if !L::ALLOW_EMPTY && value.is_empty() { return self.remove(key) @@ -2172,8 +2208,13 @@ where let value = Bytes::from(value); let root_handle = self.root_handle(); - let (new_handle, _changed) = - self.insert_at(root_handle, &mut NibbleSlice::new(key), value, &mut old_val)?; + let (new_handle, _changed) = self.insert_at( + root_handle, + &mut NibbleSlice::new(key), + value, + &mut old_val, + child_changes, + )?; #[cfg(feature = "std")] trace!(target: "trie", "insert: altered trie={}", _changed); @@ -2183,6 +2224,14 @@ where } pub fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { + self.remove_with_child_changes(key, None) + } + + pub fn remove_with_child_changes( + &mut self, + key: &[u8], + child_changes: Option>, + ) -> Result>, TrieHash, CError> { #[cfg(feature = "std")] trace!(target: "trie", "remove: key={:?}", ToHex(key)); @@ -2190,7 +2239,7 @@ where let mut key_slice = NibbleSlice::new(key); let mut old_val = None; - match self.remove_at(root_handle, &mut key_slice, &mut old_val)? { + match self.remove_at(root_handle, &mut key_slice, &mut old_val, child_changes)? { Some((handle, _changed)) => { #[cfg(feature = "std")] trace!(target: "trie", "remove: altered trie={}", _changed); From 314179deae3050f00757887298b1e4f01da87bce Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 15 Jan 2024 10:24:06 +0100 Subject: [PATCH 13/90] attach in triedbmut node --- trie-db/src/triedbmut.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 286aeabc..d968c578 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -223,7 +223,7 @@ enum Node { /// A leaf node contains the end of a key and a value. /// This key is encoded from a `NibbleSlice`, meaning it contains /// a flag indicating it is a leaf. - Leaf(NodeKey, Value), + Leaf(NodeKey, Value, Option>), /// An extension contains a shared portion of a key and a child node. /// The shared portion is encoded from a `NibbleSlice` meaning it contains /// a flag indicating it is an extension. @@ -233,12 +233,14 @@ enum Node { Branch( Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, Option>, + Option>, ), /// Branch node with support for a nibble (to avoid extension node). NibbledBranch( NodeKey, Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, Option>, + Option>, ), } @@ -561,10 +563,8 @@ impl InsertAction { enum Stored { // A new node. New(Node), - //New(Node, Option>), // A cached node, loaded from the DB. Cached(Node, TrieHash, L::Location), - //Cached(Node, TrieHash, L::Location, Option>), } /// Used to build a collection of child nodes from a collection of `NodeHandle`s From 269ef62da0715c6ff22284d342e72fddf79a92ad Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 15 Jan 2024 12:09:12 +0100 Subject: [PATCH 14/90] push changeset (warn need to init key individually for compat) --- trie-db/src/triedbmut.rs | 320 +++++++++++++++++++++++++-------------- 1 file changed, 208 insertions(+), 112 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index d968c578..64b70974 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -276,14 +276,12 @@ where L::Location: Debug, { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - match *self { + match self { Self::Empty => write!(fmt, "Empty"), - Self::Leaf((ref a, ref b), ref c) => - write!(fmt, "Leaf({:?}, {:?})", (a, ToHex(&*b)), c), - Self::Extension((ref a, ref b), ref c) => - write!(fmt, "Extension({:?}, {:?})", (a, ToHex(&*b)), c), - Self::Branch(ref a, ref b) => write!(fmt, "Branch({:?}, {:?}", a, b), - Self::NibbledBranch((ref a, ref b), ref c, ref d) => + Self::Leaf((a, b), c, _) => write!(fmt, "Leaf({:?}, {:?})", (a, ToHex(&*b)), c), + Self::Extension((a, b), c) => write!(fmt, "Extension({:?}, {:?})", (a, ToHex(&*b)), c), + Self::Branch(a, b, _) => write!(fmt, "Branch({:?}, {:?}", a, b), + Self::NibbledBranch((a, b), c, d, _) => write!(fmt, "NibbledBranch({:?}, {:?}, {:?})", (a, ToHex(&*b)), c, d), } } @@ -335,7 +333,7 @@ impl Node { .map_err(|e| Box::new(TrieError::DecoderError(node_hash, e)))?; let node = match encoded_node { EncodedNode::Empty => Node::Empty, - EncodedNode::Leaf(k, v) => Node::Leaf(k.into(), v.into()), + EncodedNode::Leaf(k, v) => Node::Leaf(k.into(), v.into(), None), EncodedNode::Extension(key, cb) => Node::Extension(key.into(), Self::inline_or_hash(node_hash, cb, storage)?), EncodedNode::Branch(encoded_children, val) => { @@ -363,7 +361,7 @@ impl Node { child(15)?, ]); - Node::Branch(children, val.map(Into::into)) + Node::Branch(children, val.map(Into::into), None) }, EncodedNode::NibbledBranch(k, encoded_children, val) => { let mut child = |i: usize| match encoded_children[i] { @@ -390,7 +388,7 @@ impl Node { child(15)?, ]); - Node::NibbledBranch(k.into(), children, val.map(Into::into)) + Node::NibbledBranch(k.into(), children, val.map(Into::into), None) }, }; Ok(node) @@ -403,7 +401,7 @@ impl Node { ) -> Self { match node_owned { NodeOwned::Empty => Node::Empty, - NodeOwned::Leaf(k, v) => Node::Leaf(k.into(), v.into()), + NodeOwned::Leaf(k, v) => Node::Leaf(k.into(), v.into(), None), NodeOwned::Extension(key, cb) => Node::Extension(key.into(), Self::inline_or_hash_owned(cb, storage)), NodeOwned::Branch(encoded_children, val) => { @@ -432,7 +430,7 @@ impl Node { child(15), ]); - Node::Branch(children, val.as_ref().map(Into::into)) + Node::Branch(children, val.as_ref().map(Into::into), None) }, NodeOwned::NibbledBranch(k, encoded_children, val) => { let mut child = |i: usize| { @@ -460,7 +458,7 @@ impl Node { child(15), ]); - Node::NibbledBranch(k.into(), children, val.as_ref().map(Into::into)) + Node::NibbledBranch(k.into(), children, val.as_ref().map(Into::into), None) }, NodeOwned::Value(_, _) => unreachable!("`NodeOwned::Value` can only be returned for the hash of a value."), @@ -470,7 +468,7 @@ impl Node { // TODO: parallelize /// Here `child_cb` should process the first parameter to either insert an external /// node value or to encode and add a new branch child node. - fn into_encoded(self, mut child_cb: F) -> Vec + fn into_encoded(self, mut child_cb: F) -> (Vec, Option>) where F: FnMut( NodeToEncode, L::Location>, @@ -479,46 +477,60 @@ impl Node { ) -> ChildReference, L::Location>, { match self { - Node::Empty => L::Codec::empty_node().to_vec(), - Node::Leaf(partial, mut value) => { + Node::Empty => (L::Codec::empty_node().to_vec(), None), + Node::Leaf(partial, mut value, child_changes) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let value = value.into_encoded::(Some(&pr), &mut child_cb); - L::Codec::leaf_node(pr.right_iter(), pr.len(), value) + (L::Codec::leaf_node(pr.right_iter(), pr.len(), value), child_changes) }, Node::Extension(partial, child) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let it = pr.right_iter(); let c = child_cb(NodeToEncode::TrieNode(child), Some(&pr), None); - L::Codec::extension_node(it, pr.len(), c) + (L::Codec::extension_node(it, pr.len(), c), None) }, - Node::Branch(mut children, mut value) => { + Node::Branch(mut children, mut value, child_changes) => { let value = value.as_mut().map(|v| v.into_encoded::(None, &mut child_cb)); - L::Codec::branch_node( - // map the `NodeHandle`s from the Branch to `ChildReferences` - children.iter_mut().map(Option::take).enumerate().map(|(i, maybe_child)| { - maybe_child.map(|child| { - child_cb(NodeToEncode::TrieNode(child), None, Some(i as u8)) - }) - }), - value, + ( + L::Codec::branch_node( + // map the `NodeHandle`s from the Branch to `ChildReferences` + children.iter_mut().map(Option::take).enumerate().map( + |(i, maybe_child)| { + maybe_child.map(|child| { + child_cb(NodeToEncode::TrieNode(child), None, Some(i as u8)) + }) + }, + ), + value, + ), + child_changes, ) }, - Node::NibbledBranch(partial, mut children, mut value) => { + Node::NibbledBranch(partial, mut children, mut value, child_changes) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let value = value.as_mut().map(|v| v.into_encoded::(Some(&pr), &mut child_cb)); let it = pr.right_iter(); - L::Codec::branch_node_nibbled( - it, - pr.len(), - // map the `NodeHandle`s from the Branch to `ChildReferences` - children.iter_mut().map(Option::take).enumerate().map(|(i, maybe_child)| { - //let branch_index = [i as u8]; - maybe_child.map(|child| { - let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); - child_cb(NodeToEncode::TrieNode(child), Some(&pr), Some(i as u8)) - }) - }), - value, + ( + L::Codec::branch_node_nibbled( + it, + pr.len(), + // map the `NodeHandle`s from the Branch to `ChildReferences` + children.iter_mut().map(Option::take).enumerate().map( + |(i, maybe_child)| { + //let branch_index = [i as u8]; + maybe_child.map(|child| { + let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); + child_cb( + NodeToEncode::TrieNode(child), + Some(&pr), + Some(i as u8), + ) + }) + }, + ), + value, + ), + child_changes, ) }, } @@ -532,7 +544,7 @@ enum Action { // Restore the original node. This trusts that the node is actually the original. Restore(Node), // if it is a new node, just clears the storage. - Delete, + Delete(Option>), } // post-insert action. Same as action without delete @@ -721,6 +733,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { hash_count: 0, storage: NodeStorage::empty(), death_row: Default::default(), + death_row_child: Default::default(), root_handle, } } @@ -732,6 +745,8 @@ pub struct NewChangesetNode { pub prefix: OwnedPrefix, pub data: Vec, pub children: Vec>, + // Storing the key (only needed for old trie). + pub key_childset: Option>, } #[derive(Debug)] @@ -887,6 +902,7 @@ where root: TrieHash, root_handle: NodeHandle, L::Location>, death_row: Set<(TrieHash, OwnedPrefix)>, + death_row_child: Vec>, /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. hash_count: usize, @@ -964,7 +980,10 @@ where Stored::New(node) => match inspector(self, node, key)? { Action::Restore(node) => Some((Stored::New(node), false)), Action::Replace(node) => Some((Stored::New(node), true)), - Action::Delete => None, + Action::Delete(child_changes) => { + child_changes.map(|c| self.death_row_child.push(c)); + None + }, }, Stored::Cached(node, hash, location) => match inspector(self, node, key)? { Action::Restore(node) => Some((Stored::Cached(node, hash, location), false)), @@ -972,8 +991,9 @@ where self.death_row.insert((hash, current_key.left_owned())); Some((Stored::New(node), true)) }, - Action::Delete => { + Action::Delete(child_changes) => { self.death_row.insert((hash, current_key.left_owned())); + child_changes.map(|c| self.death_row_child.push(c)); None }, }, @@ -1008,7 +1028,7 @@ where }, NodeHandle::InMemory(handle) => match &self.storage[handle] { Node::Empty => return Ok(None), - Node::Leaf(slice, value) => + Node::Leaf(slice, value, _) => if NibbleSlice::from_stored(slice) == partial { return Ok(value.in_memory_fetched_value( prefix, @@ -1027,7 +1047,7 @@ where return Ok(None) } }, - Node::Branch(children, value) => + Node::Branch(children, value, _) => if partial.is_empty() { return Ok(if let Some(v) = value.as_ref() { v.in_memory_fetched_value( @@ -1046,7 +1066,7 @@ where None => return Ok(None), } }, - Node::NibbledBranch(slice, children, value) => { + Node::NibbledBranch(slice, children, value, _) => { let slice = NibbleSlice::from_stored(slice); if slice == partial { return Ok(if let Some(v) = value.as_ref() { @@ -1140,9 +1160,9 @@ where #[cfg(feature = "std")] trace!(target: "trie", "empty: COMPOSE"); let value = Value::new(value, L::MAX_INLINE_VALUE); - InsertAction::Replace(Node::Leaf(partial.to_stored(), value)) + InsertAction::Replace(Node::Leaf(partial.to_stored(), value, child_changes)) }, - Node::Branch(mut children, stored_value) => { + Node::Branch(mut children, stored_value, b_child_changes) => { debug_assert!(L::USE_EXTENSION); #[cfg(feature = "std")] trace!(target: "trie", "branch: ROUTE,AUGMENT"); @@ -1150,7 +1170,7 @@ where if partial.is_empty() { let value = Some(Value::new(value, L::MAX_INLINE_VALUE)); let unchanged = stored_value == value; - let branch = Node::Branch(children, value); + let branch = Node::Branch(children, value, child_changes); self.replace_old_value(old_val, stored_value, key.left()); @@ -1170,20 +1190,27 @@ where if !changed { // The new node we composed didn't change. // It means our branch is untouched too. - return Ok(InsertAction::Restore(Node::Branch(children, stored_value))) + return Ok(InsertAction::Restore(Node::Branch( + children, + stored_value, + b_child_changes, + ))) } } else { // Original had nothing there. compose a leaf. let value = Value::new(value, L::MAX_INLINE_VALUE); - let leaf = - self.storage.alloc(Stored::New(Node::Leaf(key.to_stored(), value))); + let leaf = self.storage.alloc(Stored::New(Node::Leaf( + key.to_stored(), + value, + child_changes, + ))); children[idx] = Some(leaf.into()); } - InsertAction::Replace(Node::Branch(children, stored_value)) + InsertAction::Replace(Node::Branch(children, stored_value, b_child_changes)) } }, - Node::NibbledBranch(encoded, mut children, stored_value) => { + Node::NibbledBranch(encoded, mut children, stored_value, b_child_changes) => { debug_assert!(!L::USE_EXTENSION); #[cfg(feature = "std")] trace!(target: "trie", "branch: ROUTE,AUGMENT"); @@ -1193,7 +1220,12 @@ where if common == existing_key.len() && common == partial.len() { let value = Some(Value::new(value, L::MAX_INLINE_VALUE)); let unchanged = stored_value == value; - let branch = Node::NibbledBranch(existing_key.to_stored(), children, value); + let branch = Node::NibbledBranch( + existing_key.to_stored(), + children, + value, + child_changes, + ); let mut key_val = key.clone(); key_val.advance(existing_key.len()); @@ -1216,7 +1248,12 @@ where common, ); let nbranch_partial = existing_key.mid(common + 1).to_stored(); - let low = Node::NibbledBranch(nbranch_partial, children, stored_value); + let low = Node::NibbledBranch( + nbranch_partial, + children, + stored_value, + b_child_changes, + ); let ix = existing_key.at(common); let mut children = empty_children(); let alloc_storage = self.storage.alloc(Stored::New(low)); @@ -1229,10 +1266,12 @@ where existing_key.to_stored_range(common), children, Some(value), + child_changes, )) } else { let ix = partial.at(common); - let stored_leaf = Node::Leaf(partial.mid(common + 1).to_stored(), value); + let stored_leaf = + Node::Leaf(partial.mid(common + 1).to_stored(), value, child_changes); let leaf = self.storage.alloc(Stored::New(stored_leaf)); @@ -1241,6 +1280,7 @@ where existing_key.to_stored_range(common), children, None, + None, )) } } else { @@ -1261,14 +1301,18 @@ where existing_key.to_stored(), children, stored_value, + b_child_changes, ); return Ok(InsertAction::Restore(n_branch)) } } else { // Original had nothing there. compose a leaf. let value = Value::new(value, L::MAX_INLINE_VALUE); - let leaf = - self.storage.alloc(Stored::New(Node::Leaf(key.to_stored(), value))); + let leaf = self.storage.alloc(Stored::New(Node::Leaf( + key.to_stored(), + value, + child_changes, + ))); children[idx] = Some(leaf.into()); } @@ -1276,10 +1320,11 @@ where existing_key.to_stored(), children, stored_value, + b_child_changes, )) } }, - Node::Leaf(encoded, stored_value) => { + Node::Leaf(encoded, stored_value, l_child_changes) => { let existing_key = NibbleSlice::from_stored(&encoded); let common = partial.common_prefix(&existing_key); if common == existing_key.len() && common == partial.len() { @@ -1293,9 +1338,9 @@ where self.replace_old_value(old_val, Some(stored_value), key_val.left()); if unchanged { // unchanged. restore - InsertAction::Restore(Node::Leaf(encoded.clone(), value)) + InsertAction::Restore(Node::Leaf(encoded.clone(), value, l_child_changes)) } else { - InsertAction::Replace(Node::Leaf(encoded.clone(), value)) + InsertAction::Replace(Node::Leaf(encoded.clone(), value, child_changes)) } } else if (L::USE_EXTENSION && common == 0) || (!L::USE_EXTENSION && common < existing_key.len()) @@ -1313,17 +1358,25 @@ where let mut children = empty_children(); let branch = if L::USE_EXTENSION && existing_key.is_empty() { // always replace since branch isn't leaf. - Node::Branch(children, Some(stored_value)) + Node::Branch(children, Some(stored_value), l_child_changes) } else { let idx = existing_key.at(common) as usize; - let new_leaf = - Node::Leaf(existing_key.mid(common + 1).to_stored(), stored_value); + let new_leaf = Node::Leaf( + existing_key.mid(common + 1).to_stored(), + stored_value, + l_child_changes, + ); children[idx] = Some(self.storage.alloc(Stored::New(new_leaf)).into()); if L::USE_EXTENSION { - Node::Branch(children, None) + Node::Branch(children, None, None) } else { - Node::NibbledBranch(partial.to_stored_range(common), children, None) + Node::NibbledBranch( + partial.to_stored_range(common), + children, + None, + None, + ) } }; @@ -1343,6 +1396,7 @@ where existing_key.to_stored(), empty_children(), Some(stored_value), + l_child_changes, ); // augment the new branch. let branch = self @@ -1357,7 +1411,8 @@ where // fully-shared prefix for an extension. // make a stub branch and an extension. - let branch = Node::Branch(empty_children(), Some(stored_value)); + let branch = + Node::Branch(empty_children(), Some(stored_value), l_child_changes); // augment the new branch. key.advance(common); let branch = self @@ -1381,7 +1436,11 @@ where // partially-shared prefix for an extension. // start by making a leaf. - let low = Node::Leaf(existing_key.mid(common).to_stored(), stored_value); + let low = Node::Leaf( + existing_key.mid(common).to_stored(), + stored_value, + l_child_changes, + ); // augment it. this will result in the Leaf -> common == 0 routine, // which creates a branch. @@ -1429,7 +1488,7 @@ where // continue inserting. let branch_action = self .insert_inspector( - Node::Branch(children, None), + Node::Branch(children, None, None), key, value, old_val, @@ -1520,21 +1579,24 @@ where ) -> Result, TrieHash, CError> { let partial = *key; Ok(match (node, partial.is_empty()) { - (Node::Empty, _) => Action::Delete, - (Node::Branch(c, None), true) => Action::Restore(Node::Branch(c, None)), - (Node::NibbledBranch(n, c, None), true) => - Action::Restore(Node::NibbledBranch(n, c, None)), - (Node::Branch(children, val), true) => { + (Node::Empty, _) => Action::Delete(None), + (Node::Branch(c, None, _), true) => + Action::Restore(Node::Branch(c, None, child_changes)), + (Node::NibbledBranch(n, c, None, _), true) => + Action::Restore(Node::NibbledBranch(n, c, None, child_changes)), + (Node::Branch(children, val, _), true) => { self.replace_old_value(old_val, val, key.left()); // always replace since we took the value out. - Action::Replace(self.fix(Node::Branch(children, None), *key)?) + Action::Replace(self.fix(Node::Branch(children, None, child_changes), *key)?) }, - (Node::NibbledBranch(n, children, val), true) => { + (Node::NibbledBranch(n, children, val, _), true) => { self.replace_old_value(old_val, val, key.left()); // always replace since we took the value out. - Action::Replace(self.fix(Node::NibbledBranch(n, children, None), *key)?) + Action::Replace( + self.fix(Node::NibbledBranch(n, children, None, child_changes), *key)?, + ) }, - (Node::Branch(mut children, value), false) => { + (Node::Branch(mut children, value, b_childset), false) => { let idx = partial.at(0) as usize; if let Some(child) = children[idx].take() { #[cfg(feature = "std")] @@ -1548,7 +1610,7 @@ where match self.remove_at(child, key, old_val, child_changes)? { Some((new, changed)) => { children[idx] = Some(new.into()); - let branch = Node::Branch(children, value); + let branch = Node::Branch(children, value, b_childset); match changed { // child was changed, so we were too. true => Action::Replace(branch), @@ -1561,15 +1623,17 @@ where // the node may need fixing. #[cfg(feature = "std")] trace!(target: "trie", "branch child deleted, partial={:?}", partial); - Action::Replace(self.fix(Node::Branch(children, value), prefix)?) + Action::Replace( + self.fix(Node::Branch(children, value, b_childset), prefix)?, + ) }, } } else { // no change needed. - Action::Restore(Node::Branch(children, value)) + Action::Restore(Node::Branch(children, value, b_childset)) } }, - (Node::NibbledBranch(encoded, mut children, value), false) => { + (Node::NibbledBranch(encoded, mut children, value, b_childset), false) => { let (common, existing_length) = { let existing_key = NibbleSlice::from_stored(&encoded); (existing_key.common_prefix(&partial), existing_key.len()) @@ -1580,14 +1644,15 @@ where let mut key_val = key.clone(); key_val.advance(existing_length); self.replace_old_value(old_val, Some(value), key_val.left()); - let f = self.fix(Node::NibbledBranch(encoded, children, None), *key); + let f = self + .fix(Node::NibbledBranch(encoded, children, None, child_changes), *key); Action::Replace(f?) } else { - Action::Restore(Node::NibbledBranch(encoded, children, None)) + Action::Restore(Node::NibbledBranch(encoded, children, None, child_changes)) } } else if common < existing_length { // partway through an extension -- nothing to do here. - Action::Restore(Node::NibbledBranch(encoded, children, value)) + Action::Restore(Node::NibbledBranch(encoded, children, value, b_childset)) } else { // common == existing_length && common < partial.len() : check children let idx = partial.at(common) as usize; @@ -1604,7 +1669,8 @@ where match self.remove_at(child, key, old_val, child_changes)? { Some((new, changed)) => { children[idx] = Some(new.into()); - let branch = Node::NibbledBranch(encoded, children, value); + let branch = + Node::NibbledBranch(encoded, children, value, b_childset); match changed { // child was changed, so we were too. true => Action::Replace(branch), @@ -1621,28 +1687,26 @@ where "branch child deleted, partial={:?}", partial, ); - Action::Replace( - self.fix( - Node::NibbledBranch(encoded, children, value), - prefix, - )?, - ) + Action::Replace(self.fix( + Node::NibbledBranch(encoded, children, value, b_childset), + prefix, + )?) }, } } else { // no change needed. - Action::Restore(Node::NibbledBranch(encoded, children, value)) + Action::Restore(Node::NibbledBranch(encoded, children, value, b_childset)) } } }, - (Node::Leaf(encoded, value), _) => { + (Node::Leaf(encoded, value, l_childset), _) => { let existing_key = NibbleSlice::from_stored(&encoded); if existing_key == partial { // this is the node we were looking for. Let's delete it. let mut key_val = key.clone(); key_val.advance(existing_key.len()); self.replace_old_value(old_val, Some(value), key_val.left()); - Action::Delete + Action::Delete(child_changes) } else { // leaf the node alone. #[cfg(feature = "std")] @@ -1652,7 +1716,7 @@ where partial, NibbleSlice::from_stored(&encoded), ); - Action::Restore(Node::Leaf(encoded, value)) + Action::Restore(Node::Leaf(encoded, value, l_childset)) } }, (Node::Extension(encoded, child_branch), _) => { @@ -1681,7 +1745,7 @@ where None => { // the whole branch got deleted. // that means that this extension is useless. - Action::Delete + Action::Delete(None) }, } } else { @@ -1708,7 +1772,7 @@ where recurse_extension: bool, ) -> Result, TrieHash, CError> { match node { - Node::Branch(mut children, value) => { + Node::Branch(mut children, value, b_childset) => { // if only a single value, transmute to leaf/extension and feed through fixed. #[cfg_attr(feature = "std", derive(Debug))] enum UsedIndex { @@ -1746,17 +1810,17 @@ where // make a leaf. #[cfg(feature = "std")] trace!(target: "trie", "fixing: branch -> leaf"); - Ok(Node::Leaf(NibbleSlice::new(&[]).to_stored(), value)) + Ok(Node::Leaf(NibbleSlice::new(&[]).to_stored(), value, b_childset)) }, (_, value) => { // all is well. #[cfg(feature = "std")] trace!(target: "trie", "fixing: restoring branch"); - Ok(Node::Branch(children, value)) + Ok(Node::Branch(children, value, b_childset)) }, } }, - Node::NibbledBranch(enc_nibble, mut children, value) => { + Node::NibbledBranch(enc_nibble, mut children, value, b_childset) => { // if only a single value, transmute to leaf/extension and feed through fixed. #[cfg_attr(feature = "std", derive(Debug))] enum UsedIndex { @@ -1816,24 +1880,30 @@ where node }, }; + b_childset.map(|c| self.death_row_child.push(c)); match child_node { - Node::Leaf(sub_partial, value) => { + Node::Leaf(sub_partial, value, l_childset) => { let mut enc_nibble = enc_nibble; combine_key( &mut enc_nibble, (nibble_ops::NIBBLE_PER_BYTE - 1, &[a][..]), ); combine_key(&mut enc_nibble, (sub_partial.0, &sub_partial.1[..])); - Ok(Node::Leaf(enc_nibble, value)) + Ok(Node::Leaf(enc_nibble, value, l_childset)) }, - Node::NibbledBranch(sub_partial, ch_children, ch_value) => { + Node::NibbledBranch(sub_partial, ch_children, ch_value, n_childset) => { let mut enc_nibble = enc_nibble; combine_key( &mut enc_nibble, (nibble_ops::NIBBLE_PER_BYTE - 1, &[a][..]), ); combine_key(&mut enc_nibble, (sub_partial.0, &sub_partial.1[..])); - Ok(Node::NibbledBranch(enc_nibble, ch_children, ch_value)) + Ok(Node::NibbledBranch( + enc_nibble, + ch_children, + ch_value, + n_childset, + )) }, _ => unreachable!(), } @@ -1842,13 +1912,13 @@ where // make a leaf. #[cfg(feature = "std")] trace!(target: "trie", "fixing: branch -> leaf"); - Ok(Node::Leaf(enc_nibble, value)) + Ok(Node::Leaf(enc_nibble, value, b_childset)) }, (_, value) => { // all is well. #[cfg(feature = "std")] trace!(target: "trie", "fixing: restoring branch"); - Ok(Node::NibbledBranch(enc_nibble, children, value)) + Ok(Node::NibbledBranch(enc_nibble, children, value, b_childset)) }, } }, @@ -1919,7 +1989,7 @@ where self.fix_inner(Node::Extension(partial, sub_child), key.into(), true) }, - Node::Leaf(sub_partial, value) => { + Node::Leaf(sub_partial, value, l_childset) => { // combine with node below. if let Some(hash) = maybe_hash { // delete the cached child since we are going to replace it. @@ -1935,7 +2005,7 @@ where "fixing: extension -> leaf. new_partial={:?}", partial, ); - Ok(Node::Leaf(partial, value)) + Ok(Node::Leaf(partial, value, l_childset)) }, child_node => { #[cfg(feature = "std")] @@ -1987,7 +2057,7 @@ where let mut k = NibbleVec::new(); let mut children = Vec::new(); - let encoded_root = node.into_encoded(|node, o_slice, o_index| { + let (encoded_root, root_childset) = node.into_encoded(|node, o_slice, o_index| { let mov = k.append_optional_slice_and_nibble(o_slice, o_index); match node { NodeToEncode::Node(value) => { @@ -1998,6 +2068,7 @@ where prefix: k.as_owned_prefix(), data: value.to_vec(), //TODO: avoid allocation children: Default::default(), + key_childset: None, })); k.drop_lasts(mov); @@ -2010,6 +2081,7 @@ where }, } }); + root_childset.map(|c| children.push(*c)); #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:?}", ToHex(&encoded_root[..])); @@ -2025,6 +2097,7 @@ where prefix: Default::default(), data: encoded_root, children, + key_childset: None, }), removed, } @@ -2094,7 +2167,7 @@ where }, Stored::New(node) => { let mut sub_children = Vec::new(); - let encoded = { + let (encoded, child_set) = { let commit_child = |node: NodeToEncode, L::Location>, o_slice: Option<&NibbleSlice>, o_index: Option| { @@ -2108,6 +2181,7 @@ where prefix: prefix.as_owned_prefix(), data: value.to_vec(), //TODO: avoid allocation children: Default::default(), + key_childset: None, }, )); @@ -2129,6 +2203,7 @@ where }; node.into_encoded(commit_child) }; + child_set.map(|c| children.push(*c)); if encoded.len() >= L::Hash::LENGTH { let hash = self.db.hash(&encoded); self.cache_node(hash); @@ -2137,10 +2212,12 @@ where prefix: prefix.as_owned_prefix(), data: encoded, children: sub_children, + key_childset: None, })); self.hash_count += 1; ChildReference::Hash(hash, Default::default()) } else { + debug_assert!(children.len() == 0); // it's a small value, so we cram it into a `TrieHash` // and tag with length let mut h = >::default(); @@ -2197,6 +2274,15 @@ where value: &[u8], child_changes: Option>, ) -> Result>, TrieHash, CError> { + // expect for the child changes to have a key. + debug_assert!(child_changes + .as_ref() + .map(|c| if let ChangesetNodeRef::New(set) = c.as_ref() { + set.key_childset.is_some() + } else { + true + }) + .unwrap_or(true)); if !L::ALLOW_EMPTY && value.is_empty() { return self.remove(key) } @@ -2232,6 +2318,16 @@ where key: &[u8], child_changes: Option>, ) -> Result>, TrieHash, CError> { + // expect for the child changes to have a key. + debug_assert!(child_changes + .as_ref() + .map(|c| if let ChangesetNodeRef::New(set) = c.as_ref() { + set.key_childset.is_some() + } else { + true + }) + .unwrap_or(true)); + #[cfg(feature = "std")] trace!(target: "trie", "remove: key={:?}", ToHex(key)); From 675ae01f6f19c485615c34a2b6fcd0b177560bcf Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 15 Jan 2024 12:12:39 +0100 Subject: [PATCH 15/90] fix test and assert --- test-support/mem-tree-db/src/lib.rs | 6 ++++++ trie-db/src/triedbmut.rs | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs index c83e3210..4b8064ba 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/test-support/mem-tree-db/src/lib.rs @@ -223,6 +223,7 @@ mod tests { prefix: Default::default(), data: vec![1, 2, 3], children: vec![], + key_childset: None, }); let new_location = db.apply(&new_node); @@ -245,6 +246,7 @@ mod tests { prefix: Default::default(), data: vec![1, 2, 3], children: vec![], + key_childset: None, }); let location = db.apply(&node); assert_eq!(location, db.nodes.len() - 1); @@ -259,6 +261,7 @@ mod tests { prefix: Default::default(), data: vec![1, 2, 3], children: vec![], + key_childset: None, }), removed: Default::default(), }; @@ -275,12 +278,14 @@ mod tests { prefix: Default::default(), data: vec![1, 2, 3], children: vec![], + key_childset: None, }); let child2 = ChangesetNodeRef::New(NewChangesetNode { hash: hash(2), prefix: Default::default(), data: vec![4, 5, 6], children: vec![], + key_childset: None, }); // Create a root node that refers to the child nodes @@ -289,6 +294,7 @@ mod tests { prefix: Default::default(), data: vec![7, 8, 9], children: vec![child1, child2], + key_childset: None, }); let commit = Changeset:: { root, removed: Default::default() }; diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 64b70974..2ae67930 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -2203,7 +2203,12 @@ where }; node.into_encoded(commit_child) }; - child_set.map(|c| children.push(*c)); + let child_set = child_set + .map(|c| { + children.push(*c); + true + }) + .unwrap_or(false); if encoded.len() >= L::Hash::LENGTH { let hash = self.db.hash(&encoded); self.cache_node(hash); @@ -2217,7 +2222,7 @@ where self.hash_count += 1; ChildReference::Hash(hash, Default::default()) } else { - debug_assert!(children.len() == 0); + debug_assert!(!child_set); // it's a small value, so we cram it into a `TrieHash` // and tag with length let mut h = >::default(); From 87d60c8fdb2d534eda44fc5292642a94b4cc46f0 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 16 Jan 2024 09:21:10 +0100 Subject: [PATCH 16/90] push attached changeset at last. --- trie-db/src/triedbmut.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 2ae67930..5f5e174e 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -2203,13 +2203,7 @@ where }; node.into_encoded(commit_child) }; - let child_set = child_set - .map(|c| { - children.push(*c); - true - }) - .unwrap_or(false); - if encoded.len() >= L::Hash::LENGTH { + let result = if encoded.len() >= L::Hash::LENGTH { let hash = self.db.hash(&encoded); self.cache_node(hash); children.push(ChangesetNodeRef::New(NewChangesetNode { @@ -2222,7 +2216,7 @@ where self.hash_count += 1; ChildReference::Hash(hash, Default::default()) } else { - debug_assert!(!child_set); + debug_assert!(child_set.is_some()); // it's a small value, so we cram it into a `TrieHash` // and tag with length let mut h = >::default(); @@ -2230,7 +2224,9 @@ where h.as_mut()[..len].copy_from_slice(&encoded[..len]); ChildReference::Inline(h, len) - } + }; + child_set.map(|c| children.push(*c)); + result }, } }, From e031dbf61054bf37b83e5b309b4269dfced5ab19 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 16 Jan 2024 09:58:11 +0100 Subject: [PATCH 17/90] init a test --- test-support/mem-tree-db/src/lib.rs | 2 ++ trie-db/src/triedbmut.rs | 1 - trie-db/test/src/lib.rs | 3 +++ trie-db/test/src/triedbmut.rs | 42 ++++++++++++++++++++++++++++- 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs index 4b8064ba..7f0b424d 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/test-support/mem-tree-db/src/lib.rs @@ -268,6 +268,7 @@ mod tests { db.apply_commit(commit); assert_eq!(db.roots.len(), 1); } + #[test] fn test_commit_changeset_with_children() { let mut db = MemTreeDB::::default(); @@ -304,6 +305,7 @@ mod tests { assert_eq!(db.nodes.len(), 3); assert_eq!(db.roots.len(), 1); } + #[test] fn test_get() { let mut db = MemTreeDB::::default(); diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 5f5e174e..3d1bd14d 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -2216,7 +2216,6 @@ where self.hash_count += 1; ChildReference::Hash(hash, Default::default()) } else { - debug_assert!(child_set.is_some()); // it's a small value, so we cram it into a `TrieHash` // and tag with length let mut h = >::default(); diff --git a/trie-db/test/src/lib.rs b/trie-db/test/src/lib.rs index 51fa1f94..24b70109 100644 --- a/trie-db/test/src/lib.rs +++ b/trie-db/test/src/lib.rs @@ -42,6 +42,7 @@ trait TestDB: hash_db::HashDB + Cl ) -> TrieHash; fn remove(&mut self, hash: &::Out, prefix: Prefix); fn is_empty(&self) -> bool; + fn support_location() -> bool { false } } impl, H, KF> TestDB for MemoryDB @@ -82,6 +83,8 @@ where fn is_empty(&self) -> bool { MemTreeDB::is_empty(self) } + + fn support_location() -> bool { true } } trait TestCommit { diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 99d2c572..0d245341 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -25,7 +25,7 @@ use reference_trie::{ }; use trie_db::{ CachedValue, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, - TrieDBMutBuilder, TrieError, TrieLayout, Value, + TrieDBMutBuilder, TrieError, TrieLayout, Value, TrieHash, }; use trie_standardmap::*; @@ -855,3 +855,43 @@ fn test_two_assets_memory_db_inner_2() { assert_eq!(state.get(key2.as_ref()).unwrap().unwrap(), data2); assert_eq!(state.get(key3.as_ref()).unwrap().unwrap(), data3); } + +test_layouts!(child_trie, child_trie_internal); +fn child_trie_internal>() { + use std::collections::BTreeMap; + // Running a tipical child trie scenario: + // different trie, child trie root written all + // at once with childset before parent tree commit. + // Direct copy if using ref counting and location in db. + // Pruning. + let mut seed = Default::default(); + let nb_child_trie = 10; + let nb_child_trie_removed = 2; + let nb_child_trie_copied = 5; + let nb_child_trie_copied_half_removed = 1; + let nb_child_trie_copied_removed = 1; + let support_location = DB::support_location(); + let mut memdb = DB::default(); + let mut child_tree: BTreeMap, TrieHash> = Default::default(); + let mut main_root: TrieHash = Default::default(); + for i in 0..10 + 1 { + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 3, + journal_key: 0, + value_mode: ValueMode::Index, + count: 20, + } + .make_with(&mut seed); + + let mut memdb = DB::default(); + let memtrie = populate_trie::(&mut memdb, &x); + let root = memtrie.commit().commit_to(&mut memdb); + if i == 0 { + main_root = root; + } else { + let child_trie_root_key = x[0].0.clone(); + child_tree.insert(child_trie_root_key, root); + } + } +} From 7faf3943eef127e72f065c26688498432338e542 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 16 Jan 2024 16:12:01 +0100 Subject: [PATCH 18/90] meh, removed key pass through nodemaybe --- trie-db/src/lib.rs | 2 +- trie-db/src/nibble/mod.rs | 2 +- trie-db/src/triedbmut.rs | 69 ++++++++++++++++++++++++++++++++++- trie-db/test/src/lib.rs | 8 +++- trie-db/test/src/triedbmut.rs | 55 ++++++++++++++++++++++------ 5 files changed, 119 insertions(+), 17 deletions(-) diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index e8e00453..5ff7c0d8 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -68,7 +68,7 @@ pub use self::{ triedb::{TrieDB, TrieDBBuilder, TrieDBIterator, TrieDBKeyIterator}, triedbmut::{ Changeset, ChangesetNodeRef, ChildReference, ExistingChangesetNode, NewChangesetNode, - TrieDBMut, TrieDBMutBuilder, Value, + OwnedPrefix, TrieDBMut, TrieDBMutBuilder, Value, }, }; pub use crate::{ diff --git a/trie-db/src/nibble/mod.rs b/trie-db/src/nibble/mod.rs index e1d758e9..413cbfef 100644 --- a/trie-db/src/nibble/mod.rs +++ b/trie-db/src/nibble/mod.rs @@ -142,7 +142,7 @@ pub mod nibble_ops { } /// Backing storage for `NibbleVec`s. -pub(crate) type BackingByteVec = smallvec::SmallVec<[u8; 40]>; +pub type BackingByteVec = smallvec::SmallVec<[u8; 40]>; /// Owning, nibble-oriented byte vector. Counterpart to `NibbleSlice`. /// Nibbles are always left aligned, so making a `NibbleVec` from diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 3d1bd14d..3f99c767 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -29,12 +29,16 @@ use crate::{ use hash_db::{HashDB, Hasher, Prefix}; +#[cfg(feature = "std")] +use std::collections::BTreeMap; #[cfg(feature = "std")] use std::collections::HashSet as Set; #[cfg(not(feature = "std"))] use alloc::boxed::Box; #[cfg(not(feature = "std"))] +use alloc::collections::btree_set::BTreeMap; +#[cfg(not(feature = "std"))] use alloc::collections::btree_set::BTreeSet as Set; #[cfg(feature = "std")] @@ -775,6 +779,8 @@ impl ChangesetNodeRef { pub struct Changeset { pub root: ChangesetNodeRef, pub removed: Vec<(H, OwnedPrefix)>, + // Only needed for apply ks + pub removed_ks: BTreeMap, Vec<(H, OwnedPrefix)>>, } impl Changeset { @@ -847,6 +853,63 @@ impl Changeset { self.root_hash() } + pub fn apply_with_as_prefix(&self, mem_db: &mut MemoryDB) -> H + where + K: memory_db::KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { + fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { + let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); + result.extend_from_slice(ks); + result.extend_from_slice(prefix.0); + (result, prefix.1) + } + + fn apply_node( + node: &ChangesetNodeRef, + ks: Option<&[u8]>, + mem_db: &mut MemoryDB, + ) where + K: memory_db::KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { + match node { + ChangesetNodeRef::New(node) => { + let ks = if ks.is_some() { + ks + } else if let Some(k) = node.key_childset.as_ref() { + Some(k.as_slice()) + } else { + None + }; + for child in &node.children { + apply_node(child, ks, mem_db); + } + if let Some(ks) = ks { + let prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); + mem_db.insert((prefixed.0.as_slice(), prefixed.1), &node.data); + } else { + mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); + } + }, + ChangesetNodeRef::Existing(_) => {}, + } + } + + for (hash, p) in &self.removed { + mem_db.remove(hash, (p.0.as_slice(), p.1)); + } + for (ks, removed) in &self.removed_ks { + for (hash, p) in removed { + let prefixed = prefix_prefix(ks, (p.0.as_slice(), p.1)); + mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); + } + } + + apply_node(&self.root, None, mem_db); + self.root_hash() + } + pub fn root_hash(&self) -> H { match &self.root { ChangesetNodeRef::New(node) => node.hash, @@ -862,11 +925,12 @@ impl Changeset { location: Default::default(), }), removed: Default::default(), + removed_ks: Default::default(), } } } -type OwnedPrefix = (BackingByteVec, Option); +pub type OwnedPrefix = (BackingByteVec, Option); /// A `Trie` implementation using a generic `HashDB` backing database. /// @@ -2048,6 +2112,7 @@ where location, }), removed, + removed_ks: Default::default(), }, // no changes necessary. NodeHandle::InMemory(h) => h, }; @@ -2100,6 +2165,7 @@ where key_childset: None, }), removed, + removed_ks: Default::default(), } }, Stored::Cached(node, hash, location) => { @@ -2117,6 +2183,7 @@ where location, }), removed, + removed_ks: Default::default(), } }, } diff --git a/trie-db/test/src/lib.rs b/trie-db/test/src/lib.rs index 24b70109..6eda5060 100644 --- a/trie-db/test/src/lib.rs +++ b/trie-db/test/src/lib.rs @@ -42,7 +42,9 @@ trait TestDB: hash_db::HashDB + Cl ) -> TrieHash; fn remove(&mut self, hash: &::Out, prefix: Prefix); fn is_empty(&self) -> bool; - fn support_location() -> bool { false } + fn support_location() -> bool { + false + } } impl, H, KF> TestDB for MemoryDB @@ -84,7 +86,9 @@ where MemTreeDB::is_empty(self) } - fn support_location() -> bool { true } + fn support_location() -> bool { + true + } } trait TestCommit { diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 0d245341..654841f1 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -24,8 +24,8 @@ use reference_trie::{ ReferenceNodeCodecNoExt, TestTrieCache, }; use trie_db::{ - CachedValue, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, - TrieDBMutBuilder, TrieError, TrieLayout, Value, TrieHash, + CachedValue, Changeset, ChangesetNodeRef, DBValue, NodeCodec, Recorder, Trie, TrieCache, + TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, TrieError, TrieHash, TrieLayout, Value, }; use trie_standardmap::*; @@ -859,6 +859,12 @@ fn test_two_assets_memory_db_inner_2() { test_layouts!(child_trie, child_trie_internal); fn child_trie_internal>() { use std::collections::BTreeMap; + use trie_db::OwnedPrefix; + struct ATrie { + root: TrieHash, + data: BTreeMap, Vec>, + changeset: Option, T::Location>>, + } // Running a tipical child trie scenario: // different trie, child trie root written all // at once with childset before parent tree commit. @@ -872,9 +878,10 @@ fn child_trie_internal>() { let nb_child_trie_copied_removed = 1; let support_location = DB::support_location(); let mut memdb = DB::default(); - let mut child_tree: BTreeMap, TrieHash> = Default::default(); - let mut main_root: TrieHash = Default::default(); - for i in 0..10 + 1 { + let mut child_tries: BTreeMap, ATrie> = Default::default(); + let mut main_trie: ATrie = + ATrie { root: Default::default(), data: Default::default(), changeset: None }; + for i in 0..nb_child_trie + 1 { let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), min_key: 3, @@ -884,14 +891,38 @@ fn child_trie_internal>() { } .make_with(&mut seed); - let mut memdb = DB::default(); - let memtrie = populate_trie::(&mut memdb, &x); - let root = memtrie.commit().commit_to(&mut memdb); - if i == 0 { - main_root = root; + let mut memtrie = populate_trie::(&mut memdb, &x); + let data: BTreeMap, Vec> = x.iter().cloned().collect(); + if i == nb_child_trie { + let mut removed_ks: BTreeMap, Vec<(TrieHash, OwnedPrefix)>> = + Default::default(); + for (k, c) in child_tries.iter_mut() { + let key: &[u8] = &k[..]; + let val: &[u8] = c.root.as_ref(); + let mut changeset = c.changeset.take().unwrap(); + memtrie + .insert_with_child_changes(key, val, Some(Box::new(changeset.root))) + .unwrap(); + removed_ks.insert(k.clone(), changeset.removed); + } + let mut change_set = memtrie.commit(); + change_set.removed_ks = removed_ks; + let root = change_set.commit_to(&mut memdb); + main_trie.root = root; + main_trie.data = data; } else { - let child_trie_root_key = x[0].0.clone(); - child_tree.insert(child_trie_root_key, root); + let child_trie_root_key = data.iter().next().unwrap().0.clone(); + let mut changeset = memtrie.commit(); + let root = changeset.root_hash(); + match &mut changeset.root { + ChangesetNodeRef::New(node) => { + // needed for prefixed key. + node.key_childset = Some(child_trie_root_key.clone()); + }, + _ => unreachable!(), + } + child_tries + .insert(child_trie_root_key, ATrie { root, data, changeset: Some(changeset) }); } } } From 557b52d93abae0407b23c071bebb96a36c841d80 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 16 Jan 2024 16:29:51 +0100 Subject: [PATCH 19/90] better --- trie-db/src/triedbmut.rs | 91 +++++++++-------------------------- trie-db/test/src/triedbmut.rs | 12 ++--- 2 files changed, 28 insertions(+), 75 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 3f99c767..5bd76a9c 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -749,8 +749,8 @@ pub struct NewChangesetNode { pub prefix: OwnedPrefix, pub data: Vec, pub children: Vec>, - // Storing the key (only needed for old trie). - pub key_childset: Option>, + // Storing the key and removed nodes (only needed for old trie). + pub key_childset: Option<(Vec, Vec<(H, OwnedPrefix)>)>, } #[derive(Debug)] @@ -779,8 +779,6 @@ impl ChangesetNodeRef { pub struct Changeset { pub root: ChangesetNodeRef, pub removed: Vec<(H, OwnedPrefix)>, - // Only needed for apply ks - pub removed_ks: BTreeMap, Vec<(H, OwnedPrefix)>>, } impl Changeset { @@ -789,27 +787,47 @@ impl Changeset { K: memory_db::KeyFunction + Send + Sync, MH: Hasher + Send + Sync, { + fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { + let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); + result.extend_from_slice(ks); + result.extend_from_slice(prefix.0); + (result, prefix.1) + } + for (hash, prefix) in &self.removed { mem_db.remove(hash, (prefix.0.as_slice(), prefix.1)); } + fn apply_node( node: &ChangesetNodeRef, mem_db: &mut MemoryDB, + ks: Option<&[u8]>, ) where K: memory_db::KeyFunction + Send + Sync, MH: Hasher + Send + Sync, { match node { ChangesetNodeRef::New(node) => { + let ks = if ks.is_some() { + ks + } else if let Some((k, removed)) = node.key_childset.as_ref() { + for (hash, p) in removed.iter() { + let prefixed = prefix_prefix(k.as_slice(), (p.0.as_slice(), p.1)); + mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); + } + Some(k.as_slice()) + } else { + None + }; for child in &node.children { - apply_node(child, mem_db); + apply_node(child, mem_db, ks); } mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); }, ChangesetNodeRef::Existing(_) => {}, } } - apply_node::(&self.root, mem_db); + apply_node::(&self.root, mem_db, None); self.root_hash() } @@ -853,63 +871,6 @@ impl Changeset { self.root_hash() } - pub fn apply_with_as_prefix(&self, mem_db: &mut MemoryDB) -> H - where - K: memory_db::KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, - { - fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { - let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); - result.extend_from_slice(ks); - result.extend_from_slice(prefix.0); - (result, prefix.1) - } - - fn apply_node( - node: &ChangesetNodeRef, - ks: Option<&[u8]>, - mem_db: &mut MemoryDB, - ) where - K: memory_db::KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, - { - match node { - ChangesetNodeRef::New(node) => { - let ks = if ks.is_some() { - ks - } else if let Some(k) = node.key_childset.as_ref() { - Some(k.as_slice()) - } else { - None - }; - for child in &node.children { - apply_node(child, ks, mem_db); - } - if let Some(ks) = ks { - let prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); - mem_db.insert((prefixed.0.as_slice(), prefixed.1), &node.data); - } else { - mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); - } - }, - ChangesetNodeRef::Existing(_) => {}, - } - } - - for (hash, p) in &self.removed { - mem_db.remove(hash, (p.0.as_slice(), p.1)); - } - for (ks, removed) in &self.removed_ks { - for (hash, p) in removed { - let prefixed = prefix_prefix(ks, (p.0.as_slice(), p.1)); - mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); - } - } - - apply_node(&self.root, None, mem_db); - self.root_hash() - } - pub fn root_hash(&self) -> H { match &self.root { ChangesetNodeRef::New(node) => node.hash, @@ -925,7 +886,6 @@ impl Changeset { location: Default::default(), }), removed: Default::default(), - removed_ks: Default::default(), } } } @@ -2112,7 +2072,6 @@ where location, }), removed, - removed_ks: Default::default(), }, // no changes necessary. NodeHandle::InMemory(h) => h, }; @@ -2165,7 +2124,6 @@ where key_childset: None, }), removed, - removed_ks: Default::default(), } }, Stored::Cached(node, hash, location) => { @@ -2183,7 +2141,6 @@ where location, }), removed, - removed_ks: Default::default(), } }, } diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 654841f1..1cf7b3f6 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -863,7 +863,7 @@ fn child_trie_internal>() { struct ATrie { root: TrieHash, data: BTreeMap, Vec>, - changeset: Option, T::Location>>, + changeset: Option, T::Location>>, } // Running a tipical child trie scenario: // different trie, child trie root written all @@ -894,19 +894,15 @@ fn child_trie_internal>() { let mut memtrie = populate_trie::(&mut memdb, &x); let data: BTreeMap, Vec> = x.iter().cloned().collect(); if i == nb_child_trie { - let mut removed_ks: BTreeMap, Vec<(TrieHash, OwnedPrefix)>> = - Default::default(); for (k, c) in child_tries.iter_mut() { let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let mut changeset = c.changeset.take().unwrap(); memtrie - .insert_with_child_changes(key, val, Some(Box::new(changeset.root))) + .insert_with_child_changes(key, val, Some(Box::new(changeset))) .unwrap(); - removed_ks.insert(k.clone(), changeset.removed); } let mut change_set = memtrie.commit(); - change_set.removed_ks = removed_ks; let root = change_set.commit_to(&mut memdb); main_trie.root = root; main_trie.data = data; @@ -917,12 +913,12 @@ fn child_trie_internal>() { match &mut changeset.root { ChangesetNodeRef::New(node) => { // needed for prefixed key. - node.key_childset = Some(child_trie_root_key.clone()); + node.key_childset = Some((child_trie_root_key.clone(), changeset.removed)); }, _ => unreachable!(), } child_tries - .insert(child_trie_root_key, ATrie { root, data, changeset: Some(changeset) }); + .insert(child_trie_root_key, ATrie { root, data, changeset: Some(changeset.root) }); } } } From ed42db67caf69e9ff625e766f48066468d9ddb96 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 16 Jan 2024 16:38:57 +0100 Subject: [PATCH 20/90] memtrie issue --- trie-db/test/src/triedbmut.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 1cf7b3f6..a8d3fe20 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -872,6 +872,7 @@ fn child_trie_internal>() { // Pruning. let mut seed = Default::default(); let nb_child_trie = 10; + let nb_child_trie = 1; let nb_child_trie_removed = 2; let nb_child_trie_copied = 5; let nb_child_trie_copied_half_removed = 1; @@ -887,7 +888,8 @@ fn child_trie_internal>() { min_key: 3, journal_key: 0, value_mode: ValueMode::Index, - count: 20, + //count: 20, + count: 2, } .make_with(&mut seed); @@ -920,5 +922,13 @@ fn child_trie_internal>() { child_tries .insert(child_trie_root_key, ATrie { root, data, changeset: Some(changeset.root) }); } + + // check data + { + let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); + for (k, v) in main_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } + } } } From 8d334d2264155fe22091af8df6e89e3ebefb3d82 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 16 Jan 2024 18:50:31 +0100 Subject: [PATCH 21/90] change children location --- test-support/reference-trie/src/lib.rs | 8 ++++++++ trie-db/src/triedbmut.rs | 8 ++++++-- trie-db/test/src/triedbmut.rs | 17 ++++++++--------- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index ff012378..63962b0f 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -56,6 +56,14 @@ macro_rules! test_layouts { ($test:ident, $test_internal:ident) => { #[test] fn $test() { + eprintln!("Running with layout `HashedValueNoExtThreshold` and MemTreeDB"); + $test_internal::< + $crate::HashedValueNoExtThreshold<1, $crate::MemLocation>, + $crate::MemTreeDB<$crate::RefHasher>, + >(); + + + eprintln!("Running with layout `HashedValueNoExtThreshold` and MemoryDB"); $test_internal::< $crate::HashedValueNoExtThreshold<1, ()>, diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 5bd76a9c..27fc12a7 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -2105,7 +2105,9 @@ where }, } }); - root_childset.map(|c| children.push(*c)); + root_childset.map(|c| { + children.push(*c); + }); #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:?}", ToHex(&encoded_root[..])); @@ -2230,6 +2232,9 @@ where let result = if encoded.len() >= L::Hash::LENGTH { let hash = self.db.hash(&encoded); self.cache_node(hash); + child_set.map(|c| { + sub_children.push(*c); + }); children.push(ChangesetNodeRef::New(NewChangesetNode { hash, prefix: prefix.as_owned_prefix(), @@ -2248,7 +2253,6 @@ where ChildReference::Inline(h, len) }; - child_set.map(|c| children.push(*c)); result }, } diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index a8d3fe20..46a1238c 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -899,12 +899,12 @@ fn child_trie_internal>() { for (k, c) in child_tries.iter_mut() { let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); - let mut changeset = c.changeset.take().unwrap(); + let changeset = c.changeset.take().unwrap(); memtrie .insert_with_child_changes(key, val, Some(Box::new(changeset))) .unwrap(); } - let mut change_set = memtrie.commit(); + let change_set = memtrie.commit(); let root = change_set.commit_to(&mut memdb); main_trie.root = root; main_trie.data = data; @@ -922,13 +922,12 @@ fn child_trie_internal>() { child_tries .insert(child_trie_root_key, ATrie { root, data, changeset: Some(changeset.root) }); } - - // check data - { - let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); - for (k, v) in main_trie.data.iter() { - assert_eq!(&trie.get(k).unwrap().unwrap(), v); - } + } + // check data + { + let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); + for (k, v) in main_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } } From a244144b05b6dc68b2b91dc2a55d4e849f3b4a86 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 17 Jan 2024 11:08:59 +0100 Subject: [PATCH 22/90] access to child trie through location. --- test-support/reference-trie/src/lib.rs | 13 +-- .../reference-trie/src/substrate_like.rs | 4 +- trie-db/src/iterator.rs | 91 +++++++++++-------- trie-db/src/lib.rs | 12 ++- trie-db/src/lookup.rs | 10 +- trie-db/src/node.rs | 33 +++++++ trie-db/src/triedb.rs | 22 ++++- trie-db/src/triedbmut.rs | 8 +- trie-db/test/src/triedb.rs | 9 +- trie-db/test/src/triedbmut.rs | 61 +++++++++++-- 10 files changed, 202 insertions(+), 61 deletions(-) diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 63962b0f..b1f6424d 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -23,8 +23,8 @@ use trie_db::{ node::{NibbleSlicePlan, NodeHandlePlan, NodeOwned, NodePlan, Value, ValuePlan}, trie_visit, triedbmut::ChildReference, - DBValue, NodeCodec, Trie, TrieBuilder, TrieConfiguration, TrieDBBuilder, TrieDBMutBuilder, - TrieHash, TrieLayout, TrieRoot, + DBValue, Location, NodeCodec, Trie, TrieBuilder, TrieConfiguration, TrieDBBuilder, + TrieDBMutBuilder, TrieHash, TrieLayout, TrieRoot, }; pub use trie_root::TrieStream; use trie_root::{Hasher, Value as TrieStreamValue}; @@ -62,8 +62,6 @@ macro_rules! test_layouts { $crate::MemTreeDB<$crate::RefHasher>, >(); - - eprintln!("Running with layout `HashedValueNoExtThreshold` and MemoryDB"); $test_internal::< $crate::HashedValueNoExtThreshold<1, ()>, @@ -160,7 +158,7 @@ impl Clone for GenericNoExtensionLayout { } } -impl TrieLayout for GenericNoExtensionLayout { +impl TrieLayout for GenericNoExtensionLayout { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = false; const MAX_INLINE_VALUE: Option = None; @@ -182,10 +180,7 @@ impl TrieLayout for AllowEmptyLayout { type Location = (); } -impl TrieConfiguration - for GenericNoExtensionLayout -{ -} +impl TrieConfiguration for GenericNoExtensionLayout {} /// Trie layout without extension nodes. pub type NoExtensionLayout = GenericNoExtensionLayout; diff --git a/test-support/reference-trie/src/substrate_like.rs b/test-support/reference-trie/src/substrate_like.rs index a6290737..de7b634a 100644 --- a/test-support/reference-trie/src/substrate_like.rs +++ b/test-support/reference-trie/src/substrate_like.rs @@ -34,9 +34,7 @@ impl TrieLayout for HashedValueNoExt { type Location = mem_tree_db::Location; } -impl TrieLayout - for HashedValueNoExtThreshold -{ +impl TrieLayout for HashedValueNoExtThreshold { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = false; const MAX_INLINE_VALUE: Option = Some(C); diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index c4bd11b2..2bb8ac8d 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -76,7 +76,7 @@ impl TrieDBRawIterator { TrieDBRawIterator { trail: Vec::with_capacity(8), key_nibbles: NibbleVec::new() }; let (root_node, root_hash) = db.get_raw_or_lookup( *db.root(), - NodeHandle::Hash(db.root().as_ref(), Default::default()), + NodeHandle::Hash(db.root().as_ref(), db.root_location().unwrap_or_default()), EMPTY_PREFIX, true, )?; @@ -453,46 +453,57 @@ impl TrieDBRawIterator { Err(err) => return Some(Err(err)), }; - let mut prefix = prefix.clone(); - let value = match node.node() { - Node::Leaf(partial, value) => { - prefix.append_partial(partial.right()); - value - }, - Node::Branch(_, value) => match value { - Some(value) => value, - None => continue, - }, - Node::NibbledBranch(partial, _, value) => { - prefix.append_partial(partial.right()); - match value { - Some(value) => value, - None => continue, - } - }, - _ => continue, - }; - - let (key_slice, maybe_extra_nibble) = prefix.as_prefix(); - let key = key_slice.to_vec(); - if let Some(extra_nibble) = maybe_extra_nibble { - return Some(Err(Box::new(TrieError::ValueAtIncompleteKey(key, extra_nibble)))) + match Self::value_from_raw(prefix, node, db) { + Some(r) => return Some(r), + None => continue, } - - let value = match value { - Value::Node(hash, location) => - match Self::fetch_value(db, &hash, (key_slice, None), location) { - Ok(value) => value, - Err(err) => return Some(Err(err)), - }, - Value::Inline(value) => value.to_vec(), - }; - - return Some(Ok((key, value))) } None } + pub(crate) fn value_from_raw( + prefix: &NibbleVec, + node: &Arc>, + db: &TrieDB, + ) -> Option, CError>> { + let mut prefix = prefix.clone(); + let value = match node.node() { + Node::Leaf(partial, value) => { + prefix.append_partial(partial.right()); + value + }, + Node::Branch(_, value) => match value { + Some(value) => value, + None => return None, + }, + Node::NibbledBranch(partial, _, value) => { + prefix.append_partial(partial.right()); + match value { + Some(value) => value, + None => return None, + } + }, + _ => return None, + }; + + let (key_slice, maybe_extra_nibble) = prefix.as_prefix(); + let key = key_slice.to_vec(); + if let Some(extra_nibble) = maybe_extra_nibble { + return Some(Err(Box::new(TrieError::ValueAtIncompleteKey(key, extra_nibble)))) + } + + let value = match value { + Value::Node(hash, location) => + match Self::fetch_value(db, &hash, (key_slice, None), location) { + Ok(value) => value, + Err(err) => return Some(Err(err)), + }, + Value::Inline(value) => value.to_vec(), + }; + + return Some(Ok((key, value))) + } + /// Fetches the next key. /// /// Must be called with the same `db` as when the iterator was created. @@ -585,6 +596,14 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeIterator<'a, 'cache, L> { pub fn db(&self) -> &dyn hash_db::HashDB { self.db.db() } + + /// Access value of an item. + pub fn item_from_raw( + &self, + item: &(NibbleVec, Option>, Arc>), + ) -> Option, CError>> { + TrieDBRawIterator::value_from_raw(&item.0, &item.2, self.db) + } } impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBNodeIterator<'a, 'cache, L> { diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 5ff7c0d8..49dd2b94 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -260,6 +260,11 @@ pub trait Trie { /// Return the root of the trie. fn root(&self) -> &TrieHash; + /// Return the root location of the trie if it was set. + fn root_location(&self) -> Option { + None + } + /// Is the trie empty? fn is_empty(&self) -> bool { *self.root() == L::Codec::hashed_null_node() @@ -354,9 +359,14 @@ pub trait TrieLayout { type Hash: Hasher; /// Codec to use (needs to match hasher and nibble ops). type Codec: NodeCodec::Out>; - type Location: Copy + Default + Eq + PartialEq; + type Location: Location; } +/// Trait alias for requirement of location with `TrieLayout`. +pub trait Location: Copy + Default + Eq + PartialEq + MaybeDebug {} + +impl Location for T {} + /// This trait associates a trie definition with preferred methods. /// It also contains own default implementations and can be /// used to allow switching implementation. diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs index cb6f4869..38d915be 100644 --- a/trie-db/src/lookup.rs +++ b/trie-db/src/lookup.rs @@ -32,6 +32,8 @@ pub struct Lookup<'a, 'cache, L: TrieLayout, Q: Query> { pub query: Q, /// Hash to start at pub hash: TrieHash, + /// Optionally location to start at. + pub location: Option, /// Optional cache that should be used to speed up the lookup. pub cache: Option<&'cache mut dyn TrieCache>, /// Optional recorder that will be called to record all trie accesses. @@ -160,6 +162,9 @@ where ) -> Result>>, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; + if let Some(location_start) = self.location { + location = location_start; + } let mut key_nibbles = 0; let mut cache = self.cache.take(); @@ -601,10 +606,9 @@ where ) -> Result, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; + let mut location = self.location.unwrap_or(location); let mut key_nibbles = 0; - let mut location = location; - // this loop iterates through non-inline nodes. for depth in 0.. { let mut node = cache.get_or_insert_node(hash, location, &mut || { @@ -780,8 +784,8 @@ where ) -> Result, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; + let mut location = self.location.unwrap_or(location); let mut key_nibbles = 0; - let mut location = location; // this loop iterates through non-inline nodes. for depth in 0.. { diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index 27de0a9f..f8c8a3c0 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -649,6 +649,39 @@ impl NodePlan { value.as_mut(), } } + + /// Check if the node has a location for value. + pub fn has_location_value(&self) -> bool { + self.value_plan().map(|v| !v.is_inline()).unwrap_or(false) + } + + /// Check how many children location value node has. + pub fn nb_location_children(&self) -> usize { + match self { + NodePlan::Extension { child: NodeHandlePlan::Hash(_), .. } => 1, + NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { + let mut count = 0; + for child in children { + if let Some(NodeHandlePlan::Hash(_)) = child { + count += 1; + } + } + count + }, + _ => 0, + } + } + + pub fn attached_change_set_location(&self, locations: &[L]) -> Option { + let offset = if self.has_location_value() { 1 } else { 0 } + self.nb_location_children(); + if locations.len() > offset { + // only one additional location expected with current code. + debug_assert!(locations.len() == offset + 1); + Some(locations[offset]) + } else { + None + } + } } /// An `OwnedNode` is an owned type from which a `Node` can be constructed which borrows data from diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index d721a586..bad1ecba 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -33,6 +33,7 @@ use hash_db::{HashDB, Prefix, EMPTY_PREFIX}; pub struct TrieDBBuilder<'db, 'cache, L: TrieLayout> { db: &'db dyn HashDB, root: &'db TrieHash, + root_location: Option, cache: Option<&'cache mut dyn TrieCache>, recorder: Option<&'cache mut dyn TrieRecorder, L::Location>>, } @@ -44,7 +45,17 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// when trying to lookup any key. #[inline] pub fn new(db: &'db dyn HashDB, root: &'db TrieHash) -> Self { - Self { db, root, cache: None, recorder: None } + Self { db, root, cache: None, recorder: None, root_location: None } + } + + /// Same as `new` but indicating db location of root. Warning root hash will not be checked. + #[inline] + pub fn new_with_db_location( + db: &'db dyn HashDB, + root: &'db TrieHash, + root_location: L::Location, + ) -> Self { + Self { db, root, cache: None, recorder: None, root_location: Some(root_location) } } /// Use the given `cache` for the db. @@ -92,6 +103,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { TrieDB { db: self.db, root: self.root, + root_location: self.root_location, cache: self.cache.map(core::cell::RefCell::new), recorder: self.recorder.map(core::cell::RefCell::new), } @@ -126,6 +138,7 @@ where { db: &'db dyn HashDB, root: &'db TrieHash, + root_location: Option, cache: Option>>, recorder: Option, L::Location>>>, } @@ -225,6 +238,10 @@ where self.root } + fn root_location(&self) -> Option { + self.root_location + } + fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { let mut cache = self.cache.as_ref().map(|c| c.borrow_mut()); let mut recorder = self.recorder.as_ref().map(|r| r.borrow_mut()); @@ -233,6 +250,7 @@ where db: self.db, query: |_: &[u8]| (), hash: *self.root, + location: self.root_location, cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), recorder: recorder .as_mut() @@ -253,6 +271,7 @@ where db: self.db, query, hash: *self.root, + location: self.root_location, cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), recorder: recorder .as_mut() @@ -272,6 +291,7 @@ where db: self.db, query: |_: &[u8]| (), hash: *self.root, + location: self.root_location, cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), recorder: recorder .as_mut() diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 27fc12a7..0cf143bb 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -822,7 +822,12 @@ impl Changeset { for child in &node.children { apply_node(child, mem_db, ks); } - mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); + if let Some(ks) = ks { + let prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); + mem_db.insert((prefixed.0.as_slice(), prefixed.1), &node.data); + } else { + mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); + } }, ChangesetNodeRef::Existing(_) => {}, } @@ -1043,6 +1048,7 @@ where db: self.db, query: |v: &[u8]| v.to_vec(), hash: *hash, + location: None, // TODO support building from location cache: None, recorder: recorder .as_mut() diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index 944332ef..da54b31a 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -395,7 +395,14 @@ fn test_lookup_with_corrupt_data_returns_decoder_error_internal { db: t.db(), query: q, hash: root, cache: None, recorder: None }; + let lookup = Lookup:: { + db: t.db(), + query: q, + hash: root, + location: None, + cache: None, + recorder: None, + }; let query_result = lookup.look_up(&b"A"[..], NibbleSlice::new(b"A"), Default::default()); assert_eq!(query_result.unwrap().unwrap(), true); } diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 46a1238c..b1f29350 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -25,7 +25,8 @@ use reference_trie::{ }; use trie_db::{ CachedValue, Changeset, ChangesetNodeRef, DBValue, NodeCodec, Recorder, Trie, TrieCache, - TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, TrieError, TrieHash, TrieLayout, Value, + TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, + TrieLayout, Value, }; use trie_standardmap::*; @@ -900,12 +901,10 @@ fn child_trie_internal>() { let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let changeset = c.changeset.take().unwrap(); - memtrie - .insert_with_child_changes(key, val, Some(Box::new(changeset))) - .unwrap(); + memtrie.insert_with_child_changes(key, val, Some(Box::new(changeset))).unwrap(); } - let change_set = memtrie.commit(); - let root = change_set.commit_to(&mut memdb); + let changeset = memtrie.commit(); + let root = changeset.commit_to(&mut memdb); main_trie.root = root; main_trie.data = data; } else { @@ -930,4 +929,54 @@ fn child_trie_internal>() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } + for (root_key, child_trie) in child_tries { + if support_location { + let (child_trie_location, child_trie_root) = { + let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); + // TODO get root location + let root_loc = trie + .get_with(&root_key, |x: &[u8]| { + // panic!("{:?}", x); + x.len() + }) + .unwrap() + .unwrap(); + // Note could have a variant of get_with here that goes into + // encoded node hash and locations. + let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); + use trie_db::TrieIterator; + iter.seek(&root_key).unwrap(); + let item = iter.next().unwrap().unwrap(); + let node = &item.2; + let location = + node.node_plan().attached_change_set_location(node.locations()).unwrap(); + let root = iter.item_from_raw(&item).unwrap().unwrap(); + let mut root_hash = TrieHash::::default(); + root_hash.as_mut().copy_from_slice(&root.1); + (location, root_hash) + }; + let trie = TrieDBBuilder::::new_with_db_location( + &memdb, + &child_trie_root, + child_trie_location, + ) + .build(); + for (k, v) in child_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } + } else { + let root = { + let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); + let root_vec = trie.get(&root_key).unwrap().unwrap(); + let mut root = TrieHash::::default(); + // TODO use a prefixed memorydb over ks. + root.as_mut().copy_from_slice(root_vec.as_slice()); + root + }; + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + for (k, v) in child_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } + } + } } From f5bc93c3487b2d789b7e01f57fbf64a79f8d5af2 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 17 Jan 2024 14:20:34 +0100 Subject: [PATCH 23/90] allow access to prefixed child trie --- trie-db/src/triedbmut.rs | 21 +++++++-------------- trie-db/test/src/triedbmut.rs | 31 +++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 0cf143bb..afed6ea5 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -781,19 +781,19 @@ pub struct Changeset { pub removed: Vec<(H, OwnedPrefix)>, } +pub fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { + let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); + result.extend_from_slice(ks); + result.extend_from_slice(prefix.0); + (result, prefix.1) +} + impl Changeset { pub fn apply_to(&self, mem_db: &mut MemoryDB) -> H where K: memory_db::KeyFunction + Send + Sync, MH: Hasher + Send + Sync, { - fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { - let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); - result.extend_from_slice(ks); - result.extend_from_slice(prefix.0); - (result, prefix.1) - } - for (hash, prefix) in &self.removed { mem_db.remove(hash, (prefix.0.as_slice(), prefix.1)); } @@ -841,13 +841,6 @@ impl Changeset { K: memory_db::KeyFunction + Send + Sync, MH: Hasher + Send + Sync, { - fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { - let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); - result.extend_from_slice(ks); - result.extend_from_slice(prefix.0); - (result, prefix.1) - } - fn apply_node( node: &ChangesetNodeRef, ks: &[u8], diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index b1f29350..cdc2ff8f 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -15,7 +15,7 @@ use std::ops::Deref; use env_logger; -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; +use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; use log::debug; use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ @@ -531,9 +531,9 @@ fn insert_empty_allowed() { #[test] fn register_proof_without_value() { - use hash_db::Prefix; use reference_trie::HashedValueNoExtThreshold; use std::{cell::RefCell, collections::HashMap}; + use Prefix; type Layout = HashedValueNoExtThreshold<1, ()>; type MemoryDB = memory_db::MemoryDB, DBValue>; @@ -973,6 +973,8 @@ fn child_trie_internal>() { root.as_mut().copy_from_slice(root_vec.as_slice()); root }; + + let memdb = KeySpacedDB::new(&memdb, &root_key[..]); let trie = TrieDBBuilder::::new(&memdb, &root).build(); for (k, v) in child_trie.data.iter() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); @@ -980,3 +982,28 @@ fn child_trie_internal>() { } } } + +pub struct KeySpacedDB<'a, H, T, DL>(&'a dyn hash_db::HashDB, &'a [u8]); + +impl<'a, H, T, DL> KeySpacedDB<'a, H, T, DL> { + #[inline] + pub fn new(db: &'a dyn hash_db::HashDB, ks: &'a [u8]) -> Self { + KeySpacedDB(db, ks) + } +} + +impl<'a, H, T, L> hash_db::HashDB for KeySpacedDB<'a, H, T, L> +where + H: Hasher, + T: From<&'static [u8]>, +{ + fn get(&self, key: &H::Out, prefix: Prefix, location: L) -> Option<(T, Vec)> { + let derived_prefix = trie_db::triedbmut::prefix_prefix(self.1, prefix); + self.0.get(key, (&derived_prefix.0, derived_prefix.1), location) + } + + fn contains(&self, key: &H::Out, prefix: Prefix, location: L) -> bool { + let derived_prefix = trie_db::triedbmut::prefix_prefix(self.1, prefix); + self.0.contains(key, (&derived_prefix.0, derived_prefix.1), location) + } +} From caf67f3310104745efc8c0c95ae3fc786806fdd7 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 17 Jan 2024 16:48:12 +0100 Subject: [PATCH 24/90] share more code in test --- trie-db/test/src/triedbmut.rs | 73 +++++++++++++++++------------------ 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index cdc2ff8f..009b5f53 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -873,7 +873,7 @@ fn child_trie_internal>() { // Pruning. let mut seed = Default::default(); let nb_child_trie = 10; - let nb_child_trie = 1; + // let nb_child_trie = 1; let nb_child_trie_removed = 2; let nb_child_trie_copied = 5; let nb_child_trie_copied_half_removed = 1; @@ -889,8 +889,8 @@ fn child_trie_internal>() { min_key: 3, journal_key: 0, value_mode: ValueMode::Index, - //count: 20, - count: 2, + count: 20, + //count: 2, } .make_with(&mut seed); @@ -930,52 +930,49 @@ fn child_trie_internal>() { } } for (root_key, child_trie) in child_tries { + let (child_trie_location, child_trie_root) = { + let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); + // TODO get root location + let root_loc = trie + .get_with(&root_key, |x: &[u8]| { + // panic!("{:?}", x); + x.len() + }) + .unwrap() + .unwrap(); + // Note could have a variant of get_with here that goes into + // encoded node hash and locations. + let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); + use trie_db::TrieIterator; + iter.seek(&root_key).unwrap(); + let item = iter.next().unwrap().unwrap(); + let node = &item.2; + let location = node.node_plan().attached_change_set_location(node.locations()); + let root = iter.item_from_raw(&item).unwrap().unwrap(); + let mut root_hash = TrieHash::::default(); + root_hash.as_mut().copy_from_slice(&root.1); + (location, root_hash) + }; if support_location { - let (child_trie_location, child_trie_root) = { - let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); - // TODO get root location - let root_loc = trie - .get_with(&root_key, |x: &[u8]| { - // panic!("{:?}", x); - x.len() - }) - .unwrap() - .unwrap(); - // Note could have a variant of get_with here that goes into - // encoded node hash and locations. - let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); - use trie_db::TrieIterator; - iter.seek(&root_key).unwrap(); - let item = iter.next().unwrap().unwrap(); - let node = &item.2; - let location = - node.node_plan().attached_change_set_location(node.locations()).unwrap(); - let root = iter.item_from_raw(&item).unwrap().unwrap(); - let mut root_hash = TrieHash::::default(); - root_hash.as_mut().copy_from_slice(&root.1); - (location, root_hash) - }; let trie = TrieDBBuilder::::new_with_db_location( &memdb, &child_trie_root, - child_trie_location, + child_trie_location.unwrap(), ) .build(); for (k, v) in child_trie.data.iter() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } else { - let root = { - let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); - let root_vec = trie.get(&root_key).unwrap().unwrap(); - let mut root = TrieHash::::default(); - // TODO use a prefixed memorydb over ks. - root.as_mut().copy_from_slice(root_vec.as_slice()); - root - }; - let memdb = KeySpacedDB::new(&memdb, &root_key[..]); - let trie = TrieDBBuilder::::new(&memdb, &root).build(); + assert!(child_trie_location.is_none()); + let trie = TrieDBBuilder::::new_with_db_location( + &memdb, + &child_trie_root, + Default::default(), + ) + .build(); + for (k, v) in child_trie.data.iter() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } From de3bc22a330166f9ed8efa3706d15f575c30255d Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 17 Jan 2024 17:29:24 +0100 Subject: [PATCH 25/90] extract in fn --- trie-db/test/src/triedbmut.rs | 52 ++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 009b5f53..3072dbfa 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -930,29 +930,7 @@ fn child_trie_internal>() { } } for (root_key, child_trie) in child_tries { - let (child_trie_location, child_trie_root) = { - let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); - // TODO get root location - let root_loc = trie - .get_with(&root_key, |x: &[u8]| { - // panic!("{:?}", x); - x.len() - }) - .unwrap() - .unwrap(); - // Note could have a variant of get_with here that goes into - // encoded node hash and locations. - let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); - use trie_db::TrieIterator; - iter.seek(&root_key).unwrap(); - let item = iter.next().unwrap().unwrap(); - let node = &item.2; - let location = node.node_plan().attached_change_set_location(node.locations()); - let root = iter.item_from_raw(&item).unwrap().unwrap(); - let mut root_hash = TrieHash::::default(); - root_hash.as_mut().copy_from_slice(&root.1); - (location, root_hash) - }; + let (child_trie_root, child_trie_location) = child_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); if support_location { let trie = TrieDBBuilder::::new_with_db_location( &memdb, @@ -964,8 +942,8 @@ fn child_trie_internal>() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } else { - let memdb = KeySpacedDB::new(&memdb, &root_key[..]); assert!(child_trie_location.is_none()); + let memdb = KeySpacedDB::new(&memdb, &root_key[..]); let trie = TrieDBBuilder::::new_with_db_location( &memdb, &child_trie_root, @@ -977,7 +955,31 @@ fn child_trie_internal>() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } - } + // Modifying an existing child trie. + } +} + +fn child_trie_root>( + memdb: &DB, + main_root: &TrieHash, + root_key: &[u8], +) -> Option<(TrieHash, Option)> { + let trie = TrieDBBuilder::::new(memdb, main_root).build(); + // Note could have a variant of get_with here that goes into + // encoded node hash and locations. + let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); + use trie_db::TrieIterator; + iter.seek(root_key).unwrap(); + let item = iter.next()?.unwrap(); + let node = &item.2; + let location = node.node_plan().attached_change_set_location(node.locations()); + let root = iter.item_from_raw(&item)?.unwrap(); + if root.0.as_slice() != root_key { + return None; + } + let mut root_hash = TrieHash::::default(); + root_hash.as_mut().copy_from_slice(&root.1); + Some((root_hash, location)) } pub struct KeySpacedDB<'a, H, T, DL>(&'a dyn hash_db::HashDB, &'a [u8]); From 8edcc926082395a875e3f2edac1ff85fbbc1795a Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 17 Jan 2024 18:10:57 +0100 Subject: [PATCH 26/90] need support of location to init triedbmut --- trie-db/test/src/triedbmut.rs | 114 +++++++++++++++++++++++++++------- 1 file changed, 90 insertions(+), 24 deletions(-) diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 3072dbfa..d69690f6 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -881,6 +881,7 @@ fn child_trie_internal>() { let support_location = DB::support_location(); let mut memdb = DB::default(); let mut child_tries: BTreeMap, ATrie> = Default::default(); + let mut keyspaced_memdb; let mut main_trie: ATrie = ATrie { root: Default::default(), data: Default::default(), changeset: None }; for i in 0..nb_child_trie + 1 { @@ -929,34 +930,99 @@ fn child_trie_internal>() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } - for (root_key, child_trie) in child_tries { - let (child_trie_root, child_trie_location) = child_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); - if support_location { - let trie = TrieDBBuilder::::new_with_db_location( - &memdb, - &child_trie_root, - child_trie_location.unwrap(), - ) - .build(); - for (k, v) in child_trie.data.iter() { - assert_eq!(&trie.get(k).unwrap().unwrap(), v); - } + for (root_key, child_trie) in &child_tries { + let (child_trie_root, child_trie_location) = + child_trie_root(&memdb, &main_trie.root, root_key).unwrap(); + + let child_memdb: &dyn HashDB<_, _, _> = if support_location { + &memdb } else { assert!(child_trie_location.is_none()); - let memdb = KeySpacedDB::new(&memdb, &root_key[..]); - let trie = TrieDBBuilder::::new_with_db_location( - &memdb, - &child_trie_root, - Default::default(), - ) - .build(); - - for (k, v) in child_trie.data.iter() { - assert_eq!(&trie.get(k).unwrap().unwrap(), v); - } + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + + let trie = TrieDBBuilder::::new_with_db_location( + child_memdb, + &child_trie_root, + child_trie_location.unwrap_or_default(), + ) + .build(); + for (k, v) in child_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } + } + // Modifying an existing child trie. + let (root_key, a_child_trie) = child_tries.iter().next().unwrap(); + let (a_child_trie_root, child_trie_location) = + child_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); + let (child_changeset, child_root_hash) = { + assert_eq!(a_child_trie_root, a_child_trie.root); + let child_memdb: &dyn HashDB<_, _, _> = if support_location { + &memdb + } else { + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let mut child_trie = TrieDBMutBuilder::::from_existing_with_location( + child_memdb, + a_child_trie_root, + child_trie_location.unwrap_or_default(), + ) + .build(); + child_trie.remove(a_child_trie.data.iter().next().unwrap().0).unwrap(); + child_trie.insert(b"make_sur_it_changes", b"value").unwrap(); + let mut changeset = child_trie.commit(); + let new_root = changeset.root_hash(); + match &mut changeset.root { + ChangesetNodeRef::New(node) => { + // needed for prefixed key. + // Note if unchanged we don't need this actually unchange should only + // be a thing in case of a copy. + node.key_childset = Some((root_key.clone(), changeset.removed)); + }, + _ => unreachable!(), + } + assert!(new_root != a_child_trie_root); + (changeset.root, new_root) + }; + let mut main_trie = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); + main_trie + .insert_with_child_changes( + root_key, + child_root_hash.as_ref(), + Some(Box::new(child_changeset)), + ) + .unwrap(); + let changeset = main_trie.commit(); + let main_root = changeset.root_hash(); + changeset.commit_to(&mut memdb); + // checking modification + let (a_child_trie_root, child_trie_location) = + child_trie_root(&memdb, &main_root, root_key).unwrap(); + let child_memdb: &dyn HashDB<_, _, _> = if support_location { + &memdb + } else { + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let trie = TrieDBBuilder::::new_with_db_location( + child_memdb, + &a_child_trie_root, + child_trie_location.unwrap_or_default(), + ) + .build(); + trie.get(b"make_sur_it_changes").unwrap().unwrap(); + let mut first = true; + for (k, v) in a_child_trie.data.iter() { + if first { + assert!(&trie.get(k).unwrap().is_none()); + first = false; + } else { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); } - // Modifying an existing child trie. } + trie.get(b"make_sur_it_changes").unwrap().unwrap(); } fn child_trie_root>( From 1f9b8edb49e3a91eeb32e2d8005d8e6cac71e67d Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 17 Jan 2024 18:24:22 +0100 Subject: [PATCH 27/90] ok --- trie-db/src/triedbmut.rs | 20 +++++++++++++++++--- trie-db/test/src/triedbmut.rs | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index afed6ea5..e86a792e 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -667,6 +667,7 @@ impl<'a, L: TrieLayout> Index<&'a StorageHandle> for NodeStorage { pub struct TrieDBMutBuilder<'db, L: TrieLayout> { db: &'db dyn HashDB, root: TrieHash, + root_location: L::Location, cache: Option<&'db mut dyn TrieCache>, recorder: Option<&'db mut dyn TrieRecorder, L::Location>>, } @@ -676,7 +677,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// `root`. pub fn new(db: &'db dyn HashDB) -> Self { let root = L::Codec::hashed_null_node(); - Self { root, db, cache: None, recorder: None } + Self { root, db, cache: None, recorder: None, root_location: Default::default() } } /// Create a builder for constructing a new trie with the backing database `db` and `root`. @@ -687,9 +688,20 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { db: &'db dyn HashDB, root: TrieHash, ) -> Self { - Self { db, root, cache: None, recorder: None } + Self { db, root, cache: None, recorder: None, root_location: Default::default() } } + /// Same as `from_existing` but force a db location to access root. + /// Note root in parameter is not checked. + pub fn from_existing_with_db_location( + db: &'db dyn HashDB, + root: TrieHash, + root_location: L::Location + ) -> Self { + Self { db, root, cache: None, recorder: None, root_location } + } + + /// Use the given `cache` for the db. pub fn with_cache(mut self, cache: &'db mut dyn TrieCache) -> Self { self.cache = Some(cache); @@ -727,7 +739,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Build the [`TrieDBMut`]. pub fn build(self) -> TrieDBMut<'db, L> { - let root_handle = NodeHandle::Hash(self.root, Default::default()); + let root_handle = NodeHandle::Hash(self.root, self.root_location); TrieDBMut { db: self.db, @@ -2133,6 +2145,8 @@ where self.root_handle = NodeHandle::InMemory(self.storage.alloc(Stored::Cached( node, hash, + // TODO should we use location here?? likely yes TODO write a test with cache usage and location in + // triedbmut Default::default(), ))); Changeset { diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index d69690f6..e042f62b 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -964,7 +964,7 @@ fn child_trie_internal>() { keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); &keyspaced_memdb }; - let mut child_trie = TrieDBMutBuilder::::from_existing_with_location( + let mut child_trie = TrieDBMutBuilder::::from_existing_with_db_location( child_memdb, a_child_trie_root, child_trie_location.unwrap_or_default(), From 270aa818c8018696a7083d1783cc34b948c83568 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 17 Jan 2024 18:30:35 +0100 Subject: [PATCH 28/90] default location instead of none --- trie-db/src/iterator.rs | 2 +- trie-db/src/lib.rs | 4 ++-- trie-db/src/lookup.rs | 10 ++++------ trie-db/src/triedb.rs | 10 +++++----- trie-db/src/triedbmut.rs | 9 ++++----- trie-db/test/src/triedb.rs | 2 +- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index 2bb8ac8d..ade4acc7 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -76,7 +76,7 @@ impl TrieDBRawIterator { TrieDBRawIterator { trail: Vec::with_capacity(8), key_nibbles: NibbleVec::new() }; let (root_node, root_hash) = db.get_raw_or_lookup( *db.root(), - NodeHandle::Hash(db.root().as_ref(), db.root_location().unwrap_or_default()), + NodeHandle::Hash(db.root().as_ref(), db.root_location()), EMPTY_PREFIX, true, )?; diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 49dd2b94..0a2b4a5d 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -261,8 +261,8 @@ pub trait Trie { fn root(&self) -> &TrieHash; /// Return the root location of the trie if it was set. - fn root_location(&self) -> Option { - None + fn root_location(&self) -> L::Location { + Default::default() } /// Is the trie empty? diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs index 38d915be..a677525c 100644 --- a/trie-db/src/lookup.rs +++ b/trie-db/src/lookup.rs @@ -33,7 +33,7 @@ pub struct Lookup<'a, 'cache, L: TrieLayout, Q: Query> { /// Hash to start at pub hash: TrieHash, /// Optionally location to start at. - pub location: Option, + pub location: L::Location, /// Optional cache that should be used to speed up the lookup. pub cache: Option<&'cache mut dyn TrieCache>, /// Optional recorder that will be called to record all trie accesses. @@ -162,9 +162,7 @@ where ) -> Result>>, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; - if let Some(location_start) = self.location { - location = location_start; - } + let mut location = self.location; let mut key_nibbles = 0; let mut cache = self.cache.take(); @@ -606,7 +604,7 @@ where ) -> Result, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; - let mut location = self.location.unwrap_or(location); + let mut location = self.location; let mut key_nibbles = 0; // this loop iterates through non-inline nodes. @@ -784,7 +782,7 @@ where ) -> Result, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; - let mut location = self.location.unwrap_or(location); + let mut location = self.location; let mut key_nibbles = 0; // this loop iterates through non-inline nodes. diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index bad1ecba..45e1c28c 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -33,7 +33,7 @@ use hash_db::{HashDB, Prefix, EMPTY_PREFIX}; pub struct TrieDBBuilder<'db, 'cache, L: TrieLayout> { db: &'db dyn HashDB, root: &'db TrieHash, - root_location: Option, + root_location: L::Location, cache: Option<&'cache mut dyn TrieCache>, recorder: Option<&'cache mut dyn TrieRecorder, L::Location>>, } @@ -45,7 +45,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// when trying to lookup any key. #[inline] pub fn new(db: &'db dyn HashDB, root: &'db TrieHash) -> Self { - Self { db, root, cache: None, recorder: None, root_location: None } + Self { db, root, cache: None, recorder: None, root_location: Default::default() } } /// Same as `new` but indicating db location of root. Warning root hash will not be checked. @@ -55,7 +55,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { root: &'db TrieHash, root_location: L::Location, ) -> Self { - Self { db, root, cache: None, recorder: None, root_location: Some(root_location) } + Self { db, root, cache: None, recorder: None, root_location } } /// Use the given `cache` for the db. @@ -138,7 +138,7 @@ where { db: &'db dyn HashDB, root: &'db TrieHash, - root_location: Option, + root_location: L::Location, cache: Option>>, recorder: Option, L::Location>>>, } @@ -238,7 +238,7 @@ where self.root } - fn root_location(&self) -> Option { + fn root_location(&self) -> L::Location { self.root_location } diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index e86a792e..3b022bff 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -696,12 +696,11 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { pub fn from_existing_with_db_location( db: &'db dyn HashDB, root: TrieHash, - root_location: L::Location + root_location: L::Location, ) -> Self { Self { db, root, cache: None, recorder: None, root_location } } - /// Use the given `cache` for the db. pub fn with_cache(mut self, cache: &'db mut dyn TrieCache) -> Self { self.cache = Some(cache); @@ -1053,7 +1052,7 @@ where db: self.db, query: |v: &[u8]| v.to_vec(), hash: *hash, - location: None, // TODO support building from location + location: *location, cache: None, recorder: recorder .as_mut() @@ -2145,8 +2144,8 @@ where self.root_handle = NodeHandle::InMemory(self.storage.alloc(Stored::Cached( node, hash, - // TODO should we use location here?? likely yes TODO write a test with cache usage and location in - // triedbmut + // TODO should we use location here?? likely yes TODO write a test with cache + // usage and location in triedbmut Default::default(), ))); Changeset { diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index da54b31a..09de89ef 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -399,7 +399,7 @@ fn test_lookup_with_corrupt_data_returns_decoder_error_internal Date: Wed, 17 Jan 2024 18:36:13 +0100 Subject: [PATCH 29/90] remove unused location parameters from lookup (root location is a lookup field). --- trie-db/src/lookup.rs | 12 ++---------- trie-db/src/triedb.rs | 4 ++-- trie-db/src/triedbmut.rs | 2 +- trie-db/test/src/triedb.rs | 2 +- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs index a677525c..27c77b7c 100644 --- a/trie-db/src/lookup.rs +++ b/trie-db/src/lookup.rs @@ -158,7 +158,6 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, - mut location: L::Location, ) -> Result>>, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; @@ -385,11 +384,10 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, - location: L::Location, ) -> Result, TrieHash, CError> { match self.cache.take() { - Some(cache) => self.look_up_with_cache(full_key, nibble_key, location, cache), - None => self.look_up_without_cache(nibble_key, location, full_key, Self::load_value), + Some(cache) => self.look_up_with_cache(full_key, nibble_key, cache), + None => self.look_up_without_cache(nibble_key, full_key, Self::load_value), } } @@ -407,7 +405,6 @@ where Some(cache) => self.look_up_hash_with_cache(full_key, nibble_key, location, cache), None => self.look_up_without_cache( nibble_key, - location, full_key, |v, _, full_key, _, recorder, _| { Ok(match v { @@ -462,7 +459,6 @@ where } else { let hash_and_value = self.look_up_with_cache_internal( nibble_key, - location, full_key, cache, |value, _, full_key, _, _, recorder| match value { @@ -508,7 +504,6 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, - location: L::Location, cache: &mut dyn crate::TrieCache, ) -> Result, TrieHash, CError> { let trie_nodes_recorded = @@ -530,7 +525,6 @@ where -> Result, TrieHash, CError> { let data = lookup.look_up_with_cache_internal( nibble_key, - location, full_key, cache, Self::load_owned_value, @@ -590,7 +584,6 @@ where fn look_up_with_cache_internal( &mut self, nibble_key: NibbleSlice, - location: L::Location, full_key: &[u8], cache: &mut dyn crate::TrieCache, load_value_owned: impl Fn( @@ -769,7 +762,6 @@ where fn look_up_without_cache( mut self, nibble_key: NibbleSlice, - location: L::Location, full_key: &[u8], load_value: impl Fn( Value, diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index 45e1c28c..20efa26f 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -277,7 +277,7 @@ where .as_mut() .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } - .look_up(key, NibbleSlice::new(key), Default::default()) + .look_up(key, NibbleSlice::new(key)) } fn lookup_first_descendant( @@ -297,7 +297,7 @@ where .as_mut() .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } - .lookup_first_descendant(key, NibbleSlice::new(key), Default::default()) + .lookup_first_descendant(key, NibbleSlice::new(key)) } fn iter<'a>( diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 3b022bff..756874d6 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -1058,7 +1058,7 @@ where .as_mut() .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } - .look_up(full_key, partial, *location) + .look_up(full_key, partial) }, NodeHandle::InMemory(handle) => match &self.storage[handle] { Node::Empty => return Ok(None), diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index 09de89ef..21e94c72 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -403,7 +403,7 @@ fn test_lookup_with_corrupt_data_returns_decoder_error_internal Date: Thu, 18 Jan 2024 15:30:39 +0100 Subject: [PATCH 30/90] apply_with_prefix should not be needed. --- trie-db/src/triedbmut.rs | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 756874d6..4b2c5eb9 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -847,39 +847,6 @@ impl Changeset { self.root_hash() } - pub fn apply_with_prefix(&self, mem_db: &mut MemoryDB, ks: &[u8]) -> H - where - K: memory_db::KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, - { - fn apply_node( - node: &ChangesetNodeRef, - ks: &[u8], - mem_db: &mut MemoryDB, - ) where - K: memory_db::KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, - { - match node { - ChangesetNodeRef::New(node) => { - for child in &node.children { - apply_node(child, ks, mem_db); - } - let prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); - mem_db.insert((prefixed.0.as_slice(), prefixed.1), &node.data); - }, - ChangesetNodeRef::Existing(_) => {}, - } - } - - for (hash, p) in &self.removed { - let prefixed = prefix_prefix(ks, (p.0.as_slice(), p.1)); - mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); - } - apply_node(&self.root, ks, mem_db); - self.root_hash() - } - pub fn root_hash(&self) -> H { match &self.root { ChangesetNodeRef::New(node) => node.hash, From 39e81cb61161bb860028d28196fb3391ab19c2a1 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 18 Jan 2024 16:10:55 +0100 Subject: [PATCH 31/90] attach changeset in explicit function --- trie-db/src/lookup.rs | 4 +--- trie-db/src/triedb.rs | 2 +- trie-db/src/triedbmut.rs | 17 +++++++++++---- trie-db/test/src/triedbmut.rs | 39 +++++++++-------------------------- 4 files changed, 25 insertions(+), 37 deletions(-) diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs index 27c77b7c..d2cd9661 100644 --- a/trie-db/src/lookup.rs +++ b/trie-db/src/lookup.rs @@ -399,10 +399,9 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, - location: L::Location, ) -> Result>, TrieHash, CError> { match self.cache.take() { - Some(cache) => self.look_up_hash_with_cache(full_key, nibble_key, location, cache), + Some(cache) => self.look_up_hash_with_cache(full_key, nibble_key, cache), None => self.look_up_without_cache( nibble_key, full_key, @@ -440,7 +439,6 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, - location: L::Location, cache: &mut dyn crate::TrieCache, ) -> Result>, TrieHash, CError> { let value_cache_allowed = self diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index 20efa26f..45986edb 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -256,7 +256,7 @@ where .as_mut() .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } - .look_up_hash(key, NibbleSlice::new(key), Default::default()) + .look_up_hash(key, NibbleSlice::new(key)) } fn get_with>( diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 4b2c5eb9..d77c5661 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -29,16 +29,12 @@ use crate::{ use hash_db::{HashDB, Hasher, Prefix}; -#[cfg(feature = "std")] -use std::collections::BTreeMap; #[cfg(feature = "std")] use std::collections::HashSet as Set; #[cfg(not(feature = "std"))] use alloc::boxed::Box; #[cfg(not(feature = "std"))] -use alloc::collections::btree_set::BTreeMap; -#[cfg(not(feature = "std"))] use alloc::collections::btree_set::BTreeSet as Set; #[cfg(feature = "std")] @@ -864,6 +860,19 @@ impl Changeset { removed: Default::default(), } } + + pub fn to_insert_in_other_trie(mut self, root_at: Vec) -> Box> { + match &mut self.root { + ChangesetNodeRef::New(node) => { + // needed for prefixed key. + // Note if unchanged we don't need this actually unchange should only + // be a thing in case of a copy. + node.key_childset = Some((root_at, self.removed)); + }, + _ => (), + } + Box::new(self.root) + } } pub type OwnedPrefix = (BackingByteVec, Option); diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index e042f62b..dbf6f4a9 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -24,7 +24,7 @@ use reference_trie::{ ReferenceNodeCodecNoExt, TestTrieCache, }; use trie_db::{ - CachedValue, Changeset, ChangesetNodeRef, DBValue, NodeCodec, Recorder, Trie, TrieCache, + CachedValue, ChangesetNodeRef, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, TrieLayout, Value, }; @@ -860,11 +860,10 @@ fn test_two_assets_memory_db_inner_2() { test_layouts!(child_trie, child_trie_internal); fn child_trie_internal>() { use std::collections::BTreeMap; - use trie_db::OwnedPrefix; struct ATrie { root: TrieHash, data: BTreeMap, Vec>, - changeset: Option, T::Location>>, + changeset: Option, T::Location>>>, } // Running a tipical child trie scenario: // different trie, child trie root written all @@ -874,10 +873,6 @@ fn child_trie_internal>() { let mut seed = Default::default(); let nb_child_trie = 10; // let nb_child_trie = 1; - let nb_child_trie_removed = 2; - let nb_child_trie_copied = 5; - let nb_child_trie_copied_half_removed = 1; - let nb_child_trie_copied_removed = 1; let support_location = DB::support_location(); let mut memdb = DB::default(); let mut child_tries: BTreeMap, ATrie> = Default::default(); @@ -902,7 +897,7 @@ fn child_trie_internal>() { let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let changeset = c.changeset.take().unwrap(); - memtrie.insert_with_child_changes(key, val, Some(Box::new(changeset))).unwrap(); + memtrie.insert_with_child_changes(key, val, Some(changeset)).unwrap(); } let changeset = memtrie.commit(); let root = changeset.commit_to(&mut memdb); @@ -910,17 +905,11 @@ fn child_trie_internal>() { main_trie.data = data; } else { let child_trie_root_key = data.iter().next().unwrap().0.clone(); - let mut changeset = memtrie.commit(); + let changeset = memtrie.commit(); let root = changeset.root_hash(); - match &mut changeset.root { - ChangesetNodeRef::New(node) => { - // needed for prefixed key. - node.key_childset = Some((child_trie_root_key.clone(), changeset.removed)); - }, - _ => unreachable!(), - } + let change_to_insert = changeset.to_insert_in_other_trie(child_trie_root_key.clone()); child_tries - .insert(child_trie_root_key, ATrie { root, data, changeset: Some(changeset.root) }); + .insert(child_trie_root_key, ATrie { root, data, changeset: Some(change_to_insert) }); } } // check data @@ -972,26 +961,18 @@ fn child_trie_internal>() { .build(); child_trie.remove(a_child_trie.data.iter().next().unwrap().0).unwrap(); child_trie.insert(b"make_sur_it_changes", b"value").unwrap(); - let mut changeset = child_trie.commit(); + let changeset = child_trie.commit(); let new_root = changeset.root_hash(); - match &mut changeset.root { - ChangesetNodeRef::New(node) => { - // needed for prefixed key. - // Note if unchanged we don't need this actually unchange should only - // be a thing in case of a copy. - node.key_childset = Some((root_key.clone(), changeset.removed)); - }, - _ => unreachable!(), - } + let change_to_insert = changeset.to_insert_in_other_trie(root_key.clone()); assert!(new_root != a_child_trie_root); - (changeset.root, new_root) + (change_to_insert, new_root) }; let mut main_trie = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); main_trie .insert_with_child_changes( root_key, child_root_hash.as_ref(), - Some(Box::new(child_changeset)), + Some(child_changeset), ) .unwrap(); let changeset = main_trie.commit(); From 04e7a1889773cecac2b767615f138362d4ec2574 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 18 Jan 2024 17:09:36 +0100 Subject: [PATCH 32/90] no_std --- trie-db/src/triedbmut.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index d77c5661..dafbf0e9 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -32,8 +32,6 @@ use hash_db::{HashDB, Hasher, Prefix}; #[cfg(feature = "std")] use std::collections::HashSet as Set; -#[cfg(not(feature = "std"))] -use alloc::boxed::Box; #[cfg(not(feature = "std"))] use alloc::collections::btree_set::BTreeSet as Set; @@ -2042,9 +2040,11 @@ where trace!(target: "trie", "Committing trie changes to db."); // always kill all the nodes on death row. + #[cfg(feature = "std")] trace!(target: "trie", "{:?} nodes to remove from db", self.death_row.len()); let mut removed = Vec::with_capacity(self.death_row.len()); + #[cfg(feature = "std")] for (hash, prefix) in self.death_row.drain() { removed.push((hash, prefix)); } From 956ed99d64624d6b2cc0069511e7fd6709f5f0e5 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 22 Jan 2024 17:36:27 +0100 Subject: [PATCH 33/90] fix --- trie-db/src/iterator.rs | 11 ++++++----- trie-db/src/triedb.rs | 8 -------- trie-db/test/src/double_ended_iterator.rs | 21 +++++++++++---------- trie-db/test/src/triedb.rs | 15 +++++++-------- 4 files changed, 24 insertions(+), 31 deletions(-) diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index a2461a5e..aeddf068 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -616,11 +616,11 @@ impl TrieDBRawIterator { /// extra nibble (prefix padding), and the node value. fn extract_key_from_raw_item<'a>( raw_item: Result< - (&NibbleVec, Option<&TrieHash>, &'a Arc>), + (&NibbleVec, Option<&TrieHash>, &'a Arc>), TrieHash, CError, >, - ) -> Option, Option, Value<'a>), TrieHash, CError>> { + ) -> Option, Option, Value<'a, L::Location>), TrieHash, CError>> { let (prefix, _, node) = match raw_item { Ok(raw_item) => raw_item, Err(err) => return Some(Err(err)), @@ -779,8 +779,9 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeDoubleEndedIterator<'a, 'cache, L> { &self, key: &[u8], prefix: Prefix, + location: L::Location, ) -> Result, CError> { - TrieDBRawIterator::fetch_value(self.db, key, prefix) + TrieDBRawIterator::fetch_value(self.db, key, prefix, location) } /// Advance the iterator into a prefix, no value out of the prefix will be accessed @@ -802,7 +803,7 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeDoubleEndedIterator<'a, 'cache, L> { } /// Access inner hash db. - pub fn db(&self) -> &dyn hash_db::HashDBRef { + pub fn db(&self) -> &dyn hash_db::HashDB { self.db.db() } } @@ -818,7 +819,7 @@ impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBNodeDoubleEndedIterato impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBNodeDoubleEndedIterator<'a, 'cache, L> { type Item = - Result<(NibbleVec, Option>, Arc>), TrieHash, CError>; + Result<(NibbleVec, Option>, Arc>), TrieHash, CError>; fn next(&mut self) -> Option { self.raw_iter.next_raw_item(self.db, true).map(|result| { diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index f81b5159..a934806e 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -13,7 +13,6 @@ // limitations under the License. use crate::{ - fatdb::FatDBDoubleEndedIterator, iterator::{TrieDBNodeDoubleEndedIterator, TrieDBRawIterator}, lookup::Lookup, nibble::NibbleSlice, @@ -175,13 +174,6 @@ where TrieDBKeyDoubleEndedIterator::new(&self) } - /// create `FatDBDoubleEndedIterator` from `TrieDB`. - pub fn into_fat_double_ended_iter( - &'db self, - ) -> Result, TrieHash, CError> { - FatDBDoubleEndedIterator::new(&self) - } - /// Given some node-describing data `node`, and node key return the actual node RLP. /// This could be a simple identity operation in the case that the node is sufficiently small, /// but may require a database lookup. diff --git a/trie-db/test/src/double_ended_iterator.rs b/trie-db/test/src/double_ended_iterator.rs index 59c06700..751b3641 100644 --- a/trie-db/test/src/double_ended_iterator.rs +++ b/trie-db/test/src/double_ended_iterator.rs @@ -21,16 +21,17 @@ use trie_db::{ }; use crate::iterator::{build_trie_db, nibble_vec}; +use crate::TestDB; test_layouts!(node_double_ended_iterator_works, node_double_ended_iterator); -fn node_double_ended_iterator() { +fn node_double_ended_iterator>() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db::(&pairs); + let (memdb, root) = build_trie_db::(&pairs); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); @@ -155,8 +156,8 @@ fn node_double_ended_iterator() { } test_layouts!(seek_back_over_empty_works, seek_back_over_empty_works_internal); -fn seek_back_over_empty_works_internal() { - let (memdb, root) = build_trie_db::(&[]); +fn seek_back_over_empty_works_internal>() { + let (memdb, root) = build_trie_db::(&[]); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); @@ -188,7 +189,7 @@ fn seek_back_over_empty_works_internal() { } test_layouts!(seek_back_works, seek_back_works_internal); -fn seek_back_works_internal() { +fn seek_back_works_internal>() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), @@ -196,7 +197,7 @@ fn seek_back_works_internal() { (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db::(&pairs); + let (memdb, root) = build_trie_db::(&pairs); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); @@ -279,7 +280,7 @@ fn seek_back_works_internal() { } test_layouts!(prefix_back_works, prefix_back_works_internal); -fn prefix_back_works_internal() { +fn prefix_back_works_internal>() { let can_expand = T::MAX_INLINE_VALUE.unwrap_or(T::Hash::LENGTH as u32) < T::Hash::LENGTH as u32; let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), @@ -288,7 +289,7 @@ fn prefix_back_works_internal() { (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db::(&pairs); + let (memdb, root) = build_trie_db::(&pairs); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); @@ -384,8 +385,8 @@ fn prefix_back_works_internal() { } test_layouts!(prefix_over_empty_works, prefix_over_empty_works_internal); -fn prefix_over_empty_works_internal() { - let (memdb, root) = build_trie_db::(&[]); +fn prefix_over_empty_works_internal>() { + let (memdb, root) = build_trie_db::(&[]); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); iter.prefix(&hex!("")[..]).unwrap(); diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index f868a86c..4efd34d1 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -93,7 +93,7 @@ fn iterator_seek_works_internal>() { } test_layouts!(double_ended_iterator, double_ended_iterator_internal); -fn double_ended_iterator_internal() { +fn double_ended_iterator_internal>() { let pairs = vec![ (hex!("01").to_vec(), hex!("01").to_vec()), (hex!("02").to_vec(), hex!("02").to_vec()), @@ -102,14 +102,13 @@ fn double_ended_iterator_internal() { (hex!("11").to_vec(), hex!("11").to_vec()), ]; - let mut memdb = MemoryDB::<::Hash, PrefixedKey<_>, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (x, y) in &pairs { - t.insert(x, y).unwrap(); - } + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (x, y) in &pairs { + t.insert(x, y).unwrap(); } + let commit = t.commit(); + let root = memdb.commit(commit); let t = TrieDBBuilder::::new(&memdb, &root).build(); assert_eq!(pairs, t.iter().unwrap().map(|x| x.unwrap()).collect::>()); From 37a05559887bf770642cf6f37aa389cbc17ead41 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 22 Jan 2024 17:38:46 +0100 Subject: [PATCH 34/90] crumb on layout --- trie-db/src/iterator.rs | 20 +++++++++++--------- trie-db/test/src/double_ended_iterator.rs | 6 ++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index aeddf068..22ec0f05 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -19,7 +19,7 @@ use crate::{ triedb::TrieDB, TrieDoubleEndedIterator, TrieError, TrieItem, TrieKeyItem, }; -use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; +use hash_db::{Prefix, EMPTY_PREFIX}; use crate::rstd::{boxed::Box, sync::Arc, vec::Vec}; @@ -33,16 +33,15 @@ enum Status { AftExiting, } -// TODO Crumb L: Layout #[cfg_attr(feature = "std", derive(Debug))] #[derive(Eq, PartialEq)] -struct Crumb { - hash: Option, - node: Arc>, +struct Crumb { + hash: Option>, + node: Arc>, status: Status, } -impl Crumb { +impl Crumb { /// Move on to the next status in the node's sequence in a direction. fn step(&mut self, fwd: bool) { self.status = match (self.status, self.node.node_plan()) { @@ -73,7 +72,7 @@ impl Crumb { /// Iterator for going through all nodes in the trie in pre-order traversal order. pub struct TrieDBRawIterator { /// Forward trail of nodes to visit. - trail: Vec>, + trail: Vec>, /// Forward iteration key nibbles of the current node. key_nibbles: NibbleVec, } @@ -818,8 +817,11 @@ impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBNodeDoubleEndedIterato } impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBNodeDoubleEndedIterator<'a, 'cache, L> { - type Item = - Result<(NibbleVec, Option>, Arc>), TrieHash, CError>; + type Item = Result< + (NibbleVec, Option>, Arc>), + TrieHash, + CError, + >; fn next(&mut self) -> Option { self.raw_iter.next_raw_item(self.db, true).map(|result| { diff --git a/trie-db/test/src/double_ended_iterator.rs b/trie-db/test/src/double_ended_iterator.rs index 751b3641..fd218469 100644 --- a/trie-db/test/src/double_ended_iterator.rs +++ b/trie-db/test/src/double_ended_iterator.rs @@ -20,8 +20,10 @@ use trie_db::{ TrieLayout, }; -use crate::iterator::{build_trie_db, nibble_vec}; -use crate::TestDB; +use crate::{ + iterator::{build_trie_db, nibble_vec}, + TestDB, +}; test_layouts!(node_double_ended_iterator_works, node_double_ended_iterator); fn node_double_ended_iterator>() { From ac8a09f4938c6cf564ee4d7fd8aca939abee7bbc Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 22 Jan 2024 18:16:27 +0100 Subject: [PATCH 35/90] less node decoding, but would need to put location directly in node plan --- trie-db/src/iterator.rs | 94 ++++++++++++++++++++++++++--------------- trie-db/src/node.rs | 85 ++++++++++++++++--------------------- 2 files changed, 97 insertions(+), 82 deletions(-) diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index 22ec0f05..35dcda56 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -168,21 +168,20 @@ impl TrieDBRawIterator { "descend pushes a crumb onto the trail; \ thus the trail is non-empty; qed", ); - // TODO remove this node calc (only when needed - let node = crumb.node.node(); - - match node { - //match crumb.node.node_plan() { - Node::Leaf(slice, _) => { - //Node::Leaf { partial: partial_plan, .. } => { - //let slice = partial_plan.build(node_data); + let node_data = crumb.node.data(); + let locations = crumb.node.locations(); + + match crumb.node.node_plan() { + NodePlan::Leaf { partial: partial_plan, .. } => { + let slice = partial_plan.build(node_data); if (fwd && slice < partial) || (!fwd && slice > partial) { crumb.status = Status::Exiting; return Ok(false); } return Ok(slice.starts_with(&partial)); }, - Node::Extension(slice, child) => { + NodePlan::Extension { partial: partial_plan, child } => { + let slice = partial_plan.build(node_data); if !partial.starts_with(&slice) { if (fwd && slice < partial) || (!fwd && slice > partial) { crumb.status = Status::Exiting; @@ -200,12 +199,12 @@ impl TrieDBRawIterator { let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - child, + child.build(node_data, locations.first().copied().unwrap_or_default()), prefix.left(), true, )? }, - Node::Branch(children, _value) => { + NodePlan::Branch { value, children } => { if partial.is_empty() { return Ok(true); } @@ -214,14 +213,23 @@ impl TrieDBRawIterator { crumb.status = Status::AtChild(i as usize); self.key_nibbles.push(i); - if let Some(child) = &children[i as usize] { + if children[i as usize].is_some() { + // TODO would make sense to put location in NodePlan: this is rather + // costy + let (_, children) = NodePlan::build_value_and_children( + value.as_ref(), + children, + node_data, + locations, + ); + full_key_nibbles += 1; partial = partial.mid(1); let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - *child, + children[i as usize].unwrap(), prefix.left(), true, )? @@ -229,7 +237,8 @@ impl TrieDBRawIterator { return Ok(false); } }, - Node::NibbledBranch(slice, children, _value) => { + NodePlan::NibbledBranch { partial: partial_plan, value, children } => { + let slice = partial_plan.build(node_data); if !partial.starts_with(&slice) { if (fwd && slice < partial) || (!fwd && slice > partial) { crumb.status = Status::Exiting; @@ -252,14 +261,23 @@ impl TrieDBRawIterator { self.key_nibbles.append_partial(slice.right()); self.key_nibbles.push(i); - if let Some(child) = &children[i as usize] { + if children[i as usize].is_some() { + // TODO would make sense to put location in NodePlan: this is rather + // costy + let (_, children) = NodePlan::build_value_and_children( + value.as_ref(), + children, + node_data, + locations, + ); + full_key_nibbles += 1; partial = partial.mid(1); let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - *child, + children[i as usize].unwrap(), prefix.left(), true, )? @@ -267,7 +285,7 @@ impl TrieDBRawIterator { return Ok(false); } }, - Node::Empty => { + NodePlan::Empty => { if !partial.is_empty() { crumb.status = Status::Exiting; return Ok(false); @@ -386,11 +404,10 @@ impl TrieDBRawIterator { > { loop { let crumb = self.trail.last_mut()?; - // TODO restore on nodeplan - let node = crumb.node.node(); + let node_data = crumb.node.data(); + let locations = crumb.node.locations(); - //match (crumb.status, crumb.node.node_plan()) { - match (crumb.status, node) { + match (crumb.status, crumb.node.node_plan()) { (Status::Entering, _) => if fwd { let crumb = self.trail.last_mut().expect("we've just fetched the last element using `last_mut` so this cannot fail; qed"); @@ -405,14 +422,14 @@ impl TrieDBRawIterator { }, (Status::Exiting, node) => { match node { - Node::Empty | Node::Leaf { .. } => {}, - Node::Extension(partial, ..) => { + NodePlan::Empty | NodePlan::Leaf { .. } => {}, + NodePlan::Extension { partial, .. } => { self.key_nibbles.drop_lasts(partial.len()); }, - Node::Branch { .. } => { + NodePlan::Branch { .. } => { self.key_nibbles.pop(); }, - Node::NibbledBranch(partial, ..) => { + NodePlan::NibbledBranch { partial, .. } => { self.key_nibbles.drop_lasts(partial.len() + 1); }, } @@ -422,12 +439,13 @@ impl TrieDBRawIterator { return Some(Ok((&self.key_nibbles, crumb.hash.as_ref(), &crumb.node))); } }, - (Status::At, Node::Extension(partial, child)) => { + (Status::At, NodePlan::Extension { partial: partial_plan, child }) => { + let partial = partial_plan.build(node_data); self.key_nibbles.append_partial(partial.right()); match db.get_raw_or_lookup( crumb.hash.unwrap_or_default(), - child, + child.build(node_data, locations.first().copied().unwrap_or_default()), self.key_nibbles.as_prefix(), true, ) { @@ -440,8 +458,7 @@ impl TrieDBRawIterator { }, } }, - (Status::At, Node::Branch { .. }) => { - //(Status::At, NodePlan::Branch { .. }) => { + (Status::At, NodePlan::Branch { .. }) => { self.key_nibbles.push(if fwd { 0 } else { @@ -449,7 +466,8 @@ impl TrieDBRawIterator { }); crumb.step(fwd); }, - (Status::At, Node::NibbledBranch(partial, ..)) => { + (Status::At, NodePlan::NibbledBranch { partial: partial_plan, .. }) => { + let partial = partial_plan.build(node_data); self.key_nibbles.append_partial(partial.right()); self.key_nibbles.push(if fwd { 0 @@ -458,15 +476,23 @@ impl TrieDBRawIterator { }); crumb.step(fwd); }, - (Status::AtChild(i), Node::Branch(children, ..)) | - (Status::AtChild(i), Node::NibbledBranch(_, children, ..)) => { - if let Some(child) = &children[i] { + (Status::AtChild(i), NodePlan::Branch { value, children, .. }) | + (Status::AtChild(i), NodePlan::NibbledBranch { value, children, .. }) => { + if children[i].is_some() { + // TODO would make sense to put location in NodePlan: this is rather costy + let (_, children) = NodePlan::build_value_and_children( + value.as_ref(), + children, + node_data, + locations, + ); + self.key_nibbles.pop(); self.key_nibbles.push(i as u8); match db.get_raw_or_lookup( crumb.hash.unwrap_or_default(), - *child, + children[i].unwrap(), self.key_nibbles.as_prefix(), true, ) { diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index f8c8a3c0..a8eca9f9 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -572,62 +572,51 @@ impl NodePlan { child.build(data, locations.first().copied().unwrap_or_default()), ), NodePlan::Branch { value, children } => { - let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; - let mut nc = 0; - let value = if let Some(v) = value { - if v.is_inline() { - Some(v.build(data, Default::default())) - } else { - nc += 1; - Some(v.build(data, locations.first().copied().unwrap_or_default())) - } - } else { - None - }; - for i in 0..nibble_ops::NIBBLE_LENGTH { - if let Some(child) = &children[i] { - let location = if child.is_inline() { - Default::default() - } else { - let l = locations.get(nc).copied().unwrap_or_default(); - nc += 1; - l - }; - child_slices[i] = Some(child.build(data, location)); - } - } + let (value, child_slices) = + Self::build_value_and_children(value.as_ref(), children, data, locations); Node::Branch(child_slices, value) }, NodePlan::NibbledBranch { partial, value, children } => { - let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; - let mut nc = 0; - let value = if let Some(v) = value { - if v.is_inline() { - Some(v.build(data, Default::default())) - } else { - nc += 1; - Some(v.build(data, locations.first().copied().unwrap_or_default())) - } - } else { - None - }; - for i in 0..nibble_ops::NIBBLE_LENGTH { - if let Some(child) = &children[i] { - let location = if child.is_inline() { - Default::default() - } else { - let l = locations.get(nc).copied().unwrap_or_default(); - nc += 1; - l - }; - child_slices[i] = Some(child.build(data, location)); - } - } + let (value, child_slices) = + Self::build_value_and_children(value.as_ref(), children, data, locations); Node::NibbledBranch(partial.build(data), child_slices, value) }, } } + pub(crate) fn build_value_and_children<'a, 'b, L: Copy + Default>( + value: Option<&'a ValuePlan>, + children: &'a [Option; nibble_ops::NIBBLE_LENGTH], + data: &'b [u8], + locations: &[L], + ) -> (Option>, [Option>; nibble_ops::NIBBLE_LENGTH]) { + let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; + let mut nc = 0; + let value = if let Some(v) = value { + if v.is_inline() { + Some(v.build(data, Default::default())) + } else { + nc += 1; + Some(v.build(data, locations.first().copied().unwrap_or_default())) + } + } else { + None + }; + for i in 0..nibble_ops::NIBBLE_LENGTH { + if let Some(child) = &children[i] { + let location = if child.is_inline() { + Default::default() + } else { + let l = locations.get(nc).copied().unwrap_or_default(); + nc += 1; + l + }; + child_slices[i] = Some(child.build(data, location)); + } + } + (value, child_slices) + } + /// Access value plan from node plan, return `None` for /// node that cannot contain a `ValuePlan`. pub fn value_plan(&self) -> Option<&ValuePlan> { From bc76a212bf9f19f89a17710315c32fa7d9a53463 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 11:46:46 +0100 Subject: [PATCH 36/90] minor changes --- test-support/mem-tree-db/Cargo.toml | 2 -- test-support/mem-tree-db/src/lib.rs | 5 +++-- test-support/reference-trie/src/lib.rs | 20 +++++++------------- trie-db/src/node.rs | 1 + 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/test-support/mem-tree-db/Cargo.toml b/test-support/mem-tree-db/Cargo.toml index 08f8e5b0..23e4f0aa 100644 --- a/test-support/mem-tree-db/Cargo.toml +++ b/test-support/mem-tree-db/Cargo.toml @@ -13,5 +13,3 @@ keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } trie-db = { path = "../../trie-db", version = "0.28.0" } trie-root = { path = "../../trie-root", version = "0.18.0" } parity-scale-codec = { version = "3.0.0", features = ["derive"] } - -[dev-dependencies] \ No newline at end of file diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs index 7f0b424d..b51ff9d7 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/test-support/mem-tree-db/src/lib.rs @@ -84,9 +84,10 @@ where } pub fn remove_node(&mut self, k: &H::Out) { - self.roots.remove(k); - for node in self.nodes.iter_mut() { + let rem_root = self.roots.remove(k); + for (i, node) in self.nodes.iter_mut().enumerate() { if matches!(node, NodeEntry::Live { key, .. } if key == k) { + debug_assert!(rem_root.map(|r| r == i).unwrap_or(false)); *node = NodeEntry::Removed; } } diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index b1f6424d..7a511f30 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -61,6 +61,13 @@ macro_rules! test_layouts { $crate::HashedValueNoExtThreshold<1, $crate::MemLocation>, $crate::MemTreeDB<$crate::RefHasher>, >(); + eprintln!("Running with layout `HashedValueNoExt` and MemTreeDB"); + $test_internal::<$crate::HashedValueNoExt, $crate::MemTreeDB<$crate::RefHasher>>(); + eprintln!("Running with layout `NoExtensionLayout` and MemTreeDB"); + $test_internal::< + $crate::GenericNoExtensionLayout<$crate::RefHasher, $crate::MemLocation>, + $crate::MemTreeDB<$crate::RefHasher>, + >(); eprintln!("Running with layout `HashedValueNoExtThreshold` and MemoryDB"); $test_internal::< @@ -82,19 +89,6 @@ macro_rules! test_layouts { $crate::ExtensionLayout, $crate::PrefixedMemoryDB<$crate::ExtensionLayout>, >(); - - eprintln!("Running with layout `HashedValueNoExtThreshold` and MemTreeDB"); - $test_internal::< - $crate::HashedValueNoExtThreshold<1, $crate::MemLocation>, - $crate::MemTreeDB<$crate::RefHasher>, - >(); - eprintln!("Running with layout `HashedValueNoExt` and MemTreeDB"); - $test_internal::<$crate::HashedValueNoExt, $crate::MemTreeDB<$crate::RefHasher>>(); - eprintln!("Running with layout `NoExtensionLayout` and MemTreeDB"); - $test_internal::< - $crate::GenericNoExtensionLayout<$crate::RefHasher, $crate::MemLocation>, - $crate::MemTreeDB<$crate::RefHasher>, - >(); } }; } diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index a8eca9f9..03438988 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -699,6 +699,7 @@ impl, L: Default + Copy> OwnedNode { pub fn locations(&self) -> &[L] { &self.locations } + /// Returns a reference to the node decode plan. pub fn node_plan(&self) -> &NodePlan { &self.plan From 5df1cd8aacc477379858d1e5e2f316ce66ffdc1f Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 12:13:57 +0100 Subject: [PATCH 37/90] fix --- test-support/mem-tree-db/src/lib.rs | 19 +++++++++++++++++-- trie-db/test/src/lib.rs | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs index b51ff9d7..d68005f5 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/test-support/mem-tree-db/src/lib.rs @@ -85,9 +85,24 @@ where pub fn remove_node(&mut self, k: &H::Out) { let rem_root = self.roots.remove(k); - for (i, node) in self.nodes.iter_mut().enumerate() { + #[cfg(debug_assertions)] + { + for (i, node) in self.nodes.iter_mut().enumerate() { + if matches!(node, NodeEntry::Live { key, .. } if key == k) { + assert!(rem_root.map(|r| r == i).unwrap_or(false)); + *node = NodeEntry::Removed; + } + } + } + if let Some(rem_root_ix) = rem_root { + self.nodes[rem_root_ix] = NodeEntry::Removed; + } + } + + pub fn test_remove_node(&mut self, k: &H::Out) { + self.roots.remove(k); + for node in self.nodes.iter_mut() { if matches!(node, NodeEntry::Live { key, .. } if key == k) { - debug_assert!(rem_root.map(|r| r == i).unwrap_or(false)); *node = NodeEntry::Removed; } } diff --git a/trie-db/test/src/lib.rs b/trie-db/test/src/lib.rs index 9fd193ae..6472cd1a 100644 --- a/trie-db/test/src/lib.rs +++ b/trie-db/test/src/lib.rs @@ -81,7 +81,7 @@ where } fn remove(&mut self, hash: &H::Out, _prefix: Prefix) { - MemTreeDB::remove_node(self, hash); + MemTreeDB::test_remove_node(self, hash); } fn is_empty(&self) -> bool { From aeec4fa0877332dd19f75aed1e72965dd6125f51 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 14:31:33 +0100 Subject: [PATCH 38/90] avoid using child trie name --- trie-db/src/triedbmut.rs | 3 ++ trie-db/test/src/triedbmut.rs | 72 +++++++++++++++++------------------ 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index dafbf0e9..06841c20 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -2147,6 +2147,9 @@ where /// Cache the given `value`. /// /// `hash` is the hash of `value`. + /// + /// Cache is not done here as we want to cache the location from the db, + /// and location on new_nodes are not resolved here. fn cache_value(&mut self, _full_key: &[u8], _value: impl Into, _hash: TrieHash) {} /// Commit a node by hashing it and writing it to the db. Returns a diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 100c0302..ce1f50e5 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -856,8 +856,8 @@ fn test_two_assets_memory_db_inner_2() { assert_eq!(state.get(key3.as_ref()).unwrap().unwrap(), data3); } -test_layouts!(child_trie, child_trie_internal); -fn child_trie_internal>() { +test_layouts!(attached_trie, attached_trie_internal); +fn attached_trie_internal>() { use std::collections::BTreeMap; struct ATrie { root: TrieHash, @@ -870,15 +870,15 @@ fn child_trie_internal>() { // Direct copy if using ref counting and location in db. // Pruning. let mut seed = Default::default(); - let nb_child_trie = 10; - // let nb_child_trie = 1; + let nb_attached_trie = 10; + // let nb_attached_trie = 1; let support_location = DB::support_location(); let mut memdb = DB::default(); - let mut child_tries: BTreeMap, ATrie> = Default::default(); + let mut attached_tries: BTreeMap, ATrie> = Default::default(); let mut keyspaced_memdb; let mut main_trie: ATrie = ATrie { root: Default::default(), data: Default::default(), changeset: None }; - for i in 0..nb_child_trie + 1 { + for i in 0..nb_attached_trie + 1 { let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), min_key: 3, @@ -891,8 +891,8 @@ fn child_trie_internal>() { let mut memtrie = populate_trie::(&mut memdb, &x); let data: BTreeMap, Vec> = x.iter().cloned().collect(); - if i == nb_child_trie { - for (k, c) in child_tries.iter_mut() { + if i == nb_attached_trie { + for (k, c) in attached_tries.iter_mut() { let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let changeset = c.changeset.take().unwrap(); @@ -903,12 +903,12 @@ fn child_trie_internal>() { main_trie.root = root; main_trie.data = data; } else { - let child_trie_root_key = data.iter().next().unwrap().0.clone(); + let attached_trie_root_key = data.iter().next().unwrap().0.clone(); let changeset = memtrie.commit(); let root = changeset.root_hash(); - let change_to_insert = changeset.to_insert_in_other_trie(child_trie_root_key.clone()); - child_tries.insert( - child_trie_root_key, + let change_to_insert = changeset.to_insert_in_other_trie(attached_trie_root_key.clone()); + attached_tries.insert( + attached_trie_root_key, ATrie { root, data, changeset: Some(change_to_insert) }, ); } @@ -920,52 +920,52 @@ fn child_trie_internal>() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } - for (root_key, child_trie) in &child_tries { - let (child_trie_root, child_trie_location) = - child_trie_root(&memdb, &main_trie.root, root_key).unwrap(); + for (root_key, attached_trie) in &attached_tries { + let (attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); let child_memdb: &dyn HashDB<_, _, _> = if support_location { &memdb } else { - assert!(child_trie_location.is_none()); + assert!(attached_trie_location.is_none()); keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); &keyspaced_memdb }; let trie = TrieDBBuilder::::new_with_db_location( child_memdb, - &child_trie_root, - child_trie_location.unwrap_or_default(), + &attached_trie_root, + attached_trie_location.unwrap_or_default(), ) .build(); - for (k, v) in child_trie.data.iter() { + for (k, v) in attached_trie.data.iter() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } // Modifying an existing child trie. - let (root_key, a_child_trie) = child_tries.iter().next().unwrap(); - let (a_child_trie_root, child_trie_location) = - child_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); + let (root_key, a_attached_trie) = attached_tries.iter().next().unwrap(); + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); let (child_changeset, child_root_hash) = { - assert_eq!(a_child_trie_root, a_child_trie.root); + assert_eq!(a_attached_trie_root, a_attached_trie.root); let child_memdb: &dyn HashDB<_, _, _> = if support_location { &memdb } else { keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); &keyspaced_memdb }; - let mut child_trie = TrieDBMutBuilder::::from_existing_with_db_location( + let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( child_memdb, - a_child_trie_root, - child_trie_location.unwrap_or_default(), + a_attached_trie_root, + attached_trie_location.unwrap_or_default(), ) .build(); - child_trie.remove(a_child_trie.data.iter().next().unwrap().0).unwrap(); - child_trie.insert(b"make_sur_it_changes", b"value").unwrap(); - let changeset = child_trie.commit(); + attached_trie.remove(a_attached_trie.data.iter().next().unwrap().0).unwrap(); + attached_trie.insert(b"make_sur_it_changes", b"value").unwrap(); + let changeset = attached_trie.commit(); let new_root = changeset.root_hash(); let change_to_insert = changeset.to_insert_in_other_trie(root_key.clone()); - assert!(new_root != a_child_trie_root); + assert!(new_root != a_attached_trie_root); (change_to_insert, new_root) }; let mut main_trie = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); @@ -976,8 +976,8 @@ fn child_trie_internal>() { let main_root = changeset.root_hash(); changeset.commit_to(&mut memdb); // checking modification - let (a_child_trie_root, child_trie_location) = - child_trie_root(&memdb, &main_root, root_key).unwrap(); + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_root, root_key).unwrap(); let child_memdb: &dyn HashDB<_, _, _> = if support_location { &memdb } else { @@ -986,13 +986,13 @@ fn child_trie_internal>() { }; let trie = TrieDBBuilder::::new_with_db_location( child_memdb, - &a_child_trie_root, - child_trie_location.unwrap_or_default(), + &a_attached_trie_root, + attached_trie_location.unwrap_or_default(), ) .build(); trie.get(b"make_sur_it_changes").unwrap().unwrap(); let mut first = true; - for (k, v) in a_child_trie.data.iter() { + for (k, v) in a_attached_trie.data.iter() { if first { assert!(&trie.get(k).unwrap().is_none()); first = false; @@ -1003,7 +1003,7 @@ fn child_trie_internal>() { trie.get(b"make_sur_it_changes").unwrap().unwrap(); } -fn child_trie_root>( +fn attached_trie_root>( memdb: &DB, main_root: &TrieHash, root_key: &[u8], From f0d36620d270be7a16d833d4dfdd101949356e9c Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 14:53:42 +0100 Subject: [PATCH 39/90] more renamings --- trie-db/src/node.rs | 9 +- trie-db/src/triedbmut.rs | 195 +++++++++++++++------------------- trie-db/test/src/triedbmut.rs | 11 +- 3 files changed, 95 insertions(+), 120 deletions(-) diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index 03438988..ab36a222 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -640,12 +640,12 @@ impl NodePlan { } /// Check if the node has a location for value. - pub fn has_location_value(&self) -> bool { + pub fn has_location_for_value(&self) -> bool { self.value_plan().map(|v| !v.is_inline()).unwrap_or(false) } /// Check how many children location value node has. - pub fn nb_location_children(&self) -> usize { + pub fn num_children_locations(&self) -> usize { match self { NodePlan::Extension { child: NodeHandlePlan::Hash(_), .. } => 1, NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { @@ -661,8 +661,9 @@ impl NodePlan { } } - pub fn attached_change_set_location(&self, locations: &[L]) -> Option { - let offset = if self.has_location_value() { 1 } else { 0 } + self.nb_location_children(); + pub fn additional_ref_location(&self, locations: &[L]) -> Option { + let offset = + if self.has_location_for_value() { 1 } else { 0 } + self.num_children_locations(); if locations.len() > offset { // only one additional location expected with current code. debug_assert!(locations.len() == offset + 1); diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 06841c20..611e41bf 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -69,7 +69,7 @@ fn empty_children() -> Box<[Option>; nibble_ops::NIBBLE_L ]) } -pub type ChildChangeset = Box, ::Location>>; +pub type TreeRefChangeset = Box, ::Location>>; /// Type alias to indicate the nible covers a full key, /// therefore its left side is a full prefix. @@ -221,7 +221,7 @@ enum Node { /// A leaf node contains the end of a key and a value. /// This key is encoded from a `NibbleSlice`, meaning it contains /// a flag indicating it is a leaf. - Leaf(NodeKey, Value, Option>), + Leaf(NodeKey, Value, Option>), /// An extension contains a shared portion of a key and a child node. /// The shared portion is encoded from a `NibbleSlice` meaning it contains /// a flag indicating it is an extension. @@ -231,14 +231,14 @@ enum Node { Branch( Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, Option>, - Option>, + Option>, ), /// Branch node with support for a nibble (to avoid extension node). NibbledBranch( NodeKey, Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, Option>, - Option>, + Option>, ), } @@ -466,7 +466,7 @@ impl Node { // TODO: parallelize /// Here `child_cb` should process the first parameter to either insert an external /// node value or to encode and add a new branch child node. - fn into_encoded(self, mut child_cb: F) -> (Vec, Option>) + fn into_encoded(self, mut child_cb: F) -> (Vec, Option>) where F: FnMut( NodeToEncode, L::Location>, @@ -476,10 +476,10 @@ impl Node { { match self { Node::Empty => (L::Codec::empty_node().to_vec(), None), - Node::Leaf(partial, mut value, child_changes) => { + Node::Leaf(partial, mut value, tree_ref) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let value = value.into_encoded::(Some(&pr), &mut child_cb); - (L::Codec::leaf_node(pr.right_iter(), pr.len(), value), child_changes) + (L::Codec::leaf_node(pr.right_iter(), pr.len(), value), tree_ref) }, Node::Extension(partial, child) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); @@ -487,7 +487,7 @@ impl Node { let c = child_cb(NodeToEncode::TrieNode(child), Some(&pr), None); (L::Codec::extension_node(it, pr.len(), c), None) }, - Node::Branch(mut children, mut value, child_changes) => { + Node::Branch(mut children, mut value, tree_ref) => { let value = value.as_mut().map(|v| v.into_encoded::(None, &mut child_cb)); ( L::Codec::branch_node( @@ -501,10 +501,10 @@ impl Node { ), value, ), - child_changes, + tree_ref, ) }, - Node::NibbledBranch(partial, mut children, mut value, child_changes) => { + Node::NibbledBranch(partial, mut children, mut value, tree_ref) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let value = value.as_mut().map(|v| v.into_encoded::(Some(&pr), &mut child_cb)); let it = pr.right_iter(); @@ -528,7 +528,7 @@ impl Node { ), value, ), - child_changes, + tree_ref, ) }, } @@ -542,7 +542,7 @@ enum Action { // Restore the original node. This trusts that the node is actually the original. Restore(Node), // if it is a new node, just clears the storage. - Delete(Option>), + Delete(Option>), } // post-insert action. Same as action without delete @@ -909,7 +909,7 @@ where root: TrieHash, root_handle: NodeHandle, L::Location>, death_row: Set<(TrieHash, OwnedPrefix)>, - death_row_child: Vec>, + death_row_child: Vec>, /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. hash_count: usize, @@ -987,8 +987,8 @@ where Stored::New(node) => match inspector(self, node, key)? { Action::Restore(node) => Some((Stored::New(node), false)), Action::Replace(node) => Some((Stored::New(node), true)), - Action::Delete(child_changes) => { - child_changes.map(|c| self.death_row_child.push(c)); + Action::Delete(tree_ref) => { + tree_ref.map(|c| self.death_row_child.push(c)); None }, }, @@ -998,9 +998,9 @@ where self.death_row.insert((hash, current_key.left_owned())); Some((Stored::New(node), true)) }, - Action::Delete(child_changes) => { + Action::Delete(tree_ref) => { self.death_row.insert((hash, current_key.left_owned())); - child_changes.map(|c| self.death_row_child.push(c)); + tree_ref.map(|c| self.death_row_child.push(c)); None }, }, @@ -1112,7 +1112,7 @@ where key: &mut NibbleFullKey, value: Bytes, old_val: &mut Option>, - child_changes: Option>, + tree_ref: Option>, ) -> Result<(StorageHandle, bool), TrieHash, CError> { let h = match handle { NodeHandle::InMemory(h) => h, @@ -1122,7 +1122,7 @@ where let stored = self.storage.destroy(h); let (new_stored, changed) = self .inspect(stored, key, move |trie, stored, key| { - trie.insert_inspector(stored, key, value, old_val, child_changes) + trie.insert_inspector(stored, key, value, old_val, tree_ref) .map(|a| a.into_action()) })? .expect("Insertion never deletes."); @@ -1156,7 +1156,7 @@ where key: &mut NibbleFullKey, value: Bytes, old_val: &mut Option>, - child_changes: Option>, + tree_ref: Option>, ) -> Result, TrieHash, CError> { let partial = *key; @@ -1168,9 +1168,9 @@ where #[cfg(feature = "std")] trace!(target: "trie", "empty: COMPOSE"); let value = Value::new(value, L::MAX_INLINE_VALUE); - InsertAction::Replace(Node::Leaf(partial.to_stored(), value, child_changes)) + InsertAction::Replace(Node::Leaf(partial.to_stored(), value, tree_ref)) }, - Node::Branch(mut children, stored_value, b_child_changes) => { + Node::Branch(mut children, stored_value, b_tree_ref) => { debug_assert!(L::USE_EXTENSION); #[cfg(feature = "std")] trace!(target: "trie", "branch: ROUTE,AUGMENT"); @@ -1178,7 +1178,7 @@ where if partial.is_empty() { let value = Some(Value::new(value, L::MAX_INLINE_VALUE)); let unchanged = stored_value == value; - let branch = Node::Branch(children, value, child_changes); + let branch = Node::Branch(children, value, tree_ref); self.replace_old_value(old_val, stored_value, key.left()); @@ -1193,7 +1193,7 @@ where if let Some(child) = children[idx].take() { // Original had something there. recurse down into it. let (new_child, changed) = - self.insert_at(child, key, value, old_val, child_changes)?; + self.insert_at(child, key, value, old_val, tree_ref)?; children[idx] = Some(new_child.into()); if !changed { // The new node we composed didn't change. @@ -1201,7 +1201,7 @@ where return Ok(InsertAction::Restore(Node::Branch( children, stored_value, - b_child_changes, + b_tree_ref, ))) } } else { @@ -1210,15 +1210,15 @@ where let leaf = self.storage.alloc(Stored::New(Node::Leaf( key.to_stored(), value, - child_changes, + tree_ref, ))); children[idx] = Some(leaf.into()); } - InsertAction::Replace(Node::Branch(children, stored_value, b_child_changes)) + InsertAction::Replace(Node::Branch(children, stored_value, b_tree_ref)) } }, - Node::NibbledBranch(encoded, mut children, stored_value, b_child_changes) => { + Node::NibbledBranch(encoded, mut children, stored_value, b_tree_ref) => { debug_assert!(!L::USE_EXTENSION); #[cfg(feature = "std")] trace!(target: "trie", "branch: ROUTE,AUGMENT"); @@ -1228,12 +1228,8 @@ where if common == existing_key.len() && common == partial.len() { let value = Some(Value::new(value, L::MAX_INLINE_VALUE)); let unchanged = stored_value == value; - let branch = Node::NibbledBranch( - existing_key.to_stored(), - children, - value, - child_changes, - ); + let branch = + Node::NibbledBranch(existing_key.to_stored(), children, value, tree_ref); let mut key_val = key.clone(); key_val.advance(existing_key.len()); @@ -1256,12 +1252,8 @@ where common, ); let nbranch_partial = existing_key.mid(common + 1).to_stored(); - let low = Node::NibbledBranch( - nbranch_partial, - children, - stored_value, - b_child_changes, - ); + let low = + Node::NibbledBranch(nbranch_partial, children, stored_value, b_tree_ref); let ix = existing_key.at(common); let mut children = empty_children(); let alloc_storage = self.storage.alloc(Stored::New(low)); @@ -1274,12 +1266,12 @@ where existing_key.to_stored_range(common), children, Some(value), - child_changes, + tree_ref, )) } else { let ix = partial.at(common); let stored_leaf = - Node::Leaf(partial.mid(common + 1).to_stored(), value, child_changes); + Node::Leaf(partial.mid(common + 1).to_stored(), value, tree_ref); let leaf = self.storage.alloc(Stored::New(stored_leaf)); @@ -1300,7 +1292,7 @@ where if let Some(child) = children[idx].take() { // Original had something there. recurse down into it. let (new_child, changed) = - self.insert_at(child, key, value, old_val, child_changes)?; + self.insert_at(child, key, value, old_val, tree_ref)?; children[idx] = Some(new_child.into()); if !changed { // The new node we composed didn't change. @@ -1309,7 +1301,7 @@ where existing_key.to_stored(), children, stored_value, - b_child_changes, + b_tree_ref, ); return Ok(InsertAction::Restore(n_branch)) } @@ -1319,7 +1311,7 @@ where let leaf = self.storage.alloc(Stored::New(Node::Leaf( key.to_stored(), value, - child_changes, + tree_ref, ))); children[idx] = Some(leaf.into()); @@ -1328,11 +1320,11 @@ where existing_key.to_stored(), children, stored_value, - b_child_changes, + b_tree_ref, )) } }, - Node::Leaf(encoded, stored_value, l_child_changes) => { + Node::Leaf(encoded, stored_value, l_tree_ref) => { let existing_key = NibbleSlice::from_stored(&encoded); let common = partial.common_prefix(&existing_key); if common == existing_key.len() && common == partial.len() { @@ -1346,9 +1338,9 @@ where self.replace_old_value(old_val, Some(stored_value), key_val.left()); if unchanged { // unchanged. restore - InsertAction::Restore(Node::Leaf(encoded.clone(), value, l_child_changes)) + InsertAction::Restore(Node::Leaf(encoded.clone(), value, l_tree_ref)) } else { - InsertAction::Replace(Node::Leaf(encoded.clone(), value, child_changes)) + InsertAction::Replace(Node::Leaf(encoded.clone(), value, tree_ref)) } } else if (L::USE_EXTENSION && common == 0) || (!L::USE_EXTENSION && common < existing_key.len()) @@ -1366,13 +1358,13 @@ where let mut children = empty_children(); let branch = if L::USE_EXTENSION && existing_key.is_empty() { // always replace since branch isn't leaf. - Node::Branch(children, Some(stored_value), l_child_changes) + Node::Branch(children, Some(stored_value), l_tree_ref) } else { let idx = existing_key.at(common) as usize; let new_leaf = Node::Leaf( existing_key.mid(common + 1).to_stored(), stored_value, - l_child_changes, + l_tree_ref, ); children[idx] = Some(self.storage.alloc(Stored::New(new_leaf)).into()); @@ -1390,9 +1382,8 @@ where // always replace because whatever we get out here // is not the branch we started with. - let branch_action = self - .insert_inspector(branch, key, value, old_val, child_changes)? - .unwrap_node(); + let branch_action = + self.insert_inspector(branch, key, value, old_val, tree_ref)?.unwrap_node(); InsertAction::Replace(branch_action) } else if !L::USE_EXTENSION { #[cfg(feature = "std")] @@ -1404,12 +1395,11 @@ where existing_key.to_stored(), empty_children(), Some(stored_value), - l_child_changes, + l_tree_ref, ); // augment the new branch. - let branch = self - .insert_inspector(branch, key, value, old_val, child_changes)? - .unwrap_node(); + let branch = + self.insert_inspector(branch, key, value, old_val, tree_ref)?.unwrap_node(); InsertAction::Replace(branch) } else if common == existing_key.len() { @@ -1419,13 +1409,11 @@ where // fully-shared prefix for an extension. // make a stub branch and an extension. - let branch = - Node::Branch(empty_children(), Some(stored_value), l_child_changes); + let branch = Node::Branch(empty_children(), Some(stored_value), l_tree_ref); // augment the new branch. key.advance(common); - let branch = self - .insert_inspector(branch, key, value, old_val, child_changes)? - .unwrap_node(); + let branch = + self.insert_inspector(branch, key, value, old_val, tree_ref)?.unwrap_node(); // always replace since we took a leaf and made an extension. let leaf = self.storage.alloc(Stored::New(branch)); @@ -1444,18 +1432,14 @@ where // partially-shared prefix for an extension. // start by making a leaf. - let low = Node::Leaf( - existing_key.mid(common).to_stored(), - stored_value, - l_child_changes, - ); + let low = + Node::Leaf(existing_key.mid(common).to_stored(), stored_value, l_tree_ref); // augment it. this will result in the Leaf -> common == 0 routine, // which creates a branch. key.advance(common); - let augmented_low = self - .insert_inspector(low, key, value, old_val, child_changes)? - .unwrap_node(); + let augmented_low = + self.insert_inspector(low, key, value, old_val, tree_ref)?.unwrap_node(); // make an extension using it. this is a replacement. InsertAction::Replace(Node::Extension( existing_key.to_stored_range(common), @@ -1500,7 +1484,7 @@ where key, value, old_val, - child_changes, + tree_ref, )? .unwrap_node(); InsertAction::Replace(branch_action) @@ -1513,7 +1497,7 @@ where // insert into the child node. key.advance(common); let (new_child, changed) = - self.insert_at(child_branch, key, value, old_val, child_changes)?; + self.insert_at(child_branch, key, value, old_val, tree_ref)?; let new_ext = Node::Extension(existing_key.to_stored(), new_child.into()); @@ -1539,9 +1523,8 @@ where // augment the extension. this will take the common == 0 path, // creating a branch. key.advance(common); - let augmented_low = self - .insert_inspector(low, key, value, old_val, child_changes)? - .unwrap_node(); + let augmented_low = + self.insert_inspector(low, key, value, old_val, tree_ref)?.unwrap_node(); // always replace, since this extension is not the one we started with. // this is known because the partial key is only the common prefix. @@ -1560,7 +1543,7 @@ where handle: NodeHandle, L::Location>, key: &mut NibbleFullKey, old_val: &mut Option>, - child_changes: Option>, + tree_ref: Option>, ) -> Result, TrieHash, CError> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), @@ -1571,7 +1554,7 @@ where }; let opt = self.inspect(stored, key, move |trie, node, key| { - trie.remove_inspector(node, key, old_val, child_changes) + trie.remove_inspector(node, key, old_val, tree_ref) })?; Ok(opt.map(|(new, changed)| (self.storage.alloc(new), changed))) @@ -1583,26 +1566,23 @@ where node: Node, key: &mut NibbleFullKey, old_val: &mut Option>, - child_changes: Option>, + tree_ref: Option>, ) -> Result, TrieHash, CError> { let partial = *key; Ok(match (node, partial.is_empty()) { (Node::Empty, _) => Action::Delete(None), - (Node::Branch(c, None, _), true) => - Action::Restore(Node::Branch(c, None, child_changes)), + (Node::Branch(c, None, _), true) => Action::Restore(Node::Branch(c, None, tree_ref)), (Node::NibbledBranch(n, c, None, _), true) => - Action::Restore(Node::NibbledBranch(n, c, None, child_changes)), + Action::Restore(Node::NibbledBranch(n, c, None, tree_ref)), (Node::Branch(children, val, _), true) => { self.replace_old_value(old_val, val, key.left()); // always replace since we took the value out. - Action::Replace(self.fix(Node::Branch(children, None, child_changes), *key)?) + Action::Replace(self.fix(Node::Branch(children, None, tree_ref), *key)?) }, (Node::NibbledBranch(n, children, val, _), true) => { self.replace_old_value(old_val, val, key.left()); // always replace since we took the value out. - Action::Replace( - self.fix(Node::NibbledBranch(n, children, None, child_changes), *key)?, - ) + Action::Replace(self.fix(Node::NibbledBranch(n, children, None, tree_ref), *key)?) }, (Node::Branch(mut children, value, b_childset), false) => { let idx = partial.at(0) as usize; @@ -1615,7 +1595,7 @@ where ); let prefix = *key; key.advance(1); - match self.remove_at(child, key, old_val, child_changes)? { + match self.remove_at(child, key, old_val, tree_ref)? { Some((new, changed)) => { children[idx] = Some(new.into()); let branch = Node::Branch(children, value, b_childset); @@ -1652,11 +1632,11 @@ where let mut key_val = key.clone(); key_val.advance(existing_length); self.replace_old_value(old_val, Some(value), key_val.left()); - let f = self - .fix(Node::NibbledBranch(encoded, children, None, child_changes), *key); + let f = + self.fix(Node::NibbledBranch(encoded, children, None, tree_ref), *key); Action::Replace(f?) } else { - Action::Restore(Node::NibbledBranch(encoded, children, None, child_changes)) + Action::Restore(Node::NibbledBranch(encoded, children, None, tree_ref)) } } else if common < existing_length { // partway through an extension -- nothing to do here. @@ -1674,7 +1654,7 @@ where ); let prefix = *key; key.advance(common + 1); - match self.remove_at(child, key, old_val, child_changes)? { + match self.remove_at(child, key, old_val, tree_ref)? { Some((new, changed)) => { children[idx] = Some(new.into()); let branch = @@ -1714,7 +1694,7 @@ where let mut key_val = key.clone(); key_val.advance(existing_key.len()); self.replace_old_value(old_val, Some(value), key_val.left()); - Action::Delete(child_changes) + Action::Delete(tree_ref) } else { // leaf the node alone. #[cfg(feature = "std")] @@ -1738,7 +1718,7 @@ where trace!(target: "trie", "removing from extension child, partial={:?}", partial); let prefix = *key; key.advance(common); - match self.remove_at(child_branch, key, old_val, child_changes)? { + match self.remove_at(child_branch, key, old_val, tree_ref)? { Some((new_child, changed)) => { // if the child branch was unchanged, then the extension is too. // otherwise, this extension may need fixing. @@ -2146,8 +2126,6 @@ where /// Cache the given `value`. /// - /// `hash` is the hash of `value`. - /// /// Cache is not done here as we want to cache the location from the db, /// and location on new_nodes are not resolved here. fn cache_value(&mut self, _full_key: &[u8], _value: impl Into, _hash: TrieHash) {} @@ -2284,17 +2262,17 @@ where key: &[u8], value: &[u8], ) -> Result>, TrieHash, CError> { - self.insert_with_child_changes(key, value, None) + self.insert_with_tree_ref(key, value, None) } - pub fn insert_with_child_changes( + pub fn insert_with_tree_ref( &mut self, key: &[u8], value: &[u8], - child_changes: Option>, + tree_ref: Option>, ) -> Result>, TrieHash, CError> { // expect for the child changes to have a key. - debug_assert!(child_changes + debug_assert!(tree_ref .as_ref() .map(|c| if let ChangesetNodeRef::New(set) = c.as_ref() { set.key_childset.is_some() @@ -2313,13 +2291,8 @@ where let value = Bytes::from(value); let root_handle = self.root_handle(); - let (new_handle, _changed) = self.insert_at( - root_handle, - &mut NibbleSlice::new(key), - value, - &mut old_val, - child_changes, - )?; + let (new_handle, _changed) = + self.insert_at(root_handle, &mut NibbleSlice::new(key), value, &mut old_val, tree_ref)?; #[cfg(feature = "std")] trace!(target: "trie", "insert: altered trie={}", _changed); @@ -2329,16 +2302,16 @@ where } pub fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { - self.remove_with_child_changes(key, None) + self.remove_with_tree_ref(key, None) } - pub fn remove_with_child_changes( + pub fn remove_with_tree_ref( &mut self, key: &[u8], - child_changes: Option>, + tree_ref: Option>, ) -> Result>, TrieHash, CError> { // expect for the child changes to have a key. - debug_assert!(child_changes + debug_assert!(tree_ref .as_ref() .map(|c| if let ChangesetNodeRef::New(set) = c.as_ref() { set.key_childset.is_some() @@ -2354,7 +2327,7 @@ where let mut key_slice = NibbleSlice::new(key); let mut old_val = None; - match self.remove_at(root_handle, &mut key_slice, &mut old_val, child_changes)? { + match self.remove_at(root_handle, &mut key_slice, &mut old_val, tree_ref)? { Some((handle, _changed)) => { #[cfg(feature = "std")] trace!(target: "trie", "remove: altered trie={}", _changed); diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index ce1f50e5..68fc151b 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -896,7 +896,7 @@ fn attached_trie_internal>() { let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let changeset = c.changeset.take().unwrap(); - memtrie.insert_with_child_changes(key, val, Some(changeset)).unwrap(); + memtrie.insert_with_tree_ref(key, val, Some(changeset)).unwrap(); } let changeset = memtrie.commit(); let root = changeset.commit_to(&mut memdb); @@ -906,7 +906,8 @@ fn attached_trie_internal>() { let attached_trie_root_key = data.iter().next().unwrap().0.clone(); let changeset = memtrie.commit(); let root = changeset.root_hash(); - let change_to_insert = changeset.to_insert_in_other_trie(attached_trie_root_key.clone()); + let change_to_insert = + changeset.to_insert_in_other_trie(attached_trie_root_key.clone()); attached_tries.insert( attached_trie_root_key, ATrie { root, data, changeset: Some(change_to_insert) }, @@ -946,7 +947,7 @@ fn attached_trie_internal>() { let (root_key, a_attached_trie) = attached_tries.iter().next().unwrap(); let (a_attached_trie_root, attached_trie_location) = attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); - let (child_changeset, child_root_hash) = { + let (tree_ref_changeset, child_root_hash) = { assert_eq!(a_attached_trie_root, a_attached_trie.root); let child_memdb: &dyn HashDB<_, _, _> = if support_location { &memdb @@ -970,7 +971,7 @@ fn attached_trie_internal>() { }; let mut main_trie = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); main_trie - .insert_with_child_changes(root_key, child_root_hash.as_ref(), Some(child_changeset)) + .insert_with_tree_ref(root_key, child_root_hash.as_ref(), Some(tree_ref_changeset)) .unwrap(); let changeset = main_trie.commit(); let main_root = changeset.root_hash(); @@ -1016,7 +1017,7 @@ fn attached_trie_root>( iter.seek(root_key).unwrap(); let item = iter.next()?.unwrap(); let node = &item.2; - let location = node.node_plan().attached_change_set_location(node.locations()); + let location = node.node_plan().additional_ref_location(node.locations()); let root = iter.item_from_raw(&item)?.unwrap(); if root.0.as_slice() != root_key { return None; From e83c8f08ef8acc0d803f672f861bf0194fc5a723 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 14:55:07 +0100 Subject: [PATCH 40/90] rename --- trie-db/test/src/triedbmut.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 68fc151b..c6756517 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -962,7 +962,7 @@ fn attached_trie_internal>() { ) .build(); attached_trie.remove(a_attached_trie.data.iter().next().unwrap().0).unwrap(); - attached_trie.insert(b"make_sur_it_changes", b"value").unwrap(); + attached_trie.insert(b"make_sure_it_changes", b"value").unwrap(); let changeset = attached_trie.commit(); let new_root = changeset.root_hash(); let change_to_insert = changeset.to_insert_in_other_trie(root_key.clone()); @@ -991,7 +991,7 @@ fn attached_trie_internal>() { attached_trie_location.unwrap_or_default(), ) .build(); - trie.get(b"make_sur_it_changes").unwrap().unwrap(); + trie.get(b"make_sure_it_changes").unwrap().unwrap(); let mut first = true; for (k, v) in a_attached_trie.data.iter() { if first { @@ -1001,7 +1001,7 @@ fn attached_trie_internal>() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } - trie.get(b"make_sur_it_changes").unwrap().unwrap(); + trie.get(b"make_sure_it_changes").unwrap().unwrap(); } fn attached_trie_root>( From 367c0e7e76eb6f187f93ec2a328a96fbdb8a0e77 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 15:02:17 +0100 Subject: [PATCH 41/90] HashDB to NodeDB --- README.md | 4 ++-- hash-db/README.md | 6 +++--- hash-db/src/lib.rs | 2 +- memory-db/README.md | 2 +- memory-db/src/lib.rs | 10 +++++----- test-support/mem-tree-db/src/lib.rs | 10 +++++----- test-support/reference-trie/src/lib.rs | 10 +++++----- trie-db/src/iterator.rs | 4 ++-- trie-db/src/lib.rs | 2 +- trie-db/src/lookup.rs | 14 +++++++------- trie-db/src/proof/generate.rs | 4 ++-- trie-db/src/triedb.rs | 14 +++++++------- trie-db/src/triedbmut.rs | 18 +++++++++--------- trie-db/test/src/lib.rs | 2 +- trie-db/test/src/triedbmut.rs | 20 ++++++++++---------- 15 files changed, 61 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 8f523fe4..3660d968 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,10 @@ Trie Hash alone is able to be used in `no_std` builds by disabling its (default) In addition to these, several support crates are provided: - `hash-db` crate, used to provide `Hasher` (trait for all things that - can make cryptographic hashes) and `HashDB` (trait for databases that can have byte + can make cryptographic hashes) and `NodeDB` (trait for databases that can have byte slices pushed into them and allow for them to be retrieved based on their hash). Suitable for `no_std`, though in this case will only provide `Hasher`. -- `memory-db` crate, contains `MemoryDB`, an implementation of a `HashDB` using only +- `memory-db` crate, contains `MemoryDB`, an implementation of a `NodeDB` using only in in-memory map. - `hash256-std-hasher` crate, an implementation of a `std::hash::Hasher` for 32-byte keys that have already been hashed. Useful to build the backing `HashMap` for `MemoryDB`. diff --git a/hash-db/README.md b/hash-db/README.md index 23b49ae8..d910c8fd 100644 --- a/hash-db/README.md +++ b/hash-db/README.md @@ -1,4 +1,4 @@ -# HashDB -`HashDB` defines a common interface for databases of byte-slices keyed to their hash. It is generic over hash type through the `Hasher` trait. +# NodeDB +`NodeDB` defines a common interface for databases of byte-slices keyed to their hash. It is generic over hash type through the `Hasher` trait. -The `Hasher` trait can be used in a `no_std` context. \ No newline at end of file +The `Hasher` trait can be used in a `no_std` context. diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs index ac5ca6c8..c542617a 100644 --- a/hash-db/src/lib.rs +++ b/hash-db/src/lib.rs @@ -79,7 +79,7 @@ pub trait Hasher: Sync + Send { } /// Trait modelling datastore keyed by a hash defined by the `Hasher` and optional location tag. -pub trait HashDB: Send + Sync { +pub trait NodeDB: Send + Sync { /// Look up a trie node by hash and location. /// Returns the node bytes and the list of children node locations if any. fn get(&self, key: &H::Out, prefix: Prefix, location: L) -> Option<(T, Vec)>; diff --git a/memory-db/README.md b/memory-db/README.md index fc0c6309..497359e2 100644 --- a/memory-db/README.md +++ b/memory-db/README.md @@ -1 +1 @@ -MemoryDB is a reference counted memory-based [`HashDB`](https://github.com/paritytech/parity-common/tree/master/hash-db) implementation backed by a `HashMap`. \ No newline at end of file +MemoryDB is a reference counted memory-based [`NodeDB`](https://github.com/paritytech/parity-common/tree/master/hash-db) implementation backed by a `HashMap`. diff --git a/memory-db/src/lib.rs b/memory-db/src/lib.rs index 65d42528..7331a008 100644 --- a/memory-db/src/lib.rs +++ b/memory-db/src/lib.rs @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Reference-counted memory-based `HashDB` implementation. +//! Reference-counted memory-based `NodeDB` implementation. #![cfg_attr(not(feature = "std"), no_std)] #[cfg(not(feature = "std"))] extern crate alloc; -use hash_db::{HashDB, Hasher as KeyHasher, MaybeDebug, Prefix}; +use hash_db::{Hasher as KeyHasher, MaybeDebug, NodeDB, Prefix}; #[cfg(feature = "std")] use std::{ cmp::Eq, collections::hash_map::Entry, collections::HashMap as Map, hash, marker::PhantomData, @@ -35,7 +35,7 @@ use core::{cmp::Eq, hash, marker::PhantomData, mem}; #[cfg(not(feature = "std"))] use alloc::vec::Vec; -/// Reference-counted memory-based `HashDB` implementation. +/// Reference-counted memory-based `NodeDB` implementation. /// /// Use `new()` to create a new database. Insert items with `insert()`, remove items /// with `remove()`, check for existence with `contains()` and lookup a hash to derive @@ -287,7 +287,7 @@ where /// extern crate keccak_hasher; /// extern crate memory_db; /// - /// use hash_db::{Hasher, HashDB, EMPTY_PREFIX}; + /// use hash_db::{Hasher, NodeDB, EMPTY_PREFIX}; /// use keccak_hasher::KeccakHasher; /// use memory_db::{MemoryDB, HashKey}; /// @@ -356,7 +356,7 @@ where } } -impl HashDB for MemoryDB +impl NodeDB for MemoryDB where H: KeyHasher, T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs index d68005f5..47043206 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/test-support/mem-tree-db/src/lib.rs @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Flat memory-based `HashDB` implementation. +//! Flat memory-based `NodeDB` implementation. use std::collections::HashMap; -use hash_db::{HashDB, Hasher, Prefix}; +use hash_db::{Hasher, NodeDB, Prefix}; use trie_db::{Changeset, ChangesetNodeRef}; /// Node location which is just an index into the `nodes` vector. pub type Location = Option; -/// Tree based `HashDB` implementation. +/// Tree based `NodeDB` implementation. #[derive(Clone)] pub struct MemTreeDB where @@ -173,7 +173,7 @@ where } } -impl HashDB, Location> for MemTreeDB +impl NodeDB, Location> for MemTreeDB where H: Hasher, { @@ -221,7 +221,7 @@ where mod tests { use super::{Location, MemTreeDB, NodeEntry}; - use hash_db::{HashDB, Hasher}; + use hash_db::{Hasher, NodeDB}; use keccak_hasher::{KeccakHash, KeccakHasher}; use trie_db::{Changeset, ChangesetNodeRef, ExistingChangesetNode, NewChangesetNode}; diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 7a511f30..8f5690e1 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -961,7 +961,7 @@ where }; if root_new != root { { - let db: &dyn hash_db::HashDB<_, _, _> = &mem_db1; + let db: &dyn hash_db::NodeDB<_, _, _> = &mem_db1; let t = TrieDBBuilder::::new(db, &root_new).build(); println!("{:?}", t); for a in t.iter().unwrap() { @@ -969,7 +969,7 @@ where } } { - let db: &dyn hash_db::HashDB<_, _, _> = &mem_db2; + let db: &dyn hash_db::NodeDB<_, _, _> = &mem_db2; let t = TrieDBBuilder::::new(db, &root).build(); println!("{:?}", t); for a in t.iter().unwrap() { @@ -984,7 +984,7 @@ where } /// Compare trie builder and trie root implementations. -pub fn compare_root>( +pub fn compare_root>( data: Vec<(Vec, Vec)>, memdb: DB, ) { @@ -1083,7 +1083,7 @@ where if root != root_new { { - let db: &dyn hash_db::HashDB<_, _, _> = &mem_db1; + let db: &dyn hash_db::NodeDB<_, _, _> = &mem_db1; let t = TrieDBBuilder::::new(db, &root).build(); println!("{:?}", t); for a in t.iter().unwrap() { @@ -1091,7 +1091,7 @@ where } } { - let db: &dyn hash_db::HashDB<_, _, _> = &mem_db2; + let db: &dyn hash_db::NodeDB<_, _, _> = &mem_db2; let t = TrieDBBuilder::::new(db, &root_new).build(); println!("{:?}", t); for a in t.iter().unwrap() { diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index 35dcda56..e83ec47d 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -730,7 +730,7 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeIterator<'a, 'cache, L> { } /// Access inner hash db. - pub fn db(&self) -> &dyn hash_db::HashDB { + pub fn db(&self) -> &dyn hash_db::NodeDB { self.db.db() } @@ -828,7 +828,7 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeDoubleEndedIterator<'a, 'cache, L> { } /// Access inner hash db. - pub fn db(&self) -> &dyn hash_db::HashDB { + pub fn db(&self) -> &dyn hash_db::NodeDB { self.db.db() } } diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 00860c56..2745470a 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -78,7 +78,7 @@ pub use crate::{ node_codec::{NodeCodec, Partial}, trie_codec::{decode_compact, decode_compact_from_iter, encode_compact}, }; -pub use hash_db::{HashDB, Hasher}; +pub use hash_db::{Hasher, NodeDB}; #[cfg(feature = "std")] pub use crate::iter_build::TrieRootPrint; diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs index d2cd9661..6bf037f8 100644 --- a/trie-db/src/lookup.rs +++ b/trie-db/src/lookup.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Trie lookup via HashDB. +//! Trie lookup via NodeDB. use crate::{ nibble::NibbleSlice, @@ -22,12 +22,12 @@ use crate::{ Bytes, CError, CachedValue, DBValue, MerkleValue, Query, RecordedForKey, Result, TrieAccess, TrieCache, TrieError, TrieHash, TrieLayout, TrieRecorder, }; -use hash_db::{HashDB, Hasher, Prefix}; +use hash_db::{Hasher, NodeDB, Prefix}; /// Trie lookup helper object. pub struct Lookup<'a, 'cache, L: TrieLayout, Q: Query> { /// database to query from. - pub db: &'a dyn HashDB, + pub db: &'a dyn NodeDB, /// Query object to record nodes and transform data. pub query: Q, /// Hash to start at @@ -55,7 +55,7 @@ where v: Value, prefix: Prefix, full_key: &[u8], - db: &dyn HashDB, + db: &dyn NodeDB, recorder: &mut Option<&mut dyn TrieRecorder, L::Location>>, query: Q, ) -> Result, CError> { @@ -98,7 +98,7 @@ where prefix: Prefix, full_key: &[u8], cache: &mut dyn crate::TrieCache, - db: &dyn HashDB, + db: &dyn NodeDB, recorder: &mut Option<&mut dyn TrieRecorder, L::Location>>, ) -> Result<(Bytes, TrieHash), TrieHash, CError> { match v { @@ -589,7 +589,7 @@ where Prefix, &[u8], &mut dyn crate::TrieCache, - &dyn HashDB, + &dyn NodeDB, &mut Option<&mut dyn TrieRecorder, L::Location>>, ) -> Result, CError>, ) -> Result, TrieHash, CError> { @@ -765,7 +765,7 @@ where Value, Prefix, &[u8], - &dyn HashDB, + &dyn NodeDB, &mut Option<&mut dyn TrieRecorder, L::Location>>, Q, ) -> Result, CError>, diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index a4ca9c21..f9a58ece 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -16,7 +16,7 @@ use crate::rstd::{boxed::Box, convert::TryInto, marker::PhantomData, vec, vec::Vec}; -use hash_db::{HashDB, Hasher}; +use hash_db::{Hasher, NodeDB}; use crate::{ nibble::LeftNibbleSlice, @@ -225,7 +225,7 @@ pub fn generate_proof<'a, D, L, I, K>( keys: I, ) -> TrieResult>, TrieHash, CError> where - D: HashDB, + D: NodeDB, L: TrieLayout, I: IntoIterator, K: 'a + AsRef<[u8]>, diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index a934806e..aa23b2b1 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -28,11 +28,11 @@ use crate::{ node::Node, rstd::{fmt, vec::Vec}, }; -use hash_db::{HashDB, Prefix, EMPTY_PREFIX}; +use hash_db::{NodeDB, Prefix, EMPTY_PREFIX}; /// A builder for creating a [`TrieDB`]. pub struct TrieDBBuilder<'db, 'cache, L: TrieLayout> { - db: &'db dyn HashDB, + db: &'db dyn NodeDB, root: &'db TrieHash, root_location: L::Location, cache: Option<&'cache mut dyn TrieCache>, @@ -45,14 +45,14 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// This doesn't check if `root` exists in the given `db`. If `root` doesn't exist it will fail /// when trying to lookup any key. #[inline] - pub fn new(db: &'db dyn HashDB, root: &'db TrieHash) -> Self { + pub fn new(db: &'db dyn NodeDB, root: &'db TrieHash) -> Self { Self { db, root, cache: None, recorder: None, root_location: Default::default() } } /// Same as `new` but indicating db location of root. Warning root hash will not be checked. #[inline] pub fn new_with_db_location( - db: &'db dyn HashDB, + db: &'db dyn NodeDB, root: &'db TrieHash, root_location: L::Location, ) -> Self { @@ -111,7 +111,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { } } -/// A `Trie` implementation using a generic `HashDB` backing database, a `Hasher` +/// A `Trie` implementation using a generic `NodeDB` backing database, a `Hasher` /// implementation to generate keys and a `NodeCodec` implementation to encode/decode /// the nodes. /// @@ -137,7 +137,7 @@ pub struct TrieDB<'db, 'cache, L> where L: TrieLayout, { - db: &'db dyn HashDB, + db: &'db dyn NodeDB, root: &'db TrieHash, root_location: L::Location, cache: Option>>, @@ -149,7 +149,7 @@ where L: TrieLayout, { /// Get the backing database. - pub fn db(&'db self) -> &'db dyn HashDB { + pub fn db(&'db self) -> &'db dyn NodeDB { self.db } diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 611e41bf..1561b733 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -27,7 +27,7 @@ use crate::{ TrieRecorder, }; -use hash_db::{HashDB, Hasher, Prefix}; +use hash_db::{Hasher, NodeDB, Prefix}; #[cfg(feature = "std")] use std::collections::HashSet as Set; @@ -189,7 +189,7 @@ impl Value { fn in_memory_fetched_value( &self, prefix: Prefix, - db: &dyn HashDB, + db: &dyn NodeDB, recorder: &Option, L::Location>>>, full_key: &[u8], ) -> Result, TrieHash, CError> { @@ -659,7 +659,7 @@ impl<'a, L: TrieLayout> Index<&'a StorageHandle> for NodeStorage { /// A builder for creating a [`TrieDBMut`]. pub struct TrieDBMutBuilder<'db, L: TrieLayout> { - db: &'db dyn HashDB, + db: &'db dyn NodeDB, root: TrieHash, root_location: L::Location, cache: Option<&'db mut dyn TrieCache>, @@ -669,7 +669,7 @@ pub struct TrieDBMutBuilder<'db, L: TrieLayout> { impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Create a builder for constructing a new trie with the backing database `db` and empty /// `root`. - pub fn new(db: &'db dyn HashDB) -> Self { + pub fn new(db: &'db dyn NodeDB) -> Self { let root = L::Codec::hashed_null_node(); Self { root, db, cache: None, recorder: None, root_location: Default::default() } } @@ -679,7 +679,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// This doesn't check if `root` exists in the given `db`. If `root` doesn't exist it will fail /// when trying to lookup any key. pub fn from_existing( - db: &'db dyn HashDB, + db: &'db dyn NodeDB, root: TrieHash, ) -> Self { Self { db, root, cache: None, recorder: None, root_location: Default::default() } @@ -688,7 +688,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Same as `from_existing` but force a db location to access root. /// Note root in parameter is not checked. pub fn from_existing_with_db_location( - db: &'db dyn HashDB, + db: &'db dyn NodeDB, root: TrieHash, root_location: L::Location, ) -> Self { @@ -875,7 +875,7 @@ impl Changeset { pub type OwnedPrefix = (BackingByteVec, Option); -/// A `Trie` implementation using a generic `HashDB` backing database. +/// A `Trie` implementation using a generic `NodeDB` backing database. /// /// Use it as a `TrieMut` trait object. You can use `db()` to get the backing database object. /// Note that changes are not committed to the database until `commit` is called. @@ -905,7 +905,7 @@ where L: TrieLayout, { storage: NodeStorage, - db: &'a dyn HashDB, + db: &'a dyn NodeDB, root: TrieHash, root_handle: NodeHandle, L::Location>, death_row: Set<(TrieHash, OwnedPrefix)>, @@ -924,7 +924,7 @@ where L: TrieLayout, { /// Get the backing database. - pub fn db(&self) -> &dyn HashDB { + pub fn db(&self) -> &dyn NodeDB { self.db } diff --git a/trie-db/test/src/lib.rs b/trie-db/test/src/lib.rs index 6472cd1a..5f5f4cd9 100644 --- a/trie-db/test/src/lib.rs +++ b/trie-db/test/src/lib.rs @@ -37,7 +37,7 @@ use hash_db::{Hasher, Prefix}; use mem_tree_db::MemTreeDB; use trie_db::{Changeset, DBValue, TrieHash, TrieLayout}; -trait TestDB: hash_db::HashDB + Clone + Default { +trait TestDB: hash_db::NodeDB + Clone + Default { fn commit( &mut self, commit: trie_db::Changeset<<::Hash as Hasher>::Out, T::Location>, diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index c6756517..7693155b 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -15,7 +15,7 @@ use std::ops::Deref; use env_logger; -use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; +use hash_db::{Hasher, NodeDB, Prefix, EMPTY_PREFIX}; use log::debug; use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ @@ -35,7 +35,7 @@ type MemoryDBProof = MemoryDB<::Hash, HashKey<::Hash>, DBValue>; fn populate_trie<'db, T: TrieLayout>( - db: &'db dyn HashDB, + db: &'db dyn NodeDB, v: &[(Vec, Vec)], ) -> TrieDBMut<'db, T> { let mut t = TrieDBMutBuilder::::new(db).build(); @@ -558,14 +558,14 @@ fn register_proof_without_value() { unsafe impl Send for ProofRecorder {} unsafe impl Sync for ProofRecorder {} - impl HashDB for ProofRecorder { + impl NodeDB for ProofRecorder { fn get( &self, key: &::Out, prefix: Prefix, _location: (), ) -> Option<(DBValue, Vec<()>)> { - let v = HashDB::get(&self.db, key, prefix, ()); + let v = NodeDB::get(&self.db, key, prefix, ()); if let Some((v, _)) = v.as_ref() { self.record.borrow_mut().entry(key[..].to_vec()).or_insert_with(|| v.clone()); } @@ -925,7 +925,7 @@ fn attached_trie_internal>() { let (attached_trie_root, attached_trie_location) = attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); - let child_memdb: &dyn HashDB<_, _, _> = if support_location { + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { &memdb } else { assert!(attached_trie_location.is_none()); @@ -949,7 +949,7 @@ fn attached_trie_internal>() { attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); let (tree_ref_changeset, child_root_hash) = { assert_eq!(a_attached_trie_root, a_attached_trie.root); - let child_memdb: &dyn HashDB<_, _, _> = if support_location { + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { &memdb } else { keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); @@ -979,7 +979,7 @@ fn attached_trie_internal>() { // checking modification let (a_attached_trie_root, attached_trie_location) = attached_trie_root(&memdb, &main_root, root_key).unwrap(); - let child_memdb: &dyn HashDB<_, _, _> = if support_location { + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { &memdb } else { keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); @@ -1027,16 +1027,16 @@ fn attached_trie_root>( Some((root_hash, location)) } -pub struct KeySpacedDB<'a, H, T, DL>(&'a dyn hash_db::HashDB, &'a [u8]); +pub struct KeySpacedDB<'a, H, T, DL>(&'a dyn hash_db::NodeDB, &'a [u8]); impl<'a, H, T, DL> KeySpacedDB<'a, H, T, DL> { #[inline] - pub fn new(db: &'a dyn hash_db::HashDB, ks: &'a [u8]) -> Self { + pub fn new(db: &'a dyn hash_db::NodeDB, ks: &'a [u8]) -> Self { KeySpacedDB(db, ks) } } -impl<'a, H, T, L> hash_db::HashDB for KeySpacedDB<'a, H, T, L> +impl<'a, H, T, L> hash_db::NodeDB for KeySpacedDB<'a, H, T, L> where H: Hasher, T: From<&'static [u8]>, From c7547c696865587158c143cc744b8b7c1296bf53 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 15:58:43 +0100 Subject: [PATCH 42/90] cleaning more references to child trie --- test-support/mem-tree-db/src/lib.rs | 12 ++--- trie-db/src/triedbmut.rs | 78 ++++++++++++++++------------- 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs index 47043206..a93b77ab 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/test-support/mem-tree-db/src/lib.rs @@ -239,7 +239,7 @@ mod tests { prefix: Default::default(), data: vec![1, 2, 3], children: vec![], - key_childset: None, + removed_keys: None, }); let new_location = db.apply(&new_node); @@ -262,7 +262,7 @@ mod tests { prefix: Default::default(), data: vec![1, 2, 3], children: vec![], - key_childset: None, + removed_keys: None, }); let location = db.apply(&node); assert_eq!(location, db.nodes.len() - 1); @@ -277,7 +277,7 @@ mod tests { prefix: Default::default(), data: vec![1, 2, 3], children: vec![], - key_childset: None, + removed_keys: None, }), removed: Default::default(), }; @@ -295,14 +295,14 @@ mod tests { prefix: Default::default(), data: vec![1, 2, 3], children: vec![], - key_childset: None, + removed_keys: None, }); let child2 = ChangesetNodeRef::New(NewChangesetNode { hash: hash(2), prefix: Default::default(), data: vec![4, 5, 6], children: vec![], - key_childset: None, + removed_keys: None, }); // Create a root node that refers to the child nodes @@ -311,7 +311,7 @@ mod tests { prefix: Default::default(), data: vec![7, 8, 9], children: vec![child1, child2], - key_childset: None, + removed_keys: None, }); let commit = Changeset:: { root, removed: Default::default() }; diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 1561b733..8e8e37e8 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -754,8 +754,9 @@ pub struct NewChangesetNode { pub prefix: OwnedPrefix, pub data: Vec, pub children: Vec>, - // Storing the key and removed nodes (only needed for old trie). - pub key_childset: Option<(Vec, Vec<(H, OwnedPrefix)>)>, + // Storing the key and removed nodes related + // to this change set node (only needed for old trie). + pub removed_keys: Option<(Vec, Vec<(H, OwnedPrefix)>)>, } #[derive(Debug)] @@ -815,7 +816,7 @@ impl Changeset { ChangesetNodeRef::New(node) => { let ks = if ks.is_some() { ks - } else if let Some((k, removed)) = node.key_childset.as_ref() { + } else if let Some((k, removed)) = node.removed_keys.as_ref() { for (hash, p) in removed.iter() { let prefixed = prefix_prefix(k.as_slice(), (p.0.as_slice(), p.1)); mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); @@ -865,7 +866,7 @@ impl Changeset { // needed for prefixed key. // Note if unchanged we don't need this actually unchange should only // be a thing in case of a copy. - node.key_childset = Some((root_at, self.removed)); + node.removed_keys = Some((root_at, self.removed)); }, _ => (), } @@ -1584,7 +1585,7 @@ where // always replace since we took the value out. Action::Replace(self.fix(Node::NibbledBranch(n, children, None, tree_ref), *key)?) }, - (Node::Branch(mut children, value, b_childset), false) => { + (Node::Branch(mut children, value, btreerefset), false) => { let idx = partial.at(0) as usize; if let Some(child) = children[idx].take() { #[cfg(feature = "std")] @@ -1598,7 +1599,7 @@ where match self.remove_at(child, key, old_val, tree_ref)? { Some((new, changed)) => { children[idx] = Some(new.into()); - let branch = Node::Branch(children, value, b_childset); + let branch = Node::Branch(children, value, btreerefset); match changed { // child was changed, so we were too. true => Action::Replace(branch), @@ -1612,16 +1613,16 @@ where #[cfg(feature = "std")] trace!(target: "trie", "branch child deleted, partial={:?}", partial); Action::Replace( - self.fix(Node::Branch(children, value, b_childset), prefix)?, + self.fix(Node::Branch(children, value, btreerefset), prefix)?, ) }, } } else { // no change needed. - Action::Restore(Node::Branch(children, value, b_childset)) + Action::Restore(Node::Branch(children, value, btreerefset)) } }, - (Node::NibbledBranch(encoded, mut children, value, b_childset), false) => { + (Node::NibbledBranch(encoded, mut children, value, btreerefset), false) => { let (common, existing_length) = { let existing_key = NibbleSlice::from_stored(&encoded); (existing_key.common_prefix(&partial), existing_key.len()) @@ -1640,7 +1641,7 @@ where } } else if common < existing_length { // partway through an extension -- nothing to do here. - Action::Restore(Node::NibbledBranch(encoded, children, value, b_childset)) + Action::Restore(Node::NibbledBranch(encoded, children, value, btreerefset)) } else { // common == existing_length && common < partial.len() : check children let idx = partial.at(common) as usize; @@ -1658,7 +1659,7 @@ where Some((new, changed)) => { children[idx] = Some(new.into()); let branch = - Node::NibbledBranch(encoded, children, value, b_childset); + Node::NibbledBranch(encoded, children, value, btreerefset); match changed { // child was changed, so we were too. true => Action::Replace(branch), @@ -1676,18 +1677,18 @@ where partial, ); Action::Replace(self.fix( - Node::NibbledBranch(encoded, children, value, b_childset), + Node::NibbledBranch(encoded, children, value, btreerefset), prefix, )?) }, } } else { // no change needed. - Action::Restore(Node::NibbledBranch(encoded, children, value, b_childset)) + Action::Restore(Node::NibbledBranch(encoded, children, value, btreerefset)) } } }, - (Node::Leaf(encoded, value, l_childset), _) => { + (Node::Leaf(encoded, value, ltreerefset), _) => { let existing_key = NibbleSlice::from_stored(&encoded); if existing_key == partial { // this is the node we were looking for. Let's delete it. @@ -1704,7 +1705,7 @@ where partial, NibbleSlice::from_stored(&encoded), ); - Action::Restore(Node::Leaf(encoded, value, l_childset)) + Action::Restore(Node::Leaf(encoded, value, ltreerefset)) } }, (Node::Extension(encoded, child_branch), _) => { @@ -1760,7 +1761,7 @@ where recurse_extension: bool, ) -> Result, TrieHash, CError> { match node { - Node::Branch(mut children, value, b_childset) => { + Node::Branch(mut children, value, btreerefset) => { // if only a single value, transmute to leaf/extension and feed through fixed. #[cfg_attr(feature = "std", derive(Debug))] enum UsedIndex { @@ -1798,17 +1799,17 @@ where // make a leaf. #[cfg(feature = "std")] trace!(target: "trie", "fixing: branch -> leaf"); - Ok(Node::Leaf(NibbleSlice::new(&[]).to_stored(), value, b_childset)) + Ok(Node::Leaf(NibbleSlice::new(&[]).to_stored(), value, btreerefset)) }, (_, value) => { // all is well. #[cfg(feature = "std")] trace!(target: "trie", "fixing: restoring branch"); - Ok(Node::Branch(children, value, b_childset)) + Ok(Node::Branch(children, value, btreerefset)) }, } }, - Node::NibbledBranch(enc_nibble, mut children, value, b_childset) => { + Node::NibbledBranch(enc_nibble, mut children, value, btreerefset) => { // if only a single value, transmute to leaf/extension and feed through fixed. #[cfg_attr(feature = "std", derive(Debug))] enum UsedIndex { @@ -1868,18 +1869,23 @@ where node }, }; - b_childset.map(|c| self.death_row_child.push(c)); + btreerefset.map(|c| self.death_row_child.push(c)); match child_node { - Node::Leaf(sub_partial, value, l_childset) => { + Node::Leaf(sub_partial, value, ltreerefset) => { let mut enc_nibble = enc_nibble; combine_key( &mut enc_nibble, (nibble_ops::NIBBLE_PER_BYTE - 1, &[a][..]), ); combine_key(&mut enc_nibble, (sub_partial.0, &sub_partial.1[..])); - Ok(Node::Leaf(enc_nibble, value, l_childset)) + Ok(Node::Leaf(enc_nibble, value, ltreerefset)) }, - Node::NibbledBranch(sub_partial, ch_children, ch_value, n_childset) => { + Node::NibbledBranch( + sub_partial, + ch_children, + ch_value, + ntreerefset, + ) => { let mut enc_nibble = enc_nibble; combine_key( &mut enc_nibble, @@ -1890,7 +1896,7 @@ where enc_nibble, ch_children, ch_value, - n_childset, + ntreerefset, )) }, _ => unreachable!(), @@ -1900,13 +1906,13 @@ where // make a leaf. #[cfg(feature = "std")] trace!(target: "trie", "fixing: branch -> leaf"); - Ok(Node::Leaf(enc_nibble, value, b_childset)) + Ok(Node::Leaf(enc_nibble, value, btreerefset)) }, (_, value) => { // all is well. #[cfg(feature = "std")] trace!(target: "trie", "fixing: restoring branch"); - Ok(Node::NibbledBranch(enc_nibble, children, value, b_childset)) + Ok(Node::NibbledBranch(enc_nibble, children, value, btreerefset)) }, } }, @@ -1977,7 +1983,7 @@ where self.fix_inner(Node::Extension(partial, sub_child), key.into(), true) }, - Node::Leaf(sub_partial, value, l_childset) => { + Node::Leaf(sub_partial, value, ltreerefset) => { // combine with node below. if let Some(hash) = maybe_hash { // delete the cached child since we are going to replace it. @@ -1993,7 +1999,7 @@ where "fixing: extension -> leaf. new_partial={:?}", partial, ); - Ok(Node::Leaf(partial, value, l_childset)) + Ok(Node::Leaf(partial, value, ltreerefset)) }, child_node => { #[cfg(feature = "std")] @@ -2047,7 +2053,7 @@ where let mut k = NibbleVec::new(); let mut children = Vec::new(); - let (encoded_root, root_childset) = node.into_encoded(|node, o_slice, o_index| { + let (encoded_root, roottreerefset) = node.into_encoded(|node, o_slice, o_index| { let mov = k.append_optional_slice_and_nibble(o_slice, o_index); match node { NodeToEncode::Node(value) => { @@ -2058,7 +2064,7 @@ where prefix: k.as_owned_prefix(), data: value.to_vec(), //TODO: avoid allocation children: Default::default(), - key_childset: None, + removed_keys: None, })); k.drop_lasts(mov); @@ -2071,7 +2077,7 @@ where }, } }); - root_childset.map(|c| { + roottreerefset.map(|c| { children.push(*c); }); #[cfg(feature = "std")] @@ -2089,7 +2095,7 @@ where prefix: Default::default(), data: encoded_root, children, - key_childset: None, + removed_keys: None, }), removed, } @@ -2176,7 +2182,7 @@ where prefix: prefix.as_owned_prefix(), data: value.to_vec(), //TODO: avoid allocation children: Default::default(), - key_childset: None, + removed_keys: None, }, )); @@ -2209,7 +2215,7 @@ where prefix: prefix.as_owned_prefix(), data: encoded, children: sub_children, - key_childset: None, + removed_keys: None, })); self.hash_count += 1; ChildReference::Hash(hash, Default::default()) @@ -2275,7 +2281,7 @@ where debug_assert!(tree_ref .as_ref() .map(|c| if let ChangesetNodeRef::New(set) = c.as_ref() { - set.key_childset.is_some() + set.removed_keys.is_some() } else { true }) @@ -2314,7 +2320,7 @@ where debug_assert!(tree_ref .as_ref() .map(|c| if let ChangesetNodeRef::New(set) = c.as_ref() { - set.key_childset.is_some() + set.removed_keys.is_some() } else { true }) From 1a969ad602802796e3fb6569ee9107aae9ba027c Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 17:07:03 +0100 Subject: [PATCH 43/90] use removed key from node only --- test-support/mem-tree-db/src/lib.rs | 13 +++-- trie-db/src/triedbmut.rs | 78 +++++++++++++---------------- trie-db/test/src/triedbmut.rs | 25 ++++----- 3 files changed, 57 insertions(+), 59 deletions(-) diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs index a93b77ab..c105a9d3 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/test-support/mem-tree-db/src/lib.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use hash_db::{Hasher, NodeDB, Prefix}; -use trie_db::{Changeset, ChangesetNodeRef}; +use trie_db::{Changeset, ChangesetNodeRef, NewChangesetNode}; /// Node location which is just an index into the `nodes` vector. pub type Location = Option; @@ -167,8 +167,15 @@ where let key = commit.root.hash(); self.roots.insert(*key, root); } - for (k, _) in commit.removed { - self.remove_root(&k); + // In non test use, the root should be store before calling commit (known + // from tree where commit was build from). + if let ChangesetNodeRef::New(NewChangesetNode { + removed_keys: Some((_, removed)), .. + }) = &commit.root + { + for (k, _) in removed { + self.remove_root(&k); + } } } } diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 8e8e37e8..fd84c2ce 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -756,7 +756,7 @@ pub struct NewChangesetNode { pub children: Vec>, // Storing the key and removed nodes related // to this change set node (only needed for old trie). - pub removed_keys: Option<(Vec, Vec<(H, OwnedPrefix)>)>, + pub removed_keys: Option<(Option>, Vec<(H, OwnedPrefix)>)>, } #[derive(Debug)] @@ -784,7 +784,6 @@ impl ChangesetNodeRef { #[derive(Debug)] pub struct Changeset { pub root: ChangesetNodeRef, - pub removed: Vec<(H, OwnedPrefix)>, } pub fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { @@ -800,31 +799,27 @@ impl Changeset { K: memory_db::KeyFunction + Send + Sync, MH: Hasher + Send + Sync, { - for (hash, prefix) in &self.removed { - mem_db.remove(hash, (prefix.0.as_slice(), prefix.1)); - } - - fn apply_node( - node: &ChangesetNodeRef, + fn apply_node<'a, H, DL, MH, K>( + node: &'a ChangesetNodeRef, mem_db: &mut MemoryDB, - ks: Option<&[u8]>, + mut ks: Option<&'a [u8]>, ) where K: memory_db::KeyFunction + Send + Sync, MH: Hasher + Send + Sync, { match node { ChangesetNodeRef::New(node) => { - let ks = if ks.is_some() { - ks - } else if let Some((k, removed)) = node.removed_keys.as_ref() { + if let Some((k, removed)) = node.removed_keys.as_ref() { for (hash, p) in removed.iter() { - let prefixed = prefix_prefix(k.as_slice(), (p.0.as_slice(), p.1)); - mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); + if let Some(k) = k { + let prefixed = prefix_prefix(k.as_slice(), (p.0.as_slice(), p.1)); + mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); + ks = Some(k.as_slice()); + } else { + mem_db.remove(hash, (p.0.as_slice(), p.1)); + } } - Some(k.as_slice()) - } else { - None - }; + } for child in &node.children { apply_node(child, mem_db, ks); } @@ -856,22 +851,8 @@ impl Changeset { prefix: (BackingByteVec::new(), None), location: Default::default(), }), - removed: Default::default(), } } - - pub fn to_insert_in_other_trie(mut self, root_at: Vec) -> Box> { - match &mut self.root { - ChangesetNodeRef::New(node) => { - // needed for prefixed key. - // Note if unchanged we don't need this actually unchange should only - // be a thing in case of a copy. - node.removed_keys = Some((root_at, self.removed)); - }, - _ => (), - } - Box::new(self.root) - } } pub type OwnedPrefix = (BackingByteVec, Option); @@ -2021,7 +2002,21 @@ where } /// Calculate the changeset for the trie. - pub fn commit(mut self) -> Changeset, L::Location> { + /// Note `keyspace` only apply for hash base storage to avoid key collision + /// between composed tree states. + pub fn commit(self) -> Changeset, L::Location> { + self.commit_inner(None) + } + + /// Same as commit but use a keyspace to isolate + /// stored date. + /// `keyspace` only apply for hash base storage to avoid key collision + /// between composed tree states. + pub fn commit_with_keyspace(self, keyspace: &[u8]) -> Changeset, L::Location> { + self.commit_inner(Some(keyspace)) + } + + fn commit_inner(mut self, keyspace: Option<&[u8]>) -> Changeset, L::Location> { #[cfg(feature = "std")] trace!(target: "trie", "Committing trie changes to db."); @@ -2036,15 +2031,16 @@ where } let handle = match self.root_handle() { - NodeHandle::Hash(hash, location) => + NodeHandle::Hash(hash, location) => { + debug_assert!(removed.is_empty()); return Changeset { root: ChangesetNodeRef::Existing(ExistingChangesetNode { hash, prefix: Default::default(), location, }), - removed, - }, // no changes necessary. + } + }, // no changes necessary. NodeHandle::InMemory(h) => h, }; @@ -2095,9 +2091,8 @@ where prefix: Default::default(), data: encoded_root, children, - removed_keys: None, + removed_keys: Some((keyspace.map(|s| s.to_vec()), removed)), }), - removed, } }, Stored::Cached(node, hash, location) => { @@ -2110,13 +2105,13 @@ where // usage and location in triedbmut Default::default(), ))); + debug_assert!(removed.is_empty()); Changeset { root: ChangesetNodeRef::Existing(ExistingChangesetNode { hash, prefix: Default::default(), location, }), - removed, } }, } @@ -2342,9 +2337,8 @@ where None => { #[cfg(feature = "std")] trace!(target: "trie", "remove: obliterated trie"); - self.root_handle = - NodeHandle::Hash(L::Codec::hashed_null_node(), Default::default()); - self.root = L::Codec::hashed_null_node(); + let handle = self.storage.alloc(Stored::New(Node::Empty)); + self.root_handle = NodeHandle::InMemory(handle); }, } diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 7693155b..3a326810 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -864,9 +864,9 @@ fn attached_trie_internal>() { data: BTreeMap, Vec>, changeset: Option, T::Location>>>, } - // Running a tipical child trie scenario: - // different trie, child trie root written all - // at once with childset before parent tree commit. + // Running a typical attached trie scenario (childtrie on substrate): + // different trie, attached trie root written all + // at once with treerefset before parent tree commit. // Direct copy if using ref counting and location in db. // Pruning. let mut seed = Default::default(); @@ -903,14 +903,12 @@ fn attached_trie_internal>() { main_trie.root = root; main_trie.data = data; } else { - let attached_trie_root_key = data.iter().next().unwrap().0.clone(); - let changeset = memtrie.commit(); + let attached_trie_root_key = data.iter().next().unwrap().0; + let changeset = memtrie.commit_with_keyspace(attached_trie_root_key); let root = changeset.root_hash(); - let change_to_insert = - changeset.to_insert_in_other_trie(attached_trie_root_key.clone()); attached_tries.insert( - attached_trie_root_key, - ATrie { root, data, changeset: Some(change_to_insert) }, + attached_trie_root_key.clone(), + ATrie { root, data, changeset: Some(changeset.root.into()) }, ); } } @@ -947,7 +945,7 @@ fn attached_trie_internal>() { let (root_key, a_attached_trie) = attached_tries.iter().next().unwrap(); let (a_attached_trie_root, attached_trie_location) = attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); - let (tree_ref_changeset, child_root_hash) = { + let (tree_ref_changeset, treeref_root_hash) = { assert_eq!(a_attached_trie_root, a_attached_trie.root); let child_memdb: &dyn NodeDB<_, _, _> = if support_location { &memdb @@ -963,15 +961,14 @@ fn attached_trie_internal>() { .build(); attached_trie.remove(a_attached_trie.data.iter().next().unwrap().0).unwrap(); attached_trie.insert(b"make_sure_it_changes", b"value").unwrap(); - let changeset = attached_trie.commit(); + let changeset = attached_trie.commit_with_keyspace(root_key); let new_root = changeset.root_hash(); - let change_to_insert = changeset.to_insert_in_other_trie(root_key.clone()); assert!(new_root != a_attached_trie_root); - (change_to_insert, new_root) + (changeset.root, new_root) }; let mut main_trie = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); main_trie - .insert_with_tree_ref(root_key, child_root_hash.as_ref(), Some(tree_ref_changeset)) + .insert_with_tree_ref(root_key, treeref_root_hash.as_ref(), Some(tree_ref_changeset.into())) .unwrap(); let changeset = main_trie.commit(); let main_root = changeset.root_hash(); From 6cc83637307ff9b582b9c578ff7f95ac283edd33 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 17:17:40 +0100 Subject: [PATCH 44/90] remove ChangesetNodeRef --- test-support/mem-tree-db/src/lib.rs | 33 ++++---- test-support/reference-trie/src/lib.rs | 6 +- trie-db/src/lib.rs | 4 +- trie-db/src/triedbmut.rs | 111 ++++++++++--------------- trie-db/test/src/iter_build.rs | 2 +- trie-db/test/src/recorder.rs | 2 +- trie-db/test/src/triedbmut.rs | 8 +- 7 files changed, 70 insertions(+), 96 deletions(-) diff --git a/test-support/mem-tree-db/src/lib.rs b/test-support/mem-tree-db/src/lib.rs index c105a9d3..2291e9a5 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/test-support/mem-tree-db/src/lib.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use hash_db::{Hasher, NodeDB, Prefix}; -use trie_db::{Changeset, ChangesetNodeRef, NewChangesetNode}; +use trie_db::{Changeset, NewChangesetNode}; /// Node location which is just an index into the `nodes` vector. pub type Location = Option; @@ -133,9 +133,9 @@ where self.roots.is_empty() } - fn apply(&mut self, c: &ChangesetNodeRef) -> usize { + fn apply(&mut self, c: &Changeset) -> usize { match c { - ChangesetNodeRef::Existing(e) => { + Changeset::Existing(e) => { let location = e.location.unwrap_or_else(|| *self.roots.get(&e.hash).unwrap()); let entry = self.nodes.get_mut(location).unwrap(); match entry { @@ -148,7 +148,7 @@ where }; location }, - ChangesetNodeRef::New(n) => { + Changeset::New(n) => { let children = n.children.iter().map(|c| self.apply(c)).collect(); self.nodes.push(NodeEntry::Live { key: n.hash, @@ -163,16 +163,13 @@ where pub fn apply_commit(&mut self, commit: Changeset) { if commit.root_hash() != self.hashed_null_node { - let root = self.apply(&commit.root); - let key = commit.root.hash(); + let root = self.apply(&commit); + let key = commit.hash(); self.roots.insert(*key, root); } // In non test use, the root should be store before calling commit (known // from tree where commit was build from). - if let ChangesetNodeRef::New(NewChangesetNode { - removed_keys: Some((_, removed)), .. - }) = &commit.root - { + if let Changeset::New(NewChangesetNode { removed_keys: Some((_, removed)), .. }) = &commit { for (k, _) in removed { self.remove_root(&k); } @@ -230,7 +227,7 @@ mod tests { use super::{Location, MemTreeDB, NodeEntry}; use hash_db::{Hasher, NodeDB}; use keccak_hasher::{KeccakHash, KeccakHasher}; - use trie_db::{Changeset, ChangesetNodeRef, ExistingChangesetNode, NewChangesetNode}; + use trie_db::{Changeset, ExistingChangesetNode, NewChangesetNode}; fn hash(i: u32) -> KeccakHash { KeccakHasher::hash(&i.to_le_bytes()) @@ -241,7 +238,7 @@ mod tests { let mut db = MemTreeDB::::default(); // First, apply a new node - let new_node = ChangesetNodeRef::New(NewChangesetNode { + let new_node = Changeset::New(NewChangesetNode { hash: hash(1), prefix: Default::default(), data: vec![1, 2, 3], @@ -251,7 +248,7 @@ mod tests { let new_location = db.apply(&new_node); // Then, apply an existing node that refers to the new node - let existing_node = ChangesetNodeRef::Existing(ExistingChangesetNode { + let existing_node = Changeset::Existing(ExistingChangesetNode { hash: hash(1), location: Some(new_location), prefix: Default::default(), @@ -264,7 +261,7 @@ mod tests { #[test] fn test_apply_new_node() { let mut db = MemTreeDB::::default(); - let node = ChangesetNodeRef::New(NewChangesetNode { + let node = Changeset::New(NewChangesetNode { hash: KeccakHash::default(), prefix: Default::default(), data: vec![1, 2, 3], @@ -279,7 +276,7 @@ mod tests { fn test_apply_commit() { let mut db = MemTreeDB::::default(); let commit = Changeset:: { - root: ChangesetNodeRef::New(NewChangesetNode { + root: Changeset::New(NewChangesetNode { hash: KeccakHash::default(), prefix: Default::default(), data: vec![1, 2, 3], @@ -297,14 +294,14 @@ mod tests { let mut db = MemTreeDB::::default(); // Create two child nodes - let child1 = ChangesetNodeRef::New(NewChangesetNode { + let child1 = Changeset::New(NewChangesetNode { hash: hash(1), prefix: Default::default(), data: vec![1, 2, 3], children: vec![], removed_keys: None, }); - let child2 = ChangesetNodeRef::New(NewChangesetNode { + let child2 = Changeset::New(NewChangesetNode { hash: hash(2), prefix: Default::default(), data: vec![4, 5, 6], @@ -313,7 +310,7 @@ mod tests { }); // Create a root node that refers to the child nodes - let root = ChangesetNodeRef::New(NewChangesetNode { + let root = Changeset::New(NewChangesetNode { hash: hash(0), prefix: Default::default(), data: vec![7, 8, 9], diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 8f5690e1..eb0f74bd 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -994,7 +994,7 @@ pub fn compare_root::new(&mut mem_db2); @@ -1115,7 +1115,7 @@ where let mut memdb = MemoryDB::::default(); let mut root = { let t = TrieDBMutBuilder::::new(&memdb).build(); - *t.commit().root.hash() + *t.commit().hash() }; while a < data.len() { // new triemut every 3 element diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 2745470a..6f4b32ff 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -68,8 +68,8 @@ pub use self::{ recorder::Recorder, triedb::{TrieDB, TrieDBBuilder, TrieDBIterator, TrieDBKeyIterator}, triedbmut::{ - Changeset, ChangesetNodeRef, ChildReference, ExistingChangesetNode, NewChangesetNode, - OwnedPrefix, TrieDBMut, TrieDBMutBuilder, Value, + Changeset, ChildReference, ExistingChangesetNode, NewChangesetNode, OwnedPrefix, TrieDBMut, + TrieDBMutBuilder, Value, }, }; pub use crate::{ diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index fd84c2ce..94e2f975 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -69,7 +69,7 @@ fn empty_children() -> Box<[Option>; nibble_ops::NIBBLE_L ]) } -pub type TreeRefChangeset = Box, ::Location>>; +pub type TreeRefChangeset = Box, ::Location>>; /// Type alias to indicate the nible covers a full key, /// therefore its left side is a full prefix. @@ -753,7 +753,7 @@ pub struct NewChangesetNode { pub hash: H, pub prefix: OwnedPrefix, pub data: Vec, - pub children: Vec>, + pub children: Vec>, // Storing the key and removed nodes related // to this change set node (only needed for old trie). pub removed_keys: Option<(Option>, Vec<(H, OwnedPrefix)>)>, @@ -767,25 +767,20 @@ pub struct ExistingChangesetNode { } #[derive(Debug)] -pub enum ChangesetNodeRef { +pub enum Changeset { New(NewChangesetNode), Existing(ExistingChangesetNode), } -impl ChangesetNodeRef { +impl Changeset { pub fn hash(&self) -> &H { match self { - ChangesetNodeRef::New(node) => &node.hash, - ChangesetNodeRef::Existing(node) => &node.hash, + Changeset::New(node) => &node.hash, + Changeset::Existing(node) => &node.hash, } } } -#[derive(Debug)] -pub struct Changeset { - pub root: ChangesetNodeRef, -} - pub fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); result.extend_from_slice(ks); @@ -800,7 +795,7 @@ impl Changeset { MH: Hasher + Send + Sync, { fn apply_node<'a, H, DL, MH, K>( - node: &'a ChangesetNodeRef, + node: &'a Changeset, mem_db: &mut MemoryDB, mut ks: Option<&'a [u8]>, ) where @@ -808,7 +803,7 @@ impl Changeset { MH: Hasher + Send + Sync, { match node { - ChangesetNodeRef::New(node) => { + Changeset::New(node) => { if let Some((k, removed)) = node.removed_keys.as_ref() { for (hash, p) in removed.iter() { if let Some(k) = k { @@ -830,27 +825,17 @@ impl Changeset { mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); } }, - ChangesetNodeRef::Existing(_) => {}, + Changeset::Existing(_) => {}, } } - apply_node::(&self.root, mem_db, None); + apply_node::(&self, mem_db, None); self.root_hash() } pub fn root_hash(&self) -> H { - match &self.root { - ChangesetNodeRef::New(node) => node.hash, - ChangesetNodeRef::Existing(node) => node.hash, - } - } - - pub fn empty(root: H) -> Self { - Self { - root: ChangesetNodeRef::Existing(ExistingChangesetNode { - hash: root, - prefix: (BackingByteVec::new(), None), - location: Default::default(), - }), + match &self { + Changeset::New(node) => node.hash, + Changeset::Existing(node) => node.hash, } } } @@ -2033,13 +2018,11 @@ where let handle = match self.root_handle() { NodeHandle::Hash(hash, location) => { debug_assert!(removed.is_empty()); - return Changeset { - root: ChangesetNodeRef::Existing(ExistingChangesetNode { - hash, - prefix: Default::default(), - location, - }), - } + return Changeset::Existing(ExistingChangesetNode { + hash, + prefix: Default::default(), + location, + }); }, // no changes necessary. NodeHandle::InMemory(h) => h, }; @@ -2055,7 +2038,7 @@ where NodeToEncode::Node(value) => { let value_hash = self.db.hash(value); self.cache_value(k.inner(), value, value_hash); - children.push(ChangesetNodeRef::New(NewChangesetNode { + children.push(Changeset::New(NewChangesetNode { hash: value_hash, prefix: k.as_owned_prefix(), data: value.to_vec(), //TODO: avoid allocation @@ -2085,15 +2068,13 @@ where self.cache_node(self.root); self.root_handle = NodeHandle::Hash(self.root, Default::default()); - Changeset { - root: ChangesetNodeRef::New(NewChangesetNode { - hash: self.root.clone(), - prefix: Default::default(), - data: encoded_root, - children, - removed_keys: Some((keyspace.map(|s| s.to_vec()), removed)), - }), - } + Changeset::New(NewChangesetNode { + hash: self.root.clone(), + prefix: Default::default(), + data: encoded_root, + children, + removed_keys: Some((keyspace.map(|s| s.to_vec()), removed)), + }) }, Stored::Cached(node, hash, location) => { // probably won't happen, but update the root and move on. @@ -2106,13 +2087,11 @@ where Default::default(), ))); debug_assert!(removed.is_empty()); - Changeset { - root: ChangesetNodeRef::Existing(ExistingChangesetNode { - hash, - prefix: Default::default(), - location, - }), - } + Changeset::Existing(ExistingChangesetNode { + hash, + prefix: Default::default(), + location, + }) }, } } @@ -2140,11 +2119,11 @@ where &mut self, handle: NodeHandle, L::Location>, prefix: &mut NibbleVec, - children: &mut Vec, L::Location>>, + children: &mut Vec, L::Location>>, ) -> ChildReference, L::Location> { match handle { NodeHandle::Hash(hash, location) => { - children.push(ChangesetNodeRef::Existing(ExistingChangesetNode { + children.push(Changeset::Existing(ExistingChangesetNode { hash, prefix: prefix.as_owned_prefix(), location, @@ -2154,7 +2133,7 @@ where NodeHandle::InMemory(storage_handle) => { match self.storage.destroy(storage_handle) { Stored::Cached(_, hash, location) => { - children.push(ChangesetNodeRef::Existing(ExistingChangesetNode { + children.push(Changeset::Existing(ExistingChangesetNode { hash, prefix: prefix.as_owned_prefix(), location, @@ -2171,15 +2150,13 @@ where match node { NodeToEncode::Node(value) => { let value_hash = self.db.hash(value); - sub_children.push(ChangesetNodeRef::New( - NewChangesetNode { - hash: value_hash, - prefix: prefix.as_owned_prefix(), - data: value.to_vec(), //TODO: avoid allocation - children: Default::default(), - removed_keys: None, - }, - )); + sub_children.push(Changeset::New(NewChangesetNode { + hash: value_hash, + prefix: prefix.as_owned_prefix(), + data: value.to_vec(), //TODO: avoid allocation + children: Default::default(), + removed_keys: None, + })); self.cache_value(prefix.inner(), value, value_hash); @@ -2205,7 +2182,7 @@ where child_set.map(|c| { sub_children.push(*c); }); - children.push(ChangesetNodeRef::New(NewChangesetNode { + children.push(Changeset::New(NewChangesetNode { hash, prefix: prefix.as_owned_prefix(), data: encoded, @@ -2275,7 +2252,7 @@ where // expect for the child changes to have a key. debug_assert!(tree_ref .as_ref() - .map(|c| if let ChangesetNodeRef::New(set) = c.as_ref() { + .map(|c| if let Changeset::New(set) = c.as_ref() { set.removed_keys.is_some() } else { true @@ -2314,7 +2291,7 @@ where // expect for the child changes to have a key. debug_assert!(tree_ref .as_ref() - .map(|c| if let ChangesetNodeRef::New(set) = c.as_ref() { + .map(|c| if let Changeset::New(set) = c.as_ref() { set.removed_keys.is_some() } else { true diff --git a/trie-db/test/src/iter_build.rs b/trie-db/test/src/iter_build.rs index 55a0b811..c2f424b0 100644 --- a/trie-db/test/src/iter_build.rs +++ b/trie-db/test/src/iter_build.rs @@ -50,7 +50,7 @@ fn test_iter(data: Vec<(Vec, Vec)>) { t.commit() }; changeset.apply_to(&mut db); - let t = TrieDBBuilder::::new(&db, changeset.root.hash()).build(); + let t = TrieDBBuilder::::new(&db, changeset.hash()).build(); for (i, kv) in t.iter().unwrap().enumerate() { let (k, v) = kv.unwrap(); let key: &[u8] = &data[i].0; diff --git a/trie-db/test/src/recorder.rs b/trie-db/test/src/recorder.rs index 3ee92375..2956520d 100644 --- a/trie-db/test/src/recorder.rs +++ b/trie-db/test/src/recorder.rs @@ -32,7 +32,7 @@ fn trie_record() { x.insert(b"pirate", b"aargh!").unwrap(); x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); let commit = x.commit(); - let root = *commit.root.hash(); + let root = *commit.hash(); commit.apply_to(&mut db); { diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 3a326810..3c8d9eca 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -24,7 +24,7 @@ use reference_trie::{ ReferenceNodeCodecNoExt, TestTrieCache, }; use trie_db::{ - CachedValue, ChangesetNodeRef, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, + CachedValue, Changeset, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, TrieLayout, Value, }; use trie_standardmap::*; @@ -862,7 +862,7 @@ fn attached_trie_internal>() { struct ATrie { root: TrieHash, data: BTreeMap, Vec>, - changeset: Option, T::Location>>>, + changeset: Option, T::Location>>>, } // Running a typical attached trie scenario (childtrie on substrate): // different trie, attached trie root written all @@ -908,7 +908,7 @@ fn attached_trie_internal>() { let root = changeset.root_hash(); attached_tries.insert( attached_trie_root_key.clone(), - ATrie { root, data, changeset: Some(changeset.root.into()) }, + ATrie { root, data, changeset: Some(changeset.into()) }, ); } } @@ -964,7 +964,7 @@ fn attached_trie_internal>() { let changeset = attached_trie.commit_with_keyspace(root_key); let new_root = changeset.root_hash(); assert!(new_root != a_attached_trie_root); - (changeset.root, new_root) + (changeset, new_root) }; let mut main_trie = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); main_trie From 28a5476ac9b30faff2eb650e1c21740ca9d0d0a9 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 18:56:55 +0100 Subject: [PATCH 45/90] move memorydb hashdb and mem-tree-db into triedb, except Hasher trait. --- Cargo.toml | 3 -- hash-db/CHANGELOG.md | 10 ----- hash-db/Cargo.toml | 14 ------ hash-db/README.md | 4 -- memory-db/CHANGELOG.md | 45 ------------------- memory-db/Cargo.toml | 25 ----------- test-support/keccak-hasher/Cargo.toml | 2 +- test-support/mem-tree-db/Cargo.toml | 15 ------- test-support/reference-trie/Cargo.toml | 3 -- test-support/reference-trie/src/lib.rs | 28 ++++++------ test-support/reference-trie/src/substrate.rs | 2 +- .../reference-trie/src/substrate_like.rs | 2 +- test-support/trie-bench/Cargo.toml | 2 - test-support/trie-bench/src/lib.rs | 10 +++-- test-support/trie-standardmap/Cargo.toml | 2 +- test-support/trie-standardmap/src/lib.rs | 4 +- trie-db/Cargo.toml | 13 ++++-- {memory-db => trie-db}/benches/bench.rs | 6 ++- trie-db/fuzz/Cargo.toml | 2 - trie-db/fuzz/src/lib.rs | 9 ++-- trie-db/src/iter_build.rs | 12 ++--- trie-db/src/iterator.rs | 6 +-- trie-db/src/lib.rs | 7 ++- trie-db/src/lookup.rs | 2 +- .../src/lib.rs => trie-db/src/mem_tree_db.rs | 35 +++++++-------- .../src/lib.rs => trie-db/src/memory_db.rs | 24 +++++----- trie-db/src/nibble/nibbleslice.rs | 3 +- trie-db/src/nibble/nibblevec.rs | 2 +- trie-db/src/node.rs | 2 +- trie-db/src/node_codec.rs | 2 +- hash-db/src/lib.rs => trie-db/src/node_db.rs | 30 +------------ trie-db/src/proof/generate.rs | 2 +- trie-db/src/proof/verify.rs | 2 +- trie-db/src/trie_codec.rs | 8 ++-- trie-db/src/triedb.rs | 4 +- trie-db/src/triedbmut.rs | 12 ++--- trie-db/test/Cargo.toml | 4 +- trie-db/test/src/double_ended_iterator.rs | 5 +-- trie-db/test/src/iter_build.rs | 6 ++- trie-db/test/src/iterator.rs | 2 +- trie-db/test/src/lib.rs | 18 ++++---- trie-db/test/src/proof.rs | 6 +-- trie-db/test/src/recorder.rs | 6 ++- trie-db/test/src/trie_codec.rs | 10 ++--- trie-db/test/src/triedb.rs | 9 ++-- trie-db/test/src/triedbmut.rs | 16 +++---- trie-root/Cargo.toml | 2 +- 47 files changed, 152 insertions(+), 286 deletions(-) delete mode 100644 hash-db/CHANGELOG.md delete mode 100644 hash-db/Cargo.toml delete mode 100644 hash-db/README.md delete mode 100644 memory-db/CHANGELOG.md delete mode 100644 memory-db/Cargo.toml delete mode 100644 test-support/mem-tree-db/Cargo.toml rename {memory-db => trie-db}/benches/bench.rs (96%) rename test-support/mem-tree-db/src/lib.rs => trie-db/src/mem_tree_db.rs (92%) rename memory-db/src/lib.rs => trie-db/src/memory_db.rs (96%) rename hash-db/src/lib.rs => trie-db/src/node_db.rs (75%) diff --git a/Cargo.toml b/Cargo.toml index e1db486b..4ae84c1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,7 @@ [workspace] members = [ - "hash-db", - "memory-db", "hash256-std-hasher", "test-support/keccak-hasher", - "test-support/mem-tree-db", "test-support/reference-trie", "test-support/trie-standardmap", "test-support/trie-bench", diff --git a/hash-db/CHANGELOG.md b/hash-db/CHANGELOG.md deleted file mode 100644 index e5a6a55b..00000000 --- a/hash-db/CHANGELOG.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [0.16.0] - 2023-03-14 -- Requires Hash to be Ord. [#188](https://github.com/paritytech/trie/pull/188) - - diff --git a/hash-db/Cargo.toml b/hash-db/Cargo.toml deleted file mode 100644 index d0c6f104..00000000 --- a/hash-db/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "hash-db" -version = "0.16.0" -authors = ["Parity Technologies "] -description = "Trait for hash-keyed databases." -license = "Apache-2.0" -categories = [ "no-std" ] -repository = "https://github.com/paritytech/trie" -edition = "2018" - -[features] -default = ["std"] -std = [ -] diff --git a/hash-db/README.md b/hash-db/README.md deleted file mode 100644 index d910c8fd..00000000 --- a/hash-db/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# NodeDB -`NodeDB` defines a common interface for databases of byte-slices keyed to their hash. It is generic over hash type through the `Hasher` trait. - -The `Hasher` trait can be used in a `no_std` context. diff --git a/memory-db/CHANGELOG.md b/memory-db/CHANGELOG.md deleted file mode 100644 index 532c86a4..00000000 --- a/memory-db/CHANGELOG.md +++ /dev/null @@ -1,45 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [0.32.0] - 2023-03-14 -- Switch no_std storage to BtreeMap. [#188](https://github.com/paritytech/trie/pull/188) - -## [0.31.0] - 2022-11-29 -- Removed `parity-util-mem` support. [#172](https://github.com/paritytech/trie/pull/172) - -## [0.30.0] - 2022-09-20 -- Update `parity-util-mem` to 0.12. [#166](https://github.com/paritytech/trie/pull/166) - -## [0.29.0] - 2022-02-04 -- Update `parity-util-mem` to 0.11. [#150](https://github.com/paritytech/trie/pull/150) - -## [0.28.0] - 2021-10-19 -- Change in api bound. [#142](https://github.com/paritytech/trie/pull/142) - -## [0.27.0] - 2021-07-02 -- Update `parity-util-mem` to 0.10. [#137](https://github.com/paritytech/trie/pull/137) - -## [0.26.0] - 2021-01-27 -- Update `parity-util-mem` to 0.9. [#123](https://github.com/paritytech/trie/pull/123) - -## [0.25.0] - 2021-01-05 -- Update `parity-util-mem` and `hashbrown`, removed `heapsize`. [#118](https://github.com/paritytech/trie/pull/118) - -## [0.24.1] - 2020-07-20 -- Add `shrink_to_fit` method. [#102](https://github.com/paritytech/trie/pull/102) - -## [0.24.0] - 2020-07-07 -- Disable memory tracking for no_std target by default. [#99](https://github.com/paritytech/trie/pull/99) - -## [0.22.0] - 2020-07-06 -- Type parameter to count `malloc_size_of` on memory-db. [#94](https://github.com/paritytech/trie/pull/94) -- Update hashbrown to 0.8. [#97](https://github.com/paritytech/trie/pull/97) - -## [0.20.0] - 2020-03-21 -- Update parity-util-mem to v0.6 [#82](https://github.com/paritytech/trie/pull/82) - -## [0.19.0] - 2020-02-07 -- Update parity-util-mem to v0.5.1 [#78](https://github.com/paritytech/trie/pull/78) diff --git a/memory-db/Cargo.toml b/memory-db/Cargo.toml deleted file mode 100644 index e800a2a2..00000000 --- a/memory-db/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "memory-db" -version = "0.32.0" -authors = ["Parity Technologies "] -description = "In-memory implementation of hash-db, useful for tests" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -hash-db = { version = "0.16.0", path = "../hash-db", default-features = false } - -[dev-dependencies] -keccak-hasher = { path = "../test-support/keccak-hasher" } -criterion = "0.5.1" - -[features] -default = ["std"] -std = [ - "hash-db/std", -] - -[[bench]] -name = "bench" -harness = false diff --git a/test-support/keccak-hasher/Cargo.toml b/test-support/keccak-hasher/Cargo.toml index d933d44d..7ed18d39 100644 --- a/test-support/keccak-hasher/Cargo.toml +++ b/test-support/keccak-hasher/Cargo.toml @@ -9,7 +9,7 @@ edition = "2018" [dependencies] tiny-keccak = { version = "2.0.2", features = ["keccak"] } -hash-db = { path = "../../hash-db", default-features = false, version = "0.16.0" } +hash-db = { version = "0.16.0", default-features = false } hash256-std-hasher = { path = "../../hash256-std-hasher", version = "0.15.2" } [features] diff --git a/test-support/mem-tree-db/Cargo.toml b/test-support/mem-tree-db/Cargo.toml deleted file mode 100644 index 23e4f0aa..00000000 --- a/test-support/mem-tree-db/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "mem-tree-db" -version = "0.0.1" -authors = ["Parity Technologies "] -description = "Simple reference trie format" -repository = "https://github.com/paritytech/trie/" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -hash-db = { path = "../../hash-db", version = "0.16.0"} -keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } -trie-db = { path = "../../trie-db", version = "0.28.0" } -trie-root = { path = "../../trie-root", version = "0.18.0" } -parity-scale-codec = { version = "3.0.0", features = ["derive"] } diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index b3145ca8..a60612e3 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -8,9 +8,6 @@ license = "Apache-2.0" edition = "2018" [dependencies] -hash-db = { path = "../../hash-db", version = "0.16.0"} -memory-db = { path = "../../memory-db", default-features = false, version = "0.32.0"} -mem-tree-db = { path = "../mem-tree-db", default-features = false, version = "0.0.1"} keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } trie-db = { path = "../../trie-db", default-features = false, version = "0.28.0" } trie-root = { path = "../../trie-root", default-features = false, version = "0.18.0" } diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index eb0f74bd..cd081e4d 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -15,13 +15,13 @@ //! Reference implementation of a streamer. use hashbrown::{hash_map::Entry, HashMap}; -use memory_db::MemoryDB; use parity_scale_codec::{Compact, Decode, Encode, Error as CodecError, Input, Output}; use std::{borrow::Borrow, fmt, iter::once, marker::PhantomData, ops::Range}; use trie_db::{ + memory_db::{KeyFunction, MemoryDB, PrefixedKey}, nibble_ops, node::{NibbleSlicePlan, NodeHandlePlan, NodeOwned, NodePlan, Value, ValuePlan}, - trie_visit, + node_db, trie_visit, triedbmut::ChildReference, DBValue, Location, NodeCodec, Trie, TrieBuilder, TrieConfiguration, TrieDBBuilder, TrieDBMutBuilder, TrieHash, TrieLayout, TrieRoot, @@ -40,15 +40,15 @@ pub use substrate_like::{ NodeCodec as ReferenceNodeCodecNoExtMeta, ReferenceTrieStreamNoExt, }; -pub use mem_tree_db::{Location as MemLocation, MemTreeDB}; pub use paste::paste; pub use substrate::{LayoutV0 as SubstrateV0, LayoutV1 as SubstrateV1}; +pub use trie_db::mem_tree_db::{Location as MemLocation, MemTreeDB}; /// Reference hasher is a keccak hasher. pub type RefHasher = keccak_hasher::KeccakHasher; pub type PrefixedMemoryDB = - MemoryDB<::Hash, memory_db::PrefixedKey<::Hash>, DBValue>; + MemoryDB<::Hash, PrefixedKey<::Hash>, DBValue>; /// Apply a test method on every test layouts. #[macro_export] @@ -947,7 +947,7 @@ pub fn compare_implementations(data: Vec<(Vec, Vec)>) where T: TrieLayout, T::Location: std::fmt::Debug, - K: memory_db::KeyFunction + Send + Sync, + K: KeyFunction + Send + Sync, { let (mut mem_db1, _) = MemoryDB::::default_with_root(); let (mut mem_db2, _) = MemoryDB::::default_with_root(); @@ -961,7 +961,7 @@ where }; if root_new != root { { - let db: &dyn hash_db::NodeDB<_, _, _> = &mem_db1; + let db: &dyn node_db::NodeDB<_, _, _> = &mem_db1; let t = TrieDBBuilder::::new(db, &root_new).build(); println!("{:?}", t); for a in t.iter().unwrap() { @@ -969,7 +969,7 @@ where } } { - let db: &dyn hash_db::NodeDB<_, _, _> = &mem_db2; + let db: &dyn node_db::NodeDB<_, _, _> = &mem_db2; let t = TrieDBBuilder::::new(db, &root).build(); println!("{:?}", t); for a in t.iter().unwrap() { @@ -984,7 +984,7 @@ where } /// Compare trie builder and trie root implementations. -pub fn compare_root>( +pub fn compare_root>( data: Vec<(Vec, Vec)>, memdb: DB, ) { @@ -1041,14 +1041,14 @@ where /// Trie builder trie building utility. pub fn calc_root_build( data: I, - memdb: &mut memory_db::MemoryDB, + memdb: &mut MemoryDB, ) -> TrieHash where T: TrieLayout, I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - K: memory_db::KeyFunction + Send + Sync, + K: KeyFunction + Send + Sync, { let mut cb = TrieBuilder::::new(memdb); trie_visit::(data.into_iter(), &mut cb); @@ -1062,7 +1062,7 @@ pub fn compare_implementations_unordered(data: Vec<(Vec, Vec)>) where T: TrieLayout, T::Location: std::fmt::Debug, - K: memory_db::KeyFunction + Send + Sync, + K: KeyFunction + Send + Sync, { let mut mem_db1 = MemoryDB::::default(); let mut mem_db2 = MemoryDB::::default(); @@ -1083,7 +1083,7 @@ where if root != root_new { { - let db: &dyn hash_db::NodeDB<_, _, _> = &mem_db1; + let db: &dyn node_db::NodeDB<_, _, _> = &mem_db1; let t = TrieDBBuilder::::new(db, &root).build(); println!("{:?}", t); for a in t.iter().unwrap() { @@ -1091,7 +1091,7 @@ where } } { - let db: &dyn hash_db::NodeDB<_, _, _> = &mem_db2; + let db: &dyn node_db::NodeDB<_, _, _> = &mem_db2; let t = TrieDBBuilder::::new(db, &root_new).build(); println!("{:?}", t); for a in t.iter().unwrap() { @@ -1108,7 +1108,7 @@ where pub fn compare_insert_remove(data: Vec<(bool, Vec, Vec)>) where T: TrieLayout, - K: memory_db::KeyFunction + Send + Sync, + K: KeyFunction + Send + Sync, { let mut data2 = std::collections::BTreeMap::new(); let mut a = 0; diff --git a/test-support/reference-trie/src/substrate.rs b/test-support/reference-trie/src/substrate.rs index 7e5d1d60..a99c301a 100644 --- a/test-support/reference-trie/src/substrate.rs +++ b/test-support/reference-trie/src/substrate.rs @@ -17,12 +17,12 @@ //! Codec and layout directly copy-pasted from substrate with minimal modifications. use core::{borrow::Borrow, iter::once, marker::PhantomData, ops::Range}; -use hash_db::Hasher; use parity_scale_codec as codec; use parity_scale_codec::{Compact, Decode, Encode, Input, Output}; use trie_db::{ nibble_ops, node::{NibbleSlicePlan, NodeHandlePlan, NodePlan, Value, ValuePlan}, + node_db::Hasher, ChildReference, NodeCodec as NodeCodecT, TrieConfiguration, TrieLayout, }; diff --git a/test-support/reference-trie/src/substrate_like.rs b/test-support/reference-trie/src/substrate_like.rs index de7b634a..183a8a7e 100644 --- a/test-support/reference-trie/src/substrate_like.rs +++ b/test-support/reference-trie/src/substrate_like.rs @@ -31,7 +31,7 @@ impl TrieLayout for HashedValueNoExt { type Hash = RefHasher; type Codec = ReferenceNodeCodecNoExtMeta; - type Location = mem_tree_db::Location; + type Location = trie_db::mem_tree_db::Location; } impl TrieLayout for HashedValueNoExtThreshold { diff --git a/test-support/trie-bench/Cargo.toml b/test-support/trie-bench/Cargo.toml index 694548a4..b79f4b86 100644 --- a/test-support/trie-bench/Cargo.toml +++ b/test-support/trie-bench/Cargo.toml @@ -10,8 +10,6 @@ edition = "2018" [dependencies] keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } trie-standardmap = { path = "../trie-standardmap", version = "0.16.0" } -hash-db = { path = "../../hash-db" , version = "0.16.0"} -memory-db = { path = "../../memory-db", version = "0.32.0" } trie-root = { path = "../../trie-root", version = "0.18.0" } trie-db = { path = "../../trie-db", version = "0.28.0" } criterion = "0.5.1" diff --git a/test-support/trie-bench/src/lib.rs b/test-support/trie-bench/src/lib.rs index 2ab7c578..6b6c44a8 100644 --- a/test-support/trie-bench/src/lib.rs +++ b/test-support/trie-bench/src/lib.rs @@ -15,12 +15,14 @@ //! Standard trie benchmarking tool. use criterion::{black_box, BenchmarkId, Criterion}; -use hash_db::Hasher; use keccak_hasher::KeccakHasher; -use memory_db::{HashKey, MemoryDB}; use parity_scale_codec::{Compact, Encode}; use std::default::Default; -use trie_db::{NodeCodec, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout}; +use trie_db::{ + memory_db::{HashKey, MemoryDB}, + node_db::Hasher, + NodeCodec, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, +}; use trie_root::{trie_root, TrieStream}; use trie_standardmap::*; @@ -79,7 +81,7 @@ fn benchmark( } t.commit() }; - let root = commit.root.hash(); + let root = commit.hash(); commit.apply_to(&mut memdb); b.iter(&mut || { let t = TrieDBBuilder::::new(&memdb, &root).build(); diff --git a/test-support/trie-standardmap/Cargo.toml b/test-support/trie-standardmap/Cargo.toml index b1d2d199..7769b1c2 100644 --- a/test-support/trie-standardmap/Cargo.toml +++ b/test-support/trie-standardmap/Cargo.toml @@ -8,4 +8,4 @@ edition = "2018" [dependencies] keccak-hasher = { path = "../keccak-hasher", version = "0.16.0"} -hash-db = { path = "../../hash-db" , version = "0.16.0"} +trie-db = { path = "../../trie-db" , version = "0.28.0"} diff --git a/test-support/trie-standardmap/src/lib.rs b/test-support/trie-standardmap/src/lib.rs index 8e01ad12..23f98d7f 100644 --- a/test-support/trie-standardmap/src/lib.rs +++ b/test-support/trie-standardmap/src/lib.rs @@ -14,10 +14,10 @@ //! Key-value datastore with a modified Merkle tree. -use hash_db::Hasher; use keccak_hasher::KeccakHasher; +use trie_db::node_db::Hasher; -type H256 = ::Out; +type H256 = ::Out; /// Alphabet to use when creating words for insertion into tries. pub enum Alphabet { diff --git a/trie-db/Cargo.toml b/trie-db/Cargo.toml index 4dfa6932..4828356d 100644 --- a/trie-db/Cargo.toml +++ b/trie-db/Cargo.toml @@ -7,16 +7,23 @@ repository = "https://github.com/paritytech/trie" license = "Apache-2.0" edition = "2018" +[[bench]] +name = "bench" +harness = false + [dependencies] log = "0.4" smallvec = { version = "1.0.0", features = ["union", "const_new"] } -hash-db = { path = "../hash-db", default-features = false, version = "0.16.0"} -memory-db = { path = "../memory-db", default-features = false, version = "0.32.0"} rustc-hex = { version = "2.1.0", default-features = false, optional = true } +hash-db = { version = "0.16.0", default-features = false } + +[dev-dependencies] +keccak-hasher = { path = "../test-support/keccak-hasher", version = "0.16.0"} +criterion = "0.5.1" [features] default = ["std"] std = [ - "hash-db/std", "rustc-hex", + "hash-db/std", ] diff --git a/memory-db/benches/bench.rs b/trie-db/benches/bench.rs similarity index 96% rename from memory-db/benches/bench.rs rename to trie-db/benches/bench.rs index 455d0a30..e665bf7c 100644 --- a/memory-db/benches/bench.rs +++ b/trie-db/benches/bench.rs @@ -13,9 +13,11 @@ // limitations under the License. use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use hash_db::{Hasher, EMPTY_PREFIX}; use keccak_hasher::KeccakHasher; -use memory_db::{HashKey, MemoryDB}; +use trie_db::{ + memory_db::{HashKey, MemoryDB}, + node_db::{Hasher, EMPTY_PREFIX}, +}; criterion_group!( benches, diff --git a/trie-db/fuzz/Cargo.toml b/trie-db/fuzz/Cargo.toml index 84a03257..20fd9146 100644 --- a/trie-db/fuzz/Cargo.toml +++ b/trie-db/fuzz/Cargo.toml @@ -9,8 +9,6 @@ edition = "2018" cargo-fuzz = true [dependencies] -hash-db = { path = "../../hash-db", version = "0.16.0" } -memory-db = { path = "../../memory-db", version = "0.32.0" } reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } arbitrary = { version = "1.3.0", features = ["derive"] } array-bytes = "6.0.0" diff --git a/trie-db/fuzz/src/lib.rs b/trie-db/fuzz/src/lib.rs index 69dbeb28..a923b747 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/trie-db/fuzz/src/lib.rs @@ -13,13 +13,13 @@ // limitations under the License. use arbitrary::Arbitrary; -use hash_db::Hasher; -use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ calc_root, compare_insert_remove, reference_trie_root_iter_build as reference_trie_root, }; use std::{convert::TryInto, fmt::Debug}; use trie_db::{ + memory_db::{HashKey, MemoryDB, PrefixedKey}, + node_db::Hasher, proof::{generate_proof, verify_proof}, DBValue, Trie, TrieDBBuilder, TrieDBIterator, TrieDBMutBuilder, TrieLayout, }; @@ -130,7 +130,7 @@ fn data_sorted_unique(input: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec)> m.into_iter().collect() } -pub fn fuzz_that_compare_implementations(input: &[u8]) +pub fn fuzz_that_compare_implementations(input: &[u8]) where T::Location: Debug, { let data = data_sorted_unique(fuzz_to_data(input)); @@ -405,8 +405,7 @@ fn test_generate_proof( } fn test_trie_codec_proof(entries: Vec<(Vec, Vec)>, keys: Vec>) { - use hash_db::EMPTY_PREFIX; - use trie_db::{decode_compact, encode_compact, Recorder}; + use trie_db::{node_db::EMPTY_PREFIX, decode_compact, encode_compact, Recorder}; // Populate DB with full trie from entries. let (db, root) = { diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 683165a9..3fc2404a 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -18,15 +18,15 @@ //! See `trie_visit` function. use crate::{ + memory_db::{KeyFunction, MemoryDB}, nibble::{nibble_ops, NibbleSlice}, node::Value, node_codec::NodeCodec, + node_db::{Hasher, Prefix, EMPTY_PREFIX}, rstd::{cmp::max, marker::PhantomData, vec::Vec}, triedbmut::ChildReference, DBValue, TrieHash, TrieLayout, }; -use hash_db::{Hasher, Prefix}; -use memory_db::MemoryDB; macro_rules! exponential_out { (@3, [$($inpp:expr),*]) => { exponential_out!(@2, [$($inpp,)* $($inpp),*]) }; @@ -331,7 +331,7 @@ where } } else { // nothing null root corner case - callback.process(hash_db::EMPTY_PREFIX, T::Codec::empty_node().to_vec(), true); + callback.process(EMPTY_PREFIX, T::Codec::empty_node().to_vec(), true); } } @@ -358,18 +358,18 @@ pub trait ProcessEncodedNode { /// Get trie root and insert visited node in a hash_db. /// As for all `ProcessEncodedNode` implementation, it /// is only for full trie parsing (not existing trie). -pub struct TrieBuilder<'a, T: TrieLayout, K: memory_db::KeyFunction + Send + Sync> { +pub struct TrieBuilder<'a, T: TrieLayout, K: KeyFunction + Send + Sync> { db: &'a mut MemoryDB, pub root: Option>, } -impl<'a, T: TrieLayout, K: memory_db::KeyFunction + Send + Sync> TrieBuilder<'a, T, K> { +impl<'a, T: TrieLayout, K: KeyFunction + Send + Sync> TrieBuilder<'a, T, K> { pub fn new(db: &'a mut MemoryDB) -> Self { TrieBuilder { db, root: None } } } -impl<'a, T, K: memory_db::KeyFunction + Send + Sync> ProcessEncodedNode> +impl<'a, T, K: KeyFunction + Send + Sync> ProcessEncodedNode> for TrieBuilder<'a, T, K> where T: TrieLayout, diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index e83ec47d..dd077719 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -16,10 +16,10 @@ use super::{CError, DBValue, Result, Trie, TrieHash, TrieIterator, TrieLayout}; use crate::{ nibble::{nibble_ops, NibbleSlice, NibbleVec}, node::{Node, NodeHandle, NodePlan, OwnedNode, Value}, + node_db::{self, Prefix, EMPTY_PREFIX}, triedb::TrieDB, TrieDoubleEndedIterator, TrieError, TrieItem, TrieKeyItem, }; -use hash_db::{Prefix, EMPTY_PREFIX}; use crate::rstd::{boxed::Box, sync::Arc, vec::Vec}; @@ -730,7 +730,7 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeIterator<'a, 'cache, L> { } /// Access inner hash db. - pub fn db(&self) -> &dyn hash_db::NodeDB { + pub fn db(&self) -> &dyn node_db::NodeDB { self.db.db() } @@ -828,7 +828,7 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeDoubleEndedIterator<'a, 'cache, L> { } /// Access inner hash db. - pub fn db(&self) -> &dyn hash_db::NodeDB { + pub fn db(&self) -> &dyn node_db::NodeDB { self.db.db() } } diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 6f4b32ff..4fa599d2 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -45,11 +45,14 @@ mod rstd { use self::rstd::{fmt, Error}; use self::rstd::{boxed::Box, vec::Vec}; -use hash_db::MaybeDebug; pub use iterator::TrieDBNodeDoubleEndedIterator; use node::NodeOwned; +use node_db::MaybeDebug; +pub mod mem_tree_db; +pub mod memory_db; pub mod node; +pub mod node_db; pub mod proof; pub mod recorder; pub mod triedb; @@ -72,13 +75,13 @@ pub use self::{ TrieDBMutBuilder, Value, }, }; +use crate::node_db::Hasher; pub use crate::{ iter_build::{trie_visit, ProcessEncodedNode, TrieBuilder, TrieRoot, TrieRootUnhashed}, iterator::{TrieDBNodeIterator, TrieDBRawIterator}, node_codec::{NodeCodec, Partial}, trie_codec::{decode_compact, decode_compact_from_iter, encode_compact}, }; -pub use hash_db::{Hasher, NodeDB}; #[cfg(feature = "std")] pub use crate::iter_build::TrieRootPrint; diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs index 6bf037f8..45accfeb 100644 --- a/trie-db/src/lookup.rs +++ b/trie-db/src/lookup.rs @@ -18,11 +18,11 @@ use crate::{ nibble::NibbleSlice, node::{decode_hash, Node, NodeHandle, NodeHandleOwned, NodeOwned, Value, ValueOwned}, node_codec::NodeCodec, + node_db::{Hasher, NodeDB, Prefix}, rstd::{boxed::Box, vec::Vec}, Bytes, CError, CachedValue, DBValue, MerkleValue, Query, RecordedForKey, Result, TrieAccess, TrieCache, TrieError, TrieHash, TrieLayout, TrieRecorder, }; -use hash_db::{Hasher, NodeDB, Prefix}; /// Trie lookup helper object. pub struct Lookup<'a, 'cache, L: TrieLayout, Q: Query> { diff --git a/test-support/mem-tree-db/src/lib.rs b/trie-db/src/mem_tree_db.rs similarity index 92% rename from test-support/mem-tree-db/src/lib.rs rename to trie-db/src/mem_tree_db.rs index 2291e9a5..05ac2c44 100644 --- a/test-support/mem-tree-db/src/lib.rs +++ b/trie-db/src/mem_tree_db.rs @@ -16,8 +16,10 @@ use std::collections::HashMap; -use hash_db::{Hasher, NodeDB, Prefix}; -use trie_db::{Changeset, NewChangesetNode}; +use crate::{ + node_db::{Hasher, NodeDB, Prefix}, + Changeset, NewChangesetNode, +}; /// Node location which is just an index into the `nodes` vector. pub type Location = Option; @@ -223,11 +225,12 @@ where #[cfg(test)] mod tests { - - use super::{Location, MemTreeDB, NodeEntry}; - use hash_db::{Hasher, NodeDB}; + use super::{MemTreeDB, NodeEntry}; + use crate::{ + node_db::{Hasher, NodeDB}, + Changeset, ExistingChangesetNode, NewChangesetNode, + }; use keccak_hasher::{KeccakHash, KeccakHasher}; - use trie_db::{Changeset, ExistingChangesetNode, NewChangesetNode}; fn hash(i: u32) -> KeccakHash { KeccakHasher::hash(&i.to_le_bytes()) @@ -275,16 +278,13 @@ mod tests { #[test] fn test_apply_commit() { let mut db = MemTreeDB::::default(); - let commit = Changeset:: { - root: Changeset::New(NewChangesetNode { - hash: KeccakHash::default(), - prefix: Default::default(), - data: vec![1, 2, 3], - children: vec![], - removed_keys: None, - }), - removed: Default::default(), - }; + let commit = Changeset::New(NewChangesetNode { + hash: KeccakHash::default(), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + removed_keys: None, + }); db.apply_commit(commit); assert_eq!(db.roots.len(), 1); } @@ -310,7 +310,7 @@ mod tests { }); // Create a root node that refers to the child nodes - let root = Changeset::New(NewChangesetNode { + let commit = Changeset::New(NewChangesetNode { hash: hash(0), prefix: Default::default(), data: vec![7, 8, 9], @@ -318,7 +318,6 @@ mod tests { removed_keys: None, }); - let commit = Changeset:: { root, removed: Default::default() }; db.apply_commit(commit); // Check that the root node and child nodes are in the database diff --git a/memory-db/src/lib.rs b/trie-db/src/memory_db.rs similarity index 96% rename from memory-db/src/lib.rs rename to trie-db/src/memory_db.rs index 7331a008..fbcc4cac 100644 --- a/memory-db/src/lib.rs +++ b/trie-db/src/memory_db.rs @@ -19,7 +19,7 @@ #[cfg(not(feature = "std"))] extern crate alloc; -use hash_db::{Hasher as KeyHasher, MaybeDebug, NodeDB, Prefix}; +use crate::node_db::{Hasher as KeyHasher, MaybeDebug, NodeDB, Prefix}; #[cfg(feature = "std")] use std::{ cmp::Eq, collections::hash_map::Entry, collections::HashMap as Map, hash, marker::PhantomData, @@ -44,9 +44,10 @@ use alloc::vec::Vec; /// /// # Example /// ```rust -/// use hash_db::{Hasher, EMPTY_PREFIX}; +/// use trie_db::node_db::Hasher; +/// use trie_db::node_db::{EMPTY_PREFIX}; /// use keccak_hasher::KeccakHasher; -/// use memory_db::{MemoryDB, HashKey}; +/// use trie_db::memory_db::{MemoryDB, HashKey}; /// /// let mut m = MemoryDB::, Vec>::default(); /// let d = "Hello world!".as_bytes(); @@ -283,13 +284,10 @@ where /// /// # Examples /// ```rust - /// extern crate hash_db; - /// extern crate keccak_hasher; - /// extern crate memory_db; - /// - /// use hash_db::{Hasher, NodeDB, EMPTY_PREFIX}; + /// use trie_db::node_db::Hasher; + /// use trie_db::node_db::{NodeDB, EMPTY_PREFIX}; /// use keccak_hasher::KeccakHasher; - /// use memory_db::{MemoryDB, HashKey}; + /// use trie_db::memory_db::{MemoryDB, HashKey}; /// /// fn main() { /// let mut m = MemoryDB::, Vec>::default(); @@ -451,9 +449,11 @@ where } #[cfg(test)] -mod tests { - use super::{HashKey, KeyHasher, MemoryDB}; - use hash_db::EMPTY_PREFIX; +mod test { + use crate::{ + memory_db::{HashKey, MemoryDB}, + node_db::{Hasher as KeyHasher, EMPTY_PREFIX}, + }; use keccak_hasher::KeccakHasher; #[test] diff --git a/trie-db/src/nibble/nibbleslice.rs b/trie-db/src/nibble/nibbleslice.rs index f91fad7f..e5e062a9 100644 --- a/trie-db/src/nibble/nibbleslice.rs +++ b/trie-db/src/nibble/nibbleslice.rs @@ -17,8 +17,7 @@ use super::{nibble_ops, BackingByteVec, NibbleSlice, NibbleSliceIterator, NibbleVec}; #[cfg(feature = "std")] use crate::rstd::fmt; -use crate::{node::NodeKey, node_codec::Partial, rstd::cmp::*}; -use hash_db::Prefix; +use crate::{node::NodeKey, node_codec::Partial, node_db::Prefix, rstd::cmp::*}; impl<'a> Iterator for NibbleSliceIterator<'a> { type Item = u8; diff --git a/trie-db/src/nibble/nibblevec.rs b/trie-db/src/nibble/nibblevec.rs index 3d3c47bf..c2256a49 100644 --- a/trie-db/src/nibble/nibblevec.rs +++ b/trie-db/src/nibble/nibblevec.rs @@ -19,8 +19,8 @@ use crate::{ nibble::{nibble_ops, BackingByteVec, NibbleSlice}, node::NodeKey, node_codec::Partial, + node_db::Prefix, }; -use hash_db::Prefix; impl Default for NibbleVec { fn default() -> Self { diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index ab36a222..57ebc660 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -15,11 +15,11 @@ use crate::{ nibble::{self, nibble_ops, NibbleSlice, NibbleVec}, node_codec::NodeCodec, + node_db::Hasher, Bytes, CError, ChildReference, Result, TrieError, TrieHash, TrieLayout, }; #[cfg(not(feature = "std"))] use alloc::{boxed::Box, vec::Vec}; -use hash_db::Hasher; use crate::rstd::{borrow::Borrow, mem, ops::Range}; diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index fd7ec536..c4aa2635 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Generic trait for trie node encoding/decoding. Takes a `hash_db::Hasher` +//! Generic trait for trie node encoding/decoding. Takes a `trie_db::node_db::Hasher` //! to parametrize the hashes used in the codec. use crate::{ diff --git a/hash-db/src/lib.rs b/trie-db/src/node_db.rs similarity index 75% rename from hash-db/src/lib.rs rename to trie-db/src/node_db.rs index c542617a..745e4577 100644 --- a/hash-db/src/lib.rs +++ b/trie-db/src/node_db.rs @@ -25,8 +25,8 @@ use alloc::vec::Vec; use core::hash; #[cfg(feature = "std")] use std::fmt::Debug; -#[cfg(feature = "std")] -use std::hash; + +pub use hash_db::Hasher; #[cfg(feature = "std")] pub trait MaybeDebug: Debug {} @@ -52,32 +52,6 @@ pub type Prefix<'a> = (&'a [u8], Option); /// or for root nodes. pub static EMPTY_PREFIX: Prefix<'static> = (&[], None); -/// Trait describing an object that can hash a slice of bytes. Used to abstract -/// other types over the hashing algorithm. Defines a single `hash` method and an -/// `Out` associated type with the necessary bounds. -pub trait Hasher: Sync + Send { - /// The output type of the `Hasher` - type Out: AsRef<[u8]> - + AsMut<[u8]> - + Default - + MaybeDebug - + core::cmp::Ord - + PartialEq - + Eq - + hash::Hash - + Send - + Sync - + Clone - + Copy; - /// What to use to build `HashMap`s with this `Hasher`. - type StdHasher: Sync + Send + Default + hash::Hasher; - /// The length in bytes of the `Hasher` output. - const LENGTH: usize; - - /// Compute the hash of the provided slice of bytes returning the `Out` type of the `Hasher`. - fn hash(x: &[u8]) -> Self::Out; -} - /// Trait modelling datastore keyed by a hash defined by the `Hasher` and optional location tag. pub trait NodeDB: Send + Sync { /// Look up a trie node by hash and location. diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index f9a58ece..a3b813fd 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -16,7 +16,7 @@ use crate::rstd::{boxed::Box, convert::TryInto, marker::PhantomData, vec, vec::Vec}; -use hash_db::{Hasher, NodeDB}; +use crate::node_db::{Hasher, NodeDB}; use crate::{ nibble::LeftNibbleSlice, diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index df76fe43..d0e0cde1 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -18,10 +18,10 @@ use crate::{ nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, Value}, + node_db::Hasher, rstd::{convert::TryInto, iter::Peekable, marker::PhantomData, result::Result, vec, vec::Vec}, CError, ChildReference, NodeCodec, TrieHash, TrieLayout, }; -use hash_db::Hasher; /// Errors that may occur during proof verification. Most of the errors types simply indicate that /// the proof is invalid with respect to the statement being verified, and the exact error type can diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 25bce661..c401f55c 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -26,14 +26,14 @@ //! trie. use crate::{ + memory_db::{HashKey, MemoryDB}, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, ValuePlan}, + node_db::Prefix, rstd::{boxed::Box, convert::TryInto, marker::PhantomData, result, sync::Arc, vec, vec::Vec}, CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, TrieDB, TrieDBRawIterator, TrieError, TrieHash, TrieLayout, }; -use hash_db::Prefix; -use memory_db::MemoryDB; const OMIT_VALUE_HASH: crate::node::Value<'static, ()> = crate::node::Value::Inline(&[]); @@ -465,7 +465,7 @@ impl<'a, C: NodeCodec, L: Copy + Default> DecoderStackEntry<'a, C, L> { /// This function makes the assumption that all child references in an inline trie node are inline /// references. pub fn decode_compact( - db: &mut MemoryDB, DBValue>, + db: &mut MemoryDB, DBValue>, encoded: &[Vec], ) -> Result<(TrieHash, usize), TrieHash, CError> where @@ -476,7 +476,7 @@ where /// Variant of 'decode_compact' that accept an iterator of encoded nodes as input. pub fn decode_compact_from_iter<'a, L, I>( - db: &mut MemoryDB, DBValue>, + db: &mut MemoryDB, DBValue>, encoded: I, ) -> Result<(TrieHash, usize), TrieHash, CError> where diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index aa23b2b1..7306e74c 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -17,6 +17,7 @@ use crate::{ lookup::Lookup, nibble::NibbleSlice, node::{decode_hash, NodeHandle, OwnedNode}, + node_db::{NodeDB, Prefix, EMPTY_PREFIX}, rstd::boxed::Box, CError, DBValue, MerkleValue, Query, Result, Trie, TrieAccess, TrieCache, TrieDoubleEndedIterator, TrieError, TrieHash, TrieItem, TrieIterator, TrieKeyItem, TrieLayout, @@ -28,7 +29,6 @@ use crate::{ node::Node, rstd::{fmt, vec::Vec}, }; -use hash_db::{NodeDB, Prefix, EMPTY_PREFIX}; /// A builder for creating a [`TrieDB`]. pub struct TrieDBBuilder<'db, 'cache, L: TrieLayout> { @@ -120,7 +120,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// /// # Example /// ```ignore -/// use hash_db::Hasher; +/// use trie_db::node_db::Hasher; /// use reference_trie::{RefTrieDBMut, RefTrieDB, Trie, TrieMut}; /// use trie_db::DBValue; /// use keccak_hasher::KeccakHasher; diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 94e2f975..1fe93844 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -27,7 +27,7 @@ use crate::{ TrieRecorder, }; -use hash_db::{Hasher, NodeDB, Prefix}; +use crate::node_db::{Hasher, NodeDB, Prefix}; #[cfg(feature = "std")] use std::collections::HashSet as Set; @@ -35,9 +35,9 @@ use std::collections::HashSet as Set; #[cfg(not(feature = "std"))] use alloc::collections::btree_set::BTreeSet as Set; +use crate::memory_db::{KeyFunction, MemoryDB}; #[cfg(feature = "std")] use log::trace; -use memory_db::MemoryDB; #[cfg(feature = "std")] use crate::rstd::fmt::{self, Debug}; @@ -791,7 +791,7 @@ pub fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { impl Changeset { pub fn apply_to(&self, mem_db: &mut MemoryDB) -> H where - K: memory_db::KeyFunction + Send + Sync, + K: KeyFunction + Send + Sync, MH: Hasher + Send + Sync, { fn apply_node<'a, H, DL, MH, K>( @@ -799,7 +799,7 @@ impl Changeset { mem_db: &mut MemoryDB, mut ks: Option<&'a [u8]>, ) where - K: memory_db::KeyFunction + Send + Sync, + K: KeyFunction + Send + Sync, MH: Hasher + Send + Sync, { match node { @@ -850,11 +850,11 @@ pub type OwnedPrefix = (BackingByteVec, Option); /// /// # Example /// ```ignore -/// use hash_db::Hasher; +/// use trie_db::node_db::Hasher; /// use reference_trie::{RefTrieDBMut, TrieMut}; /// use trie_db::DBValue; /// use keccak_hasher::KeccakHasher; -/// use memory_db::*; +/// use trie_db::memory_db::*; /// /// let mut memdb = MemoryDB::, DBValue>::default(); /// let mut root = Default::default(); diff --git a/trie-db/test/Cargo.toml b/trie-db/test/Cargo.toml index a1fa3f03..6eb436c5 100644 --- a/trie-db/test/Cargo.toml +++ b/trie-db/test/Cargo.toml @@ -13,12 +13,10 @@ harness = false [dependencies] trie-db = { path = "..", version = "0.28.0"} -hash-db = { path = "../../hash-db", version = "0.16.0"} -memory-db = { path = "../../memory-db", version = "0.32.0" } rand = { version = "0.8", default-features = false, features = ["small_rng"] } trie-standardmap = { path = "../../test-support/trie-standardmap", version = "0.16.0" } reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } -mem-tree-db = { path = "../../test-support/mem-tree-db", version = "0.0.1" } +keccak-hasher = { path = "../../test-support/keccak-hasher", version = "0.16.0" } hex-literal = "0.4" criterion = "0.5.1" env_logger = { version = "0.10", default-features = false } diff --git a/trie-db/test/src/double_ended_iterator.rs b/trie-db/test/src/double_ended_iterator.rs index fd218469..77fa4ce1 100644 --- a/trie-db/test/src/double_ended_iterator.rs +++ b/trie-db/test/src/double_ended_iterator.rs @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::Hasher; use hex_literal::hex; use reference_trie::test_layouts; use trie_db::{ - node::Node, NibbleSlice, TrieDBBuilder, TrieDBNodeDoubleEndedIterator, TrieDoubleEndedIterator, - TrieLayout, + node::Node, node_db::Hasher, NibbleSlice, TrieDBBuilder, TrieDBNodeDoubleEndedIterator, + TrieDoubleEndedIterator, TrieLayout, }; use crate::{ diff --git a/trie-db/test/src/iter_build.rs b/trie-db/test/src/iter_build.rs index c2f424b0..546f6ed9 100644 --- a/trie-db/test/src/iter_build.rs +++ b/trie-db/test/src/iter_build.rs @@ -12,11 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ test_layouts, ExtensionLayout, HashedValueNoExt, HashedValueNoExtThreshold, NoExtensionLayout, }; -use trie_db::{DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout}; +use trie_db::{ + memory_db::{HashKey, MemoryDB, PrefixedKey}, + DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, +}; use crate::TestDB; diff --git a/trie-db/test/src/iterator.rs b/trie-db/test/src/iterator.rs index 6cd79335..4225e5b0 100644 --- a/trie-db/test/src/iterator.rs +++ b/trie-db/test/src/iterator.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::Hasher; use hex_literal::hex; use reference_trie::test_layouts; use trie_db::{ node::{Node, Value}, + node_db::Hasher, NibbleSlice, NibbleVec, TrieDBBuilder, TrieDBNodeIterator, TrieError, TrieIterator, TrieLayout, }; diff --git a/trie-db/test/src/lib.rs b/trie-db/test/src/lib.rs index 5f5f4cd9..31ad62ae 100644 --- a/trie-db/test/src/lib.rs +++ b/trie-db/test/src/lib.rs @@ -14,7 +14,7 @@ //! Tests for trie-db crate. -use memory_db::MemoryDB; +use trie_db::memory_db::{KeyFunction, MemoryDB}; #[cfg(test)] mod double_ended_iterator; @@ -33,11 +33,13 @@ mod triedb; #[cfg(test)] mod triedbmut; -use hash_db::{Hasher, Prefix}; -use mem_tree_db::MemTreeDB; -use trie_db::{Changeset, DBValue, TrieHash, TrieLayout}; +use trie_db::{ + mem_tree_db::{Location, MemTreeDB}, + node_db::{self, Hasher, Prefix}, + Changeset, DBValue, TrieHash, TrieLayout, +}; -trait TestDB: hash_db::NodeDB + Clone + Default { +trait TestDB: node_db::NodeDB + Clone + Default { fn commit( &mut self, commit: trie_db::Changeset<<::Hash as Hasher>::Out, T::Location>, @@ -52,7 +54,7 @@ trait TestDB: hash_db::NodeDB + Cl impl, H, KF> TestDB for MemoryDB where H: Hasher, - KF: memory_db::KeyFunction + Send + Sync, + KF: KeyFunction + Send + Sync, { fn commit( &mut self, @@ -70,11 +72,11 @@ where } } -impl, H> TestDB for MemTreeDB +impl, H> TestDB for MemTreeDB where H: Hasher + Clone, { - fn commit(&mut self, commit: trie_db::Changeset) -> H::Out { + fn commit(&mut self, commit: trie_db::Changeset) -> H::Out { let root = commit.root_hash(); self.apply_commit(commit); root diff --git a/trie-db/test/src/proof.rs b/trie-db/test/src/proof.rs index e1ee342d..331f43b0 100644 --- a/trie-db/test/src/proof.rs +++ b/trie-db/test/src/proof.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::Hasher; use reference_trie::{test_layouts, NoExtensionLayout}; +use trie_db::node_db::Hasher; use trie_db::{ proof::{generate_proof, verify_proof, VerifyError}, @@ -22,9 +22,9 @@ use trie_db::{ use crate::TestDB; -type MemoryDB = memory_db::MemoryDB< +type MemoryDB = trie_db::memory_db::MemoryDB< ::Hash, - memory_db::HashKey<::Hash>, + trie_db::memory_db::HashKey<::Hash>, DBValue, >; diff --git a/trie-db/test/src/recorder.rs b/trie-db/test/src/recorder.rs index 2956520d..baa39ad8 100644 --- a/trie-db/test/src/recorder.rs +++ b/trie-db/test/src/recorder.rs @@ -14,9 +14,11 @@ //! Trie query recorder. -use memory_db::{HashKey, MemoryDB}; use reference_trie::{NoExtensionLayout, RefHasher, RefTrieDBBuilder, RefTrieDBMutBuilder}; -use trie_db::{Recorder, Trie}; +use trie_db::{ + memory_db::{HashKey, MemoryDB}, + Recorder, Trie, +}; #[test] fn trie_record() { diff --git a/trie-db/test/src/trie_codec.rs b/trie-db/test/src/trie_codec.rs index 77ffcaa8..6227bd2b 100644 --- a/trie-db/test/src/trie_codec.rs +++ b/trie-db/test/src/trie_codec.rs @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::{Hasher, EMPTY_PREFIX}; use reference_trie::{test_layouts, ExtensionLayout}; use trie_db::{ - decode_compact, encode_compact, DBValue, NodeCodec, Recorder, Trie, TrieDBBuilder, - TrieDBMutBuilder, TrieError, TrieLayout, + decode_compact, encode_compact, + node_db::{Hasher, EMPTY_PREFIX}, + DBValue, NodeCodec, Recorder, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieError, TrieLayout, }; use crate::TestDB; -type MemoryDB = memory_db::MemoryDB< +type MemoryDB = trie_db::memory_db::MemoryDB< ::Hash, - memory_db::HashKey<::Hash>, + trie_db::memory_db::HashKey<::Hash>, DBValue, >; diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index 4efd34d1..0ff0b18a 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -14,16 +14,17 @@ use std::ops::Deref; -use hash_db::{Hasher, EMPTY_PREFIX}; use hex_literal::hex; -use memory_db::{HashKey, MemoryDB}; use reference_trie::{ test_layouts, test_layouts_substrate, HashedValueNoExtThreshold, PrefixedMemoryDB, TestTrieCache, }; use trie_db::{ - encode_compact, CachedValue, DBValue, Lookup, NibbleSlice, RecordedForKey, Recorder, Trie, - TrieCache, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieRecorder, + encode_compact, + memory_db::{HashKey, MemoryDB}, + node_db::{Hasher, EMPTY_PREFIX}, + CachedValue, DBValue, Lookup, NibbleSlice, RecordedForKey, Recorder, Trie, TrieCache, + TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieRecorder, }; use crate::{TestCommit, TestDB}; diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 3c8d9eca..10bf1dec 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -15,15 +15,15 @@ use std::ops::Deref; use env_logger; -use hash_db::{Hasher, NodeDB, Prefix, EMPTY_PREFIX}; use log::debug; -use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ reference_trie_root, test_layouts, ExtensionLayout, HashedValueNoExt, HashedValueNoExtThreshold, NoExtensionLayout, PrefixedMemoryDB, RefHasher, ReferenceNodeCodec, ReferenceNodeCodecNoExt, TestTrieCache, }; use trie_db::{ + memory_db::{HashKey, MemoryDB, PrefixedKey}, + node_db::{Hasher, NodeDB, Prefix, EMPTY_PREFIX}, CachedValue, Changeset, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, TrieLayout, Value, }; @@ -535,7 +535,7 @@ fn register_proof_without_value() { use Prefix; type Layout = HashedValueNoExtThreshold<1, ()>; - type MemoryDB = memory_db::MemoryDB, DBValue>; + type MemoryDB = trie_db::memory_db::MemoryDB, DBValue>; let x = [ (b"test1".to_vec(), vec![1; 32]), // inline (b"test1234".to_vec(), vec![2; 36]), @@ -593,10 +593,10 @@ fn register_proof_without_value() { trie.remove(b"test1234").unwrap(); // proof should contain value for 'te' only. - type MemoryDBProof = memory_db::MemoryDB, DBValue>; + type MemoryDBProof = trie_db::memory_db::MemoryDB, DBValue>; let mut memdb_from_proof = MemoryDBProof::default(); for (_key, value) in memdb.record.into_inner().into_iter() { - memdb_from_proof.insert(hash_db::EMPTY_PREFIX, value.as_slice()); + memdb_from_proof.insert(EMPTY_PREFIX, value.as_slice()); } let db_unpacked = memdb_from_proof.clone(); @@ -1024,16 +1024,16 @@ fn attached_trie_root>( Some((root_hash, location)) } -pub struct KeySpacedDB<'a, H, T, DL>(&'a dyn hash_db::NodeDB, &'a [u8]); +pub struct KeySpacedDB<'a, H, T, DL>(&'a dyn NodeDB, &'a [u8]); impl<'a, H, T, DL> KeySpacedDB<'a, H, T, DL> { #[inline] - pub fn new(db: &'a dyn hash_db::NodeDB, ks: &'a [u8]) -> Self { + pub fn new(db: &'a dyn NodeDB, ks: &'a [u8]) -> Self { KeySpacedDB(db, ks) } } -impl<'a, H, T, L> hash_db::NodeDB for KeySpacedDB<'a, H, T, L> +impl<'a, H, T, L> NodeDB for KeySpacedDB<'a, H, T, L> where H: Hasher, T: From<&'static [u8]>, diff --git a/trie-root/Cargo.toml b/trie-root/Cargo.toml index 7a39bddd..64c60e64 100644 --- a/trie-root/Cargo.toml +++ b/trie-root/Cargo.toml @@ -9,7 +9,7 @@ categories = [ "no-std" ] edition = "2018" [dependencies] -hash-db = { path = "../hash-db", default-features = false, version = "0.16.0" } +hash-db = { version = "0.16.0", default-features = false } [features] default = ["std"] From 93433ca47fe50eb08b10c3921ef0fd70fc71587c Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 19:09:14 +0100 Subject: [PATCH 46/90] fix --- trie-db/test/benches/bench.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/trie-db/test/benches/bench.rs b/trie-db/test/benches/bench.rs index 8688730c..e75dc233 100644 --- a/trie-db/test/benches/bench.rs +++ b/trie-db/test/benches/bench.rs @@ -17,6 +17,7 @@ use std::collections::BTreeMap; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; use reference_trie::ExtensionLayout as Layout; use trie_db::{ + memory_db, proof::{generate_proof, verify_proof}, NibbleSlice, Trie, }; From 16f1cd57383ecc71e2451bea4981e42c16a9aeb8 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 21:56:00 +0100 Subject: [PATCH 47/90] no_std --- trie-db/src/memory_db.rs | 20 +++++--------------- trie-db/src/node_db.rs | 9 +-------- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/trie-db/src/memory_db.rs b/trie-db/src/memory_db.rs index fbcc4cac..9ccc25f2 100644 --- a/trie-db/src/memory_db.rs +++ b/trie-db/src/memory_db.rs @@ -14,27 +14,17 @@ //! Reference-counted memory-based `NodeDB` implementation. -#![cfg_attr(not(feature = "std"), no_std)] - -#[cfg(not(feature = "std"))] -extern crate alloc; +use crate::{ + node_db::{Hasher as KeyHasher, MaybeDebug, NodeDB, Prefix}, + rstd::{cmp::Eq, hash, marker::PhantomData, mem, vec::Vec}, +}; -use crate::node_db::{Hasher as KeyHasher, MaybeDebug, NodeDB, Prefix}; #[cfg(feature = "std")] -use std::{ - cmp::Eq, collections::hash_map::Entry, collections::HashMap as Map, hash, marker::PhantomData, - mem, -}; +use std::collections::hash_map::{Entry, HashMap as Map}; #[cfg(not(feature = "std"))] use alloc::collections::btree_map::{BTreeMap as Map, Entry}; -#[cfg(not(feature = "std"))] -use core::{cmp::Eq, hash, marker::PhantomData, mem}; - -#[cfg(not(feature = "std"))] -use alloc::vec::Vec; - /// Reference-counted memory-based `NodeDB` implementation. /// /// Use `new()` to create a new database. Insert items with `insert()`, remove items diff --git a/trie-db/src/node_db.rs b/trie-db/src/node_db.rs index 745e4577..704e84c7 100644 --- a/trie-db/src/node_db.rs +++ b/trie-db/src/node_db.rs @@ -14,15 +14,8 @@ //! Database of byte-slices keyed to their hash. -#![cfg_attr(not(feature = "std"), no_std)] +use crate::rstd::vec::Vec; -#[cfg(not(feature = "std"))] -extern crate alloc; - -#[cfg(not(feature = "std"))] -use alloc::vec::Vec; -#[cfg(not(feature = "std"))] -use core::hash; #[cfg(feature = "std")] use std::fmt::Debug; From 04a1ee4b9f5271668098e2aaa75f3f876690f0fe Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 22:08:42 +0100 Subject: [PATCH 48/90] missing file write --- trie-db/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 4fa599d2..8379cadf 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -49,6 +49,7 @@ pub use iterator::TrieDBNodeDoubleEndedIterator; use node::NodeOwned; use node_db::MaybeDebug; +#[cfg(feature = "std")] pub mod mem_tree_db; pub mod memory_db; pub mod node; From 5e2662840daf6088f73b4c9324ea84e441ca20d0 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 23 Jan 2024 22:21:25 +0100 Subject: [PATCH 49/90] don't check memorydb --- .github/workflows/rust.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 12de8153..8373528a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -43,12 +43,6 @@ jobs: command: check args: --manifest-path trie-db/Cargo.toml --no-default-features - - name: Check memory-db Without Std - uses: actions-rs/cargo@v1 - with: - command: check - args: --manifest-path memory-db/Cargo.toml --no-default-features - - name: Check trie-root Without Std uses: actions-rs/cargo@v1 with: From 554187bd82e8b3db5ae5463395d74e8720fb65f3 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 24 Jan 2024 11:25:08 +0100 Subject: [PATCH 50/90] flatten crate, only keep h256-std-hasher. --- .github/workflows/rust.yml | 4 +- Cargo.toml | 9 +-- .../CHANGELOG.md | 0 .../Cargo.toml | 6 +- .../benches/bench.rs | 2 +- .../src/lib.rs | 9 +-- .../src/substrate.rs | 4 +- .../src/substrate_like.rs | 4 +- test-support/keccak-hasher/CHANGELOG.md | 11 --- test-support/keccak-hasher/Cargo.toml | 19 ----- test-support/trie-bench/CHANGELOG.md | 69 ------------------- test-support/trie-bench/Cargo.toml | 16 ----- test-support/trie-standardmap/CHANGELOG.md | 8 --- test-support/trie-standardmap/Cargo.toml | 11 --- {trie-db/test => test}/Cargo.toml | 10 +-- {trie-db/test => test}/benches/bench.rs | 2 +- .../bench.rs => test/benches/memory_db.rs | 2 +- .../src/double_ended_iterator.rs | 0 {trie-db/test => test}/src/iter_build.rs | 0 {trie-db/test => test}/src/iterator.rs | 0 {trie-db/test => test}/src/lib.rs | 2 + {trie-db/test => test}/src/proof.rs | 0 {trie-db/test => test}/src/recorder.rs | 0 {trie-db/test => test}/src/trie_codec.rs | 0 test/src/trie_root.rs | 44 ++++++++++++ {trie-db/test => test}/src/triedb.rs | 0 {trie-db/test => test}/src/triedbmut.rs | 2 +- trie-db/Cargo.toml | 18 +++-- .../src/lib.rs => trie-db/src/bench.rs | 8 +-- trie-db/src/iter_build.rs | 4 +- .../lib.rs => trie-db/src/keccak_hasher.rs | 2 +- trie-db/src/lib.rs | 7 ++ trie-db/src/mem_tree_db.rs | 2 +- trie-db/src/memory_db.rs | 12 +++- trie-db/src/node_db.rs | 30 +++++++- .../src/lib.rs => trie-db/src/test_utils.rs | 5 +- .../src/lib.rs => trie-db/src/trie_root.rs | 23 +------ trie-root/CHANGELOG.md | 17 ----- trie-root/Cargo.toml | 18 ----- trie-root/README.md | 2 - trie-root/test/Cargo.toml | 15 ---- trie-root/test/src/lib.rs | 45 ------------ 42 files changed, 135 insertions(+), 307 deletions(-) rename {test-support/reference-trie => reference-trie}/CHANGELOG.md (100%) rename {test-support/reference-trie => reference-trie}/Cargo.toml (67%) rename {test-support/reference-trie => reference-trie}/benches/bench.rs (95%) rename {test-support/reference-trie => reference-trie}/src/lib.rs (99%) rename {test-support/reference-trie => reference-trie}/src/substrate.rs (99%) rename {test-support/reference-trie => reference-trie}/src/substrate_like.rs (99%) delete mode 100644 test-support/keccak-hasher/CHANGELOG.md delete mode 100644 test-support/keccak-hasher/Cargo.toml delete mode 100644 test-support/trie-bench/CHANGELOG.md delete mode 100644 test-support/trie-bench/Cargo.toml delete mode 100644 test-support/trie-standardmap/CHANGELOG.md delete mode 100644 test-support/trie-standardmap/Cargo.toml rename {trie-db/test => test}/Cargo.toml (63%) rename {trie-db/test => test}/benches/bench.rs (99%) rename trie-db/benches/bench.rs => test/benches/memory_db.rs (98%) rename {trie-db/test => test}/src/double_ended_iterator.rs (100%) rename {trie-db/test => test}/src/iter_build.rs (100%) rename {trie-db/test => test}/src/iterator.rs (100%) rename {trie-db/test => test}/src/lib.rs (98%) rename {trie-db/test => test}/src/proof.rs (100%) rename {trie-db/test => test}/src/recorder.rs (100%) rename {trie-db/test => test}/src/trie_codec.rs (100%) create mode 100644 test/src/trie_root.rs rename {trie-db/test => test}/src/triedb.rs (100%) rename {trie-db/test => test}/src/triedbmut.rs (99%) rename test-support/trie-bench/src/lib.rs => trie-db/src/bench.rs (99%) rename test-support/keccak-hasher/src/lib.rs => trie-db/src/keccak_hasher.rs (98%) rename test-support/trie-standardmap/src/lib.rs => trie-db/src/test_utils.rs (97%) rename trie-root/src/lib.rs => trie-db/src/trie_root.rs (96%) delete mode 100644 trie-root/CHANGELOG.md delete mode 100644 trie-root/Cargo.toml delete mode 100644 trie-root/README.md delete mode 100644 trie-root/test/Cargo.toml delete mode 100644 trie-root/test/src/lib.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8373528a..d7d432c5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -43,11 +43,11 @@ jobs: command: check args: --manifest-path trie-db/Cargo.toml --no-default-features - - name: Check trie-root Without Std + - name: Check trie-db-test With bench uses: actions-rs/cargo@v1 with: command: check - args: --manifest-path trie-root/Cargo.toml --no-default-features + args: --manifest-path trie-db/test/Cargo.toml --benches fmt: name: Rustfmt diff --git a/Cargo.toml b/Cargo.toml index 4ae84c1c..22cf63e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,7 @@ [workspace] members = [ "hash256-std-hasher", - "test-support/keccak-hasher", - "test-support/reference-trie", - "test-support/trie-standardmap", - "test-support/trie-bench", "trie-db", - "trie-db/test", - "trie-root", - "trie-root/test" + "test", + "reference-trie", ] diff --git a/test-support/reference-trie/CHANGELOG.md b/reference-trie/CHANGELOG.md similarity index 100% rename from test-support/reference-trie/CHANGELOG.md rename to reference-trie/CHANGELOG.md diff --git a/test-support/reference-trie/Cargo.toml b/reference-trie/Cargo.toml similarity index 67% rename from test-support/reference-trie/Cargo.toml rename to reference-trie/Cargo.toml index a60612e3..ad461be3 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/reference-trie/Cargo.toml @@ -8,15 +8,12 @@ license = "Apache-2.0" edition = "2018" [dependencies] -keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } -trie-db = { path = "../../trie-db", default-features = false, version = "0.28.0" } -trie-root = { path = "../../trie-root", default-features = false, version = "0.18.0" } +trie-db = { path = "../trie-db", default-features = false, version = "0.28.0" } parity-scale-codec = { version = "3.0.0", features = ["derive"] } hashbrown = { version = "0.14.1", default-features = false, features = ["ahash"] } paste = "1.0.12" [dev-dependencies] -trie-bench = { path = "../trie-bench" } criterion = "0.5.1" [[bench]] @@ -28,5 +25,4 @@ default = ["std"] # no actual support for std, only to avoid a cargo issues std = [ "trie-db/std", - "trie-root/std", ] diff --git a/test-support/reference-trie/benches/bench.rs b/reference-trie/benches/bench.rs similarity index 95% rename from test-support/reference-trie/benches/bench.rs rename to reference-trie/benches/bench.rs index b3466d28..4e1cb969 100644 --- a/test-support/reference-trie/benches/bench.rs +++ b/reference-trie/benches/bench.rs @@ -18,7 +18,7 @@ criterion_group!(benches, benchmark); criterion_main!(benches); fn benchmark(c: &mut Criterion) { - trie_bench::standard_benchmark::< + trie_db::bench::standard_benchmark::< reference_trie::ExtensionLayout, reference_trie::ReferenceTrieStream, >(c, "ref"); diff --git a/test-support/reference-trie/src/lib.rs b/reference-trie/src/lib.rs similarity index 99% rename from test-support/reference-trie/src/lib.rs rename to reference-trie/src/lib.rs index cd081e4d..c07ce963 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/reference-trie/src/lib.rs @@ -21,13 +21,14 @@ use trie_db::{ memory_db::{KeyFunction, MemoryDB, PrefixedKey}, nibble_ops, node::{NibbleSlicePlan, NodeHandlePlan, NodeOwned, NodePlan, Value, ValuePlan}, - node_db, trie_visit, + node_db, + node_db::Hasher, + trie_root::{self, TrieStream, Value as TrieStreamValue}, + trie_visit, triedbmut::ChildReference, DBValue, Location, NodeCodec, Trie, TrieBuilder, TrieConfiguration, TrieDBBuilder, TrieDBMutBuilder, TrieHash, TrieLayout, TrieRoot, }; -pub use trie_root::TrieStream; -use trie_root::{Hasher, Value as TrieStreamValue}; mod substrate; mod substrate_like; @@ -45,7 +46,7 @@ pub use substrate::{LayoutV0 as SubstrateV0, LayoutV1 as SubstrateV1}; pub use trie_db::mem_tree_db::{Location as MemLocation, MemTreeDB}; /// Reference hasher is a keccak hasher. -pub type RefHasher = keccak_hasher::KeccakHasher; +pub type RefHasher = trie_db::keccak_hasher::KeccakHasher; pub type PrefixedMemoryDB = MemoryDB<::Hash, PrefixedKey<::Hash>, DBValue>; diff --git a/test-support/reference-trie/src/substrate.rs b/reference-trie/src/substrate.rs similarity index 99% rename from test-support/reference-trie/src/substrate.rs rename to reference-trie/src/substrate.rs index a99c301a..0adf6257 100644 --- a/test-support/reference-trie/src/substrate.rs +++ b/reference-trie/src/substrate.rs @@ -78,8 +78,8 @@ fn fuse_nibbles_node(nibbles: &[u8], kind: NodeKind) -> impl Iterator .chain(nibbles[nibbles.len() % 2..].chunks(2).map(|ch| ch[0] << 4 | ch[1])) } -use trie_root::Value as TrieStreamValue; -impl trie_root::TrieStream for TrieStream { +use trie_db::trie_root::{self, Value as TrieStreamValue}; +impl trie_db::trie_root::TrieStream for TrieStream { fn new() -> Self { Self { buffer: Vec::new() } } diff --git a/test-support/reference-trie/src/substrate_like.rs b/reference-trie/src/substrate_like.rs similarity index 99% rename from test-support/reference-trie/src/substrate_like.rs rename to reference-trie/src/substrate_like.rs index 183a8a7e..73c7839b 100644 --- a/test-support/reference-trie/src/substrate_like.rs +++ b/reference-trie/src/substrate_like.rs @@ -465,8 +465,8 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator Self { Self { buffer: Vec::new() } } diff --git a/test-support/keccak-hasher/CHANGELOG.md b/test-support/keccak-hasher/CHANGELOG.md deleted file mode 100644 index 4da7ff58..00000000 --- a/test-support/keccak-hasher/CHANGELOG.md +++ /dev/null @@ -1,11 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [0.16.0] - 2023-03-14 -- Switch to `hash-db` 0.16. [#188](https://github.com/paritytech/trie/pull/188) - -## [0.15.3] - 2020-07-24 -- Update `tiny-keccak` to 0.2. [#105](https://github.com/paritytech/trie/pull/105) diff --git a/test-support/keccak-hasher/Cargo.toml b/test-support/keccak-hasher/Cargo.toml deleted file mode 100644 index 7ed18d39..00000000 --- a/test-support/keccak-hasher/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "keccak-hasher" -version = "0.16.0" -authors = ["Parity Technologies "] -description = "Keccak-256 implementation of the Hasher trait" -repository = "https://github.com/paritytech/parity/" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -tiny-keccak = { version = "2.0.2", features = ["keccak"] } -hash-db = { version = "0.16.0", default-features = false } -hash256-std-hasher = { path = "../../hash256-std-hasher", version = "0.15.2" } - -[features] -default = ["std"] -std = [ - "hash-db/std", -] diff --git a/test-support/trie-bench/CHANGELOG.md b/test-support/trie-bench/CHANGELOG.md deleted file mode 100644 index 6982dff9..00000000 --- a/test-support/trie-bench/CHANGELOG.md +++ /dev/null @@ -1,69 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [Unreleased] - -## [0.38.0] - 2023-09-12 -- chore: Release trie-db 0.28.0 [#200](https://github.com/paritytech/trie/pull/200) - -## [0.37.0] - 2023-03-14 -- Update dependencies. [#188](https://github.com/paritytech/trie/pull/188) and [#187](https://github.com/paritytech/trie/pull/187) - -## [0.36.0] - 2023-02-24 -- Updated `trie-db` to 0.26.0. [#185](https://github.com/paritytech/trie/pull/185) - -## [0.35.0] - 2023-02-03 -- Updated `trie-db` to 0.25.0. - -## [0.34.0] - 2023-01-23 -- Updated `criterion` to 0.4. [#176](https://github.com/paritytech/trie/pull/176) - -## [0.33.0] - 2022-11-29 -- Updated `memory-db` to 0.31. [#172](https://github.com/paritytech/trie/pull/172) - -## [0.32.0] - 2022-09-20 -- Updated `memory-db` to 0.30. [#166](https://github.com/paritytech/trie/pull/166) - -## [0.31.0] - 2022-08-04 -- Update trie-db to 0.24.0. [#163](https://github.com/paritytech/trie/pull/163) - -## [0.30.0] - 2022-02-04 -- Updated `memory-db` to 0.29. [#150](https://github.com/paritytech/trie/pull/150) - -## [0.29.0] - 2021-10-19 -- Updated memory-db, triedb and trie-root. [#142](https://github.com/paritytech/trie/pull/142) - -## [0.28.0] - 2021-07-02 -- Updated memory-db to 0.27. [#139](https://github.com/paritytech/trie/pull/139) -- Updated parity-scale-codec to 2.0. [#137](https://github.com/paritytech/trie/pull/137) - -## [0.27.1] - 2021-06-24 -- Updated parity-scale-codec to 2.2.0-rc.2. - -## [0.27.0] - 2021-01-27 -- Updated memory-db to 0.26. -- Updated parity-scale-codec to 2.0. - -## [0.26.0] - 2021-01-05 -- Updated memory-db to 0.25. [#118](https://github.com/paritytech/trie/pull/118) - -## [0.25.0] - 2020-07-24 -- Updated criterion to 0.3. [#106](https://github.com/paritytech/trie/pull/106) - -## [0.24.0] - 2020-07-07 -- Updated memory-db to 0.24. [#99](https://github.com/paritytech/trie/pull/99) - -## [0.23.0] - 2020-07-06 -- Updated memory-db to 0.22. [#98](https://github.com/paritytech/trie/pull/98) - -## [0.21.0] - 2020-03-21 -- Updated memory-db to 0.20.0 [#82](https://github.com/paritytech/trie/pull/82) - -## [0.20.0] - 2020-02-07 -- Updated trie-root to v0.16.0 and memory-db to v0.19.0 and trie-db to v0.20.0 [#78](https://github.com/paritytech/trie/pull/78) - -## [0.19.0] - 2020-01-17 -- Updated trie-db to 0.19.0. [#75](https://github.com/paritytech/trie/pull/75) diff --git a/test-support/trie-bench/Cargo.toml b/test-support/trie-bench/Cargo.toml deleted file mode 100644 index b79f4b86..00000000 --- a/test-support/trie-bench/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "trie-bench" -description = "Standard benchmarking suite for tries" -version = "0.38.0" -authors = ["Parity Technologies "] -repository = "https://github.com/paritytech/trie/" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } -trie-standardmap = { path = "../trie-standardmap", version = "0.16.0" } -trie-root = { path = "../../trie-root", version = "0.18.0" } -trie-db = { path = "../../trie-db", version = "0.28.0" } -criterion = "0.5.1" -parity-scale-codec = "3.0.0" diff --git a/test-support/trie-standardmap/CHANGELOG.md b/test-support/trie-standardmap/CHANGELOG.md deleted file mode 100644 index 39a5cbbf..00000000 --- a/test-support/trie-standardmap/CHANGELOG.md +++ /dev/null @@ -1,8 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [0.16.0] - 2023-03-14 -- Update dependencies. [#188](https://github.com/paritytech/trie/pull/188) and [#187](https://github.com/paritytech/trie/pull/187) diff --git a/test-support/trie-standardmap/Cargo.toml b/test-support/trie-standardmap/Cargo.toml deleted file mode 100644 index 7769b1c2..00000000 --- a/test-support/trie-standardmap/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "trie-standardmap" -description = "Standard test map for profiling tries" -version = "0.16.0" -authors = ["Parity Technologies "] -license = "Apache-2.0" -edition = "2018" - -[dependencies] -keccak-hasher = { path = "../keccak-hasher", version = "0.16.0"} -trie-db = { path = "../../trie-db" , version = "0.28.0"} diff --git a/trie-db/test/Cargo.toml b/test/Cargo.toml similarity index 63% rename from trie-db/test/Cargo.toml rename to test/Cargo.toml index 6eb436c5..2cdf75fd 100644 --- a/trie-db/test/Cargo.toml +++ b/test/Cargo.toml @@ -11,12 +11,14 @@ edition = "2018" name = "bench" harness = false +[[bench]] +name = "memory_db" +harness = false + [dependencies] -trie-db = { path = "..", version = "0.28.0"} +trie-db = { path = "../trie-db", version = "0.28.0", features = ["test_utils"]} rand = { version = "0.8", default-features = false, features = ["small_rng"] } -trie-standardmap = { path = "../../test-support/trie-standardmap", version = "0.16.0" } -reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } -keccak-hasher = { path = "../../test-support/keccak-hasher", version = "0.16.0" } +reference-trie = { path = "../reference-trie", version = "0.29.0" } hex-literal = "0.4" criterion = "0.5.1" env_logger = { version = "0.10", default-features = false } diff --git a/trie-db/test/benches/bench.rs b/test/benches/bench.rs similarity index 99% rename from trie-db/test/benches/bench.rs rename to test/benches/bench.rs index e75dc233..6916cfde 100644 --- a/trie-db/test/benches/bench.rs +++ b/test/benches/bench.rs @@ -18,10 +18,10 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criteri use reference_trie::ExtensionLayout as Layout; use trie_db::{ memory_db, + test_utils::{Alphabet, StandardMap, ValueMode}, proof::{generate_proof, verify_proof}, NibbleSlice, Trie, }; -use trie_standardmap::{Alphabet, StandardMap, ValueMode}; criterion_group!( benches, diff --git a/trie-db/benches/bench.rs b/test/benches/memory_db.rs similarity index 98% rename from trie-db/benches/bench.rs rename to test/benches/memory_db.rs index e665bf7c..3bf247f2 100644 --- a/trie-db/benches/bench.rs +++ b/test/benches/memory_db.rs @@ -13,10 +13,10 @@ // limitations under the License. use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use keccak_hasher::KeccakHasher; use trie_db::{ memory_db::{HashKey, MemoryDB}, node_db::{Hasher, EMPTY_PREFIX}, + keccak_hasher::{KeccakHasher}, }; criterion_group!( diff --git a/trie-db/test/src/double_ended_iterator.rs b/test/src/double_ended_iterator.rs similarity index 100% rename from trie-db/test/src/double_ended_iterator.rs rename to test/src/double_ended_iterator.rs diff --git a/trie-db/test/src/iter_build.rs b/test/src/iter_build.rs similarity index 100% rename from trie-db/test/src/iter_build.rs rename to test/src/iter_build.rs diff --git a/trie-db/test/src/iterator.rs b/test/src/iterator.rs similarity index 100% rename from trie-db/test/src/iterator.rs rename to test/src/iterator.rs diff --git a/trie-db/test/src/lib.rs b/test/src/lib.rs similarity index 98% rename from trie-db/test/src/lib.rs rename to test/src/lib.rs index 31ad62ae..7f31769b 100644 --- a/trie-db/test/src/lib.rs +++ b/test/src/lib.rs @@ -29,6 +29,8 @@ mod recorder; #[cfg(test)] mod trie_codec; #[cfg(test)] +mod trie_root; +#[cfg(test)] mod triedb; #[cfg(test)] mod triedbmut; diff --git a/trie-db/test/src/proof.rs b/test/src/proof.rs similarity index 100% rename from trie-db/test/src/proof.rs rename to test/src/proof.rs diff --git a/trie-db/test/src/recorder.rs b/test/src/recorder.rs similarity index 100% rename from trie-db/test/src/recorder.rs rename to test/src/recorder.rs diff --git a/trie-db/test/src/trie_codec.rs b/test/src/trie_codec.rs similarity index 100% rename from trie-db/test/src/trie_codec.rs rename to test/src/trie_codec.rs diff --git a/test/src/trie_root.rs b/test/src/trie_root.rs new file mode 100644 index 00000000..bf4e5309 --- /dev/null +++ b/test/src/trie_root.rs @@ -0,0 +1,44 @@ +// Copyright 2017, 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test for trie-root module. + +use hex_literal::hex; +use reference_trie::ReferenceTrieStream; +use trie_db::{ + keccak_hasher::KeccakHasher, + trie_root::{sec_trie_root, trie_root}, +}; + +#[test] +fn previous_doc_test_1() { + let v = vec![("doe", "reindeer"), ("dog", "puppy"), ("dogglesworth", "cat")]; + + let root = hex!["d6e02b2bd48aa04fd2ad87cfac1144a29ca7f7dc60f4526c7b7040763abe3d43"]; + assert_eq!( + sec_trie_root::(v, Default::default()), + root + ); +} + +#[test] +fn previous_doc_test_2() { + let v = vec![("doe", "reindeer"), ("dog", "puppy"), ("dogglesworth", "cat")]; + + let root = hex!["0807d5393ae7f349481063ebb5dbaf6bda58db282a385ca97f37dccba717cb79"]; + assert_eq!( + trie_root::(v, Default::default()), + root + ); +} diff --git a/trie-db/test/src/triedb.rs b/test/src/triedb.rs similarity index 100% rename from trie-db/test/src/triedb.rs rename to test/src/triedb.rs diff --git a/trie-db/test/src/triedbmut.rs b/test/src/triedbmut.rs similarity index 99% rename from trie-db/test/src/triedbmut.rs rename to test/src/triedbmut.rs index 10bf1dec..6d9006f7 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -27,7 +27,7 @@ use trie_db::{ CachedValue, Changeset, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, TrieLayout, Value, }; -use trie_standardmap::*; +use trie_db::test_utils::*; use crate::{TestCommit, TestDB}; diff --git a/trie-db/Cargo.toml b/trie-db/Cargo.toml index 4828356d..d67f758e 100644 --- a/trie-db/Cargo.toml +++ b/trie-db/Cargo.toml @@ -7,23 +7,27 @@ repository = "https://github.com/paritytech/trie" license = "Apache-2.0" edition = "2018" -[[bench]] -name = "bench" -harness = false - [dependencies] +hash256-std-hasher = { path = "../hash256-std-hasher", version = "0.15.2", optional = true } log = "0.4" smallvec = { version = "1.0.0", features = ["union", "const_new"] } rustc-hex = { version = "2.1.0", default-features = false, optional = true } -hash-db = { version = "0.16.0", default-features = false } +tiny-keccak = { version = "2.0.2", features = ["keccak"], optional = true } +parity-scale-codec = { version = "3.0.0", optional = true } [dev-dependencies] -keccak-hasher = { path = "../test-support/keccak-hasher", version = "0.16.0"} criterion = "0.5.1" +hash256-std-hasher = { path = "../hash256-std-hasher", version = "0.15.2" } +tiny-keccak = { version = "2.0.2", features = ["keccak"] } [features] default = ["std"] +test_utils = [ + "std", + "hash256-std-hasher", + "tiny-keccak", + "parity-scale-codec", +] std = [ "rustc-hex", - "hash-db/std", ] diff --git a/test-support/trie-bench/src/lib.rs b/trie-db/src/bench.rs similarity index 99% rename from test-support/trie-bench/src/lib.rs rename to trie-db/src/bench.rs index 6b6c44a8..0b7d1bce 100644 --- a/test-support/trie-bench/src/lib.rs +++ b/trie-db/src/bench.rs @@ -15,16 +15,16 @@ //! Standard trie benchmarking tool. use criterion::{black_box, BenchmarkId, Criterion}; -use keccak_hasher::KeccakHasher; use parity_scale_codec::{Compact, Encode}; use std::default::Default; -use trie_db::{ +use crate::{ + keccak_hasher::KeccakHasher, memory_db::{HashKey, MemoryDB}, node_db::Hasher, + trie_root::{trie_root, TrieStream}, NodeCodec, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, }; -use trie_root::{trie_root, TrieStream}; -use trie_standardmap::*; +use crate::test_utils::*; struct TrieInsertionList(Vec<(Vec, Vec)>); impl ::std::fmt::Display for TrieInsertionList { diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index 3fc2404a..25463ddc 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -343,7 +343,7 @@ pub trait ProcessEncodedNode { /// Note that the returned value can change depending on implementation, /// but usually it should be the Hash of encoded node. /// This is not something direcly related to encoding but is here for - /// optimisation purpose (builder hash_db does return this value). + /// optimisation purpose (builder node_db does return this value). fn process( &mut self, prefix: Prefix, @@ -355,7 +355,7 @@ pub trait ProcessEncodedNode { fn process_inner_hashed_value(&mut self, prefix: Prefix, value: &[u8]) -> HO; } -/// Get trie root and insert visited node in a hash_db. +/// Get trie root and insert visited node in a node_db. /// As for all `ProcessEncodedNode` implementation, it /// is only for full trie parsing (not existing trie). pub struct TrieBuilder<'a, T: TrieLayout, K: KeyFunction + Send + Sync> { diff --git a/test-support/keccak-hasher/src/lib.rs b/trie-db/src/keccak_hasher.rs similarity index 98% rename from test-support/keccak-hasher/src/lib.rs rename to trie-db/src/keccak_hasher.rs index b4692f77..05ea572b 100644 --- a/test-support/keccak-hasher/src/lib.rs +++ b/trie-db/src/keccak_hasher.rs @@ -14,8 +14,8 @@ //! Hasher implementation for the Keccak-256 hash +use crate::node_db::Hasher; use hash256_std_hasher::Hash256StdHasher; -use hash_db::Hasher; use tiny_keccak::{Hasher as _, Keccak}; /// The `Keccak` hash output type. diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 8379cadf..f23dd0a1 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -49,6 +49,8 @@ pub use iterator::TrieDBNodeDoubleEndedIterator; use node::NodeOwned; use node_db::MaybeDebug; +#[cfg(any(feature = "test_utils", test))] +pub mod keccak_hasher; #[cfg(feature = "std")] pub mod mem_tree_db; pub mod memory_db; @@ -56,6 +58,11 @@ pub mod node; pub mod node_db; pub mod proof; pub mod recorder; +#[cfg(feature = "bench")] +pub mod bench; +#[cfg(feature = "test_utils")] +pub mod test_utils; +pub mod trie_root; pub mod triedb; pub mod triedbmut; diff --git a/trie-db/src/mem_tree_db.rs b/trie-db/src/mem_tree_db.rs index 05ac2c44..88a5d040 100644 --- a/trie-db/src/mem_tree_db.rs +++ b/trie-db/src/mem_tree_db.rs @@ -227,10 +227,10 @@ where mod tests { use super::{MemTreeDB, NodeEntry}; use crate::{ + keccak_hasher::{KeccakHash, KeccakHasher}, node_db::{Hasher, NodeDB}, Changeset, ExistingChangesetNode, NewChangesetNode, }; - use keccak_hasher::{KeccakHash, KeccakHasher}; fn hash(i: u32) -> KeccakHash { KeccakHasher::hash(&i.to_le_bytes()) diff --git a/trie-db/src/memory_db.rs b/trie-db/src/memory_db.rs index 9ccc25f2..a8a0d1ca 100644 --- a/trie-db/src/memory_db.rs +++ b/trie-db/src/memory_db.rs @@ -34,9 +34,11 @@ use alloc::collections::btree_map::{BTreeMap as Map, Entry}; /// /// # Example /// ```rust +/// #[cfg(feature = "test_utils")] +/// { /// use trie_db::node_db::Hasher; /// use trie_db::node_db::{EMPTY_PREFIX}; -/// use keccak_hasher::KeccakHasher; +/// use trie_db::keccak_hasher::KeccakHasher; /// use trie_db::memory_db::{MemoryDB, HashKey}; /// /// let mut m = MemoryDB::, Vec>::default(); @@ -67,6 +69,7 @@ use alloc::collections::btree_map::{BTreeMap as Map, Entry}; /// /// m.remove(&k, EMPTY_PREFIX); /// assert!(!m.contains(&k, EMPTY_PREFIX)); +/// } /// ``` pub struct MemoryDB where @@ -274,9 +277,11 @@ where /// /// # Examples /// ```rust + /// #[cfg(feature = "test_utils")] + /// { /// use trie_db::node_db::Hasher; /// use trie_db::node_db::{NodeDB, EMPTY_PREFIX}; - /// use keccak_hasher::KeccakHasher; + /// use trie_db::keccak_hasher::KeccakHasher; /// use trie_db::memory_db::{MemoryDB, HashKey}; /// /// fn main() { @@ -287,6 +292,7 @@ where /// m.clear(); /// assert!(!m.contains(&hash, EMPTY_PREFIX)); /// } + /// } /// ``` pub fn clear(&mut self) { self.data.clear(); @@ -441,10 +447,10 @@ where #[cfg(test)] mod test { use crate::{ + keccak_hasher::KeccakHasher, memory_db::{HashKey, MemoryDB}, node_db::{Hasher as KeyHasher, EMPTY_PREFIX}, }; - use keccak_hasher::KeccakHasher; #[test] fn memorydb_remove_and_purge() { diff --git a/trie-db/src/node_db.rs b/trie-db/src/node_db.rs index 704e84c7..8ee19879 100644 --- a/trie-db/src/node_db.rs +++ b/trie-db/src/node_db.rs @@ -14,13 +14,11 @@ //! Database of byte-slices keyed to their hash. -use crate::rstd::vec::Vec; +use crate::rstd::{hash, vec::Vec}; #[cfg(feature = "std")] use std::fmt::Debug; -pub use hash_db::Hasher; - #[cfg(feature = "std")] pub trait MaybeDebug: Debug {} #[cfg(feature = "std")] @@ -45,6 +43,32 @@ pub type Prefix<'a> = (&'a [u8], Option); /// or for root nodes. pub static EMPTY_PREFIX: Prefix<'static> = (&[], None); +/// Trait describing an object that can hash a slice of bytes. Used to abstract +/// other types over the hashing algorithm. Defines a single `hash` method and an +/// `Out` associated type with the necessary bounds. +pub trait Hasher: Sync + Send { + /// The output type of the `Hasher` + type Out: AsRef<[u8]> + + AsMut<[u8]> + + Default + + MaybeDebug + + core::cmp::Ord + + PartialEq + + Eq + + hash::Hash + + Send + + Sync + + Clone + + Copy; + /// What to use to build `HashMap`s with this `Hasher`. + type StdHasher: Sync + Send + Default + hash::Hasher; + /// The length in bytes of the `Hasher` output. + const LENGTH: usize; + + /// Compute the hash of the provided slice of bytes returning the `Out` type of the `Hasher`. + fn hash(x: &[u8]) -> Self::Out; +} + /// Trait modelling datastore keyed by a hash defined by the `Hasher` and optional location tag. pub trait NodeDB: Send + Sync { /// Look up a trie node by hash and location. diff --git a/test-support/trie-standardmap/src/lib.rs b/trie-db/src/test_utils.rs similarity index 97% rename from test-support/trie-standardmap/src/lib.rs rename to trie-db/src/test_utils.rs index 23f98d7f..8e503bdd 100644 --- a/test-support/trie-standardmap/src/lib.rs +++ b/trie-db/src/test_utils.rs @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Key-value datastore with a modified Merkle tree. +//! Test utilities. -use keccak_hasher::KeccakHasher; -use trie_db::node_db::Hasher; +use crate::{keccak_hasher::KeccakHasher, node_db::Hasher}; type H256 = ::Out; diff --git a/trie-root/src/lib.rs b/trie-db/src/trie_root.rs similarity index 96% rename from trie-root/src/lib.rs rename to trie-db/src/trie_root.rs index b827abb6..d38472fc 100644 --- a/trie-root/src/lib.rs +++ b/trie-db/src/trie_root.rs @@ -16,28 +16,7 @@ //! //! This module should be used to generate trie root hash. -#![cfg_attr(not(feature = "std"), no_std)] - -#[cfg(not(feature = "std"))] -extern crate alloc; - -#[cfg(feature = "std")] -mod rstd { - pub use std::{cmp, collections::BTreeMap, vec::Vec}; -} - -#[cfg(not(feature = "std"))] -mod rstd { - pub use alloc::{ - collections::{BTreeMap, VecDeque}, - vec::Vec, - }; - pub use core::cmp; -} - -use self::rstd::*; - -pub use hash_db::Hasher; +use crate::{node_db::Hasher, rstd::*}; /// Different possible value to use for node encoding. #[derive(Clone)] diff --git a/trie-root/CHANGELOG.md b/trie-root/CHANGELOG.md deleted file mode 100644 index a35b5b7b..00000000 --- a/trie-root/CHANGELOG.md +++ /dev/null @@ -1,17 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [Unreleased] - - -## [0.18.0] - 2023-03-14 -- Update dependencies. [#188](https://github.com/paritytech/trie/pull/188) and [#187](https://github.com/paritytech/trie/pull/187) - -## [0.17.0] - 2021-10-19 -- Support for value nodes. [#142](https://github.com/paritytech/trie/pull/142) - -## [0.16.0] - 2020-02-07 -- Update reference-trie to v0.20.0 [#78](https://github.com/paritytech/trie/pull/78) diff --git a/trie-root/Cargo.toml b/trie-root/Cargo.toml deleted file mode 100644 index 64c60e64..00000000 --- a/trie-root/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "trie-root" -version = "0.18.0" -authors = ["Parity Technologies "] -description = "In-memory patricia trie operations" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -categories = [ "no-std" ] -edition = "2018" - -[dependencies] -hash-db = { version = "0.16.0", default-features = false } - -[features] -default = ["std"] -std = [ - "hash-db/std" -] diff --git a/trie-root/README.md b/trie-root/README.md deleted file mode 100644 index 36d38b68..00000000 --- a/trie-root/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This crate provides utility functions to validate and initialize tries using flexible input. -It is used extensively in `substrate` to validate blocks (mostly transactions and receipt roots). diff --git a/trie-root/test/Cargo.toml b/trie-root/test/Cargo.toml deleted file mode 100644 index 23f99af6..00000000 --- a/trie-root/test/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "trie-root-test" -version = "0.21.0" -authors = ["Parity Technologies "] -description = "Tests fo trie-root crate" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -categories = [ ] -edition = "2018" - -[dependencies] -trie-root = { path = "..", version = "0.18.0" } -hex-literal = "0.4" -keccak-hasher = { path = "../../test-support/keccak-hasher", version = "0.16.0" } -reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } diff --git a/trie-root/test/src/lib.rs b/trie-root/test/src/lib.rs deleted file mode 100644 index 0fef011c..00000000 --- a/trie-root/test/src/lib.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Test for trie-root crate. - -#[cfg(test)] -mod test { - use hex_literal::hex; - use keccak_hasher::KeccakHasher; - use reference_trie::ReferenceTrieStream; - use trie_root::{sec_trie_root, trie_root}; - - #[test] - fn previous_doc_test_1() { - let v = vec![("doe", "reindeer"), ("dog", "puppy"), ("dogglesworth", "cat")]; - - let root = hex!["d6e02b2bd48aa04fd2ad87cfac1144a29ca7f7dc60f4526c7b7040763abe3d43"]; - assert_eq!( - sec_trie_root::(v, Default::default()), - root - ); - } - - #[test] - fn previous_doc_test_2() { - let v = vec![("doe", "reindeer"), ("dog", "puppy"), ("dogglesworth", "cat")]; - - let root = hex!["0807d5393ae7f349481063ebb5dbaf6bda58db282a385ca97f37dccba717cb79"]; - assert_eq!( - trie_root::(v, Default::default()), - root - ); - } -} From 10cc9489994e71943f72e509513c2c6a5a583316 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 24 Jan 2024 11:29:23 +0100 Subject: [PATCH 51/90] fix --- test/Cargo.toml | 2 +- trie-db/Cargo.toml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/Cargo.toml b/test/Cargo.toml index 2cdf75fd..532c36c8 100644 --- a/test/Cargo.toml +++ b/test/Cargo.toml @@ -16,7 +16,7 @@ name = "memory_db" harness = false [dependencies] -trie-db = { path = "../trie-db", version = "0.28.0", features = ["test_utils"]} +trie-db = { path = "../trie-db", version = "0.28.0", features = ["test_utils", "bench"]} rand = { version = "0.8", default-features = false, features = ["small_rng"] } reference-trie = { path = "../reference-trie", version = "0.29.0" } hex-literal = "0.4" diff --git a/trie-db/Cargo.toml b/trie-db/Cargo.toml index d67f758e..a4568a89 100644 --- a/trie-db/Cargo.toml +++ b/trie-db/Cargo.toml @@ -14,6 +14,7 @@ smallvec = { version = "1.0.0", features = ["union", "const_new"] } rustc-hex = { version = "2.1.0", default-features = false, optional = true } tiny-keccak = { version = "2.0.2", features = ["keccak"], optional = true } parity-scale-codec = { version = "3.0.0", optional = true } +criterion = { version = "0.5.1", optional = true } [dev-dependencies] criterion = "0.5.1" @@ -22,6 +23,10 @@ tiny-keccak = { version = "2.0.2", features = ["keccak"] } [features] default = ["std"] +bench = [ + "std", + "criterion", +] test_utils = [ "std", "hash256-std-hasher", From 024e964a4a963c33cc5f8d09a82e57d8e7c5b68b Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 24 Jan 2024 11:32:09 +0100 Subject: [PATCH 52/90] fmt --- test/benches/bench.rs | 2 +- test/benches/memory_db.rs | 2 +- test/src/triedbmut.rs | 2 +- trie-db/src/bench.rs | 8 ++++---- trie-db/src/lib.rs | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/benches/bench.rs b/test/benches/bench.rs index 6916cfde..93075ffa 100644 --- a/test/benches/bench.rs +++ b/test/benches/bench.rs @@ -18,8 +18,8 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criteri use reference_trie::ExtensionLayout as Layout; use trie_db::{ memory_db, - test_utils::{Alphabet, StandardMap, ValueMode}, proof::{generate_proof, verify_proof}, + test_utils::{Alphabet, StandardMap, ValueMode}, NibbleSlice, Trie, }; diff --git a/test/benches/memory_db.rs b/test/benches/memory_db.rs index 3bf247f2..148393f3 100644 --- a/test/benches/memory_db.rs +++ b/test/benches/memory_db.rs @@ -14,9 +14,9 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use trie_db::{ + keccak_hasher::KeccakHasher, memory_db::{HashKey, MemoryDB}, node_db::{Hasher, EMPTY_PREFIX}, - keccak_hasher::{KeccakHasher}, }; criterion_group!( diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index 6d9006f7..d300270e 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -24,10 +24,10 @@ use reference_trie::{ use trie_db::{ memory_db::{HashKey, MemoryDB, PrefixedKey}, node_db::{Hasher, NodeDB, Prefix, EMPTY_PREFIX}, + test_utils::*, CachedValue, Changeset, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, TrieLayout, Value, }; -use trie_db::test_utils::*; use crate::{TestCommit, TestDB}; diff --git a/trie-db/src/bench.rs b/trie-db/src/bench.rs index 0b7d1bce..870b945e 100644 --- a/trie-db/src/bench.rs +++ b/trie-db/src/bench.rs @@ -14,17 +14,17 @@ //! Standard trie benchmarking tool. -use criterion::{black_box, BenchmarkId, Criterion}; -use parity_scale_codec::{Compact, Encode}; -use std::default::Default; use crate::{ keccak_hasher::KeccakHasher, memory_db::{HashKey, MemoryDB}, node_db::Hasher, + test_utils::*, trie_root::{trie_root, TrieStream}, NodeCodec, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, }; -use crate::test_utils::*; +use criterion::{black_box, BenchmarkId, Criterion}; +use parity_scale_codec::{Compact, Encode}; +use std::default::Default; struct TrieInsertionList(Vec<(Vec, Vec)>); impl ::std::fmt::Display for TrieInsertionList { diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index f23dd0a1..51421177 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -49,6 +49,8 @@ pub use iterator::TrieDBNodeDoubleEndedIterator; use node::NodeOwned; use node_db::MaybeDebug; +#[cfg(feature = "bench")] +pub mod bench; #[cfg(any(feature = "test_utils", test))] pub mod keccak_hasher; #[cfg(feature = "std")] @@ -58,8 +60,6 @@ pub mod node; pub mod node_db; pub mod proof; pub mod recorder; -#[cfg(feature = "bench")] -pub mod bench; #[cfg(feature = "test_utils")] pub mod test_utils; pub mod trie_root; From d176e428b34bff7e56e8e72c82c979961ec14bbc Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 24 Jan 2024 11:49:09 +0100 Subject: [PATCH 53/90] fix --- trie-db/src/trie_root.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trie-db/src/trie_root.rs b/trie-db/src/trie_root.rs index d38472fc..d27f1641 100644 --- a/trie-db/src/trie_root.rs +++ b/trie-db/src/trie_root.rs @@ -16,7 +16,10 @@ //! //! This module should be used to generate trie root hash. -use crate::{node_db::Hasher, rstd::*}; +use crate::{ + node_db::Hasher, + rstd::{cmp, vec::Vec, BTreeMap}, +}; /// Different possible value to use for node encoding. #[derive(Clone)] From 7a575efaa7e4f34f713a625545ecdebc6c7b405a Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 24 Jan 2024 12:56:38 +0100 Subject: [PATCH 54/90] ci --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d7d432c5..39aa7cf9 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -47,7 +47,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: check - args: --manifest-path trie-db/test/Cargo.toml --benches + args: --manifest-path test/Cargo.toml --benches fmt: name: Rustfmt From fca95774cfc84124c5a22a4fe343f3fdf7e2e6e9 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 24 Jan 2024 13:46:48 +0100 Subject: [PATCH 55/90] add testutils when using bench --- trie-db/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/trie-db/Cargo.toml b/trie-db/Cargo.toml index a4568a89..6e62b290 100644 --- a/trie-db/Cargo.toml +++ b/trie-db/Cargo.toml @@ -26,6 +26,7 @@ default = ["std"] bench = [ "std", "criterion", + "test_utils", ] test_utils = [ "std", From 26e462da954036c11d253c0ca231844075a82515 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 24 Jan 2024 15:08:14 +0100 Subject: [PATCH 56/90] restore empty function, call it unchanged --- trie-db/src/triedbmut.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 0e34eb8f..f3bb3b5c 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -837,6 +837,14 @@ impl Changeset { Changeset::Existing(node) => node.hash, } } + + pub fn unchanged(root: H) -> Self { + Changeset::Existing(ExistingChangesetNode { + hash: root, + prefix: (BackingByteVec::new(), None), + location: Default::default(), + }) + } } pub type OwnedPrefix = (BackingByteVec, Option); From 88013700febdd0c3abb24aca40af1b3fef248f15 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 14 Feb 2024 15:51:54 +0100 Subject: [PATCH 57/90] store child index in node plan --- reference-trie/src/lib.rs | 10 ++++-- reference-trie/src/substrate.rs | 4 ++- reference-trie/src/substrate_like.rs | 4 ++- test/src/lib.rs | 9 ++++-- test/src/triedbmut.rs | 1 + trie-db/src/iterator.rs | 20 ++++++------ trie-db/src/node.rs | 48 ++++++++++++++++++++++------ trie-db/src/proof/generate.rs | 4 +-- 8 files changed, 71 insertions(+), 29 deletions(-) diff --git a/reference-trie/src/lib.rs b/reference-trie/src/lib.rs index c07ce963..51e55fd4 100644 --- a/reference-trie/src/lib.rs +++ b/reference-trie/src/lib.rs @@ -654,12 +654,14 @@ impl NodeCodec for ReferenceNodeCodec { None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; + let mut i_hash = 0; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { let count = >::decode(&mut input)?.0 as usize; let range = input.take(count)?; children[i] = Some(if count == H::LENGTH { - NodeHandlePlan::Hash(range) + i_hash += 1; + NodeHandlePlan::Hash(range, i_hash - 1) } else { NodeHandlePlan::Inline(range) }); @@ -676,7 +678,7 @@ impl NodeCodec for ReferenceNodeCodec { let count = >::decode(&mut input)?.0 as usize; let range = input.take(count)?; let child = if count == H::LENGTH { - NodeHandlePlan::Hash(range) + NodeHandlePlan::Hash(range, 0) } else { NodeHandlePlan::Inline(range) }; @@ -825,12 +827,14 @@ impl NodeCodec for ReferenceNodeCodecNoExt { None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; + let mut i_hash = 0; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { let count = >::decode(&mut input)?.0 as usize; let range = input.take(count)?; children[i] = Some(if count == H::LENGTH { - NodeHandlePlan::Hash(range) + i_hash += 1; + NodeHandlePlan::Hash(range, i_hash - 1) } else { NodeHandlePlan::Inline(range) }); diff --git a/reference-trie/src/substrate.rs b/reference-trie/src/substrate.rs index 0adf6257..a841a603 100644 --- a/reference-trie/src/substrate.rs +++ b/reference-trie/src/substrate.rs @@ -259,12 +259,14 @@ where None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; + let mut i_hash = 0; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { let count = >::decode(&mut input)?.0 as usize; let range = input.take(count)?; children[i] = Some(if count == H::LENGTH { - NodeHandlePlan::Hash(range) + i_hash += 1; + NodeHandlePlan::Hash(range, i_hash - 1) } else { NodeHandlePlan::Inline(range) }); diff --git a/reference-trie/src/substrate_like.rs b/reference-trie/src/substrate_like.rs index 73c7839b..f1493799 100644 --- a/reference-trie/src/substrate_like.rs +++ b/reference-trie/src/substrate_like.rs @@ -103,12 +103,14 @@ impl NodeCodec { None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; + let mut i_hash = 0; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { let count = >::decode(&mut input)?.0 as usize; let range = input.take(count)?; children[i] = Some(if count == H::LENGTH { - NodeHandlePlan::Hash(range) + i_hash += 1; + NodeHandlePlan::Hash(range, i_hash - 1) } else { NodeHandlePlan::Inline(range) }); diff --git a/test/src/lib.rs b/test/src/lib.rs index 7f31769b..e4430719 100644 --- a/test/src/lib.rs +++ b/test/src/lib.rs @@ -14,8 +14,6 @@ //! Tests for trie-db crate. -use trie_db::memory_db::{KeyFunction, MemoryDB}; - #[cfg(test)] mod double_ended_iterator; #[cfg(test)] @@ -35,12 +33,15 @@ mod triedb; #[cfg(test)] mod triedbmut; +#[cfg(test)] use trie_db::{ mem_tree_db::{Location, MemTreeDB}, + memory_db::{KeyFunction, MemoryDB}, node_db::{self, Hasher, Prefix}, Changeset, DBValue, TrieHash, TrieLayout, }; +#[cfg(test)] trait TestDB: node_db::NodeDB + Clone + Default { fn commit( &mut self, @@ -53,6 +54,7 @@ trait TestDB: node_db::NodeDB + Cl } } +#[cfg(test)] impl, H, KF> TestDB for MemoryDB where H: Hasher, @@ -74,6 +76,7 @@ where } } +#[cfg(test)] impl, H> TestDB for MemTreeDB where H: Hasher + Clone, @@ -97,10 +100,12 @@ where } } +#[cfg(test)] trait TestCommit { fn commit_to>(self, db: &mut DB) -> TrieHash; } +#[cfg(test)] impl> TestCommit for Changeset where T::Hash: Hasher, diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index d300270e..951d1319 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -1001,6 +1001,7 @@ fn attached_trie_internal>() { trie.get(b"make_sure_it_changes").unwrap().unwrap(); } +#[cfg(test)] fn attached_trie_root>( memdb: &DB, main_root: &TrieHash, diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index dd077719..4a18b322 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -214,11 +214,10 @@ impl TrieDBRawIterator { self.key_nibbles.push(i); if children[i as usize].is_some() { - // TODO would make sense to put location in NodePlan: this is rather - // costy - let (_, children) = NodePlan::build_value_and_children( + let child = NodePlan::build_child( value.as_ref(), children, + i as usize, node_data, locations, ); @@ -229,7 +228,7 @@ impl TrieDBRawIterator { let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - children[i as usize].unwrap(), + child.unwrap(), prefix.left(), true, )? @@ -262,11 +261,10 @@ impl TrieDBRawIterator { self.key_nibbles.push(i); if children[i as usize].is_some() { - // TODO would make sense to put location in NodePlan: this is rather - // costy - let (_, children) = NodePlan::build_value_and_children( + let child = NodePlan::build_child( value.as_ref(), children, + i as usize, node_data, locations, ); @@ -277,7 +275,7 @@ impl TrieDBRawIterator { let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - children[i as usize].unwrap(), + child.unwrap(), prefix.left(), true, )? @@ -479,10 +477,10 @@ impl TrieDBRawIterator { (Status::AtChild(i), NodePlan::Branch { value, children, .. }) | (Status::AtChild(i), NodePlan::NibbledBranch { value, children, .. }) => { if children[i].is_some() { - // TODO would make sense to put location in NodePlan: this is rather costy - let (_, children) = NodePlan::build_value_and_children( + let child = NodePlan::build_child( value.as_ref(), children, + i, node_data, locations, ); @@ -492,7 +490,7 @@ impl TrieDBRawIterator { match db.get_raw_or_lookup( crumb.hash.unwrap_or_default(), - children[i].unwrap(), + child.unwrap(), self.key_nibbles.as_prefix(), true, ) { diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index 57ebc660..d0a36804 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -440,9 +440,10 @@ impl NodeOwned { /// A `NodeHandlePlan` is a decoding plan for constructing a `NodeHandle` from an encoded trie /// node. This is used as a substructure of `NodePlan`. See `NodePlan` for details. +/// Number of existing node is stored (allow fast access to children locations). #[derive(Debug, Clone, PartialEq, Eq)] pub enum NodeHandlePlan { - Hash(Range), + Hash(Range, u8), Inline(Range), } @@ -452,7 +453,7 @@ impl NodeHandlePlan { /// data, otherwise the call may decode incorrectly or panic. pub fn build<'a, 'b, L>(&'a self, data: &'b [u8], location: L) -> NodeHandle<'b, L> { match self { - NodeHandlePlan::Hash(range) => NodeHandle::Hash(&data[range.clone()], location), + NodeHandlePlan::Hash(range, _) => NodeHandle::Hash(&data[range.clone()], location), NodeHandlePlan::Inline(range) => NodeHandle::Inline(&data[range.clone()]), } } @@ -460,8 +461,8 @@ impl NodeHandlePlan { /// Check if the node is innline. pub fn is_inline(&self) -> bool { match self { - NodeHandlePlan::Hash(_) => false, - NodeHandlePlan::Inline(_) => true, + NodeHandlePlan::Hash(..) => false, + NodeHandlePlan::Inline(..) => true, } } } @@ -584,7 +585,7 @@ impl NodePlan { } } - pub(crate) fn build_value_and_children<'a, 'b, L: Copy + Default>( + fn build_value_and_children<'a, 'b, L: Copy + Default>( value: Option<&'a ValuePlan>, children: &'a [Option; nibble_ops::NIBBLE_LENGTH], data: &'b [u8], @@ -617,6 +618,34 @@ impl NodePlan { (value, child_slices) } + pub(crate) fn build_child<'a, 'b, L: Copy + Default>( + value: Option<&'a ValuePlan>, + children: &'a [Option; nibble_ops::NIBBLE_LENGTH], + index: usize, + data: &'b [u8], + locations: &[L], + ) -> Option> { + let mut location_value_offset = 0; + if let Some(v) = value { + if !v.is_inline() { + location_value_offset = 1; + } + } + if let Some(child) = &children[index] { + let location = if let NodeHandlePlan::Hash(_, i_hash) = child { + locations + .get(location_value_offset + *i_hash as usize) + .copied() + .unwrap_or_default() + } else { + Default::default() + }; + Some(child.build(data, location)) + } else { + None + } + } + /// Access value plan from node plan, return `None` for /// node that cannot contain a `ValuePlan`. pub fn value_plan(&self) -> Option<&ValuePlan> { @@ -644,14 +673,13 @@ impl NodePlan { self.value_plan().map(|v| !v.is_inline()).unwrap_or(false) } - /// Check how many children location value node has. - pub fn num_children_locations(&self) -> usize { + fn num_children_locations(&self) -> usize { match self { - NodePlan::Extension { child: NodeHandlePlan::Hash(_), .. } => 1, + NodePlan::Extension { child: NodeHandlePlan::Hash(..), .. } => 1, NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { let mut count = 0; for child in children { - if let Some(NodeHandlePlan::Hash(_)) = child { + if let Some(NodeHandlePlan::Hash(..)) = child { count += 1; } } @@ -661,6 +689,8 @@ impl NodePlan { } } + /// Check if an extra location is defined, it can be attached state. + /// This method is counting and should be call only when needed. pub fn additional_ref_location(&self, locations: &[L]) -> Option { let offset = if self.has_location_for_value() { 1 } else { 0 } + self.num_children_locations(); diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index a3b813fd..b03f0274 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -199,8 +199,8 @@ impl<'a, C: NodeCodec, L: Copy + Default> StackEntry<'a, C, L> { child: &NodeHandlePlan, ) -> ChildReference { match child { - NodeHandlePlan::Hash(_) => ChildReference::Inline(C::HashOut::default(), 0), - NodeHandlePlan::Inline(_) => { + NodeHandlePlan::Hash(..) => ChildReference::Inline(C::HashOut::default(), 0), + NodeHandlePlan::Inline(..) => { let mut hash = C::HashOut::default(); assert!( encoded_child.len() <= hash.as_ref().len(), From bb561a022355d4b6a2cecc5af45d4eb20f951e77 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 15 Feb 2024 18:30:59 +0100 Subject: [PATCH 58/90] NodeDBMut trait for testing in polkadot sdk --- trie-db/src/mem_tree_db.rs | 11 ++++++++++- trie-db/src/memory_db.rs | 14 +++++++++++++- trie-db/src/node_db.rs | 12 +++++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/trie-db/src/mem_tree_db.rs b/trie-db/src/mem_tree_db.rs index 88a5d040..f21f3672 100644 --- a/trie-db/src/mem_tree_db.rs +++ b/trie-db/src/mem_tree_db.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use crate::{ - node_db::{Hasher, NodeDB, Prefix}, + node_db::{Hasher, NodeDB, NodeDBMut, Prefix}, Changeset, NewChangesetNode, }; @@ -223,6 +223,15 @@ where } } +impl NodeDBMut, Location> for MemTreeDB +where + H: Hasher, +{ + fn apply_changeset(&mut self, commit: Changeset) { + self.apply_commit(commit); + } +} + #[cfg(test)] mod tests { use super::{MemTreeDB, NodeEntry}; diff --git a/trie-db/src/memory_db.rs b/trie-db/src/memory_db.rs index a8a0d1ca..afafb593 100644 --- a/trie-db/src/memory_db.rs +++ b/trie-db/src/memory_db.rs @@ -15,8 +15,9 @@ //! Reference-counted memory-based `NodeDB` implementation. use crate::{ - node_db::{Hasher as KeyHasher, MaybeDebug, NodeDB, Prefix}, + node_db::{Hasher as KeyHasher, MaybeDebug, NodeDB, NodeDBMut, Prefix}, rstd::{cmp::Eq, hash, marker::PhantomData, mem, vec::Vec}, + Changeset, DBValue, }; #[cfg(feature = "std")] @@ -365,6 +366,17 @@ where } } +impl NodeDBMut for MemoryDB +where + H: KeyHasher, + KF: KeyFunction + Send + Sync, + L: Default, +{ + fn apply_changeset(&mut self, commit: Changeset) { + commit.apply_to(self); + } +} + impl MemoryDB where H: KeyHasher, diff --git a/trie-db/src/node_db.rs b/trie-db/src/node_db.rs index 8ee19879..eb692e23 100644 --- a/trie-db/src/node_db.rs +++ b/trie-db/src/node_db.rs @@ -14,7 +14,10 @@ //! Database of byte-slices keyed to their hash. -use crate::rstd::{hash, vec::Vec}; +use crate::{ + rstd::{hash, vec::Vec}, + Changeset, +}; #[cfg(feature = "std")] use std::fmt::Debug; @@ -85,3 +88,10 @@ pub trait NodeDB: Send + Sync { H::hash(value) } } + +/// Trait for node db that can get update by a CommitSet. +/// Mostly usefull for testing. +pub trait NodeDBMut: NodeDB { + /// Insert commit set to the db. + fn apply_changeset(&mut self, commit: Changeset); +} From e911ebc654f0b969ae2a62ed9c48d07c5784d726 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 16 Feb 2024 20:04:33 +0100 Subject: [PATCH 59/90] Allow non send sync db --- trie-db/src/node_db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie-db/src/node_db.rs b/trie-db/src/node_db.rs index eb692e23..c888a1cc 100644 --- a/trie-db/src/node_db.rs +++ b/trie-db/src/node_db.rs @@ -73,7 +73,7 @@ pub trait Hasher: Sync + Send { } /// Trait modelling datastore keyed by a hash defined by the `Hasher` and optional location tag. -pub trait NodeDB: Send + Sync { +pub trait NodeDB { /// Look up a trie node by hash and location. /// Returns the node bytes and the list of children node locations if any. fn get(&self, key: &H::Out, prefix: Prefix, location: L) -> Option<(T, Vec)>; From bef6e0f8124bb90a76a40d4b33f15b30760f7ff1 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 16 Feb 2024 20:10:34 +0100 Subject: [PATCH 60/90] Revert "Allow non send sync db" This reverts commit e911ebc654f0b969ae2a62ed9c48d07c5784d726. --- trie-db/src/node_db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie-db/src/node_db.rs b/trie-db/src/node_db.rs index c888a1cc..eb692e23 100644 --- a/trie-db/src/node_db.rs +++ b/trie-db/src/node_db.rs @@ -73,7 +73,7 @@ pub trait Hasher: Sync + Send { } /// Trait modelling datastore keyed by a hash defined by the `Hasher` and optional location tag. -pub trait NodeDB { +pub trait NodeDB: Send + Sync { /// Look up a trie node by hash and location. /// Returns the node bytes and the list of children node locations if any. fn get(&self, key: &H::Out, prefix: Prefix, location: L) -> Option<(T, Vec)>; From 04ac7a05fd16c570e44e240aecd1eb0040717294 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 16 Feb 2024 21:23:40 +0100 Subject: [PATCH 61/90] rem ref to TrieMut --- trie-db/src/triedb.rs | 2 +- trie-db/src/triedbmut.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index 7306e74c..668e9a02 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -121,7 +121,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// # Example /// ```ignore /// use trie_db::node_db::Hasher; -/// use reference_trie::{RefTrieDBMut, RefTrieDB, Trie, TrieMut}; +/// use reference_trie::{RefTrieDBMut, RefTrieDB, Trie}; /// use trie_db::DBValue; /// use keccak_hasher::KeccakHasher; /// use memory_db::*; diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index f3bb3b5c..d2ef286a 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -851,14 +851,14 @@ pub type OwnedPrefix = (BackingByteVec, Option); /// A `Trie` implementation using a generic `NodeDB` backing database. /// -/// Use it as a `TrieMut` trait object. You can use `db()` to get the backing database object. +/// You can use `db()` to get the backing database object. /// Note that changes are not committed to the database until `commit` is called. /// /// /// # Example /// ```ignore /// use trie_db::node_db::Hasher; -/// use reference_trie::{RefTrieDBMut, TrieMut}; +/// use reference_trie::RefTrieDBMut; /// use trie_db::DBValue; /// use keccak_hasher::KeccakHasher; /// use trie_db::memory_db::*; From e4b29e4b78d219aad99a2fa4f7a42939bd007d98 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 19 Feb 2024 09:26:55 +0100 Subject: [PATCH 62/90] return root from apply changeset --- test/src/lib.rs | 4 +--- trie-db/src/mem_tree_db.rs | 10 ++++++---- trie-db/src/memory_db.rs | 4 ++-- trie-db/src/node_db.rs | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/src/lib.rs b/test/src/lib.rs index e4430719..be5c4abb 100644 --- a/test/src/lib.rs +++ b/test/src/lib.rs @@ -82,9 +82,7 @@ where H: Hasher + Clone, { fn commit(&mut self, commit: trie_db::Changeset) -> H::Out { - let root = commit.root_hash(); - self.apply_commit(commit); - root + self.apply_commit(commit) } fn remove(&mut self, hash: &H::Out, _prefix: Prefix) { diff --git a/trie-db/src/mem_tree_db.rs b/trie-db/src/mem_tree_db.rs index f21f3672..b7c17f0e 100644 --- a/trie-db/src/mem_tree_db.rs +++ b/trie-db/src/mem_tree_db.rs @@ -163,8 +163,9 @@ where } } - pub fn apply_commit(&mut self, commit: Changeset) { - if commit.root_hash() != self.hashed_null_node { + pub fn apply_commit(&mut self, commit: Changeset) -> H::Out { + let root = commit.root_hash(); + if root != self.hashed_null_node { let root = self.apply(&commit); let key = commit.hash(); self.roots.insert(*key, root); @@ -176,6 +177,7 @@ where self.remove_root(&k); } } + root } } @@ -227,8 +229,8 @@ impl NodeDBMut, Location> for MemTreeDB where H: Hasher, { - fn apply_changeset(&mut self, commit: Changeset) { - self.apply_commit(commit); + fn apply_changeset(&mut self, commit: Changeset) -> H::Out { + self.apply_commit(commit) } } diff --git a/trie-db/src/memory_db.rs b/trie-db/src/memory_db.rs index afafb593..c08eb297 100644 --- a/trie-db/src/memory_db.rs +++ b/trie-db/src/memory_db.rs @@ -372,8 +372,8 @@ where KF: KeyFunction + Send + Sync, L: Default, { - fn apply_changeset(&mut self, commit: Changeset) { - commit.apply_to(self); + fn apply_changeset(&mut self, commit: Changeset) -> H::Out { + commit.apply_to(self) } } diff --git a/trie-db/src/node_db.rs b/trie-db/src/node_db.rs index eb692e23..95005de0 100644 --- a/trie-db/src/node_db.rs +++ b/trie-db/src/node_db.rs @@ -93,5 +93,5 @@ pub trait NodeDB: Send + Sync { /// Mostly usefull for testing. pub trait NodeDBMut: NodeDB { /// Insert commit set to the db. - fn apply_changeset(&mut self, commit: Changeset); + fn apply_changeset(&mut self, commit: Changeset) -> H::Out; } From 1b8cced3b3479bce17401efe8292cad6814ab195 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 21 Feb 2024 14:35:26 +0100 Subject: [PATCH 63/90] method to visit changeset to avoid apply_to cost in sc-client-db --- trie-db/src/memory_db.rs | 2 + trie-db/src/triedbmut.rs | 84 +++++++++++++++++++++++++++++----------- 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/trie-db/src/memory_db.rs b/trie-db/src/memory_db.rs index c08eb297..fdfa05e6 100644 --- a/trie-db/src/memory_db.rs +++ b/trie-db/src/memory_db.rs @@ -126,6 +126,7 @@ where } pub trait KeyFunction { + const NEED_PREFIX: bool = true; type Key: Send + Sync + Clone + hash::Hash + Eq + MaybeDebug + core::cmp::Ord; fn key(hash: &H::Out, prefix: Prefix) -> Self::Key; @@ -148,6 +149,7 @@ impl core::fmt::Debug for HashKey { impl KeyFunction for HashKey { type Key = H::Out; + const NEED_PREFIX: bool = false; fn key(hash: &H::Out, prefix: Prefix) -> H::Out { hash_key::(hash, prefix) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index d2ef286a..b532e6be 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -787,50 +787,88 @@ pub fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { (result, prefix.1) } +/// Use this trait to visit a changeset content. +pub trait ChangesetVisitor { + fn need_prefix(&self) -> bool { + true + } + + fn new_node(&mut self, prefix: Prefix, content: &[u8]); + + fn removed_node(&mut self, hash: &H, prefix: Prefix); +} + +impl ChangesetVisitor for MemoryDB +where + K: KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, +{ + fn need_prefix(&self) -> bool { + K::NEED_PREFIX + } + + fn new_node(&mut self, prefix: Prefix, content: &[u8]) { + self.insert(prefix, content); + } + + fn removed_node(&mut self, hash: &H, prefix: Prefix) { + self.remove(hash, prefix); + } +} + impl Changeset { - pub fn apply_to(&self, mem_db: &mut MemoryDB) -> H - where - K: KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, - { - fn apply_node<'a, H, DL, MH, K>( + pub fn visit(&self, v: &mut impl ChangesetVisitor) -> H { + fn apply_node<'a, H, DL>( node: &'a Changeset, - mem_db: &mut MemoryDB, + v: &mut impl ChangesetVisitor, mut ks: Option<&'a [u8]>, - ) where - K: KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, - { + ) { match node { Changeset::New(node) => { if let Some((k, removed)) = node.removed_keys.as_ref() { for (hash, p) in removed.iter() { - if let Some(k) = k { - let prefixed = prefix_prefix(k.as_slice(), (p.0.as_slice(), p.1)); - mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); + let prefixed; + let prefix = if !v.need_prefix() { + (&[][..], None) + } else if let Some(k) = k { ks = Some(k.as_slice()); + prefixed = prefix_prefix(k.as_slice(), (p.0.as_slice(), p.1)); + (prefixed.0.as_slice(), prefixed.1) } else { - mem_db.remove(hash, (p.0.as_slice(), p.1)); - } + (p.0.as_slice(), p.1) + }; + v.removed_node(hash, prefix); } } for child in &node.children { - apply_node(child, mem_db, ks); + apply_node(child, v, ks); } - if let Some(ks) = ks { - let prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); - mem_db.insert((prefixed.0.as_slice(), prefixed.1), &node.data); + let prefixed; + let prefix = if !v.need_prefix() { + (&[][..], None) + } else if let Some(ks) = ks { + prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); + (prefixed.0.as_slice(), prefixed.1) } else { - mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); - } + (node.prefix.0.as_slice(), node.prefix.1) + }; + v.new_node(prefix, &node.data); }, Changeset::Existing(_) => {}, } } - apply_node::(&self, mem_db, None); + apply_node::(&self, v, None); self.root_hash() } + pub fn apply_to(&self, mem_db: &mut MemoryDB) -> H + where + K: KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { + self.visit(mem_db) + } + pub fn root_hash(&self) -> H { match &self { Changeset::New(node) => node.hash, From 8ad4ee885f1049b798cf58fea812b077a1628c76 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 21 Feb 2024 15:13:34 +0100 Subject: [PATCH 64/90] Revert "method to visit changeset to avoid apply_to cost in sc-client-db" This reverts commit 1b8cced3b3479bce17401efe8292cad6814ab195. --- trie-db/src/memory_db.rs | 2 - trie-db/src/triedbmut.rs | 84 +++++++++++----------------------------- 2 files changed, 23 insertions(+), 63 deletions(-) diff --git a/trie-db/src/memory_db.rs b/trie-db/src/memory_db.rs index fdfa05e6..c08eb297 100644 --- a/trie-db/src/memory_db.rs +++ b/trie-db/src/memory_db.rs @@ -126,7 +126,6 @@ where } pub trait KeyFunction { - const NEED_PREFIX: bool = true; type Key: Send + Sync + Clone + hash::Hash + Eq + MaybeDebug + core::cmp::Ord; fn key(hash: &H::Out, prefix: Prefix) -> Self::Key; @@ -149,7 +148,6 @@ impl core::fmt::Debug for HashKey { impl KeyFunction for HashKey { type Key = H::Out; - const NEED_PREFIX: bool = false; fn key(hash: &H::Out, prefix: Prefix) -> H::Out { hash_key::(hash, prefix) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index b532e6be..d2ef286a 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -787,88 +787,50 @@ pub fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { (result, prefix.1) } -/// Use this trait to visit a changeset content. -pub trait ChangesetVisitor { - fn need_prefix(&self) -> bool { - true - } - - fn new_node(&mut self, prefix: Prefix, content: &[u8]); - - fn removed_node(&mut self, hash: &H, prefix: Prefix); -} - -impl ChangesetVisitor for MemoryDB -where - K: KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, -{ - fn need_prefix(&self) -> bool { - K::NEED_PREFIX - } - - fn new_node(&mut self, prefix: Prefix, content: &[u8]) { - self.insert(prefix, content); - } - - fn removed_node(&mut self, hash: &H, prefix: Prefix) { - self.remove(hash, prefix); - } -} - impl Changeset { - pub fn visit(&self, v: &mut impl ChangesetVisitor) -> H { - fn apply_node<'a, H, DL>( + pub fn apply_to(&self, mem_db: &mut MemoryDB) -> H + where + K: KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { + fn apply_node<'a, H, DL, MH, K>( node: &'a Changeset, - v: &mut impl ChangesetVisitor, + mem_db: &mut MemoryDB, mut ks: Option<&'a [u8]>, - ) { + ) where + K: KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { match node { Changeset::New(node) => { if let Some((k, removed)) = node.removed_keys.as_ref() { for (hash, p) in removed.iter() { - let prefixed; - let prefix = if !v.need_prefix() { - (&[][..], None) - } else if let Some(k) = k { + if let Some(k) = k { + let prefixed = prefix_prefix(k.as_slice(), (p.0.as_slice(), p.1)); + mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); ks = Some(k.as_slice()); - prefixed = prefix_prefix(k.as_slice(), (p.0.as_slice(), p.1)); - (prefixed.0.as_slice(), prefixed.1) } else { - (p.0.as_slice(), p.1) - }; - v.removed_node(hash, prefix); + mem_db.remove(hash, (p.0.as_slice(), p.1)); + } } } for child in &node.children { - apply_node(child, v, ks); + apply_node(child, mem_db, ks); } - let prefixed; - let prefix = if !v.need_prefix() { - (&[][..], None) - } else if let Some(ks) = ks { - prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); - (prefixed.0.as_slice(), prefixed.1) + if let Some(ks) = ks { + let prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); + mem_db.insert((prefixed.0.as_slice(), prefixed.1), &node.data); } else { - (node.prefix.0.as_slice(), node.prefix.1) - }; - v.new_node(prefix, &node.data); + mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); + } }, Changeset::Existing(_) => {}, } } - apply_node::(&self, v, None); + apply_node::(&self, mem_db, None); self.root_hash() } - pub fn apply_to(&self, mem_db: &mut MemoryDB) -> H - where - K: KeyFunction + Send + Sync, - MH: Hasher + Send + Sync, - { - self.visit(mem_db) - } - pub fn root_hash(&self) -> H { match &self { Changeset::New(node) => node.hash, From d352340db9efc46718b3113af353dd4852c27743 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Thu, 29 Feb 2024 15:13:54 +0100 Subject: [PATCH 65/90] empty tree change set --- trie-db/src/triedbmut.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index d2ef286a..b07b6de7 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -780,6 +780,20 @@ impl Changeset { } } +impl Changeset { + /// In case the underlying db do not + /// do empty node optimization, it can + /// make sense to insert the empty node. + pub fn new_empty_tree>() -> Self { + Self::New(NewChangesetNode { + hash: C::hashed_null_node(), + prefix: Default::default(), + data: C::empty_node().to_vec(), + children: Default::default(), + removed_keys: None, + }) + } +} pub fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); result.extend_from_slice(ks); From 8f33c2c194e720a7d51994639b2120f2402b3798 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Thu, 29 Feb 2024 16:04:18 +0100 Subject: [PATCH 66/90] better naming --- trie-db/src/triedbmut.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index b07b6de7..86735d3e 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -784,7 +784,7 @@ impl Changeset { /// In case the underlying db do not /// do empty node optimization, it can /// make sense to insert the empty node. - pub fn new_empty_tree>() -> Self { + pub fn new_empty>() -> Self { Self::New(NewChangesetNode { hash: C::hashed_null_node(), prefix: Default::default(), From 8e2852fd931d92efdf3b3d56964ad7598f14d094 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 27 Mar 2024 10:04:15 +0100 Subject: [PATCH 67/90] Switch from trie-db crate to subtrie. --- Cargo.toml | 2 +- README.md | 33 ++----------------- reference-trie/Cargo.toml | 2 +- {trie-db => subtrie}/CHANGELOG.md | 0 {trie-db => subtrie}/Cargo.toml | 4 +-- {trie-db => subtrie}/fuzz/Cargo.toml | 2 +- .../fuzz/fuzz_targets/no_ext_insert.rs | 0 .../fuzz/fuzz_targets/no_ext_insert_rem.rs | 0 .../fuzz/fuzz_targets/prefix_iter.rs | 0 .../fuzz/fuzz_targets/prefix_seek_iter.rs | 0 .../fuzz/fuzz_targets/seek_iter.rs | 0 .../fuzz/fuzz_targets/trie_codec_proof.rs | 0 .../fuzz/fuzz_targets/trie_proof_invalid.rs | 0 .../fuzz/fuzz_targets/trie_proof_valid.rs | 0 .../fuzz/fuzz_targets/trie_root.rs | 0 .../fuzz/fuzz_targets/trie_root_fix_len.rs | 0 .../fuzz/fuzz_targets/trie_root_new.rs | 0 {trie-db => subtrie}/fuzz/src/lib.rs | 6 ++-- {trie-db => subtrie}/src/bench.rs | 0 {trie-db => subtrie}/src/iter_build.rs | 0 {trie-db => subtrie}/src/iterator.rs | 0 {trie-db => subtrie}/src/keccak_hasher.rs | 0 {trie-db => subtrie}/src/lib.rs | 0 {trie-db => subtrie}/src/lookup.rs | 0 {trie-db => subtrie}/src/mem_tree_db.rs | 0 {trie-db => subtrie}/src/memory_db.rs | 0 .../src/nibble/leftnibbleslice.rs | 0 {trie-db => subtrie}/src/nibble/mod.rs | 0 .../src/nibble/nibbleslice.rs | 0 {trie-db => subtrie}/src/nibble/nibblevec.rs | 0 {trie-db => subtrie}/src/node.rs | 0 {trie-db => subtrie}/src/node_codec.rs | 0 {trie-db => subtrie}/src/node_db.rs | 0 {trie-db => subtrie}/src/proof/generate.rs | 0 {trie-db => subtrie}/src/proof/mod.rs | 0 {trie-db => subtrie}/src/proof/verify.rs | 0 {trie-db => subtrie}/src/recorder.rs | 0 {trie-db => subtrie}/src/test_utils.rs | 0 {trie-db => subtrie}/src/trie_codec.rs | 0 {trie-db => subtrie}/src/trie_root.rs | 0 {trie-db => subtrie}/src/triedb.rs | 0 {trie-db => subtrie}/src/triedbmut.rs | 0 test/Cargo.toml | 2 +- test/src/lib.rs | 2 +- 44 files changed, 12 insertions(+), 41 deletions(-) rename {trie-db => subtrie}/CHANGELOG.md (100%) rename {trie-db => subtrie}/Cargo.toml (96%) rename {trie-db => subtrie}/fuzz/Cargo.toml (98%) rename {trie-db => subtrie}/fuzz/fuzz_targets/no_ext_insert.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/no_ext_insert_rem.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/prefix_iter.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/prefix_seek_iter.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/seek_iter.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_codec_proof.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_proof_invalid.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_proof_valid.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_root.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_root_fix_len.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_root_new.rs (100%) rename {trie-db => subtrie}/fuzz/src/lib.rs (98%) rename {trie-db => subtrie}/src/bench.rs (100%) rename {trie-db => subtrie}/src/iter_build.rs (100%) rename {trie-db => subtrie}/src/iterator.rs (100%) rename {trie-db => subtrie}/src/keccak_hasher.rs (100%) rename {trie-db => subtrie}/src/lib.rs (100%) rename {trie-db => subtrie}/src/lookup.rs (100%) rename {trie-db => subtrie}/src/mem_tree_db.rs (100%) rename {trie-db => subtrie}/src/memory_db.rs (100%) rename {trie-db => subtrie}/src/nibble/leftnibbleslice.rs (100%) rename {trie-db => subtrie}/src/nibble/mod.rs (100%) rename {trie-db => subtrie}/src/nibble/nibbleslice.rs (100%) rename {trie-db => subtrie}/src/nibble/nibblevec.rs (100%) rename {trie-db => subtrie}/src/node.rs (100%) rename {trie-db => subtrie}/src/node_codec.rs (100%) rename {trie-db => subtrie}/src/node_db.rs (100%) rename {trie-db => subtrie}/src/proof/generate.rs (100%) rename {trie-db => subtrie}/src/proof/mod.rs (100%) rename {trie-db => subtrie}/src/proof/verify.rs (100%) rename {trie-db => subtrie}/src/recorder.rs (100%) rename {trie-db => subtrie}/src/test_utils.rs (100%) rename {trie-db => subtrie}/src/trie_codec.rs (100%) rename {trie-db => subtrie}/src/trie_root.rs (100%) rename {trie-db => subtrie}/src/triedb.rs (100%) rename {trie-db => subtrie}/src/triedbmut.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 22cf63e0..b0794c99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] members = [ "hash256-std-hasher", - "trie-db", + "subtrie", "test", "reference-trie", ] diff --git a/README.md b/README.md index 3660d968..921a8aad 100644 --- a/README.md +++ b/README.md @@ -4,40 +4,11 @@ A generic implementation of the Base-16 Modified Merkle Tree ("Trie") data structure, provided under the Apache2 license. -The implementation comes in two formats: - -- Trie DB (`trie-db` crate) which can be combined with a backend database to provide - a persistent trie structure whose contents can be modified and whose root hash - is recalculated efficiently. -- Trie Root (`trie-root` crate) which provides a closed-form function that accepts a - enumeration of keys and values and provides a root calculated entirely in-memory and - closed form. - Trie Hash alone is able to be used in `no_std` builds by disabling its (default) `std` feature. +Implementation is in `subtrie` crate. -In addition to these, several support crates are provided: - -- `hash-db` crate, used to provide `Hasher` (trait for all things that - can make cryptographic hashes) and `NodeDB` (trait for databases that can have byte - slices pushed into them and allow for them to be retrieved based on their hash). - Suitable for `no_std`, though in this case will only provide `Hasher`. -- `memory-db` crate, contains `MemoryDB`, an implementation of a `NodeDB` using only - in in-memory map. -- `hash256-std-hasher` crate, an implementation of a `std::hash::Hasher` for 32-byte - keys that have already been hashed. Useful to build the backing `HashMap` for `MemoryDB`. - -There are also three crates used only for testing: - -- `keccak-hasher` crate, an implementation of `Hasher` based on the Keccak-256 algorithm. -- `reference-trie` crate, an implementation of a simple trie format; this provides both - a `NodeCodec` and `TrieStream` implementation making it suitable for both Trie DB and - Trie Root. -- `trie-standardmap` crate, a key/value generation tool for creating large test datasets - to specific qualities. -- `trie-bench` crate, a comprehensive standard benchmarking tool for trie format - implementations. Works using the `criterion` project so benchmarking can be done with - the stable rustc branch. +Testing in `reference-trie` crate and `trie-db-test`. In the spirit of all things Rust, this aims to be reliable, secure, and high performance. diff --git a/reference-trie/Cargo.toml b/reference-trie/Cargo.toml index ad461be3..f26ed81e 100644 --- a/reference-trie/Cargo.toml +++ b/reference-trie/Cargo.toml @@ -8,7 +8,7 @@ license = "Apache-2.0" edition = "2018" [dependencies] -trie-db = { path = "../trie-db", default-features = false, version = "0.28.0" } +trie-db = { package = "subtrie", path = "../subtrie", default-features = false, version = "0.0.1" } parity-scale-codec = { version = "3.0.0", features = ["derive"] } hashbrown = { version = "0.14.1", default-features = false, features = ["ahash"] } paste = "1.0.12" diff --git a/trie-db/CHANGELOG.md b/subtrie/CHANGELOG.md similarity index 100% rename from trie-db/CHANGELOG.md rename to subtrie/CHANGELOG.md diff --git a/trie-db/Cargo.toml b/subtrie/Cargo.toml similarity index 96% rename from trie-db/Cargo.toml rename to subtrie/Cargo.toml index 6e62b290..a0c607ad 100644 --- a/trie-db/Cargo.toml +++ b/subtrie/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "trie-db" -version = "0.28.0" +name = "subtrie" +version = "0.0.1" authors = ["Parity Technologies "] description = "Merkle-Patricia Trie generic over key hasher and node encoding" repository = "https://github.com/paritytech/trie" diff --git a/trie-db/fuzz/Cargo.toml b/subtrie/fuzz/Cargo.toml similarity index 98% rename from trie-db/fuzz/Cargo.toml rename to subtrie/fuzz/Cargo.toml index 20fd9146..24e4a837 100644 --- a/trie-db/fuzz/Cargo.toml +++ b/subtrie/fuzz/Cargo.toml @@ -13,7 +13,7 @@ reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0 arbitrary = { version = "1.3.0", features = ["derive"] } array-bytes = "6.0.0" -[dependencies.trie-db] +[dependencies.subtrie] path = ".." [dependencies.libfuzzer-sys] diff --git a/trie-db/fuzz/fuzz_targets/no_ext_insert.rs b/subtrie/fuzz/fuzz_targets/no_ext_insert.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/no_ext_insert.rs rename to subtrie/fuzz/fuzz_targets/no_ext_insert.rs diff --git a/trie-db/fuzz/fuzz_targets/no_ext_insert_rem.rs b/subtrie/fuzz/fuzz_targets/no_ext_insert_rem.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/no_ext_insert_rem.rs rename to subtrie/fuzz/fuzz_targets/no_ext_insert_rem.rs diff --git a/trie-db/fuzz/fuzz_targets/prefix_iter.rs b/subtrie/fuzz/fuzz_targets/prefix_iter.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/prefix_iter.rs rename to subtrie/fuzz/fuzz_targets/prefix_iter.rs diff --git a/trie-db/fuzz/fuzz_targets/prefix_seek_iter.rs b/subtrie/fuzz/fuzz_targets/prefix_seek_iter.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/prefix_seek_iter.rs rename to subtrie/fuzz/fuzz_targets/prefix_seek_iter.rs diff --git a/trie-db/fuzz/fuzz_targets/seek_iter.rs b/subtrie/fuzz/fuzz_targets/seek_iter.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/seek_iter.rs rename to subtrie/fuzz/fuzz_targets/seek_iter.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_codec_proof.rs b/subtrie/fuzz/fuzz_targets/trie_codec_proof.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_codec_proof.rs rename to subtrie/fuzz/fuzz_targets/trie_codec_proof.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs b/subtrie/fuzz/fuzz_targets/trie_proof_invalid.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs rename to subtrie/fuzz/fuzz_targets/trie_proof_invalid.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_proof_valid.rs b/subtrie/fuzz/fuzz_targets/trie_proof_valid.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_proof_valid.rs rename to subtrie/fuzz/fuzz_targets/trie_proof_valid.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_root.rs b/subtrie/fuzz/fuzz_targets/trie_root.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_root.rs rename to subtrie/fuzz/fuzz_targets/trie_root.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs b/subtrie/fuzz/fuzz_targets/trie_root_fix_len.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs rename to subtrie/fuzz/fuzz_targets/trie_root_fix_len.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_root_new.rs b/subtrie/fuzz/fuzz_targets/trie_root_new.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_root_new.rs rename to subtrie/fuzz/fuzz_targets/trie_root_new.rs diff --git a/trie-db/fuzz/src/lib.rs b/subtrie/fuzz/src/lib.rs similarity index 98% rename from trie-db/fuzz/src/lib.rs rename to subtrie/fuzz/src/lib.rs index a923b747..537a75c7 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/subtrie/fuzz/src/lib.rs @@ -17,7 +17,7 @@ use reference_trie::{ calc_root, compare_insert_remove, reference_trie_root_iter_build as reference_trie_root, }; use std::{convert::TryInto, fmt::Debug}; -use trie_db::{ +use subtrie::{ memory_db::{HashKey, MemoryDB, PrefixedKey}, node_db::Hasher, proof::{generate_proof, verify_proof}, @@ -282,7 +282,7 @@ pub fn fuzz_prefix_seek_iter(mut input: PrefixSeekTestInput) { let trie = TrieDBBuilder::::new(&memdb, &root).build(); let iter = - trie_db::TrieDBIterator::new_prefixed_then_seek(&trie, &input.prefix_key, &input.seek_key) + subtrie::TrieDBIterator::new_prefixed_then_seek(&trie, &input.prefix_key, &input.seek_key) .unwrap(); let output_keys: Vec<_> = iter.map(|item| item.unwrap().0).collect(); @@ -405,7 +405,7 @@ fn test_generate_proof( } fn test_trie_codec_proof(entries: Vec<(Vec, Vec)>, keys: Vec>) { - use trie_db::{node_db::EMPTY_PREFIX, decode_compact, encode_compact, Recorder}; + use subtrie::{node_db::EMPTY_PREFIX, decode_compact, encode_compact, Recorder}; // Populate DB with full trie from entries. let (db, root) = { diff --git a/trie-db/src/bench.rs b/subtrie/src/bench.rs similarity index 100% rename from trie-db/src/bench.rs rename to subtrie/src/bench.rs diff --git a/trie-db/src/iter_build.rs b/subtrie/src/iter_build.rs similarity index 100% rename from trie-db/src/iter_build.rs rename to subtrie/src/iter_build.rs diff --git a/trie-db/src/iterator.rs b/subtrie/src/iterator.rs similarity index 100% rename from trie-db/src/iterator.rs rename to subtrie/src/iterator.rs diff --git a/trie-db/src/keccak_hasher.rs b/subtrie/src/keccak_hasher.rs similarity index 100% rename from trie-db/src/keccak_hasher.rs rename to subtrie/src/keccak_hasher.rs diff --git a/trie-db/src/lib.rs b/subtrie/src/lib.rs similarity index 100% rename from trie-db/src/lib.rs rename to subtrie/src/lib.rs diff --git a/trie-db/src/lookup.rs b/subtrie/src/lookup.rs similarity index 100% rename from trie-db/src/lookup.rs rename to subtrie/src/lookup.rs diff --git a/trie-db/src/mem_tree_db.rs b/subtrie/src/mem_tree_db.rs similarity index 100% rename from trie-db/src/mem_tree_db.rs rename to subtrie/src/mem_tree_db.rs diff --git a/trie-db/src/memory_db.rs b/subtrie/src/memory_db.rs similarity index 100% rename from trie-db/src/memory_db.rs rename to subtrie/src/memory_db.rs diff --git a/trie-db/src/nibble/leftnibbleslice.rs b/subtrie/src/nibble/leftnibbleslice.rs similarity index 100% rename from trie-db/src/nibble/leftnibbleslice.rs rename to subtrie/src/nibble/leftnibbleslice.rs diff --git a/trie-db/src/nibble/mod.rs b/subtrie/src/nibble/mod.rs similarity index 100% rename from trie-db/src/nibble/mod.rs rename to subtrie/src/nibble/mod.rs diff --git a/trie-db/src/nibble/nibbleslice.rs b/subtrie/src/nibble/nibbleslice.rs similarity index 100% rename from trie-db/src/nibble/nibbleslice.rs rename to subtrie/src/nibble/nibbleslice.rs diff --git a/trie-db/src/nibble/nibblevec.rs b/subtrie/src/nibble/nibblevec.rs similarity index 100% rename from trie-db/src/nibble/nibblevec.rs rename to subtrie/src/nibble/nibblevec.rs diff --git a/trie-db/src/node.rs b/subtrie/src/node.rs similarity index 100% rename from trie-db/src/node.rs rename to subtrie/src/node.rs diff --git a/trie-db/src/node_codec.rs b/subtrie/src/node_codec.rs similarity index 100% rename from trie-db/src/node_codec.rs rename to subtrie/src/node_codec.rs diff --git a/trie-db/src/node_db.rs b/subtrie/src/node_db.rs similarity index 100% rename from trie-db/src/node_db.rs rename to subtrie/src/node_db.rs diff --git a/trie-db/src/proof/generate.rs b/subtrie/src/proof/generate.rs similarity index 100% rename from trie-db/src/proof/generate.rs rename to subtrie/src/proof/generate.rs diff --git a/trie-db/src/proof/mod.rs b/subtrie/src/proof/mod.rs similarity index 100% rename from trie-db/src/proof/mod.rs rename to subtrie/src/proof/mod.rs diff --git a/trie-db/src/proof/verify.rs b/subtrie/src/proof/verify.rs similarity index 100% rename from trie-db/src/proof/verify.rs rename to subtrie/src/proof/verify.rs diff --git a/trie-db/src/recorder.rs b/subtrie/src/recorder.rs similarity index 100% rename from trie-db/src/recorder.rs rename to subtrie/src/recorder.rs diff --git a/trie-db/src/test_utils.rs b/subtrie/src/test_utils.rs similarity index 100% rename from trie-db/src/test_utils.rs rename to subtrie/src/test_utils.rs diff --git a/trie-db/src/trie_codec.rs b/subtrie/src/trie_codec.rs similarity index 100% rename from trie-db/src/trie_codec.rs rename to subtrie/src/trie_codec.rs diff --git a/trie-db/src/trie_root.rs b/subtrie/src/trie_root.rs similarity index 100% rename from trie-db/src/trie_root.rs rename to subtrie/src/trie_root.rs diff --git a/trie-db/src/triedb.rs b/subtrie/src/triedb.rs similarity index 100% rename from trie-db/src/triedb.rs rename to subtrie/src/triedb.rs diff --git a/trie-db/src/triedbmut.rs b/subtrie/src/triedbmut.rs similarity index 100% rename from trie-db/src/triedbmut.rs rename to subtrie/src/triedbmut.rs diff --git a/test/Cargo.toml b/test/Cargo.toml index 532c36c8..14cbc60d 100644 --- a/test/Cargo.toml +++ b/test/Cargo.toml @@ -16,7 +16,7 @@ name = "memory_db" harness = false [dependencies] -trie-db = { path = "../trie-db", version = "0.28.0", features = ["test_utils", "bench"]} +trie-db = { package = "subtrie", path = "../subtrie", version = "0.0.1", features = ["test_utils", "bench"]} rand = { version = "0.8", default-features = false, features = ["small_rng"] } reference-trie = { path = "../reference-trie", version = "0.29.0" } hex-literal = "0.4" diff --git a/test/src/lib.rs b/test/src/lib.rs index be5c4abb..d0a7a688 100644 --- a/test/src/lib.rs +++ b/test/src/lib.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Tests for trie-db crate. +//! Tests for subtrie crate. #[cfg(test)] mod double_ended_iterator; From 71bcaf8a4cd8c0fb7bf3ae95f433079c9451daa3 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 27 Mar 2024 10:20:18 +0100 Subject: [PATCH 68/90] fix doctests --- subtrie/src/memory_db.rs | 16 ++++++++-------- subtrie/src/proof/generate.rs | 4 ++-- subtrie/src/triedb.rs | 4 ++-- subtrie/src/triedbmut.rs | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/subtrie/src/memory_db.rs b/subtrie/src/memory_db.rs index c08eb297..7f3aad37 100644 --- a/subtrie/src/memory_db.rs +++ b/subtrie/src/memory_db.rs @@ -37,10 +37,10 @@ use alloc::collections::btree_map::{BTreeMap as Map, Entry}; /// ```rust /// #[cfg(feature = "test_utils")] /// { -/// use trie_db::node_db::Hasher; -/// use trie_db::node_db::{EMPTY_PREFIX}; -/// use trie_db::keccak_hasher::KeccakHasher; -/// use trie_db::memory_db::{MemoryDB, HashKey}; +/// use subtrie::node_db::Hasher; +/// use subtrie::node_db::{EMPTY_PREFIX}; +/// use subtrie::keccak_hasher::KeccakHasher; +/// use subtrie::memory_db::{MemoryDB, HashKey}; /// /// let mut m = MemoryDB::, Vec>::default(); /// let d = "Hello world!".as_bytes(); @@ -280,10 +280,10 @@ where /// ```rust /// #[cfg(feature = "test_utils")] /// { - /// use trie_db::node_db::Hasher; - /// use trie_db::node_db::{NodeDB, EMPTY_PREFIX}; - /// use trie_db::keccak_hasher::KeccakHasher; - /// use trie_db::memory_db::{MemoryDB, HashKey}; + /// use subtrie::node_db::Hasher; + /// use subtrie::node_db::{NodeDB, EMPTY_PREFIX}; + /// use subtrie::keccak_hasher::KeccakHasher; + /// use subtrie::memory_db::{MemoryDB, HashKey}; /// /// fn main() { /// let mut m = MemoryDB::, Vec>::default(); diff --git a/subtrie/src/proof/generate.rs b/subtrie/src/proof/generate.rs index b03f0274..a874e0e2 100644 --- a/subtrie/src/proof/generate.rs +++ b/subtrie/src/proof/generate.rs @@ -333,7 +333,7 @@ where assert_eq!( Some(&value), expected_value.as_ref(), - "expected_value is found using `trie_db::Lookup`; \ + "expected_value is found using `subtrie::Lookup`; \ value is found by traversing the same nodes recorded during the lookup \ using the same logic; \ thus the values found must be equal" @@ -352,7 +352,7 @@ where assert_eq!( value, expected_value.as_ref().map(|v| v.as_ref()), - "expected_value is found using `trie_db::Lookup`; \ + "expected_value is found using `subtrie::Lookup`; \ value is found by traversing the same nodes recorded during the lookup \ using the same logic; \ thus the values found must be equal" diff --git a/subtrie/src/triedb.rs b/subtrie/src/triedb.rs index 668e9a02..9d649615 100644 --- a/subtrie/src/triedb.rs +++ b/subtrie/src/triedb.rs @@ -120,9 +120,9 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// /// # Example /// ```ignore -/// use trie_db::node_db::Hasher; +/// use subtrie::node_db::Hasher; /// use reference_trie::{RefTrieDBMut, RefTrieDB, Trie}; -/// use trie_db::DBValue; +/// use subtrie::DBValue; /// use keccak_hasher::KeccakHasher; /// use memory_db::*; /// diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index 86735d3e..1b1c9c80 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -871,11 +871,11 @@ pub type OwnedPrefix = (BackingByteVec, Option); /// /// # Example /// ```ignore -/// use trie_db::node_db::Hasher; +/// use subtrie::node_db::Hasher; /// use reference_trie::RefTrieDBMut; -/// use trie_db::DBValue; +/// use subtrie::DBValue; /// use keccak_hasher::KeccakHasher; -/// use trie_db::memory_db::*; +/// use subtrie::memory_db::*; /// /// let mut memdb = MemoryDB::, DBValue>::default(); /// let mut root = Default::default(); From af1b48cd571fc0a9a8fc46d4c6282dd55461fe99 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 27 Mar 2024 10:49:48 +0100 Subject: [PATCH 69/90] ci --- .github/workflows/rust.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 39aa7cf9..4c7760b6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -37,13 +37,13 @@ jobs: command: test args: --workspace - - name: Check trie-db Without Std + - name: Check subtrie Without Std uses: actions-rs/cargo@v1 with: command: check - args: --manifest-path trie-db/Cargo.toml --no-default-features + args: --manifest-path subtrie/Cargo.toml --no-default-features - - name: Check trie-db-test With bench + - name: Check subtrie-test With bench uses: actions-rs/cargo@v1 with: command: check From 61c6261ab9a0dc1314a22e012fbc8f5c61f88c97 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 2 Apr 2024 07:31:07 +0000 Subject: [PATCH 70/90] switch to single subtrie crate dep (#1) Maintain a single 'subtrie' crate. --- .github/workflows/rust.yml | 18 +- Cargo.toml | 15 +- README.md | 33 +- hash-db/CHANGELOG.md | 10 - hash-db/Cargo.toml | 14 - hash-db/README.md | 4 - hash-db/src/lib.rs | 216 ---- hash256-std-hasher/Cargo.toml | 2 +- memory-db/CHANGELOG.md | 45 - memory-db/Cargo.toml | 25 - memory-db/README.md | 2 +- .../CHANGELOG.md | 0 .../Cargo.toml | 11 +- .../benches/bench.rs | 2 +- .../src/lib.rs | 268 +++-- .../src/substrate.rs | 40 +- .../src/substrate_like.rs | 42 +- {trie-db => subtrie}/CHANGELOG.md | 5 + subtrie/Cargo.toml | 39 + {trie-db => subtrie}/fuzz/Cargo.toml | 4 +- .../fuzz/fuzz_targets/no_ext_insert.rs | 0 .../fuzz/fuzz_targets/no_ext_insert_rem.rs | 0 .../fuzz/fuzz_targets/prefix_iter.rs | 0 .../fuzz/fuzz_targets/prefix_seek_iter.rs | 0 .../fuzz/fuzz_targets/seek_iter.rs | 0 .../fuzz/fuzz_targets/trie_codec_proof.rs | 0 .../fuzz/fuzz_targets/trie_proof_invalid.rs | 0 .../fuzz/fuzz_targets/trie_proof_valid.rs | 0 .../fuzz/fuzz_targets/trie_root.rs | 0 .../fuzz/fuzz_targets/trie_root_fix_len.rs | 0 .../fuzz/fuzz_targets/trie_root_new.rs | 0 {trie-db => subtrie}/fuzz/src/lib.rs | 100 +- .../src/lib.rs => subtrie/src/bench.rs | 27 +- {trie-db => subtrie}/src/iter_build.rs | 53 +- {trie-db => subtrie}/src/iterator.rs | 524 +++++--- .../lib.rs => subtrie/src/keccak_hasher.rs | 2 +- {trie-db => subtrie}/src/lib.rs | 311 ++--- {trie-db => subtrie}/src/lookup.rs | 406 +++++-- subtrie/src/mem_tree_db.rs | 367 ++++++ .../src/lib.rs => subtrie/src/memory_db.rs | 221 +--- .../src/nibble/leftnibbleslice.rs | 0 {trie-db => subtrie}/src/nibble/mod.rs | 2 +- .../src/nibble/nibbleslice.rs | 3 +- {trie-db => subtrie}/src/nibble/nibblevec.rs | 27 +- {trie-db => subtrie}/src/node.rs | 313 +++-- {trie-db => subtrie}/src/node_codec.rs | 31 +- subtrie/src/node_db.rs | 97 ++ {trie-db => subtrie}/src/proof/generate.rs | 188 ++- {trie-db => subtrie}/src/proof/mod.rs | 0 {trie-db => subtrie}/src/proof/verify.rs | 28 +- {trie-db => subtrie}/src/recorder.rs | 19 +- .../src/lib.rs => subtrie/src/test_utils.rs | 7 +- {trie-db => subtrie}/src/trie_codec.rs | 96 +- .../src/lib.rs => subtrie/src/trie_root.rs | 30 +- {trie-db => subtrie}/src/triedb.rs | 252 +++- {trie-db => subtrie}/src/triedbmut.rs | 1058 ++++++++++------- test-support/keccak-hasher/CHANGELOG.md | 11 - test-support/keccak-hasher/Cargo.toml | 19 - test-support/trie-bench/CHANGELOG.md | 66 - test-support/trie-bench/Cargo.toml | 18 - test-support/trie-standardmap/CHANGELOG.md | 8 - test-support/trie-standardmap/Cargo.toml | 11 - {trie-db/test => test}/Cargo.toml | 15 +- {trie-db/test => test}/benches/bench.rs | 8 +- .../bench.rs => test/benches/memory_db.rs | 8 +- test/src/double_ended_iterator.rs | 410 +++++++ {trie-db/test => test}/src/iter_build.rs | 63 +- {trie-db/test => test}/src/iterator.rs | 59 +- test/src/lib.rs | 114 ++ {trie-db/test => test}/src/proof.rs | 78 +- {trie-db/test => test}/src/recorder.rs | 30 +- {trie-db/test => test}/src/trie_codec.rs | 58 +- test/src/trie_root.rs | 44 + {trie-db/test => test}/src/triedb.rs | 569 ++++++--- test/src/triedbmut.rs | 1051 ++++++++++++++++ trie-db/Cargo.toml | 22 - trie-db/src/fatdb.rs | 187 --- trie-db/src/fatdbmut.rs | 116 -- trie-db/src/sectriedb.rs | 97 -- trie-db/src/sectriedbmut.rs | 96 -- trie-db/test/src/fatdb.rs | 37 - trie-db/test/src/fatdbmut.rs | 47 - trie-db/test/src/lib.rs | 38 - trie-db/test/src/sectriedb.rs | 30 - trie-db/test/src/sectriedbmut.rs | 30 - trie-db/test/src/triedbmut.rs | 838 ------------- trie-eip1186/CHANGELOG.md | 8 - trie-eip1186/Cargo.toml | 19 - trie-eip1186/src/eip1186.rs | 311 ----- trie-eip1186/src/lib.rs | 30 - trie-eip1186/test/Cargo.toml | 15 - trie-eip1186/test/src/eip1186.rs | 181 --- trie-eip1186/test/src/lib.rs | 18 - trie-root/CHANGELOG.md | 17 - trie-root/Cargo.toml | 18 - trie-root/README.md | 2 - trie-root/test/Cargo.toml | 15 - trie-root/test/src/lib.rs | 45 - 98 files changed, 5213 insertions(+), 4601 deletions(-) delete mode 100644 hash-db/CHANGELOG.md delete mode 100644 hash-db/Cargo.toml delete mode 100644 hash-db/README.md delete mode 100644 hash-db/src/lib.rs delete mode 100644 memory-db/CHANGELOG.md delete mode 100644 memory-db/Cargo.toml rename {test-support/reference-trie => reference-trie}/CHANGELOG.md (100%) rename {test-support/reference-trie => reference-trie}/Cargo.toml (55%) rename {test-support/reference-trie => reference-trie}/benches/bench.rs (95%) rename {test-support/reference-trie => reference-trie}/src/lib.rs (83%) rename {test-support/reference-trie => reference-trie}/src/substrate.rs (96%) rename {test-support/reference-trie => reference-trie}/src/substrate_like.rs (94%) rename {trie-db => subtrie}/CHANGELOG.md (90%) create mode 100644 subtrie/Cargo.toml rename {trie-db => subtrie}/fuzz/Cargo.toml (89%) rename {trie-db => subtrie}/fuzz/fuzz_targets/no_ext_insert.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/no_ext_insert_rem.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/prefix_iter.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/prefix_seek_iter.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/seek_iter.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_codec_proof.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_proof_invalid.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_proof_valid.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_root.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_root_fix_len.rs (100%) rename {trie-db => subtrie}/fuzz/fuzz_targets/trie_root_new.rs (100%) rename {trie-db => subtrie}/fuzz/src/lib.rs (83%) rename test-support/trie-bench/src/lib.rs => subtrie/src/bench.rs (99%) rename {trie-db => subtrie}/src/iter_build.rs (91%) rename {trie-db => subtrie}/src/iterator.rs (51%) rename test-support/keccak-hasher/src/lib.rs => subtrie/src/keccak_hasher.rs (98%) rename {trie-db => subtrie}/src/lib.rs (73%) rename {trie-db => subtrie}/src/lookup.rs (58%) create mode 100644 subtrie/src/mem_tree_db.rs rename memory-db/src/lib.rs => subtrie/src/memory_db.rs (71%) rename {trie-db => subtrie}/src/nibble/leftnibbleslice.rs (100%) rename {trie-db => subtrie}/src/nibble/mod.rs (98%) rename {trie-db => subtrie}/src/nibble/nibbleslice.rs (99%) rename {trie-db => subtrie}/src/nibble/nibblevec.rs (94%) rename {trie-db => subtrie}/src/node.rs (68%) rename {trie-db => subtrie}/src/node_codec.rs (84%) create mode 100644 subtrie/src/node_db.rs rename {trie-db => subtrie}/src/proof/generate.rs (79%) rename {trie-db => subtrie}/src/proof/mod.rs (100%) rename {trie-db => subtrie}/src/proof/verify.rs (96%) rename {trie-db => subtrie}/src/recorder.rs (79%) rename test-support/trie-standardmap/src/lib.rs => subtrie/src/test_utils.rs (96%) rename {trie-db => subtrie}/src/trie_codec.rs (87%) rename trie-root/src/lib.rs => subtrie/src/trie_root.rs (96%) rename {trie-db => subtrie}/src/triedb.rs (65%) rename {trie-db => subtrie}/src/triedbmut.rs (67%) delete mode 100644 test-support/keccak-hasher/CHANGELOG.md delete mode 100644 test-support/keccak-hasher/Cargo.toml delete mode 100644 test-support/trie-bench/CHANGELOG.md delete mode 100644 test-support/trie-bench/Cargo.toml delete mode 100644 test-support/trie-standardmap/CHANGELOG.md delete mode 100644 test-support/trie-standardmap/Cargo.toml rename {trie-db/test => test}/Cargo.toml (56%) rename {trie-db/test => test}/benches/bench.rs (98%) rename memory-db/benches/bench.rs => test/benches/memory_db.rs (95%) create mode 100644 test/src/double_ended_iterator.rs rename {trie-db/test => test}/src/iter_build.rs (84%) rename {trie-db/test => test}/src/iterator.rs (89%) create mode 100644 test/src/lib.rs rename {trie-db/test => test}/src/proof.rs (72%) rename {trie-db/test => test}/src/recorder.rs (83%) rename {trie-db/test => test}/src/trie_codec.rs (79%) create mode 100644 test/src/trie_root.rs rename {trie-db/test => test}/src/triedb.rs (65%) create mode 100644 test/src/triedbmut.rs delete mode 100644 trie-db/Cargo.toml delete mode 100644 trie-db/src/fatdb.rs delete mode 100644 trie-db/src/fatdbmut.rs delete mode 100644 trie-db/src/sectriedb.rs delete mode 100644 trie-db/src/sectriedbmut.rs delete mode 100644 trie-db/test/src/fatdb.rs delete mode 100644 trie-db/test/src/fatdbmut.rs delete mode 100644 trie-db/test/src/lib.rs delete mode 100644 trie-db/test/src/sectriedb.rs delete mode 100644 trie-db/test/src/sectriedbmut.rs delete mode 100644 trie-db/test/src/triedbmut.rs delete mode 100644 trie-eip1186/CHANGELOG.md delete mode 100644 trie-eip1186/Cargo.toml delete mode 100644 trie-eip1186/src/eip1186.rs delete mode 100644 trie-eip1186/src/lib.rs delete mode 100644 trie-eip1186/test/Cargo.toml delete mode 100644 trie-eip1186/test/src/eip1186.rs delete mode 100644 trie-eip1186/test/src/lib.rs delete mode 100644 trie-root/CHANGELOG.md delete mode 100644 trie-root/Cargo.toml delete mode 100644 trie-root/README.md delete mode 100644 trie-root/test/Cargo.toml delete mode 100644 trie-root/test/src/lib.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 348be1dd..4c7760b6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Rust Toolchain uses: actions-rs/toolchain@v1 @@ -37,29 +37,23 @@ jobs: command: test args: --workspace - - name: Check trie-db Without Std + - name: Check subtrie Without Std uses: actions-rs/cargo@v1 with: command: check - args: --manifest-path trie-db/Cargo.toml --no-default-features + args: --manifest-path subtrie/Cargo.toml --no-default-features - - name: Check memory-db Without Std + - name: Check subtrie-test With bench uses: actions-rs/cargo@v1 with: command: check - args: --manifest-path memory-db/Cargo.toml --no-default-features - - - name: Check trie-root Without Std - uses: actions-rs/cargo@v1 - with: - command: check - args: --manifest-path trie-root/Cargo.toml --no-default-features + args: --manifest-path test/Cargo.toml --benches fmt: name: Rustfmt runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: profile: minimal diff --git a/Cargo.toml b/Cargo.toml index 607558fa..b0794c99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,16 +1,7 @@ [workspace] members = [ - "hash-db", - "memory-db", "hash256-std-hasher", - "test-support/keccak-hasher", - "test-support/reference-trie", - "test-support/trie-standardmap", - "test-support/trie-bench", - "trie-db", - "trie-db/test", - "trie-eip1186", - "trie-eip1186/test", - "trie-root", - "trie-root/test" + "subtrie", + "test", + "reference-trie", ] diff --git a/README.md b/README.md index 8f523fe4..921a8aad 100644 --- a/README.md +++ b/README.md @@ -4,40 +4,11 @@ A generic implementation of the Base-16 Modified Merkle Tree ("Trie") data structure, provided under the Apache2 license. -The implementation comes in two formats: - -- Trie DB (`trie-db` crate) which can be combined with a backend database to provide - a persistent trie structure whose contents can be modified and whose root hash - is recalculated efficiently. -- Trie Root (`trie-root` crate) which provides a closed-form function that accepts a - enumeration of keys and values and provides a root calculated entirely in-memory and - closed form. - Trie Hash alone is able to be used in `no_std` builds by disabling its (default) `std` feature. +Implementation is in `subtrie` crate. -In addition to these, several support crates are provided: - -- `hash-db` crate, used to provide `Hasher` (trait for all things that - can make cryptographic hashes) and `HashDB` (trait for databases that can have byte - slices pushed into them and allow for them to be retrieved based on their hash). - Suitable for `no_std`, though in this case will only provide `Hasher`. -- `memory-db` crate, contains `MemoryDB`, an implementation of a `HashDB` using only - in in-memory map. -- `hash256-std-hasher` crate, an implementation of a `std::hash::Hasher` for 32-byte - keys that have already been hashed. Useful to build the backing `HashMap` for `MemoryDB`. - -There are also three crates used only for testing: - -- `keccak-hasher` crate, an implementation of `Hasher` based on the Keccak-256 algorithm. -- `reference-trie` crate, an implementation of a simple trie format; this provides both - a `NodeCodec` and `TrieStream` implementation making it suitable for both Trie DB and - Trie Root. -- `trie-standardmap` crate, a key/value generation tool for creating large test datasets - to specific qualities. -- `trie-bench` crate, a comprehensive standard benchmarking tool for trie format - implementations. Works using the `criterion` project so benchmarking can be done with - the stable rustc branch. +Testing in `reference-trie` crate and `trie-db-test`. In the spirit of all things Rust, this aims to be reliable, secure, and high performance. diff --git a/hash-db/CHANGELOG.md b/hash-db/CHANGELOG.md deleted file mode 100644 index e5a6a55b..00000000 --- a/hash-db/CHANGELOG.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [0.16.0] - 2023-03-14 -- Requires Hash to be Ord. [#188](https://github.com/paritytech/trie/pull/188) - - diff --git a/hash-db/Cargo.toml b/hash-db/Cargo.toml deleted file mode 100644 index d0c6f104..00000000 --- a/hash-db/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "hash-db" -version = "0.16.0" -authors = ["Parity Technologies "] -description = "Trait for hash-keyed databases." -license = "Apache-2.0" -categories = [ "no-std" ] -repository = "https://github.com/paritytech/trie" -edition = "2018" - -[features] -default = ["std"] -std = [ -] diff --git a/hash-db/README.md b/hash-db/README.md deleted file mode 100644 index 23b49ae8..00000000 --- a/hash-db/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# HashDB -`HashDB` defines a common interface for databases of byte-slices keyed to their hash. It is generic over hash type through the `Hasher` trait. - -The `Hasher` trait can be used in a `no_std` context. \ No newline at end of file diff --git a/hash-db/src/lib.rs b/hash-db/src/lib.rs deleted file mode 100644 index 4825aada..00000000 --- a/hash-db/src/lib.rs +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2017, 2021 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Database of byte-slices keyed to their hash. - -#![cfg_attr(not(feature = "std"), no_std)] - -#[cfg(not(feature = "std"))] -use core::hash; -#[cfg(feature = "std")] -use std::fmt::Debug; -#[cfg(feature = "std")] -use std::hash; - -#[cfg(feature = "std")] -pub trait MaybeDebug: Debug {} -#[cfg(feature = "std")] -impl MaybeDebug for T {} -#[cfg(not(feature = "std"))] -pub trait MaybeDebug {} -#[cfg(not(feature = "std"))] -impl MaybeDebug for T {} - -/// A trie node prefix, it is the nibble path from the trie root -/// to the trie node. -/// For a node containing no partial key value it is the full key. -/// For a value node or node containing a partial key, it is the full key minus its node partial -/// nibbles (the node key can be split into prefix and node partial). -/// Therefore it is always the leftmost portion of the node key, so its internal representation -/// is a non expanded byte slice followed by a last padded byte representation. -/// The padded byte is an optional padded value. -pub type Prefix<'a> = (&'a [u8], Option); - -/// An empty prefix constant. -/// Can be use when the prefix is not use internally -/// or for root nodes. -pub static EMPTY_PREFIX: Prefix<'static> = (&[], None); - -/// Trait describing an object that can hash a slice of bytes. Used to abstract -/// other types over the hashing algorithm. Defines a single `hash` method and an -/// `Out` associated type with the necessary bounds. -pub trait Hasher: Sync + Send { - /// The output type of the `Hasher` - type Out: AsRef<[u8]> - + AsMut<[u8]> - + Default - + MaybeDebug - + core::cmp::Ord - + PartialEq - + Eq - + hash::Hash - + Send - + Sync - + Clone - + Copy; - /// What to use to build `HashMap`s with this `Hasher`. - type StdHasher: Sync + Send + Default + hash::Hasher; - /// The length in bytes of the `Hasher` output. - const LENGTH: usize; - - /// Compute the hash of the provided slice of bytes returning the `Out` type of the `Hasher`. - fn hash(x: &[u8]) -> Self::Out; -} - -/// Trait modelling a plain datastore whose key is a fixed type. -/// The caller should ensure that a key only corresponds to -/// one value. -pub trait PlainDB: Send + Sync + AsPlainDB { - /// Look up a given hash into the bytes that hash to it, returning None if the - /// hash is not known. - fn get(&self, key: &K) -> Option; - - /// Check for the existence of a hash-key. - fn contains(&self, key: &K) -> bool; - - /// Insert a datum item into the DB. Insertions are counted and the equivalent - /// number of `remove()`s must be performed before the data is considered dead. - /// The caller should ensure that a key only corresponds to one value. - fn emplace(&mut self, key: K, value: V); - - /// Remove a datum previously inserted. Insertions can be "owed" such that the - /// same number of `insert()`s may happen without the data being eventually - /// being inserted into the DB. It can be "owed" more than once. - /// The caller should ensure that a key only corresponds to one value. - fn remove(&mut self, key: &K); -} - -/// Trait for immutable reference of PlainDB. -pub trait PlainDBRef { - /// Look up a given hash into the bytes that hash to it, returning None if the - /// hash is not known. - fn get(&self, key: &K) -> Option; - - /// Check for the existance of a hash-key. - fn contains(&self, key: &K) -> bool; -} - -impl<'a, K, V> PlainDBRef for &'a dyn PlainDB { - fn get(&self, key: &K) -> Option { - PlainDB::get(*self, key) - } - fn contains(&self, key: &K) -> bool { - PlainDB::contains(*self, key) - } -} - -impl<'a, K, V> PlainDBRef for &'a mut dyn PlainDB { - fn get(&self, key: &K) -> Option { - PlainDB::get(*self, key) - } - fn contains(&self, key: &K) -> bool { - PlainDB::contains(*self, key) - } -} - -/// Trait modelling datastore keyed by a hash defined by the `Hasher`. -pub trait HashDB: Send + Sync + AsHashDB { - /// Look up a given hash into the bytes that hash to it, returning None if the - /// hash is not known. - fn get(&self, key: &H::Out, prefix: Prefix) -> Option; - - /// Check for the existence of a hash-key. - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool; - - /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions - /// are counted and the equivalent number of `remove()`s must be performed before the data - /// is considered dead. - fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out; - - /// Like `insert()`, except you provide the key and the data is all moved. - fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T); - - /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of - /// `insert()`s may happen without the data being eventually being inserted into the DB. - /// It can be "owed" more than once. - fn remove(&mut self, key: &H::Out, prefix: Prefix); -} - -/// Trait for immutable reference of HashDB. -pub trait HashDBRef { - /// Look up a given hash into the bytes that hash to it, returning None if the - /// hash is not known. - fn get(&self, key: &H::Out, prefix: Prefix) -> Option; - - /// Check for the existance of a hash-key. - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool; -} - -impl<'a, H: Hasher, T> HashDBRef for &'a dyn HashDB { - fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - HashDB::get(*self, key, prefix) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - HashDB::contains(*self, key, prefix) - } -} - -impl<'a, H: Hasher, T> HashDBRef for &'a mut dyn HashDB { - fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - HashDB::get(*self, key, prefix) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - HashDB::contains(*self, key, prefix) - } -} - -/// Upcast trait for HashDB. -pub trait AsHashDB { - /// Perform upcast to HashDB for anything that derives from HashDB. - fn as_hash_db(&self) -> &dyn HashDB; - /// Perform mutable upcast to HashDB for anything that derives from HashDB. - fn as_hash_db_mut<'a>(&'a mut self) -> &'a mut (dyn HashDB + 'a); -} - -/// Upcast trait for PlainDB. -pub trait AsPlainDB { - /// Perform upcast to PlainDB for anything that derives from PlainDB. - fn as_plain_db(&self) -> &dyn PlainDB; - /// Perform mutable upcast to PlainDB for anything that derives from PlainDB. - fn as_plain_db_mut<'a>(&'a mut self) -> &'a mut (dyn PlainDB + 'a); -} - -// NOTE: There used to be a `impl AsHashDB for T` but that does not work with generics. -// See https://stackoverflow.com/questions/48432842/ -// implementing-a-trait-for-reference-and-non-reference-types-causes-conflicting-im -// This means we need concrete impls of AsHashDB in several places, which somewhat defeats -// the point of the trait. -impl<'a, H: Hasher, T> AsHashDB for &'a mut dyn HashDB { - fn as_hash_db(&self) -> &dyn HashDB { - &**self - } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { - &mut **self - } -} - -#[cfg(feature = "std")] -impl<'a, K, V> AsPlainDB for &'a mut dyn PlainDB { - fn as_plain_db(&self) -> &dyn PlainDB { - &**self - } - fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn PlainDB + 'b) { - &mut **self - } -} diff --git a/hash256-std-hasher/Cargo.toml b/hash256-std-hasher/Cargo.toml index addd32fb..bdeb08ef 100644 --- a/hash256-std-hasher/Cargo.toml +++ b/hash256-std-hasher/Cargo.toml @@ -16,7 +16,7 @@ harness = false crunchy = "0.2.1" [dev-dependencies] -criterion = "0.4.0" +criterion = "0.5.1" [features] default = ["std"] diff --git a/memory-db/CHANGELOG.md b/memory-db/CHANGELOG.md deleted file mode 100644 index 532c86a4..00000000 --- a/memory-db/CHANGELOG.md +++ /dev/null @@ -1,45 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [0.32.0] - 2023-03-14 -- Switch no_std storage to BtreeMap. [#188](https://github.com/paritytech/trie/pull/188) - -## [0.31.0] - 2022-11-29 -- Removed `parity-util-mem` support. [#172](https://github.com/paritytech/trie/pull/172) - -## [0.30.0] - 2022-09-20 -- Update `parity-util-mem` to 0.12. [#166](https://github.com/paritytech/trie/pull/166) - -## [0.29.0] - 2022-02-04 -- Update `parity-util-mem` to 0.11. [#150](https://github.com/paritytech/trie/pull/150) - -## [0.28.0] - 2021-10-19 -- Change in api bound. [#142](https://github.com/paritytech/trie/pull/142) - -## [0.27.0] - 2021-07-02 -- Update `parity-util-mem` to 0.10. [#137](https://github.com/paritytech/trie/pull/137) - -## [0.26.0] - 2021-01-27 -- Update `parity-util-mem` to 0.9. [#123](https://github.com/paritytech/trie/pull/123) - -## [0.25.0] - 2021-01-05 -- Update `parity-util-mem` and `hashbrown`, removed `heapsize`. [#118](https://github.com/paritytech/trie/pull/118) - -## [0.24.1] - 2020-07-20 -- Add `shrink_to_fit` method. [#102](https://github.com/paritytech/trie/pull/102) - -## [0.24.0] - 2020-07-07 -- Disable memory tracking for no_std target by default. [#99](https://github.com/paritytech/trie/pull/99) - -## [0.22.0] - 2020-07-06 -- Type parameter to count `malloc_size_of` on memory-db. [#94](https://github.com/paritytech/trie/pull/94) -- Update hashbrown to 0.8. [#97](https://github.com/paritytech/trie/pull/97) - -## [0.20.0] - 2020-03-21 -- Update parity-util-mem to v0.6 [#82](https://github.com/paritytech/trie/pull/82) - -## [0.19.0] - 2020-02-07 -- Update parity-util-mem to v0.5.1 [#78](https://github.com/paritytech/trie/pull/78) diff --git a/memory-db/Cargo.toml b/memory-db/Cargo.toml deleted file mode 100644 index adad836a..00000000 --- a/memory-db/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "memory-db" -version = "0.32.0" -authors = ["Parity Technologies "] -description = "In-memory implementation of hash-db, useful for tests" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -hash-db = { version = "0.16.0", path = "../hash-db", default-features = false } - -[dev-dependencies] -keccak-hasher = { path = "../test-support/keccak-hasher" } -criterion = "0.4.0" - -[features] -default = ["std"] -std = [ - "hash-db/std", -] - -[[bench]] -name = "bench" -harness = false diff --git a/memory-db/README.md b/memory-db/README.md index fc0c6309..497359e2 100644 --- a/memory-db/README.md +++ b/memory-db/README.md @@ -1 +1 @@ -MemoryDB is a reference counted memory-based [`HashDB`](https://github.com/paritytech/parity-common/tree/master/hash-db) implementation backed by a `HashMap`. \ No newline at end of file +MemoryDB is a reference counted memory-based [`NodeDB`](https://github.com/paritytech/parity-common/tree/master/hash-db) implementation backed by a `HashMap`. diff --git a/test-support/reference-trie/CHANGELOG.md b/reference-trie/CHANGELOG.md similarity index 100% rename from test-support/reference-trie/CHANGELOG.md rename to reference-trie/CHANGELOG.md diff --git a/test-support/reference-trie/Cargo.toml b/reference-trie/Cargo.toml similarity index 55% rename from test-support/reference-trie/Cargo.toml rename to reference-trie/Cargo.toml index 90fe577a..f26ed81e 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/reference-trie/Cargo.toml @@ -8,17 +8,13 @@ license = "Apache-2.0" edition = "2018" [dependencies] -hash-db = { path = "../../hash-db" , version = "0.16.0"} -keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } -trie-db = { path = "../../trie-db", default-features = false, version = "0.27.0" } -trie-root = { path = "../../trie-root", default-features = false, version = "0.18.0" } +trie-db = { package = "subtrie", path = "../subtrie", default-features = false, version = "0.0.1" } parity-scale-codec = { version = "3.0.0", features = ["derive"] } -hashbrown = { version = "0.13.2", default-features = false, features = ["ahash"] } +hashbrown = { version = "0.14.1", default-features = false, features = ["ahash"] } paste = "1.0.12" [dev-dependencies] -trie-bench = { path = "../trie-bench" } -criterion = "0.4.0" +criterion = "0.5.1" [[bench]] name = "bench" @@ -29,5 +25,4 @@ default = ["std"] # no actual support for std, only to avoid a cargo issues std = [ "trie-db/std", - "trie-root/std", ] diff --git a/test-support/reference-trie/benches/bench.rs b/reference-trie/benches/bench.rs similarity index 95% rename from test-support/reference-trie/benches/bench.rs rename to reference-trie/benches/bench.rs index b3466d28..4e1cb969 100644 --- a/test-support/reference-trie/benches/bench.rs +++ b/reference-trie/benches/bench.rs @@ -18,7 +18,7 @@ criterion_group!(benches, benchmark); criterion_main!(benches); fn benchmark(c: &mut Criterion) { - trie_bench::standard_benchmark::< + trie_db::bench::standard_benchmark::< reference_trie::ExtensionLayout, reference_trie::ReferenceTrieStream, >(c, "ref"); diff --git a/test-support/reference-trie/src/lib.rs b/reference-trie/src/lib.rs similarity index 83% rename from test-support/reference-trie/src/lib.rs rename to reference-trie/src/lib.rs index be626801..51e55fd4 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/reference-trie/src/lib.rs @@ -18,15 +18,17 @@ use hashbrown::{hash_map::Entry, HashMap}; use parity_scale_codec::{Compact, Decode, Encode, Error as CodecError, Input, Output}; use std::{borrow::Borrow, fmt, iter::once, marker::PhantomData, ops::Range}; use trie_db::{ + memory_db::{KeyFunction, MemoryDB, PrefixedKey}, nibble_ops, node::{NibbleSlicePlan, NodeHandlePlan, NodeOwned, NodePlan, Value, ValuePlan}, + node_db, + node_db::Hasher, + trie_root::{self, TrieStream, Value as TrieStreamValue}, trie_visit, triedbmut::ChildReference, - DBValue, NodeCodec, Trie, TrieBuilder, TrieConfiguration, TrieDBBuilder, TrieDBMutBuilder, - TrieHash, TrieLayout, TrieMut, TrieRoot, + DBValue, Location, NodeCodec, Trie, TrieBuilder, TrieConfiguration, TrieDBBuilder, + TrieDBMutBuilder, TrieHash, TrieLayout, TrieRoot, }; -pub use trie_root::TrieStream; -use trie_root::{Hasher, Value as TrieStreamValue}; mod substrate; mod substrate_like; @@ -41,9 +43,13 @@ pub use substrate_like::{ pub use paste::paste; pub use substrate::{LayoutV0 as SubstrateV0, LayoutV1 as SubstrateV1}; +pub use trie_db::mem_tree_db::{Location as MemLocation, MemTreeDB}; /// Reference hasher is a keccak hasher. -pub type RefHasher = keccak_hasher::KeccakHasher; +pub type RefHasher = trie_db::keccak_hasher::KeccakHasher; + +pub type PrefixedMemoryDB = + MemoryDB<::Hash, PrefixedKey<::Hash>, DBValue>; /// Apply a test method on every test layouts. #[macro_export] @@ -51,14 +57,39 @@ macro_rules! test_layouts { ($test:ident, $test_internal:ident) => { #[test] fn $test() { - eprintln!("Running with layout `HashedValueNoExtThreshold`"); - $test_internal::<$crate::HashedValueNoExtThreshold<1>>(); - eprintln!("Running with layout `HashedValueNoExt`"); - $test_internal::<$crate::HashedValueNoExt>(); - eprintln!("Running with layout `NoExtensionLayout`"); - $test_internal::<$crate::NoExtensionLayout>(); - eprintln!("Running with layout `ExtensionLayout`"); - $test_internal::<$crate::ExtensionLayout>(); + eprintln!("Running with layout `HashedValueNoExtThreshold` and MemTreeDB"); + $test_internal::< + $crate::HashedValueNoExtThreshold<1, $crate::MemLocation>, + $crate::MemTreeDB<$crate::RefHasher>, + >(); + eprintln!("Running with layout `HashedValueNoExt` and MemTreeDB"); + $test_internal::<$crate::HashedValueNoExt, $crate::MemTreeDB<$crate::RefHasher>>(); + eprintln!("Running with layout `NoExtensionLayout` and MemTreeDB"); + $test_internal::< + $crate::GenericNoExtensionLayout<$crate::RefHasher, $crate::MemLocation>, + $crate::MemTreeDB<$crate::RefHasher>, + >(); + + eprintln!("Running with layout `HashedValueNoExtThreshold` and MemoryDB"); + $test_internal::< + $crate::HashedValueNoExtThreshold<1, ()>, + $crate::PrefixedMemoryDB<$crate::HashedValueNoExtThreshold<1, ()>>, + >(); + eprintln!("Running with layout `HashedValueNoExt` and MemoryDB"); + $test_internal::< + $crate::HashedValueNoExt, + $crate::PrefixedMemoryDB<$crate::HashedValueNoExt>, + >(); + eprintln!("Running with layout `NoExtensionLayout` and MemoryDB"); + $test_internal::< + $crate::NoExtensionLayout, + $crate::PrefixedMemoryDB<$crate::NoExtensionLayout>, + >(); + eprintln!("Running with layout `ExtensionLayout` and MemoryDB"); + $test_internal::< + $crate::ExtensionLayout, + $crate::PrefixedMemoryDB<$crate::ExtensionLayout>, + >(); } }; } @@ -101,32 +132,34 @@ impl TrieLayout for ExtensionLayout { const MAX_INLINE_VALUE: Option = None; type Hash = RefHasher; type Codec = ReferenceNodeCodec; + type Location = (); } impl TrieConfiguration for ExtensionLayout {} /// Trie layout without extension nodes, allowing /// generic hasher. -pub struct GenericNoExtensionLayout(PhantomData); +pub struct GenericNoExtensionLayout(PhantomData<(H, L)>); -impl Default for GenericNoExtensionLayout { +impl Default for GenericNoExtensionLayout { fn default() -> Self { GenericNoExtensionLayout(PhantomData) } } -impl Clone for GenericNoExtensionLayout { +impl Clone for GenericNoExtensionLayout { fn clone(&self) -> Self { GenericNoExtensionLayout(PhantomData) } } -impl TrieLayout for GenericNoExtensionLayout { +impl TrieLayout for GenericNoExtensionLayout { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = false; const MAX_INLINE_VALUE: Option = None; type Hash = H; type Codec = ReferenceNodeCodecNoExt; + type Location = L; } /// Trie that allows empty values. @@ -139,12 +172,13 @@ impl TrieLayout for AllowEmptyLayout { const MAX_INLINE_VALUE: Option = None; type Hash = RefHasher; type Codec = ReferenceNodeCodec; + type Location = (); } -impl TrieConfiguration for GenericNoExtensionLayout {} +impl TrieConfiguration for GenericNoExtensionLayout {} /// Trie layout without extension nodes. -pub type NoExtensionLayout = GenericNoExtensionLayout; +pub type NoExtensionLayout = GenericNoExtensionLayout; /// Children bitmap codec for radix 16 trie. pub struct Bitmap(u16); @@ -184,10 +218,6 @@ pub type RefTrieDBMutAllowEmpty<'a> = trie_db::TrieDBMut<'a, AllowEmptyLayout>; pub type RefTrieDBMutAllowEmptyBuilder<'a> = trie_db::TrieDBMutBuilder<'a, AllowEmptyLayout>; pub type RefTestTrieDBCache = TestTrieCache; pub type RefTestTrieDBCacheNoExt = TestTrieCache; -pub type RefFatDB<'a, 'cache> = trie_db::FatDB<'a, 'cache, ExtensionLayout>; -pub type RefFatDBMut<'a> = trie_db::FatDBMut<'a, ExtensionLayout>; -pub type RefSecTrieDB<'a, 'cache> = trie_db::SecTrieDB<'a, 'cache, ExtensionLayout>; -pub type RefSecTrieDBMut<'a> = trie_db::SecTrieDBMut<'a, ExtensionLayout>; pub type RefLookup<'a, 'cache, Q> = trie_db::Lookup<'a, 'cache, ExtensionLayout, Q>; pub type RefLookupNoExt<'a, 'cache, Q> = trie_db::Lookup<'a, 'cache, NoExtensionLayout, Q>; @@ -624,12 +654,14 @@ impl NodeCodec for ReferenceNodeCodec { None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; + let mut i_hash = 0; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { let count = >::decode(&mut input)?.0 as usize; let range = input.take(count)?; children[i] = Some(if count == H::LENGTH { - NodeHandlePlan::Hash(range) + i_hash += 1; + NodeHandlePlan::Hash(range, i_hash - 1) } else { NodeHandlePlan::Inline(range) }); @@ -646,7 +678,7 @@ impl NodeCodec for ReferenceNodeCodec { let count = >::decode(&mut input)?.0 as usize; let range = input.take(count)?; let child = if count == H::LENGTH { - NodeHandlePlan::Hash(range) + NodeHandlePlan::Hash(range, 0) } else { NodeHandlePlan::Inline(range) }; @@ -679,7 +711,11 @@ impl NodeCodec for ReferenceNodeCodec { &[EMPTY_TRIE] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node( + partial: impl Iterator, + number_nibble: usize, + value: Value, + ) -> Vec { let mut output = partial_from_iterator_to_key(partial, number_nibble, LEAF_NODE_OFFSET, LEAF_NODE_OVER); match value { @@ -692,10 +728,10 @@ impl NodeCodec for ReferenceNodeCodec { output } - fn extension_node( + fn extension_node( partial: impl Iterator, number_nibble: usize, - child: ChildReference, + child: ChildReference, ) -> Vec { let mut output = partial_from_iterator_to_key( partial, @@ -704,16 +740,16 @@ impl NodeCodec for ReferenceNodeCodec { EXTENSION_NODE_OVER, ); match child { - ChildReference::Hash(h) => h.as_ref().encode_to(&mut output), + ChildReference::Hash(h, _) => h.as_ref().encode_to(&mut output), ChildReference::Inline(inline_data, len) => (&AsRef::<[u8]>::as_ref(&inline_data)[..len]).encode_to(&mut output), }; output } - fn branch_node( - children: impl Iterator>>>, - maybe_value: Option, + fn branch_node( + children: impl Iterator>>>, + maybe_value: Option>, ) -> Vec { let mut output = vec![0; BITMAP_LENGTH + 1]; let mut prefix: [u8; 3] = [0; 3]; @@ -727,7 +763,7 @@ impl NodeCodec for ReferenceNodeCodec { _ => unimplemented!("unsupported"), }; let has_children = children.map(|maybe_child| match maybe_child.borrow() { - Some(ChildReference::Hash(h)) => { + Some(ChildReference::Hash(h, _)) => { h.as_ref().encode_to(&mut output); true }, @@ -742,11 +778,11 @@ impl NodeCodec for ReferenceNodeCodec { output } - fn branch_node_nibbled( + fn branch_node_nibbled( _partial: impl Iterator, _number_nibble: usize, - _children: impl Iterator>>>, - _maybe_value: Option, + _children: impl Iterator>>>, + _maybe_value: Option>, ) -> Vec { unreachable!("codec with extension branch") } @@ -791,12 +827,14 @@ impl NodeCodec for ReferenceNodeCodecNoExt { None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; + let mut i_hash = 0; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { let count = >::decode(&mut input)?.0 as usize; let range = input.take(count)?; children[i] = Some(if count == H::LENGTH { - NodeHandlePlan::Hash(range) + i_hash += 1; + NodeHandlePlan::Hash(range, i_hash - 1) } else { NodeHandlePlan::Inline(range) }); @@ -835,7 +873,11 @@ impl NodeCodec for ReferenceNodeCodecNoExt { &[EMPTY_TRIE_NO_EXT] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node( + partial: impl Iterator, + number_nibble: usize, + value: Value, + ) -> Vec { let mut output = partial_from_iterator_encode(partial, number_nibble, NodeKindNoExt::Leaf); match value { Value::Inline(value) => { @@ -847,26 +889,26 @@ impl NodeCodec for ReferenceNodeCodecNoExt { output } - fn extension_node( + fn extension_node( _partial: impl Iterator, _nbnibble: usize, - _child: ChildReference<::Out>, + _child: ChildReference<::Out, L>, ) -> Vec { unreachable!("no extension codec") } - fn branch_node( - _children: impl Iterator::Out>>>>, - _maybe_value: Option, + fn branch_node( + _children: impl Iterator::Out, L>>>>, + _maybe_value: Option>, ) -> Vec { unreachable!("no extension codec") } - fn branch_node_nibbled( + fn branch_node_nibbled( partial: impl Iterator, number_nibble: usize, - children: impl Iterator>>>, - maybe_value: Option, + children: impl Iterator>>>, + maybe_value: Option>, ) -> Vec { let mut output = if maybe_value.is_none() { partial_from_iterator_encode(partial, number_nibble, NodeKindNoExt::BranchNoValue) @@ -887,7 +929,7 @@ impl NodeCodec for ReferenceNodeCodecNoExt { Bitmap::encode( children.map(|maybe_child| match maybe_child.borrow() { - Some(ChildReference::Hash(h)) => { + Some(ChildReference::Hash(h, _)) => { h.as_ref().encode_to(&mut output); true }, @@ -906,33 +948,34 @@ impl NodeCodec for ReferenceNodeCodecNoExt { } /// Compare trie builder and in memory trie. -pub fn compare_implementations(data: Vec<(Vec, Vec)>, mut memdb: DB, mut hashdb: DB) +pub fn compare_implementations(data: Vec<(Vec, Vec)>) where T: TrieLayout, - DB: hash_db::HashDB + Eq, + T::Location: std::fmt::Debug, + K: KeyFunction + Send + Sync, { - let root_new = calc_root_build::(data.clone(), &mut hashdb); + let (mut mem_db1, _) = MemoryDB::::default_with_root(); + let (mut mem_db2, _) = MemoryDB::::default_with_root(); + let root_new = calc_root_build::(data.clone(), &mut mem_db1); let root = { - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut mem_db2).build(); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); } - t.commit(); - *t.root() + t.commit().apply_to(&mut mem_db2) }; if root_new != root { { - let db: &dyn hash_db::HashDB<_, _> = &hashdb; - let t = TrieDBBuilder::::new(&db, &root_new).build(); + let db: &dyn node_db::NodeDB<_, _, _> = &mem_db1; + let t = TrieDBBuilder::::new(db, &root_new).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:x?}", a); } } { - let db: &dyn hash_db::HashDB<_, _> = &memdb; - let t = TrieDBBuilder::::new(&db, &root).build(); + let db: &dyn node_db::NodeDB<_, _, _> = &mem_db2; + let t = TrieDBBuilder::::new(db, &root).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:x?}", a); @@ -942,22 +985,21 @@ where assert_eq!(root, root_new); // compare db content for key fuzzing - assert!(memdb == hashdb); + assert!(mem_db1 == mem_db2); } /// Compare trie builder and trie root implementations. -pub fn compare_root>( +pub fn compare_root>( data: Vec<(Vec, Vec)>, - mut memdb: DB, + memdb: DB, ) { let root_new = reference_trie_root_iter_build::(data.clone()); let root = { - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); } - *t.root() + *t.commit().hash() }; assert_eq!(root, root_new); @@ -1002,57 +1044,60 @@ where } /// Trie builder trie building utility. -pub fn calc_root_build(data: I, hashdb: &mut DB) -> ::Out +pub fn calc_root_build( + data: I, + memdb: &mut MemoryDB, +) -> TrieHash where T: TrieLayout, I: IntoIterator, A: AsRef<[u8]> + Ord + fmt::Debug, B: AsRef<[u8]> + fmt::Debug, - DB: hash_db::HashDB, + K: KeyFunction + Send + Sync, { - let mut cb = TrieBuilder::::new(hashdb); + let mut cb = TrieBuilder::::new(memdb); trie_visit::(data.into_iter(), &mut cb); cb.root.unwrap_or_default() } /// `compare_implementations_no_extension` for unordered input (trie_root does /// ordering before running when trie_build expect correct ordering). -pub fn compare_implementations_unordered( - data: Vec<(Vec, Vec)>, - mut memdb: DB, - mut hashdb: DB, -) where + +pub fn compare_implementations_unordered(data: Vec<(Vec, Vec)>) +where T: TrieLayout, - DB: hash_db::HashDB + Eq, + T::Location: std::fmt::Debug, + K: KeyFunction + Send + Sync, { + let mut mem_db1 = MemoryDB::::default(); + let mut mem_db2 = MemoryDB::::default(); let mut b_map = std::collections::btree_map::BTreeMap::new(); let root = { - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut mem_db1).build(); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); b_map.insert(data[i].0.clone(), data[i].1.clone()); } - *t.root() + *t.commit().hash() }; let root_new = { - let mut cb = TrieBuilder::::new(&mut hashdb); + let mut cb = TrieBuilder::::new(&mut mem_db2); trie_visit::(b_map.into_iter(), &mut cb); cb.root.unwrap_or_default() }; if root != root_new { { - let db: &dyn hash_db::HashDB<_, _> = &memdb; - let t = TrieDBBuilder::::new(&db, &root).build(); + let db: &dyn node_db::NodeDB<_, _, _> = &mem_db1; + let t = TrieDBBuilder::::new(db, &root).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:?}", a); } } { - let db: &dyn hash_db::HashDB<_, _> = &hashdb; - let t = TrieDBBuilder::::new(&db, &root_new).build(); + let db: &dyn node_db::NodeDB<_, _, _> = &mem_db2; + let t = TrieDBBuilder::::new(db, &root_new).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:?}", a); @@ -1065,24 +1110,22 @@ pub fn compare_implementations_unordered( /// Testing utility that uses some periodic removal over /// its input test data. -pub fn compare_insert_remove>( - data: Vec<(bool, Vec, Vec)>, - mut memdb: DB, -) where +pub fn compare_insert_remove(data: Vec<(bool, Vec, Vec)>) +where T: TrieLayout, - DB: hash_db::HashDB + Eq, + K: KeyFunction + Send + Sync, { let mut data2 = std::collections::BTreeMap::new(); - let mut root = Default::default(); let mut a = 0; - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.commit(); - } + let mut memdb = MemoryDB::::default(); + let mut root = { + let t = TrieDBMutBuilder::::new(&memdb).build(); + *t.commit().hash() + }; while a < data.len() { // new triemut every 3 element root = { - let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::from_existing(&memdb, root).build(); for _ in 0..3 { if data[a].0 { // remove @@ -1099,14 +1142,12 @@ pub fn compare_insert_remove>( break } } - t.commit(); - *t.root() + t.commit().apply_to(&mut memdb) }; } - let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); // we are testing the RefTrie code here so we do not sort or check uniqueness // before. - assert_eq!(*t.root(), calc_root::(data2)); + assert_eq!(root, calc_root::(data2)); } /// Example trie cache implementation. @@ -1114,8 +1155,8 @@ pub fn compare_insert_remove>( /// Should not be used for anything in production. pub struct TestTrieCache { /// In a real implementation we need to make sure that this is unique per trie root. - value_cache: HashMap, trie_db::CachedValue>>, - node_cache: HashMap, NodeOwned>>, + value_cache: HashMap, trie_db::CachedValue, L::Location>>, + node_cache: HashMap, NodeOwned, L::Location>>, } impl TestTrieCache { @@ -1136,24 +1177,32 @@ impl Default for TestTrieCache { } } -impl trie_db::TrieCache for TestTrieCache { - fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&trie_db::CachedValue>> { +impl trie_db::TrieCache for TestTrieCache { + fn lookup_value_for_key( + &mut self, + key: &[u8], + ) -> Option<&trie_db::CachedValue, L::Location>> { self.value_cache.get(key) } - fn cache_value_for_key(&mut self, key: &[u8], value: trie_db::CachedValue>) { + fn cache_value_for_key( + &mut self, + key: &[u8], + value: trie_db::CachedValue, L::Location>, + ) { self.value_cache.insert(key.to_vec(), value); } fn get_or_insert_node( &mut self, hash: TrieHash, + _location: L::Location, fetch_node: &mut dyn FnMut() -> trie_db::Result< - NodeOwned>, + NodeOwned, L::Location>, TrieHash, trie_db::CError, >, - ) -> trie_db::Result<&NodeOwned>, TrieHash, trie_db::CError> { + ) -> trie_db::Result<&NodeOwned, L::Location>, TrieHash, trie_db::CError> { match self.node_cache.entry(hash) { Entry::Occupied(e) => Ok(e.into_mut()), Entry::Vacant(e) => { @@ -1163,9 +1212,15 @@ impl trie_db::TrieCache for TestTrieCache { } } - fn get_node(&mut self, hash: &TrieHash) -> Option<&NodeOwned>> { + fn get_node( + &mut self, + hash: &TrieHash, + _location: L::Location, + ) -> Option<&NodeOwned, L::Location>> { self.node_cache.get(hash) } + + fn insert_new_node(&mut self, _hash: &TrieHash) {} } #[cfg(test)] @@ -1203,9 +1258,10 @@ mod tests { let enc = as NodeCodec>::leaf_node( input.iter().cloned(), input.len() * NIBBLE_PER_BYTE, - Value::Inline(&[1]), + Value::<()>::Inline(&[1]), ); - let dec = as NodeCodec>::decode(&enc).unwrap(); + let dec = + as NodeCodec>::decode(&enc, &[] as &[()]).unwrap(); let o_sl = if let Node::Leaf(sl, _) = dec { Some(sl) } else { None }; assert!(o_sl.is_some()); } diff --git a/test-support/reference-trie/src/substrate.rs b/reference-trie/src/substrate.rs similarity index 96% rename from test-support/reference-trie/src/substrate.rs rename to reference-trie/src/substrate.rs index 9b1573f1..a841a603 100644 --- a/test-support/reference-trie/src/substrate.rs +++ b/reference-trie/src/substrate.rs @@ -17,12 +17,12 @@ //! Codec and layout directly copy-pasted from substrate with minimal modifications. use core::{borrow::Borrow, iter::once, marker::PhantomData, ops::Range}; -use hash_db::Hasher; use parity_scale_codec as codec; use parity_scale_codec::{Compact, Decode, Encode, Input, Output}; use trie_db::{ nibble_ops, node::{NibbleSlicePlan, NodeHandlePlan, NodePlan, Value, ValuePlan}, + node_db::Hasher, ChildReference, NodeCodec as NodeCodecT, TrieConfiguration, TrieLayout, }; @@ -78,8 +78,8 @@ fn fuse_nibbles_node(nibbles: &[u8], kind: NodeKind) -> impl Iterator .chain(nibbles[nibbles.len() % 2..].chunks(2).map(|ch| ch[0] << 4 | ch[1])) } -use trie_root::Value as TrieStreamValue; -impl trie_root::TrieStream for TrieStream { +use trie_db::trie_root::{self, Value as TrieStreamValue}; +impl trie_db::trie_root::TrieStream for TrieStream { fn new() -> Self { Self { buffer: Vec::new() } } @@ -259,12 +259,14 @@ where None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; + let mut i_hash = 0; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { let count = >::decode(&mut input)?.0 as usize; let range = input.take(count)?; children[i] = Some(if count == H::LENGTH { - NodeHandlePlan::Hash(range) + i_hash += 1; + NodeHandlePlan::Hash(range, i_hash - 1) } else { NodeHandlePlan::Inline(range) }); @@ -310,7 +312,11 @@ where &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node( + partial: impl Iterator, + number_nibble: usize, + value: Value, + ) -> Vec { let contains_hash = matches!(&value, Value::Node(..)); let mut output = if contains_hash { partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueLeaf) @@ -322,7 +328,7 @@ where Compact(value.len() as u32).encode_to(&mut output); output.extend_from_slice(value); }, - Value::Node(hash) => { + Value::Node(hash, _) => { debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, @@ -330,26 +336,26 @@ where output } - fn extension_node( + fn extension_node( _partial: impl Iterator, _nbnibble: usize, - _child: ChildReference<::Out>, + _child: ChildReference<::Out, L>, ) -> Vec { unreachable!("No extension codec.") } - fn branch_node( - _children: impl Iterator::Out>>>>, - _maybe_value: Option, + fn branch_node( + _children: impl Iterator::Out, L>>>>, + _maybe_value: Option>, ) -> Vec { unreachable!("No extension codec.") } - fn branch_node_nibbled( + fn branch_node_nibbled( partial: impl Iterator, number_nibble: usize, - children: impl Iterator::Out>>>>, - value: Option, + children: impl Iterator::Out, L>>>>, + value: Option>, ) -> Vec { let contains_hash = matches!(&value, Some(Value::Node(..))); let mut output = match (&value, contains_hash) { @@ -369,7 +375,7 @@ where Compact(value.len() as u32).encode_to(&mut output); output.extend_from_slice(value); }, - Some(Value::Node(hash)) => { + Some(Value::Node(hash, _)) => { debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, @@ -377,7 +383,7 @@ where } Bitmap::encode( children.map(|maybe_child| match maybe_child.borrow() { - Some(ChildReference::Hash(h)) => { + Some(ChildReference::Hash(h, _)) => { h.as_ref().encode_to(&mut output); true }, @@ -470,6 +476,7 @@ where type Hash = H; type Codec = NodeCodec; + type Location = (); } impl TrieConfiguration for LayoutV0 @@ -512,6 +519,7 @@ where type Hash = H; type Codec = NodeCodec; + type Location = (); } impl TrieConfiguration for LayoutV1 diff --git a/test-support/reference-trie/src/substrate_like.rs b/reference-trie/src/substrate_like.rs similarity index 94% rename from test-support/reference-trie/src/substrate_like.rs rename to reference-trie/src/substrate_like.rs index 37cccefc..f1493799 100644 --- a/test-support/reference-trie/src/substrate_like.rs +++ b/reference-trie/src/substrate_like.rs @@ -22,7 +22,7 @@ pub struct HashedValueNoExt; /// No extension trie which stores value above a static size /// as external node. -pub struct HashedValueNoExtThreshold; +pub struct HashedValueNoExtThreshold(PhantomData); impl TrieLayout for HashedValueNoExt { const USE_EXTENSION: bool = false; @@ -31,15 +31,17 @@ impl TrieLayout for HashedValueNoExt { type Hash = RefHasher; type Codec = ReferenceNodeCodecNoExtMeta; + type Location = trie_db::mem_tree_db::Location; } -impl TrieLayout for HashedValueNoExtThreshold { +impl TrieLayout for HashedValueNoExtThreshold { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = false; const MAX_INLINE_VALUE: Option = Some(C); type Hash = RefHasher; type Codec = ReferenceNodeCodecNoExtMeta; + type Location = L; } /// Constants specific to encoding with external value node support. @@ -101,12 +103,14 @@ impl NodeCodec { None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]; + let mut i_hash = 0; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { let count = >::decode(&mut input)?.0 as usize; let range = input.take(count)?; children[i] = Some(if count == H::LENGTH { - NodeHandlePlan::Hash(range) + i_hash += 1; + NodeHandlePlan::Hash(range, i_hash - 1) } else { NodeHandlePlan::Inline(range) }); @@ -169,7 +173,11 @@ where &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + fn leaf_node( + partial: impl Iterator, + number_nibble: usize, + value: Value, + ) -> Vec { let contains_hash = matches!(&value, Value::Node(..)); let mut output = if contains_hash { partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueLeaf) @@ -181,7 +189,7 @@ where Compact(value.len() as u32).encode_to(&mut output); output.extend_from_slice(value); }, - Value::Node(hash) => { + Value::Node(hash, _) => { debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, @@ -189,26 +197,26 @@ where output } - fn extension_node( + fn extension_node( _partial: impl Iterator, _nbnibble: usize, - _child: ChildReference<::Out>, + _child: ChildReference<::Out, L>, ) -> Vec { unreachable!("Codec without extension.") } - fn branch_node( - _children: impl Iterator::Out>>>>, - _maybe_value: Option, + fn branch_node( + _children: impl Iterator::Out, L>>>>, + _maybe_value: Option>, ) -> Vec { unreachable!("Codec without extension.") } - fn branch_node_nibbled( + fn branch_node_nibbled( partial: impl Iterator, number_nibble: usize, - children: impl Iterator::Out>>>>, - value: Option, + children: impl Iterator::Out, L>>>>, + value: Option>, ) -> Vec { let contains_hash = matches!(&value, Some(Value::Node(..))); let mut output = match (&value, contains_hash) { @@ -228,7 +236,7 @@ where Compact(value.len() as u32).encode_to(&mut output); output.extend_from_slice(value); }, - Some(Value::Node(hash)) => { + Some(Value::Node(hash, _)) => { debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, @@ -236,7 +244,7 @@ where } Bitmap::encode( children.map(|maybe_child| match maybe_child.borrow() { - Some(ChildReference::Hash(h)) => { + Some(ChildReference::Hash(h, _)) => { h.as_ref().encode_to(&mut output); true }, @@ -459,8 +467,8 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator Self { Self { buffer: Vec::new() } } diff --git a/trie-db/CHANGELOG.md b/subtrie/CHANGELOG.md similarity index 90% rename from trie-db/CHANGELOG.md rename to subtrie/CHANGELOG.md index ac450d40..08d51546 100644 --- a/trie-db/CHANGELOG.md +++ b/subtrie/CHANGELOG.md @@ -4,6 +4,11 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ +## [0.28.0] - 2023-09-12 +- Make `trie_nodes_recorded_for_key` work for inline values [#194](https://github.com/paritytech/trie/pull/194) +- trie-db: Fetch the closest merkle value [#199](https://github.com/paritytech/trie/pull/199) +- fixing triedbmut lookup, added some testing in test. [#198](https://github.com/paritytech/trie/pull/198) + ## [0.27.1] - 2023-03-17 - Fix `TrieDBRawIterator::prefix_then_seek` [#190](https://github.com/paritytech/trie/pull/190) diff --git a/subtrie/Cargo.toml b/subtrie/Cargo.toml new file mode 100644 index 00000000..a0c607ad --- /dev/null +++ b/subtrie/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "subtrie" +version = "0.0.1" +authors = ["Parity Technologies "] +description = "Merkle-Patricia Trie generic over key hasher and node encoding" +repository = "https://github.com/paritytech/trie" +license = "Apache-2.0" +edition = "2018" + +[dependencies] +hash256-std-hasher = { path = "../hash256-std-hasher", version = "0.15.2", optional = true } +log = "0.4" +smallvec = { version = "1.0.0", features = ["union", "const_new"] } +rustc-hex = { version = "2.1.0", default-features = false, optional = true } +tiny-keccak = { version = "2.0.2", features = ["keccak"], optional = true } +parity-scale-codec = { version = "3.0.0", optional = true } +criterion = { version = "0.5.1", optional = true } + +[dev-dependencies] +criterion = "0.5.1" +hash256-std-hasher = { path = "../hash256-std-hasher", version = "0.15.2" } +tiny-keccak = { version = "2.0.2", features = ["keccak"] } + +[features] +default = ["std"] +bench = [ + "std", + "criterion", + "test_utils", +] +test_utils = [ + "std", + "hash256-std-hasher", + "tiny-keccak", + "parity-scale-codec", +] +std = [ + "rustc-hex", +] diff --git a/trie-db/fuzz/Cargo.toml b/subtrie/fuzz/Cargo.toml similarity index 89% rename from trie-db/fuzz/Cargo.toml rename to subtrie/fuzz/Cargo.toml index 84a03257..24e4a837 100644 --- a/trie-db/fuzz/Cargo.toml +++ b/subtrie/fuzz/Cargo.toml @@ -9,13 +9,11 @@ edition = "2018" cargo-fuzz = true [dependencies] -hash-db = { path = "../../hash-db", version = "0.16.0" } -memory-db = { path = "../../memory-db", version = "0.32.0" } reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } arbitrary = { version = "1.3.0", features = ["derive"] } array-bytes = "6.0.0" -[dependencies.trie-db] +[dependencies.subtrie] path = ".." [dependencies.libfuzzer-sys] diff --git a/trie-db/fuzz/fuzz_targets/no_ext_insert.rs b/subtrie/fuzz/fuzz_targets/no_ext_insert.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/no_ext_insert.rs rename to subtrie/fuzz/fuzz_targets/no_ext_insert.rs diff --git a/trie-db/fuzz/fuzz_targets/no_ext_insert_rem.rs b/subtrie/fuzz/fuzz_targets/no_ext_insert_rem.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/no_ext_insert_rem.rs rename to subtrie/fuzz/fuzz_targets/no_ext_insert_rem.rs diff --git a/trie-db/fuzz/fuzz_targets/prefix_iter.rs b/subtrie/fuzz/fuzz_targets/prefix_iter.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/prefix_iter.rs rename to subtrie/fuzz/fuzz_targets/prefix_iter.rs diff --git a/trie-db/fuzz/fuzz_targets/prefix_seek_iter.rs b/subtrie/fuzz/fuzz_targets/prefix_seek_iter.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/prefix_seek_iter.rs rename to subtrie/fuzz/fuzz_targets/prefix_seek_iter.rs diff --git a/trie-db/fuzz/fuzz_targets/seek_iter.rs b/subtrie/fuzz/fuzz_targets/seek_iter.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/seek_iter.rs rename to subtrie/fuzz/fuzz_targets/seek_iter.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_codec_proof.rs b/subtrie/fuzz/fuzz_targets/trie_codec_proof.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_codec_proof.rs rename to subtrie/fuzz/fuzz_targets/trie_codec_proof.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs b/subtrie/fuzz/fuzz_targets/trie_proof_invalid.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_proof_invalid.rs rename to subtrie/fuzz/fuzz_targets/trie_proof_invalid.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_proof_valid.rs b/subtrie/fuzz/fuzz_targets/trie_proof_valid.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_proof_valid.rs rename to subtrie/fuzz/fuzz_targets/trie_proof_valid.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_root.rs b/subtrie/fuzz/fuzz_targets/trie_root.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_root.rs rename to subtrie/fuzz/fuzz_targets/trie_root.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs b/subtrie/fuzz/fuzz_targets/trie_root_fix_len.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_root_fix_len.rs rename to subtrie/fuzz/fuzz_targets/trie_root_fix_len.rs diff --git a/trie-db/fuzz/fuzz_targets/trie_root_new.rs b/subtrie/fuzz/fuzz_targets/trie_root_new.rs similarity index 100% rename from trie-db/fuzz/fuzz_targets/trie_root_new.rs rename to subtrie/fuzz/fuzz_targets/trie_root_new.rs diff --git a/trie-db/fuzz/src/lib.rs b/subtrie/fuzz/src/lib.rs similarity index 83% rename from trie-db/fuzz/src/lib.rs rename to subtrie/fuzz/src/lib.rs index f9de8c07..537a75c7 100644 --- a/trie-db/fuzz/src/lib.rs +++ b/subtrie/fuzz/src/lib.rs @@ -13,15 +13,15 @@ // limitations under the License. use arbitrary::Arbitrary; -use hash_db::Hasher; -use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ calc_root, compare_insert_remove, reference_trie_root_iter_build as reference_trie_root, }; -use std::convert::TryInto; -use trie_db::{ +use std::{convert::TryInto, fmt::Debug}; +use subtrie::{ + memory_db::{HashKey, MemoryDB, PrefixedKey}, + node_db::Hasher, proof::{generate_proof, verify_proof}, - DBValue, Trie, TrieDBBuilder, TrieDBIterator, TrieDBMutBuilder, TrieLayout, TrieMut, + DBValue, Trie, TrieDBBuilder, TrieDBIterator, TrieDBMutBuilder, TrieLayout, }; fn fuzz_to_data(input: &[u8]) -> Vec<(Vec, Vec)> { @@ -91,24 +91,22 @@ fn fuzz_removal(data: Vec<(Vec, Vec)>) -> Vec<(bool, Vec, Vec)> pub fn fuzz_that_reference_trie_root(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data(input)); - let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let memdb = MemoryDB::<_, HashKey<_>, _>::default(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } - assert_eq!(*t.root(), reference_trie_root::(data)); + assert_eq!(t.commit().root_hash(), reference_trie_root::(data)); } pub fn fuzz_that_reference_trie_root_fix_length(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data_fix_length(input)); - let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let memdb = MemoryDB::<_, HashKey<_>, _>::default(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } - assert_eq!(*t.root(), reference_trie_root::(data)); + assert_eq!(t.commit().root_hash(), reference_trie_root::(data)); } fn fuzz_to_data_fix_length(input: &[u8]) -> Vec<(Vec, Vec)> { @@ -132,20 +130,18 @@ fn data_sorted_unique(input: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec)> m.into_iter().collect() } -pub fn fuzz_that_compare_implementations(input: &[u8]) { +pub fn fuzz_that_compare_implementations(input: &[u8]) + where T::Location: Debug, +{ let data = data_sorted_unique(fuzz_to_data(input)); - //println!("data:{:?}", &data); - let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations::(data, memdb, hashdb); + reference_trie::compare_implementations::>(data); } pub fn fuzz_that_no_extension_insert(input: &[u8]) { let data = fuzz_to_data(input); //println!("data{:?}", data); - let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let memdb = MemoryDB::<_, HashKey<_>, _>::default(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); for a in 0..data.len() { t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } @@ -153,28 +149,25 @@ pub fn fuzz_that_no_extension_insert(input: &[u8]) { // before. let data = data_sorted_unique(fuzz_to_data(input)); //println!("data{:?}", data); - assert_eq!(*t.root(), calc_root::(data)); + assert_eq!(t.commit().root_hash(), calc_root::(data)); } pub fn fuzz_that_no_extension_insert_remove(input: &[u8]) { let data = fuzz_to_data(input); let data = fuzz_removal(data); - let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - compare_insert_remove::(data, memdb); + compare_insert_remove::>(data); } pub fn fuzz_seek_iter(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data_fix_length(input)); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for a in 0..data.len() { - t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for a in 0..data.len() { + t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } + let root = t.commit().apply_to(&mut memdb); // fuzzing around a fix prefix of 6 nibble. let prefix = &b"012"[..]; @@ -217,13 +210,11 @@ pub fn fuzz_prefix_iter(input: &[u8]) { let data = data_sorted_unique(fuzz_to_data_fix_length(input)); let mut memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for a in 0..data.len() { - t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for a in 0..data.len() { + t.insert(&data[a].0[..], &data[a].1[..]).unwrap(); } + let root = t.commit().apply_to(&mut memdb); // fuzzing around a fix prefix of 6 nibble. let prefix = &b"012"[..]; @@ -283,17 +274,15 @@ pub fn fuzz_prefix_seek_iter(mut input: PrefixSeekTestInput) { input.keys.dedup(); let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (index, key) in input.keys.iter().enumerate() { - t.insert(&key, &[index as u8]).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (index, key) in input.keys.iter().enumerate() { + t.insert(&key, &[index as u8]).unwrap(); } + let root = t.commit().apply_to(&mut memdb); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let iter = - trie_db::TrieDBIterator::new_prefixed_then_seek(&trie, &input.prefix_key, &input.seek_key) + subtrie::TrieDBIterator::new_prefixed_then_seek(&trie, &input.prefix_key, &input.seek_key) .unwrap(); let output_keys: Vec<_> = iter.map(|item| item.unwrap().0).collect(); @@ -393,13 +382,11 @@ fn test_generate_proof( // Populate DB with full trie from entries. let (db, root) = { let mut db = , _>>::default(); - let mut root = Default::default(); - { - let mut trie = TrieDBMutBuilder::::new(&mut db, &mut root).build(); - for (key, value) in entries { - trie.insert(&key, &value).unwrap(); - } + let mut trie = TrieDBMutBuilder::::new(&mut db).build(); + for (key, value) in entries { + trie.insert(&key, &value).unwrap(); } + let root = trie.commit().apply_to(&mut db); (db, root) }; @@ -418,19 +405,16 @@ fn test_generate_proof( } fn test_trie_codec_proof(entries: Vec<(Vec, Vec)>, keys: Vec>) { - use hash_db::{HashDB, EMPTY_PREFIX}; - use trie_db::{decode_compact, encode_compact, Recorder}; + use subtrie::{node_db::EMPTY_PREFIX, decode_compact, encode_compact, Recorder}; // Populate DB with full trie from entries. let (db, root) = { let mut db = , _>>::default(); - let mut root = Default::default(); - { - let mut trie = TrieDBMutBuilder::::new(&mut db, &mut root).build(); - for (key, value) in entries { - trie.insert(&key, &value).unwrap(); - } + let mut trie = TrieDBMutBuilder::::new(&db).build(); + for (key, value) in entries { + trie.insert(&key, &value).unwrap(); } + let root = trie.commit().apply_to(&mut db); (db, root) }; let expected_root = root; @@ -461,7 +445,7 @@ fn test_trie_codec_proof(entries: Vec<(Vec, Vec)>, keys: let expected_used = compact_trie.len(); // Reconstruct the partial DB from the compact encoding. let mut db = , _>>::default(); - let (root, used) = decode_compact::(&mut db, &compact_trie).unwrap(); + let (root, used) = decode_compact::(&mut db, &compact_trie).unwrap(); assert_eq!(root, expected_root); assert_eq!(used, expected_used); diff --git a/test-support/trie-bench/src/lib.rs b/subtrie/src/bench.rs similarity index 99% rename from test-support/trie-bench/src/lib.rs rename to subtrie/src/bench.rs index afa45849..870b945e 100644 --- a/test-support/trie-bench/src/lib.rs +++ b/subtrie/src/bench.rs @@ -14,15 +14,17 @@ //! Standard trie benchmarking tool. +use crate::{ + keccak_hasher::KeccakHasher, + memory_db::{HashKey, MemoryDB}, + node_db::Hasher, + test_utils::*, + trie_root::{trie_root, TrieStream}, + NodeCodec, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, +}; use criterion::{black_box, BenchmarkId, Criterion}; -use hash_db::Hasher; -use keccak_hasher::KeccakHasher; -use memory_db::{HashKey, MemoryDB}; use parity_scale_codec::{Compact, Encode}; use std::default::Default; -use trie_db::{NodeCodec, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieHash, TrieLayout, TrieMut}; -use trie_root::{trie_root, TrieStream}; -use trie_standardmap::*; struct TrieInsertionList(Vec<(Vec, Vec)>); impl ::std::fmt::Display for TrieInsertionList { @@ -60,8 +62,7 @@ fn benchmark( |b, d: &TrieInsertionList| { b.iter(&mut || { let mut memdb = MemoryDB::<_, HashKey, _>::new(L::Codec::empty_node()); - let mut root = >::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for i in d.0.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -73,13 +74,15 @@ fn benchmark( bench_list, |b, d: &TrieInsertionList| { let mut memdb = MemoryDB::<_, HashKey<_>, _>::new(L::Codec::empty_node()); - let mut root = >::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + let commit = { + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); for i in d.0.iter() { t.insert(&i.0, &i.1).unwrap(); } - } + t.commit() + }; + let root = commit.hash(); + commit.apply_to(&mut memdb); b.iter(&mut || { let t = TrieDBBuilder::::new(&memdb, &root).build(); for n in t.iter().unwrap() { diff --git a/trie-db/src/iter_build.rs b/subtrie/src/iter_build.rs similarity index 91% rename from trie-db/src/iter_build.rs rename to subtrie/src/iter_build.rs index c843c3f1..25463ddc 100644 --- a/trie-db/src/iter_build.rs +++ b/subtrie/src/iter_build.rs @@ -18,14 +18,15 @@ //! See `trie_visit` function. use crate::{ + memory_db::{KeyFunction, MemoryDB}, nibble::{nibble_ops, NibbleSlice}, node::Value, node_codec::NodeCodec, + node_db::{Hasher, Prefix, EMPTY_PREFIX}, rstd::{cmp::max, marker::PhantomData, vec::Vec}, triedbmut::ChildReference, DBValue, TrieHash, TrieLayout, }; -use hash_db::{HashDB, Hasher, Prefix}; macro_rules! exponential_out { (@3, [$($inpp:expr),*]) => { exponential_out!(@2, [$($inpp,)* $($inpp),*]) }; @@ -33,7 +34,7 @@ macro_rules! exponential_out { (@1, [$($inpp:expr),*]) => { [$($inpp,)* $($inpp),*] }; } -type CacheNode = Option>; +type CacheNode = Option>; #[inline(always)] fn new_vec_slice_buffer() -> [CacheNode; 16] { @@ -134,7 +135,7 @@ where value } else { hashed = callback.process_inner_hashed_value((k2.as_ref(), None), v2.as_ref()); - Value::Node(hashed.as_ref()) + Value::Node(hashed.as_ref(), ()) }; let encoded = T::Codec::leaf_node(nkey.right_iter(), nkey.len(), value); let hash = callback.process(pr.left(), encoded, false); @@ -184,7 +185,7 @@ where branch_d: usize, is_root: bool, nkey: Option<(usize, usize)>, - ) -> ChildReference> { + ) -> ChildReference, ()> { let last = self.0.len() - 1; assert_eq!(self.0[last].2, branch_d); @@ -201,7 +202,7 @@ where let mut prefix = NibbleSlice::new_offset(&key_branch, 0); prefix.advance(branch_d); hashed = callback.process_inner_hashed_value(prefix.left(), v.as_ref()); - Value::Node(hashed.as_ref()) + Value::Node(hashed.as_ref(), ()) }) } else { None @@ -229,7 +230,7 @@ where branch_d: usize, is_root: bool, nkey: Option<(usize, usize)>, - ) -> ChildReference> { + ) -> ChildReference, ()> { let (children, v, depth) = self.0.pop().expect("checked"); debug_assert!(branch_d == depth); @@ -244,7 +245,7 @@ where let mut prefix = NibbleSlice::new_offset(&key_branch, 0); prefix.advance(branch_d); hashed = callback.process_inner_hashed_value(prefix.left(), v.as_ref()); - Value::Node(hashed.as_ref()) + Value::Node(hashed.as_ref(), ()) }) } else { None @@ -318,7 +319,7 @@ where value } else { hashed = callback.process_inner_hashed_value((k2.as_ref(), None), v2.as_ref()); - Value::Node(hashed.as_ref()) + Value::Node(hashed.as_ref(), ()) }; let encoded = T::Codec::leaf_node(nkey.right_iter(), nkey.len(), value); @@ -330,7 +331,7 @@ where } } else { // nothing null root corner case - callback.process(hash_db::EMPTY_PREFIX, T::Codec::empty_node().to_vec(), true); + callback.process(EMPTY_PREFIX, T::Codec::empty_node().to_vec(), true); } } @@ -342,43 +343,43 @@ pub trait ProcessEncodedNode { /// Note that the returned value can change depending on implementation, /// but usually it should be the Hash of encoded node. /// This is not something direcly related to encoding but is here for - /// optimisation purpose (builder hash_db does return this value). + /// optimisation purpose (builder node_db does return this value). fn process( &mut self, prefix: Prefix, encoded_node: Vec, is_root: bool, - ) -> ChildReference; + ) -> ChildReference; /// Callback for hashed value in encoded node. fn process_inner_hashed_value(&mut self, prefix: Prefix, value: &[u8]) -> HO; } -/// Get trie root and insert visited node in a hash_db. +/// Get trie root and insert visited node in a node_db. /// As for all `ProcessEncodedNode` implementation, it /// is only for full trie parsing (not existing trie). -pub struct TrieBuilder<'a, T: TrieLayout, DB> { - db: &'a mut DB, +pub struct TrieBuilder<'a, T: TrieLayout, K: KeyFunction + Send + Sync> { + db: &'a mut MemoryDB, pub root: Option>, } -impl<'a, T: TrieLayout, DB> TrieBuilder<'a, T, DB> { - pub fn new(db: &'a mut DB) -> Self { +impl<'a, T: TrieLayout, K: KeyFunction + Send + Sync> TrieBuilder<'a, T, K> { + pub fn new(db: &'a mut MemoryDB) -> Self { TrieBuilder { db, root: None } } } -impl<'a, T, DB> ProcessEncodedNode> for TrieBuilder<'a, T, DB> +impl<'a, T, K: KeyFunction + Send + Sync> ProcessEncodedNode> + for TrieBuilder<'a, T, K> where T: TrieLayout, - DB: HashDB, { fn process( &mut self, prefix: Prefix, encoded_node: Vec, is_root: bool, - ) -> ChildReference> { + ) -> ChildReference, ()> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { let mut h = <::Out as Default>::default(); @@ -390,7 +391,7 @@ where if is_root { self.root = Some(hash); }; - ChildReference::Hash(hash) + ChildReference::Hash(hash, ()) } fn process_inner_hashed_value(&mut self, prefix: Prefix, value: &[u8]) -> TrieHash { @@ -416,7 +417,7 @@ impl ProcessEncodedNode> for TrieRoot { _: Prefix, encoded_node: Vec, is_root: bool, - ) -> ChildReference> { + ) -> ChildReference, ()> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { let mut h = <::Out as Default>::default(); @@ -428,7 +429,7 @@ impl ProcessEncodedNode> for TrieRoot { if is_root { self.root = Some(hash); }; - ChildReference::Hash(hash) + ChildReference::Hash(hash, ()) } fn process_inner_hashed_value(&mut self, _prefix: Prefix, value: &[u8]) -> TrieHash { @@ -472,7 +473,7 @@ impl ProcessEncodedNode> for TrieRootPrint { p: Prefix, encoded_node: Vec, is_root: bool, - ) -> ChildReference> { + ) -> ChildReference, ()> { println!("Encoded node: {:x?}", &encoded_node); println!(" with prefix: {:x?}", &p); let len = encoded_node.len(); @@ -488,7 +489,7 @@ impl ProcessEncodedNode> for TrieRootPrint { self.root = Some(hash); }; println!(" hashed to {:x?}", hash.as_ref()); - ChildReference::Hash(hash) + ChildReference::Hash(hash, ()) } fn process_inner_hashed_value(&mut self, _prefix: Prefix, value: &[u8]) -> TrieHash { @@ -503,7 +504,7 @@ impl ProcessEncodedNode> for TrieRootUnhashed { _: Prefix, encoded_node: Vec, is_root: bool, - ) -> ChildReference<::Out> { + ) -> ChildReference<::Out, ()> { let len = encoded_node.len(); if !is_root && len < ::LENGTH { let mut h = <::Out as Default>::default(); @@ -516,7 +517,7 @@ impl ProcessEncodedNode> for TrieRootUnhashed { if is_root { self.root = Some(encoded_node); }; - ChildReference::Hash(hash) + ChildReference::Hash(hash, ()) } fn process_inner_hashed_value(&mut self, _prefix: Prefix, value: &[u8]) -> TrieHash { diff --git a/trie-db/src/iterator.rs b/subtrie/src/iterator.rs similarity index 51% rename from trie-db/src/iterator.rs rename to subtrie/src/iterator.rs index ca382b70..4a18b322 100644 --- a/trie-db/src/iterator.rs +++ b/subtrie/src/iterator.rs @@ -16,10 +16,10 @@ use super::{CError, DBValue, Result, Trie, TrieHash, TrieIterator, TrieLayout}; use crate::{ nibble::{nibble_ops, NibbleSlice, NibbleVec}, node::{Node, NodeHandle, NodePlan, OwnedNode, Value}, + node_db::{self, Prefix, EMPTY_PREFIX}, triedb::TrieDB, - TrieError, TrieItem, TrieKeyItem, + TrieDoubleEndedIterator, TrieError, TrieItem, TrieKeyItem, }; -use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; use crate::rstd::{boxed::Box, sync::Arc, vec::Vec}; @@ -30,29 +30,40 @@ enum Status { At, AtChild(usize), Exiting, + AftExiting, } #[cfg_attr(feature = "std", derive(Debug))] #[derive(Eq, PartialEq)] -struct Crumb { - hash: Option, - node: Arc>, +struct Crumb { + hash: Option>, + node: Arc>, status: Status, } -impl Crumb { - /// Move on to next status in the node's sequence. - fn increment(&mut self) { +impl Crumb { + /// Move on to the next status in the node's sequence in a direction. + fn step(&mut self, fwd: bool) { self.status = match (self.status, self.node.node_plan()) { (Status::Entering, NodePlan::Extension { .. }) => Status::At, (Status::Entering, NodePlan::Branch { .. }) | (Status::Entering, NodePlan::NibbledBranch { .. }) => Status::At, (Status::At, NodePlan::Branch { .. }) | - (Status::At, NodePlan::NibbledBranch { .. }) => Status::AtChild(0), + (Status::At, NodePlan::NibbledBranch { .. }) => + if fwd { + Status::AtChild(0) + } else { + Status::AtChild(nibble_ops::NIBBLE_LENGTH - 1) + }, (Status::AtChild(x), NodePlan::Branch { .. }) | (Status::AtChild(x), NodePlan::NibbledBranch { .. }) - if x < (nibble_ops::NIBBLE_LENGTH - 1) => + if fwd && x < (nibble_ops::NIBBLE_LENGTH - 1) => Status::AtChild(x + 1), + (Status::AtChild(x), NodePlan::Branch { .. }) | + (Status::AtChild(x), NodePlan::NibbledBranch { .. }) + if !fwd && x > 0 => + Status::AtChild(x - 1), + (Status::Exiting, _) => Status::AftExiting, _ => Status::Exiting, } } @@ -60,7 +71,9 @@ impl Crumb { /// Iterator for going through all nodes in the trie in pre-order traversal order. pub struct TrieDBRawIterator { - trail: Vec>, + /// Forward trail of nodes to visit. + trail: Vec>, + /// Forward iteration key nibbles of the current node. key_nibbles: NibbleVec, } @@ -76,10 +89,11 @@ impl TrieDBRawIterator { TrieDBRawIterator { trail: Vec::with_capacity(8), key_nibbles: NibbleVec::new() }; let (root_node, root_hash) = db.get_raw_or_lookup( *db.root(), - NodeHandle::Hash(db.root().as_ref()), + NodeHandle::Hash(db.root().as_ref(), db.root_location()), EMPTY_PREFIX, true, )?; + r.descend(root_node, root_hash); Ok(r) } @@ -87,7 +101,7 @@ impl TrieDBRawIterator { /// Create a new iterator, but limited to a given prefix. pub fn new_prefixed(db: &TrieDB, prefix: &[u8]) -> Result, CError> { let mut iter = TrieDBRawIterator::new(db)?; - iter.prefix(db, prefix)?; + iter.prefix(db, prefix, true)?; Ok(iter) } @@ -105,8 +119,8 @@ impl TrieDBRawIterator { Ok(iter) } - /// Descend into a payload. - fn descend(&mut self, node: OwnedNode, node_hash: Option>) { + /// Descend into a node. + fn descend(&mut self, node: OwnedNode, node_hash: Option>) { self.trail .push(Crumb { hash: node_hash, status: Status::Entering, node: Arc::new(node) }); } @@ -116,10 +130,11 @@ impl TrieDBRawIterator { db: &TrieDB, key: &[u8], prefix: Prefix, + location: L::Location, ) -> Result, CError> { let mut res = TrieHash::::default(); res.as_mut().copy_from_slice(key); - db.fetch_value(res, prefix) + db.fetch_value(res, prefix, location) } /// Seek a node position at 'key' for iterator. @@ -132,6 +147,7 @@ impl TrieDBRawIterator { &mut self, db: &TrieDB, key: &[u8], + fwd: bool, ) -> Result, CError> { self.trail.clear(); self.key_nibbles.clear(); @@ -139,7 +155,7 @@ impl TrieDBRawIterator { let (mut node, mut node_hash) = db.get_raw_or_lookup( >::default(), - NodeHandle::Hash(db.root().as_ref()), + NodeHandle::Hash(db.root().as_ref(), Default::default()), EMPTY_PREFIX, true, )?; @@ -149,29 +165,30 @@ impl TrieDBRawIterator { let (next_node, next_node_hash) = { self.descend(node, node_hash); let crumb = self.trail.last_mut().expect( - "descend_into_node pushes a crumb onto the trial; \ + "descend pushes a crumb onto the trail; \ thus the trail is non-empty; qed", ); let node_data = crumb.node.data(); + let locations = crumb.node.locations(); match crumb.node.node_plan() { NodePlan::Leaf { partial: partial_plan, .. } => { let slice = partial_plan.build(node_data); - if slice < partial { + if (fwd && slice < partial) || (!fwd && slice > partial) { crumb.status = Status::Exiting; - return Ok(false) + return Ok(false); } - return Ok(slice.starts_with(&partial)) + return Ok(slice.starts_with(&partial)); }, NodePlan::Extension { partial: partial_plan, child } => { let slice = partial_plan.build(node_data); if !partial.starts_with(&slice) { - if slice < partial { + if (fwd && slice < partial) || (!fwd && slice > partial) { crumb.status = Status::Exiting; self.key_nibbles.append_partial(slice.right()); - return Ok(false) + return Ok(false); } - return Ok(slice.starts_with(&partial)) + return Ok(slice.starts_with(&partial)); } full_key_nibbles += slice.len(); @@ -182,52 +199,60 @@ impl TrieDBRawIterator { let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - child.build(node_data), + child.build(node_data, locations.first().copied().unwrap_or_default()), prefix.left(), true, )? }, - NodePlan::Branch { value: _, children } => { + NodePlan::Branch { value, children } => { if partial.is_empty() { - return Ok(true) + return Ok(true); } let i = partial.at(0); crumb.status = Status::AtChild(i as usize); self.key_nibbles.push(i); - if let Some(child) = &children[i as usize] { + if children[i as usize].is_some() { + let child = NodePlan::build_child( + value.as_ref(), + children, + i as usize, + node_data, + locations, + ); + full_key_nibbles += 1; partial = partial.mid(1); let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - child.build(node_data), + child.unwrap(), prefix.left(), true, )? } else { - return Ok(false) + return Ok(false); } }, - NodePlan::NibbledBranch { partial: partial_plan, value: _, children } => { + NodePlan::NibbledBranch { partial: partial_plan, value, children } => { let slice = partial_plan.build(node_data); if !partial.starts_with(&slice) { - if slice < partial { + if (fwd && slice < partial) || (!fwd && slice > partial) { crumb.status = Status::Exiting; self.key_nibbles.append_partial(slice.right()); self.key_nibbles.push((nibble_ops::NIBBLE_LENGTH - 1) as u8); - return Ok(false) + return Ok(false); } - return Ok(slice.starts_with(&partial)) + return Ok(slice.starts_with(&partial)); } full_key_nibbles += slice.len(); partial = partial.mid(slice.len()); if partial.is_empty() { - return Ok(true) + return Ok(true); } let i = partial.at(0); @@ -235,27 +260,35 @@ impl TrieDBRawIterator { self.key_nibbles.append_partial(slice.right()); self.key_nibbles.push(i); - if let Some(child) = &children[i as usize] { + if children[i as usize].is_some() { + let child = NodePlan::build_child( + value.as_ref(), + children, + i as usize, + node_data, + locations, + ); + full_key_nibbles += 1; partial = partial.mid(1); let prefix = key.back(full_key_nibbles); db.get_raw_or_lookup( node_hash.unwrap_or_default(), - child.build(node_data), + child.unwrap(), prefix.left(), true, )? } else { - return Ok(false) + return Ok(false); } }, NodePlan::Empty => { if !partial.is_empty() { crumb.status = Status::Exiting; - return Ok(false) + return Ok(false); } - return Ok(true) + return Ok(true); }, } }; @@ -267,8 +300,13 @@ impl TrieDBRawIterator { /// Advance the iterator into a prefix, no value out of the prefix will be accessed /// or returned after this operation. - fn prefix(&mut self, db: &TrieDB, prefix: &[u8]) -> Result<(), TrieHash, CError> { - if self.seek(db, prefix)? { + fn prefix( + &mut self, + db: &TrieDB, + prefix: &[u8], + fwd: bool, + ) -> Result<(), TrieHash, CError> { + if self.seek(db, prefix, fwd)? { if let Some(v) = self.trail.pop() { self.trail.clear(); self.trail.push(v); @@ -290,31 +328,31 @@ impl TrieDBRawIterator { ) -> Result<(), TrieHash, CError> { if prefix.is_empty() { // There's no prefix, so just seek. - return self.seek(db, seek).map(|_| ()) + return self.seek(db, seek, true).map(|_| ()); } if seek.is_empty() || seek <= prefix { // Either we're not supposed to seek anywhere, // or we're supposed to seek *before* the prefix, // so just directly go to the prefix. - return self.prefix(db, prefix) + return self.prefix(db, prefix, true); } if !seek.starts_with(prefix) { // We're supposed to seek *after* the prefix, // so just return an empty iterator. self.trail.clear(); - return Ok(()) + return Ok(()); } - if !self.seek(db, prefix)? { + if !self.seek(db, prefix, true)? { // The database doesn't have a key with such a prefix. self.trail.clear(); - return Ok(()) + return Ok(()); } // Now seek forward again. - self.seek(db, seek)?; + self.seek(db, seek, true)?; let prefix_len = prefix.len() * crate::nibble::nibble_ops::NIBBLE_PER_BYTE; let mut len = 0; @@ -338,7 +376,7 @@ impl TrieDBRawIterator { } if len > prefix_len { self.trail = self.trail.split_off(i); - return Ok(()) + return Ok(()); } } @@ -349,12 +387,15 @@ impl TrieDBRawIterator { /// Fetches the next raw item. // /// Must be called with the same `db` as when the iterator was created. + /// + /// Specify `fwd` to indicate the direction of the iteration (`true` for forward). pub(crate) fn next_raw_item( &mut self, db: &TrieDB, + fwd: bool, ) -> Option< Result< - (&NibbleVec, Option<&TrieHash>, &Arc>), + (&NibbleVec, Option<&TrieHash>, &Arc>), TrieHash, CError, >, @@ -362,13 +403,20 @@ impl TrieDBRawIterator { loop { let crumb = self.trail.last_mut()?; let node_data = crumb.node.data(); + let locations = crumb.node.locations(); match (crumb.status, crumb.node.node_plan()) { - (Status::Entering, _) => { - // This is only necessary due to current borrow checker's limitation. - let crumb = self.trail.last_mut().expect("we've just fetched the last element using `last_mut` so this cannot fail; qed"); - crumb.increment(); - return Some(Ok((&self.key_nibbles, crumb.hash.as_ref(), &crumb.node))) + (Status::Entering, _) => + if fwd { + let crumb = self.trail.last_mut().expect("we've just fetched the last element using `last_mut` so this cannot fail; qed"); + crumb.step(fwd); + return Some(Ok((&self.key_nibbles, crumb.hash.as_ref(), &crumb.node))); + } else { + crumb.step(fwd); + }, + (Status::AftExiting, _) => { + self.trail.pop().expect("we've just fetched the last element using `last_mut` so this cannot fail; qed"); + self.trail.last_mut()?.step(fwd); }, (Status::Exiting, node) => { match node { @@ -383,8 +431,11 @@ impl TrieDBRawIterator { self.key_nibbles.drop_lasts(partial.len() + 1); }, } - self.trail.pop().expect("we've just fetched the last element using `last_mut` so this cannot fail; qed"); - self.trail.last_mut()?.increment(); + self.trail.last_mut()?.step(fwd); + if !fwd { + let crumb = self.trail.last_mut().expect("we've just fetched the last element using `last_mut` so this cannot fail; qed"); + return Some(Ok((&self.key_nibbles, crumb.hash.as_ref(), &crumb.node))); + } }, (Status::At, NodePlan::Extension { partial: partial_plan, child }) => { let partial = partial_plan.build(node_data); @@ -392,7 +443,7 @@ impl TrieDBRawIterator { match db.get_raw_or_lookup( crumb.hash.unwrap_or_default(), - child.build(node_data), + child.build(node_data, locations.first().copied().unwrap_or_default()), self.key_nibbles.as_prefix(), true, ) { @@ -400,30 +451,46 @@ impl TrieDBRawIterator { self.descend(node, node_hash); }, Err(err) => { - crumb.increment(); - return Some(Err(err)) + crumb.step(fwd); + return Some(Err(err)); }, } }, (Status::At, NodePlan::Branch { .. }) => { - self.key_nibbles.push(0); - crumb.increment(); + self.key_nibbles.push(if fwd { + 0 + } else { + (nibble_ops::NIBBLE_LENGTH - 1) as u8 + }); + crumb.step(fwd); }, (Status::At, NodePlan::NibbledBranch { partial: partial_plan, .. }) => { let partial = partial_plan.build(node_data); self.key_nibbles.append_partial(partial.right()); - self.key_nibbles.push(0); - crumb.increment(); + self.key_nibbles.push(if fwd { + 0 + } else { + (nibble_ops::NIBBLE_LENGTH - 1) as u8 + }); + crumb.step(fwd); }, - (Status::AtChild(i), NodePlan::Branch { children, .. }) | - (Status::AtChild(i), NodePlan::NibbledBranch { children, .. }) => { - if let Some(child) = &children[i] { + (Status::AtChild(i), NodePlan::Branch { value, children, .. }) | + (Status::AtChild(i), NodePlan::NibbledBranch { value, children, .. }) => { + if children[i].is_some() { + let child = NodePlan::build_child( + value.as_ref(), + children, + i, + node_data, + locations, + ); + self.key_nibbles.pop(); self.key_nibbles.push(i as u8); match db.get_raw_or_lookup( crumb.hash.unwrap_or_default(), - child.build(node_data), + child.unwrap(), self.key_nibbles.as_prefix(), true, ) { @@ -431,16 +498,16 @@ impl TrieDBRawIterator { self.descend(node, node_hash); }, Err(err) => { - crumb.increment(); - return Some(Err(err)) + crumb.step(fwd); + return Some(Err(err)); }, } } else { - crumb.increment(); + crumb.step(fwd); } }, _ => panic!( - "Crumb::increment and TrieDBNodeIterator are implemented so that \ + "Crumb::step and TrieDBNodeIterator are implemented so that \ the above arms are the only possible states" ), } @@ -451,92 +518,168 @@ impl TrieDBRawIterator { /// /// Must be called with the same `db` as when the iterator was created. pub fn next_item(&mut self, db: &TrieDB) -> Option, CError>> { - while let Some(raw_item) = self.next_raw_item(db) { + while let Some(raw_item) = self.next_raw_item(db, true) { let (prefix, _, node) = match raw_item { Ok(raw_item) => raw_item, Err(err) => return Some(Err(err)), }; - let mut prefix = prefix.clone(); - let value = match node.node() { - Node::Leaf(partial, value) => { - prefix.append_partial(partial.right()); - value - }, - Node::Branch(_, value) => match value { - Some(value) => value, - None => continue, - }, - Node::NibbledBranch(partial, _, value) => { - prefix.append_partial(partial.right()); - match value { - Some(value) => value, - None => continue, - } - }, - _ => continue, + match Self::value_from_raw(prefix, node, db) { + Some(r) => return Some(r), + None => continue, + } + } + None + } + + /// Fetches the previous trie item. + /// + /// Must be called with the same `db` as when the iterator was created. + pub fn prev_item(&mut self, db: &TrieDB) -> Option, CError>> { + while let Some(raw_item) = self.next_raw_item(db, false) { + let (prefix, _, node) = match raw_item { + Ok(raw_item) => raw_item, + Err(err) => return Some(Err(err)), }; - let (key_slice, maybe_extra_nibble) = prefix.as_prefix(); - let key = key_slice.to_vec(); - if let Some(extra_nibble) = maybe_extra_nibble { - return Some(Err(Box::new(TrieError::ValueAtIncompleteKey(key, extra_nibble)))) + match Self::value_from_raw(prefix, node, db) { + Some(r) => return Some(r), + None => continue, } + } + None + } + + pub(crate) fn value_from_raw( + prefix: &NibbleVec, + node: &Arc>, + db: &TrieDB, + ) -> Option, CError>> { + let mut prefix = prefix.clone(); + let value = match node.node() { + Node::Leaf(partial, value) => { + prefix.append_partial(partial.right()); + value + }, + Node::Branch(_, value) => match value { + Some(value) => value, + None => return None, + }, + Node::NibbledBranch(partial, _, value) => { + prefix.append_partial(partial.right()); + match value { + Some(value) => value, + None => return None, + } + }, + _ => return None, + }; + + let (key_slice, maybe_extra_nibble) = prefix.as_prefix(); + let key = key_slice.to_vec(); + if let Some(extra_nibble) = maybe_extra_nibble { + return Some(Err(Box::new(TrieError::ValueAtIncompleteKey(key, extra_nibble)))) + } - let value = match value { - Value::Node(hash) => match Self::fetch_value(db, &hash, (key_slice, None)) { + let value = match value { + Value::Node(hash, location) => + match Self::fetch_value(db, &hash, (key_slice, None), location) { Ok(value) => value, Err(err) => return Some(Err(err)), }, - Value::Inline(value) => value.to_vec(), - }; + Value::Inline(value) => value.to_vec(), + }; - return Some(Ok((key, value))) - } - None + return Some(Ok((key, value))) } /// Fetches the next key. /// /// Must be called with the same `db` as when the iterator was created. pub fn next_key(&mut self, db: &TrieDB) -> Option, CError>> { - while let Some(raw_item) = self.next_raw_item(db) { - let (prefix, _, node) = match raw_item { - Ok(raw_item) => raw_item, - Err(err) => return Some(Err(err)), + while let Some(raw_item) = self.next_raw_item(db, true) { + let (key, maybe_extra_nibble, _) = match Self::extract_key_from_raw_item(raw_item) { + Some(Ok(k)) => k, + Some(Err(err)) => return Some(Err(err)), + None => continue, }; - let mut prefix = prefix.clone(); - match node.node() { - Node::Leaf(partial, _) => { - prefix.append_partial(partial.right()); - }, - Node::Branch(_, value) => - if value.is_none() { - continue - }, - Node::NibbledBranch(partial, _, value) => { - prefix.append_partial(partial.right()); - if value.is_none() { - continue - } - }, - _ => continue, + if let Some(extra_nibble) = maybe_extra_nibble { + return Some(Err(Box::new(TrieError::ValueAtIncompleteKey(key, extra_nibble)))); + } + + return Some(Ok(key)); + } + None + } + + /// Fetches the previous key. + /// + /// Must be called with the same `db` as when the iterator was created. + pub fn prev_key(&mut self, db: &TrieDB) -> Option, CError>> { + while let Some(raw_item) = self.next_raw_item(db, false) { + let (key, maybe_extra_nibble, _) = match Self::extract_key_from_raw_item(raw_item) { + Some(Ok(k)) => k, + Some(Err(err)) => return Some(Err(err)), + None => continue, }; - let (key_slice, maybe_extra_nibble) = prefix.as_prefix(); - let key = key_slice.to_vec(); if let Some(extra_nibble) = maybe_extra_nibble { - return Some(Err(Box::new(TrieError::ValueAtIncompleteKey(key, extra_nibble)))) + return Some(Err(Box::new(TrieError::ValueAtIncompleteKey(key, extra_nibble)))); } - return Some(Ok(key)) + return Some(Ok(key)); } None } + + /// Extracts the key from the result of a raw item retrieval. + /// + /// Given a raw item, it extracts the key information, including the key bytes, an optional + /// extra nibble (prefix padding), and the node value. + fn extract_key_from_raw_item<'a>( + raw_item: Result< + (&NibbleVec, Option<&TrieHash>, &'a Arc>), + TrieHash, + CError, + >, + ) -> Option, Option, Value<'a, L::Location>), TrieHash, CError>> { + let (prefix, _, node) = match raw_item { + Ok(raw_item) => raw_item, + Err(err) => return Some(Err(err)), + }; + + let mut prefix = prefix.clone(); + let value = match node.node() { + Node::Leaf(partial, value) => { + prefix.append_partial(partial.right()); + value + }, + Node::Branch(_, value) => match value { + Some(value) => value, + None => return None, + }, + Node::NibbledBranch(partial, _, value) => { + prefix.append_partial(partial.right()); + match value { + Some(value) => value, + None => return None, + } + }, + _ => return None, + }; + + let (key_slice, maybe_extra_nibble) = prefix.as_prefix(); + + Some(Ok((key_slice.to_vec(), maybe_extra_nibble, value))) + } } /// Iterator for going through all nodes in the trie in pre-order traversal order. +/// +/// You can reduce the number of iterations and simultaneously iterate in both directions with two +/// cursors by using `TrieDBNodeDoubleEndedIterator`. You can convert this iterator into a double +/// ended iterator with `into_double_ended_iter`. pub struct TrieDBNodeIterator<'a, 'cache, L: TrieLayout> { db: &'a TrieDB<'a, 'cache, L>, raw_iter: TrieDBRawIterator, @@ -563,14 +706,15 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeIterator<'a, 'cache, L> { &self, key: &[u8], prefix: Prefix, + location: L::Location, ) -> Result, CError> { - TrieDBRawIterator::fetch_value(self.db, key, prefix) + TrieDBRawIterator::fetch_value(self.db, key, prefix, location) } /// Advance the iterator into a prefix, no value out of the prefix will be accessed /// or returned after this operation. pub fn prefix(&mut self, prefix: &[u8]) -> Result<(), TrieHash, CError> { - self.raw_iter.prefix(self.db, prefix) + self.raw_iter.prefix(self.db, prefix, true) } /// Advance the iterator into a prefix, no value out of the prefix will be accessed @@ -584,23 +728,137 @@ impl<'a, 'cache, L: TrieLayout> TrieDBNodeIterator<'a, 'cache, L> { } /// Access inner hash db. - pub fn db(&self) -> &dyn hash_db::HashDBRef { + pub fn db(&self) -> &dyn node_db::NodeDB { self.db.db() } + + /// Access value of an item. + pub fn item_from_raw( + &self, + item: &(NibbleVec, Option>, Arc>), + ) -> Option, CError>> { + TrieDBRawIterator::value_from_raw(&item.0, &item.2, self.db) + } } impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBNodeIterator<'a, 'cache, L> { fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { - self.raw_iter.seek(self.db, key).map(|_| ()) + self.raw_iter.seek(self.db, key, true).map(|_| ()) } } impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBNodeIterator<'a, 'cache, L> { - type Item = - Result<(NibbleVec, Option>, Arc>), TrieHash, CError>; + type Item = Result< + (NibbleVec, Option>, Arc>), + TrieHash, + CError, + >; fn next(&mut self) -> Option { - self.raw_iter.next_raw_item(self.db).map(|result| { + self.raw_iter.next_raw_item(self.db, true).map(|result| { + result.map(|(nibble, hash, node)| (nibble.clone(), hash.cloned(), node.clone())) + }) + } +} + +/// Double ended iterator for going through all nodes in the trie in pre-order traversal order. +pub struct TrieDBNodeDoubleEndedIterator<'a, 'cache, L: TrieLayout> { + db: &'a TrieDB<'a, 'cache, L>, + raw_iter: TrieDBRawIterator, + back_raw_iter: TrieDBRawIterator, +} + +impl<'a, 'cache, L: TrieLayout> TrieDBNodeDoubleEndedIterator<'a, 'cache, L> { + /// Create a new double ended iterator. + pub fn new(db: &'a TrieDB<'a, 'cache, L>) -> Result, CError> { + Ok(Self { + db, + raw_iter: TrieDBRawIterator::new(db)?, + back_raw_iter: TrieDBRawIterator::new(db)?, + }) + } + + /// Restore an iterator from a raw iterators. + pub fn from_raw( + db: &'a TrieDB<'a, 'cache, L>, + raw_iter: TrieDBRawIterator, + back_raw_iter: TrieDBRawIterator, + ) -> Self { + Self { db, raw_iter, back_raw_iter } + } + + /// Convert the iterator to a raw forward iterator. + pub fn into_raw(self) -> TrieDBRawIterator { + self.raw_iter + } + + /// Convert the iterator to a raw backward iterator. + pub fn into_raw_back(self) -> TrieDBRawIterator { + self.back_raw_iter + } + + /// Fetch value by hash at a current node height + pub fn fetch_value( + &self, + key: &[u8], + prefix: Prefix, + location: L::Location, + ) -> Result, CError> { + TrieDBRawIterator::fetch_value(self.db, key, prefix, location) + } + + /// Advance the iterator into a prefix, no value out of the prefix will be accessed + /// or returned after this operation. + pub fn prefix(&mut self, prefix: &[u8]) -> Result<(), TrieHash, CError> { + self.raw_iter.prefix(self.db, prefix, true)?; + self.back_raw_iter.prefix(self.db, prefix, false) + } + + /// Advance the iterator into a prefix, no value out of the prefix will be accessed + /// or returned after this operation. + pub fn prefix_then_seek( + &mut self, + prefix: &[u8], + seek: &[u8], + ) -> Result<(), TrieHash, CError> { + self.raw_iter.prefix_then_seek(self.db, prefix, seek)?; + self.back_raw_iter.prefix_then_seek(self.db, prefix, seek) + } + + /// Access inner hash db. + pub fn db(&self) -> &dyn node_db::NodeDB { + self.db.db() + } +} + +impl TrieDoubleEndedIterator for TrieDBNodeDoubleEndedIterator<'_, '_, L> {} + +impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBNodeDoubleEndedIterator<'a, 'cache, L> { + fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { + self.raw_iter.seek(self.db, key, true).map(|_| ())?; + self.back_raw_iter.seek(self.db, key, false).map(|_| ()) + } +} + +impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBNodeDoubleEndedIterator<'a, 'cache, L> { + type Item = Result< + (NibbleVec, Option>, Arc>), + TrieHash, + CError, + >; + + fn next(&mut self) -> Option { + self.raw_iter.next_raw_item(self.db, true).map(|result| { + result.map(|(nibble, hash, node)| (nibble.clone(), hash.cloned(), node.clone())) + }) + } +} + +impl<'a, 'cache, L: TrieLayout> DoubleEndedIterator + for TrieDBNodeDoubleEndedIterator<'a, 'cache, L> +{ + fn next_back(&mut self) -> Option { + self.back_raw_iter.next_raw_item(self.db, false).map(|result| { result.map(|(nibble, hash, node)| (nibble.clone(), hash.cloned(), node.clone())) }) } diff --git a/test-support/keccak-hasher/src/lib.rs b/subtrie/src/keccak_hasher.rs similarity index 98% rename from test-support/keccak-hasher/src/lib.rs rename to subtrie/src/keccak_hasher.rs index b4692f77..05ea572b 100644 --- a/test-support/keccak-hasher/src/lib.rs +++ b/subtrie/src/keccak_hasher.rs @@ -14,8 +14,8 @@ //! Hasher implementation for the Keccak-256 hash +use crate::node_db::Hasher; use hash256_std_hasher::Hash256StdHasher; -use hash_db::Hasher; use tiny_keccak::{Hasher as _, Keccak}; /// The `Keccak` hash output type. diff --git a/trie-db/src/lib.rs b/subtrie/src/lib.rs similarity index 73% rename from trie-db/src/lib.rs rename to subtrie/src/lib.rs index b09372b2..51421177 100644 --- a/trie-db/src/lib.rs +++ b/subtrie/src/lib.rs @@ -21,14 +21,21 @@ extern crate alloc; #[cfg(feature = "std")] mod rstd { pub use std::{ - borrow, boxed, cmp, collections::VecDeque, convert, error::Error, fmt, hash, iter, marker, - mem, ops, rc, result, sync, vec, + borrow, boxed, cmp, + collections::{BTreeMap, VecDeque}, + convert, + error::Error, + fmt, hash, iter, marker, mem, ops, result, sync, vec, }; } #[cfg(not(feature = "std"))] mod rstd { - pub use alloc::{borrow, boxed, collections::VecDeque, rc, sync, vec}; + pub use alloc::{ + borrow, boxed, + collections::{btree_map::BTreeMap, VecDeque}, + rc, sync, vec, + }; pub use core::{cmp, convert, fmt, hash, iter, marker, mem, ops, result}; pub trait Error {} impl Error for T {} @@ -38,19 +45,27 @@ mod rstd { use self::rstd::{fmt, Error}; use self::rstd::{boxed::Box, vec::Vec}; -use hash_db::MaybeDebug; +pub use iterator::TrieDBNodeDoubleEndedIterator; use node::NodeOwned; +use node_db::MaybeDebug; +#[cfg(feature = "bench")] +pub mod bench; +#[cfg(any(feature = "test_utils", test))] +pub mod keccak_hasher; +#[cfg(feature = "std")] +pub mod mem_tree_db; +pub mod memory_db; pub mod node; +pub mod node_db; pub mod proof; pub mod recorder; -pub mod sectriedb; -pub mod sectriedbmut; +#[cfg(feature = "test_utils")] +pub mod test_utils; +pub mod trie_root; pub mod triedb; pub mod triedbmut; -mod fatdb; -mod fatdbmut; mod iter_build; mod iterator; mod lookup; @@ -59,23 +74,22 @@ mod node_codec; mod trie_codec; pub use self::{ - fatdb::{FatDB, FatDBIterator}, - fatdbmut::FatDBMut, lookup::Lookup, nibble::{nibble_ops, NibbleSlice, NibbleVec}, recorder::Recorder, - sectriedb::SecTrieDB, - sectriedbmut::SecTrieDBMut, triedb::{TrieDB, TrieDBBuilder, TrieDBIterator, TrieDBKeyIterator}, - triedbmut::{ChildReference, TrieDBMut, TrieDBMutBuilder, Value}, + triedbmut::{ + Changeset, ChildReference, ExistingChangesetNode, NewChangesetNode, OwnedPrefix, TrieDBMut, + TrieDBMutBuilder, Value, + }, }; +use crate::node_db::Hasher; pub use crate::{ iter_build::{trie_visit, ProcessEncodedNode, TrieBuilder, TrieRoot, TrieRootUnhashed}, iterator::{TrieDBNodeIterator, TrieDBRawIterator}, node_codec::{NodeCodec, Partial}, trie_codec::{decode_compact, decode_compact_from_iter, encode_compact}, }; -pub use hash_db::{HashDB, HashDBRef, Hasher}; #[cfg(feature = "std")] pub use crate::iter_build::TrieRootPrint; @@ -162,9 +176,9 @@ pub trait Query { /// If a cache is used, [`Self::Key`] and [`Self::NodeOwned`] are possible /// values. Otherwise only [`Self::EncodedNode`] is a possible value. #[cfg_attr(feature = "std", derive(Debug))] -pub enum TrieAccess<'a, H> { +pub enum TrieAccess<'a, H, L> { /// The given [`NodeOwned`] was accessed using its `hash`. - NodeOwned { hash: H, node_owned: &'a NodeOwned }, + NodeOwned { hash: H, node_owned: &'a NodeOwned }, /// The given `encoded_node` was accessed using its `hash`. EncodedNode { hash: H, encoded_node: rstd::borrow::Cow<'a, [u8]> }, /// The given `value` was accessed using its `hash`. @@ -173,6 +187,13 @@ pub enum TrieAccess<'a, H> { /// /// Should map to [`RecordedForKey::Value`] when checking the recorder. Value { hash: H, value: rstd::borrow::Cow<'a, [u8]>, full_key: &'a [u8] }, + /// A value was accessed that is stored inline a node. + /// + /// As the value is stored inline there is no need to separately record the value as it is part + /// of a node. The given `full_key` is the key to access this value in the trie. + /// + /// Should map to [`RecordedForKey::Value`] when checking the recorder. + InlineValue { full_key: &'a [u8] }, /// The hash of the value for the given `full_key` was accessed. /// /// Should map to [`RecordedForKey::Hash`] when checking the recorder. @@ -184,7 +205,7 @@ pub enum TrieAccess<'a, H> { } /// Result of [`TrieRecorder::trie_nodes_recorded_for_key`]. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RecordedForKey { /// We recorded all trie nodes up to the value for a storage key. /// @@ -223,12 +244,12 @@ impl RecordedForKey { /// /// To build a trie proof a recorder is required that records all trie accesses. These recorded trie /// accesses can then be used to create the proof. -pub trait TrieRecorder { +pub trait TrieRecorder { /// Record the given [`TrieAccess`]. /// /// Depending on the [`TrieAccess`] a call of [`Self::trie_nodes_recorded_for_key`] afterwards /// must return the correct recorded state. - fn record<'a>(&mut self, access: TrieAccess<'a, H>); + fn record<'a>(&mut self, access: TrieAccess<'a, H, L>); /// Check if we have recorded any trie nodes for the given `key`. /// @@ -251,6 +272,11 @@ pub trait Trie { /// Return the root of the trie. fn root(&self) -> &TrieHash; + /// Return the root location of the trie if it was set. + fn root_location(&self) -> L::Location { + Default::default() + } + /// Is the trie empty? fn is_empty(&self) -> bool { *self.root() == L::Codec::hashed_null_node() @@ -277,6 +303,17 @@ pub trait Trie { query: Q, ) -> Result, TrieHash, CError>; + /// Look up the [`MerkleValue`] of the node that is the closest descendant for the provided + /// key. + /// + /// When the provided key leads to a node, then the merkle value of that node + /// is returned. However, if the key does not lead to a node, then the merkle value + /// of the closest descendant is returned. `None` if no such descendant exists. + fn lookup_first_descendant( + &self, + key: &[u8], + ) -> Result>>, TrieHash, CError>; + /// Returns a depth-first iterator over the elements of trie. fn iter<'a>( &'a self, @@ -296,183 +333,26 @@ pub trait Trie { >; } -/// A key-value datastore implemented as a database-backed modified Merkle tree. -pub trait TrieMut { - /// Return the root of the trie. - fn root(&mut self) -> &TrieHash; - - /// Is the trie empty? - fn is_empty(&self) -> bool; - - /// Does the trie contain a given key? - fn contains(&self, key: &[u8]) -> Result, CError> { - self.get(key).map(|x| x.is_some()) - } - - /// What is the value of the given key in this trie? - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, TrieHash, CError> - where - 'a: 'key; - - /// Insert a `key`/`value` pair into the trie. An empty value is equivalent to removing - /// `key` from the trie. Returns the old value associated with this key, if it existed. - fn insert( - &mut self, - key: &[u8], - value: &[u8], - ) -> Result>, TrieHash, CError>; - - /// Remove a `key` from the trie. Equivalent to making it equal to the empty - /// value. Returns the old value associated with this key, if it existed. - fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError>; -} - /// A trie iterator that also supports random access (`seek()`). pub trait TrieIterator: Iterator { /// Position the iterator on the first element with key >= `key` fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError>; } +/// Extending the `TrieIterator` trait with `DoubleEndedIterator` trait. +pub trait TrieDoubleEndedIterator: TrieIterator + DoubleEndedIterator {} + /// Trie types #[derive(PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] pub enum TrieSpec { /// Generic trie. Generic, - /// Secure trie. - Secure, - /// Secure trie with fat database. - Fat, } impl Default for TrieSpec { fn default() -> TrieSpec { - TrieSpec::Secure - } -} - -/// Trie factory. -#[derive(Default, Clone)] -pub struct TrieFactory { - spec: TrieSpec, -} - -/// All different kinds of tries. -/// This is used to prevent a heap allocation for every created trie. -pub enum TrieKinds<'db, 'cache, L: TrieLayout> { - /// A generic trie db. - Generic(TrieDB<'db, 'cache, L>), - /// A secure trie db. - Secure(SecTrieDB<'db, 'cache, L>), - /// A fat trie db. - Fat(FatDB<'db, 'cache, L>), -} - -// wrapper macro for making the match easier to deal with. -macro_rules! wrapper { - ($me: ident, $f_name: ident, $($param: ident),*) => { - match *$me { - TrieKinds::Generic(ref t) => t.$f_name($($param),*), - TrieKinds::Secure(ref t) => t.$f_name($($param),*), - TrieKinds::Fat(ref t) => t.$f_name($($param),*), - } - } -} - -impl<'db, 'cache, L: TrieLayout> Trie for TrieKinds<'db, 'cache, L> { - fn root(&self) -> &TrieHash { - wrapper!(self, root,) - } - - fn is_empty(&self) -> bool { - wrapper!(self, is_empty,) - } - - fn contains(&self, key: &[u8]) -> Result, CError> { - wrapper!(self, contains, key) - } - - fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { - wrapper!(self, get_hash, key) - } - - fn get_with>( - &self, - key: &[u8], - query: Q, - ) -> Result, TrieHash, CError> { - wrapper!(self, get_with, key, query) - } - - fn iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - wrapper!(self, iter,) - } - - fn key_iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - wrapper!(self, key_iter,) - } -} - -impl TrieFactory { - /// Creates new factory. - pub fn new(spec: TrieSpec) -> Self { - TrieFactory { spec } - } - - /// Create new immutable instance of Trie. - pub fn readonly<'db, 'cache, L: TrieLayout>( - &self, - db: &'db dyn HashDBRef, - root: &'db TrieHash, - ) -> TrieKinds<'db, 'cache, L> { - match self.spec { - TrieSpec::Generic => TrieKinds::Generic(TrieDBBuilder::new(db, root).build()), - TrieSpec::Secure => TrieKinds::Secure(SecTrieDB::new(db, root)), - TrieSpec::Fat => TrieKinds::Fat(FatDB::new(db, root)), - } - } - - /// Create new mutable instance of Trie. - pub fn create<'db, L: TrieLayout + 'db>( - &self, - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, - ) -> Box + 'db> { - match self.spec { - TrieSpec::Generic => Box::new(TrieDBMutBuilder::::new(db, root).build()), - TrieSpec::Secure => Box::new(SecTrieDBMut::::new(db, root)), - TrieSpec::Fat => Box::new(FatDBMut::::new(db, root)), - } - } - - /// Create new mutable instance of trie and check for errors. - pub fn from_existing<'db, L: TrieLayout + 'db>( - &self, - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, - ) -> Box + 'db> { - match self.spec { - TrieSpec::Generic => Box::new(TrieDBMutBuilder::::from_existing(db, root).build()), - TrieSpec::Secure => Box::new(SecTrieDBMut::::from_existing(db, root)), - TrieSpec::Fat => Box::new(FatDBMut::::from_existing(db, root)), - } - } - - /// Returns true iff the trie DB is a fat DB (allows enumeration of keys). - pub fn is_fat(&self) -> bool { - self.spec == TrieSpec::Fat + TrieSpec::Generic } } @@ -494,21 +374,29 @@ pub trait TrieLayout { type Hash: Hasher; /// Codec to use (needs to match hasher and nibble ops). type Codec: NodeCodec::Out>; + type Location: Location; } +/// Trait alias for requirement of location with `TrieLayout`. +pub trait Location: Copy + Default + Eq + PartialEq + MaybeDebug {} + +impl Location for T {} + /// This trait associates a trie definition with preferred methods. /// It also contains own default implementations and can be /// used to allow switching implementation. pub trait TrieConfiguration: Sized + TrieLayout { /// Operation to build a trie db from its ordered iterator over its key/values. - fn trie_build(db: &mut DB, input: I) -> ::Out + fn trie_build( + db: &mut memory_db::MemoryDB, DBValue>, + input: I, + ) -> ::Out where - DB: HashDB, I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - let mut cb = TrieBuilder::::new(db); + let mut cb = TrieBuilder::>::new(db); trie_visit::(input.into_iter(), &mut cb); cb.root.unwrap_or_default() } @@ -560,11 +448,11 @@ pub type CError = <::Codec as NodeCodec>::Error; /// A value as cached by the [`TrieCache`]. #[derive(Clone, Debug)] -pub enum CachedValue { +pub enum CachedValue { /// The value doesn't exist in the trie. NonExisting, - /// We cached the hash, because we did not yet accessed the data. - ExistingHash(H), + /// We cached the hash and location, because we did not yet accessed the data. + ExistingHash(H, L), /// The value exists in the trie. Existing { /// The hash of the value. @@ -578,7 +466,7 @@ pub enum CachedValue { }, } -impl CachedValue { +impl CachedValue { /// Returns the data of the value. /// /// If a value doesn't exist in the trie or only the value hash is cached, this function returns @@ -596,36 +484,24 @@ impl CachedValue { /// Returns only `None` when the value doesn't exist. pub fn hash(&self) -> Option { match self { - Self::ExistingHash(hash) | Self::Existing { hash, .. } => Some(*hash), + Self::ExistingHash(hash, _) | Self::Existing { hash, .. } => Some(*hash), Self::NonExisting => None, } } } -impl From<(Bytes, H)> for CachedValue { +impl From<(Bytes, H)> for CachedValue { fn from(value: (Bytes, H)) -> Self { Self::Existing { hash: value.1, data: value.0.into() } } } -impl From for CachedValue { - fn from(value: H) -> Self { - Self::ExistingHash(value) - } -} - -impl From> for CachedValue { +impl From> for CachedValue { fn from(value: Option<(Bytes, H)>) -> Self { value.map_or(Self::NonExisting, |v| Self::Existing { hash: v.1, data: v.0.into() }) } } -impl From> for CachedValue { - fn from(value: Option) -> Self { - value.map_or(Self::NonExisting, |v| Self::ExistingHash(v)) - } -} - /// A cache that can be used to speed-up certain operations when accessing the trie. /// /// The [`TrieDB`]/[`TrieDBMut`] by default are working with the internal hash-db in a non-owning @@ -642,7 +518,7 @@ impl From> for CachedValue { /// different values under the same key, it up to the cache implementation to ensure that the /// correct value is returned. As each trie has a different root, this root can be used to /// differentiate values under the same key. -pub trait TrieCache { +pub trait TrieCache { /// Lookup value for the given `key`. /// /// Returns the `None` if the `key` is unknown or otherwise `Some(_)` with the associated @@ -656,7 +532,7 @@ pub trait TrieCache { /// The cache can be used for different tries, aka with different roots. This means /// that the cache implementation needs to take care of always returning the correct value /// for the current trie root. - fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&CachedValue>; + fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&CachedValue>; /// Cache the given `value` for the given `key`. /// @@ -665,7 +541,7 @@ pub trait TrieCache { /// The cache can be used for different tries, aka with different roots. This means /// that the cache implementation needs to take care of caching `value` for the current /// trie root. - fn cache_value_for_key(&mut self, key: &[u8], value: CachedValue); + fn cache_value_for_key(&mut self, key: &[u8], value: CachedValue); /// Get or insert a [`NodeOwned`]. /// @@ -677,11 +553,15 @@ pub trait TrieCache { fn get_or_insert_node( &mut self, hash: NC::HashOut, - fetch_node: &mut dyn FnMut() -> Result, NC::HashOut, NC::Error>, - ) -> Result<&NodeOwned, NC::HashOut, NC::Error>; + location: L, + fetch_node: &mut dyn FnMut() -> Result, NC::HashOut, NC::Error>, + ) -> Result<&NodeOwned, NC::HashOut, NC::Error>; /// Get the [`NodeOwned`] that corresponds to the given `hash`. - fn get_node(&mut self, hash: &NC::HashOut) -> Option<&NodeOwned>; + fn get_node(&mut self, hash: &NC::HashOut, location: L) -> Option<&NodeOwned>; + + /// Put a new node. This is used to clear location info for existing nodes with the same hash. + fn insert_new_node(&mut self, hash: &NC::HashOut); } /// A container for storing bytes. @@ -738,3 +618,18 @@ impl From for BytesWeak { Self(rstd::sync::Arc::downgrade(&bytes.0)) } } + +/// Either the `hash` or `value` of a node depending on its size. +/// +/// If the size of the node `value` is bigger or equal than `MAX_INLINE_VALUE` the `hash` is +/// returned. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum MerkleValue { + /// The merkle value is the node data itself when the + /// node data is smaller than `MAX_INLINE_VALUE`. + /// + /// Note: The case of inline nodes. + Node(Vec), + /// The merkle value is the hash of the node. + Hash(H), +} diff --git a/trie-db/src/lookup.rs b/subtrie/src/lookup.rs similarity index 58% rename from trie-db/src/lookup.rs rename to subtrie/src/lookup.rs index 5e34055f..45accfeb 100644 --- a/trie-db/src/lookup.rs +++ b/subtrie/src/lookup.rs @@ -12,30 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Trie lookup via HashDB. +//! Trie lookup via NodeDB. use crate::{ nibble::NibbleSlice, node::{decode_hash, Node, NodeHandle, NodeHandleOwned, NodeOwned, Value, ValueOwned}, node_codec::NodeCodec, - rstd::boxed::Box, - Bytes, CError, CachedValue, DBValue, Query, RecordedForKey, Result, TrieAccess, TrieCache, - TrieError, TrieHash, TrieLayout, TrieRecorder, + node_db::{Hasher, NodeDB, Prefix}, + rstd::{boxed::Box, vec::Vec}, + Bytes, CError, CachedValue, DBValue, MerkleValue, Query, RecordedForKey, Result, TrieAccess, + TrieCache, TrieError, TrieHash, TrieLayout, TrieRecorder, }; -use hash_db::{HashDBRef, Hasher, Prefix}; /// Trie lookup helper object. pub struct Lookup<'a, 'cache, L: TrieLayout, Q: Query> { /// database to query from. - pub db: &'a dyn HashDBRef, + pub db: &'a dyn NodeDB, /// Query object to record nodes and transform data. pub query: Q, /// Hash to start at pub hash: TrieHash, + /// Optionally location to start at. + pub location: L::Location, /// Optional cache that should be used to speed up the lookup. - pub cache: Option<&'cache mut dyn TrieCache>, + pub cache: Option<&'cache mut dyn TrieCache>, /// Optional recorder that will be called to record all trie accesses. - pub recorder: Option<&'cache mut dyn TrieRecorder>>, + pub recorder: Option<&'cache mut dyn TrieRecorder, L::Location>>, } impl<'a, 'cache, L, Q> Lookup<'a, 'cache, L, Q> @@ -50,19 +52,25 @@ where /// /// Returns the bytes representing the value. fn load_value( - v: Value, + v: Value, prefix: Prefix, full_key: &[u8], - db: &dyn HashDBRef, - recorder: &mut Option<&mut dyn TrieRecorder>>, + db: &dyn NodeDB, + recorder: &mut Option<&mut dyn TrieRecorder, L::Location>>, query: Q, ) -> Result, CError> { match v { - Value::Inline(value) => Ok(query.decode(&value)), - Value::Node(hash) => { + Value::Inline(value) => { + if let Some(recorder) = recorder { + recorder.record(TrieAccess::InlineValue { full_key }); + } + + Ok(query.decode(&value)) + }, + Value::Node(hash, location) => { let mut res = TrieHash::::default(); res.as_mut().copy_from_slice(hash); - if let Some(value) = db.get(&res, prefix) { + if let Some((value, _)) = db.get(&res, prefix, location) { if let Some(recorder) = recorder { recorder.record(TrieAccess::Value { hash: res, @@ -86,19 +94,25 @@ where /// /// Returns the bytes representing the value and its hash. fn load_owned_value( - v: ValueOwned>, + v: ValueOwned, L::Location>, prefix: Prefix, full_key: &[u8], - cache: &mut dyn crate::TrieCache, - db: &dyn HashDBRef, - recorder: &mut Option<&mut dyn TrieRecorder>>, + cache: &mut dyn crate::TrieCache, + db: &dyn NodeDB, + recorder: &mut Option<&mut dyn TrieRecorder, L::Location>>, ) -> Result<(Bytes, TrieHash), TrieHash, CError> { match v { - ValueOwned::Inline(value, hash) => Ok((value.clone(), hash)), - ValueOwned::Node(hash) => { - let node = cache.get_or_insert_node(hash, &mut || { - let value = db - .get(&hash, prefix) + ValueOwned::Inline(value, hash) => { + if let Some(recorder) = recorder { + recorder.record(TrieAccess::InlineValue { full_key }); + } + + Ok((value.clone(), hash)) + }, + ValueOwned::Node(hash, location) => { + let node = cache.get_or_insert_node(hash, location, &mut || { + let (value, _) = db + .get(&hash, prefix, location) .ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; Ok(NodeOwned::Value(value.into(), hash)) @@ -125,14 +139,240 @@ where } } - fn record<'b>(&mut self, get_access: impl FnOnce() -> TrieAccess<'b, TrieHash>) + fn record<'b>(&mut self, get_access: impl FnOnce() -> TrieAccess<'b, TrieHash, L::Location>) where TrieHash: 'b, + L::Location: 'b, { if let Some(recorder) = self.recorder.as_mut() { recorder.record(get_access()); } } + /// Look up the merkle value (hash) of the node that is the closest descendant for the provided + /// key. + /// + /// When the provided key leads to a node, then the merkle value (hash) of that node + /// is returned. However, if the key does not lead to a node, then the merkle value + /// of the closest descendant is returned. `None` if no such descendant exists. + pub fn lookup_first_descendant( + mut self, + full_key: &[u8], + nibble_key: NibbleSlice, + ) -> Result>>, TrieHash, CError> { + let mut partial = nibble_key; + let mut hash = self.hash; + let mut location = self.location; + let mut key_nibbles = 0; + + let mut cache = self.cache.take(); + + // this loop iterates through non-inline nodes. + for depth in 0.. { + // Ensure the owned node reference lives long enough. + // Value is never read, but the reference is. + let mut _owned_node = NodeOwned::Empty; + + // The binary encoded data of the node fetched from the database. + // + // Populated by `get_owned_node` to avoid one extra allocation by not + // calling `NodeOwned::to_encoded` when computing the hash of inlined nodes. + let mut node_data = Vec::new(); + + // Get the owned node representation from the database. + let mut get_owned_node = |depth: i32| { + let (data, locations) = + match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { + Some(value) => value, + None => + return Err(Box::new(match depth { + 0 => TrieError::InvalidStateRoot(hash), + _ => TrieError::IncompleteDatabase(hash), + })), + }; + + let decoded = match L::Codec::decode(&data[..], &locations) { + Ok(node) => node, + Err(e) => return Err(Box::new(TrieError::DecoderError(hash, e))), + }; + + let owned = decoded.to_owned_node::()?; + node_data = data; + Ok(owned) + }; + + let mut node = if let Some(cache) = &mut cache { + let node = + cache.get_or_insert_node(hash, location, &mut || get_owned_node(depth))?; + + self.record(|| TrieAccess::NodeOwned { hash, node_owned: node }); + + node + } else { + _owned_node = get_owned_node(depth)?; + + self.record(|| TrieAccess::EncodedNode { + hash, + encoded_node: node_data.as_slice().into(), + }); + + &_owned_node + }; + + // this loop iterates through all inline children (usually max 1) + // without incrementing the depth. + let mut is_inline = false; + loop { + let next_node = match node { + NodeOwned::Leaf(slice, _) => { + // The leaf slice can be longer than remainder of the provided key + // (descendent), but not the other way around. + if !slice.starts_with_slice(&partial) { + self.record(|| TrieAccess::NonExisting { full_key }); + return Ok(None) + } + + if partial.len() != slice.len() { + self.record(|| TrieAccess::NonExisting { full_key }); + } + + let res = is_inline + .then(|| MerkleValue::Node(node_data)) + .unwrap_or_else(|| MerkleValue::Hash(hash)); + return Ok(Some(res)) + }, + NodeOwned::Extension(slice, item) => { + if partial.len() < slice.len() { + self.record(|| TrieAccess::NonExisting { full_key }); + + // Extension slice can be longer than remainder of the provided key + // (descendent), ensure the extension slice starts with the remainder + // of the provided key. + return if slice.starts_with_slice(&partial) { + let res = is_inline + .then(|| MerkleValue::Node(node_data)) + .unwrap_or_else(|| MerkleValue::Hash(hash)); + Ok(Some(res)) + } else { + Ok(None) + } + } + + // Remainder of the provided key is longer than the extension slice, + // must advance the node iteration if and only if keys share + // a common prefix. + if partial.starts_with_vec(&slice) { + // Empties the partial key if the extension slice is longer. + partial = partial.mid(slice.len()); + key_nibbles += slice.len(); + item + } else { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + } + }, + NodeOwned::Branch(children, value) => + if partial.is_empty() { + if value.is_none() { + self.record(|| TrieAccess::NonExisting { full_key }); + } + let res = is_inline + .then(|| MerkleValue::Node(node_data)) + .unwrap_or_else(|| MerkleValue::Hash(hash)); + return Ok(Some(res)) + } else { + match &children[partial.at(0) as usize] { + Some(x) => { + partial = partial.mid(1); + key_nibbles += 1; + x + }, + None => { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + }, + } + }, + NodeOwned::NibbledBranch(slice, children, value) => { + // Not enough remainder key to continue the search. + if partial.len() < slice.len() { + self.record(|| TrieAccess::NonExisting { full_key }); + + // Branch slice starts with the remainder key, there's nothing to + // advance. + return if slice.starts_with_slice(&partial) { + let res = is_inline + .then(|| MerkleValue::Node(node_data)) + .unwrap_or_else(|| MerkleValue::Hash(hash)); + Ok(Some(res)) + } else { + Ok(None) + } + } + + // Partial key is longer or equal than the branch slice. + // Ensure partial key starts with the branch slice. + if !partial.starts_with_vec(&slice) { + self.record(|| TrieAccess::NonExisting { full_key }); + return Ok(None) + } + + // Partial key starts with the branch slice. + if partial.len() == slice.len() { + if value.is_none() { + self.record(|| TrieAccess::NonExisting { full_key }); + } + + let res = is_inline + .then(|| MerkleValue::Node(node_data)) + .unwrap_or_else(|| MerkleValue::Hash(hash)); + return Ok(Some(res)) + } else { + match &children[partial.at(slice.len()) as usize] { + Some(x) => { + partial = partial.mid(slice.len() + 1); + key_nibbles += slice.len() + 1; + x + }, + None => { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + }, + } + } + }, + NodeOwned::Empty => { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + }, + NodeOwned::Value(_, _) => { + unreachable!( + "`NodeOwned::Value` can not be reached by using the hash of a node. \ + `NodeOwned::Value` is only constructed when loading a value into memory, \ + which needs to have a different hash than any node; qed", + ) + }, + }; + + // check if new node data is inline or hash. + match next_node { + NodeHandleOwned::Hash(new_hash, new_location) => { + hash = *new_hash; + location = *new_location; + break + }, + NodeHandleOwned::Inline(inline_node) => { + node = &inline_node; + is_inline = true; + }, + } + } + } + Ok(None) + } /// Look up the given `nibble_key`. /// @@ -167,8 +407,17 @@ where full_key, |v, _, full_key, _, recorder, _| { Ok(match v { - Value::Inline(v) => L::Hash::hash(&v), - Value::Node(hash_bytes) => { + Value::Inline(v) => { + if let Some(recoder) = recorder.as_mut() { + // We can record this as `InlineValue`, even we are just returning + // the `hash`. This is done to prevent requiring to re-record this + // key. + recoder.record(TrieAccess::InlineValue { full_key }); + } + + L::Hash::hash(&v) + }, + Value::Node(hash_bytes, _location) => { if let Some(recoder) = recorder.as_mut() { recoder.record(TrieAccess::Hash { full_key }); } @@ -190,7 +439,7 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, - cache: &mut dyn crate::TrieCache, + cache: &mut dyn crate::TrieCache, ) -> Result>, TrieHash, CError> { let value_cache_allowed = self .recorder @@ -211,21 +460,31 @@ where full_key, cache, |value, _, full_key, _, _, recorder| match value { - ValueOwned::Inline(value, hash) => Ok((hash, Some(value.clone()))), - ValueOwned::Node(hash) => { + ValueOwned::Inline(value, hash) => { + if let Some(recoder) = recorder.as_mut() { + // We can record this as `InlineValue`, even we are just returning + // the `hash`. This is done to prevent requiring to re-record this + // key. + recoder.record(TrieAccess::InlineValue { full_key }); + } + + Ok((hash, Some(value.clone()), Default::default())) + }, + ValueOwned::Node(hash, location) => { if let Some(recoder) = recorder.as_mut() { recoder.record(TrieAccess::Hash { full_key }); } - Ok((hash, None)) + Ok((hash, None, location)) }, }, )?; match &hash_and_value { - Some((hash, Some(value))) => + Some((hash, Some(value), _location)) => cache.cache_value_for_key(full_key, (value.clone(), *hash).into()), - Some((hash, None)) => cache.cache_value_for_key(full_key, (*hash).into()), + Some((hash, None, location)) => + cache.cache_value_for_key(full_key, CachedValue::ExistingHash(*hash, *location)), None => cache.cache_value_for_key(full_key, CachedValue::NonExisting), } @@ -243,7 +502,7 @@ where mut self, full_key: &[u8], nibble_key: NibbleSlice, - cache: &mut dyn crate::TrieCache, + cache: &mut dyn crate::TrieCache, ) -> Result, TrieHash, CError> { let trie_nodes_recorded = self.recorder.as_ref().map(|r| r.trie_nodes_recorded_for_key(full_key)); @@ -260,7 +519,7 @@ where }; let lookup_data = |lookup: &mut Self, - cache: &mut dyn crate::TrieCache| + cache: &mut dyn crate::TrieCache| -> Result, TrieHash, CError> { let data = lookup.look_up_with_cache_internal( nibble_key, @@ -277,11 +536,11 @@ where let res = match value_cache_allowed.then(|| cache.lookup_value_for_key(full_key)).flatten() { Some(CachedValue::NonExisting) => None, - Some(CachedValue::ExistingHash(hash)) => { + Some(CachedValue::ExistingHash(hash, location)) => { let data = Self::load_owned_value( // If we only have the hash cached, this can only be a value node. // For inline nodes we cache them directly as `CachedValue::Existing`. - ValueOwned::Node(*hash), + ValueOwned::Node(*hash, *location), nibble_key.original_data_as_prefix(), full_key, cache, @@ -324,33 +583,35 @@ where &mut self, nibble_key: NibbleSlice, full_key: &[u8], - cache: &mut dyn crate::TrieCache, + cache: &mut dyn crate::TrieCache, load_value_owned: impl Fn( - ValueOwned>, + ValueOwned, L::Location>, Prefix, &[u8], - &mut dyn crate::TrieCache, - &dyn HashDBRef, - &mut Option<&mut dyn TrieRecorder>>, + &mut dyn crate::TrieCache, + &dyn NodeDB, + &mut Option<&mut dyn TrieRecorder, L::Location>>, ) -> Result, CError>, ) -> Result, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; + let mut location = self.location; let mut key_nibbles = 0; // this loop iterates through non-inline nodes. for depth in 0.. { - let mut node = cache.get_or_insert_node(hash, &mut || { - let node_data = match self.db.get(&hash, nibble_key.mid(key_nibbles).left()) { - Some(value) => value, - None => - return Err(Box::new(match depth { - 0 => TrieError::InvalidStateRoot(hash), - _ => TrieError::IncompleteDatabase(hash), - })), - }; - - let decoded = match L::Codec::decode(&node_data[..]) { + let mut node = cache.get_or_insert_node(hash, location, &mut || { + let (node_data, locations) = + match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { + Some(value) => value, + None => + return Err(Box::new(match depth { + 0 => TrieError::InvalidStateRoot(hash), + _ => TrieError::IncompleteDatabase(hash), + })), + }; + + let decoded = match L::Codec::decode(&node_data[..], &locations) { Ok(node) => node, Err(e) => return Err(Box::new(TrieError::DecoderError(hash, e))), }; @@ -367,7 +628,6 @@ where NodeOwned::Leaf(slice, value) => return if partial == *slice { let value = (*value).clone(); - drop(node); load_value_owned( value, nibble_key.original_data_as_prefix(), @@ -395,7 +655,6 @@ where NodeOwned::Branch(children, value) => if partial.is_empty() { return if let Some(value) = value.clone() { - drop(node); load_value_owned( value, nibble_key.original_data_as_prefix(), @@ -433,7 +692,6 @@ where if partial.len() == slice.len() { return if let Some(value) = value.clone() { - drop(node); load_value_owned( value, nibble_key.original_data_as_prefix(), @@ -479,8 +737,9 @@ where // check if new node data is inline or hash. match next_node { - NodeHandleOwned::Hash(new_hash) => { + NodeHandleOwned::Hash(new_hash, new_location) => { hash = *new_hash; + location = *new_location; break }, NodeHandleOwned::Inline(inline_node) => { @@ -503,28 +762,30 @@ where nibble_key: NibbleSlice, full_key: &[u8], load_value: impl Fn( - Value, + Value, Prefix, &[u8], - &dyn HashDBRef, - &mut Option<&mut dyn TrieRecorder>>, + &dyn NodeDB, + &mut Option<&mut dyn TrieRecorder, L::Location>>, Q, ) -> Result, CError>, ) -> Result, TrieHash, CError> { let mut partial = nibble_key; let mut hash = self.hash; + let mut location = self.location; let mut key_nibbles = 0; // this loop iterates through non-inline nodes. for depth in 0.. { - let node_data = match self.db.get(&hash, nibble_key.mid(key_nibbles).left()) { - Some(value) => value, - None => - return Err(Box::new(match depth { - 0 => TrieError::InvalidStateRoot(hash), - _ => TrieError::IncompleteDatabase(hash), - })), - }; + let (node_data, locations) = + match self.db.get(&hash, nibble_key.mid(key_nibbles).left(), location) { + Some(value) => value, + None => + return Err(Box::new(match depth { + 0 => TrieError::InvalidStateRoot(hash), + _ => TrieError::IncompleteDatabase(hash), + })), + }; self.record(|| TrieAccess::EncodedNode { hash, @@ -535,7 +796,7 @@ where // without incrementing the depth. let mut node_data = &node_data[..]; loop { - let decoded = match L::Codec::decode(node_data) { + let decoded = match L::Codec::decode(node_data, &locations) { Ok(node) => node, Err(e) => return Err(Box::new(TrieError::DecoderError(hash, e))), }; @@ -585,7 +846,8 @@ where Ok(None) } } else { - match children[partial.at(0) as usize] { + let i = partial.at(0) as usize; + match children[i] { Some(x) => { partial = partial.mid(1); key_nibbles += 1; @@ -622,7 +884,8 @@ where Ok(None) } } else { - match children[partial.at(slice.len()) as usize] { + let i = partial.at(slice.len()) as usize; + match children[i] { Some(x) => { partial = partial.mid(slice.len() + 1); key_nibbles += slice.len() + 1; @@ -645,9 +908,10 @@ where // check if new node data is inline or hash. match next_node { - NodeHandle::Hash(data) => { + NodeHandle::Hash(data, l) => { hash = decode_hash::(data) .ok_or_else(|| Box::new(TrieError::InvalidHash(hash, data.to_vec())))?; + location = l; break }, NodeHandle::Inline(data) => { diff --git a/subtrie/src/mem_tree_db.rs b/subtrie/src/mem_tree_db.rs new file mode 100644 index 00000000..b7c17f0e --- /dev/null +++ b/subtrie/src/mem_tree_db.rs @@ -0,0 +1,367 @@ +// Copyright 2017-2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Flat memory-based `NodeDB` implementation. + +use std::collections::HashMap; + +use crate::{ + node_db::{Hasher, NodeDB, NodeDBMut, Prefix}, + Changeset, NewChangesetNode, +}; + +/// Node location which is just an index into the `nodes` vector. +pub type Location = Option; + +/// Tree based `NodeDB` implementation. +#[derive(Clone)] +pub struct MemTreeDB +where + H: Hasher, +{ + nodes: Vec>, + roots: HashMap, + hashed_null_node: H::Out, + null_node_data: Vec, +} + +#[derive(Clone)] +enum NodeEntry { + Live { key: H, data: Vec, children: Vec, rc: u32 }, + Removed, +} + +impl Default for MemTreeDB +where + H: Hasher, +{ + fn default() -> Self { + Self::from_null_node(&[0u8][..], [0u8][..].into()) + } +} + +impl MemTreeDB +where + H: Hasher, +{ + /// Create a new `MemoryDB` from a given null key/data + pub fn from_null_node(null_key: &[u8], null_node_data: &[u8]) -> Self { + MemTreeDB { + nodes: vec![], + roots: HashMap::default(), + hashed_null_node: H::hash(null_key), + null_node_data: null_node_data.to_vec(), + } + } + + /// Create a new instance of `Self`. + pub fn new(data: &[u8]) -> Self { + Self::from_null_node(data, data.into()) + } + + pub fn clear(&mut self) { + self.nodes.clear(); + self.roots.clear(); + } + + pub fn remove_root(&mut self, key: &H::Out) { + let Some(location) = self.roots.get(key) else { + return; + }; + + if self.remove_tree(*location) { + self.roots.remove(key); + } + } + + pub fn remove_node(&mut self, k: &H::Out) { + let rem_root = self.roots.remove(k); + #[cfg(debug_assertions)] + { + for (i, node) in self.nodes.iter_mut().enumerate() { + if matches!(node, NodeEntry::Live { key, .. } if key == k) { + assert!(rem_root.map(|r| r == i).unwrap_or(false)); + *node = NodeEntry::Removed; + } + } + } + if let Some(rem_root_ix) = rem_root { + self.nodes[rem_root_ix] = NodeEntry::Removed; + } + } + + pub fn test_remove_node(&mut self, k: &H::Out) { + self.roots.remove(k); + for node in self.nodes.iter_mut() { + if matches!(node, NodeEntry::Live { key, .. } if key == k) { + *node = NodeEntry::Removed; + } + } + } + + fn remove_tree(&mut self, location: usize) -> bool { + let entry = self.nodes.get_mut(location).unwrap(); + match entry { + NodeEntry::Live { rc, children, .. } => + if *rc == 1 { + let children = std::mem::take(children); + *entry = NodeEntry::Removed; + for c in children { + self.remove_tree(c); + } + true + } else { + *rc -= 1; + false + }, + NodeEntry::Removed => { + panic!("Accessing removed node"); + }, + } + } + + pub fn is_empty(&self) -> bool { + self.roots.is_empty() + } + + fn apply(&mut self, c: &Changeset) -> usize { + match c { + Changeset::Existing(e) => { + let location = e.location.unwrap_or_else(|| *self.roots.get(&e.hash).unwrap()); + let entry = self.nodes.get_mut(location).unwrap(); + match entry { + NodeEntry::Live { rc, .. } => { + *rc += 1; + }, + NodeEntry::Removed => { + panic!("Accessing removed node"); + }, + }; + location + }, + Changeset::New(n) => { + let children = n.children.iter().map(|c| self.apply(c)).collect(); + self.nodes.push(NodeEntry::Live { + key: n.hash, + data: n.data.clone(), + children, + rc: 1, + }); + self.nodes.len() - 1 + }, + } + } + + pub fn apply_commit(&mut self, commit: Changeset) -> H::Out { + let root = commit.root_hash(); + if root != self.hashed_null_node { + let root = self.apply(&commit); + let key = commit.hash(); + self.roots.insert(*key, root); + } + // In non test use, the root should be store before calling commit (known + // from tree where commit was build from). + if let Changeset::New(NewChangesetNode { removed_keys: Some((_, removed)), .. }) = &commit { + for (k, _) in removed { + self.remove_root(&k); + } + } + root + } +} + +impl NodeDB, Location> for MemTreeDB +where + H: Hasher, +{ + fn get( + &self, + k: &H::Out, + _prefix: Prefix, + location: Location, + ) -> Option<(Vec, Vec)> { + if k == &self.hashed_null_node { + return Some((self.null_node_data.clone(), Default::default())) + } + + let location = match location { + Some(l) => l, + None => + if let Some(l) = self.roots.get(k) { + *l + } else { + return None + }, + }; + match self.nodes.get(location) { + Some(NodeEntry::Live { data, children, key, .. }) => { + assert_eq!(k, key); + Some((data.clone(), children.iter().map(|l| Some(*l)).collect())) + }, + _ => None, + } + } + + fn contains(&self, key: &H::Out, _prefix: Prefix, location: Location) -> bool { + if key == &self.hashed_null_node { + return true; + } + if let Some(l) = location { + l < self.nodes.len() && !matches!(self.nodes[l], NodeEntry::Removed) + } else { + self.roots.contains_key(key) + } + } +} + +impl NodeDBMut, Location> for MemTreeDB +where + H: Hasher, +{ + fn apply_changeset(&mut self, commit: Changeset) -> H::Out { + self.apply_commit(commit) + } +} + +#[cfg(test)] +mod tests { + use super::{MemTreeDB, NodeEntry}; + use crate::{ + keccak_hasher::{KeccakHash, KeccakHasher}, + node_db::{Hasher, NodeDB}, + Changeset, ExistingChangesetNode, NewChangesetNode, + }; + + fn hash(i: u32) -> KeccakHash { + KeccakHasher::hash(&i.to_le_bytes()) + } + + #[test] + fn test_apply_existing_node() { + let mut db = MemTreeDB::::default(); + + // First, apply a new node + let new_node = Changeset::New(NewChangesetNode { + hash: hash(1), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + removed_keys: None, + }); + let new_location = db.apply(&new_node); + + // Then, apply an existing node that refers to the new node + let existing_node = Changeset::Existing(ExistingChangesetNode { + hash: hash(1), + location: Some(new_location), + prefix: Default::default(), + }); + let existing_location = db.apply(&existing_node); + + assert_eq!(existing_location, new_location); + } + + #[test] + fn test_apply_new_node() { + let mut db = MemTreeDB::::default(); + let node = Changeset::New(NewChangesetNode { + hash: KeccakHash::default(), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + removed_keys: None, + }); + let location = db.apply(&node); + assert_eq!(location, db.nodes.len() - 1); + } + + #[test] + fn test_apply_commit() { + let mut db = MemTreeDB::::default(); + let commit = Changeset::New(NewChangesetNode { + hash: KeccakHash::default(), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + removed_keys: None, + }); + db.apply_commit(commit); + assert_eq!(db.roots.len(), 1); + } + + #[test] + fn test_commit_changeset_with_children() { + let mut db = MemTreeDB::::default(); + + // Create two child nodes + let child1 = Changeset::New(NewChangesetNode { + hash: hash(1), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + removed_keys: None, + }); + let child2 = Changeset::New(NewChangesetNode { + hash: hash(2), + prefix: Default::default(), + data: vec![4, 5, 6], + children: vec![], + removed_keys: None, + }); + + // Create a root node that refers to the child nodes + let commit = Changeset::New(NewChangesetNode { + hash: hash(0), + prefix: Default::default(), + data: vec![7, 8, 9], + children: vec![child1, child2], + removed_keys: None, + }); + + db.apply_commit(commit); + + // Check that the root node and child nodes are in the database + assert_eq!(db.nodes.len(), 3); + assert_eq!(db.roots.len(), 1); + } + + #[test] + fn test_get() { + let mut db = MemTreeDB::::default(); + let key = KeccakHash::default(); + db.nodes.push(NodeEntry::Live { + key: key.clone(), + data: vec![1, 2, 3], + children: vec![], + rc: 1, + }); + db.roots.insert(key.clone(), 0); + let result = db.get(&key, Default::default(), None); + assert_eq!(result, Some((vec![1, 2, 3], vec![]))); + } + + #[test] + fn test_contains() { + let mut db = MemTreeDB::::default(); + let key = KeccakHash::default(); + db.nodes.push(NodeEntry::Live { + key: key.clone(), + data: vec![1, 2, 3], + children: vec![], + rc: 1, + }); + db.roots.insert(key.clone(), 0); + assert!(db.contains(&key, Default::default(), None)); + } +} diff --git a/memory-db/src/lib.rs b/subtrie/src/memory_db.rs similarity index 71% rename from memory-db/src/lib.rs rename to subtrie/src/memory_db.rs index 9aff2c54..7f3aad37 100644 --- a/memory-db/src/lib.rs +++ b/subtrie/src/memory_db.rs @@ -12,33 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Reference-counted memory-based `HashDB` implementation. +//! Reference-counted memory-based `NodeDB` implementation. -#![cfg_attr(not(feature = "std"), no_std)] - -#[cfg(not(feature = "std"))] -extern crate alloc; - -use hash_db::{ - AsHashDB, AsPlainDB, HashDB, HashDBRef, Hasher as KeyHasher, MaybeDebug, PlainDB, PlainDBRef, - Prefix, +use crate::{ + node_db::{Hasher as KeyHasher, MaybeDebug, NodeDB, NodeDBMut, Prefix}, + rstd::{cmp::Eq, hash, marker::PhantomData, mem, vec::Vec}, + Changeset, DBValue, }; + #[cfg(feature = "std")] -use std::{ - borrow::Borrow, cmp::Eq, collections::hash_map::Entry, collections::HashMap as Map, hash, - marker::PhantomData, mem, -}; +use std::collections::hash_map::{Entry, HashMap as Map}; #[cfg(not(feature = "std"))] use alloc::collections::btree_map::{BTreeMap as Map, Entry}; -#[cfg(not(feature = "std"))] -use core::{borrow::Borrow, cmp::Eq, hash, marker::PhantomData, mem}; - -#[cfg(not(feature = "std"))] -use alloc::vec::Vec; - -/// Reference-counted memory-based `HashDB` implementation. +/// Reference-counted memory-based `NodeDB` implementation. /// /// Use `new()` to create a new database. Insert items with `insert()`, remove items /// with `remove()`, check for existence with `contains()` and lookup a hash to derive @@ -47,9 +35,12 @@ use alloc::vec::Vec; /// /// # Example /// ```rust -/// use hash_db::{Hasher, HashDB, EMPTY_PREFIX}; -/// use keccak_hasher::KeccakHasher; -/// use memory_db::{MemoryDB, HashKey}; +/// #[cfg(feature = "test_utils")] +/// { +/// use subtrie::node_db::Hasher; +/// use subtrie::node_db::{EMPTY_PREFIX}; +/// use subtrie::keccak_hasher::KeccakHasher; +/// use subtrie::memory_db::{MemoryDB, HashKey}; /// /// let mut m = MemoryDB::, Vec>::default(); /// let d = "Hello world!".as_bytes(); @@ -79,6 +70,7 @@ use alloc::vec::Vec; /// /// m.remove(&k, EMPTY_PREFIX); /// assert!(!m.contains(&k, EMPTY_PREFIX)); +/// } /// ``` pub struct MemoryDB where @@ -114,7 +106,7 @@ where T: Eq + MaybeDebug, { fn eq(&self, other: &MemoryDB) -> bool { - for a in self.data.iter() { + for a in self.data.iter().filter(|(_, (_, rc))| *rc > 0) { match other.data.get(a.0) { Some(v) if v != a.1 => return false, None => return false, @@ -201,43 +193,6 @@ pub fn prefixed_key(key: &H::Out, prefix: Prefix) -> Vec { prefixed_key } -/// Key function that concatenates prefix and hash. -/// This is doing useless computation and should only be -/// used for legacy purpose. -/// It shall be remove in the future. -#[derive(Clone, Debug)] -#[deprecated(since = "0.22.0")] -pub struct LegacyPrefixedKey(PhantomData); - -#[allow(deprecated)] -impl KeyFunction for LegacyPrefixedKey { - type Key = Vec; - - fn key(hash: &H::Out, prefix: Prefix) -> Vec { - legacy_prefixed_key::(hash, prefix) - } -} - -/// Legacy method for db using previous version of prefix encoding. -/// Only for trie radix 16 trie. -#[deprecated(since = "0.22.0")] -pub fn legacy_prefixed_key(key: &H::Out, prefix: Prefix) -> Vec { - let mut prefixed_key = Vec::with_capacity(key.as_ref().len() + prefix.0.len() + 1); - if let Some(last) = prefix.1 { - let mut prev = 0x01u8; - for i in prefix.0.iter() { - prefixed_key.push((prev << 4) + (*i >> 4)); - prev = *i; - } - prefixed_key.push((prev << 4) + (last >> 4)); - } else { - prefixed_key.push(0); - prefixed_key.extend_from_slice(prefix.0); - } - prefixed_key.extend_from_slice(key.as_ref()); - prefixed_key -} - impl Default for MemoryDB where H: KeyHasher, @@ -323,13 +278,12 @@ where /// /// # Examples /// ```rust - /// extern crate hash_db; - /// extern crate keccak_hasher; - /// extern crate memory_db; - /// - /// use hash_db::{Hasher, HashDB, EMPTY_PREFIX}; - /// use keccak_hasher::KeccakHasher; - /// use memory_db::{MemoryDB, HashKey}; + /// #[cfg(feature = "test_utils")] + /// { + /// use subtrie::node_db::Hasher; + /// use subtrie::node_db::{NodeDB, EMPTY_PREFIX}; + /// use subtrie::keccak_hasher::KeccakHasher; + /// use subtrie::memory_db::{MemoryDB, HashKey}; /// /// fn main() { /// let mut m = MemoryDB::, Vec>::default(); @@ -339,6 +293,7 @@ where /// m.clear(); /// assert!(!m.contains(&hash, EMPTY_PREFIX)); /// } + /// } /// ``` pub fn clear(&mut self) { self.data.clear(); @@ -396,78 +351,39 @@ where } } -impl PlainDB for MemoryDB +impl NodeDB for MemoryDB where H: KeyHasher, - T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, - KF: Send + Sync + KeyFunction, - KF::Key: Borrow<[u8]> + for<'a> From<&'a [u8]>, + T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, + KF: KeyFunction + Send + Sync, { - fn get(&self, key: &H::Out) -> Option { - match self.data.get(key.as_ref()) { - Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), - _ => None, - } + fn get(&self, key: &H::Out, prefix: Prefix, _location: L) -> Option<(T, Vec)> { + MemoryDB::get(self, key, prefix).map(|d| (d, Default::default())) } - fn contains(&self, key: &H::Out) -> bool { - match self.data.get(key.as_ref()) { - Some(&(_, x)) if x > 0 => true, - _ => false, - } - } - - fn emplace(&mut self, key: H::Out, value: T) { - match self.data.entry(key.as_ref().into()) { - Entry::Occupied(mut entry) => { - let &mut (ref mut old_value, ref mut rc) = entry.get_mut(); - if *rc <= 0 { - *old_value = value; - } - *rc += 1; - }, - Entry::Vacant(entry) => { - entry.insert((value, 1)); - }, - } - } - - fn remove(&mut self, key: &H::Out) { - match self.data.entry(key.as_ref().into()) { - Entry::Occupied(mut entry) => { - let &mut (_, ref mut rc) = entry.get_mut(); - *rc -= 1; - }, - Entry::Vacant(entry) => { - let value = T::default(); - entry.insert((value, -1)); - }, - } + fn contains(&self, key: &H::Out, prefix: Prefix, _location: L) -> bool { + MemoryDB::contains(self, key, prefix) } } -impl PlainDBRef for MemoryDB +impl NodeDBMut for MemoryDB where H: KeyHasher, - T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, - KF: Send + Sync + KeyFunction, - KF::Key: Borrow<[u8]> + for<'a> From<&'a [u8]>, + KF: KeyFunction + Send + Sync, + L: Default, { - fn get(&self, key: &H::Out) -> Option { - PlainDB::get(self, key) - } - fn contains(&self, key: &H::Out) -> bool { - PlainDB::contains(self, key) + fn apply_changeset(&mut self, commit: Changeset) -> H::Out { + commit.apply_to(self) } } -impl HashDB for MemoryDB +impl MemoryDB where H: KeyHasher, T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, KF: KeyFunction + Send + Sync, { - fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + pub fn get(&self, key: &H::Out, prefix: Prefix) -> Option { if key == &self.hashed_null_node { return Some(self.null_node_data.clone()) } @@ -479,7 +395,7 @@ where } } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + pub fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { if key == &self.hashed_null_node { return true } @@ -491,7 +407,7 @@ where } } - fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T) { + pub fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T) { if value == self.null_node_data { return } @@ -511,17 +427,17 @@ where } } - fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { + pub fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { if T::from(value) == self.null_node_data { return self.hashed_null_node } let key = H::hash(value); - HashDB::emplace(self, key, prefix, value.into()); + self.emplace(key, prefix, value.into()); key } - fn remove(&mut self, key: &H::Out, prefix: Prefix) { + pub fn remove(&mut self, key: &H::Out, prefix: Prefix) { if key == &self.hashed_null_node { return } @@ -540,54 +456,13 @@ where } } -impl HashDBRef for MemoryDB -where - H: KeyHasher, - T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, - KF: KeyFunction + Send + Sync, -{ - fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - HashDB::get(self, key, prefix) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - HashDB::contains(self, key, prefix) - } -} - -impl AsPlainDB for MemoryDB -where - H: KeyHasher, - T: Default + PartialEq + for<'a> From<&'a [u8]> + Clone + Send + Sync, - KF: KeyFunction + Send + Sync, - KF::Key: Borrow<[u8]> + for<'a> From<&'a [u8]>, -{ - fn as_plain_db(&self) -> &dyn PlainDB { - self - } - fn as_plain_db_mut(&mut self) -> &mut dyn PlainDB { - self - } -} - -impl AsHashDB for MemoryDB -where - H: KeyHasher, - T: Default + PartialEq + AsRef<[u8]> + for<'a> From<&'a [u8]> + Clone + Send + Sync, - KF: KeyFunction + Send + Sync, -{ - fn as_hash_db(&self) -> &dyn HashDB { - self - } - fn as_hash_db_mut(&mut self) -> &mut dyn HashDB { - self - } -} - #[cfg(test)] -mod tests { - use super::{HashDB, HashKey, KeyHasher, MemoryDB}; - use hash_db::EMPTY_PREFIX; - use keccak_hasher::KeccakHasher; +mod test { + use crate::{ + keccak_hasher::KeccakHasher, + memory_db::{HashKey, MemoryDB}, + node_db::{Hasher as KeyHasher, EMPTY_PREFIX}, + }; #[test] fn memorydb_remove_and_purge() { diff --git a/trie-db/src/nibble/leftnibbleslice.rs b/subtrie/src/nibble/leftnibbleslice.rs similarity index 100% rename from trie-db/src/nibble/leftnibbleslice.rs rename to subtrie/src/nibble/leftnibbleslice.rs diff --git a/trie-db/src/nibble/mod.rs b/subtrie/src/nibble/mod.rs similarity index 98% rename from trie-db/src/nibble/mod.rs rename to subtrie/src/nibble/mod.rs index e1d758e9..413cbfef 100644 --- a/trie-db/src/nibble/mod.rs +++ b/subtrie/src/nibble/mod.rs @@ -142,7 +142,7 @@ pub mod nibble_ops { } /// Backing storage for `NibbleVec`s. -pub(crate) type BackingByteVec = smallvec::SmallVec<[u8; 40]>; +pub type BackingByteVec = smallvec::SmallVec<[u8; 40]>; /// Owning, nibble-oriented byte vector. Counterpart to `NibbleSlice`. /// Nibbles are always left aligned, so making a `NibbleVec` from diff --git a/trie-db/src/nibble/nibbleslice.rs b/subtrie/src/nibble/nibbleslice.rs similarity index 99% rename from trie-db/src/nibble/nibbleslice.rs rename to subtrie/src/nibble/nibbleslice.rs index f91fad7f..e5e062a9 100644 --- a/trie-db/src/nibble/nibbleslice.rs +++ b/subtrie/src/nibble/nibbleslice.rs @@ -17,8 +17,7 @@ use super::{nibble_ops, BackingByteVec, NibbleSlice, NibbleSliceIterator, NibbleVec}; #[cfg(feature = "std")] use crate::rstd::fmt; -use crate::{node::NodeKey, node_codec::Partial, rstd::cmp::*}; -use hash_db::Prefix; +use crate::{node::NodeKey, node_codec::Partial, node_db::Prefix, rstd::cmp::*}; impl<'a> Iterator for NibbleSliceIterator<'a> { type Item = u8; diff --git a/trie-db/src/nibble/nibblevec.rs b/subtrie/src/nibble/nibblevec.rs similarity index 94% rename from trie-db/src/nibble/nibblevec.rs rename to subtrie/src/nibble/nibblevec.rs index f612585a..c2256a49 100644 --- a/trie-db/src/nibble/nibblevec.rs +++ b/subtrie/src/nibble/nibblevec.rs @@ -19,8 +19,8 @@ use crate::{ nibble::{nibble_ops, BackingByteVec, NibbleSlice}, node::NodeKey, node_codec::Partial, + node_db::Prefix, }; -use hash_db::Prefix; impl Default for NibbleVec { fn default() -> Self { @@ -117,6 +117,12 @@ impl NibbleVec { } } + /// Get `Prefix` representation of this `NibbleVec`. + pub fn as_owned_prefix(&self) -> (BackingByteVec, Option) { + let (inner, pad) = self.as_prefix(); + (inner.into(), pad) + } + /// Append another `NibbleVec`. Can be slow (alignement of second vec). pub fn append(&mut self, v: &NibbleVec) { if v.len == 0 { @@ -235,6 +241,25 @@ impl NibbleVec { true } + /// Same as [`Self::starts_with`] but using [`NibbleSlice`]. + pub fn starts_with_slice(&self, other: &NibbleSlice) -> bool { + if self.len() < other.len() { + return false + } + + match self.as_nibbleslice() { + Some(slice) => slice.starts_with(&other), + None => { + for i in 0..other.len() { + if self.at(i) != other.at(i) { + return false + } + } + true + }, + } + } + /// Return an iterator over `Partial` bytes representation. pub fn right_iter<'a>(&'a self) -> impl Iterator + 'a { let require_padding = self.len % nibble_ops::NIBBLE_PER_BYTE != 0; diff --git a/trie-db/src/node.rs b/subtrie/src/node.rs similarity index 68% rename from trie-db/src/node.rs rename to subtrie/src/node.rs index 19ed9162..d0a36804 100644 --- a/trie-db/src/node.rs +++ b/subtrie/src/node.rs @@ -15,11 +15,11 @@ use crate::{ nibble::{self, nibble_ops, NibbleSlice, NibbleVec}, node_codec::NodeCodec, + node_db::Hasher, Bytes, CError, ChildReference, Result, TrieError, TrieHash, TrieLayout, }; #[cfg(not(feature = "std"))] use alloc::{boxed::Box, vec::Vec}; -use hash_db::Hasher; use crate::rstd::{borrow::Borrow, mem, ops::Range}; @@ -29,22 +29,25 @@ pub type NodeKey = (usize, nibble::BackingByteVec); /// A reference to a trie node which may be stored within another trie node. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum NodeHandle<'a> { - Hash(&'a [u8]), +pub enum NodeHandle<'a, L> { + Hash(&'a [u8], L), Inline(&'a [u8]), } -impl NodeHandle<'_> { +impl<'a, L: Copy + Default> NodeHandle<'a, L> { /// Converts this node handle into a [`NodeHandleOwned`]. - pub fn to_owned_handle( + pub fn to_owned_handle( &self, - ) -> Result>, TrieHash, CError> { + ) -> Result, TL::Location>, TrieHash, CError> + where + TL::Location: From, + { match self { - Self::Hash(h) => decode_hash::(h) + Self::Hash(h, l) => decode_hash::(h) .ok_or_else(|| Box::new(TrieError::InvalidHash(Default::default(), h.to_vec()))) - .map(NodeHandleOwned::Hash), - Self::Inline(i) => match L::Codec::decode(i) { - Ok(node) => Ok(NodeHandleOwned::Inline(Box::new(node.to_owned_node::()?))), + .map(|h| NodeHandleOwned::Hash(h, (*l).into())), + Self::Inline(i) => match TL::Codec::decode(i, &[] as &[L]) { + Ok(node) => Ok(NodeHandleOwned::Inline(Box::new(node.to_owned_node::()?))), Err(e) => Err(Box::new(TrieError::DecoderError(Default::default(), e))), }, } @@ -54,14 +57,15 @@ impl NodeHandle<'_> { /// Owned version of [`NodeHandleOwned`]. #[derive(Clone, PartialEq, Eq)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum NodeHandleOwned { - Hash(H), - Inline(Box>), +pub enum NodeHandleOwned { + Hash(H, L), + Inline(Box>), } -impl NodeHandleOwned +impl NodeHandleOwned where H: Default + AsRef<[u8]> + AsMut<[u8]> + Copy, + L: Default + Copy, { /// Returns `self` as a [`ChildReference`]. /// @@ -69,9 +73,9 @@ where /// /// This function panics if `self == Self::Inline(_)` and the inline node encoded length is /// greater then the length of the hash. - fn as_child_reference>(&self) -> ChildReference { + fn as_child_reference>(&self) -> ChildReference { match self { - NodeHandleOwned::Hash(h) => ChildReference::Hash(*h), + NodeHandleOwned::Hash(h, l) => ChildReference::Hash(*h, *l), NodeHandleOwned::Inline(n) => { let encoded = n.to_encoded::(); let mut store = H::default(); @@ -84,11 +88,11 @@ where } } -impl NodeHandleOwned { +impl NodeHandleOwned { /// Returns `self` as inline node. - pub fn as_inline(&self) -> Option<&NodeOwned> { + pub fn as_inline(&self) -> Option<&NodeOwned> { match self { - Self::Hash(_) => None, + Self::Hash(_, _) => None, Self::Inline(node) => Some(&*node), } } @@ -107,14 +111,14 @@ pub fn decode_hash(data: &[u8]) -> Option { /// Value representation in `Node`. #[derive(Eq, PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum Value<'a> { +pub enum Value<'a, L> { /// Value byte slice as stored in a trie node. Inline(&'a [u8]), /// Hash byte slice as stored in a trie node. - Node(&'a [u8]), + Node(&'a [u8], L), } -impl<'a> Value<'a> { +impl<'a, L: Copy + Default> Value<'a, L> { pub(crate) fn new_inline(value: &'a [u8], threshold: Option) -> Option { if let Some(threshold) = threshold { if value.len() >= threshold as usize { @@ -127,14 +131,17 @@ impl<'a> Value<'a> { } } - pub fn to_owned_value(&self) -> ValueOwned> { + pub fn to_owned_value(&self) -> ValueOwned, TL::Location> + where + TL::Location: From, + { match self { - Self::Inline(data) => ValueOwned::Inline(Bytes::from(*data), L::Hash::hash(data)), - Self::Node(hash) => { - let mut res = TrieHash::::default(); + Self::Inline(data) => ValueOwned::Inline(Bytes::from(*data), TL::Hash::hash(data)), + Self::Node(hash, l) => { + let mut res = TrieHash::::default(); res.as_mut().copy_from_slice(hash); - ValueOwned::Node(res) + ValueOwned::Node(res, (*l).into()) }, } } @@ -143,19 +150,19 @@ impl<'a> Value<'a> { /// Owned value representation in `Node`. #[derive(Eq, PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum ValueOwned { +pub enum ValueOwned { /// Value bytes as stored in a trie node and its hash. Inline(Bytes, H), /// Hash byte slice as stored in a trie node. - Node(H), + Node(H, L), } -impl + Copy> ValueOwned { +impl + Copy, L: Copy + Default> ValueOwned { /// Returns self as [`Value`]. - pub fn as_value(&self) -> Value { + pub fn as_value(&self) -> Value { match self { Self::Inline(data, _) => Value::Inline(&data), - Self::Node(hash) => Value::Node(hash.as_ref()), + Self::Node(hash, location) => Value::Node(hash.as_ref(), *location), } } @@ -163,17 +170,17 @@ impl + Copy> ValueOwned { pub fn data_hash(&self) -> Option { match self { Self::Inline(_, hash) => Some(*hash), - Self::Node(hash) => Some(*hash), + Self::Node(hash, _) => Some(*hash), } } } -impl ValueOwned { +impl ValueOwned { /// Returns the data stored in self. pub fn data(&self) -> Option<&Bytes> { match self { Self::Inline(data, _) => Some(data), - Self::Node(_) => None, + Self::Node(_, _) => None, } } } @@ -181,29 +188,33 @@ impl ValueOwned { /// Type of node in the trie and essential information thereof. #[derive(Eq, PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum Node<'a> { +pub enum Node<'a, L> { /// Null trie node; could be an empty root or an empty branch entry. Empty, /// Leaf node; has key slice and value. Value may not be empty. - Leaf(NibbleSlice<'a>, Value<'a>), + Leaf(NibbleSlice<'a>, Value<'a, L>), /// Extension node; has key slice and node data. Data may not be null. - Extension(NibbleSlice<'a>, NodeHandle<'a>), + Extension(NibbleSlice<'a>, NodeHandle<'a, L>), /// Branch node; has slice of child nodes (each possibly null) /// and an optional immediate node data. - Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), + Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), /// Branch node with support for a nibble (when extension nodes are not used). NibbledBranch( NibbleSlice<'a>, - [Option>; nibble_ops::NIBBLE_LENGTH], - Option>, + [Option>; nibble_ops::NIBBLE_LENGTH], + Option>, ), } -impl Node<'_> { +impl Node<'_, Location> { /// Converts this node into a [`NodeOwned`]. pub fn to_owned_node( &self, - ) -> Result>, TrieHash, CError> { + ) -> Result, L::Location>, TrieHash, CError> + where + L::Location: From, + Location: Copy + Default, + { match self { Self::Empty => Ok(NodeOwned::Empty), Self::Leaf(n, d) => Ok(NodeOwned::Leaf((*n).into(), d.to_owned_value::())), @@ -248,21 +259,21 @@ impl Node<'_> { /// Owned version of [`Node`]. #[derive(Eq, PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum NodeOwned { +pub enum NodeOwned { /// Null trie node; could be an empty root or an empty branch entry. Empty, /// Leaf node; has key slice and value. Value may not be empty. - Leaf(NibbleVec, ValueOwned), + Leaf(NibbleVec, ValueOwned), /// Extension node; has key slice and node data. Data may not be null. - Extension(NibbleVec, NodeHandleOwned), + Extension(NibbleVec, NodeHandleOwned), /// Branch node; has slice of child nodes (each possibly null) /// and an optional immediate node data. - Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), + Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), /// Branch node with support for a nibble (when extension nodes are not used). NibbledBranch( NibbleVec, - [Option>; nibble_ops::NIBBLE_LENGTH], - Option>, + [Option>; nibble_ops::NIBBLE_LENGTH], + Option>, ), /// Node that represents a value. /// @@ -271,7 +282,7 @@ pub enum NodeOwned { Value(Bytes, H), } -impl NodeOwned +impl NodeOwned where H: Default + AsRef<[u8]> + AsMut<[u8]> + Copy, { @@ -304,15 +315,15 @@ where } /// Returns an iterator over all existing children with their optional nibble. - pub fn child_iter(&self) -> impl Iterator, &NodeHandleOwned)> { - enum ChildIter<'a, H> { + pub fn child_iter(&self) -> impl Iterator, &NodeHandleOwned)> { + enum ChildIter<'a, H, L> { Empty, - Single(&'a NodeHandleOwned, bool), - Array(&'a [Option>; nibble_ops::NIBBLE_LENGTH], usize), + Single(&'a NodeHandleOwned, bool), + Array(&'a [Option>; nibble_ops::NIBBLE_LENGTH], usize), } - impl<'a, H> Iterator for ChildIter<'a, H> { - type Item = (Option, &'a NodeHandleOwned); + impl<'a, H, L> Iterator for ChildIter<'a, H, L> { + type Item = (Option, &'a NodeHandleOwned); fn next(&mut self) -> Option { loop { @@ -362,7 +373,7 @@ where } } -impl NodeOwned { +impl NodeOwned { /// Returns the data attached to this node. pub fn data(&self) -> Option<&Bytes> { match &self { @@ -391,8 +402,8 @@ impl NodeOwned { pub fn size_in_bytes(&self) -> usize { let self_size = mem::size_of::(); - fn childs_size<'a, H: 'a>( - childs: impl Iterator>>, + fn childs_size<'a, H: 'a, L: 'a>( + childs: impl Iterator>>, ) -> usize { // If a `child` isn't an inline node, its size is already taken account for by // `self_size`. @@ -429,9 +440,10 @@ impl NodeOwned { /// A `NodeHandlePlan` is a decoding plan for constructing a `NodeHandle` from an encoded trie /// node. This is used as a substructure of `NodePlan`. See `NodePlan` for details. +/// Number of existing node is stored (allow fast access to children locations). #[derive(Debug, Clone, PartialEq, Eq)] pub enum NodeHandlePlan { - Hash(Range), + Hash(Range, u8), Inline(Range), } @@ -439,12 +451,20 @@ impl NodeHandlePlan { /// Build a node handle by decoding a byte slice according to the node handle plan. It is the /// responsibility of the caller to ensure that the node plan was created for the argument /// data, otherwise the call may decode incorrectly or panic. - pub fn build<'a, 'b>(&'a self, data: &'b [u8]) -> NodeHandle<'b> { + pub fn build<'a, 'b, L>(&'a self, data: &'b [u8], location: L) -> NodeHandle<'b, L> { match self { - NodeHandlePlan::Hash(range) => NodeHandle::Hash(&data[range.clone()]), + NodeHandlePlan::Hash(range, _) => NodeHandle::Hash(&data[range.clone()], location), NodeHandlePlan::Inline(range) => NodeHandle::Inline(&data[range.clone()]), } } + + /// Check if the node is innline. + pub fn is_inline(&self) -> bool { + match self { + NodeHandlePlan::Hash(..) => false, + NodeHandlePlan::Inline(..) => true, + } + } } /// A `NibbleSlicePlan` is a blueprint for decoding a nibble slice from a byte slice. The @@ -488,10 +508,18 @@ pub enum ValuePlan { impl ValuePlan { /// Build a value slice by decoding a byte slice according to the plan. - pub fn build<'a, 'b>(&'a self, data: &'b [u8]) -> Value<'b> { + pub fn build<'a, 'b, L>(&'a self, data: &'b [u8], location: L) -> Value<'b, L> { match self { ValuePlan::Inline(range) => Value::Inline(&data[range.clone()]), - ValuePlan::Node(range) => Value::Node(&data[range.clone()]), + ValuePlan::Node(range) => Value::Node(&data[range.clone()], location), + } + } + + /// Check if the value is inline. + pub fn is_inline(&self) -> bool { + match self { + ValuePlan::Inline(_) => true, + ValuePlan::Node(_) => false, } } } @@ -526,36 +554,98 @@ pub enum NodePlan { } impl NodePlan { - /// Build a node by decoding a byte slice according to the node plan. It is the responsibility - /// of the caller to ensure that the node plan was created for the argument data, otherwise the - /// call may decode incorrectly or panic. - pub fn build<'a, 'b>(&'a self, data: &'b [u8]) -> Node<'b> { + /// Build a node by decoding a byte slice according to the node plan and attaching location + /// dats. It is the responsibility of the caller to ensure that the node plan was created for + /// the argument data, otherwise the call may decode incorrectly or panic. + pub fn build<'a, 'b, L: Copy + Default>( + &'a self, + data: &'b [u8], + locations: &[L], + ) -> Node<'b, L> { match self { NodePlan::Empty => Node::Empty, - NodePlan::Leaf { partial, value } => Node::Leaf(partial.build(data), value.build(data)), - NodePlan::Extension { partial, child } => - Node::Extension(partial.build(data), child.build(data)), + NodePlan::Leaf { partial, value } => Node::Leaf( + partial.build(data), + value.build(data, locations.first().copied().unwrap_or_default()), + ), + NodePlan::Extension { partial, child } => Node::Extension( + partial.build(data), + child.build(data, locations.first().copied().unwrap_or_default()), + ), NodePlan::Branch { value, children } => { - let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; - for i in 0..nibble_ops::NIBBLE_LENGTH { - child_slices[i] = children[i].as_ref().map(|child| child.build(data)); - } - Node::Branch(child_slices, value.as_ref().map(|v| v.build(data))) + let (value, child_slices) = + Self::build_value_and_children(value.as_ref(), children, data, locations); + Node::Branch(child_slices, value) }, NodePlan::NibbledBranch { partial, value, children } => { - let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; - for i in 0..nibble_ops::NIBBLE_LENGTH { - child_slices[i] = children[i].as_ref().map(|child| child.build(data)); - } - Node::NibbledBranch( - partial.build(data), - child_slices, - value.as_ref().map(|v| v.build(data)), - ) + let (value, child_slices) = + Self::build_value_and_children(value.as_ref(), children, data, locations); + Node::NibbledBranch(partial.build(data), child_slices, value) }, } } + fn build_value_and_children<'a, 'b, L: Copy + Default>( + value: Option<&'a ValuePlan>, + children: &'a [Option; nibble_ops::NIBBLE_LENGTH], + data: &'b [u8], + locations: &[L], + ) -> (Option>, [Option>; nibble_ops::NIBBLE_LENGTH]) { + let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; + let mut nc = 0; + let value = if let Some(v) = value { + if v.is_inline() { + Some(v.build(data, Default::default())) + } else { + nc += 1; + Some(v.build(data, locations.first().copied().unwrap_or_default())) + } + } else { + None + }; + for i in 0..nibble_ops::NIBBLE_LENGTH { + if let Some(child) = &children[i] { + let location = if child.is_inline() { + Default::default() + } else { + let l = locations.get(nc).copied().unwrap_or_default(); + nc += 1; + l + }; + child_slices[i] = Some(child.build(data, location)); + } + } + (value, child_slices) + } + + pub(crate) fn build_child<'a, 'b, L: Copy + Default>( + value: Option<&'a ValuePlan>, + children: &'a [Option; nibble_ops::NIBBLE_LENGTH], + index: usize, + data: &'b [u8], + locations: &[L], + ) -> Option> { + let mut location_value_offset = 0; + if let Some(v) = value { + if !v.is_inline() { + location_value_offset = 1; + } + } + if let Some(child) = &children[index] { + let location = if let NodeHandlePlan::Hash(_, i_hash) = child { + locations + .get(location_value_offset + *i_hash as usize) + .copied() + .unwrap_or_default() + } else { + Default::default() + }; + Some(child.build(data, location)) + } else { + None + } + } + /// Access value plan from node plan, return `None` for /// node that cannot contain a `ValuePlan`. pub fn value_plan(&self) -> Option<&ValuePlan> { @@ -577,22 +667,58 @@ impl NodePlan { value.as_mut(), } } + + /// Check if the node has a location for value. + pub fn has_location_for_value(&self) -> bool { + self.value_plan().map(|v| !v.is_inline()).unwrap_or(false) + } + + fn num_children_locations(&self) -> usize { + match self { + NodePlan::Extension { child: NodeHandlePlan::Hash(..), .. } => 1, + NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { + let mut count = 0; + for child in children { + if let Some(NodeHandlePlan::Hash(..)) = child { + count += 1; + } + } + count + }, + _ => 0, + } + } + + /// Check if an extra location is defined, it can be attached state. + /// This method is counting and should be call only when needed. + pub fn additional_ref_location(&self, locations: &[L]) -> Option { + let offset = + if self.has_location_for_value() { 1 } else { 0 } + self.num_children_locations(); + if locations.len() > offset { + // only one additional location expected with current code. + debug_assert!(locations.len() == offset + 1); + Some(locations[offset]) + } else { + None + } + } } /// An `OwnedNode` is an owned type from which a `Node` can be constructed which borrows data from /// the `OwnedNode`. This is useful for trie iterators. #[cfg_attr(feature = "std", derive(Debug))] #[derive(PartialEq, Eq)] -pub struct OwnedNode> { +pub struct OwnedNode, L> { data: D, plan: NodePlan, + locations: Vec, } -impl> OwnedNode { +impl, L: Default + Copy> OwnedNode { /// Construct an `OwnedNode` by decoding an owned data source according to some codec. - pub fn new(data: D) -> core::result::Result { + pub fn new(data: D, locations: Vec) -> core::result::Result { let plan = C::decode_plan(data.borrow())?; - Ok(OwnedNode { data, plan }) + Ok(OwnedNode { data, plan, locations }) } /// Returns a reference to the backing data. @@ -600,6 +726,11 @@ impl> OwnedNode { self.data.borrow() } + /// Returns a reference to children locations. + pub fn locations(&self) -> &[L] { + &self.locations + } + /// Returns a reference to the node decode plan. pub fn node_plan(&self) -> &NodePlan { &self.plan @@ -611,7 +742,7 @@ impl> OwnedNode { } /// Construct a `Node` by borrowing data from this struct. - pub fn node(&self) -> Node { - self.plan.build(self.data.borrow()) + pub fn node(&self) -> Node { + self.plan.build(self.data.borrow(), &self.locations) } } diff --git a/trie-db/src/node_codec.rs b/subtrie/src/node_codec.rs similarity index 84% rename from trie-db/src/node_codec.rs rename to subtrie/src/node_codec.rs index eb9b1f67..c4aa2635 100644 --- a/trie-db/src/node_codec.rs +++ b/subtrie/src/node_codec.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Generic trait for trie node encoding/decoding. Takes a `hash_db::Hasher` +//! Generic trait for trie node encoding/decoding. Takes a `trie_db::node_db::Hasher` //! to parametrize the hashes used in the codec. use crate::{ @@ -59,8 +59,11 @@ pub trait NodeCodec: Sized { fn decode_plan(data: &[u8]) -> Result; /// Decode bytes to a `Node`. Returns `Self::E` on failure. - fn decode<'a>(data: &'a [u8]) -> Result, Self::Error> { - Ok(Self::decode_plan(data)?.build(data)) + fn decode<'a, L: Copy + Default>( + data: &'a [u8], + locations: &[L], + ) -> Result, Self::Error> { + Ok(Self::decode_plan(data)?.build(data, locations)) } /// Check if the provided bytes correspond to the codecs "empty" node. @@ -74,32 +77,36 @@ pub trait NodeCodec: Sized { /// Note that number_nibble is the number of element of the iterator /// it can possibly be obtain by `Iterator` `size_hint`, but /// for simplicity it is used directly as a parameter. - fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec; + fn leaf_node( + partial: impl Iterator, + number_nibble: usize, + value: Value, + ) -> Vec; /// Returns an encoded extension node /// /// Note that number_nibble is the number of element of the iterator /// it can possibly be obtain by `Iterator` `size_hint`, but /// for simplicity it is used directly as a parameter. - fn extension_node( + fn extension_node( partial: impl Iterator, number_nibble: usize, - child_ref: ChildReference, + child_ref: ChildReference, ) -> Vec; /// Returns an encoded branch node. /// Takes an iterator yielding `ChildReference` and an optional value. - fn branch_node( - children: impl Iterator>>>, - value: Option, + fn branch_node( + children: impl Iterator>>>, + value: Option>, ) -> Vec; /// Returns an encoded branch node with a possible partial path. /// `number_nibble` is the partial path length as in `extension_node`. - fn branch_node_nibbled( + fn branch_node_nibbled( partial: impl Iterator, number_nibble: usize, - children: impl Iterator>>>, - value: Option, + children: impl Iterator>>>, + value: Option>, ) -> Vec; } diff --git a/subtrie/src/node_db.rs b/subtrie/src/node_db.rs new file mode 100644 index 00000000..95005de0 --- /dev/null +++ b/subtrie/src/node_db.rs @@ -0,0 +1,97 @@ +// Copyright 2017, 2021 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Database of byte-slices keyed to their hash. + +use crate::{ + rstd::{hash, vec::Vec}, + Changeset, +}; + +#[cfg(feature = "std")] +use std::fmt::Debug; + +#[cfg(feature = "std")] +pub trait MaybeDebug: Debug {} +#[cfg(feature = "std")] +impl MaybeDebug for T {} +#[cfg(not(feature = "std"))] +pub trait MaybeDebug {} +#[cfg(not(feature = "std"))] +impl MaybeDebug for T {} + +/// A trie node prefix, it is the nibble path from the trie root +/// to the trie node. +/// For a node containing no partial key value it is the full key. +/// For a value node or node containing a partial key, it is the full key minus its node partial +/// nibbles (the node key can be split into prefix and node partial). +/// Therefore it is always the leftmost portion of the node key, so its internal representation +/// is a non expanded byte slice followed by a last padded byte representation. +/// The padded byte is an optional padded value. +pub type Prefix<'a> = (&'a [u8], Option); + +/// An empty prefix constant. +/// Can be use when the prefix is not use internally +/// or for root nodes. +pub static EMPTY_PREFIX: Prefix<'static> = (&[], None); + +/// Trait describing an object that can hash a slice of bytes. Used to abstract +/// other types over the hashing algorithm. Defines a single `hash` method and an +/// `Out` associated type with the necessary bounds. +pub trait Hasher: Sync + Send { + /// The output type of the `Hasher` + type Out: AsRef<[u8]> + + AsMut<[u8]> + + Default + + MaybeDebug + + core::cmp::Ord + + PartialEq + + Eq + + hash::Hash + + Send + + Sync + + Clone + + Copy; + /// What to use to build `HashMap`s with this `Hasher`. + type StdHasher: Sync + Send + Default + hash::Hasher; + /// The length in bytes of the `Hasher` output. + const LENGTH: usize; + + /// Compute the hash of the provided slice of bytes returning the `Out` type of the `Hasher`. + fn hash(x: &[u8]) -> Self::Out; +} + +/// Trait modelling datastore keyed by a hash defined by the `Hasher` and optional location tag. +pub trait NodeDB: Send + Sync { + /// Look up a trie node by hash and location. + /// Returns the node bytes and the list of children node locations if any. + fn get(&self, key: &H::Out, prefix: Prefix, location: L) -> Option<(T, Vec)>; + + /// Check for the existence of a hash-key at the location. + fn contains(&self, key: &H::Out, prefix: Prefix, location: L) -> bool { + self.get(key, prefix, location).is_some() + } + + /// Compute value hash. + fn hash(&self, value: &[u8]) -> H::Out { + H::hash(value) + } +} + +/// Trait for node db that can get update by a CommitSet. +/// Mostly usefull for testing. +pub trait NodeDBMut: NodeDB { + /// Insert commit set to the db. + fn apply_changeset(&mut self, commit: Changeset) -> H::Out; +} diff --git a/trie-db/src/proof/generate.rs b/subtrie/src/proof/generate.rs similarity index 79% rename from trie-db/src/proof/generate.rs rename to subtrie/src/proof/generate.rs index 2db1eafa..a874e0e2 100644 --- a/trie-db/src/proof/generate.rs +++ b/subtrie/src/proof/generate.rs @@ -16,22 +16,22 @@ use crate::rstd::{boxed::Box, convert::TryInto, marker::PhantomData, vec, vec::Vec}; -use hash_db::{HashDBRef, Hasher}; +use crate::node_db::{Hasher, NodeDB}; use crate::{ nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, - node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, Value, ValuePlan}, + node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, Value, ValuePlan}, recorder::Record, CError, ChildReference, DBValue, NibbleSlice, NodeCodec, Recorder, Result as TrieResult, Trie, TrieDBBuilder, TrieError, TrieHash, TrieLayout, }; -struct StackEntry<'a, C: NodeCodec> { +struct StackEntry<'a, C: NodeCodec, L> { /// The prefix is the nibble path to the node in the trie. prefix: LeftNibbleSlice<'a>, /// Stacked node. - node: OwnedNode>, + node: OwnedNode, L>, /// The hash of the node or None if it is referenced inline. node_hash: Option, /// Whether the value should be omitted in the generated proof. @@ -40,20 +40,21 @@ struct StackEntry<'a, C: NodeCodec> { /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, /// The child references to use in constructing the proof nodes. - children: Vec>>, + children: Vec>>, /// The index into the proof vector that the encoding of this entry should be placed at. output_index: Option, _marker: PhantomData, } -impl<'a, C: NodeCodec> StackEntry<'a, C> { +impl<'a, C: NodeCodec, L: Copy + Default> StackEntry<'a, C, L> { fn new( prefix: LeftNibbleSlice<'a>, node_data: Vec, + locations: Vec, node_hash: Option, output_index: Option, ) -> TrieResult { - let node = OwnedNode::new::(node_data) + let node = OwnedNode::new::(node_data, locations) .map_err(|err| Box::new(TrieError::DecoderError(node_hash.unwrap_or_default(), err)))?; let children_len = match node.node_plan() { NodePlan::Empty | NodePlan::Leaf { .. } => 0, @@ -76,9 +77,9 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { fn encode_node(mut self) -> TrieResult, C::HashOut, C::Error> { let omit_value = self.omit_value; let node_data = self.node.data(); - let value_with_omission = |value_range: ValuePlan| -> Option { + let value_with_omission = |value_range: ValuePlan| -> Option> { if !omit_value { - Some(value_range.build(&node_data)) + Some(value_range.build(&node_data, Default::default())) } else { None } @@ -87,12 +88,12 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { NodePlan::Empty => node_data.to_vec(), NodePlan::Leaf { .. } if !omit_value => node_data.to_vec(), NodePlan::Leaf { partial, value: _ } => { - let partial = partial.build(node_data); - C::leaf_node(partial.right_iter(), partial.len(), Value::Inline(&[])) + let partial = partial.build(&node_data); + C::leaf_node::(partial.right_iter(), partial.len(), Value::Inline(&[])) }, NodePlan::Extension { .. } if self.child_index == 0 => node_data.to_vec(), NodePlan::Extension { partial: partial_plan, child: _ } => { - let partial = partial_plan.build(node_data); + let partial = partial_plan.build(&node_data); let child = self.children[0].expect( "for extension nodes, children[0] is guaranteed to be Some when \ child_index > 0; \ @@ -102,8 +103,8 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { }, NodePlan::Branch { value, children } => { Self::complete_branch_children( - node_data, - children, + &node_data, + &children, self.child_index, &mut self.children, )?; @@ -113,10 +114,10 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { ) }, NodePlan::NibbledBranch { partial: partial_plan, value, children } => { - let partial = partial_plan.build(node_data); + let partial = partial_plan.build(&node_data); Self::complete_branch_children( - node_data, - children, + &node_data, + &children, self.child_index, &mut self.children, )?; @@ -140,13 +141,13 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { node_data: &[u8], child_handles: &[Option; NIBBLE_LENGTH], child_index: usize, - children: &mut [Option>], + children: &mut [Option>], ) -> TrieResult<(), C::HashOut, C::Error> { for i in child_index..NIBBLE_LENGTH { children[i] = child_handles[i] .as_ref() .map(|child_plan| { - child_plan.build(node_data).try_into().map_err(|hash| { + child_plan.build(node_data, Default::default()).try_into().map_err(|hash| { Box::new(TrieError::InvalidHash(C::HashOut::default(), hash)) }) }) @@ -172,7 +173,7 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { set_child is called when the only child is popped from the stack; \ child_index is 0 before child is pushed to the stack; qed" ); - Some(Self::replacement_child_ref(encoded_child, child)) + Some(Self::replacement_child_ref(encoded_child, &child)) }, NodePlan::Branch { children, .. } | NodePlan::NibbledBranch { children, .. } => { assert!( @@ -196,10 +197,10 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { fn replacement_child_ref( encoded_child: &[u8], child: &NodeHandlePlan, - ) -> ChildReference { + ) -> ChildReference { match child { - NodeHandlePlan::Hash(_) => ChildReference::Inline(C::HashOut::default(), 0), - NodeHandlePlan::Inline(_) => { + NodeHandlePlan::Hash(..) => ChildReference::Inline(C::HashOut::default(), 0), + NodeHandlePlan::Inline(..) => { let mut hash = C::HashOut::default(); assert!( encoded_child.len() <= hash.as_ref().len(), @@ -224,7 +225,7 @@ pub fn generate_proof<'a, D, L, I, K>( keys: I, ) -> TrieResult>, TrieHash, CError> where - D: HashDBRef, + D: NodeDB, L: TrieLayout, I: IntoIterator, K: 'a + AsRef<[u8]>, @@ -236,7 +237,7 @@ where // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. - let mut stack = >>::new(); + let mut stack = >>::new(); // The mutated trie nodes comprising the final proof. let mut proof_nodes = Vec::new(); @@ -274,9 +275,8 @@ where loop { let step = match stack.last_mut() { - Some(entry) => match_key_to_node::( - entry.node.data(), - entry.node.node_plan(), + Some(entry) => match_key_to_node::( + &entry.node, &mut entry.omit_value, &mut entry.child_index, &mut entry.children, @@ -285,15 +285,17 @@ where &mut recorded_nodes, )?, // If stack is empty, descend into the root node. - None => - Step::Descend { child_prefix_len: 0, child: NodeHandle::Hash(root.as_ref()) }, + None => Step::Descend { + child_prefix_len: 0, + child: NodeHandle::Hash(root.as_ref(), Default::default()), + }, }; match step { Step::Descend { child_prefix_len, child } => { let child_prefix = key.truncate(child_prefix_len); let child_entry = match child { - NodeHandle::Hash(hash) => { + NodeHandle::Hash(hash, _) => { let child_record = recorded_nodes.next().expect( "this function's trie traversal logic mirrors that of Lookup; \ thus the sequence of traversed nodes must be the same; \ @@ -310,6 +312,7 @@ where StackEntry::new( child_prefix, child_record.data, + Vec::new(), Some(child_record.hash), Some(output_index), )? @@ -321,7 +324,7 @@ where data.to_vec(), ))) } - StackEntry::new(child_prefix, data.to_vec(), None, None)? + StackEntry::new(child_prefix, data.to_vec(), Vec::new(), None, None)? }, }; stack.push(child_entry); @@ -330,7 +333,7 @@ where assert_eq!( Some(&value), expected_value.as_ref(), - "expected_value is found using `trie_db::Lookup`; \ + "expected_value is found using `subtrie::Lookup`; \ value is found by traversing the same nodes recorded during the lookup \ using the same logic; \ thus the values found must be equal" @@ -349,7 +352,7 @@ where assert_eq!( value, expected_value.as_ref().map(|v| v.as_ref()), - "expected_value is found using `trie_db::Lookup`; \ + "expected_value is found using `subtrie::Lookup`; \ value is found by traversing the same nodes recorded during the lookup \ using the same logic; \ thus the values found must be equal" @@ -368,19 +371,19 @@ where } } - unwind_stack::(&mut stack, &mut proof_nodes, None)?; + unwind_stack::(&mut stack, &mut proof_nodes, None)?; Ok(proof_nodes) } -enum Step<'a> { - Descend { child_prefix_len: usize, child: NodeHandle<'a> }, +enum Step<'a, L> { + Descend { child_prefix_len: usize, child: NodeHandle<'a, L> }, FoundValue(Option<&'a [u8]>), FoundHashedValue(Vec), } -fn resolve_value( +fn resolve_value( recorded_nodes: &mut dyn Iterator>, -) -> TrieResult, C::HashOut, C::Error> { +) -> TrieResult, C::HashOut, C::Error> { if let Some(resolve_value) = recorded_nodes.next() { Ok(Step::FoundHashedValue(resolve_value.data)) } else { @@ -390,49 +393,44 @@ fn resolve_value( /// Determine the next algorithmic step to take by matching the current key against the current top /// entry on the stack. -fn match_key_to_node<'a, C: NodeCodec>( - node_data: &'a [u8], - node_plan: &NodePlan, +fn match_key_to_node<'a, C: NodeCodec, L: Copy + Default>( + node: &'a OwnedNode, L>, omit_value: &mut bool, child_index: &mut usize, - children: &mut [Option>], - key: &LeftNibbleSlice, + children: &mut [Option>], + key: &'a LeftNibbleSlice, prefix_len: usize, recorded_nodes: &mut dyn Iterator>, -) -> TrieResult, C::HashOut, C::Error> { - Ok(match node_plan { - NodePlan::Empty => Step::FoundValue(None), - NodePlan::Leaf { partial: partial_plan, value: value_range } => { - let partial = partial_plan.build(node_data); +) -> TrieResult, C::HashOut, C::Error> { + let node = node.node(); + Ok(match node { + Node::Empty => Step::FoundValue(None), + Node::Leaf(partial, value) => { if key.contains(&partial, prefix_len) && key.len() == prefix_len + partial.len() { - match value_range { - ValuePlan::Inline(value_range) => { + match value { + Value::Inline(data) => { *omit_value = true; - Step::FoundValue(Some(&node_data[value_range.clone()])) + Step::FoundValue(Some(data)) }, - ValuePlan::Node(..) => { + Value::Node(..) => { *omit_value = true; - resolve_value::(recorded_nodes)? + resolve_value::(recorded_nodes)? }, } } else { Step::FoundValue(None) } }, - NodePlan::Extension { partial: partial_plan, child: child_plan } => { - let partial = partial_plan.build(node_data); + Node::Extension(partial, child) => if key.contains(&partial, prefix_len) { assert_eq!(*child_index, 0); let child_prefix_len = prefix_len + partial.len(); - let child = child_plan.build(&node_data); Step::Descend { child_prefix_len, child } } else { Step::FoundValue(None) - } - }, - NodePlan::Branch { value, children: child_handles } => match_key_to_branch_node::( - node_data, - value.as_ref(), + }, + Node::Branch(child_handles, value) => match_key_to_branch_node::( + value.clone(), &child_handles, omit_value, child_index, @@ -442,47 +440,44 @@ fn match_key_to_node<'a, C: NodeCodec>( NibbleSlice::new(&[]), recorded_nodes, )?, - NodePlan::NibbledBranch { partial: partial_plan, value, children: child_handles } => - match_key_to_branch_node::( - node_data, - value.as_ref(), - &child_handles, - omit_value, - child_index, - children, - key, - prefix_len, - partial_plan.build(node_data), - recorded_nodes, - )?, + Node::NibbledBranch(partial, child_handles, value) => match_key_to_branch_node::( + value.clone(), + &child_handles, + omit_value, + child_index, + children, + key, + prefix_len, + partial, + recorded_nodes, + )?, }) } -fn match_key_to_branch_node<'a, 'b, C: NodeCodec>( - node_data: &'a [u8], - value_range: Option<&'b ValuePlan>, - child_handles: &'b [Option; NIBBLE_LENGTH], +fn match_key_to_branch_node<'a, 'b, C: NodeCodec, L: Copy + Default>( + value: Option>, + child_handles: &'b [Option>; NIBBLE_LENGTH], omit_value: &mut bool, child_index: &mut usize, - children: &mut [Option>], - key: &'b LeftNibbleSlice<'b>, + children: &mut [Option>], + key: &'b LeftNibbleSlice<'a>, prefix_len: usize, - partial: NibbleSlice<'b>, + partial: NibbleSlice<'a>, recorded_nodes: &mut dyn Iterator>, -) -> TrieResult, C::HashOut, C::Error> { +) -> TrieResult, C::HashOut, C::Error> { if !key.contains(&partial, prefix_len) { return Ok(Step::FoundValue(None)) } if key.len() == prefix_len + partial.len() { - let value = match value_range { - Some(ValuePlan::Inline(range)) => { + let value = match value { + Some(Value::Inline(data)) => { *omit_value = true; - Some(&node_data[range.clone()]) + Some(data) }, - Some(ValuePlan::Node(..)) => { + Some(Value::Node(_, _)) => { *omit_value = true; - return resolve_value::(recorded_nodes) + return resolve_value::(recorded_nodes) }, None => None, }; @@ -499,20 +494,17 @@ fn match_key_to_branch_node<'a, 'b, C: NodeCodec>( while *child_index < new_index { children[*child_index] = child_handles[*child_index] .as_ref() - .map(|child_plan| { - child_plan - .build(node_data) + .map(|child| { + child + .clone() .try_into() .map_err(|hash| Box::new(TrieError::InvalidHash(C::HashOut::default(), hash))) }) .transpose()?; *child_index += 1; } - if let Some(child_plan) = &child_handles[*child_index] { - Ok(Step::Descend { - child_prefix_len: prefix_len + partial.len() + 1, - child: child_plan.build(node_data), - }) + if let Some(child) = &child_handles[*child_index] { + Ok(Step::Descend { child_prefix_len: prefix_len + partial.len() + 1, child: child.clone() }) } else { Ok(Step::FoundValue(None)) } @@ -521,8 +513,8 @@ fn match_key_to_branch_node<'a, 'b, C: NodeCodec>( /// Unwind the stack until the given key is prefixed by the entry at the top of the stack. If the /// key is None, unwind the stack completely. As entries are popped from the stack, they are /// encoded into proof nodes and added to the finalized proof. -fn unwind_stack( - stack: &mut Vec>, +fn unwind_stack( + stack: &mut Vec>, proof_nodes: &mut Vec>, maybe_key: Option<&LeftNibbleSlice>, ) -> TrieResult<(), C::HashOut, C::Error> { diff --git a/trie-db/src/proof/mod.rs b/subtrie/src/proof/mod.rs similarity index 100% rename from trie-db/src/proof/mod.rs rename to subtrie/src/proof/mod.rs diff --git a/trie-db/src/proof/verify.rs b/subtrie/src/proof/verify.rs similarity index 96% rename from trie-db/src/proof/verify.rs rename to subtrie/src/proof/verify.rs index fedd0579..d0e0cde1 100644 --- a/trie-db/src/proof/verify.rs +++ b/subtrie/src/proof/verify.rs @@ -18,10 +18,10 @@ use crate::{ nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, Value}, + node_db::Hasher, rstd::{convert::TryInto, iter::Peekable, marker::PhantomData, result::Result, vec, vec::Vec}, CError, ChildReference, NodeCodec, TrieHash, TrieLayout, }; -use hash_db::Hasher; /// Errors that may occur during proof verification. Most of the errors types simply indicate that /// the proof is invalid with respect to the statement being verified, and the exact error type can @@ -89,15 +89,15 @@ impl std::error::Error for struct StackEntry<'a, L: TrieLayout> { /// The prefix is the nibble path to the node in the trie. prefix: LeftNibbleSlice<'a>, - node: Node<'a>, + node: Node<'a, ()>, is_inline: bool, /// The value associated with this trie node. - value: Option>, + value: Option>, /// The next entry in the stack is a child of the preceding entry at this index. For branch /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, /// The child references to use in reconstructing the trie nodes. - children: Vec>>>, + children: Vec, ()>>>, /// Technical to attach lifetime to entry. next_value_hash: Option>, _marker: PhantomData, @@ -109,7 +109,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { prefix: LeftNibbleSlice<'a>, is_inline: bool, ) -> Result, CError>> { - let node = L::Codec::decode(&node_data[..]).map_err(Error::DecodeError)?; + let node = L::Codec::decode(&node_data[..], &[]).map_err(Error::DecodeError)?; let children_len = match &node { Node::Empty | Node::Leaf(..) => 0, Node::Extension(..) => 1, @@ -132,9 +132,9 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { }) } - fn value(&self) -> Option { + fn value(&self) -> Option> { if let Some(hash) = self.next_value_hash.as_ref() { - Some(Value::Node(hash.as_ref())) + Some(Value::Node(hash.as_ref(), ())) } else { self.value.clone() } @@ -226,7 +226,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { fn make_child_entry( proof_iter: &mut I, - child: NodeHandle<'a>, + child: NodeHandle<'a, ()>, prefix: LeftNibbleSlice<'a>, ) -> Result, CError>> where @@ -240,7 +240,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { } else { StackEntry::new(data, prefix, true) }, - NodeHandle::Hash(data) => { + NodeHandle::Hash(data, _) => { let mut hash = TrieHash::::default(); if data.len() != hash.as_ref().len() { return Err(Error::InvalidChildReference(data.to_vec())) @@ -323,7 +323,7 @@ enum ValueMatch<'a> { fn match_key_to_node<'a>( key: &LeftNibbleSlice<'a>, prefix_len: usize, - node: &Node, + node: &Node<()>, ) -> ValueMatch<'a> { match node { Node::Empty => ValueMatch::NotFound, @@ -365,8 +365,8 @@ fn match_key_to_node<'a>( fn match_key_to_branch_node<'a>( key: &LeftNibbleSlice<'a>, prefix_plus_partial_len: usize, - children: &[Option; NIBBLE_LENGTH], - value: Option<&Value>, + children: &[Option>; NIBBLE_LENGTH], + value: Option<&Value<()>>, ) -> ValueMatch<'a> { if key.len() == prefix_plus_partial_len { if value.is_none() { @@ -455,7 +455,7 @@ where ChildReference::Inline(hash, node_data.len()) } else { let hash = L::Hash::hash(&node_data); - ChildReference::Hash(hash) + ChildReference::Hash(hash, ()) }; if let Some(entry) = stack.pop() { @@ -467,7 +467,7 @@ where return Err(Error::ExtraneousNode) } let computed_root = match child_ref { - ChildReference::Hash(hash) => hash, + ChildReference::Hash(hash, _) => hash, ChildReference::Inline(_, _) => panic!("the bottom item on the stack has is_inline = false; qed"), }; diff --git a/trie-db/src/recorder.rs b/subtrie/src/recorder.rs similarity index 79% rename from trie-db/src/recorder.rs rename to subtrie/src/recorder.rs index ba4c77e5..1c081d7c 100644 --- a/trie-db/src/recorder.rs +++ b/subtrie/src/recorder.rs @@ -14,8 +14,10 @@ //! Trie query recorder. -use crate::{rstd::vec::Vec, RecordedForKey, TrieAccess, TrieHash, TrieLayout, TrieRecorder}; -use hashbrown::HashMap; +use crate::{ + rstd::{vec::Vec, BTreeMap}, + RecordedForKey, TrieAccess, TrieHash, TrieLayout, TrieRecorder, +}; /// The record of a visited node. #[cfg_attr(feature = "std", derive(Debug))] @@ -31,7 +33,7 @@ pub struct Record { #[cfg_attr(feature = "std", derive(Debug))] pub struct Recorder { nodes: Vec>>, - recorded_keys: HashMap, RecordedForKey>, + recorded_keys: BTreeMap, RecordedForKey>, } impl Default for Recorder { @@ -53,8 +55,8 @@ impl Recorder { } } -impl TrieRecorder> for Recorder { - fn record<'a>(&mut self, access: TrieAccess<'a, TrieHash>) { +impl TrieRecorder, L::Location> for Recorder { + fn record<'a>(&mut self, access: TrieAccess<'a, TrieHash, L::Location>) { match access { TrieAccess::EncodedNode { hash, encoded_node, .. } => { self.nodes.push(Record { hash, data: encoded_node.to_vec() }); @@ -64,14 +66,17 @@ impl TrieRecorder> for Recorder { }, TrieAccess::Value { hash, value, full_key } => { self.nodes.push(Record { hash, data: value.to_vec() }); - self.recorded_keys.entry(full_key.to_vec()).insert(RecordedForKey::Value); + self.recorded_keys.insert(full_key.to_vec(), RecordedForKey::Value); }, TrieAccess::Hash { full_key } => { self.recorded_keys.entry(full_key.to_vec()).or_insert(RecordedForKey::Hash); }, TrieAccess::NonExisting { full_key } => { // We handle the non existing value/hash like having recorded the value. - self.recorded_keys.entry(full_key.to_vec()).insert(RecordedForKey::Value); + self.recorded_keys.insert(full_key.to_vec(), RecordedForKey::Value); + }, + TrieAccess::InlineValue { full_key } => { + self.recorded_keys.insert(full_key.to_vec(), RecordedForKey::Value); }, } } diff --git a/test-support/trie-standardmap/src/lib.rs b/subtrie/src/test_utils.rs similarity index 96% rename from test-support/trie-standardmap/src/lib.rs rename to subtrie/src/test_utils.rs index 8e01ad12..8e503bdd 100644 --- a/test-support/trie-standardmap/src/lib.rs +++ b/subtrie/src/test_utils.rs @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Key-value datastore with a modified Merkle tree. +//! Test utilities. -use hash_db::Hasher; -use keccak_hasher::KeccakHasher; +use crate::{keccak_hasher::KeccakHasher, node_db::Hasher}; -type H256 = ::Out; +type H256 = ::Out; /// Alphabet to use when creating words for insertion into tries. pub enum Alphabet { diff --git a/trie-db/src/trie_codec.rs b/subtrie/src/trie_codec.rs similarity index 87% rename from trie-db/src/trie_codec.rs rename to subtrie/src/trie_codec.rs index 9a1f51b3..c401f55c 100644 --- a/trie-db/src/trie_codec.rs +++ b/subtrie/src/trie_codec.rs @@ -26,21 +26,22 @@ //! trie. use crate::{ + memory_db::{HashKey, MemoryDB}, nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, ValuePlan}, + node_db::Prefix, rstd::{boxed::Box, convert::TryInto, marker::PhantomData, result, sync::Arc, vec, vec::Vec}, CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, TrieDB, TrieDBRawIterator, TrieError, TrieHash, TrieLayout, }; -use hash_db::{HashDB, Prefix}; -const OMIT_VALUE_HASH: crate::node::Value<'static> = crate::node::Value::Inline(&[]); +const OMIT_VALUE_HASH: crate::node::Value<'static, ()> = crate::node::Value::Inline(&[]); -struct EncoderStackEntry { +struct EncoderStackEntry { /// The prefix is the nibble path to the node in the trie. prefix: NibbleVec, /// Node in memory content. - node: Arc>, + node: Arc>, /// The next entry in the stack is a child of the preceding entry at this index. For branch /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, @@ -54,7 +55,7 @@ struct EncoderStackEntry { _marker: PhantomData, } -impl EncoderStackEntry { +impl EncoderStackEntry { /// Given the prefix of the next child node, identify its index and advance `child_index` to /// that. For a given entry, this must be called sequentially only with strictly increasing /// child prefixes. Returns an error if the child prefix is not a child of this entry or if @@ -116,14 +117,14 @@ impl EncoderStackEntry { node_data.to_vec() } else { let partial = partial.build(node_data); - let empty_child = ChildReference::Inline(C::HashOut::default(), 0); + let empty_child = ChildReference::<_, ()>::Inline(C::HashOut::default(), 0); C::extension_node(partial.right_iter(), partial.len(), empty_child) }, NodePlan::Branch { value, children } => { let value = if self.omit_value { value.is_some().then_some(OMIT_VALUE_HASH) } else { - value.as_ref().map(|v| v.build(node_data)) + value.as_ref().map(|v| v.build(node_data, ())) }; C::branch_node( Self::branch_children(node_data, &children, &self.omit_children)?.iter(), @@ -135,7 +136,7 @@ impl EncoderStackEntry { let value = if self.omit_value { value.is_some().then_some(OMIT_VALUE_HASH) } else { - value.as_ref().map(|v| v.build(node_data)) + value.as_ref().map(|v| v.build(node_data, ())) }; C::branch_node_nibbled( partial.right_iter(), @@ -165,14 +166,14 @@ impl EncoderStackEntry { node_data: &[u8], child_handles: &[Option; NIBBLE_LENGTH], omit_children: &[bool], - ) -> Result<[Option>; NIBBLE_LENGTH], C::HashOut, C::Error> { + ) -> Result<[Option>; NIBBLE_LENGTH], C::HashOut, C::Error> { let empty_child = ChildReference::Inline(C::HashOut::default(), 0); let mut children = [None; NIBBLE_LENGTH]; for i in 0..NIBBLE_LENGTH { children[i] = if omit_children[i] { Some(empty_child) } else if let Some(child_plan) = &child_handles[i] { - let child_ref = child_plan.build(node_data).try_into().map_err(|hash| { + let child_ref = child_plan.build(node_data, ()).try_into().map_err(|hash| { Box::new(TrieError::InvalidHash(C::HashOut::default(), hash)) })?; Some(child_ref) @@ -192,13 +193,17 @@ fn detached_value( value: &ValuePlan, node_data: &[u8], node_prefix: Prefix, + location: L::Location, ) -> Option> { let fetched; match value { ValuePlan::Node(hash_plan) => { - if let Ok(value) = - TrieDBRawIterator::fetch_value(db, &node_data[hash_plan.clone()], node_prefix) - { + if let Ok(value) = TrieDBRawIterator::fetch_value( + db, + &node_data[hash_plan.clone()], + node_prefix, + location, + ) { fetched = value; } else { return None @@ -224,7 +229,7 @@ where // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. - let mut stack: Vec> = Vec::new(); + let mut stack: Vec> = Vec::new(); // TrieDBRawIterator guarantees that: // - It yields at least one node. @@ -236,7 +241,7 @@ where // iteration of the loop below, the stack always has at least one entry and the bottom (front) // of the stack is the root node, which is not inline. Furthermore, the iterator is not empty, // so at least one iteration always occurs. - while let Some(item) = iter.next_raw_item(db) { + while let Some(item) = iter.next_raw_item(db, true) { match item { Ok((prefix, node_hash, node)) => { // Skip inline nodes, as they cannot contain hash references to other nodes by @@ -268,12 +273,28 @@ where let (children_len, detached_value) = match node.node_plan() { NodePlan::Empty => (0, None), - NodePlan::Leaf { value, .. } => - (0, detached_value(db, value, node.data(), prefix.as_prefix())), + NodePlan::Leaf { value, .. } => ( + 0, + detached_value( + db, + &value, + node.data(), + prefix.as_prefix(), + node.locations().first().copied().unwrap_or_default(), + ), + ), NodePlan::Extension { .. } => (1, None), NodePlan::NibbledBranch { value: Some(value), .. } | - NodePlan::Branch { value: Some(value), .. } => - (NIBBLE_LENGTH, detached_value(db, value, node.data(), prefix.as_prefix())), + NodePlan::Branch { value: Some(value), .. } => ( + NIBBLE_LENGTH, + detached_value( + db, + &value, + node.data(), + prefix.as_prefix(), + node.locations().last().copied().unwrap_or_default(), + ), + ), NodePlan::NibbledBranch { value: None, .. } | NodePlan::Branch { value: None, .. } => (NIBBLE_LENGTH, None), }; @@ -311,19 +332,19 @@ where Ok(output) } -struct DecoderStackEntry<'a, C: NodeCodec> { - node: Node<'a>, +struct DecoderStackEntry<'a, C: NodeCodec, L> { + node: Node<'a, L>, /// The next entry in the stack is a child of the preceding entry at this index. For branch /// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in [0, 1]. child_index: usize, /// The reconstructed child references. - children: Vec>>, + children: Vec>>, /// A value attached as a node. The node will need to use its hash as value. attached_value: Option<&'a [u8]>, _marker: PhantomData, } -impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { +impl<'a, C: NodeCodec, L: Copy + Default> DecoderStackEntry<'a, C, L> { /// Advance the child index until either it exceeds the number of children or the child is /// marked as omitted. Omitted children are indicated by an empty inline reference. For each /// child that is passed over and not omitted, copy over the child reference from the node to @@ -407,7 +428,7 @@ impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { /// Preconditions: /// - if node is an extension node, then `children[0]` is Some. fn encode_node(self, attached_hash: Option<&[u8]>) -> Vec { - let attached_hash = attached_hash.map(|h| crate::node::Value::Node(h)); + let attached_hash = attached_hash.map(|h| crate::node::Value::Node(h, Default::default())); match self.node { Node::Empty => C::empty_node().to_vec(), Node::Leaf(partial, value) => @@ -433,40 +454,38 @@ impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { /// Reconstructs a partial trie DB from a compact representation. The encoding is a vector of /// mutated trie nodes with those child references omitted. The decode function reads them in order -/// from the given slice, reconstructing the full nodes and inserting them into the given `HashDB`. -/// It stops after fully constructing one partial trie and returns the root hash and the number of -/// nodes read. If an error occurs during decoding, there are no guarantees about which entries -/// were or were not added to the DB. +/// from the given slice, reconstructing the full nodes and inserting them into the given +/// `MemoryDB`. It stops after fully constructing one partial trie and returns the root hash and the +/// number of nodes read. If an error occurs during decoding, there are no guarantees about which +/// entries were or were not added to the DB. /// /// The number of nodes read may be fewer than the total number of items in `encoded`. This allows /// one to concatenate multiple compact encodings together and still reconstruct them all. // /// This function makes the assumption that all child references in an inline trie node are inline /// references. -pub fn decode_compact( - db: &mut DB, +pub fn decode_compact( + db: &mut MemoryDB, DBValue>, encoded: &[Vec], ) -> Result<(TrieHash, usize), TrieHash, CError> where L: TrieLayout, - DB: HashDB, { - decode_compact_from_iter::(db, encoded.iter().map(Vec::as_slice)) + decode_compact_from_iter::(db, encoded.iter().map(Vec::as_slice)) } /// Variant of 'decode_compact' that accept an iterator of encoded nodes as input. -pub fn decode_compact_from_iter<'a, L, DB, I>( - db: &mut DB, +pub fn decode_compact_from_iter<'a, L, I>( + db: &mut MemoryDB, DBValue>, encoded: I, ) -> Result<(TrieHash, usize), TrieHash, CError> where L: TrieLayout, - DB: HashDB, I: IntoIterator, { // The stack of nodes through a path in the trie. Each entry is a child node of the preceding // entry. - let mut stack: Vec> = Vec::new(); + let mut stack: Vec> = Vec::new(); // The prefix of the next item to be read from the slice of encoded items. let mut prefix = NibbleVec::new(); @@ -479,7 +498,7 @@ where attached_node = 1; } } - let node = L::Codec::decode(&encoded_node[attached_node..]) + let node = L::Codec::decode(&encoded_node[attached_node..], &[]) .map_err(|err| Box::new(TrieError::DecoderError(>::default(), err)))?; let children_len = match node { @@ -523,7 +542,8 @@ where if let Some(entry) = stack.pop() { last_entry = entry; last_entry.pop_from_prefix(&mut prefix); - last_entry.children[last_entry.child_index] = Some(ChildReference::Hash(node_hash)); + last_entry.children[last_entry.child_index] = + Some(ChildReference::Hash(node_hash, Default::default())); last_entry.child_index += 1; } else { return Ok((node_hash, i + 1)) diff --git a/trie-root/src/lib.rs b/subtrie/src/trie_root.rs similarity index 96% rename from trie-root/src/lib.rs rename to subtrie/src/trie_root.rs index e184cfbb..d27f1641 100644 --- a/trie-root/src/lib.rs +++ b/subtrie/src/trie_root.rs @@ -16,32 +16,10 @@ //! //! This module should be used to generate trie root hash. -#![cfg_attr(not(feature = "std"), no_std)] - -#[cfg(not(feature = "std"))] -extern crate alloc; - -#[cfg(feature = "std")] -mod rstd { - pub use std::{ - cmp, - collections::{BTreeMap, VecDeque}, - vec::Vec, - }; -} - -#[cfg(not(feature = "std"))] -mod rstd { - pub use alloc::{ - collections::{BTreeMap, VecDeque}, - vec::Vec, - }; - pub use core::cmp; -} - -use self::rstd::*; - -pub use hash_db::Hasher; +use crate::{ + node_db::Hasher, + rstd::{cmp, vec::Vec, BTreeMap}, +}; /// Different possible value to use for node encoding. #[derive(Clone)] diff --git a/trie-db/src/triedb.rs b/subtrie/src/triedb.rs similarity index 65% rename from trie-db/src/triedb.rs rename to subtrie/src/triedb.rs index ad6166eb..9d649615 100644 --- a/trie-db/src/triedb.rs +++ b/subtrie/src/triedb.rs @@ -13,13 +13,15 @@ // limitations under the License. use crate::{ - iterator::TrieDBRawIterator, + iterator::{TrieDBNodeDoubleEndedIterator, TrieDBRawIterator}, lookup::Lookup, nibble::NibbleSlice, node::{decode_hash, NodeHandle, OwnedNode}, + node_db::{NodeDB, Prefix, EMPTY_PREFIX}, rstd::boxed::Box, - CError, DBValue, Query, Result, Trie, TrieAccess, TrieCache, TrieError, TrieHash, TrieItem, - TrieIterator, TrieKeyItem, TrieLayout, TrieRecorder, + CError, DBValue, MerkleValue, Query, Result, Trie, TrieAccess, TrieCache, + TrieDoubleEndedIterator, TrieError, TrieHash, TrieItem, TrieIterator, TrieKeyItem, TrieLayout, + TrieRecorder, }; #[cfg(feature = "std")] use crate::{ @@ -27,14 +29,14 @@ use crate::{ node::Node, rstd::{fmt, vec::Vec}, }; -use hash_db::{HashDBRef, Prefix, EMPTY_PREFIX}; /// A builder for creating a [`TrieDB`]. pub struct TrieDBBuilder<'db, 'cache, L: TrieLayout> { - db: &'db dyn HashDBRef, + db: &'db dyn NodeDB, root: &'db TrieHash, - cache: Option<&'cache mut dyn TrieCache>, - recorder: Option<&'cache mut dyn TrieRecorder>>, + root_location: L::Location, + cache: Option<&'cache mut dyn TrieCache>, + recorder: Option<&'cache mut dyn TrieRecorder, L::Location>>, } impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { @@ -43,13 +45,23 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// This doesn't check if `root` exists in the given `db`. If `root` doesn't exist it will fail /// when trying to lookup any key. #[inline] - pub fn new(db: &'db dyn HashDBRef, root: &'db TrieHash) -> Self { - Self { db, root, cache: None, recorder: None } + pub fn new(db: &'db dyn NodeDB, root: &'db TrieHash) -> Self { + Self { db, root, cache: None, recorder: None, root_location: Default::default() } + } + + /// Same as `new` but indicating db location of root. Warning root hash will not be checked. + #[inline] + pub fn new_with_db_location( + db: &'db dyn NodeDB, + root: &'db TrieHash, + root_location: L::Location, + ) -> Self { + Self { db, root, cache: None, recorder: None, root_location } } /// Use the given `cache` for the db. #[inline] - pub fn with_cache(mut self, cache: &'cache mut dyn TrieCache) -> Self { + pub fn with_cache(mut self, cache: &'cache mut dyn TrieCache) -> Self { self.cache = Some(cache); self } @@ -58,7 +70,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { #[inline] pub fn with_optional_cache<'ocache: 'cache>( mut self, - cache: Option<&'ocache mut dyn TrieCache>, + cache: Option<&'ocache mut dyn TrieCache>, ) -> Self { // Make the compiler happy by "converting" the lifetime self.cache = cache.map(|c| c as _); @@ -67,7 +79,10 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// Use the given `recorder` to record trie accesses. #[inline] - pub fn with_recorder(mut self, recorder: &'cache mut dyn TrieRecorder>) -> Self { + pub fn with_recorder( + mut self, + recorder: &'cache mut dyn TrieRecorder, L::Location>, + ) -> Self { self.recorder = Some(recorder); self } @@ -76,7 +91,7 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { #[inline] pub fn with_optional_recorder<'recorder: 'cache>( mut self, - recorder: Option<&'recorder mut dyn TrieRecorder>>, + recorder: Option<&'recorder mut dyn TrieRecorder, L::Location>>, ) -> Self { // Make the compiler happy by "converting" the lifetime self.recorder = recorder.map(|r| r as _); @@ -89,13 +104,14 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { TrieDB { db: self.db, root: self.root, + root_location: self.root_location, cache: self.cache.map(core::cell::RefCell::new), recorder: self.recorder.map(core::cell::RefCell::new), } } } -/// A `Trie` implementation using a generic `HashDB` backing database, a `Hasher` +/// A `Trie` implementation using a generic `NodeDB` backing database, a `Hasher` /// implementation to generate keys and a `NodeCodec` implementation to encode/decode /// the nodes. /// @@ -104,9 +120,9 @@ impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { /// /// # Example /// ```ignore -/// use hash_db::Hasher; -/// use reference_trie::{RefTrieDBMut, RefTrieDB, Trie, TrieMut}; -/// use trie_db::DBValue; +/// use subtrie::node_db::Hasher; +/// use reference_trie::{RefTrieDBMut, RefTrieDB, Trie}; +/// use subtrie::DBValue; /// use keccak_hasher::KeccakHasher; /// use memory_db::*; /// @@ -121,10 +137,11 @@ pub struct TrieDB<'db, 'cache, L> where L: TrieLayout, { - db: &'db dyn HashDBRef, + db: &'db dyn NodeDB, root: &'db TrieHash, - cache: Option>>, - recorder: Option>>>, + root_location: L::Location, + cache: Option>>, + recorder: Option, L::Location>>>, } impl<'db, 'cache, L> TrieDB<'db, 'cache, L> @@ -132,10 +149,31 @@ where L: TrieLayout, { /// Get the backing database. - pub fn db(&'db self) -> &'db dyn HashDBRef { + pub fn db(&'db self) -> &'db dyn NodeDB { self.db } + /// Create `TrieDBDoubleEndedIterator` from `TrieDB`. + pub fn into_double_ended_iter( + &'db self, + ) -> Result, TrieHash, CError> { + TrieDBDoubleEndedIterator::new(&self) + } + + /// Create `TrieDBNodeDoubleEndedIterator` from `TrieDB`. + pub fn into_node_double_ended_iter( + &'db self, + ) -> Result, TrieHash, CError> { + TrieDBNodeDoubleEndedIterator::new(&self) + } + + /// create `TrieDBKeyDoubleEndedIterator` from `TrieDB`. + pub fn into_key_double_ended_iter( + &'db self, + ) -> Result, TrieHash, CError> { + TrieDBKeyDoubleEndedIterator::new(&self) + } + /// Given some node-describing data `node`, and node key return the actual node RLP. /// This could be a simple identity operation in the case that the node is sufficiently small, /// but may require a database lookup. @@ -150,27 +188,28 @@ where pub(crate) fn get_raw_or_lookup( &self, parent_hash: TrieHash, - node_handle: NodeHandle, + node_handle: NodeHandle, partial_key: Prefix, record_access: bool, - ) -> Result<(OwnedNode, Option>), TrieHash, CError> { - let (node_hash, node_data) = match node_handle { - NodeHandle::Hash(data) => { + ) -> Result<(OwnedNode, Option>), TrieHash, CError> { + let (node_hash, (node_data, locations)) = match node_handle { + NodeHandle::Hash(data, location) => { let node_hash = decode_hash::(data) .ok_or_else(|| Box::new(TrieError::InvalidHash(parent_hash, data.to_vec())))?; - let node_data = self.db.get(&node_hash, partial_key).ok_or_else(|| { - if partial_key == EMPTY_PREFIX { - Box::new(TrieError::InvalidStateRoot(node_hash)) - } else { - Box::new(TrieError::IncompleteDatabase(node_hash)) - } - })?; + let node_data = + self.db.get(&node_hash, partial_key, location).ok_or_else(|| { + if partial_key == EMPTY_PREFIX { + Box::new(TrieError::InvalidStateRoot(node_hash)) + } else { + Box::new(TrieError::IncompleteDatabase(node_hash)) + } + })?; (Some(node_hash), node_data) }, - NodeHandle::Inline(data) => (None, data.to_vec()), + NodeHandle::Inline(data) => (None, (data.to_vec(), Default::default())), }; - let owned_node = OwnedNode::new::(node_data) + let owned_node = OwnedNode::new::(node_data, locations) .map_err(|e| Box::new(TrieError::DecoderError(node_hash.unwrap_or(parent_hash), e)))?; if record_access { @@ -192,10 +231,11 @@ where &self, hash: TrieHash, prefix: Prefix, + location: L::Location, ) -> Result, CError> { - let value = self + let (value, _) = self .db - .get(&hash, prefix) + .get(&hash, prefix, location) .ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; if let Some(recorder) = self.recorder.as_ref() { @@ -220,6 +260,10 @@ where self.root } + fn root_location(&self) -> L::Location { + self.root_location + } + fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { let mut cache = self.cache.as_ref().map(|c| c.borrow_mut()); let mut recorder = self.recorder.as_ref().map(|r| r.borrow_mut()); @@ -228,8 +272,11 @@ where db: self.db, query: |_: &[u8]| (), hash: *self.root, - cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), - recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder>), + location: self.root_location, + cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), + recorder: recorder + .as_mut() + .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } .look_up_hash(key, NibbleSlice::new(key)) } @@ -246,12 +293,35 @@ where db: self.db, query, hash: *self.root, - cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), - recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder>), + location: self.root_location, + cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), + recorder: recorder + .as_mut() + .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } .look_up(key, NibbleSlice::new(key)) } + fn lookup_first_descendant( + &self, + key: &[u8], + ) -> Result>>, TrieHash, CError> { + let mut cache = self.cache.as_ref().map(|c| c.borrow_mut()); + let mut recorder = self.recorder.as_ref().map(|r| r.borrow_mut()); + + Lookup:: { + db: self.db, + query: |_: &[u8]| (), + hash: *self.root, + location: self.root_location, + cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), + recorder: recorder + .as_mut() + .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), + } + .lookup_first_descendant(key, NibbleSlice::new(key)) + } + fn iter<'a>( &'a self, ) -> Result< @@ -280,7 +350,7 @@ where L: TrieLayout, { trie: &'db TrieDB<'db, 'cache, L>, - node_key: NodeHandle<'a>, + node_key: NodeHandle<'a, L::Location>, partial_key: NibbleVec, index: Option, } @@ -289,6 +359,7 @@ where impl<'db, 'cache, 'a, L> fmt::Debug for TrieAwareDebugNode<'db, 'cache, 'a, L> where L: TrieLayout, + L::Location: fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self.trie.get_raw_or_lookup( @@ -386,6 +457,7 @@ where impl<'db, 'cache, L> fmt::Debug for TrieDB<'db, 'cache, L> where L: TrieLayout, + L::Location: fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("TrieDB") @@ -393,7 +465,7 @@ where "root", &TrieAwareDebugNode { trie: self, - node_key: NodeHandle::Hash(self.root().as_ref()), + node_key: NodeHandle::Hash(self.root().as_ref(), Default::default()), partial_key: NibbleVec::new(), index: None, }, @@ -414,6 +486,45 @@ pub struct TrieDBKeyIterator<'a, 'cache, L: TrieLayout> { raw_iter: TrieDBRawIterator, } +/// Double ended iterator for going through all of key with values in the trie in pre-order +/// traversal order. +pub struct TrieDBKeyDoubleEndedIterator<'a, 'cache, L: TrieLayout> { + db: &'a TrieDB<'a, 'cache, L>, + raw_iter: TrieDBRawIterator, + back_raw_iter: TrieDBRawIterator, +} + +impl<'a, 'cache, L: TrieLayout> TrieDBKeyDoubleEndedIterator<'a, 'cache, L> { + /// Create a new double ended iterator. + pub fn new(db: &'a TrieDB<'a, 'cache, L>) -> Result, CError> { + Ok(Self { + db, + raw_iter: TrieDBRawIterator::new(db)?, + back_raw_iter: TrieDBRawIterator::new(db)?, + }) + } +} + +/// Double ended iterator for going through all values in the trie in pre-order traversal order. +pub struct TrieDBDoubleEndedIterator<'a, 'cache, L: TrieLayout> { + db: &'a TrieDB<'a, 'cache, L>, + raw_iter: TrieDBRawIterator, + back_raw_iter: TrieDBRawIterator, +} + +impl<'a, 'cache, L: TrieLayout> TrieDBDoubleEndedIterator<'a, 'cache, L> { + /// Create a new double ended iterator. + pub fn new(db: &'a TrieDB<'a, 'cache, L>) -> Result, CError> { + Ok(Self { + db, + raw_iter: TrieDBRawIterator::new(db)?, + back_raw_iter: TrieDBRawIterator::new(db)?, + }) + } +} + +impl TrieDoubleEndedIterator for TrieDBDoubleEndedIterator<'_, '_, L> {} + impl<'a, 'cache, L: TrieLayout> TrieDBIterator<'a, 'cache, L> { /// Create a new iterator. pub fn new(db: &'a TrieDB<'a, 'cache, L>) -> Result, CError> { @@ -453,7 +564,36 @@ impl<'a, 'cache, L: TrieLayout> TrieDBIterator<'a, 'cache, L> { impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBIterator<'a, 'cache, L> { /// Position the iterator on the first element with key >= `key` fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { - self.raw_iter.seek(self.db, key).map(|_| ()) + self.raw_iter.seek(self.db, key, true).map(|_| ()) + } +} + +impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBIterator<'a, 'cache, L> { + type Item = TrieItem, CError>; + + fn next(&mut self) -> Option { + self.raw_iter.next_item(self.db) + } +} + +impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBDoubleEndedIterator<'a, 'cache, L> { + fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { + self.raw_iter.seek(self.db, key, true).map(|_| ())?; + self.back_raw_iter.seek(self.db, key, false).map(|_| ()) + } +} + +impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBDoubleEndedIterator<'a, 'cache, L> { + type Item = TrieItem, CError>; + + fn next(&mut self) -> Option { + self.raw_iter.next_item(self.db) + } +} + +impl<'a, 'cache, L: TrieLayout> DoubleEndedIterator for TrieDBDoubleEndedIterator<'a, 'cache, L> { + fn next_back(&mut self) -> Option { + self.back_raw_iter.prev_item(self.db) } } @@ -496,22 +636,38 @@ impl<'a, 'cache, L: TrieLayout> TrieDBKeyIterator<'a, 'cache, L> { impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBKeyIterator<'a, 'cache, L> { /// Position the iterator on the first element with key >= `key` fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { - self.raw_iter.seek(self.db, key).map(|_| ()) + self.raw_iter.seek(self.db, key, true).map(|_| ()) } } -impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBIterator<'a, 'cache, L> { - type Item = TrieItem, CError>; +impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBKeyIterator<'a, 'cache, L> { + type Item = TrieKeyItem, CError>; fn next(&mut self) -> Option { - self.raw_iter.next_item(self.db) + self.raw_iter.next_key(self.db) } } -impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBKeyIterator<'a, 'cache, L> { +impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBKeyDoubleEndedIterator<'a, 'cache, L> { + /// Position the iterator on the first element with key >= `key` + fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { + self.raw_iter.seek(self.db, key, true).map(|_| ())?; + self.back_raw_iter.seek(self.db, key, false).map(|_| ()) + } +} + +impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBKeyDoubleEndedIterator<'a, 'cache, L> { type Item = TrieKeyItem, CError>; fn next(&mut self) -> Option { self.raw_iter.next_key(self.db) } } + +impl<'a, 'cache, L: TrieLayout> DoubleEndedIterator + for TrieDBKeyDoubleEndedIterator<'a, 'cache, L> +{ + fn next_back(&mut self) -> Option { + self.back_raw_iter.prev_key(self.db) + } +} diff --git a/trie-db/src/triedbmut.rs b/subtrie/src/triedbmut.rs similarity index 67% rename from trie-db/src/triedbmut.rs rename to subtrie/src/triedbmut.rs index bf3edbcb..1b1c9c80 100644 --- a/trie-db/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -23,13 +23,19 @@ use crate::{ }, node_codec::NodeCodec, rstd::{boxed::Box, convert::TryFrom, mem, ops::Index, result, vec::Vec, VecDeque}, - Bytes, CError, CachedValue, DBValue, Result, TrieAccess, TrieCache, TrieError, TrieHash, - TrieLayout, TrieMut, TrieRecorder, + Bytes, CError, DBValue, Result, TrieAccess, TrieCache, TrieError, TrieHash, TrieLayout, + TrieRecorder, }; -use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; -use hashbrown::HashSet; +use crate::node_db::{Hasher, NodeDB, Prefix}; +#[cfg(feature = "std")] +use std::collections::HashSet as Set; + +#[cfg(not(feature = "std"))] +use alloc::collections::btree_set::BTreeSet as Set; + +use crate::memory_db::{KeyFunction, MemoryDB}; #[cfg(feature = "std")] use log::trace; @@ -43,26 +49,28 @@ struct StorageHandle(usize); // Handles to nodes in the trie. #[cfg_attr(feature = "std", derive(Debug))] -enum NodeHandle { +enum NodeHandle { /// Loaded into memory. InMemory(StorageHandle), /// Either a hash or an inline node - Hash(H), + Hash(H, L), } -impl From for NodeHandle { +impl From for NodeHandle { fn from(handle: StorageHandle) -> Self { NodeHandle::InMemory(handle) } } -fn empty_children() -> Box<[Option>; nibble_ops::NIBBLE_LENGTH]> { +fn empty_children() -> Box<[Option>; nibble_ops::NIBBLE_LENGTH]> { Box::new([ None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ]) } +pub type TreeRefChangeset = Box, ::Location>>; + /// Type alias to indicate the nible covers a full key, /// therefore its left side is a full prefix. type NibbleFullKey<'key> = NibbleSlice<'key>; @@ -73,7 +81,7 @@ pub enum Value { /// Value bytes inlined in a trie node. Inline(Bytes), /// Hash of the value. - Node(TrieHash), + Node(TrieHash, L::Location), /// Hash of value bytes if calculated and value bytes. /// The hash may be undefined until it node is added /// to the db. @@ -84,7 +92,7 @@ impl PartialEq for Value { fn eq(&self, other: &Self) -> bool { match (self, other) { (Value::Inline(v), Value::Inline(ov)) => v == ov, - (Value::Node(h), Value::Node(oh)) => h == oh, + (Value::Node(h, _), Value::Node(oh, _)) => h == oh, (Value::NewNode(Some(h), _), Value::NewNode(Some(oh), _)) => h == oh, (Value::NewNode(_, v), Value::NewNode(_, ov)) => v == ov, // Note that for uncalculated hash we do not calculate it and default to true. @@ -94,24 +102,24 @@ impl PartialEq for Value { } } -impl<'a, L: TrieLayout> From> for Value { - fn from(v: EncodedValue<'a>) -> Self { +impl<'a, L: TrieLayout> From> for Value { + fn from(v: EncodedValue<'a, L::Location>) -> Self { match v { EncodedValue::Inline(value) => Value::Inline(value.into()), - EncodedValue::Node(hash) => { + EncodedValue::Node(hash, l) => { let mut h = TrieHash::::default(); h.as_mut().copy_from_slice(hash); - Value::Node(h) + Value::Node(h, l) }, } } } -impl From<&ValueOwned>> for Value { - fn from(val: &ValueOwned>) -> Self { +impl From<&ValueOwned, L::Location>> for Value { + fn from(val: &ValueOwned, L::Location>) -> Self { match val { ValueOwned::Inline(data, _) => Self::Inline(data.clone()), - ValueOwned::Node(hash) => Self::Node(*hash), + ValueOwned::Node(hash, location) => Self::Node(*hash, *location), } } } @@ -120,7 +128,7 @@ impl From<(Bytes, Option)> for Value { fn from((v, threshold): (Bytes, Option)) -> Self { match v { value => - if threshold.map(|threshold| value.len() >= threshold as usize).unwrap_or(false) { + if threshold.map_or(false, |threshold| value.len() >= threshold as usize) { Value::NewNode(None, value) } else { Value::Inline(value) @@ -129,9 +137,9 @@ impl From<(Bytes, Option)> for Value { } } -enum NodeToEncode<'a, H> { +enum NodeToEncode<'a, H, L> { Node(&'a [u8]), - TrieNode(NodeHandle), + TrieNode(NodeHandle), } impl Value { @@ -143,31 +151,35 @@ impl Value { &'a mut self, partial: Option<&NibbleSlice>, f: &mut F, - ) -> EncodedValue<'a> + ) -> EncodedValue<'a, L::Location> where F: FnMut( - NodeToEncode>, + NodeToEncode, L::Location>, Option<&NibbleSlice>, Option, - ) -> ChildReference>, + ) -> ChildReference, L::Location>, { if let Value::NewNode(hash, value) = self { - let new_hash = - if let ChildReference::Hash(hash) = f(NodeToEncode::Node(&value), partial, None) { - hash - } else { - unreachable!("Value node can never be inlined; qed") - }; + let new_hash = if let ChildReference::Hash(hash, _) = + f(NodeToEncode::Node(&value), partial, None) + { + hash + } else { + unreachable!("Value node can never be inlined; qed") + }; if let Some(h) = hash.as_ref() { debug_assert!(h == &new_hash); } else { *hash = Some(new_hash); } + } else if let Value::Node(hash, location) = self { + f(NodeToEncode::TrieNode(NodeHandle::Hash(*hash, *location)), partial, None); } let value = match &*self { Value::Inline(value) => EncodedValue::Inline(&value), - Value::Node(hash) => EncodedValue::Node(hash.as_ref()), - Value::NewNode(Some(hash), _value) => EncodedValue::Node(hash.as_ref()), + Value::Node(hash, location) => EncodedValue::Node(hash.as_ref(), *location), + Value::NewNode(Some(hash), _value) => + EncodedValue::Node(hash.as_ref(), Default::default()), Value::NewNode(None, _value) => unreachable!("New external value are always added before encoding anode"), }; @@ -177,15 +189,15 @@ impl Value { fn in_memory_fetched_value( &self, prefix: Prefix, - db: &dyn HashDB, - recorder: &Option>>>, + db: &dyn NodeDB, + recorder: &Option, L::Location>>>, full_key: &[u8], ) -> Result, TrieHash, CError> { Ok(Some(match self { Value::Inline(value) => value.to_vec(), Value::NewNode(_, value) => value.to_vec(), - Value::Node(hash) => - if let Some(value) = db.get(hash, prefix) { + Value::Node(hash, location) => + if let Some((value, _)) = db.get(hash, prefix, *location) { recorder.as_ref().map(|r| { r.borrow_mut().record(TrieAccess::Value { hash: *hash, @@ -209,19 +221,24 @@ enum Node { /// A leaf node contains the end of a key and a value. /// This key is encoded from a `NibbleSlice`, meaning it contains /// a flag indicating it is a leaf. - Leaf(NodeKey, Value), + Leaf(NodeKey, Value, Option>), /// An extension contains a shared portion of a key and a child node. /// The shared portion is encoded from a `NibbleSlice` meaning it contains /// a flag indicating it is an extension. /// The child node is always a branch. - Extension(NodeKey, NodeHandle>), + Extension(NodeKey, NodeHandle, L::Location>), /// A branch has up to 16 children and an optional value. - Branch(Box<[Option>>; nibble_ops::NIBBLE_LENGTH]>, Option>), + Branch( + Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, + Option>, + Option>, + ), /// Branch node with support for a nibble (to avoid extension node). NibbledBranch( NodeKey, - Box<[Option>>; nibble_ops::NIBBLE_LENGTH]>, + Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, Option>, + Option>, ), } @@ -243,7 +260,7 @@ impl Debug for Value { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { match self { Self::Inline(value) => write!(fmt, "Some({:?})", ToHex(value)), - Self::Node(hash) => write!(fmt, "Hash({:?})", ToHex(hash.as_ref())), + Self::Node(hash, _l) => write!(fmt, "Hash({:?})", ToHex(hash.as_ref())), Self::NewNode(Some(hash), _) => write!(fmt, "Hash({:?})", ToHex(hash.as_ref())), Self::NewNode(_hash, value) => write!(fmt, "Some({:?})", ToHex(value)), } @@ -254,16 +271,15 @@ impl Debug for Value { impl Debug for Node where L::Hash: Debug, + L::Location: Debug, { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - match *self { + match self { Self::Empty => write!(fmt, "Empty"), - Self::Leaf((ref a, ref b), ref c) => - write!(fmt, "Leaf({:?}, {:?})", (a, ToHex(&*b)), c), - Self::Extension((ref a, ref b), ref c) => - write!(fmt, "Extension({:?}, {:?})", (a, ToHex(&*b)), c), - Self::Branch(ref a, ref b) => write!(fmt, "Branch({:?}, {:?}", a, b), - Self::NibbledBranch((ref a, ref b), ref c, ref d) => + Self::Leaf((a, b), c, _) => write!(fmt, "Leaf({:?}, {:?})", (a, ToHex(&*b)), c), + Self::Extension((a, b), c) => write!(fmt, "Extension({:?}, {:?})", (a, ToHex(&*b)), c), + Self::Branch(a, b, _) => write!(fmt, "Branch({:?}, {:?}", a, b), + Self::NibbledBranch((a, b), c, d, _) => write!(fmt, "NibbledBranch({:?}, {:?}, {:?})", (a, ToHex(&*b)), c, d), } } @@ -273,17 +289,17 @@ impl Node { // load an inline node into memory or get the hash to do the lookup later. fn inline_or_hash( parent_hash: TrieHash, - child: EncodedNodeHandle, + child: EncodedNodeHandle, storage: &mut NodeStorage, - ) -> Result>, TrieHash, CError> { + ) -> Result, L::Location>, TrieHash, CError> { let handle = match child { - EncodedNodeHandle::Hash(data) => { + EncodedNodeHandle::Hash(data, location) => { let hash = decode_hash::(data) .ok_or_else(|| Box::new(TrieError::InvalidHash(parent_hash, data.to_vec())))?; - NodeHandle::Hash(hash) + NodeHandle::Hash(hash, location) }, EncodedNodeHandle::Inline(data) => { - let child = Node::from_encoded(parent_hash, data, storage)?; + let child = Node::from_encoded(parent_hash, data, &[], storage)?; NodeHandle::InMemory(storage.alloc(Stored::New(child))) }, }; @@ -292,11 +308,11 @@ impl Node { // load an inline node into memory or get the hash to do the lookup later. fn inline_or_hash_owned( - child: &NodeHandleOwned>, + child: &NodeHandleOwned, L::Location>, storage: &mut NodeStorage, - ) -> NodeHandle> { + ) -> NodeHandle, L::Location> { match child { - NodeHandleOwned::Hash(hash) => NodeHandle::Hash(*hash), + NodeHandleOwned::Hash(hash, location) => NodeHandle::Hash(*hash, *location), NodeHandleOwned::Inline(node) => { let child = Node::from_node_owned(&**node, storage); NodeHandle::InMemory(storage.alloc(Stored::New(child))) @@ -308,13 +324,14 @@ impl Node { fn from_encoded<'a, 'b>( node_hash: TrieHash, data: &'a [u8], + locations: &[L::Location], storage: &'b mut NodeStorage, ) -> Result, CError> { - let encoded_node = - L::Codec::decode(data).map_err(|e| Box::new(TrieError::DecoderError(node_hash, e)))?; + let encoded_node = L::Codec::decode(data, locations) + .map_err(|e| Box::new(TrieError::DecoderError(node_hash, e)))?; let node = match encoded_node { EncodedNode::Empty => Node::Empty, - EncodedNode::Leaf(k, v) => Node::Leaf(k.into(), v.into()), + EncodedNode::Leaf(k, v) => Node::Leaf(k.into(), v.into(), None), EncodedNode::Extension(key, cb) => Node::Extension(key.into(), Self::inline_or_hash(node_hash, cb, storage)?), EncodedNode::Branch(encoded_children, val) => { @@ -342,7 +359,7 @@ impl Node { child(15)?, ]); - Node::Branch(children, val.map(Into::into)) + Node::Branch(children, val.map(Into::into), None) }, EncodedNode::NibbledBranch(k, encoded_children, val) => { let mut child = |i: usize| match encoded_children[i] { @@ -369,17 +386,20 @@ impl Node { child(15)?, ]); - Node::NibbledBranch(k.into(), children, val.map(Into::into)) + Node::NibbledBranch(k.into(), children, val.map(Into::into), None) }, }; Ok(node) } /// Decode a node from a [`NodeOwned`]. - fn from_node_owned(node_owned: &NodeOwned>, storage: &mut NodeStorage) -> Self { + fn from_node_owned( + node_owned: &NodeOwned, L::Location>, + storage: &mut NodeStorage, + ) -> Self { match node_owned { NodeOwned::Empty => Node::Empty, - NodeOwned::Leaf(k, v) => Node::Leaf(k.into(), v.into()), + NodeOwned::Leaf(k, v) => Node::Leaf(k.into(), v.into(), None), NodeOwned::Extension(key, cb) => Node::Extension(key.into(), Self::inline_or_hash_owned(cb, storage)), NodeOwned::Branch(encoded_children, val) => { @@ -408,7 +428,7 @@ impl Node { child(15), ]); - Node::Branch(children, val.as_ref().map(Into::into)) + Node::Branch(children, val.as_ref().map(Into::into), None) }, NodeOwned::NibbledBranch(k, encoded_children, val) => { let mut child = |i: usize| { @@ -436,7 +456,7 @@ impl Node { child(15), ]); - Node::NibbledBranch(k.into(), children, val.as_ref().map(Into::into)) + Node::NibbledBranch(k.into(), children, val.as_ref().map(Into::into), None) }, NodeOwned::Value(_, _) => unreachable!("`NodeOwned::Value` can only be returned for the hash of a value."), @@ -446,70 +466,73 @@ impl Node { // TODO: parallelize /// Here `child_cb` should process the first parameter to either insert an external /// node value or to encode and add a new branch child node. - fn into_encoded(self, mut child_cb: F) -> Vec + fn into_encoded(self, mut child_cb: F) -> (Vec, Option>) where F: FnMut( - NodeToEncode>, + NodeToEncode, L::Location>, Option<&NibbleSlice>, Option, - ) -> ChildReference>, + ) -> ChildReference, L::Location>, { match self { - Node::Empty => L::Codec::empty_node().to_vec(), - Node::Leaf(partial, mut value) => { + Node::Empty => (L::Codec::empty_node().to_vec(), None), + Node::Leaf(partial, mut value, tree_ref) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let value = value.into_encoded::(Some(&pr), &mut child_cb); - L::Codec::leaf_node(pr.right_iter(), pr.len(), value) + (L::Codec::leaf_node(pr.right_iter(), pr.len(), value), tree_ref) }, Node::Extension(partial, child) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let it = pr.right_iter(); let c = child_cb(NodeToEncode::TrieNode(child), Some(&pr), None); - L::Codec::extension_node(it, pr.len(), c) + (L::Codec::extension_node(it, pr.len(), c), None) }, - Node::Branch(mut children, mut value) => { + Node::Branch(mut children, mut value, tree_ref) => { let value = value.as_mut().map(|v| v.into_encoded::(None, &mut child_cb)); - L::Codec::branch_node( - // map the `NodeHandle`s from the Branch to `ChildReferences` - children.iter_mut().map(Option::take).enumerate().map(|(i, maybe_child)| { - maybe_child.map(|child| { - child_cb(NodeToEncode::TrieNode(child), None, Some(i as u8)) - }) - }), - value, + ( + L::Codec::branch_node( + // map the `NodeHandle`s from the Branch to `ChildReferences` + children.iter_mut().map(Option::take).enumerate().map( + |(i, maybe_child)| { + maybe_child.map(|child| { + child_cb(NodeToEncode::TrieNode(child), None, Some(i as u8)) + }) + }, + ), + value, + ), + tree_ref, ) }, - Node::NibbledBranch(partial, mut children, mut value) => { + Node::NibbledBranch(partial, mut children, mut value, tree_ref) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let value = value.as_mut().map(|v| v.into_encoded::(Some(&pr), &mut child_cb)); let it = pr.right_iter(); - L::Codec::branch_node_nibbled( - it, - pr.len(), - // map the `NodeHandle`s from the Branch to `ChildReferences` - children.iter_mut().map(Option::take).enumerate().map(|(i, maybe_child)| { - //let branch_index = [i as u8]; - maybe_child.map(|child| { - let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); - child_cb(NodeToEncode::TrieNode(child), Some(&pr), Some(i as u8)) - }) - }), - value, + ( + L::Codec::branch_node_nibbled( + it, + pr.len(), + // map the `NodeHandle`s from the Branch to `ChildReferences` + children.iter_mut().map(Option::take).enumerate().map( + |(i, maybe_child)| { + //let branch_index = [i as u8]; + maybe_child.map(|child| { + let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); + child_cb( + NodeToEncode::TrieNode(child), + Some(&pr), + Some(i as u8), + ) + }) + }, + ), + value, + ), + tree_ref, ) }, } } - - /// Returns the key partial key of this node. - fn partial_key(&self) -> Option<&NodeKey> { - match &self { - Self::Empty => None, - Self::Leaf(key, _) => Some(key), - Self::Branch(_, _) => None, - Self::NibbledBranch(key, _, _) => Some(key), - Self::Extension(key, _) => Some(key), - } - } } // post-inspect action. @@ -519,7 +542,7 @@ enum Action { // Restore the original node. This trusts that the node is actually the original. Restore(Node), // if it is a new node, just clears the storage. - Delete, + Delete(Option>), } // post-insert action. Same as action without delete @@ -551,33 +574,33 @@ enum Stored { // A new node. New(Node), // A cached node, loaded from the DB. - Cached(Node, TrieHash), + Cached(Node, TrieHash, L::Location), } /// Used to build a collection of child nodes from a collection of `NodeHandle`s #[derive(Clone, Copy)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum ChildReference { +pub enum ChildReference { // `HO` is e.g. `H256`, i.e. the output of a `Hasher` - Hash(HO), + Hash(HO, L), Inline(HO, usize), // usize is the length of the node data we store in the `H::Out` } -impl<'a, HO> TryFrom> for ChildReference +impl<'a, HO, L> TryFrom> for ChildReference where HO: AsRef<[u8]> + AsMut<[u8]> + Default + Clone + Copy, { type Error = Vec; - fn try_from(handle: EncodedNodeHandle<'a>) -> result::Result> { + fn try_from(handle: EncodedNodeHandle<'a, L>) -> result::Result> { match handle { - EncodedNodeHandle::Hash(data) => { + EncodedNodeHandle::Hash(data, location) => { let mut hash = HO::default(); if data.len() != hash.as_ref().len() { return Err(data.to_vec()) } hash.as_mut().copy_from_slice(data); - Ok(ChildReference::Hash(hash)) + Ok(ChildReference::Hash(hash, location)) }, EncodedNodeHandle::Inline(data) => { let mut hash = HO::default(); @@ -629,26 +652,26 @@ impl<'a, L: TrieLayout> Index<&'a StorageHandle> for NodeStorage { fn index(&self, handle: &'a StorageHandle) -> &Node { match self.nodes[handle.0] { Stored::New(ref node) => node, - Stored::Cached(ref node, _) => node, + Stored::Cached(ref node, _, _) => node, } } } /// A builder for creating a [`TrieDBMut`]. pub struct TrieDBMutBuilder<'db, L: TrieLayout> { - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, - cache: Option<&'db mut dyn TrieCache>, - recorder: Option<&'db mut dyn TrieRecorder>>, + db: &'db dyn NodeDB, + root: TrieHash, + root_location: L::Location, + cache: Option<&'db mut dyn TrieCache>, + recorder: Option<&'db mut dyn TrieRecorder, L::Location>>, } impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Create a builder for constructing a new trie with the backing database `db` and empty /// `root`. - pub fn new(db: &'db mut dyn HashDB, root: &'db mut TrieHash) -> Self { - *root = L::Codec::hashed_null_node(); - - Self { root, db, cache: None, recorder: None } + pub fn new(db: &'db dyn NodeDB) -> Self { + let root = L::Codec::hashed_null_node(); + Self { root, db, cache: None, recorder: None, root_location: Default::default() } } /// Create a builder for constructing a new trie with the backing database `db` and `root`. @@ -656,14 +679,24 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// This doesn't check if `root` exists in the given `db`. If `root` doesn't exist it will fail /// when trying to lookup any key. pub fn from_existing( - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, + db: &'db dyn NodeDB, + root: TrieHash, ) -> Self { - Self { db, root, cache: None, recorder: None } + Self { db, root, cache: None, recorder: None, root_location: Default::default() } + } + + /// Same as `from_existing` but force a db location to access root. + /// Note root in parameter is not checked. + pub fn from_existing_with_db_location( + db: &'db dyn NodeDB, + root: TrieHash, + root_location: L::Location, + ) -> Self { + Self { db, root, cache: None, recorder: None, root_location } } /// Use the given `cache` for the db. - pub fn with_cache(mut self, cache: &'db mut dyn TrieCache) -> Self { + pub fn with_cache(mut self, cache: &'db mut dyn TrieCache) -> Self { self.cache = Some(cache); self } @@ -671,7 +704,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Use the given optional `cache` for the db. pub fn with_optional_cache<'cache: 'db>( mut self, - cache: Option<&'cache mut dyn TrieCache>, + cache: Option<&'cache mut dyn TrieCache>, ) -> Self { // Make the compiler happy by "converting" the lifetime self.cache = cache.map(|c| c as _); @@ -679,7 +712,10 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { } /// Use the given `recorder` to record trie accesses. - pub fn with_recorder(mut self, recorder: &'db mut dyn TrieRecorder>) -> Self { + pub fn with_recorder( + mut self, + recorder: &'db mut dyn TrieRecorder, L::Location>, + ) -> Self { self.recorder = Some(recorder); self } @@ -687,7 +723,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Use the given optional `recorder` to record trie accesses. pub fn with_optional_recorder<'recorder: 'db>( mut self, - recorder: Option<&'recorder mut dyn TrieRecorder>>, + recorder: Option<&'recorder mut dyn TrieRecorder, L::Location>>, ) -> Self { // Make the compiler happy by "converting" the lifetime self.recorder = recorder.map(|r| r as _); @@ -696,36 +732,150 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { /// Build the [`TrieDBMut`]. pub fn build(self) -> TrieDBMut<'db, L> { - let root_handle = NodeHandle::Hash(*self.root); + let root_handle = NodeHandle::Hash(self.root, self.root_location); TrieDBMut { db: self.db, root: self.root, cache: self.cache, recorder: self.recorder.map(core::cell::RefCell::new), - hash_count: 0, storage: NodeStorage::empty(), death_row: Default::default(), + death_row_child: Default::default(), root_handle, } } } -/// A `Trie` implementation using a generic `HashDB` backing database. +#[derive(Debug)] +pub struct NewChangesetNode { + pub hash: H, + pub prefix: OwnedPrefix, + pub data: Vec, + pub children: Vec>, + // Storing the key and removed nodes related + // to this change set node (only needed for old trie). + pub removed_keys: Option<(Option>, Vec<(H, OwnedPrefix)>)>, +} + +#[derive(Debug)] +pub struct ExistingChangesetNode { + pub hash: H, + pub prefix: OwnedPrefix, + pub location: DL, +} + +#[derive(Debug)] +pub enum Changeset { + New(NewChangesetNode), + Existing(ExistingChangesetNode), +} + +impl Changeset { + pub fn hash(&self) -> &H { + match self { + Changeset::New(node) => &node.hash, + Changeset::Existing(node) => &node.hash, + } + } +} + +impl Changeset { + /// In case the underlying db do not + /// do empty node optimization, it can + /// make sense to insert the empty node. + pub fn new_empty>() -> Self { + Self::New(NewChangesetNode { + hash: C::hashed_null_node(), + prefix: Default::default(), + data: C::empty_node().to_vec(), + children: Default::default(), + removed_keys: None, + }) + } +} +pub fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { + let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); + result.extend_from_slice(ks); + result.extend_from_slice(prefix.0); + (result, prefix.1) +} + +impl Changeset { + pub fn apply_to(&self, mem_db: &mut MemoryDB) -> H + where + K: KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { + fn apply_node<'a, H, DL, MH, K>( + node: &'a Changeset, + mem_db: &mut MemoryDB, + mut ks: Option<&'a [u8]>, + ) where + K: KeyFunction + Send + Sync, + MH: Hasher + Send + Sync, + { + match node { + Changeset::New(node) => { + if let Some((k, removed)) = node.removed_keys.as_ref() { + for (hash, p) in removed.iter() { + if let Some(k) = k { + let prefixed = prefix_prefix(k.as_slice(), (p.0.as_slice(), p.1)); + mem_db.remove(hash, (prefixed.0.as_slice(), prefixed.1)); + ks = Some(k.as_slice()); + } else { + mem_db.remove(hash, (p.0.as_slice(), p.1)); + } + } + } + for child in &node.children { + apply_node(child, mem_db, ks); + } + if let Some(ks) = ks { + let prefixed = prefix_prefix(ks, (node.prefix.0.as_slice(), node.prefix.1)); + mem_db.insert((prefixed.0.as_slice(), prefixed.1), &node.data); + } else { + mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); + } + }, + Changeset::Existing(_) => {}, + } + } + apply_node::(&self, mem_db, None); + self.root_hash() + } + + pub fn root_hash(&self) -> H { + match &self { + Changeset::New(node) => node.hash, + Changeset::Existing(node) => node.hash, + } + } + + pub fn unchanged(root: H) -> Self { + Changeset::Existing(ExistingChangesetNode { + hash: root, + prefix: (BackingByteVec::new(), None), + location: Default::default(), + }) + } +} + +pub type OwnedPrefix = (BackingByteVec, Option); + +/// A `Trie` implementation using a generic `NodeDB` backing database. /// -/// Use it as a `TrieMut` trait object. You can use `db()` to get the backing database object. +/// You can use `db()` to get the backing database object. /// Note that changes are not committed to the database until `commit` is called. /// -/// Querying the root or dropping the trie will commit automatically. -/// /// /// # Example /// ```ignore -/// use hash_db::Hasher; -/// use reference_trie::{RefTrieDBMut, TrieMut}; -/// use trie_db::DBValue; +/// use subtrie::node_db::Hasher; +/// use reference_trie::RefTrieDBMut; +/// use subtrie::DBValue; /// use keccak_hasher::KeccakHasher; -/// use memory_db::*; +/// use subtrie::memory_db::*; /// /// let mut memdb = MemoryDB::, DBValue>::default(); /// let mut root = Default::default(); @@ -743,17 +893,15 @@ where L: TrieLayout, { storage: NodeStorage, - db: &'a mut dyn HashDB, - root: &'a mut TrieHash, - root_handle: NodeHandle>, - death_row: HashSet<(TrieHash, (BackingByteVec, Option))>, - /// The number of hash operations this trie has performed. - /// Note that none are performed until changes are committed. - hash_count: usize, + db: &'a dyn NodeDB, + root: TrieHash, + root_handle: NodeHandle, L::Location>, + death_row: Set<(TrieHash, OwnedPrefix)>, + death_row_child: Vec>, /// Optional cache for speeding up the lookup of nodes. - cache: Option<&'a mut dyn TrieCache>, + cache: Option<&'a mut dyn TrieCache>, /// Optional trie recorder for recording trie accesses. - recorder: Option>>>, + recorder: Option, L::Location>>>, } impl<'a, L> TrieDBMut<'a, L> @@ -761,12 +909,7 @@ where L: TrieLayout, { /// Get the backing database. - pub fn db(&self) -> &dyn HashDB { - self.db - } - - /// Get the backing database mutably. - pub fn db_mut(&mut self) -> &mut dyn HashDB { + pub fn db(&self) -> &dyn NodeDB { self.db } @@ -775,12 +918,13 @@ where &mut self, hash: TrieHash, key: Prefix, + location: L::Location, ) -> Result, CError> { // We only check the `cache` for a node with `get_node` and don't insert // the node if it wasn't there, because in substrate we only access the node while computing // a new trie (aka some branch). We assume that this node isn't that important // to have it being cached. - let node = match self.cache.as_mut().and_then(|c| c.get_node(&hash)) { + let node = match self.cache.as_mut().and_then(|c| c.get_node(&hash, location)) { Some(node) => { if let Some(recorder) = self.recorder.as_mut() { recorder.borrow_mut().record(TrieAccess::NodeOwned { hash, node_owned: &node }); @@ -789,9 +933,9 @@ where Node::from_node_owned(&node, &mut self.storage) }, None => { - let node_encoded = self + let (node_encoded, locations) = self .db - .get(&hash, key) + .get(&hash, key, location) .ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; if let Some(recorder) = self.recorder.as_mut() { @@ -801,11 +945,11 @@ where }); } - Node::from_encoded(hash, &node_encoded, &mut self.storage)? + Node::from_encoded(hash, &node_encoded, &locations, &mut self.storage)? }, }; - Ok(self.storage.alloc(Stored::Cached(node, hash))) + Ok(self.storage.alloc(Stored::Cached(node, hash, location))) } // Inspect a node, choosing either to replace, restore, or delete it. @@ -828,16 +972,20 @@ where Stored::New(node) => match inspector(self, node, key)? { Action::Restore(node) => Some((Stored::New(node), false)), Action::Replace(node) => Some((Stored::New(node), true)), - Action::Delete => None, + Action::Delete(tree_ref) => { + tree_ref.map(|c| self.death_row_child.push(c)); + None + }, }, - Stored::Cached(node, hash) => match inspector(self, node, key)? { - Action::Restore(node) => Some((Stored::Cached(node, hash), false)), + Stored::Cached(node, hash, location) => match inspector(self, node, key)? { + Action::Restore(node) => Some((Stored::Cached(node, hash, location), false)), Action::Replace(node) => { self.death_row.insert((hash, current_key.left_owned())); Some((Stored::New(node), true)) }, - Action::Delete => { + Action::Delete(tree_ref) => { self.death_row.insert((hash, current_key.left_owned())); + tree_ref.map(|c| self.death_row_child.push(c)); None }, }, @@ -849,30 +997,32 @@ where &self, full_key: &[u8], mut partial: NibbleSlice, - handle: &NodeHandle>, + handle: &NodeHandle, L::Location>, ) -> Result, TrieHash, CError> { let mut handle = handle; + // prefix only use for value node access, so this is always correct. let prefix = (full_key, None); loop { let (mid, child) = match handle { - NodeHandle::Hash(hash) => { + NodeHandle::Hash(hash, location) => { let mut recorder = self.recorder.as_ref().map(|r| r.borrow_mut()); return Lookup:: { - db: &self.db, + db: self.db, query: |v: &[u8]| v.to_vec(), hash: *hash, + location: *location, cache: None, recorder: recorder .as_mut() - .map(|r| &mut ***r as &mut dyn TrieRecorder>), + .map(|r| &mut ***r as &mut dyn TrieRecorder, L::Location>), } .look_up(full_key, partial) }, NodeHandle::InMemory(handle) => match &self.storage[handle] { Node::Empty => return Ok(None), - Node::Leaf(key, value) => - if NibbleSlice::from_stored(key) == partial { + Node::Leaf(slice, value, _) => + if NibbleSlice::from_stored(slice) == partial { return Ok(value.in_memory_fetched_value( prefix, self.db, @@ -890,7 +1040,7 @@ where return Ok(None) } }, - Node::Branch(children, value) => + Node::Branch(children, value, _) => if partial.is_empty() { return Ok(if let Some(v) = value.as_ref() { v.in_memory_fetched_value( @@ -909,7 +1059,7 @@ where None => return Ok(None), } }, - Node::NibbledBranch(slice, children, value) => { + Node::NibbledBranch(slice, children, value, _) => { let slice = NibbleSlice::from_stored(slice); if slice == partial { return Ok(if let Some(v) = value.as_ref() { @@ -923,7 +1073,7 @@ where None }) } else if partial.starts_with(&slice) { - let idx = partial.at(0); + let idx = partial.at(slice.len()); match children[idx as usize].as_ref() { Some(child) => (1 + slice.len(), child), None => return Ok(None), @@ -943,20 +1093,22 @@ where /// Insert a key-value pair into the trie, creating new nodes if necessary. fn insert_at( &mut self, - handle: NodeHandle>, + handle: NodeHandle, L::Location>, key: &mut NibbleFullKey, value: Bytes, old_val: &mut Option>, + tree_ref: Option>, ) -> Result<(StorageHandle, bool), TrieHash, CError> { let h = match handle { NodeHandle::InMemory(h) => h, - NodeHandle::Hash(h) => self.cache(h, key.left())?, + NodeHandle::Hash(h, l) => self.cache(h, key.left(), l)?, }; // cache then destroy for hash handle (handle being root in most case) let stored = self.storage.destroy(h); let (new_stored, changed) = self .inspect(stored, key, move |trie, stored, key| { - trie.insert_inspector(stored, key, value, old_val).map(|a| a.into_action()) + trie.insert_inspector(stored, key, value, old_val, tree_ref) + .map(|a| a.into_action()) })? .expect("Insertion never deletes."); @@ -971,7 +1123,7 @@ where ) { match &stored_value { Some(Value::NewNode(Some(hash), _)) // also removing new node in case commit is called multiple times - | Some(Value::Node(hash)) => { + | Some(Value::Node(hash, _)) => { self.death_row.insert(( hash.clone(), (prefix.0.into(), prefix.1), @@ -989,6 +1141,7 @@ where key: &mut NibbleFullKey, value: Bytes, old_val: &mut Option>, + tree_ref: Option>, ) -> Result, TrieHash, CError> { let partial = *key; @@ -1000,9 +1153,9 @@ where #[cfg(feature = "std")] trace!(target: "trie", "empty: COMPOSE"); let value = Value::new(value, L::MAX_INLINE_VALUE); - InsertAction::Replace(Node::Leaf(partial.to_stored(), value)) + InsertAction::Replace(Node::Leaf(partial.to_stored(), value, tree_ref)) }, - Node::Branch(mut children, stored_value) => { + Node::Branch(mut children, stored_value, b_tree_ref) => { debug_assert!(L::USE_EXTENSION); #[cfg(feature = "std")] trace!(target: "trie", "branch: ROUTE,AUGMENT"); @@ -1010,7 +1163,7 @@ where if partial.is_empty() { let value = Some(Value::new(value, L::MAX_INLINE_VALUE)); let unchanged = stored_value == value; - let branch = Node::Branch(children, value); + let branch = Node::Branch(children, value, tree_ref); self.replace_old_value(old_val, stored_value, key.left()); @@ -1024,25 +1177,33 @@ where key.advance(1); if let Some(child) = children[idx].take() { // Original had something there. recurse down into it. - let (new_child, changed) = self.insert_at(child, key, value, old_val)?; + let (new_child, changed) = + self.insert_at(child, key, value, old_val, tree_ref)?; children[idx] = Some(new_child.into()); if !changed { // The new node we composed didn't change. // It means our branch is untouched too. - return Ok(InsertAction::Restore(Node::Branch(children, stored_value))) + return Ok(InsertAction::Restore(Node::Branch( + children, + stored_value, + b_tree_ref, + ))) } } else { // Original had nothing there. compose a leaf. let value = Value::new(value, L::MAX_INLINE_VALUE); - let leaf = - self.storage.alloc(Stored::New(Node::Leaf(key.to_stored(), value))); + let leaf = self.storage.alloc(Stored::New(Node::Leaf( + key.to_stored(), + value, + tree_ref, + ))); children[idx] = Some(leaf.into()); } - InsertAction::Replace(Node::Branch(children, stored_value)) + InsertAction::Replace(Node::Branch(children, stored_value, b_tree_ref)) } }, - Node::NibbledBranch(encoded, mut children, stored_value) => { + Node::NibbledBranch(encoded, mut children, stored_value, b_tree_ref) => { debug_assert!(!L::USE_EXTENSION); #[cfg(feature = "std")] trace!(target: "trie", "branch: ROUTE,AUGMENT"); @@ -1052,7 +1213,8 @@ where if common == existing_key.len() && common == partial.len() { let value = Some(Value::new(value, L::MAX_INLINE_VALUE)); let unchanged = stored_value == value; - let branch = Node::NibbledBranch(existing_key.to_stored(), children, value); + let branch = + Node::NibbledBranch(existing_key.to_stored(), children, value, tree_ref); let mut key_val = key.clone(); key_val.advance(existing_key.len()); @@ -1075,7 +1237,8 @@ where common, ); let nbranch_partial = existing_key.mid(common + 1).to_stored(); - let low = Node::NibbledBranch(nbranch_partial, children, stored_value); + let low = + Node::NibbledBranch(nbranch_partial, children, stored_value, b_tree_ref); let ix = existing_key.at(common); let mut children = empty_children(); let alloc_storage = self.storage.alloc(Stored::New(low)); @@ -1088,10 +1251,12 @@ where existing_key.to_stored_range(common), children, Some(value), + tree_ref, )) } else { let ix = partial.at(common); - let stored_leaf = Node::Leaf(partial.mid(common + 1).to_stored(), value); + let stored_leaf = + Node::Leaf(partial.mid(common + 1).to_stored(), value, tree_ref); let leaf = self.storage.alloc(Stored::New(stored_leaf)); @@ -1100,6 +1265,7 @@ where existing_key.to_stored_range(common), children, None, + None, )) } } else { @@ -1110,7 +1276,8 @@ where key.advance(common + 1); if let Some(child) = children[idx].take() { // Original had something there. recurse down into it. - let (new_child, changed) = self.insert_at(child, key, value, old_val)?; + let (new_child, changed) = + self.insert_at(child, key, value, old_val, tree_ref)?; children[idx] = Some(new_child.into()); if !changed { // The new node we composed didn't change. @@ -1119,14 +1286,18 @@ where existing_key.to_stored(), children, stored_value, + b_tree_ref, ); return Ok(InsertAction::Restore(n_branch)) } } else { // Original had nothing there. compose a leaf. let value = Value::new(value, L::MAX_INLINE_VALUE); - let leaf = - self.storage.alloc(Stored::New(Node::Leaf(key.to_stored(), value))); + let leaf = self.storage.alloc(Stored::New(Node::Leaf( + key.to_stored(), + value, + tree_ref, + ))); children[idx] = Some(leaf.into()); } @@ -1134,10 +1305,11 @@ where existing_key.to_stored(), children, stored_value, + b_tree_ref, )) } }, - Node::Leaf(encoded, stored_value) => { + Node::Leaf(encoded, stored_value, l_tree_ref) => { let existing_key = NibbleSlice::from_stored(&encoded); let common = partial.common_prefix(&existing_key); if common == existing_key.len() && common == partial.len() { @@ -1151,9 +1323,9 @@ where self.replace_old_value(old_val, Some(stored_value), key_val.left()); if unchanged { // unchanged. restore - InsertAction::Restore(Node::Leaf(encoded.clone(), value)) + InsertAction::Restore(Node::Leaf(encoded.clone(), value, l_tree_ref)) } else { - InsertAction::Replace(Node::Leaf(encoded.clone(), value)) + InsertAction::Replace(Node::Leaf(encoded.clone(), value, tree_ref)) } } else if (L::USE_EXTENSION && common == 0) || (!L::USE_EXTENSION && common < existing_key.len()) @@ -1171,24 +1343,32 @@ where let mut children = empty_children(); let branch = if L::USE_EXTENSION && existing_key.is_empty() { // always replace since branch isn't leaf. - Node::Branch(children, Some(stored_value)) + Node::Branch(children, Some(stored_value), l_tree_ref) } else { let idx = existing_key.at(common) as usize; - let new_leaf = - Node::Leaf(existing_key.mid(common + 1).to_stored(), stored_value); + let new_leaf = Node::Leaf( + existing_key.mid(common + 1).to_stored(), + stored_value, + l_tree_ref, + ); children[idx] = Some(self.storage.alloc(Stored::New(new_leaf)).into()); if L::USE_EXTENSION { - Node::Branch(children, None) + Node::Branch(children, None, None) } else { - Node::NibbledBranch(partial.to_stored_range(common), children, None) + Node::NibbledBranch( + partial.to_stored_range(common), + children, + None, + None, + ) } }; // always replace because whatever we get out here // is not the branch we started with. let branch_action = - self.insert_inspector(branch, key, value, old_val)?.unwrap_node(); + self.insert_inspector(branch, key, value, old_val, tree_ref)?.unwrap_node(); InsertAction::Replace(branch_action) } else if !L::USE_EXTENSION { #[cfg(feature = "std")] @@ -1200,9 +1380,11 @@ where existing_key.to_stored(), empty_children(), Some(stored_value), + l_tree_ref, ); // augment the new branch. - let branch = self.insert_inspector(branch, key, value, old_val)?.unwrap_node(); + let branch = + self.insert_inspector(branch, key, value, old_val, tree_ref)?.unwrap_node(); InsertAction::Replace(branch) } else if common == existing_key.len() { @@ -1212,10 +1394,11 @@ where // fully-shared prefix for an extension. // make a stub branch and an extension. - let branch = Node::Branch(empty_children(), Some(stored_value)); + let branch = Node::Branch(empty_children(), Some(stored_value), l_tree_ref); // augment the new branch. key.advance(common); - let branch = self.insert_inspector(branch, key, value, old_val)?.unwrap_node(); + let branch = + self.insert_inspector(branch, key, value, old_val, tree_ref)?.unwrap_node(); // always replace since we took a leaf and made an extension. let leaf = self.storage.alloc(Stored::New(branch)); @@ -1234,13 +1417,14 @@ where // partially-shared prefix for an extension. // start by making a leaf. - let low = Node::Leaf(existing_key.mid(common).to_stored(), stored_value); + let low = + Node::Leaf(existing_key.mid(common).to_stored(), stored_value, l_tree_ref); // augment it. this will result in the Leaf -> common == 0 routine, // which creates a branch. key.advance(common); let augmented_low = - self.insert_inspector(low, key, value, old_val)?.unwrap_node(); + self.insert_inspector(low, key, value, old_val, tree_ref)?.unwrap_node(); // make an extension using it. this is a replacement. InsertAction::Replace(Node::Extension( existing_key.to_stored_range(common), @@ -1280,7 +1464,13 @@ where // continue inserting. let branch_action = self - .insert_inspector(Node::Branch(children, None), key, value, old_val)? + .insert_inspector( + Node::Branch(children, None, None), + key, + value, + old_val, + tree_ref, + )? .unwrap_node(); InsertAction::Replace(branch_action) } else if common == existing_key.len() { @@ -1291,7 +1481,8 @@ where // insert into the child node. key.advance(common); - let (new_child, changed) = self.insert_at(child_branch, key, value, old_val)?; + let (new_child, changed) = + self.insert_at(child_branch, key, value, old_val, tree_ref)?; let new_ext = Node::Extension(existing_key.to_stored(), new_child.into()); @@ -1318,7 +1509,7 @@ where // creating a branch. key.advance(common); let augmented_low = - self.insert_inspector(low, key, value, old_val)?.unwrap_node(); + self.insert_inspector(low, key, value, old_val, tree_ref)?.unwrap_node(); // always replace, since this extension is not the one we started with. // this is known because the partial key is only the common prefix. @@ -1334,20 +1525,21 @@ where /// Removes a node from the trie based on key. fn remove_at( &mut self, - handle: NodeHandle>, + handle: NodeHandle, L::Location>, key: &mut NibbleFullKey, old_val: &mut Option>, + tree_ref: Option>, ) -> Result, TrieHash, CError> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), - NodeHandle::Hash(h) => { - let handle = self.cache(h, key.left())?; + NodeHandle::Hash(h, l) => { + let handle = self.cache(h, key.left(), l)?; self.storage.destroy(handle) }, }; let opt = self.inspect(stored, key, move |trie, node, key| { - trie.remove_inspector(node, key, old_val) + trie.remove_inspector(node, key, old_val, tree_ref) })?; Ok(opt.map(|(new, changed)| (self.storage.alloc(new), changed))) @@ -1359,24 +1551,25 @@ where node: Node, key: &mut NibbleFullKey, old_val: &mut Option>, + tree_ref: Option>, ) -> Result, TrieHash, CError> { let partial = *key; Ok(match (node, partial.is_empty()) { - (Node::Empty, _) => Action::Delete, - (Node::Branch(c, None), true) => Action::Restore(Node::Branch(c, None)), - (Node::NibbledBranch(n, c, None), true) => - Action::Restore(Node::NibbledBranch(n, c, None)), - (Node::Branch(children, val), true) => { + (Node::Empty, _) => Action::Delete(None), + (Node::Branch(c, None, _), true) => Action::Restore(Node::Branch(c, None, tree_ref)), + (Node::NibbledBranch(n, c, None, _), true) => + Action::Restore(Node::NibbledBranch(n, c, None, tree_ref)), + (Node::Branch(children, val, _), true) => { self.replace_old_value(old_val, val, key.left()); // always replace since we took the value out. - Action::Replace(self.fix(Node::Branch(children, None), *key)?) + Action::Replace(self.fix(Node::Branch(children, None, tree_ref), *key)?) }, - (Node::NibbledBranch(n, children, val), true) => { + (Node::NibbledBranch(n, children, val, _), true) => { self.replace_old_value(old_val, val, key.left()); // always replace since we took the value out. - Action::Replace(self.fix(Node::NibbledBranch(n, children, None), *key)?) + Action::Replace(self.fix(Node::NibbledBranch(n, children, None, tree_ref), *key)?) }, - (Node::Branch(mut children, value), false) => { + (Node::Branch(mut children, value, btreerefset), false) => { let idx = partial.at(0) as usize; if let Some(child) = children[idx].take() { #[cfg(feature = "std")] @@ -1387,10 +1580,10 @@ where ); let prefix = *key; key.advance(1); - match self.remove_at(child, key, old_val)? { + match self.remove_at(child, key, old_val, tree_ref)? { Some((new, changed)) => { children[idx] = Some(new.into()); - let branch = Node::Branch(children, value); + let branch = Node::Branch(children, value, btreerefset); match changed { // child was changed, so we were too. true => Action::Replace(branch), @@ -1403,15 +1596,17 @@ where // the node may need fixing. #[cfg(feature = "std")] trace!(target: "trie", "branch child deleted, partial={:?}", partial); - Action::Replace(self.fix(Node::Branch(children, value), prefix)?) + Action::Replace( + self.fix(Node::Branch(children, value, btreerefset), prefix)?, + ) }, } } else { // no change needed. - Action::Restore(Node::Branch(children, value)) + Action::Restore(Node::Branch(children, value, btreerefset)) } }, - (Node::NibbledBranch(encoded, mut children, value), false) => { + (Node::NibbledBranch(encoded, mut children, value, btreerefset), false) => { let (common, existing_length) = { let existing_key = NibbleSlice::from_stored(&encoded); (existing_key.common_prefix(&partial), existing_key.len()) @@ -1422,14 +1617,15 @@ where let mut key_val = key.clone(); key_val.advance(existing_length); self.replace_old_value(old_val, Some(value), key_val.left()); - let f = self.fix(Node::NibbledBranch(encoded, children, None), *key); + let f = + self.fix(Node::NibbledBranch(encoded, children, None, tree_ref), *key); Action::Replace(f?) } else { - Action::Restore(Node::NibbledBranch(encoded, children, None)) + Action::Restore(Node::NibbledBranch(encoded, children, None, tree_ref)) } } else if common < existing_length { // partway through an extension -- nothing to do here. - Action::Restore(Node::NibbledBranch(encoded, children, value)) + Action::Restore(Node::NibbledBranch(encoded, children, value, btreerefset)) } else { // common == existing_length && common < partial.len() : check children let idx = partial.at(common) as usize; @@ -1443,10 +1639,11 @@ where ); let prefix = *key; key.advance(common + 1); - match self.remove_at(child, key, old_val)? { + match self.remove_at(child, key, old_val, tree_ref)? { Some((new, changed)) => { children[idx] = Some(new.into()); - let branch = Node::NibbledBranch(encoded, children, value); + let branch = + Node::NibbledBranch(encoded, children, value, btreerefset); match changed { // child was changed, so we were too. true => Action::Replace(branch), @@ -1463,28 +1660,26 @@ where "branch child deleted, partial={:?}", partial, ); - Action::Replace( - self.fix( - Node::NibbledBranch(encoded, children, value), - prefix, - )?, - ) + Action::Replace(self.fix( + Node::NibbledBranch(encoded, children, value, btreerefset), + prefix, + )?) }, } } else { // no change needed. - Action::Restore(Node::NibbledBranch(encoded, children, value)) + Action::Restore(Node::NibbledBranch(encoded, children, value, btreerefset)) } } }, - (Node::Leaf(encoded, value), _) => { + (Node::Leaf(encoded, value, ltreerefset), _) => { let existing_key = NibbleSlice::from_stored(&encoded); if existing_key == partial { // this is the node we were looking for. Let's delete it. let mut key_val = key.clone(); key_val.advance(existing_key.len()); self.replace_old_value(old_val, Some(value), key_val.left()); - Action::Delete + Action::Delete(tree_ref) } else { // leaf the node alone. #[cfg(feature = "std")] @@ -1494,7 +1689,7 @@ where partial, NibbleSlice::from_stored(&encoded), ); - Action::Restore(Node::Leaf(encoded, value)) + Action::Restore(Node::Leaf(encoded, value, ltreerefset)) } }, (Node::Extension(encoded, child_branch), _) => { @@ -1508,7 +1703,7 @@ where trace!(target: "trie", "removing from extension child, partial={:?}", partial); let prefix = *key; key.advance(common); - match self.remove_at(child_branch, key, old_val)? { + match self.remove_at(child_branch, key, old_val, tree_ref)? { Some((new_child, changed)) => { // if the child branch was unchanged, then the extension is too. // otherwise, this extension may need fixing. @@ -1523,7 +1718,7 @@ where None => { // the whole branch got deleted. // that means that this extension is useless. - Action::Delete + Action::Delete(None) }, } } else { @@ -1550,7 +1745,7 @@ where recurse_extension: bool, ) -> Result, TrieHash, CError> { match node { - Node::Branch(mut children, value) => { + Node::Branch(mut children, value, btreerefset) => { // if only a single value, transmute to leaf/extension and feed through fixed. #[cfg_attr(feature = "std", derive(Debug))] enum UsedIndex { @@ -1588,17 +1783,17 @@ where // make a leaf. #[cfg(feature = "std")] trace!(target: "trie", "fixing: branch -> leaf"); - Ok(Node::Leaf(NibbleSlice::new(&[]).to_stored(), value)) + Ok(Node::Leaf(NibbleSlice::new(&[]).to_stored(), value, btreerefset)) }, (_, value) => { // all is well. #[cfg(feature = "std")] trace!(target: "trie", "fixing: restoring branch"); - Ok(Node::Branch(children, value)) + Ok(Node::Branch(children, value, btreerefset)) }, } }, - Node::NibbledBranch(enc_nibble, mut children, value) => { + Node::NibbledBranch(enc_nibble, mut children, value, btreerefset) => { // if only a single value, transmute to leaf/extension and feed through fixed. #[cfg_attr(feature = "std", derive(Debug))] enum UsedIndex { @@ -1645,37 +1840,48 @@ where ); let stored = match child { NodeHandle::InMemory(h) => self.storage.destroy(h), - NodeHandle::Hash(h) => { - let handle = self.cache(h, child_prefix)?; + NodeHandle::Hash(h, l) => { + let handle = self.cache(h, child_prefix, l)?; self.storage.destroy(handle) }, }; let child_node = match stored { Stored::New(node) => node, - Stored::Cached(node, hash) => { + Stored::Cached(node, hash, _location) => { self.death_row .insert((hash, (child_prefix.0[..].into(), child_prefix.1))); node }, }; + btreerefset.map(|c| self.death_row_child.push(c)); match child_node { - Node::Leaf(sub_partial, value) => { + Node::Leaf(sub_partial, value, ltreerefset) => { let mut enc_nibble = enc_nibble; combine_key( &mut enc_nibble, (nibble_ops::NIBBLE_PER_BYTE - 1, &[a][..]), ); combine_key(&mut enc_nibble, (sub_partial.0, &sub_partial.1[..])); - Ok(Node::Leaf(enc_nibble, value)) + Ok(Node::Leaf(enc_nibble, value, ltreerefset)) }, - Node::NibbledBranch(sub_partial, ch_children, ch_value) => { + Node::NibbledBranch( + sub_partial, + ch_children, + ch_value, + ntreerefset, + ) => { let mut enc_nibble = enc_nibble; combine_key( &mut enc_nibble, (nibble_ops::NIBBLE_PER_BYTE - 1, &[a][..]), ); combine_key(&mut enc_nibble, (sub_partial.0, &sub_partial.1[..])); - Ok(Node::NibbledBranch(enc_nibble, ch_children, ch_value)) + Ok(Node::NibbledBranch( + enc_nibble, + ch_children, + ch_value, + ntreerefset, + )) }, _ => unreachable!(), } @@ -1684,13 +1890,13 @@ where // make a leaf. #[cfg(feature = "std")] trace!(target: "trie", "fixing: branch -> leaf"); - Ok(Node::Leaf(enc_nibble, value)) + Ok(Node::Leaf(enc_nibble, value, btreerefset)) }, (_, value) => { // all is well. #[cfg(feature = "std")] trace!(target: "trie", "fixing: restoring branch"); - Ok(Node::NibbledBranch(enc_nibble, children, value)) + Ok(Node::NibbledBranch(enc_nibble, children, value, btreerefset)) }, } }, @@ -1730,15 +1936,15 @@ where let stored = match child { NodeHandle::InMemory(h) => self.storage.destroy(h), - NodeHandle::Hash(h) => { - let handle = self.cache(h, child_prefix)?; + NodeHandle::Hash(h, l) => { + let handle = self.cache(h, child_prefix, l)?; self.storage.destroy(handle) }, }; - let (child_node, maybe_hash) = match stored { - Stored::New(node) => (node, None), - Stored::Cached(node, hash) => (node, Some(hash)), + let (child_node, maybe_hash, child_location) = match stored { + Stored::New(node) => (node, None, Default::default()), + Stored::Cached(node, hash, location) => (node, Some(hash), location), }; match child_node { @@ -1761,7 +1967,7 @@ where self.fix_inner(Node::Extension(partial, sub_child), key.into(), true) }, - Node::Leaf(sub_partial, value) => { + Node::Leaf(sub_partial, value, ltreerefset) => { // combine with node below. if let Some(hash) = maybe_hash { // delete the cached child since we are going to replace it. @@ -1777,7 +1983,7 @@ where "fixing: extension -> leaf. new_partial={:?}", partial, ); - Ok(Node::Leaf(partial, value)) + Ok(Node::Leaf(partial, value, ltreerefset)) }, child_node => { #[cfg(feature = "std")] @@ -1785,7 +1991,7 @@ where // reallocate the child node. let stored = if let Some(hash) = maybe_hash { - Stored::Cached(child_node, hash) + Stored::Cached(child_node, hash, child_location) } else { Stored::New(child_node) }; @@ -1798,146 +2004,128 @@ where } } - /// Commit the in-memory changes to disk, freeing their storage and - /// updating the state root. - pub fn commit(&mut self) { + /// Calculate the changeset for the trie. + /// Note `keyspace` only apply for hash base storage to avoid key collision + /// between composed tree states. + pub fn commit(self) -> Changeset, L::Location> { + self.commit_inner(None) + } + + /// Same as commit but use a keyspace to isolate + /// stored date. + /// `keyspace` only apply for hash base storage to avoid key collision + /// between composed tree states. + pub fn commit_with_keyspace(self, keyspace: &[u8]) -> Changeset, L::Location> { + self.commit_inner(Some(keyspace)) + } + + fn commit_inner(mut self, keyspace: Option<&[u8]>) -> Changeset, L::Location> { #[cfg(feature = "std")] trace!(target: "trie", "Committing trie changes to db."); // always kill all the nodes on death row. #[cfg(feature = "std")] trace!(target: "trie", "{:?} nodes to remove from db", self.death_row.len()); + let mut removed = Vec::with_capacity(self.death_row.len()); + + #[cfg(feature = "std")] for (hash, prefix) in self.death_row.drain() { - self.db.remove(&hash, (&prefix.0[..], prefix.1)); + removed.push((hash, prefix)); } let handle = match self.root_handle() { - NodeHandle::Hash(_) => return, // no changes necessary. + NodeHandle::Hash(hash, location) => { + debug_assert!(removed.is_empty()); + return Changeset::Existing(ExistingChangesetNode { + hash, + prefix: Default::default(), + location, + }); + }, // no changes necessary. NodeHandle::InMemory(h) => h, }; match self.storage.destroy(handle) { Stored::New(node) => { - // Reconstructs the full key for root node. - let full_key = self.cache.as_ref().and_then(|_| { - node.partial_key().and_then(|k| Some(NibbleSlice::from_stored(k).into())) - }); - let mut k = NibbleVec::new(); + let mut children = Vec::new(); - let encoded_root = node.into_encoded(|node, o_slice, o_index| { + let (encoded_root, roottreerefset) = node.into_encoded(|node, o_slice, o_index| { let mov = k.append_optional_slice_and_nibble(o_slice, o_index); match node { NodeToEncode::Node(value) => { - let value_hash = self.db.insert(k.as_prefix(), value); + let value_hash = self.db.hash(value); self.cache_value(k.inner(), value, value_hash); + children.push(Changeset::New(NewChangesetNode { + hash: value_hash, + prefix: k.as_owned_prefix(), + data: value.to_vec(), //TODO: avoid allocation + children: Default::default(), + removed_keys: None, + })); + k.drop_lasts(mov); - ChildReference::Hash(value_hash) + ChildReference::Hash(value_hash, Default::default()) }, NodeToEncode::TrieNode(child) => { - let result = self.commit_child(child, &mut k); + let result = self.commit_child(child, &mut k, &mut children); k.drop_lasts(mov); result }, } }); + roottreerefset.map(|c| { + children.push(*c); + }); #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:?}", ToHex(&encoded_root[..])); - *self.root = self.db.insert(EMPTY_PREFIX, &encoded_root); - self.hash_count += 1; + self.root = self.db.hash(&encoded_root); - self.cache_node(*self.root, &encoded_root, full_key); + self.cache_node(self.root); - self.root_handle = NodeHandle::Hash(*self.root); + self.root_handle = NodeHandle::Hash(self.root, Default::default()); + Changeset::New(NewChangesetNode { + hash: self.root.clone(), + prefix: Default::default(), + data: encoded_root, + children, + removed_keys: Some((keyspace.map(|s| s.to_vec()), removed)), + }) }, - Stored::Cached(node, hash) => { + Stored::Cached(node, hash, location) => { // probably won't happen, but update the root and move on. - *self.root = hash; - self.root_handle = - NodeHandle::InMemory(self.storage.alloc(Stored::Cached(node, hash))); + self.root = hash; + self.root_handle = NodeHandle::InMemory(self.storage.alloc(Stored::Cached( + node, + hash, + // TODO should we use location here?? likely yes TODO write a test with cache + // usage and location in triedbmut + Default::default(), + ))); + debug_assert!(removed.is_empty()); + Changeset::Existing(ExistingChangesetNode { + hash, + prefix: Default::default(), + location, + }) }, } } /// Cache the given `encoded` node. - fn cache_node(&mut self, hash: TrieHash, encoded: &[u8], full_key: Option) { - // If we have a cache, cache our node directly. + fn cache_node(&mut self, hash: TrieHash) { + // Mark the node as new so that it is removed from the shared cache. if let Some(cache) = self.cache.as_mut() { - let node = cache.get_or_insert_node(hash, &mut || { - Ok(L::Codec::decode(&encoded) - .ok() - .and_then(|n| n.to_owned_node::().ok()) - .expect("Just encoded the node, so it should decode without any errors; qed")) - }); - - // `node` should always be `OK`, but let's play it safe. - let node = if let Ok(node) = node { node } else { return }; - - let mut values_to_cache = Vec::new(); - - // If the given node has data attached, the `full_key` is the full key to this node. - if let Some(full_key) = full_key { - node.data().and_then(|v| node.data_hash().map(|h| (&full_key, v, h))).map( - |(k, v, h)| { - values_to_cache.push((k.inner().to_vec(), (v.clone(), h).into())); - }, - ); - - fn cache_child_values( - node: &NodeOwned>, - values_to_cache: &mut Vec<(Vec, CachedValue>)>, - full_key: NibbleVec, - ) { - node.child_iter().flat_map(|(n, c)| c.as_inline().map(|c| (n, c))).for_each( - |(n, c)| { - let mut key = full_key.clone(); - n.map(|n| key.push(n)); - c.partial_key().map(|p| key.append(p)); - - if let Some((hash, data)) = - c.data().and_then(|d| c.data_hash().map(|h| (h, d))) - { - values_to_cache - .push((key.inner().to_vec(), (data.clone(), hash).into())); - } - - cache_child_values::(c, values_to_cache, key); - }, - ); - } - - // Also cache values of inline nodes. - cache_child_values::(&node, &mut values_to_cache, full_key.clone()); - } - - drop(node); - values_to_cache.into_iter().for_each(|(k, v)| cache.cache_value_for_key(&k, v)); + cache.insert_new_node(&hash); } } /// Cache the given `value`. /// - /// `hash` is the hash of `value`. - fn cache_value(&mut self, full_key: &[u8], value: impl Into, hash: TrieHash) { - if let Some(cache) = self.cache.as_mut() { - let value = value.into(); - - // `get_or_insert` should always return `Ok`, but be safe. - let value = if let Ok(value) = cache - .get_or_insert_node(hash, &mut || Ok(NodeOwned::Value(value.clone(), hash))) - .map(|n| n.data().cloned()) - { - value - } else { - None - }; - - if let Some(value) = value { - cache.cache_value_for_key(full_key, (value, hash).into()) - } - } - } + /// Cache is not done here as we want to cache the location from the db, + /// and location on new_nodes are not resolved here. + fn cache_value(&mut self, _full_key: &[u8], _value: impl Into, _hash: TrieHash) {} /// Commit a node by hashing it and writing it to the db. Returns a /// `ChildReference` which in most cases carries a normal hash but for the @@ -1946,40 +2134,58 @@ where /// `into_encoded` method of `Node`. fn commit_child( &mut self, - handle: NodeHandle>, + handle: NodeHandle, L::Location>, prefix: &mut NibbleVec, - ) -> ChildReference> { + children: &mut Vec, L::Location>>, + ) -> ChildReference, L::Location> { match handle { - NodeHandle::Hash(hash) => ChildReference::Hash(hash), + NodeHandle::Hash(hash, location) => { + children.push(Changeset::Existing(ExistingChangesetNode { + hash, + prefix: prefix.as_owned_prefix(), + location, + })); + ChildReference::Hash(hash, location) + }, NodeHandle::InMemory(storage_handle) => { match self.storage.destroy(storage_handle) { - Stored::Cached(_, hash) => ChildReference::Hash(hash), + Stored::Cached(_, hash, location) => { + children.push(Changeset::Existing(ExistingChangesetNode { + hash, + prefix: prefix.as_owned_prefix(), + location, + })); + ChildReference::Hash(hash, location) + }, Stored::New(node) => { - // Reconstructs the full key - let full_key = self.cache.as_ref().and_then(|_| { - let mut prefix = prefix.clone(); - if let Some(partial) = node.partial_key() { - prefix.append_partial(NibbleSlice::from_stored(partial).right()); - } - Some(prefix) - }); - - let encoded = { - let commit_child = |node: NodeToEncode>, + let mut sub_children = Vec::new(); + let (encoded, child_set) = { + let commit_child = |node: NodeToEncode, L::Location>, o_slice: Option<&NibbleSlice>, o_index: Option| { let mov = prefix.append_optional_slice_and_nibble(o_slice, o_index); match node { NodeToEncode::Node(value) => { - let value_hash = self.db.insert(prefix.as_prefix(), value); + let value_hash = self.db.hash(value); + sub_children.push(Changeset::New(NewChangesetNode { + hash: value_hash, + prefix: prefix.as_owned_prefix(), + data: value.to_vec(), //TODO: avoid allocation + children: Default::default(), + removed_keys: None, + })); self.cache_value(prefix.inner(), value, value_hash); prefix.drop_lasts(mov); - ChildReference::Hash(value_hash) + ChildReference::Hash(value_hash, Default::default()) }, NodeToEncode::TrieNode(node_handle) => { - let result = self.commit_child(node_handle, prefix); + let result = self.commit_child( + node_handle, + prefix, + &mut sub_children, + ); prefix.drop_lasts(mov); result }, @@ -1987,13 +2193,20 @@ where }; node.into_encoded(commit_child) }; - if encoded.len() >= L::Hash::LENGTH { - let hash = self.db.insert(prefix.as_prefix(), &encoded); - self.hash_count += 1; - - self.cache_node(hash, &encoded, full_key); - - ChildReference::Hash(hash) + let result = if encoded.len() >= L::Hash::LENGTH { + let hash = self.db.hash(&encoded); + self.cache_node(hash); + child_set.map(|c| { + sub_children.push(*c); + }); + children.push(Changeset::New(NewChangesetNode { + hash, + prefix: prefix.as_owned_prefix(), + data: encoded, + children: sub_children, + removed_keys: None, + })); + ChildReference::Hash(hash, Default::default()) } else { // it's a small value, so we cram it into a `TrieHash` // and tag with length @@ -2002,7 +2215,8 @@ where h.as_mut()[..len].copy_from_slice(&encoded[..len]); ChildReference::Inline(h, len) - } + }; + result }, } }, @@ -2010,26 +2224,16 @@ where } // a hack to get the root node's handle - fn root_handle(&self) -> NodeHandle> { + fn root_handle(&self) -> NodeHandle, L::Location> { match self.root_handle { - NodeHandle::Hash(h) => NodeHandle::Hash(h), + NodeHandle::Hash(h, l) => NodeHandle::Hash(h, l), NodeHandle::InMemory(StorageHandle(x)) => NodeHandle::InMemory(StorageHandle(x)), } } -} -impl<'a, L> TrieMut for TrieDBMut<'a, L> -where - L: TrieLayout, -{ - fn root(&mut self) -> &TrieHash { - self.commit(); - self.root - } - - fn is_empty(&self) -> bool { + pub fn is_empty(&self) -> bool { match self.root_handle { - NodeHandle::Hash(h) => h == L::Codec::hashed_null_node(), + NodeHandle::Hash(h, _) => h == L::Codec::hashed_null_node(), NodeHandle::InMemory(ref h) => match self.storage[h] { Node::Empty => true, _ => false, @@ -2037,18 +2241,39 @@ where } } - fn get<'x, 'key>(&'x self, key: &'key [u8]) -> Result, TrieHash, CError> + pub fn get<'x, 'key>( + &'x self, + key: &'key [u8], + ) -> Result, TrieHash, CError> where 'x: 'key, { self.lookup(key, NibbleSlice::new(key), &self.root_handle) } - fn insert( + pub fn insert( + &mut self, + key: &[u8], + value: &[u8], + ) -> Result>, TrieHash, CError> { + self.insert_with_tree_ref(key, value, None) + } + + pub fn insert_with_tree_ref( &mut self, key: &[u8], value: &[u8], + tree_ref: Option>, ) -> Result>, TrieHash, CError> { + // expect for the child changes to have a key. + debug_assert!(tree_ref + .as_ref() + .map(|c| if let Changeset::New(set) = c.as_ref() { + set.removed_keys.is_some() + } else { + true + }) + .unwrap_or(true)); if !L::ALLOW_EMPTY && value.is_empty() { return self.remove(key) } @@ -2061,7 +2286,7 @@ where let value = Bytes::from(value); let root_handle = self.root_handle(); let (new_handle, _changed) = - self.insert_at(root_handle, &mut NibbleSlice::new(key), value, &mut old_val)?; + self.insert_at(root_handle, &mut NibbleSlice::new(key), value, &mut old_val, tree_ref)?; #[cfg(feature = "std")] trace!(target: "trie", "insert: altered trie={}", _changed); @@ -2070,7 +2295,25 @@ where Ok(old_val) } - fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { + pub fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { + self.remove_with_tree_ref(key, None) + } + + pub fn remove_with_tree_ref( + &mut self, + key: &[u8], + tree_ref: Option>, + ) -> Result>, TrieHash, CError> { + // expect for the child changes to have a key. + debug_assert!(tree_ref + .as_ref() + .map(|c| if let Changeset::New(set) = c.as_ref() { + set.removed_keys.is_some() + } else { + true + }) + .unwrap_or(true)); + #[cfg(feature = "std")] trace!(target: "trie", "remove: key={:?}", ToHex(key)); @@ -2078,7 +2321,7 @@ where let mut key_slice = NibbleSlice::new(key); let mut old_val = None; - match self.remove_at(root_handle, &mut key_slice, &mut old_val)? { + match self.remove_at(root_handle, &mut key_slice, &mut old_val, tree_ref)? { Some((handle, _changed)) => { #[cfg(feature = "std")] trace!(target: "trie", "remove: altered trie={}", _changed); @@ -2087,8 +2330,8 @@ where None => { #[cfg(feature = "std")] trace!(target: "trie", "remove: obliterated trie"); - self.root_handle = NodeHandle::Hash(L::Codec::hashed_null_node()); - *self.root = L::Codec::hashed_null_node(); + let handle = self.storage.alloc(Stored::New(Node::Empty)); + self.root_handle = NodeHandle::InMemory(handle); }, } @@ -2096,15 +2339,6 @@ where } } -impl<'a, L> Drop for TrieDBMut<'a, L> -where - L: TrieLayout, -{ - fn drop(&mut self) { - self.commit(); - } -} - /// combine two NodeKeys fn combine_key(start: &mut NodeKey, end: (usize, &[u8])) { debug_assert!(start.0 < nibble_ops::NIBBLE_PER_BYTE); diff --git a/test-support/keccak-hasher/CHANGELOG.md b/test-support/keccak-hasher/CHANGELOG.md deleted file mode 100644 index 4da7ff58..00000000 --- a/test-support/keccak-hasher/CHANGELOG.md +++ /dev/null @@ -1,11 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [0.16.0] - 2023-03-14 -- Switch to `hash-db` 0.16. [#188](https://github.com/paritytech/trie/pull/188) - -## [0.15.3] - 2020-07-24 -- Update `tiny-keccak` to 0.2. [#105](https://github.com/paritytech/trie/pull/105) diff --git a/test-support/keccak-hasher/Cargo.toml b/test-support/keccak-hasher/Cargo.toml deleted file mode 100644 index d933d44d..00000000 --- a/test-support/keccak-hasher/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "keccak-hasher" -version = "0.16.0" -authors = ["Parity Technologies "] -description = "Keccak-256 implementation of the Hasher trait" -repository = "https://github.com/paritytech/parity/" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -tiny-keccak = { version = "2.0.2", features = ["keccak"] } -hash-db = { path = "../../hash-db", default-features = false, version = "0.16.0" } -hash256-std-hasher = { path = "../../hash256-std-hasher", version = "0.15.2" } - -[features] -default = ["std"] -std = [ - "hash-db/std", -] diff --git a/test-support/trie-bench/CHANGELOG.md b/test-support/trie-bench/CHANGELOG.md deleted file mode 100644 index 8634244b..00000000 --- a/test-support/trie-bench/CHANGELOG.md +++ /dev/null @@ -1,66 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [Unreleased] - -## [0.37.0] - 2023-03-14 -- Update dependencies. [#188](https://github.com/paritytech/trie/pull/188) and [#187](https://github.com/paritytech/trie/pull/187) - -## [0.36.0] - 2023-02-24 -- Updated `trie-db` to 0.26.0. [#185](https://github.com/paritytech/trie/pull/185) - -## [0.35.0] - 2023-02-03 -- Updated `trie-db` to 0.25.0. - -## [0.34.0] - 2023-01-23 -- Updated `criterion` to 0.4. [#176](https://github.com/paritytech/trie/pull/176) - -## [0.33.0] - 2022-11-29 -- Updated `memory-db` to 0.31. [#172](https://github.com/paritytech/trie/pull/172) - -## [0.32.0] - 2022-09-20 -- Updated `memory-db` to 0.30. [#166](https://github.com/paritytech/trie/pull/166) - -## [0.31.0] - 2022-08-04 -- Update trie-db to 0.24.0. [#163](https://github.com/paritytech/trie/pull/163) - -## [0.30.0] - 2022-02-04 -- Updated `memory-db` to 0.29. [#150](https://github.com/paritytech/trie/pull/150) - -## [0.29.0] - 2021-10-19 -- Updated memory-db, triedb and trie-root. [#142](https://github.com/paritytech/trie/pull/142) - -## [0.28.0] - 2021-07-02 -- Updated memory-db to 0.27. [#139](https://github.com/paritytech/trie/pull/139) -- Updated parity-scale-codec to 2.0. [#137](https://github.com/paritytech/trie/pull/137) - -## [0.27.1] - 2021-06-24 -- Updated parity-scale-codec to 2.2.0-rc.2. - -## [0.27.0] - 2021-01-27 -- Updated memory-db to 0.26. -- Updated parity-scale-codec to 2.0. - -## [0.26.0] - 2021-01-05 -- Updated memory-db to 0.25. [#118](https://github.com/paritytech/trie/pull/118) - -## [0.25.0] - 2020-07-24 -- Updated criterion to 0.3. [#106](https://github.com/paritytech/trie/pull/106) - -## [0.24.0] - 2020-07-07 -- Updated memory-db to 0.24. [#99](https://github.com/paritytech/trie/pull/99) - -## [0.23.0] - 2020-07-06 -- Updated memory-db to 0.22. [#98](https://github.com/paritytech/trie/pull/98) - -## [0.21.0] - 2020-03-21 -- Updated memory-db to 0.20.0 [#82](https://github.com/paritytech/trie/pull/82) - -## [0.20.0] - 2020-02-07 -- Updated trie-root to v0.16.0 and memory-db to v0.19.0 and trie-db to v0.20.0 [#78](https://github.com/paritytech/trie/pull/78) - -## [0.19.0] - 2020-01-17 -- Updated trie-db to 0.19.0. [#75](https://github.com/paritytech/trie/pull/75) diff --git a/test-support/trie-bench/Cargo.toml b/test-support/trie-bench/Cargo.toml deleted file mode 100644 index f8de1f96..00000000 --- a/test-support/trie-bench/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "trie-bench" -description = "Standard benchmarking suite for tries" -version = "0.37.0" -authors = ["Parity Technologies "] -repository = "https://github.com/paritytech/trie/" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -keccak-hasher = { path = "../keccak-hasher", version = "0.16.0" } -trie-standardmap = { path = "../trie-standardmap", version = "0.16.0" } -hash-db = { path = "../../hash-db" , version = "0.16.0"} -memory-db = { path = "../../memory-db", version = "0.32.0" } -trie-root = { path = "../../trie-root", version = "0.18.0" } -trie-db = { path = "../../trie-db", version = "0.27.0" } -criterion = "0.4.0" -parity-scale-codec = "3.0.0" diff --git a/test-support/trie-standardmap/CHANGELOG.md b/test-support/trie-standardmap/CHANGELOG.md deleted file mode 100644 index 39a5cbbf..00000000 --- a/test-support/trie-standardmap/CHANGELOG.md +++ /dev/null @@ -1,8 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [0.16.0] - 2023-03-14 -- Update dependencies. [#188](https://github.com/paritytech/trie/pull/188) and [#187](https://github.com/paritytech/trie/pull/187) diff --git a/test-support/trie-standardmap/Cargo.toml b/test-support/trie-standardmap/Cargo.toml deleted file mode 100644 index b1d2d199..00000000 --- a/test-support/trie-standardmap/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "trie-standardmap" -description = "Standard test map for profiling tries" -version = "0.16.0" -authors = ["Parity Technologies "] -license = "Apache-2.0" -edition = "2018" - -[dependencies] -keccak-hasher = { path = "../keccak-hasher", version = "0.16.0"} -hash-db = { path = "../../hash-db" , version = "0.16.0"} diff --git a/trie-db/test/Cargo.toml b/test/Cargo.toml similarity index 56% rename from trie-db/test/Cargo.toml rename to test/Cargo.toml index 547df945..14cbc60d 100644 --- a/trie-db/test/Cargo.toml +++ b/test/Cargo.toml @@ -11,15 +11,16 @@ edition = "2018" name = "bench" harness = false +[[bench]] +name = "memory_db" +harness = false + [dependencies] -trie-db = { path = "..", version = "0.27.0"} -hash-db = { path = "../../hash-db", version = "0.16.0"} -memory-db = { path = "../../memory-db", version = "0.32.0" } +trie-db = { package = "subtrie", path = "../subtrie", version = "0.0.1", features = ["test_utils", "bench"]} rand = { version = "0.8", default-features = false, features = ["small_rng"] } -trie-standardmap = { path = "../../test-support/trie-standardmap", version = "0.16.0" } -reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } -hex-literal = "0.3" -criterion = "0.4.0" +reference-trie = { path = "../reference-trie", version = "0.29.0" } +hex-literal = "0.4" +criterion = "0.5.1" env_logger = { version = "0.10", default-features = false } log = "0.4" diff --git a/trie-db/test/benches/bench.rs b/test/benches/bench.rs similarity index 98% rename from trie-db/test/benches/bench.rs rename to test/benches/bench.rs index bd2611d5..93075ffa 100644 --- a/trie-db/test/benches/bench.rs +++ b/test/benches/bench.rs @@ -17,10 +17,11 @@ use std::collections::BTreeMap; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; use reference_trie::ExtensionLayout as Layout; use trie_db::{ + memory_db, proof::{generate_proof, verify_proof}, + test_utils::{Alphabet, StandardMap, ValueMode}, NibbleSlice, Trie, }; -use trie_standardmap::{Alphabet, StandardMap, ValueMode}; criterion_group!( benches, @@ -254,7 +255,6 @@ fn trie_mut_ref_root(c: &mut Criterion) { fn trie_mut(c: &mut Criterion) { use memory_db::HashKey; - use trie_db::TrieMut; let params = vec![(29, 204800 / 2, 512 * 2), (29, 204800, 32)]; @@ -265,10 +265,8 @@ fn trie_mut(c: &mut Criterion) { let param = format!("seed({}), len({}), value_length({})", seed, len, value_length); group.bench_with_input(BenchmarkId::new("trie_mut", param), &input, |b, i| { b.iter(|| { - let mut root = Default::default(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - let mut trie = - trie_db::TrieDBMutBuilder::::new(&mut mdb, &mut root).build(); + let mut trie = trie_db::TrieDBMutBuilder::::new(&mut mdb).build(); for (key, value) in i { trie.insert(&key, &value).expect( "changes trie: insertion to trie is not allowed to fail within runtime", diff --git a/memory-db/benches/bench.rs b/test/benches/memory_db.rs similarity index 95% rename from memory-db/benches/bench.rs rename to test/benches/memory_db.rs index b3e0fd9a..148393f3 100644 --- a/memory-db/benches/bench.rs +++ b/test/benches/memory_db.rs @@ -13,9 +13,11 @@ // limitations under the License. use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; -use keccak_hasher::KeccakHasher; -use memory_db::{HashKey, MemoryDB}; +use trie_db::{ + keccak_hasher::KeccakHasher, + memory_db::{HashKey, MemoryDB}, + node_db::{Hasher, EMPTY_PREFIX}, +}; criterion_group!( benches, diff --git a/test/src/double_ended_iterator.rs b/test/src/double_ended_iterator.rs new file mode 100644 index 00000000..77fa4ce1 --- /dev/null +++ b/test/src/double_ended_iterator.rs @@ -0,0 +1,410 @@ +// Copyright 2017, 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use hex_literal::hex; +use reference_trie::test_layouts; +use trie_db::{ + node::Node, node_db::Hasher, NibbleSlice, TrieDBBuilder, TrieDBNodeDoubleEndedIterator, + TrieDoubleEndedIterator, TrieLayout, +}; + +use crate::{ + iterator::{build_trie_db, nibble_vec}, + TestDB, +}; + +test_layouts!(node_double_ended_iterator_works, node_double_ended_iterator); +fn node_double_ended_iterator>() { + let pairs = vec![ + (hex!("01").to_vec(), b"aaaa".to_vec()), + (hex!("0123").to_vec(), b"bbbb".to_vec()), + (hex!("02").to_vec(), vec![1; 32]), + ]; + + let (memdb, root) = build_trie_db::(&pairs); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); + + if T::USE_EXTENSION { + match iter.next_back() { + Some(Ok((prefix, Some(_), node))) => { + assert_eq!(prefix, nibble_vec(hex!("02"), 2)); + match node.node() { + Node::Leaf(partial, _) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, None, node))) => { + assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); + match node.node() { + Node::Leaf(partial, _) => + assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)), + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, None, node))) => { + assert_eq!(prefix, nibble_vec(hex!("01"), 2)); + match node.node() { + Node::Branch(_, _) => {}, + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, Some(_), node))) => { + assert_eq!(prefix, nibble_vec(hex!("00"), 1)); + match node.node() { + Node::Branch(_, _) => {}, + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, Some(_), node))) => { + assert_eq!(prefix, nibble_vec(hex!(""), 0)); + match node.node() { + Node::Extension(partial, _) => + assert_eq!(partial, NibbleSlice::new_offset(&hex!("00")[..], 1)), + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + assert!(iter.next_back().is_none()); + } else { + let can_expand = + T::MAX_INLINE_VALUE.unwrap_or(T::Hash::LENGTH as u32) < T::Hash::LENGTH as u32; + + match iter.next_back() { + Some(Ok((prefix, Some(_), node))) => { + assert_eq!(prefix, nibble_vec(hex!("02"), 2)); + match node.node() { + Node::Leaf(partial, _) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, hash, node))) => { + if !can_expand { + assert!(hash.is_none()); + } + assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); + match node.node() { + Node::Leaf(partial, _) => + assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)), + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, hash, node))) => { + if !can_expand { + assert!(hash.is_none()); + } + assert_eq!(prefix, nibble_vec(hex!("01"), 2)); + match node.node() { + Node::NibbledBranch(partial, _, _) => + assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)), + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, Some(_), node))) => { + assert_eq!(prefix, nibble_vec(hex!(""), 0)); + match node.node() { + Node::NibbledBranch(partial, _, _) => + assert_eq!(partial, NibbleSlice::new_offset(&hex!("00")[..], 1)), + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + assert!(iter.next_back().is_none()); + } +} + +test_layouts!(seek_back_over_empty_works, seek_back_over_empty_works_internal); +fn seek_back_over_empty_works_internal>() { + let (memdb, root) = build_trie_db::(&[]); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); + + >::seek(&mut iter, &hex!("")[..]).unwrap(); + match iter.next_back() { + Some(Ok((prefix, _, node))) => { + assert_eq!(prefix, nibble_vec(hex!(""), 0)); + match node.node() { + Node::Empty => {}, + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + assert!(iter.next_back().is_none()); + + >::seek(&mut iter, &hex!("00")[..]).unwrap(); + match iter.next_back() { + Some(Ok((prefix, _, node))) => { + assert_eq!(prefix, nibble_vec(hex!(""), 0)); + match node.node() { + Node::Empty => {}, + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } +} + +test_layouts!(seek_back_works, seek_back_works_internal); +fn seek_back_works_internal>() { + let pairs = vec![ + (hex!("01").to_vec(), b"aaaa".to_vec()), + (hex!("0123").to_vec(), b"bbbb".to_vec()), + (hex!("0122").to_vec(), b"cccc".to_vec()), + (hex!("02").to_vec(), vec![1; 32]), + ]; + + let (memdb, root) = build_trie_db::(&pairs); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); + + >::seek(&mut iter, &hex!("")[..]).unwrap(); + match iter.next_back() { + Some(Ok((prefix, _, _))) => assert_eq!(prefix, nibble_vec(hex!(""), 0)), + _ => panic!("unexpected item"), + } + + >::seek(&mut iter, &hex!("03")[..]).unwrap(); + match iter.next_back() { + Some(Ok((prefix, _, _))) => assert_eq!(prefix, nibble_vec(hex!("02"), 2)), + _ => panic!("unexpected item"), + } + + >::seek(&mut iter, &hex!("02")[..]).unwrap(); + match iter.next_back() { + Some(Ok((prefix, _, _))) => assert_eq!(prefix, nibble_vec(hex!("02"), 2)), + _ => panic!("unexpected item"), + } + + >::seek(&mut iter, &hex!("01")[..]).unwrap(); + match iter.next_back() { + Some(Ok((prefix, _, _))) => { + assert_eq!(prefix, nibble_vec(hex!("0123"), 4)); + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, _, _))) => { + assert_eq!(prefix, nibble_vec(hex!("0122"), 4)); + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, _, _))) => { + assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, _, _))) => { + assert_eq!(prefix, nibble_vec(hex!("01"), 2)); + }, + _ => panic!("unexpected item"), + } + + >::seek(&mut iter, &hex!("0125")[..]).unwrap(); + match iter.next_back() { + Some(Ok((prefix, _, _))) => { + assert_eq!(prefix, nibble_vec(hex!("0123"), 4)); + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, _, _))) => { + assert_eq!(prefix, nibble_vec(hex!("0122"), 4)); + }, + _ => panic!("unexpected item"), + } + + >::seek(&mut iter, &hex!("0120")[..]).unwrap(); + match iter.next_back() { + Some(Ok((prefix, _, _))) => { + assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, _, _))) => { + assert_eq!(prefix, nibble_vec(hex!("01"), 2)); + }, + _ => panic!("unexpected item"), + } +} + +test_layouts!(prefix_back_works, prefix_back_works_internal); +fn prefix_back_works_internal>() { + let can_expand = T::MAX_INLINE_VALUE.unwrap_or(T::Hash::LENGTH as u32) < T::Hash::LENGTH as u32; + let pairs = vec![ + (hex!("01").to_vec(), b"aaaa".to_vec()), + (hex!("0123").to_vec(), b"bbbb".to_vec()), + (hex!("0122").to_vec(), b"cccc".to_vec()), + (hex!("02").to_vec(), vec![1; 32]), + ]; + + let (memdb, root) = build_trie_db::(&pairs); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); + + iter.prefix(&hex!("01").to_vec()[..]).unwrap(); + + if T::USE_EXTENSION { + match iter.next_back() { + Some(Ok((prefix, None, node))) => { + assert_eq!(prefix, nibble_vec(hex!("0123"), 4)); + match node.node() { + Node::Leaf(partial, _) => { + assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)) + }, + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + } else { + match iter.next_back() { + Some(Ok((prefix, hash, node))) => { + if !can_expand { + debug_assert!(hash.is_none()); + } + assert_eq!(prefix, nibble_vec(hex!("0123"), 4)); + match node.node() { + Node::Leaf(partial, _) => { + assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)) + }, + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + } + + match iter.next_back() { + Some(Ok((prefix, hash, node))) => { + if !can_expand { + debug_assert!(hash.is_none()); + } + assert_eq!(prefix, nibble_vec(hex!("0122"), 4)); + match node.node() { + Node::Leaf(partial, _) => { + assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)) + }, + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, hash, node))) => { + if !can_expand { + debug_assert!(hash.is_none()); + } + assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); + match node.node() { + Node::NibbledBranch(partial, _, _) => + assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)), + Node::Branch(_, _) => {}, + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + match iter.next_back() { + Some(Ok((prefix, hash, node))) => { + if !can_expand { + debug_assert!(hash.is_none()); + } + assert_eq!(prefix, nibble_vec(hex!("01"), 2)); + match node.node() { + Node::NibbledBranch(partial, _, _) => + assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)), + Node::Branch(_, _) => {}, + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + assert!(iter.next_back().is_none()); + + let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); + iter.prefix(&hex!("0010").to_vec()[..]).unwrap(); + assert!(iter.next_back().is_none()); + let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); + iter.prefix(&hex!("10").to_vec()[..]).unwrap(); + assert!(iter.next_back().is_none()); +} + +test_layouts!(prefix_over_empty_works, prefix_over_empty_works_internal); +fn prefix_over_empty_works_internal>() { + let (memdb, root) = build_trie_db::(&[]); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); + iter.prefix(&hex!("")[..]).unwrap(); + match iter.next_back() { + Some(Ok((prefix, Some(_), node))) => { + assert_eq!(prefix, nibble_vec(hex!(""), 0)); + match node.node() { + Node::Empty => {}, + _ => panic!("unexpected node"), + } + }, + _ => panic!("unexpected item"), + } + + assert!(iter.next_back().is_none()); + + let mut iter = TrieDBNodeDoubleEndedIterator::new(&trie).unwrap(); + iter.prefix(&hex!("00")[..]).unwrap(); + assert!(iter.next_back().is_none()); +} diff --git a/trie-db/test/src/iter_build.rs b/test/src/iter_build.rs similarity index 84% rename from trie-db/test/src/iter_build.rs rename to test/src/iter_build.rs index bb4c0d85..546f6ed9 100644 --- a/trie-db/test/src/iter_build.rs +++ b/test/src/iter_build.rs @@ -12,12 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ test_layouts, ExtensionLayout, HashedValueNoExt, HashedValueNoExtThreshold, NoExtensionLayout, - RefHasher, }; -use trie_db::{DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieMut}; +use trie_db::{ + memory_db::{HashKey, MemoryDB, PrefixedKey}, + DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, +}; + +use crate::TestDB; #[test] fn trie_root_empty() { @@ -39,16 +42,17 @@ fn root_extension_one() { fn test_iter(data: Vec<(Vec, Vec)>) { let mut db = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); + let changeset = { + let mut t = TrieDBMutBuilder::::new(&mut db).build(); for i in 0..data.len() { let key: &[u8] = &data[i].0; let value: &[u8] = &data[i].1; t.insert(key, value).unwrap(); } - } - let t = TrieDBBuilder::::new(&db, &root).build(); + t.commit() + }; + changeset.apply_to(&mut db); + let t = TrieDBBuilder::::new(&db, changeset.hash()).build(); for (i, kv) in t.iter().unwrap().enumerate() { let (k, v) = kv.unwrap(); let key: &[u8] = &data[i].0; @@ -62,7 +66,7 @@ fn test_iter(data: Vec<(Vec, Vec)>) { } fn compare_implementations(data: Vec<(Vec, Vec)>) { - test_iter::>(data.clone()); + test_iter::>(data.clone()); test_iter::(data.clone()); test_iter::(data.clone()); test_iter::(data.clone()); @@ -71,38 +75,37 @@ fn compare_implementations(data: Vec<(Vec, Vec)>) { } fn compare_implementations_prefixed(data: Vec<(Vec, Vec)>) { - compare_implementations_prefixed_internal::>(data.clone()); + compare_implementations_prefixed_internal::>(data.clone()); compare_implementations_prefixed_internal::(data.clone()); compare_implementations_prefixed_internal::(data.clone()); compare_implementations_prefixed_internal::(data.clone()); } -fn compare_implementations_prefixed_internal(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations::(data, memdb, hashdb); +fn compare_implementations_prefixed_internal(data: Vec<(Vec, Vec)>) +where + T::Location: std::fmt::Debug, +{ + reference_trie::compare_implementations::>(data); } fn compare_implementations_h(data: Vec<(Vec, Vec)>) { - compare_implementations_h_internal::>(data.clone()); + compare_implementations_h_internal::>(data.clone()); compare_implementations_h_internal::(data.clone()); compare_implementations_h_internal::(data.clone()); compare_implementations_h_internal::(data.clone()); } -fn compare_implementations_h_internal(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations::(data.clone(), memdb, hashdb); +fn compare_implementations_h_internal(data: Vec<(Vec, Vec)>) +where + T::Location: std::fmt::Debug, +{ + reference_trie::compare_implementations::>(data.clone()); } fn compare_implementations_no_extension_unordered(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::<_, HashKey<_>, _>::default(); - let hashdb = MemoryDB::, DBValue>::default(); - reference_trie::compare_implementations_unordered::(data, memdb, hashdb); + reference_trie::compare_implementations_unordered::>(data); } fn compare_insert_remove(data: Vec<(bool, Vec, Vec)>) { - let memdb = MemoryDB::<_, PrefixedKey<_>, _>::default(); - reference_trie::compare_insert_remove::(data, memdb); + reference_trie::compare_insert_remove::>(data); } -fn compare_root(data: Vec<(Vec, Vec)>) { - let memdb = MemoryDB::, _>::default(); +fn compare_root>(data: Vec<(Vec, Vec)>) { + let memdb = DB::default(); reference_trie::compare_root::(data, memdb); } fn compare_unhashed(data: Vec<(Vec, Vec)>) { @@ -132,8 +135,8 @@ fn trie_middle_node2() { ]); } test_layouts!(root_extension_bis, root_extension_bis_internal); -fn root_extension_bis_internal() { - compare_root::(vec![ +fn root_extension_bis_internal>() { + compare_root::(vec![ (vec![1u8, 2u8, 3u8, 3u8], vec![8u8; 32]), (vec![1u8, 2u8, 3u8, 4u8], vec![7u8; 32]), ]); @@ -239,7 +242,7 @@ fn fuzz_no_extension4() { ]); } test_layouts!(fuzz_no_extension_insert_remove_1, fuzz_no_extension_insert_remove_1_internal); -fn fuzz_no_extension_insert_remove_1_internal() { +fn fuzz_no_extension_insert_remove_1_internal>() { let data = vec![ (false, vec![0], vec![251, 255]), (false, vec![0, 1], vec![251, 255]), @@ -249,7 +252,7 @@ fn fuzz_no_extension_insert_remove_1_internal() { compare_insert_remove::(data); } test_layouts!(fuzz_no_extension_insert_remove_2, fuzz_no_extension_insert_remove_2_internal); -fn fuzz_no_extension_insert_remove_2_internal() { +fn fuzz_no_extension_insert_remove_2_internal>() { let data = vec![ (false, vec![0x00], vec![0xfd, 0xff]), (false, vec![0x10, 0x00], vec![1; 32]), diff --git a/trie-db/test/src/iterator.rs b/test/src/iterator.rs similarity index 89% rename from trie-db/test/src/iterator.rs rename to test/src/iterator.rs index e36ef6d0..4225e5b0 100644 --- a/trie-db/test/src/iterator.rs +++ b/test/src/iterator.rs @@ -12,36 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::{HashDB, Hasher}; use hex_literal::hex; use reference_trie::test_layouts; use trie_db::{ node::{Node, Value}, - DBValue, NibbleSlice, NibbleVec, TrieDBBuilder, TrieDBNodeIterator, TrieError, TrieIterator, - TrieLayout, TrieMut, + node_db::Hasher, + NibbleSlice, NibbleVec, TrieDBBuilder, TrieDBNodeIterator, TrieError, TrieIterator, TrieLayout, }; -type MemoryDB = memory_db::MemoryDB< - ::Hash, - memory_db::PrefixedKey<::Hash>, - DBValue, ->; +use crate::TestDB; -fn build_trie_db( +pub(crate) fn build_trie_db>( pairs: &[(Vec, Vec)], -) -> (MemoryDB, ::Out) { - let mut memdb = MemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = trie_db::TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); +) -> (DB, ::Out) { + let mut memdb = DB::default(); + let changeset = { + let mut t = trie_db::TrieDBMutBuilder::::new(&mut memdb).build(); for (x, y) in pairs.iter() { t.insert(x, y).unwrap(); } - } + t.commit() + }; + let root = memdb.commit(changeset); (memdb, root) } -fn nibble_vec>(bytes: T, len: usize) -> NibbleVec { +pub(crate) fn nibble_vec>(bytes: T, len: usize) -> NibbleVec { let slice = NibbleSlice::new(bytes.as_ref()); let mut v = NibbleVec::new(); @@ -52,14 +48,14 @@ fn nibble_vec>(bytes: T, len: usize) -> NibbleVec { } test_layouts!(iterator_works, iterator_works_internal); -fn iterator_works_internal() { +fn iterator_works_internal>() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db::(&pairs); + let (memdb, root) = build_trie_db::(&pairs); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); @@ -184,8 +180,8 @@ fn iterator_works_internal() { } test_layouts!(iterator_over_empty_works, iterator_over_empty_works_internal); -fn iterator_over_empty_works_internal() { - let (memdb, root) = build_trie_db::(&[]); +fn iterator_over_empty_works_internal>() { + let (memdb, root) = build_trie_db::(&[]); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); @@ -204,14 +200,14 @@ fn iterator_over_empty_works_internal() { } test_layouts!(seek_works, seek_works_internal); -fn seek_works_internal() { +fn seek_works_internal>() { let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db::(&pairs); + let (memdb, root) = build_trie_db::(&pairs); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); @@ -244,8 +240,8 @@ fn seek_works_internal() { } test_layouts!(seek_over_empty_works, seek_over_empty_works_internal); -fn seek_over_empty_works_internal() { - let (memdb, root) = build_trie_db::(&[]); +fn seek_over_empty_works_internal>() { + let (memdb, root) = build_trie_db::(&[]); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); @@ -266,7 +262,10 @@ fn seek_over_empty_works_internal() { } test_layouts!(iterate_over_incomplete_db, iterate_over_incomplete_db_internal); -fn iterate_over_incomplete_db_internal() { +fn iterate_over_incomplete_db_internal>() +where + T::Location: std::fmt::Debug, +{ let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), (hex!("0123").to_vec(), b"bbbb".to_vec()), @@ -274,7 +273,7 @@ fn iterate_over_incomplete_db_internal() { (hex!("03").to_vec(), vec![2; 32]), ]; - let (mut memdb, root) = build_trie_db::(&pairs); + let (mut memdb, root) = build_trie_db::(&pairs); // Look up the leaf node with prefix "02". let leaf_hash = { @@ -338,7 +337,7 @@ fn iterate_over_incomplete_db_internal() { } test_layouts!(prefix_works, prefix_works_internal); -fn prefix_works_internal() { +fn prefix_works_internal>() { let can_expand = T::MAX_INLINE_VALUE.unwrap_or(T::Hash::LENGTH as u32) < T::Hash::LENGTH as u32; let pairs = vec![ (hex!("01").to_vec(), b"aaaa".to_vec()), @@ -346,7 +345,7 @@ fn prefix_works_internal() { (hex!("02").to_vec(), vec![1; 32]), ]; - let (memdb, root) = build_trie_db::(&pairs); + let (memdb, root) = build_trie_db::(&pairs); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); @@ -407,8 +406,8 @@ fn prefix_works_internal() { } test_layouts!(prefix_over_empty_works, prefix_over_empty_works_internal); -fn prefix_over_empty_works_internal() { - let (memdb, root) = build_trie_db::(&[]); +fn prefix_over_empty_works_internal>() { + let (memdb, root) = build_trie_db::(&[]); let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); iter.prefix(&hex!("")[..]).unwrap(); diff --git a/test/src/lib.rs b/test/src/lib.rs new file mode 100644 index 00000000..d0a7a688 --- /dev/null +++ b/test/src/lib.rs @@ -0,0 +1,114 @@ +// Copyright 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Tests for subtrie crate. + +#[cfg(test)] +mod double_ended_iterator; +#[cfg(test)] +mod iter_build; +#[cfg(test)] +mod iterator; +#[cfg(test)] +mod proof; +#[cfg(test)] +mod recorder; +#[cfg(test)] +mod trie_codec; +#[cfg(test)] +mod trie_root; +#[cfg(test)] +mod triedb; +#[cfg(test)] +mod triedbmut; + +#[cfg(test)] +use trie_db::{ + mem_tree_db::{Location, MemTreeDB}, + memory_db::{KeyFunction, MemoryDB}, + node_db::{self, Hasher, Prefix}, + Changeset, DBValue, TrieHash, TrieLayout, +}; + +#[cfg(test)] +trait TestDB: node_db::NodeDB + Clone + Default { + fn commit( + &mut self, + commit: trie_db::Changeset<<::Hash as Hasher>::Out, T::Location>, + ) -> TrieHash; + fn remove(&mut self, hash: &::Out, prefix: Prefix); + fn is_empty(&self) -> bool; + fn support_location() -> bool { + false + } +} + +#[cfg(test)] +impl, H, KF> TestDB for MemoryDB +where + H: Hasher, + KF: KeyFunction + Send + Sync, +{ + fn commit( + &mut self, + commit: trie_db::Changeset::Location>, + ) -> H::Out { + commit.apply_to(self) + } + + fn remove(&mut self, hash: &::Out, prefix: Prefix) { + MemoryDB::remove(self, hash, prefix); + } + + fn is_empty(&self) -> bool { + self.keys().is_empty() + } +} + +#[cfg(test)] +impl, H> TestDB for MemTreeDB +where + H: Hasher + Clone, +{ + fn commit(&mut self, commit: trie_db::Changeset) -> H::Out { + self.apply_commit(commit) + } + + fn remove(&mut self, hash: &H::Out, _prefix: Prefix) { + MemTreeDB::test_remove_node(self, hash); + } + + fn is_empty(&self) -> bool { + MemTreeDB::is_empty(self) + } + + fn support_location() -> bool { + true + } +} + +#[cfg(test)] +trait TestCommit { + fn commit_to>(self, db: &mut DB) -> TrieHash; +} + +#[cfg(test)] +impl> TestCommit for Changeset +where + T::Hash: Hasher, +{ + fn commit_to>(self, db: &mut DB) -> TrieHash { + db.commit(self) + } +} diff --git a/trie-db/test/src/proof.rs b/test/src/proof.rs similarity index 72% rename from trie-db/test/src/proof.rs rename to test/src/proof.rs index cca2c70e..331f43b0 100644 --- a/trie-db/test/src/proof.rs +++ b/test/src/proof.rs @@ -12,17 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::Hasher; use reference_trie::{test_layouts, NoExtensionLayout}; +use trie_db::node_db::Hasher; use trie_db::{ proof::{generate_proof, verify_proof, VerifyError}, - DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieMut, + DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, }; -type MemoryDB = memory_db::MemoryDB< +use crate::TestDB; + +type MemoryDB = trie_db::memory_db::MemoryDB< ::Hash, - memory_db::HashKey<::Hash>, + trie_db::memory_db::HashKey<::Hash>, DBValue, >; @@ -44,20 +46,19 @@ fn test_entries() -> Vec<(&'static [u8], &'static [u8])> { ] } -fn test_generate_proof( +fn test_generate_proof>( entries: Vec<(&'static [u8], &'static [u8])>, keys: Vec<&'static [u8]>, ) -> (::Out, Vec>, Vec<(&'static [u8], Option)>) { // Populate DB with full trie from entries. let (db, root) = { - let mut db = >::default(); - let mut root = Default::default(); - { - let mut trie = >::new(&mut db, &mut root).build(); - for (key, value) in entries.iter() { - trie.insert(key, value).unwrap(); - } + let mut db = DB::default(); + let mut trie = >::new(&db).build(); + for (key, value) in entries.iter() { + trie.insert(key, value).unwrap(); } + let commit = trie.commit(); + let root = db.commit(commit); (db, root) }; @@ -70,8 +71,8 @@ fn test_generate_proof( } test_layouts!(trie_proof_works2, trie_proof_works_internal2); -fn trie_proof_works_internal2() { - let (root, proof, items) = test_generate_proof::( +fn trie_proof_works_internal2>() { + let (root, proof, items) = test_generate_proof::( vec![ // "do" is at a hash-referenced branch node. (&b"do"[..], b"verb"), @@ -85,8 +86,8 @@ fn trie_proof_works_internal2() { } test_layouts!(trie_proof_works, trie_proof_works_internal); -fn trie_proof_works_internal() { - let (root, proof, items) = test_generate_proof::( +fn trie_proof_works_internal>() { + let (root, proof, items) = test_generate_proof::( test_entries(), vec![ b"do", b"dog", b"doge", b"bravo", b"alfabet", // None, not found under leaf node @@ -100,16 +101,16 @@ fn trie_proof_works_internal() { } test_layouts!(trie_proof_works_for_empty_trie, trie_proof_works_for_empty_trie_internal); -fn trie_proof_works_for_empty_trie_internal() { +fn trie_proof_works_for_empty_trie_internal>() { let (root, proof, items) = - test_generate_proof::(vec![], vec![b"alpha", b"bravo", b"\x42\x42"]); + test_generate_proof::(vec![], vec![b"alpha", b"bravo", b"\x42\x42"]); verify_proof::(&root, &proof, items.iter()).unwrap(); } test_layouts!(test_verify_duplicate_keys, test_verify_duplicate_keys_internal); -fn test_verify_duplicate_keys_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); +fn test_verify_duplicate_keys_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); let items = vec![(b"bravo", Some(b"bravo")), (b"bravo", Some(b"bravo"))]; assert!(if let Err(VerifyError::DuplicateKey(key)) = @@ -122,8 +123,8 @@ fn test_verify_duplicate_keys_internal() { } test_layouts!(test_verify_extraneaous_node, test_verify_extraneaous_node_internal); -fn test_verify_extraneaous_node_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo", b"do"]); +fn test_verify_extraneaous_node_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo", b"do"]); let items = vec![(b"bravo", Some(b"bravo"))]; assert!(matches!( @@ -133,8 +134,8 @@ fn test_verify_extraneaous_node_internal() { } test_layouts!(test_verify_extraneaous_value, test_verify_extraneaous_value_internal); -fn test_verify_extraneaous_value_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"doge"]); +fn test_verify_extraneaous_value_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"doge"]); let items = vec![(&b"do"[..], Some(&b"verb"[..])), (&b"doge"[..], Some(&[0; 32][..]))]; assert!(if let Err(VerifyError::ExtraneousValue(val)) = @@ -148,7 +149,10 @@ fn test_verify_extraneaous_value_internal() { #[test] fn test_verify_extraneous_hash_reference() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"do"]); + let (root, proof, _) = test_generate_proof::>( + test_entries(), + vec![b"do"], + ); let items = vec![(&b"alfa"[..], Some(&[0; 32][..])), (&b"do"[..], Some(&b"verb"[..]))]; match verify_proof::(&root, &proof, items.iter()) { @@ -158,8 +162,8 @@ fn test_verify_extraneous_hash_reference() { } test_layouts!(test_verify_invalid_child_reference, test_verify_invalid_child_reference_internal); -fn test_verify_invalid_child_reference_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); +fn test_verify_invalid_child_reference_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); if T::MAX_INLINE_VALUE.map_or(false, |t| t as usize <= b"bravo".len()) { // node will not be inline: ignore test @@ -178,8 +182,8 @@ test_layouts!( test_verify_value_mismatch_some_to_none, test_verify_value_mismatch_some_to_none_internal ); -fn test_verify_value_mismatch_some_to_none_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"horse"]); +fn test_verify_value_mismatch_some_to_none_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"horse"]); let items = vec![(&b"horse"[..], Some(&b"stallion"[..])), (&b"halp"[..], Some(&b"plz"[..]))]; assert!(if let Err(VerifyError::ValueMismatch(val)) = @@ -195,8 +199,8 @@ test_layouts!( test_verify_value_mismatch_none_to_some, test_verify_value_mismatch_none_to_some_internal ); -fn test_verify_value_mismatch_none_to_some_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"alfa", b"bravo"]); +fn test_verify_value_mismatch_none_to_some_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"alfa", b"bravo"]); let items = vec![(&b"alfa"[..], Some(&[0; 32][..])), (&b"bravo"[..], None)]; assert!(if let Err(VerifyError::ValueMismatch(val)) = @@ -209,8 +213,8 @@ fn test_verify_value_mismatch_none_to_some_internal() { } test_layouts!(test_verify_incomplete_proof, test_verify_incomplete_proof_internal); -fn test_verify_incomplete_proof_internal() { - let (root, mut proof, items) = test_generate_proof::(test_entries(), vec![b"alfa"]); +fn test_verify_incomplete_proof_internal>() { + let (root, mut proof, items) = test_generate_proof::(test_entries(), vec![b"alfa"]); proof.pop(); assert!(matches!( @@ -220,8 +224,8 @@ fn test_verify_incomplete_proof_internal() { } test_layouts!(test_verify_root_mismatch, test_verify_root_mismatch_internal); -fn test_verify_root_mismatch_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); +fn test_verify_root_mismatch_internal>() { + let (root, proof, _) = test_generate_proof::(test_entries(), vec![b"bravo"]); let items = vec![(b"bravo", Some("incorrect"))]; match verify_proof::(&root, &proof, items.iter()) { @@ -231,8 +235,8 @@ fn test_verify_root_mismatch_internal() { } test_layouts!(test_verify_decode_error, test_verify_decode_error_internal); -fn test_verify_decode_error_internal() { - let (root, mut proof, items) = test_generate_proof::(test_entries(), vec![b"bravo"]); +fn test_verify_decode_error_internal>() { + let (root, mut proof, items) = test_generate_proof::(test_entries(), vec![b"bravo"]); proof.insert(0, b"this is not a trie node".to_vec()); match verify_proof::(&root, &proof, items.iter()) { diff --git a/trie-db/test/src/recorder.rs b/test/src/recorder.rs similarity index 83% rename from trie-db/test/src/recorder.rs rename to test/src/recorder.rs index 485d0153..baa39ad8 100644 --- a/trie-db/test/src/recorder.rs +++ b/test/src/recorder.rs @@ -14,26 +14,28 @@ //! Trie query recorder. -use memory_db::{HashKey, MemoryDB}; use reference_trie::{NoExtensionLayout, RefHasher, RefTrieDBBuilder, RefTrieDBMutBuilder}; -use trie_db::{Recorder, Trie, TrieMut}; +use trie_db::{ + memory_db::{HashKey, MemoryDB}, + Recorder, Trie, +}; #[test] fn trie_record() { let mut db = MemoryDB::, _>::default(); - let mut root = Default::default(); - { - let mut x = RefTrieDBMutBuilder::new(&mut db, &mut root).build(); + let mut x = RefTrieDBMutBuilder::new(&mut db).build(); - x.insert(b"dog", b"cat").unwrap(); - x.insert(b"lunch", b"time").unwrap(); - x.insert(b"notdog", b"notcat").unwrap(); - x.insert(b"hotdog", b"hotcat").unwrap(); - x.insert(b"letter", b"confusion").unwrap(); - x.insert(b"insert", b"remove").unwrap(); - x.insert(b"pirate", b"aargh!").unwrap(); - x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); - } + x.insert(b"dog", b"cat").unwrap(); + x.insert(b"lunch", b"time").unwrap(); + x.insert(b"notdog", b"notcat").unwrap(); + x.insert(b"hotdog", b"hotcat").unwrap(); + x.insert(b"letter", b"confusion").unwrap(); + x.insert(b"insert", b"remove").unwrap(); + x.insert(b"pirate", b"aargh!").unwrap(); + x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); + let commit = x.commit(); + let root = *commit.hash(); + commit.apply_to(&mut db); { let mut recorder = Recorder::::new(); diff --git a/trie-db/test/src/trie_codec.rs b/test/src/trie_codec.rs similarity index 79% rename from trie-db/test/src/trie_codec.rs rename to test/src/trie_codec.rs index 17b52cb5..6227bd2b 100644 --- a/trie-db/test/src/trie_codec.rs +++ b/test/src/trie_codec.rs @@ -12,33 +12,34 @@ // See the License for the specific language governing permissions and // limitations under the License. -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; use reference_trie::{test_layouts, ExtensionLayout}; use trie_db::{ - decode_compact, encode_compact, DBValue, NodeCodec, Recorder, Trie, TrieDBBuilder, - TrieDBMutBuilder, TrieError, TrieLayout, TrieMut, + decode_compact, encode_compact, + node_db::{Hasher, EMPTY_PREFIX}, + DBValue, NodeCodec, Recorder, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieError, TrieLayout, }; -type MemoryDB = memory_db::MemoryDB< +use crate::TestDB; + +type MemoryDB = trie_db::memory_db::MemoryDB< ::Hash, - memory_db::HashKey<::Hash>, + trie_db::memory_db::HashKey<::Hash>, DBValue, >; -fn test_encode_compact( +fn test_encode_compact>( entries: Vec<(&'static [u8], &'static [u8])>, keys: Vec<&'static [u8]>, ) -> (::Out, Vec>, Vec<(&'static [u8], Option)>) { // Populate DB with full trie from entries. let (db, root) = { - let mut db = >::default(); - let mut root = Default::default(); - { - let mut trie = >::new(&mut db, &mut root).build(); - for (key, value) in entries.iter() { - trie.insert(key, value).unwrap(); - } + let mut db = DB::default(); + let mut trie = >::new(&mut db).build(); + for (key, value) in entries.iter() { + trie.insert(key, value).unwrap(); } + let commit = trie.commit(); + let root = db.commit(commit); (db, root) }; @@ -77,7 +78,7 @@ fn test_decode_compact( ) { // Reconstruct the partial DB from the compact encoding. let mut db = MemoryDB::::default(); - let (root, used) = decode_compact::(&mut db, encoded).unwrap(); + let (root, used) = decode_compact::(&mut db, encoded).unwrap(); assert_eq!(root, expected_root); assert_eq!(used, expected_used); @@ -89,8 +90,8 @@ fn test_decode_compact( } test_layouts!(trie_compact_encoding_works, trie_compact_encoding_works_internal); -fn trie_compact_encoding_works_internal() { - let (root, mut encoded, items) = test_encode_compact::( +fn trie_compact_encoding_works_internal>() { + let (root, mut encoded, items) = test_encode_compact::( vec![ // "alfa" is at a hash-referenced leaf node. (b"alfa", &[0; 32]), @@ -122,15 +123,17 @@ test_layouts!( trie_decoding_fails_with_incomplete_database, trie_decoding_fails_with_incomplete_database_internal ); -fn trie_decoding_fails_with_incomplete_database_internal() { - let (_, encoded, _) = - test_encode_compact::(vec![(b"alfa", &[0; 32]), (b"bravo", b"bravo")], vec![b"alfa"]); +fn trie_decoding_fails_with_incomplete_database_internal>() { + let (_, encoded, _) = test_encode_compact::( + vec![(b"alfa", &[0; 32]), (b"bravo", b"bravo")], + vec![b"alfa"], + ); assert!(encoded.len() > 1); // Reconstruct the partial DB from the compact encoding. let mut db = MemoryDB::::default(); - match decode_compact::(&mut db, &encoded[..encoded.len() - 1]) { + match decode_compact::(&mut db, &encoded[..encoded.len() - 1]) { Err(err) => match *err { TrieError::IncompleteDatabase(_) => {}, _ => panic!("got unexpected TrieError"), @@ -160,14 +163,14 @@ fn encoding_node_owned_and_decoding_node_works() { // Populate DB with full trie from entries. let mut recorder = { let mut db = >::default(); - let mut root = Default::default(); let mut recorder = Recorder::::new(); - { - let mut trie = >::new(&mut db, &mut root).build(); - for (key, value) in entries.iter() { - trie.insert(key, value).unwrap(); - } + let mut trie = >::new(&mut db).build(); + for (key, value) in entries.iter() { + trie.insert(key, value).unwrap(); } + let commit = trie.commit(); + commit.apply_to(&mut db); + let root = commit.root_hash(); let trie = TrieDBBuilder::::new(&db, &root) .with_recorder(&mut recorder) @@ -181,7 +184,8 @@ fn encoding_node_owned_and_decoding_node_works() { for record in recorder.drain() { let node = - <::Codec as NodeCodec>::decode(&record.data).unwrap(); + <::Codec as NodeCodec>::decode(&record.data, &[(); 0]) + .unwrap(); let node_owned = node.to_owned_node::().unwrap(); assert_eq!(record.data, node_owned.to_encoded::<::Codec>()); diff --git a/test/src/trie_root.rs b/test/src/trie_root.rs new file mode 100644 index 00000000..bf4e5309 --- /dev/null +++ b/test/src/trie_root.rs @@ -0,0 +1,44 @@ +// Copyright 2017, 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test for trie-root module. + +use hex_literal::hex; +use reference_trie::ReferenceTrieStream; +use trie_db::{ + keccak_hasher::KeccakHasher, + trie_root::{sec_trie_root, trie_root}, +}; + +#[test] +fn previous_doc_test_1() { + let v = vec![("doe", "reindeer"), ("dog", "puppy"), ("dogglesworth", "cat")]; + + let root = hex!["d6e02b2bd48aa04fd2ad87cfac1144a29ca7f7dc60f4526c7b7040763abe3d43"]; + assert_eq!( + sec_trie_root::(v, Default::default()), + root + ); +} + +#[test] +fn previous_doc_test_2() { + let v = vec![("doe", "reindeer"), ("dog", "puppy"), ("dogglesworth", "cat")]; + + let root = hex!["0807d5393ae7f349481063ebb5dbaf6bda58db282a385ca97f37dccba717cb79"]; + assert_eq!( + trie_root::(v, Default::default()), + root + ); +} diff --git a/trie-db/test/src/triedb.rs b/test/src/triedb.rs similarity index 65% rename from trie-db/test/src/triedb.rs rename to test/src/triedb.rs index 9825ab50..0ff0b18a 100644 --- a/trie-db/test/src/triedb.rs +++ b/test/src/triedb.rs @@ -14,37 +14,38 @@ use std::ops::Deref; -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; use hex_literal::hex; -use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ - test_layouts, test_layouts_substrate, HashedValueNoExtThreshold, TestTrieCache, + test_layouts, test_layouts_substrate, HashedValueNoExtThreshold, PrefixedMemoryDB, + TestTrieCache, }; use trie_db::{ - encode_compact, CachedValue, DBValue, Lookup, NibbleSlice, Recorder, Trie, TrieCache, - TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieMut, + encode_compact, + memory_db::{HashKey, MemoryDB}, + node_db::{Hasher, EMPTY_PREFIX}, + CachedValue, DBValue, Lookup, NibbleSlice, RecordedForKey, Recorder, Trie, TrieCache, + TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieRecorder, }; -type PrefixedMemoryDB = - MemoryDB<::Hash, PrefixedKey<::Hash>, DBValue>; +use crate::{TestCommit, TestDB}; + type MemoryDBProof = MemoryDB<::Hash, HashKey<::Hash>, DBValue>; test_layouts!(iterator_works, iterator_works_internal); -fn iterator_works_internal() { +fn iterator_works_internal>() { let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()), (hex!("0103000000000010000469").to_vec(), hex!("ffffffffff").to_vec()), ]; - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (x, y) in &pairs { - t.insert(x, y).unwrap(); - } + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (x, y) in &pairs { + t.insert(x, y).unwrap(); } + let commit = t.commit(); + let root = memdb.commit(commit); let trie = TrieDBBuilder::::new(&memdb, &root).build(); @@ -59,20 +60,18 @@ fn iterator_works_internal() { } test_layouts!(iterator_seek_works, iterator_seek_works_internal); -fn iterator_seek_works_internal() { +fn iterator_seek_works_internal>() { let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("fffffffffe").to_vec()), (hex!("0103000000000000000469").to_vec(), hex!("ffffffffff").to_vec()), ]; - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (x, y) in &pairs { - t.insert(x, y).unwrap(); - } + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (x, y) in &pairs { + t.insert(x, y).unwrap(); } + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); @@ -94,18 +93,50 @@ fn iterator_seek_works_internal() { ); } +test_layouts!(double_ended_iterator, double_ended_iterator_internal); +fn double_ended_iterator_internal>() { + let pairs = vec![ + (hex!("01").to_vec(), hex!("01").to_vec()), + (hex!("02").to_vec(), hex!("02").to_vec()), + (hex!("03").to_vec(), hex!("03").to_vec()), + (hex!("10").to_vec(), hex!("10").to_vec()), + (hex!("11").to_vec(), hex!("11").to_vec()), + ]; + + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (x, y) in &pairs { + t.insert(x, y).unwrap(); + } + let commit = t.commit(); + let root = memdb.commit(commit); + + let t = TrieDBBuilder::::new(&memdb, &root).build(); + assert_eq!(pairs, t.iter().unwrap().map(|x| x.unwrap()).collect::>()); + + let mut iter = t.into_double_ended_iter().unwrap(); + + for i in 0..pairs.len() { + assert_eq!(iter.next().unwrap().unwrap(), pairs[i].clone()); + } + assert!(iter.next().is_none()); + + for i in (0..pairs.len()).rev() { + assert_eq!(iter.next_back().unwrap().unwrap(), pairs[i].clone()); + } + assert!(iter.next_back().is_none()); +} + test_layouts!(iterator, iterator_internal); -fn iterator_internal() { +fn iterator_internal>() { let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"B".to_vec()]; - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for x in &d { - t.insert(x, x).unwrap(); - } + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for x in &d { + t.insert(x, x).unwrap(); } + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); assert_eq!( @@ -116,18 +147,16 @@ fn iterator_internal() { } test_layouts!(iterator_seek, iterator_seek_internal); -fn iterator_seek_internal() { +fn iterator_seek_internal>() { let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"AS".to_vec(), b"B".to_vec()]; let vals = vec![vec![0; 32], vec![1; 32], vec![2; 32], vec![4; 32], vec![3; 32]]; - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (k, val) in d.iter().zip(vals.iter()) { - t.insert(k, val.as_slice()).unwrap(); - } + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (k, val) in d.iter().zip(vals.iter()) { + t.insert(k, val.as_slice()).unwrap(); } + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = t.iter().unwrap(); @@ -171,24 +200,24 @@ fn iterator_seek_internal() { assert_eq!(&vals[5..], &iter.map(|x| x.unwrap().1).collect::>()[..]); } -fn trie_from_hex_keys(keys: &[&str], callback: impl FnOnce(&mut trie_db::TrieDB)) -where +fn trie_from_hex_keys>( + keys: &[&str], + callback: impl FnOnce(&mut trie_db::TrieDB), +) where T: TrieLayout, { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (index, key) in keys.iter().enumerate() { - t.insert(&array_bytes::hex2bytes(key).unwrap(), &[index as u8]).unwrap(); - } + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (index, key) in keys.iter().enumerate() { + t.insert(&array_bytes::hex2bytes(key).unwrap(), &[index as u8]).unwrap(); } + let root = t.commit().commit_to(&mut memdb); let mut t = TrieDBBuilder::::new(&memdb, &root).build(); callback(&mut t); } -fn test_prefixed_then_seek( +fn test_prefixed_then_seek>( keys: &[&str], prefix_key: &str, seek_key: &str, @@ -197,7 +226,7 @@ fn test_prefixed_then_seek( let prefix_key = array_bytes::hex2bytes(prefix_key).unwrap(); let seek_key = array_bytes::hex2bytes(seek_key).unwrap(); - trie_from_hex_keys::(keys, |trie| { + trie_from_hex_keys::(keys, |trie| { let iter = trie_db::TrieDBIterator::new_prefixed_then_seek(&trie, &prefix_key, &seek_key).unwrap(); let output: Vec<_> = iter.map(|x| array_bytes::bytes2hex("", x.unwrap().0)).collect(); @@ -248,36 +277,36 @@ fn iterator_prefixed_then_seek_real_world() { ]; let target_key = "7474449cca95dc5d0c00e71735a6d17d3cd15a3fd6e04e47bee3922dbfa92c8da7dad55cf08ffe8194efa962146801b0503092b1ed6a3fa6aee9107334aefd7965bbe568c3d24c6d"; - test_prefixed_then_seek::(keys, target_key, target_key, &[]); + test_prefixed_then_seek::>(keys, target_key, target_key, &[]); } // This is the real-word test, but simplified. test_layouts_substrate!(iterator_prefixed_then_seek_simple); fn iterator_prefixed_then_seek_simple() { - test_prefixed_then_seek::(&["0100"], "00", "00", &[]); + test_prefixed_then_seek::>(&["0100"], "00", "00", &[]); } // These are just tests that the fuzzer barfed out while working on the fix for the real-world // issue. test_layouts_substrate!(iterator_prefixed_then_seek_testcase_1); fn iterator_prefixed_then_seek_testcase_1() { - test_prefixed_then_seek::(&["00"], "00", "", &["00"]) + test_prefixed_then_seek::>(&["00"], "00", "", &["00"]) } test_layouts_substrate!(iterator_prefixed_then_seek_testcase_2); fn iterator_prefixed_then_seek_testcase_2() { - test_prefixed_then_seek::(&["00", "0003"], "00", "", &["00", "0003"]) + test_prefixed_then_seek::>(&["00", "0003"], "00", "", &["00", "0003"]) } test_layouts_substrate!(iterator_prefixed_then_seek_testcase_3); fn iterator_prefixed_then_seek_testcase_3() { - test_prefixed_then_seek::(&["20"], "20", "0700", &["20"]) + test_prefixed_then_seek::>(&["20"], "20", "0700", &["20"]) } test_layouts_substrate!(iterator_prefixed_then_seek_testcase_4); fn iterator_prefixed_then_seek_testcase_4() { let keys = &["1701", "ffffffffffffffffffffffdfffffffffffffffffffffffffffffffffffffffff"]; - test_prefixed_then_seek::( + test_prefixed_then_seek::>( keys, "1701", "ffff27272727274949494949ce494949494949494949491768687b737373732b", @@ -287,18 +316,16 @@ fn iterator_prefixed_then_seek_testcase_4() { test_layouts_substrate!(iterator_prefixed_then_seek_testcase_5); fn iterator_prefixed_then_seek_testcase_5() { - test_prefixed_then_seek::(&["20"], "20", "20", &["20"]) + test_prefixed_then_seek::>(&["20"], "20", "20", &["20"]) } test_layouts!(get_length_with_extension, get_length_with_extension_internal); -fn get_length_with_extension_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(b"A", b"ABC").unwrap(); - t.insert(b"B", b"ABCBAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(); - } +fn get_length_with_extension_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(b"A", b"ABC").unwrap(); + t.insert(b"B", b"ABCBAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(); + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()).unwrap(), Some(3)); @@ -307,18 +334,19 @@ fn get_length_with_extension_internal() { } test_layouts!(debug_output_supports_pretty_print, debug_output_supports_pretty_print_internal); -fn debug_output_supports_pretty_print_internal() { +fn debug_output_supports_pretty_print_internal>() +where + T::Location: std::fmt::Debug, +{ let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"B".to_vec()]; - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let root = { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for x in &d { - t.insert(x, x).unwrap(); - } - t.root().clone() - }; + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for x in &d { + t.insert(x, x).unwrap(); + } + let root = t.commit().commit_to(&mut memdb); + let t = TrieDBBuilder::::new(&memdb, &root).build(); if T::USE_EXTENSION { @@ -391,26 +419,31 @@ test_layouts!( test_lookup_with_corrupt_data_returns_decoder_error, test_lookup_with_corrupt_data_returns_decoder_error_internal ); -fn test_lookup_with_corrupt_data_returns_decoder_error_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(b"A", b"ABC").unwrap(); - t.insert(b"B", b"ABCBA").unwrap(); - } +fn test_lookup_with_corrupt_data_returns_decoder_error_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(b"A", b"ABC").unwrap(); + t.insert(b"B", b"ABCBA").unwrap(); + let root = t.commit().commit_to(&mut memdb); let t = TrieDBBuilder::::new(&memdb, &root).build(); // query for an invalid data type to trigger an error let q = |x: &[u8]| x.len() < 64; - let lookup = Lookup:: { db: t.db(), query: q, hash: root, cache: None, recorder: None }; + let lookup = Lookup:: { + db: t.db(), + query: q, + hash: root, + location: Default::default(), + cache: None, + recorder: None, + }; let query_result = lookup.look_up(&b"A"[..], NibbleSlice::new(b"A")); assert_eq!(query_result.unwrap().unwrap(), true); } test_layouts!(test_recorder, test_recorder_internal); -fn test_recorder_internal() { +fn test_recorder_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -418,14 +451,12 @@ fn test_recorder_internal() { (b"B".to_vec(), vec![4; 64]), ]; - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in &key_value { - t.insert(key, value).unwrap(); - } + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); } + let root = memdb.commit(t.commit()); let mut recorder = Recorder::::new(); { @@ -452,7 +483,7 @@ fn test_recorder_internal() { } test_layouts!(test_recorder_with_cache, test_recorder_with_cache_internal); -fn test_recorder_with_cache_internal() { +fn test_recorder_with_cache_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -460,15 +491,13 @@ fn test_recorder_with_cache_internal() { (b"B".to_vec(), vec![4; 64]), ]; - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); + let mut memdb = DB::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in &key_value { - t.insert(key, value).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); } + let root = t.commit().commit_to(&mut memdb); let mut cache = TestTrieCache::::default(); @@ -480,7 +509,7 @@ fn test_recorder_with_cache_internal() { } // Root should now be cached. - assert!(cache.get_node(&root).is_some()); + assert!(cache.get_node(&root, Default::default()).is_some()); // Also the data should be cached. let value = cache.lookup_value_for_key(&key_value[1].0).unwrap(); @@ -539,7 +568,7 @@ fn test_recorder_with_cache_internal() { } test_layouts!(test_recorder_with_cache_get_hash, test_recorder_with_cache_get_hash_internal); -fn test_recorder_with_cache_get_hash_internal() { +fn test_recorder_with_cache_get_hash_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -547,15 +576,13 @@ fn test_recorder_with_cache_get_hash_internal() { (b"B".to_vec(), vec![4; 64]), ]; - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); + let mut memdb = DB::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in &key_value { - t.insert(key, value).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); } + let root = t.commit().commit_to(&mut memdb); let mut cache = TestTrieCache::::default(); @@ -570,7 +597,7 @@ fn test_recorder_with_cache_get_hash_internal() { } // Root should now be cached. - assert!(cache.get_node(&root).is_some()); + assert!(cache.get_node(&root, Default::default()).is_some()); // Also the data should be cached. if T::MAX_INLINE_VALUE.map_or(true, |l| l as usize > key_value[1].1.len()) { @@ -581,7 +608,7 @@ fn test_recorder_with_cache_get_hash_internal() { } else { assert!(matches!( cache.lookup_value_for_key(&key_value[1].0).unwrap(), - CachedValue::ExistingHash(hash) if *hash == T::Hash::hash(&key_value[1].1) + CachedValue::ExistingHash(hash, _) if *hash == T::Hash::hash(&key_value[1].1) )); } @@ -644,19 +671,211 @@ fn test_recorder_with_cache_get_hash_internal() { } } +test_layouts!(test_merkle_value, test_merkle_value_internal); +fn test_merkle_value_internal>() { + let mut memdb = DB::default(); + + // Data set. + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + (b"AAAA".to_vec(), vec![3; 64]), + (b"AAB".to_vec(), vec![4; 64]), + (b"AABBBB".to_vec(), vec![4; 1]), + (b"AB".to_vec(), vec![5; 1]), + (b"B".to_vec(), vec![6; 1]), + ]; + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); + } + let root = memdb.commit(t.commit()); + + // Ensure we can fetch the merkle values for all present keys. + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + for (key, _) in &key_value { + trie.lookup_first_descendant(key).unwrap().unwrap(); + } + + // Key is not present and has no descedant, but shares a prefix. + let hash = trie.lookup_first_descendant(b"AAAAX").unwrap(); + assert!(hash.is_none()); + let hash = trie.lookup_first_descendant(b"AABX").unwrap(); + assert!(hash.is_none()); + let hash = trie.lookup_first_descendant(b"AABC").unwrap(); + assert!(hash.is_none()); + let hash = trie.lookup_first_descendant(b"ABX").unwrap(); + assert!(hash.is_none()); + let hash = trie.lookup_first_descendant(b"AABBBBX").unwrap(); + assert!(hash.is_none()); + let hash = trie.lookup_first_descendant(b"BX").unwrap(); + assert!(hash.is_none()); + let hash = trie.lookup_first_descendant(b"AC").unwrap(); + assert!(hash.is_none()); + let hash = trie.lookup_first_descendant(b"BC").unwrap(); + assert!(hash.is_none()); + let hash = trie.lookup_first_descendant(b"AAAAX").unwrap(); + assert!(hash.is_none()); + // Key shares the first nibble with b"A". + let hash = trie.lookup_first_descendant(b"C").unwrap(); + assert!(hash.is_none()); + + // Key not present, but has a descendent. + let hash = trie.lookup_first_descendant(b"AAA").unwrap().unwrap(); + let expected = trie.lookup_first_descendant(b"AAAA").unwrap().unwrap(); + assert_eq!(hash, expected); + let hash = trie.lookup_first_descendant(b"AABB").unwrap().unwrap(); + let expected = trie.lookup_first_descendant(b"AABBBB").unwrap().unwrap(); + assert_eq!(hash, expected); + let hash = trie.lookup_first_descendant(b"AABBB").unwrap().unwrap(); + let expected = trie.lookup_first_descendant(b"AABBBB").unwrap().unwrap(); + assert_eq!(hash, expected); + + // Prefix AABB in between AAB and AABBBB, but has different ending char. + let hash = trie.lookup_first_descendant(b"AABBX").unwrap(); + assert!(hash.is_none()); +} + +test_layouts!(test_merkle_value_single_key, test_merkle_value_single_key_internal); +fn test_merkle_value_single_key_internal>() { + let mut memdb = DB::default(); + + // Data set. + let key_value = vec![(b"AAA".to_vec(), vec![1; 64])]; + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); + } + let root = memdb.commit(t.commit()); + + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + + let hash = trie.lookup_first_descendant(b"AA").unwrap().unwrap(); + let expected = trie.lookup_first_descendant(b"AAA").unwrap().unwrap(); + assert_eq!(hash, expected); + + // Trie does not contain AAC or AAAA. + let hash = trie.lookup_first_descendant(b"AAC").unwrap(); + assert!(hash.is_none()); + let hash = trie.lookup_first_descendant(b"AAAA").unwrap(); + assert!(hash.is_none()); +} + +test_layouts!(test_merkle_value_branches, test_merkle_value_branches_internal); +fn test_merkle_value_branches_internal>() { + let mut memdb = DB::default(); + + // Data set. + let key_value = vec![(b"AAAA".to_vec(), vec![1; 64]), (b"AABA".to_vec(), vec![2; 64])]; + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); + } + let root = memdb.commit(t.commit()); + + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + + // The hash is returned from the branch node. + let hash = trie.lookup_first_descendant(b"A").unwrap().unwrap(); + let aaaa_hash = trie.lookup_first_descendant(b"AAAA").unwrap().unwrap(); + let aaba_hash = trie.lookup_first_descendant(b"AABA").unwrap().unwrap(); + // Ensure the hash is not from any leaf. + assert_ne!(hash, aaaa_hash); + assert_ne!(hash, aaba_hash); +} + +test_layouts!(test_merkle_value_empty_trie, test_merkle_value_empty_trie_internal); +fn test_merkle_value_empty_trie_internal>() { + let mut memdb = DB::default(); + + // Valid state root. + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + t.insert(&[], &[]).unwrap(); + let root = memdb.commit(t.commit()); + + // Data set is empty. + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + + let hash = trie.lookup_first_descendant(b"").unwrap(); + assert!(hash.is_none()); + + let hash = trie.lookup_first_descendant(b"A").unwrap(); + assert!(hash.is_none()); + + let hash = trie.lookup_first_descendant(b"AA").unwrap(); + assert!(hash.is_none()); + + let hash = trie.lookup_first_descendant(b"AAA").unwrap(); + assert!(hash.is_none()); + + let hash = trie.lookup_first_descendant(b"AAAA").unwrap(); + assert!(hash.is_none()); +} + +test_layouts!(test_merkle_value_modification, test_merkle_value_modification_internal); +fn test_merkle_value_modification_internal>() { + let mut memdb = DB::default(); + + let key_value = vec![(b"AAAA".to_vec(), vec![1; 64]), (b"AABA".to_vec(), vec![2; 64])]; + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); + } + let root = memdb.commit(t.commit()); + + let (a_hash_lhs, aaaa_hash_lhs, aaba_hash_lhs) = { + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + + // The hash is returned from the branch node. + let hash = trie.lookup_first_descendant(b"A").unwrap().unwrap(); + let aaaa_hash = trie.lookup_first_descendant(b"AAAA").unwrap().unwrap(); + let aaba_hash = trie.lookup_first_descendant(b"AABA").unwrap().unwrap(); + + // Ensure the hash is not from any leaf. + assert_ne!(hash, aaaa_hash); + assert_ne!(hash, aaba_hash); + + (hash, aaaa_hash, aaba_hash) + }; + + // Modify AABA and expect AAAA to return the same merkle value. + let mut t = TrieDBMutBuilder::::from_existing(&memdb, root).build(); + t.insert(b"AABA", &vec![3; 64]).unwrap(); + let root = memdb.commit(t.commit()); + + let (a_hash_rhs, aaaa_hash_rhs, aaba_hash_rhs) = { + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + + // The hash is returned from the branch node. + let hash = trie.lookup_first_descendant(b"A").unwrap().unwrap(); + let aaaa_hash = trie.lookup_first_descendant(b"AAAA").unwrap().unwrap(); + let aaba_hash = trie.lookup_first_descendant(b"AABA").unwrap().unwrap(); + + // Ensure the hash is not from any leaf. + assert_ne!(hash, aaaa_hash); + assert_ne!(hash, aaba_hash); + + (hash, aaaa_hash, aaba_hash) + }; + + // AAAA was not modified. + assert_eq!(aaaa_hash_lhs, aaaa_hash_rhs); + // Changes to AABA must propagate to the root. + assert_ne!(aaba_hash_lhs, aaba_hash_rhs); + assert_ne!(a_hash_lhs, a_hash_rhs); +} + test_layouts!(iterator_seek_with_recorder, iterator_seek_with_recorder_internal); -fn iterator_seek_with_recorder_internal() { +fn iterator_seek_with_recorder_internal>() { let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"B".to_vec()]; let vals = vec![vec![0; 64], vec![1; 64], vec![2; 64], vec![3; 64]]; - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (k, val) in d.iter().zip(vals.iter()) { - t.insert(k, val.as_slice()).unwrap(); - } + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (k, val) in d.iter().zip(vals.iter()) { + t.insert(k, val.as_slice()).unwrap(); } + let root = t.commit().commit_to(&mut memdb); let mut recorder = Recorder::::new(); { @@ -682,7 +901,7 @@ fn iterator_seek_with_recorder_internal() { } test_layouts!(test_cache, test_cache_internal); -fn test_cache_internal() { +fn test_cache_internal>() { let key_value = vec![ (b"A".to_vec(), vec![1; 64]), (b"AA".to_vec(), vec![2; 64]), @@ -691,16 +910,20 @@ fn test_cache_internal() { (b"BC".to_vec(), vec![4; 64]), ]; - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); + let mut memdb = DB::default(); let mut cache = TestTrieCache::::default(); - { - let mut t = - TrieDBMutBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); + let changeset = { + let mut t = TrieDBMutBuilder::::new(&memdb).with_cache(&mut cache).build(); for (key, value) in &key_value { t.insert(key, value).unwrap(); } + t.commit() + }; + let root = memdb.commit(changeset); + let t = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); + for (key, _) in &key_value { + t.get(key).unwrap(); } // Ensure that when we cache the same value multiple times under different keys, @@ -719,13 +942,11 @@ fn test_cache_internal() { let cached_value = cache.lookup_value_for_key(&b"AB"[..]).unwrap().clone(); assert_eq!(cached_value.data().flatten().unwrap(), vec![3u8; 4]); - { - let mut t = - TrieDBMutBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); - for (key, value) in &key_value { - t.insert(key, value).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&memdb).with_cache(&mut cache).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); } + let root = memdb.commit(t.commit()); assert_eq!( cache.lookup_value_for_key(&b"AB"[..]).unwrap().data().flatten().unwrap(), @@ -737,7 +958,7 @@ fn test_cache_internal() { cache.clear_node_cache(); { - let t = TrieDBBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); + let t = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); for (key, value) in &key_value { assert_eq!(*value, t.get(key).unwrap().unwrap()); } @@ -747,7 +968,7 @@ fn test_cache_internal() { cache.clear_node_cache(); { - let t = TrieDBBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); + let t = TrieDBBuilder::::new(&mut memdb, &root).with_cache(&mut cache).build(); for (key, value) in &key_value { assert_eq!(T::Hash::hash(value), t.get_hash(key).unwrap().unwrap()); } @@ -756,19 +977,17 @@ fn test_cache_internal() { #[test] fn test_record_value() { - type L = HashedValueNoExtThreshold<33>; + type L = HashedValueNoExtThreshold<33, ()>; // one root branch and two leaf, one with inline value, the other with node value. let key_value = vec![(b"A".to_vec(), vec![1; 32]), (b"B".to_vec(), vec![1; 33])]; // Add some initial data to the trie let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in key_value.iter() { - t.insert(key, value).unwrap(); - } + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (key, value) in key_value.iter() { + t.insert(key, value).unwrap(); } + let root = t.commit().apply_to(&mut memdb); // Value access would record a two nodes (branch and leaf with value 32 len inline). let mut recorder = Recorder::::new(); @@ -896,3 +1115,73 @@ fn test_record_value() { // leaf with value hash only. assert_eq!(compact_proof[1].len(), 33); } + +test_layouts!(test_trie_nodes_recorded, test_trie_nodes_recorded_internal); +fn test_trie_nodes_recorded_internal>() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + (b"AB".to_vec(), vec![3; 4]), + (b"B".to_vec(), vec![4; 64]), + (b"BC".to_vec(), vec![4; 64]), + ]; + const NON_EXISTENT_KEY: &[u8] = &*b"NOT"; + + let mut memdb = DB::default(); + + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); + } + let root = memdb.commit(t.commit()); + + for mut cache in [Some(TestTrieCache::::default()), None] { + for get_hash in [true, false] { + let mut recorder = Recorder::::default(); + { + let trie = TrieDBBuilder::::new(&memdb, &root) + .with_recorder(&mut recorder) + .with_optional_cache(cache.as_mut().map(|c| c as &mut _)) + .build(); + for (key, _) in &key_value { + if get_hash { + assert!(trie.get_hash(key).unwrap().is_some()); + } else { + assert!(trie.get(key).unwrap().is_some()); + } + } + + if get_hash { + assert!(trie.get_hash(&NON_EXISTENT_KEY).unwrap().is_none()); + } else { + assert!(trie.get(&NON_EXISTENT_KEY).unwrap().is_none()); + } + } + + for (key, value) in &key_value { + let recorded = recorder.trie_nodes_recorded_for_key(&key); + + let is_inline = T::MAX_INLINE_VALUE.map_or(true, |m| value.len() < m as usize); + + let expected = if get_hash && !is_inline { + RecordedForKey::Hash + } else { + RecordedForKey::Value + }; + + assert_eq!( + expected, + recorded, + "{:?} max_inline: {:?} get_hash: {get_hash}", + String::from_utf8(key.to_vec()), + T::MAX_INLINE_VALUE + ); + } + + assert_eq!( + RecordedForKey::Value, + recorder.trie_nodes_recorded_for_key(&NON_EXISTENT_KEY), + ); + } + } +} diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs new file mode 100644 index 00000000..951d1319 --- /dev/null +++ b/test/src/triedbmut.rs @@ -0,0 +1,1051 @@ +// Copyright 2017, 2020 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ops::Deref; + +use env_logger; +use log::debug; +use reference_trie::{ + reference_trie_root, test_layouts, ExtensionLayout, HashedValueNoExt, + HashedValueNoExtThreshold, NoExtensionLayout, PrefixedMemoryDB, RefHasher, ReferenceNodeCodec, + ReferenceNodeCodecNoExt, TestTrieCache, +}; +use trie_db::{ + memory_db::{HashKey, MemoryDB, PrefixedKey}, + node_db::{Hasher, NodeDB, Prefix, EMPTY_PREFIX}, + test_utils::*, + CachedValue, Changeset, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, + TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, TrieLayout, Value, +}; + +use crate::{TestCommit, TestDB}; + +type MemoryDBProof = + MemoryDB<::Hash, HashKey<::Hash>, DBValue>; + +fn populate_trie<'db, T: TrieLayout>( + db: &'db dyn NodeDB, + v: &[(Vec, Vec)], +) -> TrieDBMut<'db, T> { + let mut t = TrieDBMutBuilder::::new(db).build(); + for i in 0..v.len() { + let key: &[u8] = &v[i].0; + let val: &[u8] = &v[i].1; + t.insert(key, val).unwrap(); + } + t +} + +fn unpopulate_trie<'db, T: TrieLayout>( + t: &mut TrieDBMut<'db, T>, + v: &[(Vec, Vec)], +) -> bool { + for (_ix, i) in v.into_iter().enumerate() { + let key: &[u8] = &i.0; + if t.remove(key).is_err() { + return false + } + } + true +} + +fn reference_hashed_null_node() -> ::Out { + if T::USE_EXTENSION { + as NodeCodec>::hashed_null_node() + } else { + as NodeCodec>::hashed_null_node() + } +} + +#[test] +fn playpen() { + env_logger::init(); + playpen_internal::>(); + playpen_internal::(); + playpen_internal::(); + playpen_internal::(); +} +fn playpen_internal() { + let mut seed = [0u8; 32]; + for test_i in 0..10_000 { + if test_i % 50 == 0 { + debug!("{:?} of 10000 stress tests done", test_i); + } + let initial_seed = seed.clone(); + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 100, + } + .make_with(&mut seed); + + let real = reference_trie_root::(x.clone()); + let mut memdb = PrefixedMemoryDB::::default(); + let memtrie = populate_trie::(&memdb, &x); + // avoid duplicate + let value_set: std::collections::BTreeMap<&[u8], &[u8]> = + x.iter().map(|(k, v)| (k.as_slice(), v.as_slice())).collect(); + for (k, v) in value_set { + assert_eq!(memtrie.get(k).unwrap().unwrap(), v); + } + let commit = memtrie.commit(); + let root = commit.apply_to(&mut memdb); + + if root != real { + println!("TRIE MISMATCH"); + println!(); + println!("{:?} vs {:?}", root, real); + for i in &x { + println!("{:#x?} -> {:#x?}", i.0, i.1); + } + } + assert_eq!(root, real); + + let mut memtrie = TrieDBMutBuilder::::from_existing(&memdb, root).build(); + assert!(unpopulate_trie(&mut memtrie, &x), "{:?}", (test_i, initial_seed)); + let root = memtrie.commit().apply_to(&mut memdb); + let hashed_null_node = reference_hashed_null_node::(); + if root != hashed_null_node { + println!("- TRIE MISMATCH"); + println!(); + println!("{:#x?} vs {:#x?}", root, hashed_null_node); + for i in &x { + println!("{:#x?} -> {:#x?}", i.0, i.1); + } + } + assert_eq!(root, hashed_null_node); + } +} + +test_layouts!(init, init_internal); +fn init_internal>() { + let memdb = DB::default(); + let t = TrieDBMutBuilder::::new(&memdb).build(); + let hashed_null_node = reference_hashed_null_node::(); + assert_eq!(t.commit().root_hash(), hashed_null_node); +} + +test_layouts!(insert_on_empty, insert_on_empty_internal); +fn insert_on_empty_internal>() { + let memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + assert_eq!( + t.commit().root_hash(), + reference_trie_root::(vec![(vec![0x01u8, 0x23], vec![0x01u8, 0x23])]), + ); +} + +test_layouts!(remove_to_empty, remove_to_empty_internal); +fn remove_to_empty_internal>() { + let big_value = b"00000000000000000000000000000000"; + + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + + t.insert(&[0x01], big_value).unwrap(); + t.insert(&[0x01, 0x23], big_value).unwrap(); + t.insert(&[0x01, 0x34], big_value).unwrap(); + t.remove(&[0x01]).unwrap(); + t.remove(&[0x01, 0x23]).unwrap(); + t.remove(&[0x01, 0x34]).unwrap(); + t.commit().commit_to(&mut memdb); + assert!(memdb.is_empty()); +} + +test_layouts!(remove_to_empty_checked, remove_to_empty_checked_internal); +fn remove_to_empty_checked_internal>() { + let big_value = b"00000000000000000000000000000000"; + + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01], big_value).unwrap(); + t.insert(&[0x01, 0x23], big_value).unwrap(); + t.insert(&[0x01, 0x34], big_value).unwrap(); + let root = t.commit().commit_to(&mut memdb); + let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); + assert_eq!(t.get(&[0x01]).unwrap(), Some(big_value.to_vec()),); + assert_eq!(t.get(&[0x01, 0x34]).unwrap(), Some(big_value.to_vec()),); + t.remove(&[0x01]).unwrap(); + t.remove(&[0x01, 0x23]).unwrap(); + t.remove(&[0x01, 0x34]).unwrap(); + t.commit().commit_to(&mut memdb); + assert!(memdb.is_empty()); +} + +test_layouts!(remove_to_empty_no_extension, remove_to_empty_no_extension_internal); +fn remove_to_empty_no_extension_internal>() { + let big_value = b"00000000000000000000000000000000"; + let big_value2 = b"00000000000000000000000000000002"; + let big_value3 = b"00000000000000000000000000000004"; + + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + + t.insert(&[0x01, 0x23], big_value3).unwrap(); + t.insert(&[0x01], big_value2).unwrap(); + t.insert(&[0x01, 0x34], big_value).unwrap(); + t.remove(&[0x01]).unwrap(); + + let root = t.commit().commit_to(&mut memdb); + assert_eq!( + &root, + &reference_trie::calc_root::(vec![ + (vec![0x01u8, 0x23], big_value3.to_vec()), + (vec![0x01u8, 0x34], big_value.to_vec()), + ]) + ); +} + +test_layouts!(insert_replace_root, insert_replace_root_internal); +fn insert_replace_root_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); + let root = t.commit().commit_to(&mut memdb); + assert_eq!( + root, + reference_trie_root::(vec![(vec![0x01u8, 0x23], vec![0x23u8, 0x45])]), + ); +} + +test_layouts!(insert_make_branch_root, insert_make_branch_root_internal); +fn insert_make_branch_root_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); + let root = t.commit().commit_to(&mut memdb); + assert_eq!( + root, + reference_trie_root::(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) + ]) + ); +} + +test_layouts!(insert_into_branch_root, insert_into_branch_root_internal); +fn insert_into_branch_root_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); + let root = t.commit().commit_to(&mut memdb); + assert_eq!( + root, + reference_trie_root::(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ]) + ); +} + +test_layouts!(insert_value_into_branch_root, insert_value_into_branch_root_internal); +fn insert_value_into_branch_root_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[], &[0x0]).unwrap(); + let root = t.commit().commit_to(&mut memdb); + assert_eq!( + root, + reference_trie_root::(vec![ + (vec![], vec![0x0]), + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + ]) + ); +} + +test_layouts!(insert_split_leaf, insert_split_leaf_internal); +fn insert_split_leaf_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); + let root = t.commit().commit_to(&mut memdb); + assert_eq!( + root, + reference_trie_root::(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), + ]) + ); +} + +test_layouts!(insert_split_extenstion, insert_split_extenstion_internal); +fn insert_split_extenstion_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); + t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); + t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); + let root = t.commit().commit_to(&mut memdb); + assert_eq!( + root, + reference_trie_root::(vec![ + (vec![0x01, 0x23, 0x45], vec![0x01]), + (vec![0x01, 0xf3, 0x45], vec![0x02]), + (vec![0x01, 0xf3, 0xf5], vec![0x03]), + ]) + ); +} + +test_layouts!(insert_big_value, insert_big_value_internal); +fn insert_big_value_internal>() { + let big_value0 = b"00000000000000000000000000000000"; + let big_value1 = b"11111111111111111111111111111111"; + + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01u8, 0x23], big_value0).unwrap(); + t.insert(&[0x11u8, 0x23], big_value1).unwrap(); + let root = t.commit().commit_to(&mut memdb); + assert_eq!( + root, + reference_trie_root::(vec![ + (vec![0x01u8, 0x23], big_value0.to_vec()), + (vec![0x11u8, 0x23], big_value1.to_vec()) + ]) + ); +} + +test_layouts!(insert_duplicate_value, insert_duplicate_value_internal); +fn insert_duplicate_value_internal>() { + let big_value = b"00000000000000000000000000000000"; + + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01u8, 0x23], big_value).unwrap(); + t.insert(&[0x11u8, 0x23], big_value).unwrap(); + let root = t.commit().commit_to(&mut memdb); + assert_eq!( + root, + reference_trie_root::(vec![ + (vec![0x01u8, 0x23], big_value.to_vec()), + (vec![0x11u8, 0x23], big_value.to_vec()) + ]) + ); +} + +test_layouts!(test_at_empty, test_at_empty_internal); +fn test_at_empty_internal>() { + let mut memdb = DB::default(); + let t = TrieDBMutBuilder::::new(&mut memdb).build(); + assert_eq!(t.get(&[0x5]).unwrap(), None); +} + +test_layouts!(test_at_one_and_two, test_at_one_and_two_internal); +fn test_at_one_and_two_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); + let root = t.commit().commit_to(&mut memdb); + let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); + assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); + t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x24]).unwrap(); + let root = t.commit().commit_to(&mut memdb); + let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); + t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x25]).unwrap(); + // This test that middle node get resolved correctly (modified + // triedbmut node due to change of child node). + assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); +} + +test_layouts!(test_at_three, test_at_three_internal); +fn test_at_three_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); + assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); + assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), vec![0xf1u8, 0x23]); + assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), vec![0x81u8, 0x23]); + assert_eq!(t.get(&[0x82, 0x23]).unwrap(), None); + let root = memdb.commit(t.commit()); + let t = TrieDBMutBuilder::::from_existing(&memdb, root).build(); + assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); + assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), vec![0xf1u8, 0x23]); + assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), vec![0x81u8, 0x23]); + assert_eq!(t.get(&[0x82, 0x23]).unwrap(), None); +} + +#[test] +fn test_nibbled_branch_changed_value() { + let memdb = MemoryDB::, DBValue>::default(); + let mut t = reference_trie::RefTrieDBMutNoExtBuilder::new(&memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0x01u8, 0x23, 0x11], &[0xf1u8, 0x23]).unwrap(); + assert_eq!(t.get(&[0x01u8, 0x23]).unwrap(), Some(vec![0x01u8, 0x23])); +} + +test_layouts!(stress, stress_internal); +fn stress_internal>() { + let mut seed = Default::default(); + for _ in 0..1000 { + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 4, + } + .make_with(&mut seed); + + let real = reference_trie_root::(x.clone()); + let mut memdb = DB::default(); + let memtrie = populate_trie::(&mut memdb, &x); + let mut y = x.clone(); + y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); + let mut memdb2 = DB::default(); + let memtrie_sorted = populate_trie::(&mut memdb2, &y); + let root = memtrie.commit().commit_to(&mut memdb); + let root2 = memtrie_sorted.commit().commit_to(&mut memdb2); + if root != real || root2 != real { + println!("TRIE MISMATCH"); + println!(); + println!("ORIGINAL... {:#x?}", root); + for i in &x { + println!("{:#x?} -> {:#x?}", i.0, i.1); + } + println!("SORTED... {:#x?}", root2); + for i in &y { + println!("{:#x?} -> {:#x?}", i.0, i.1); + } + } + assert_eq!(root, real); + assert_eq!(root2, real); + } +} + +test_layouts!(test_trie_existing, test_trie_existing_internal); +fn test_trie_existing_internal>() { + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + let root = t.commit().commit_to(&mut memdb); + let _ = TrieDBMutBuilder::::from_existing(&memdb, root); +} + +test_layouts!(insert_empty, insert_empty_internal); +fn insert_empty_internal>() { + let mut seed = Default::default(); + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 4, + } + .make_with(&mut seed); + + let mut db = DB::default(); + let mut t = TrieDBMutBuilder::::new(&db).build(); + for &(ref key, ref value) in &x { + t.insert(key, value).unwrap(); + } + let root = db.commit(t.commit()); + + assert_eq!(root, reference_trie_root::(x.clone())); + + let mut t = TrieDBMutBuilder::::from_existing(&db, root).build(); + for &(ref key, _) in &x { + t.insert(key, &[]).unwrap(); + } + assert!(t.is_empty()); + let root = db.commit(t.commit()); + + let hashed_null_node = reference_hashed_null_node::(); + assert_eq!(root, hashed_null_node); +} + +test_layouts!(return_old_values, return_old_values_internal); +fn return_old_values_internal>() { + let threshold = T::MAX_INLINE_VALUE; + let mut seed = Default::default(); + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 2, + } + .make_with(&mut seed); + + let mut db = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut db).build(); + for &(ref key, ref value) in &x { + assert!(t.insert(key, value).unwrap() == None); + if threshold.map(|t| value.len() < t as usize).unwrap_or(true) { + assert_eq!(t.insert(key, value).unwrap(), Some(Value::Inline(value.clone().into()))); + } else { + assert!(matches!(t.insert(key, value).unwrap(), Some(Value::NewNode(..)))); + } + } + for (key, value) in x { + if threshold.map(|t| value.len() < t as usize).unwrap_or(true) { + assert_eq!(t.remove(&key).unwrap(), Some(Value::Inline(value.into()))); + } else { + assert!(matches!(t.remove(&key).unwrap(), Some(Value::NewNode(..)))); + } + assert_eq!(t.remove(&key).unwrap(), None); + } +} + +#[test] +fn insert_empty_allowed() { + let mut db = MemoryDB::, DBValue>::default(); + let mut t = reference_trie::RefTrieDBMutAllowEmptyBuilder::new(&db).build(); + t.insert(b"test", &[]).unwrap(); + let root = t.commit().apply_to(&mut db); + + assert_eq!( + root, + reference_trie_root::(vec![( + b"test".to_vec(), + Vec::new() + )],) + ); + let t = reference_trie::RefTrieDBMutAllowEmptyBuilder::from_existing(&db, root).build(); + assert_eq!(t.get(b"test").unwrap(), Some(Vec::new())); +} + +#[test] +fn register_proof_without_value() { + use reference_trie::HashedValueNoExtThreshold; + use std::{cell::RefCell, collections::HashMap}; + use Prefix; + + type Layout = HashedValueNoExtThreshold<1, ()>; + type MemoryDB = trie_db::memory_db::MemoryDB, DBValue>; + let x = [ + (b"test1".to_vec(), vec![1; 32]), // inline + (b"test1234".to_vec(), vec![2; 36]), + (b"te".to_vec(), vec![3; 32]), + ]; + + let mut memdb = MemoryDB::default(); + let t = populate_trie::(&mut memdb, &x); + let root = t.commit().apply_to(&mut memdb); + { + let trie = TrieDBBuilder::::new(&memdb, &root).build(); + println!("{:?}", trie); + } + + struct ProofRecorder { + db: MemoryDB, + record: RefCell, Vec>>, + } + // Only to test without threads. + unsafe impl Send for ProofRecorder {} + unsafe impl Sync for ProofRecorder {} + + impl NodeDB for ProofRecorder { + fn get( + &self, + key: &::Out, + prefix: Prefix, + _location: (), + ) -> Option<(DBValue, Vec<()>)> { + let v = NodeDB::get(&self.db, key, prefix, ()); + if let Some((v, _)) = v.as_ref() { + self.record.borrow_mut().entry(key[..].to_vec()).or_insert_with(|| v.clone()); + } + v + } + fn contains( + &self, + key: &::Out, + prefix: Prefix, + _locatoin: (), + ) -> bool { + self.get(key, prefix, ()).is_some() + } + } + + let mut memdb = ProofRecorder { db: memdb, record: Default::default() }; + + let root_proof = root.clone(); + let mut trie = TrieDBMutBuilder::::from_existing(&mut memdb, root).build(); + // touch te value (test1 remains untouch). + trie.get(b"te").unwrap(); + // cut test_1234 prefix + trie.insert(b"test12", &[2u8; 36][..]).unwrap(); + // remove 1234 + trie.remove(b"test1234").unwrap(); + + // proof should contain value for 'te' only. + type MemoryDBProof = trie_db::memory_db::MemoryDB, DBValue>; + let mut memdb_from_proof = MemoryDBProof::default(); + for (_key, value) in memdb.record.into_inner().into_iter() { + memdb_from_proof.insert(EMPTY_PREFIX, value.as_slice()); + } + + let db_unpacked = memdb_from_proof.clone(); + let root_unpacked = root_proof.clone(); + + let mut memdb_from_proof = db_unpacked.clone(); + let root_proof = root_unpacked.clone(); + { + let mut trie = + TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, root_proof).build(); + trie.get(b"te").unwrap(); + trie.insert(b"test12", &[2u8; 36][..]).unwrap(); + trie.remove(b"test1234").unwrap(); + } + + let mut memdb_from_proof = db_unpacked.clone(); + let root_proof = root_unpacked.clone(); + { + use trie_db::Trie; + let trie = TrieDBBuilder::::new(&memdb_from_proof, &root_proof).build(); + assert!(trie.get(b"te").unwrap().is_some()); + assert!(matches!( + trie.get(b"test1").map_err(|e| *e), + Err(TrieError::IncompleteDatabase(..)) + )); + } + + { + let trie = + TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, root_proof).build(); + assert!(trie.get(b"te").unwrap().is_some()); + assert!(matches!( + trie.get(b"test1").map_err(|e| *e), + Err(TrieError::IncompleteDatabase(..)) + )); + } +} + +test_layouts!(test_recorder, test_recorder_internal); +fn test_recorder_internal>() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + (b"AB".to_vec(), vec![3; 64]), + (b"B".to_vec(), vec![4; 64]), + ]; + + // Add some initial data to the trie + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (key, value) in key_value.iter().take(1) { + t.insert(key, value).unwrap(); + } + let root = t.commit().commit_to(&mut memdb); + + // Add more data, but this time only to the overlay. + // While doing that we record all trie accesses to replay this operation. + let mut recorder = Recorder::::new(); + let mut overlay = memdb.clone(); + let new_root = root; + { + let mut trie = TrieDBMutBuilder::::from_existing(&mut overlay, new_root) + .with_recorder(&mut recorder) + .build(); + + for (key, value) in key_value.iter().skip(1) { + trie.insert(key, value).unwrap(); + } + } + + let mut partial_db = MemoryDBProof::::default(); + for record in recorder.drain() { + partial_db.insert(EMPTY_PREFIX, &record.data); + } + + // Replay the it, but this time we use the proof. + let validated_root = root; + { + let mut trie = + TrieDBMutBuilder::::from_existing(&mut partial_db, validated_root).build(); + + for (key, value) in key_value.iter().skip(1) { + trie.insert(key, value).unwrap(); + } + } + + assert_eq!(new_root, validated_root); +} + +test_layouts!(test_recorder_witch_cache, test_recorder_with_cache_internal); +fn test_recorder_with_cache_internal>() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + (b"AB".to_vec(), vec![3; 64]), + (b"B".to_vec(), vec![4; 64]), + ]; + + // Add some initial data to the trie + let mut memdb = DB::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb).build(); + for (key, value) in key_value.iter().take(1) { + t.insert(key, value).unwrap(); + } + let root = t.commit().commit_to(&mut memdb); + let mut validated_root = root; + + let mut cache = TestTrieCache::::default(); + + { + let trie = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); + + // Only read one entry. + assert_eq!(key_value[0].1, trie.get(&key_value[0].0).unwrap().unwrap()); + } + + // Root should now be cached. + assert!(cache.get_node(&root, Default::default()).is_some()); + + // Add more data, but this time only to the overlay. + // While doing that we record all trie accesses to replay this operation. + let mut recorder = Recorder::::new(); + let mut overlay = memdb.clone(); + let mut trie = TrieDBMutBuilder::::from_existing(&mut overlay, root) + .with_recorder(&mut recorder) + .with_cache(&mut cache) + .build(); + + for (key, value) in key_value.iter().skip(1) { + trie.insert(key, value).unwrap(); + } + let new_root = trie.commit().commit_to(&mut overlay); + + let t = TrieDBBuilder::::new(&overlay, &new_root).with_cache(&mut cache).build(); + for (key, _) in key_value.iter().skip(1) { + t.get(key).unwrap(); + } + + for (key, value) in key_value.iter().skip(1) { + let cached_value = cache.lookup_value_for_key(key).unwrap(); + + assert_eq!(value, cached_value.data().flatten().unwrap().deref()); + assert_eq!(T::Hash::hash(&value), cached_value.hash().unwrap()); + } + + let mut partial_db = MemoryDBProof::::default(); + for record in recorder.drain() { + partial_db.insert(EMPTY_PREFIX, &record.data); + } + + // Replay the it, but this time we use the proof. + { + let mut trie = TrieDBMutBuilder::::from_existing(&partial_db, validated_root).build(); + + for (key, value) in key_value.iter().skip(1) { + trie.insert(key, value).unwrap(); + } + validated_root = trie.commit().apply_to(&mut partial_db); + } + + assert_eq!(new_root, validated_root); +} + +test_layouts!(test_insert_remove_data_with_cache, test_insert_remove_data_with_cache_internal); +fn test_insert_remove_data_with_cache_internal>() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + // Should be inlined + (b"AC".to_vec(), vec![7; 4]), + (b"AB".to_vec(), vec![3; 64]), + (b"B".to_vec(), vec![4; 64]), + ]; + + let mut cache = TestTrieCache::::default(); + let mut recorder = Recorder::::new(); + let mut memdb = DB::default(); + let mut trie = TrieDBMutBuilder::::new(&mut memdb) + .with_recorder(&mut recorder) + .with_cache(&mut cache) + .build(); + + // Add all values + for (key, value) in key_value.iter() { + trie.insert(key, value).unwrap(); + } + + // Remove only the last 2 elements + for (key, _) in key_value.iter().skip(3) { + let _ = trie.remove(key); + } + let mut memdb = MemoryDB::, DBValue>::default(); + let root = trie.commit().apply_to(&mut memdb); + let t = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); + for (key, _) in &key_value { + t.get(key).unwrap(); + } + + // Then only the first 3 elements should be in the cache and the last + // two ones should be added as non-existent. + for (key, value) in key_value.iter().take(3) { + let key_str = String::from_utf8_lossy(key); + + let cached_value = cache + .lookup_value_for_key(key) + .unwrap_or_else(|| panic!("Failed to lookup `{}`", key_str)); + + assert_eq!(value, cached_value.data().flatten().unwrap().deref(), "{:?}", key_str); + assert_eq!(T::Hash::hash(&value), cached_value.hash().unwrap()); + } + + for (key, _) in key_value.iter().skip(3) { + assert!(matches!(cache.lookup_value_for_key(key).unwrap(), CachedValue::NonExisting)); + } +} + +#[test] +fn test_two_assets_memory_db() { + test_two_assets_memory_db_inner_1::>(); + test_two_assets_memory_db_inner_2::>(); +} +fn test_two_assets_memory_db_inner_1() { + let memdb = PrefixedMemoryDB::::new(&[0u8]); + let mut state = TrieDBMutBuilder::::new(&memdb).build(); + + let key1 = [1u8; 3]; + let data1 = [1u8; 2]; + state.insert(key1.as_ref(), &data1).unwrap(); + assert_eq!(state.get(key1.as_ref()).unwrap().unwrap(), data1); //PASSING + let key2 = [2u8; 3]; + let data2 = [2u8; 2]; + state.insert(key2.as_ref(), &data2).unwrap(); + assert_eq!(state.get(key1.as_ref()).unwrap().unwrap(), data1); + + state.commit(); +} + +fn test_two_assets_memory_db_inner_2() { + let memdb = PrefixedMemoryDB::::new(&[0u8]); + let mut state = TrieDBMutBuilder::::new(&memdb).build(); + + let key1 = [1u8]; + let data1 = [1u8; 2]; + state.insert(key1.as_ref(), &data1).unwrap(); + assert_eq!(state.get(key1.as_ref()).unwrap().unwrap(), data1); + let key2 = [1u8, 2]; + let data2 = [2u8; 2]; + state.insert(key2.as_ref(), &data2).unwrap(); + assert_eq!(state.get(key1.as_ref()).unwrap().unwrap(), data1); + assert_eq!(state.get(key2.as_ref()).unwrap().unwrap(), data2); + + let key3 = [1u8, 3]; + let data3 = [3u8; 2]; + state.insert(key3.as_ref(), &data3).unwrap(); + assert_eq!(state.get(key1.as_ref()).unwrap().unwrap(), data1); + assert_eq!(state.get(key2.as_ref()).unwrap().unwrap(), data2); + assert_eq!(state.get(key3.as_ref()).unwrap().unwrap(), data3); +} + +test_layouts!(attached_trie, attached_trie_internal); +fn attached_trie_internal>() { + use std::collections::BTreeMap; + struct ATrie { + root: TrieHash, + data: BTreeMap, Vec>, + changeset: Option, T::Location>>>, + } + // Running a typical attached trie scenario (childtrie on substrate): + // different trie, attached trie root written all + // at once with treerefset before parent tree commit. + // Direct copy if using ref counting and location in db. + // Pruning. + let mut seed = Default::default(); + let nb_attached_trie = 10; + // let nb_attached_trie = 1; + let support_location = DB::support_location(); + let mut memdb = DB::default(); + let mut attached_tries: BTreeMap, ATrie> = Default::default(); + let mut keyspaced_memdb; + let mut main_trie: ATrie = + ATrie { root: Default::default(), data: Default::default(), changeset: None }; + for i in 0..nb_attached_trie + 1 { + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 3, + journal_key: 0, + value_mode: ValueMode::Index, + count: 20, + //count: 2, + } + .make_with(&mut seed); + + let mut memtrie = populate_trie::(&mut memdb, &x); + let data: BTreeMap, Vec> = x.iter().cloned().collect(); + if i == nb_attached_trie { + for (k, c) in attached_tries.iter_mut() { + let key: &[u8] = &k[..]; + let val: &[u8] = c.root.as_ref(); + let changeset = c.changeset.take().unwrap(); + memtrie.insert_with_tree_ref(key, val, Some(changeset)).unwrap(); + } + let changeset = memtrie.commit(); + let root = changeset.commit_to(&mut memdb); + main_trie.root = root; + main_trie.data = data; + } else { + let attached_trie_root_key = data.iter().next().unwrap().0; + let changeset = memtrie.commit_with_keyspace(attached_trie_root_key); + let root = changeset.root_hash(); + attached_tries.insert( + attached_trie_root_key.clone(), + ATrie { root, data, changeset: Some(changeset.into()) }, + ); + } + } + // check data + { + let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); + for (k, v) in main_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } + } + for (root_key, attached_trie) in &attached_tries { + let (attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); + + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + assert!(attached_trie_location.is_none()); + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + + let trie = TrieDBBuilder::::new_with_db_location( + child_memdb, + &attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + for (k, v) in attached_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } + } + // Modifying an existing child trie. + let (root_key, a_attached_trie) = attached_tries.iter().next().unwrap(); + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); + let (tree_ref_changeset, treeref_root_hash) = { + assert_eq!(a_attached_trie_root, a_attached_trie.root); + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( + child_memdb, + a_attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + attached_trie.remove(a_attached_trie.data.iter().next().unwrap().0).unwrap(); + attached_trie.insert(b"make_sure_it_changes", b"value").unwrap(); + let changeset = attached_trie.commit_with_keyspace(root_key); + let new_root = changeset.root_hash(); + assert!(new_root != a_attached_trie_root); + (changeset, new_root) + }; + let mut main_trie = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); + main_trie + .insert_with_tree_ref(root_key, treeref_root_hash.as_ref(), Some(tree_ref_changeset.into())) + .unwrap(); + let changeset = main_trie.commit(); + let main_root = changeset.root_hash(); + changeset.commit_to(&mut memdb); + // checking modification + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_root, root_key).unwrap(); + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let trie = TrieDBBuilder::::new_with_db_location( + child_memdb, + &a_attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + trie.get(b"make_sure_it_changes").unwrap().unwrap(); + let mut first = true; + for (k, v) in a_attached_trie.data.iter() { + if first { + assert!(&trie.get(k).unwrap().is_none()); + first = false; + } else { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } + } + trie.get(b"make_sure_it_changes").unwrap().unwrap(); +} + +#[cfg(test)] +fn attached_trie_root>( + memdb: &DB, + main_root: &TrieHash, + root_key: &[u8], +) -> Option<(TrieHash, Option)> { + let trie = TrieDBBuilder::::new(memdb, main_root).build(); + // Note could have a variant of get_with here that goes into + // encoded node hash and locations. + let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); + use trie_db::TrieIterator; + iter.seek(root_key).unwrap(); + let item = iter.next()?.unwrap(); + let node = &item.2; + let location = node.node_plan().additional_ref_location(node.locations()); + let root = iter.item_from_raw(&item)?.unwrap(); + if root.0.as_slice() != root_key { + return None; + } + let mut root_hash = TrieHash::::default(); + root_hash.as_mut().copy_from_slice(&root.1); + Some((root_hash, location)) +} + +pub struct KeySpacedDB<'a, H, T, DL>(&'a dyn NodeDB, &'a [u8]); + +impl<'a, H, T, DL> KeySpacedDB<'a, H, T, DL> { + #[inline] + pub fn new(db: &'a dyn NodeDB, ks: &'a [u8]) -> Self { + KeySpacedDB(db, ks) + } +} + +impl<'a, H, T, L> NodeDB for KeySpacedDB<'a, H, T, L> +where + H: Hasher, + T: From<&'static [u8]>, +{ + fn get(&self, key: &H::Out, prefix: Prefix, location: L) -> Option<(T, Vec)> { + let derived_prefix = trie_db::triedbmut::prefix_prefix(self.1, prefix); + self.0.get(key, (&derived_prefix.0, derived_prefix.1), location) + } + + fn contains(&self, key: &H::Out, prefix: Prefix, location: L) -> bool { + let derived_prefix = trie_db::triedbmut::prefix_prefix(self.1, prefix); + self.0.contains(key, (&derived_prefix.0, derived_prefix.1), location) + } +} diff --git a/trie-db/Cargo.toml b/trie-db/Cargo.toml deleted file mode 100644 index f8e6b00c..00000000 --- a/trie-db/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "trie-db" -version = "0.27.1" -authors = ["Parity Technologies "] -description = "Merkle-Patricia Trie generic over key hasher and node encoding" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -log = "0.4" -smallvec = { version = "1.0.0", features = ["union", "const_new"] } -hash-db = { path = "../hash-db", default-features = false, version = "0.16.0"} -hashbrown = { version = "0.13.2", default-features = false, features = ["ahash"] } -rustc-hex = { version = "2.1.0", default-features = false, optional = true } - -[features] -default = ["std"] -std = [ - "hash-db/std", - "rustc-hex", -] diff --git a/trie-db/src/fatdb.rs b/trie-db/src/fatdb.rs deleted file mode 100644 index 16883394..00000000 --- a/trie-db/src/fatdb.rs +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2017, 2021 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::{ - CError, DBValue, Query, Result, Trie, TrieDB, TrieDBIterator, TrieDBKeyIterator, TrieHash, - TrieItem, TrieIterator, TrieKeyItem, TrieLayout, -}; -use hash_db::{HashDBRef, Hasher}; - -use crate::{rstd::boxed::Box, TrieDBBuilder}; - -/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. -/// Additionaly it stores inserted hash-key mappings for later retrieval. -/// -/// Use it as a `Trie` or `TrieMut` trait object. -pub struct FatDB<'db, 'cache, L> -where - L: TrieLayout, -{ - raw: TrieDB<'db, 'cache, L>, -} - -impl<'db, 'cache, L> FatDB<'db, 'cache, L> -where - L: TrieLayout, -{ - /// Create a new trie with the backing database `db` and empty `root` - /// Initialise to the state entailed by the genesis block. - /// This guarantees the trie is built correctly. - pub fn new(db: &'db dyn HashDBRef, root: &'db TrieHash) -> Self { - FatDB { raw: TrieDBBuilder::new(db, root).build() } - } - - /// Get the backing database. - pub fn db(&self) -> &dyn HashDBRef { - self.raw.db() - } -} - -impl<'db, 'cache, L> Trie for FatDB<'db, 'cache, L> -where - L: TrieLayout, -{ - fn root(&self) -> &TrieHash { - self.raw.root() - } - - fn contains(&self, key: &[u8]) -> Result, CError> { - self.raw.contains(L::Hash::hash(key).as_ref()) - } - - fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { - self.raw.get_hash(key) - } - - fn get_with>( - &self, - key: &[u8], - query: Q, - ) -> Result, TrieHash, CError> { - self.raw.get_with(L::Hash::hash(key).as_ref(), query) - } - - fn iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - FatDBIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) - } - - fn key_iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - FatDBKeyIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) - } -} - -/// Iterator over inserted pairs of key values. -pub struct FatDBIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - trie_iterator: TrieDBIterator<'db, 'cache, L>, - trie: &'db TrieDB<'db, 'cache, L>, -} - -impl<'db, 'cache, L> FatDBIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - /// Creates new iterator. - pub fn new(trie: &'db TrieDB<'db, 'cache, L>) -> Result, CError> { - Ok(FatDBIterator { trie_iterator: TrieDBIterator::new(trie)?, trie }) - } -} - -impl<'db, 'cache, L> TrieIterator for FatDBIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { - let hashed_key = L::Hash::hash(key); - self.trie_iterator.seek(hashed_key.as_ref()) - } -} - -impl<'db, 'cache, L> Iterator for FatDBIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - type Item = TrieItem, CError>; - - fn next(&mut self) -> Option { - self.trie_iterator.next().map(|res| { - res.map(|(hash, value)| { - let aux_hash = L::Hash::hash(&hash); - ( - self.trie.db().get(&aux_hash, Default::default()).expect("Missing fatdb hash"), - value, - ) - }) - }) - } -} - -/// Iterator over inserted keys. -pub struct FatDBKeyIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - trie_iterator: TrieDBKeyIterator<'db, 'cache, L>, - trie: &'db TrieDB<'db, 'cache, L>, -} - -impl<'db, 'cache, L> FatDBKeyIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - /// Creates new iterator. - pub fn new(trie: &'db TrieDB<'db, 'cache, L>) -> Result, CError> { - Ok(FatDBKeyIterator { trie_iterator: TrieDBKeyIterator::new(trie)?, trie }) - } -} - -impl<'db, 'cache, L> TrieIterator for FatDBKeyIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { - let hashed_key = L::Hash::hash(key); - self.trie_iterator.seek(hashed_key.as_ref()) - } -} - -impl<'db, 'cache, L> Iterator for FatDBKeyIterator<'db, 'cache, L> -where - L: TrieLayout, -{ - type Item = TrieKeyItem, CError>; - - fn next(&mut self) -> Option { - self.trie_iterator.next().map(|res| { - res.map(|hash| { - let aux_hash = L::Hash::hash(&hash); - self.trie.db().get(&aux_hash, Default::default()).expect("Missing fatdb hash") - }) - }) - } -} diff --git a/trie-db/src/fatdbmut.rs b/trie-db/src/fatdbmut.rs deleted file mode 100644 index fa7a8f07..00000000 --- a/trie-db/src/fatdbmut.rs +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{ - triedbmut::{TrieDBMutBuilder, Value}, - CError, DBValue, Result, TrieDBMut, TrieHash, TrieLayout, TrieMut, -}; -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; - -/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. -/// Additionaly it stores inserted hash-key mappings for later retrieval. -/// -/// Use it as a `Trie` or `TrieMut` trait object. -pub struct FatDBMut<'db, L> -where - L: TrieLayout, -{ - raw: TrieDBMut<'db, L>, -} - -impl<'db, L> FatDBMut<'db, L> -where - L: TrieLayout, -{ - /// Create a new trie with the backing database `db` and empty `root` - /// Initialise to the state entailed by the genesis block. - /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut dyn HashDB, root: &'db mut TrieHash) -> Self { - FatDBMut { raw: TrieDBMutBuilder::new(db, root).build() } - } - - /// Create a new trie with the backing database `db` and `root`. - /// - /// Returns an error if root does not exist. - pub fn from_existing( - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, - ) -> Self { - FatDBMut { raw: TrieDBMutBuilder::from_existing(db, root).build() } - } - - /// Get the backing database. - pub fn db(&self) -> &dyn HashDB { - self.raw.db() - } - - /// Get the backing database. - pub fn db_mut(&mut self) -> &mut dyn HashDB { - self.raw.db_mut() - } -} - -impl<'db, L> TrieMut for FatDBMut<'db, L> -where - L: TrieLayout, -{ - fn root(&mut self) -> &TrieHash { - self.raw.root() - } - - fn is_empty(&self) -> bool { - self.raw.is_empty() - } - - fn contains(&self, key: &[u8]) -> Result, CError> { - self.raw.contains(L::Hash::hash(key).as_ref()) - } - - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, TrieHash, CError> - where - 'a: 'key, - { - self.raw.get(L::Hash::hash(key).as_ref()) - } - - fn insert( - &mut self, - key: &[u8], - value: &[u8], - ) -> Result>, TrieHash, CError> { - let hash = L::Hash::hash(key); - let out = self.raw.insert(hash.as_ref(), value)?; - let db = self.raw.db_mut(); - - // insert if it doesn't exist. - if out.is_none() { - let aux_hash = L::Hash::hash(hash.as_ref()); - db.emplace(aux_hash, EMPTY_PREFIX, key.to_vec()); - } - Ok(out) - } - - fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { - let hash = L::Hash::hash(key); - let out = self.raw.remove(hash.as_ref())?; - - // remove if it already exists. - if out.is_some() { - let aux_hash = L::Hash::hash(hash.as_ref()); - self.raw.db_mut().remove(&aux_hash, EMPTY_PREFIX); - } - - Ok(out) - } -} diff --git a/trie-db/src/sectriedb.rs b/trie-db/src/sectriedb.rs deleted file mode 100644 index ccfbd971..00000000 --- a/trie-db/src/sectriedb.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{ - rstd::boxed::Box, triedb::TrieDB, CError, DBValue, Query, Result, Trie, TrieDBBuilder, - TrieHash, TrieItem, TrieIterator, TrieKeyItem, TrieLayout, -}; -use hash_db::{HashDBRef, Hasher}; - -/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. -/// -/// Use it as a `Trie` trait object. You can use `raw()` to get the backing `TrieDB` object. -pub struct SecTrieDB<'db, 'cache, L> -where - L: TrieLayout, -{ - raw: TrieDB<'db, 'cache, L>, -} - -impl<'db, 'cache, L> SecTrieDB<'db, 'cache, L> -where - L: TrieLayout, -{ - /// Create a new trie with the backing database `db` and `root`. - /// - /// Initialise to the state entailed by the genesis block. - /// This guarantees the trie is built correctly. - pub fn new(db: &'db dyn HashDBRef, root: &'db TrieHash) -> Self { - SecTrieDB { raw: TrieDBBuilder::new(db, root).build() } - } - - /// Get a reference to the underlying raw `TrieDB` struct. - pub fn raw(&self) -> &TrieDB<'db, 'cache, L> { - &self.raw - } - - /// Get a mutable reference to the underlying raw `TrieDB` struct. - pub fn raw_mut(&mut self) -> &mut TrieDB<'db, 'cache, L> { - &mut self.raw - } -} - -impl<'db, 'cache, L> Trie for SecTrieDB<'db, 'cache, L> -where - L: TrieLayout, -{ - fn root(&self) -> &TrieHash { - self.raw.root() - } - - fn contains(&self, key: &[u8]) -> Result, CError> { - self.raw.contains(L::Hash::hash(key).as_ref()) - } - - fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { - self.raw.get_hash(key) - } - - fn get_with>( - &self, - key: &[u8], - query: Q, - ) -> Result, TrieHash, CError> { - self.raw.get_with(L::Hash::hash(key).as_ref(), query) - } - - fn iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - TrieDB::iter(&self.raw) - } - - fn key_iter<'a>( - &'a self, - ) -> Result< - Box, CError>> + 'a>, - TrieHash, - CError, - > { - TrieDB::key_iter(&self.raw) - } -} diff --git a/trie-db/src/sectriedbmut.rs b/trie-db/src/sectriedbmut.rs deleted file mode 100644 index b56dc55a..00000000 --- a/trie-db/src/sectriedbmut.rs +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{ - triedbmut::TrieDBMutBuilder, CError, DBValue, Result, TrieDBMut, TrieHash, TrieLayout, TrieMut, - Value, -}; -use hash_db::{HashDB, Hasher}; - -/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. -/// -/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` -/// object. -pub struct SecTrieDBMut<'db, L> -where - L: TrieLayout, -{ - raw: TrieDBMut<'db, L>, -} - -impl<'db, L> SecTrieDBMut<'db, L> -where - L: TrieLayout, -{ - /// Create a new trie with the backing database `db` and empty `root` - /// Initialize to the state entailed by the genesis block. - /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut dyn HashDB, root: &'db mut TrieHash) -> Self { - SecTrieDBMut { raw: TrieDBMutBuilder::new(db, root).build() } - } - - /// Create a new trie with the backing database `db` and `root`. - pub fn from_existing( - db: &'db mut dyn HashDB, - root: &'db mut TrieHash, - ) -> Self { - SecTrieDBMut { raw: TrieDBMutBuilder::from_existing(db, root).build() } - } - - /// Get the backing database. - pub fn db(&self) -> &dyn HashDB { - self.raw.db() - } - - /// Get the backing database. - pub fn db_mut(&mut self) -> &mut dyn HashDB { - self.raw.db_mut() - } -} - -impl<'db, L> TrieMut for SecTrieDBMut<'db, L> -where - L: TrieLayout, -{ - fn root(&mut self) -> &TrieHash { - self.raw.root() - } - - fn is_empty(&self) -> bool { - self.raw.is_empty() - } - - fn contains(&self, key: &[u8]) -> Result, CError> { - self.raw.contains(&L::Hash::hash(key).as_ref()) - } - - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, TrieHash, CError> - where - 'a: 'key, - { - self.raw.get(&L::Hash::hash(key).as_ref()) - } - - fn insert( - &mut self, - key: &[u8], - value: &[u8], - ) -> Result>, TrieHash, CError> { - self.raw.insert(&L::Hash::hash(key).as_ref(), value) - } - - fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { - self.raw.remove(&L::Hash::hash(key).as_ref()) - } -} diff --git a/trie-db/test/src/fatdb.rs b/trie-db/test/src/fatdb.rs deleted file mode 100644 index fb8e7350..00000000 --- a/trie-db/test/src/fatdb.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefFatDB, RefFatDBMut, RefHasher}; -use trie_db::{DBValue, Trie, TrieMut}; - -#[test] -fn fatdb_to_trie() { - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = RefFatDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - let t = RefFatDB::new(&memdb, &root); - assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); - assert_eq!( - t.iter().unwrap().map(Result::unwrap).collect::>(), - vec![(vec![0x01u8, 0x23], vec![0x01u8, 0x23])] - ); - assert_eq!( - t.key_iter().unwrap().map(Result::unwrap).collect::>(), - vec![vec![0x01u8, 0x23]] - ); -} diff --git a/trie-db/test/src/fatdbmut.rs b/trie-db/test/src/fatdbmut.rs deleted file mode 100644 index 519f5f0f..00000000 --- a/trie-db/test/src/fatdbmut.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use hash_db::{Hasher, EMPTY_PREFIX}; -use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefFatDBMut, RefHasher, RefTrieDBBuilder}; -use trie_db::{Trie, TrieMut}; - -#[test] -fn fatdbmut_to_trie() { - let mut memdb = MemoryDB::, _>::default(); - let mut root = Default::default(); - { - let mut t = RefFatDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - - let t = RefTrieDBBuilder::new(&memdb, &root).build(); - assert_eq!(t.get(&RefHasher::hash(&[0x01u8, 0x23])), Ok(Some(vec![0x01u8, 0x23])),); -} - -#[test] -fn fatdbmut_insert_remove_key_mapping() { - let mut memdb = MemoryDB::, _>::default(); - let mut root = Default::default(); - let key = [0x01u8, 0x23]; - let val = [0x01u8, 0x24]; - let key_hash = RefHasher::hash(&key); - let aux_hash = RefHasher::hash(&key_hash); - let mut t = RefFatDBMut::new(&mut memdb, &mut root); - t.insert(&key, &val).unwrap(); - assert_eq!(t.get(&key), Ok(Some(val.to_vec()))); - assert_eq!(t.db().get(&aux_hash, EMPTY_PREFIX), Some(key.to_vec())); - t.remove(&key).unwrap(); - assert_eq!(t.db().get(&aux_hash, EMPTY_PREFIX), None); -} diff --git a/trie-db/test/src/lib.rs b/trie-db/test/src/lib.rs deleted file mode 100644 index 7802247e..00000000 --- a/trie-db/test/src/lib.rs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Tests for trie-db crate. - -#[cfg(test)] -mod fatdb; -#[cfg(test)] -mod fatdbmut; -#[cfg(test)] -mod iter_build; -#[cfg(test)] -mod iterator; -#[cfg(test)] -mod proof; -#[cfg(test)] -mod recorder; -#[cfg(test)] -mod sectriedb; -#[cfg(test)] -mod sectriedbmut; -#[cfg(test)] -mod trie_codec; -#[cfg(test)] -mod triedb; -#[cfg(test)] -mod triedbmut; diff --git a/trie-db/test/src/sectriedb.rs b/trie-db/test/src/sectriedb.rs deleted file mode 100644 index bc04a5c3..00000000 --- a/trie-db/test/src/sectriedb.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use hash_db::Hasher; -use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefHasher, RefSecTrieDB, RefTrieDBMutBuilder}; -use trie_db::{DBValue, Trie, TrieMut}; - -#[test] -fn trie_to_sectrie() { - let mut db = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = RefTrieDBMutBuilder::new(&mut db, &mut root).build(); - t.insert(&RefHasher::hash(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); - } - let t = RefSecTrieDB::new(&db, &root); - assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); -} diff --git a/trie-db/test/src/sectriedbmut.rs b/trie-db/test/src/sectriedbmut.rs deleted file mode 100644 index e984d2e2..00000000 --- a/trie-db/test/src/sectriedbmut.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use hash_db::Hasher; -use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefHasher, RefSecTrieDBMut, RefTrieDBBuilder}; -use trie_db::{DBValue, Trie, TrieMut}; - -#[test] -fn sectrie_to_trie() { - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - { - let mut t = RefSecTrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - let t = RefTrieDBBuilder::new(&memdb, &root).build(); - assert_eq!(t.get(&RefHasher::hash(&[0x01u8, 0x23])).unwrap().unwrap(), vec![0x01u8, 0x23],); -} diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs deleted file mode 100644 index 02316892..00000000 --- a/trie-db/test/src/triedbmut.rs +++ /dev/null @@ -1,838 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::ops::Deref; - -use env_logger; -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; -use log::debug; -use memory_db::{HashKey, MemoryDB, PrefixedKey}; -use reference_trie::{ - reference_trie_root, test_layouts, ExtensionLayout, HashedValueNoExt, - HashedValueNoExtThreshold, NoExtensionLayout, RefHasher, ReferenceNodeCodec, - ReferenceNodeCodecNoExt, TestTrieCache, -}; -use trie_db::{ - DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, - TrieError, TrieLayout, TrieMut, Value, -}; -use trie_standardmap::*; - -type PrefixedMemoryDB = - MemoryDB<::Hash, PrefixedKey<::Hash>, DBValue>; -type MemoryDBProof = - MemoryDB<::Hash, HashKey<::Hash>, DBValue>; - -fn populate_trie<'db, T: TrieLayout>( - db: &'db mut dyn HashDB, - root: &'db mut ::Out, - v: &[(Vec, Vec)], -) -> TrieDBMut<'db, T> { - let mut t = TrieDBMutBuilder::::new(db, root).build(); - - for i in 0..v.len() { - let key: &[u8] = &v[i].0; - let val: &[u8] = &v[i].1; - t.insert(key, val).unwrap(); - } - t -} - -fn unpopulate_trie<'db, T: TrieLayout>( - t: &mut TrieDBMut<'db, T>, - v: &[(Vec, Vec)], -) -> bool { - for (_ix, i) in v.into_iter().enumerate() { - let key: &[u8] = &i.0; - if t.remove(key).is_err() { - return false - } - } - true -} - -fn reference_hashed_null_node() -> ::Out { - if T::USE_EXTENSION { - as NodeCodec>::hashed_null_node() - } else { - as NodeCodec>::hashed_null_node() - } -} - -#[test] -fn playpen() { - env_logger::init(); - playpen_internal::>(); - playpen_internal::(); - playpen_internal::(); - playpen_internal::(); -} -fn playpen_internal() { - let mut seed = [0u8; 32]; - for test_i in 0..10_000 { - if test_i % 50 == 0 { - debug!("{:?} of 10000 stress tests done", test_i); - } - let initial_seed = seed.clone(); - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, - journal_key: 0, - value_mode: ValueMode::Index, - count: 100, - } - .make_with(&mut seed); - - let real = reference_trie_root::(x.clone()); - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); - - memtrie.commit(); - if *memtrie.root() != real { - println!("TRIE MISMATCH"); - println!(); - println!("{:?} vs {:?}", memtrie.root(), real); - for i in &x { - println!("{:#x?} -> {:#x?}", i.0, i.1); - } - } - assert_eq!(*memtrie.root(), real); - assert!(unpopulate_trie(&mut memtrie, &x), "{:?}", (test_i, initial_seed)); - memtrie.commit(); - let hashed_null_node = reference_hashed_null_node::(); - if *memtrie.root() != hashed_null_node { - println!("- TRIE MISMATCH"); - println!(); - println!("{:#x?} vs {:#x?}", memtrie.root(), hashed_null_node); - for i in &x { - println!("{:#x?} -> {:#x?}", i.0, i.1); - } - } - assert_eq!(*memtrie.root(), hashed_null_node); - } -} - -test_layouts!(init, init_internal); -fn init_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - let hashed_null_node = reference_hashed_null_node::(); - assert_eq!(*t.root(), hashed_null_node); -} - -test_layouts!(insert_on_empty, insert_on_empty_internal); -fn insert_on_empty_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - assert_eq!( - *t.root(), - reference_trie_root::(vec![(vec![0x01u8, 0x23], vec![0x01u8, 0x23])]), - ); -} - -test_layouts!(remove_to_empty, remove_to_empty_internal); -fn remove_to_empty_internal() { - let big_value = b"00000000000000000000000000000000"; - - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - - t.insert(&[0x01], big_value).unwrap(); - t.insert(&[0x01, 0x23], big_value).unwrap(); - t.insert(&[0x01, 0x34], big_value).unwrap(); - t.remove(&[0x01]).unwrap(); - t.remove(&[0x01, 0x23]).unwrap(); - t.remove(&[0x01, 0x34]).unwrap(); - } - assert_eq!(memdb.keys().len(), 0); -} - -test_layouts!(remove_to_empty_checked, remove_to_empty_checked_internal); -fn remove_to_empty_checked_internal() { - let big_value = b"00000000000000000000000000000000"; - - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - - t.insert(&[0x01], big_value).unwrap(); - t.insert(&[0x01, 0x23], big_value).unwrap(); - t.insert(&[0x01, 0x34], big_value).unwrap(); - t.commit(); - assert_eq!(t.get(&[0x01]).unwrap(), Some(big_value.to_vec()),); - assert_eq!(t.get(&[0x01, 0x34]).unwrap(), Some(big_value.to_vec()),); - t.commit(); - t.remove(&[0x01]).unwrap(); - t.remove(&[0x01, 0x23]).unwrap(); - t.remove(&[0x01, 0x34]).unwrap(); - } - assert_eq!(memdb.keys().len(), 0); -} - -test_layouts!(remove_to_empty_no_extension, remove_to_empty_no_extension_internal); -fn remove_to_empty_no_extension_internal() { - let big_value = b"00000000000000000000000000000000"; - let big_value2 = b"00000000000000000000000000000002"; - let big_value3 = b"00000000000000000000000000000004"; - - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - - t.insert(&[0x01, 0x23], big_value3).unwrap(); - t.insert(&[0x01], big_value2).unwrap(); - t.insert(&[0x01, 0x34], big_value).unwrap(); - t.remove(&[0x01]).unwrap(); - // commit on drop - } - assert_eq!( - &root, - &reference_trie::calc_root::(vec![ - (vec![0x01u8, 0x23], big_value3.to_vec()), - (vec![0x01u8, 0x34], big_value.to_vec()), - ]) - ); -} - -test_layouts!(insert_replace_root, insert_replace_root_internal); -fn insert_replace_root_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); - assert_eq!( - *t.root(), - reference_trie_root::(vec![(vec![0x01u8, 0x23], vec![0x23u8, 0x45])]), - ); -} - -test_layouts!(insert_make_branch_root, insert_make_branch_root_internal); -fn insert_make_branch_root_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); - assert_eq!( - *t.root(), - reference_trie_root::(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) - ]) - ); -} - -test_layouts!(insert_into_branch_root, insert_into_branch_root_internal); -fn insert_into_branch_root_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); - t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); - assert_eq!( - *t.root(), - reference_trie_root::(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), - (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), - ]) - ); -} - -test_layouts!(insert_value_into_branch_root, insert_value_into_branch_root_internal); -fn insert_value_into_branch_root_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[], &[0x0]).unwrap(); - assert_eq!( - *t.root(), - reference_trie_root::(vec![ - (vec![], vec![0x0]), - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - ]) - ); -} - -test_layouts!(insert_split_leaf, insert_split_leaf_internal); -fn insert_split_leaf_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); - assert_eq!( - *t.root(), - reference_trie_root::(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), - ]) - ); -} - -test_layouts!(insert_split_extenstion, insert_split_extenstion_internal); -fn insert_split_extenstion_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); - t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); - t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); - assert_eq!( - *t.root(), - reference_trie_root::(vec![ - (vec![0x01, 0x23, 0x45], vec![0x01]), - (vec![0x01, 0xf3, 0x45], vec![0x02]), - (vec![0x01, 0xf3, 0xf5], vec![0x03]), - ]) - ); -} - -test_layouts!(insert_big_value, insert_big_value_internal); -fn insert_big_value_internal() { - let big_value0 = b"00000000000000000000000000000000"; - let big_value1 = b"11111111111111111111111111111111"; - - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], big_value0).unwrap(); - t.insert(&[0x11u8, 0x23], big_value1).unwrap(); - assert_eq!( - *t.root(), - reference_trie_root::(vec![ - (vec![0x01u8, 0x23], big_value0.to_vec()), - (vec![0x11u8, 0x23], big_value1.to_vec()) - ]) - ); -} - -test_layouts!(insert_duplicate_value, insert_duplicate_value_internal); -fn insert_duplicate_value_internal() { - let big_value = b"00000000000000000000000000000000"; - - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], big_value).unwrap(); - t.insert(&[0x11u8, 0x23], big_value).unwrap(); - assert_eq!( - *t.root(), - reference_trie_root::(vec![ - (vec![0x01u8, 0x23], big_value.to_vec()), - (vec![0x11u8, 0x23], big_value.to_vec()) - ]) - ); -} - -test_layouts!(test_at_empty, test_at_empty_internal); -fn test_at_empty_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - assert_eq!(t.get(&[0x5]).unwrap(), None); -} - -test_layouts!(test_at_one_and_two, test_at_one_and_two_internal); -fn test_at_one_and_two_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); - t.commit(); - assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); - t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x24]).unwrap(); - } - let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x25]).unwrap(); - // This test that middle node get resolved correctly (modified - // triedbmut node due to change of child node). - assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); -} - -test_layouts!(test_at_three, test_at_three_internal); -fn test_at_three_internal() { - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); - t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); - assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); - assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), vec![0xf1u8, 0x23]); - assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), vec![0x81u8, 0x23]); - assert_eq!(t.get(&[0x82, 0x23]).unwrap(), None); - t.commit(); - assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), vec![0x01u8, 0x23]); - assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), vec![0xf1u8, 0x23]); - assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), vec![0x81u8, 0x23]); - assert_eq!(t.get(&[0x82, 0x23]).unwrap(), None); -} - -#[test] -fn test_nibbled_branch_changed_value() { - let mut memdb = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - let mut t = reference_trie::RefTrieDBMutNoExtBuilder::new(&mut memdb, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0x01u8, 0x23, 0x11], &[0xf1u8, 0x23]).unwrap(); - assert_eq!(t.get(&[0x01u8, 0x23]).unwrap(), Some(vec![0x01u8, 0x23])); -} - -test_layouts!(stress, stress_internal); -fn stress_internal() { - let mut seed = Default::default(); - for _ in 0..1000 { - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, - journal_key: 0, - value_mode: ValueMode::Index, - count: 4, - } - .make_with(&mut seed); - - let real = reference_trie_root::(x.clone()); - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); - let mut y = x.clone(); - y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); - let mut memdb2 = PrefixedMemoryDB::::default(); - let mut root2 = Default::default(); - let mut memtrie_sorted = populate_trie::(&mut memdb2, &mut root2, &y); - if *memtrie.root() != real || *memtrie_sorted.root() != real { - println!("TRIE MISMATCH"); - println!(); - println!("ORIGINAL... {:#x?}", memtrie.root()); - for i in &x { - println!("{:#x?} -> {:#x?}", i.0, i.1); - } - println!("SORTED... {:#x?}", memtrie_sorted.root()); - for i in &y { - println!("{:#x?} -> {:#x?}", i.0, i.1); - } - } - assert_eq!(*memtrie.root(), real); - assert_eq!(*memtrie_sorted.root(), real); - } -} - -test_layouts!(test_trie_existing, test_trie_existing_internal); -fn test_trie_existing_internal() { - let mut db = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - - { - let _ = TrieDBMutBuilder::::from_existing(&mut db, &mut root); - } -} - -test_layouts!(insert_empty, insert_empty_internal); -fn insert_empty_internal() { - let mut seed = Default::default(); - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, - journal_key: 0, - value_mode: ValueMode::Index, - count: 4, - } - .make_with(&mut seed); - - let mut db = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); - for &(ref key, ref value) in &x { - t.insert(key, value).unwrap(); - } - - assert_eq!(*t.root(), reference_trie_root::(x.clone())); - - for &(ref key, _) in &x { - t.insert(key, &[]).unwrap(); - } - - assert!(t.is_empty()); - let hashed_null_node = reference_hashed_null_node::(); - assert_eq!(*t.root(), hashed_null_node); -} - -test_layouts!(return_old_values, return_old_values_internal); -fn return_old_values_internal() { - let threshold = T::MAX_INLINE_VALUE; - let mut seed = Default::default(); - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, - journal_key: 0, - value_mode: ValueMode::Index, - count: 2, - } - .make_with(&mut seed); - - let mut db = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); - for &(ref key, ref value) in &x { - assert!(t.insert(key, value).unwrap() == None); - if threshold.map(|t| value.len() < t as usize).unwrap_or(true) { - assert_eq!(t.insert(key, value).unwrap(), Some(Value::Inline(value.clone().into()))); - } else { - assert!(matches!(t.insert(key, value).unwrap(), Some(Value::NewNode(..)))); - } - } - for (key, value) in x { - if threshold.map(|t| value.len() < t as usize).unwrap_or(true) { - assert_eq!(t.remove(&key).unwrap(), Some(Value::Inline(value.into()))); - } else { - assert!(matches!(t.remove(&key).unwrap(), Some(Value::NewNode(..)))); - } - assert_eq!(t.remove(&key).unwrap(), None); - } -} - -#[test] -fn insert_empty_allowed() { - let mut db = MemoryDB::, DBValue>::default(); - let mut root = Default::default(); - let mut t = reference_trie::RefTrieDBMutAllowEmptyBuilder::new(&mut db, &mut root).build(); - t.insert(b"test", &[]).unwrap(); - - assert_eq!( - *t.root(), - reference_trie_root::(vec![( - b"test".to_vec(), - Vec::new() - )],) - ); - assert_eq!(t.get(b"test").unwrap(), Some(Vec::new())); -} - -#[test] -fn register_proof_without_value() { - use hash_db::{AsHashDB, Prefix}; - use reference_trie::HashedValueNoExtThreshold; - use std::{cell::RefCell, collections::HashMap}; - - type Layout = HashedValueNoExtThreshold<1>; - type MemoryDB = memory_db::MemoryDB, DBValue>; - let x = [ - (b"test1".to_vec(), vec![1; 32]), // inline - (b"test1234".to_vec(), vec![2; 36]), - (b"te".to_vec(), vec![3; 32]), - ]; - - let mut memdb = MemoryDB::default(); - let mut root = Default::default(); - let _ = populate_trie::(&mut memdb, &mut root, &x); - { - let trie = TrieDBBuilder::::new(&memdb, &root).build(); - println!("{:?}", trie); - } - - struct ProofRecorder { - db: MemoryDB, - record: RefCell, Vec>>, - } - // Only to test without threads. - unsafe impl Send for ProofRecorder {} - unsafe impl Sync for ProofRecorder {} - - impl HashDB for ProofRecorder { - fn get(&self, key: &::Out, prefix: Prefix) -> Option { - let v = self.db.get(key, prefix); - if let Some(v) = v.as_ref() { - self.record.borrow_mut().entry(key[..].to_vec()).or_insert_with(|| v.clone()); - } - v - } - - fn contains(&self, key: &::Out, prefix: Prefix) -> bool { - self.get(key, prefix).is_some() - } - - fn emplace(&mut self, key: ::Out, prefix: Prefix, value: DBValue) { - self.db.emplace(key, prefix, value) - } - - fn insert(&mut self, prefix: Prefix, value: &[u8]) -> ::Out { - self.db.insert(prefix, value) - } - - fn remove(&mut self, key: &::Out, prefix: Prefix) { - self.db.remove(key, prefix) - } - } - - impl AsHashDB for ProofRecorder { - fn as_hash_db(&self) -> &dyn HashDB { - self - } - fn as_hash_db_mut<'a>(&'a mut self) -> &'a mut (dyn HashDB + 'a) { - self - } - } - - let mut memdb = ProofRecorder { db: memdb, record: Default::default() }; - - let root_proof = root.clone(); - { - let mut trie = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); - // touch te value (test1 remains untouch). - trie.get(b"te").unwrap(); - // cut test_1234 prefix - trie.insert(b"test12", &[2u8; 36][..]).unwrap(); - // remove 1234 - trie.remove(b"test1234").unwrap(); - - // proof should contain value for 'te' only. - } - - type MemoryDBProof = memory_db::MemoryDB, DBValue>; - let mut memdb_from_proof = MemoryDBProof::default(); - for (_key, value) in memdb.record.into_inner().into_iter() { - memdb_from_proof.insert(hash_db::EMPTY_PREFIX, value.as_slice()); - } - - let db_unpacked = memdb_from_proof.clone(); - let root_unpacked = root_proof.clone(); - - let mut memdb_from_proof = db_unpacked.clone(); - let mut root_proof = root_unpacked.clone(); - { - let mut trie = - TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, &mut root_proof) - .build(); - trie.get(b"te").unwrap(); - trie.insert(b"test12", &[2u8; 36][..]).unwrap(); - trie.remove(b"test1234").unwrap(); - } - - let mut memdb_from_proof = db_unpacked.clone(); - let mut root_proof = root_unpacked.clone(); - { - use trie_db::Trie; - let trie = TrieDBBuilder::::new(&memdb_from_proof, &root_proof).build(); - assert!(trie.get(b"te").unwrap().is_some()); - assert!(matches!( - trie.get(b"test1").map_err(|e| *e), - Err(TrieError::IncompleteDatabase(..)) - )); - } - - { - let trie = - TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, &mut root_proof) - .build(); - assert!(trie.get(b"te").unwrap().is_some()); - assert!(matches!( - trie.get(b"test1").map_err(|e| *e), - Err(TrieError::IncompleteDatabase(..)) - )); - } -} - -test_layouts!(test_recorder, test_recorder_internal); -fn test_recorder_internal() { - let key_value = vec![ - (b"A".to_vec(), vec![1; 64]), - (b"AA".to_vec(), vec![2; 64]), - (b"AB".to_vec(), vec![3; 64]), - (b"B".to_vec(), vec![4; 64]), - ]; - - // Add some initial data to the trie - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in key_value.iter().take(1) { - t.insert(key, value).unwrap(); - } - } - - // Add more data, but this time only to the overlay. - // While doing that we record all trie accesses to replay this operation. - let mut recorder = Recorder::::new(); - let mut overlay = memdb.clone(); - let mut new_root = root; - { - let mut trie = TrieDBMutBuilder::::from_existing(&mut overlay, &mut new_root) - .with_recorder(&mut recorder) - .build(); - - for (key, value) in key_value.iter().skip(1) { - trie.insert(key, value).unwrap(); - } - } - - let mut partial_db = MemoryDBProof::::default(); - for record in recorder.drain() { - partial_db.insert(EMPTY_PREFIX, &record.data); - } - - // Replay the it, but this time we use the proof. - let mut validated_root = root; - { - let mut trie = - TrieDBMutBuilder::::from_existing(&mut partial_db, &mut validated_root).build(); - - for (key, value) in key_value.iter().skip(1) { - trie.insert(key, value).unwrap(); - } - } - - assert_eq!(new_root, validated_root); -} - -test_layouts!(test_recorder_witch_cache, test_recorder_with_cache_internal); -fn test_recorder_with_cache_internal() { - let key_value = vec![ - (b"A".to_vec(), vec![1; 64]), - (b"AA".to_vec(), vec![2; 64]), - (b"AB".to_vec(), vec![3; 64]), - (b"B".to_vec(), vec![4; 64]), - ]; - - // Add some initial data to the trie - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); - for (key, value) in key_value.iter().take(1) { - t.insert(key, value).unwrap(); - } - } - - let mut cache = TestTrieCache::::default(); - - { - let trie = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); - - // Only read one entry. - assert_eq!(key_value[0].1, trie.get(&key_value[0].0).unwrap().unwrap()); - } - - // Root should now be cached. - assert!(cache.get_node(&root).is_some()); - - // Add more data, but this time only to the overlay. - // While doing that we record all trie accesses to replay this operation. - let mut recorder = Recorder::::new(); - let mut overlay = memdb.clone(); - let mut new_root = root; - { - let mut trie = TrieDBMutBuilder::::from_existing(&mut overlay, &mut new_root) - .with_recorder(&mut recorder) - .with_cache(&mut cache) - .build(); - - for (key, value) in key_value.iter().skip(1) { - trie.insert(key, value).unwrap(); - } - } - - for (key, value) in key_value.iter().skip(1) { - let cached_value = cache.lookup_value_for_key(key).unwrap(); - - assert_eq!(value, cached_value.data().flatten().unwrap().deref()); - assert_eq!(T::Hash::hash(&value), cached_value.hash().unwrap()); - } - - let mut partial_db = MemoryDBProof::::default(); - for record in recorder.drain() { - partial_db.insert(EMPTY_PREFIX, &record.data); - } - - // Replay the it, but this time we use the proof. - let mut validated_root = root; - { - let mut trie = - TrieDBMutBuilder::::from_existing(&mut partial_db, &mut validated_root).build(); - - for (key, value) in key_value.iter().skip(1) { - trie.insert(key, value).unwrap(); - } - } - - assert_eq!(new_root, validated_root); -} - -test_layouts!(test_insert_remove_data_with_cache, test_insert_remove_data_with_cache_internal); -fn test_insert_remove_data_with_cache_internal() { - let key_value = vec![ - (b"A".to_vec(), vec![1; 64]), - (b"AA".to_vec(), vec![2; 64]), - // Should be inlined - (b"AC".to_vec(), vec![7; 4]), - (b"AB".to_vec(), vec![3; 64]), - (b"B".to_vec(), vec![4; 64]), - ]; - - let mut cache = TestTrieCache::::default(); - let mut recorder = Recorder::::new(); - let mut memdb = PrefixedMemoryDB::::default(); - let mut root = Default::default(); - { - let mut trie = TrieDBMutBuilder::::new(&mut memdb, &mut root) - .with_recorder(&mut recorder) - .with_cache(&mut cache) - .build(); - - // Add all values - for (key, value) in key_value.iter() { - trie.insert(key, value).unwrap(); - } - - // Remove only the last 2 elements - for (key, _) in key_value.iter().skip(3) { - let _ = trie.remove(key); - } - } - - // Then only the first 3 elements should be in the cache and the last - // two ones should not be there. - for (key, value) in key_value.iter().take(3) { - let key_str = String::from_utf8_lossy(key); - - let cached_value = cache - .lookup_value_for_key(key) - .unwrap_or_else(|| panic!("Failed to lookup `{}`", key_str)); - - assert_eq!(value, cached_value.data().flatten().unwrap().deref(), "{:?}", key_str); - assert_eq!(T::Hash::hash(&value), cached_value.hash().unwrap()); - } - - for (key, _) in key_value.iter().skip(3) { - assert!(cache.lookup_value_for_key(key).is_none()); - } -} diff --git a/trie-eip1186/CHANGELOG.md b/trie-eip1186/CHANGELOG.md deleted file mode 100644 index 38b4e85f..00000000 --- a/trie-eip1186/CHANGELOG.md +++ /dev/null @@ -1,8 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [Unreleased] -Support eip 1186 trie proofs. [#146](https://github.com/paritytech/trie/pull/146) diff --git a/trie-eip1186/Cargo.toml b/trie-eip1186/Cargo.toml deleted file mode 100644 index 5254d261..00000000 --- a/trie-eip1186/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "trie-eip1186" -version = "0.5.0" -authors = ["Parity Technologies "] -description = "EIP-1186 compliant proof generation and verification" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -trie-db = { path = "../trie-db", default-features = false, version = "0.27.0"} -hash-db = { path = "../hash-db", default-features = false, version = "0.16.0"} - -[features] -default = ["std"] -std = [ - "trie-db/std", - "hash-db/std", -] diff --git a/trie-eip1186/src/eip1186.rs b/trie-eip1186/src/eip1186.rs deleted file mode 100644 index 37ad106d..00000000 --- a/trie-eip1186/src/eip1186.rs +++ /dev/null @@ -1,311 +0,0 @@ -use crate::rstd::{result::Result, vec::Vec}; -use hash_db::{HashDBRef, Hasher}; -use trie_db::{ - node::{decode_hash, Node, NodeHandle, Value}, - recorder::Recorder, - CError, DBValue, NibbleSlice, NodeCodec, Result as TrieResult, Trie, TrieDBBuilder, TrieHash, - TrieLayout, -}; - -/// Generate an eip-1186 compatible proof for key-value pairs in a trie given a key. -pub fn generate_proof( - db: &dyn HashDBRef, - root: &TrieHash, - key: &[u8], -) -> TrieResult<(Vec>, Option>), TrieHash, CError> -where - L: TrieLayout, -{ - let mut recorder = Recorder::::new(); - - let item = { - let trie = TrieDBBuilder::::new(db, root).with_recorder(&mut recorder).build(); - trie.get(key)? - }; - - let proof: Vec> = recorder.drain().into_iter().map(|r| r.data).collect(); - Ok((proof, item)) -} - -/// Errors that may occur during proof verification. Most of the errors types simply indicate that -/// the proof is invalid with respect to the statement being verified, and the exact error type can -/// be used for debugging. -#[derive(PartialEq, Eq)] -#[cfg_attr(feature = "std", derive(Debug))] -pub enum VerifyError<'a, HO, CE> { - /// The proof does not contain any value for the given key - /// the error carries the nibbles left after traversing the trie - NonExistingValue(NibbleSlice<'a>), - /// The proof contains a value for the given key - /// while we were expecting to find a non-existence proof - ExistingValue(Vec), - /// The proof indicates that the trie contains a different value. - /// the error carries the value contained in the trie - ValueMismatch(Vec), - /// The proof is missing trie nodes required to verify. - IncompleteProof, - /// The node hash computed from the proof is not matching. - HashMismatch(HO), - /// One of the proof nodes could not be decoded. - DecodeError(CE), - /// Error in converting a plain hash into a HO - HashDecodeError(&'a [u8]), -} - -#[cfg(feature = "std")] -impl<'a, HO: std::fmt::Debug, CE: std::error::Error> std::fmt::Display for VerifyError<'a, HO, CE> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - match self { - VerifyError::NonExistingValue(key) => { - write!(f, "Key does not exist in trie: reaming key={:?}", key) - }, - VerifyError::ExistingValue(value) => { - write!(f, "trie contains a value for given key value={:?}", value) - }, - VerifyError::ValueMismatch(key) => { - write!(f, "Expected value was not found in the trie: key={:?}", key) - }, - VerifyError::IncompleteProof => write!(f, "Proof is incomplete -- expected more nodes"), - VerifyError::HashMismatch(hash) => write!(f, "hash mismatch found: hash={:?}", hash), - VerifyError::DecodeError(err) => write!(f, "Unable to decode proof node: {}", err), - VerifyError::HashDecodeError(plain_hash) => { - write!(f, "Unable to decode hash value plain_hash: {:?}", plain_hash) - }, - } - } -} - -#[cfg(feature = "std")] -impl<'a, HO: std::fmt::Debug, CE: std::error::Error + 'static> std::error::Error - for VerifyError<'a, HO, CE> -{ - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - VerifyError::DecodeError(err) => Some(err), - _ => None, - } - } -} - -/// Verify a compact proof for key-value pairs in a trie given a root hash. -pub fn verify_proof<'a, L>( - root: &::Out, - proof: &'a [Vec], - raw_key: &'a [u8], - expected_value: Option<&[u8]>, -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if proof.is_empty() { - return Err(VerifyError::IncompleteProof) - } - let key = NibbleSlice::new(raw_key); - process_node::(Some(root), &proof[0], key, expected_value, &proof[1..]) -} - -fn process_node<'a, L>( - expected_node_hash: Option<&::Out>, - encoded_node: &'a [u8], - key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if let Some(value) = expected_value { - if encoded_node == value { - return Ok(()) - } - } - if let Some(expected) = expected_node_hash { - let calculated_node_hash = ::hash(encoded_node); - if calculated_node_hash != *expected { - return Err(VerifyError::HashMismatch(calculated_node_hash)) - } - } - let node = ::decode(encoded_node).map_err(VerifyError::DecodeError)?; - match node { - Node::Empty => process_empty::(key, expected_value, proof), - Node::Leaf(nib, data) => process_leaf::(nib, data, key, expected_value, proof), - Node::Extension(nib, handle) => - process_extension::(&nib, handle, key, expected_value, proof), - Node::Branch(children, maybe_data) => - process_branch::(children, maybe_data, key, expected_value, proof), - Node::NibbledBranch(nib, children, maybe_data) => - process_nibbledbranch::(nib, children, maybe_data, key, expected_value, proof), - } -} - -fn process_empty<'a, L>( - key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - _: &[Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if expected_value.is_none() { - Ok(()) - } else { - Err(VerifyError::NonExistingValue(key)) - } -} - -fn process_leaf<'a, L>( - nib: NibbleSlice, - data: Value<'a>, - key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if key != nib && expected_value.is_none() { - return Ok(()) - } else if key != nib { - return Err(VerifyError::NonExistingValue(key)) - } - match_value::(Some(data), key, expected_value, proof) -} -fn process_extension<'a, L>( - nib: &NibbleSlice, - handle: NodeHandle<'a>, - mut key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if !key.starts_with(nib) && expected_value.is_none() { - return Ok(()) - } else if !key.starts_with(nib) { - return Err(VerifyError::NonExistingValue(key)) - } - key.advance(nib.len()); - - match handle { - NodeHandle::Inline(encoded_node) => - process_node::(None, encoded_node, key, expected_value, proof), - NodeHandle::Hash(plain_hash) => { - let new_root = decode_hash::(plain_hash) - .ok_or(VerifyError::HashDecodeError(plain_hash))?; - process_node::(Some(&new_root), &proof[0], key, expected_value, &proof[1..]) - }, - } -} - -fn process_nibbledbranch<'a, L>( - nib: NibbleSlice, - children: [Option>; 16], - maybe_data: Option>, - mut key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if !key.starts_with(&nib) && expected_value.is_none() { - return Ok(()) - } else if !key.starts_with(&nib) && expected_value.is_some() { - return Err(VerifyError::NonExistingValue(key)) - } - key.advance(nib.len()); - - if key.is_empty() { - match_value::(maybe_data, key, expected_value, proof) - } else { - match_children::(children, key, expected_value, proof) - } -} - -fn process_branch<'a, L>( - children: [Option>; 16], - maybe_data: Option>, - key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - if key.is_empty() { - match_value::(maybe_data, key, expected_value, proof) - } else { - match_children::(children, key, expected_value, proof) - } -} -fn match_children<'a, L>( - children: [Option>; 16], - mut key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - match children.get(key.at(0) as usize) { - Some(Some(NodeHandle::Hash(hash))) => - if proof.is_empty() { - Err(VerifyError::IncompleteProof) - } else { - key.advance(1); - let new_root = - decode_hash::(hash).ok_or(VerifyError::HashDecodeError(hash))?; - process_node::(Some(&new_root), &proof[0], key, expected_value, &proof[1..]) - }, - Some(Some(NodeHandle::Inline(encoded_node))) => { - key.advance(1); - process_node::(None, encoded_node, key, expected_value, proof) - }, - Some(None) => - if expected_value.is_none() { - Ok(()) - } else { - Err(VerifyError::NonExistingValue(key)) - }, - None => panic!("key index is out of range in children array"), - } -} - -fn match_value<'a, L>( - maybe_data: Option>, - key: NibbleSlice<'a>, - expected_value: Option<&[u8]>, - proof: &'a [Vec], -) -> Result<(), VerifyError<'a, TrieHash, CError>> -where - L: TrieLayout, -{ - match (maybe_data, proof.first(), expected_value) { - (None, _, None) => Ok(()), - (None, _, Some(_)) => Err(VerifyError::NonExistingValue(key)), - (Some(Value::Inline(inline_data)), _, Some(value)) => - if inline_data == value { - Ok(()) - } else { - Err(VerifyError::ValueMismatch(inline_data.to_vec())) - }, - (Some(Value::Inline(inline_data)), _, None) => - Err(VerifyError::ExistingValue(inline_data.to_vec())), - (Some(Value::Node(plain_hash)), Some(next_proof_item), Some(value)) => { - let value_hash = L::Hash::hash(value); - let node_hash = decode_hash::(plain_hash) - .ok_or(VerifyError::HashDecodeError(plain_hash))?; - if node_hash != value_hash { - Err(VerifyError::HashMismatch(node_hash)) - } else if next_proof_item != value { - Err(VerifyError::ValueMismatch(next_proof_item.to_vec())) - } else { - Ok(()) - } - }, - (Some(Value::Node(_)), None, _) => Err(VerifyError::IncompleteProof), - (Some(Value::Node(_)), Some(proof_item), None) => - Err(VerifyError::ExistingValue(proof_item.to_vec())), - } -} diff --git a/trie-eip1186/src/lib.rs b/trie-eip1186/src/lib.rs deleted file mode 100644 index 1483b857..00000000 --- a/trie-eip1186/src/lib.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2021, 2021 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#![cfg_attr(not(feature = "std"), no_std)] - -#[cfg(feature = "std")] -mod rstd { - pub use std::{result, vec}; -} - -#[cfg(not(feature = "std"))] -mod rstd { - pub use alloc::vec; - pub use core::result; - pub trait Error {} - impl Error for T {} -} - -mod eip1186; -pub use eip1186::{generate_proof, verify_proof, VerifyError}; diff --git a/trie-eip1186/test/Cargo.toml b/trie-eip1186/test/Cargo.toml deleted file mode 100644 index 4ee7c0fc..00000000 --- a/trie-eip1186/test/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "trie-eip1186-test" -version = "0.5.0" -authors = ["Parity Technologies "] -description = "Tests for trie-eip1186 crate" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -edition = "2018" - -[dependencies] -trie-eip1186 = { path = "..", version = "0.5.0"} -trie-db = { path = "../../trie-db", version = "0.27.0"} -hash-db = { path = "../../hash-db", version = "0.16.0"} -reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } -memory-db = { path = "../../memory-db", version = "0.32.0" } diff --git a/trie-eip1186/test/src/eip1186.rs b/trie-eip1186/test/src/eip1186.rs deleted file mode 100644 index 05d89edc..00000000 --- a/trie-eip1186/test/src/eip1186.rs +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2019, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use hash_db::Hasher; -use reference_trie::test_layouts; -use trie_db::{DBValue, TrieDBMutBuilder, TrieLayout, TrieMut}; -use trie_eip1186::{generate_proof, verify_proof, VerifyError}; - -type MemoryDB = memory_db::MemoryDB< - ::Hash, - memory_db::HashKey<::Hash>, - DBValue, ->; - -fn test_entries() -> Vec<(&'static [u8], &'static [u8])> { - vec![ - // "alfa" is at a hash-referenced leaf node. - (b"alfa", &[0; 32]), - // "bravo" is at an inline leaf node. - (b"bravo", b"bravo"), - // "do" is at a hash-referenced branch node. - (b"do", b"verb"), - // "dog" is at a hash-referenced branch node. - (b"dog", b"puppy"), - // "doge" is at a hash-referenced leaf node. - (b"doge", &[0; 32]), - // extension node "o" (plus nibble) to next branch. - (b"horse", b"stallion"), - (b"house", b"building"), - ] -} - -fn test_generate_proof( - entries: Vec<(&'static [u8], &'static [u8])>, - key: &[u8], -) -> (::Out, Vec>, Option>) { - // Populate DB with full trie from entries. - let (db, root) = { - let mut db = >::default(); - let mut root = Default::default(); - { - let mut trie = >::new(&mut db, &mut root).build(); - for (key, value) in entries.iter() { - trie.insert(key, value).unwrap(); - } - } - (db, root) - }; - // Generate proof for the given keys.. - let proof = generate_proof::(&db, &root, key).unwrap(); - (root, proof.0, proof.1) -} - -test_layouts!(trie_proof_works2, trie_proof_works_internal2); -fn trie_proof_works_internal2() { - let (root, proof, item) = test_generate_proof::( - vec![ - // "do" is at a hash-referenced branch node. - (b"do", b"verb"), - // "dog" is at a hash-referenced branch node. - (b"dog", b"puppy"), - ], - b"do", - ); - assert_eq!(Some(b"verb".as_ref()), item.as_deref(), "verb is the item"); - assert!(verify_proof::(&root, &proof, b"do", Some(b"verb")).is_ok(), "verifying do"); - - let (root, proof, item) = test_generate_proof::( - vec![ - // "do" is at a hash-referenced branch node. - (b"do", b"verb"), - // "dog" is at a hash-referenced branch node. - (b"dog", b"puppy"), - ], - b"dog", - ); - assert_eq!(Some(b"puppy".as_ref()), item.as_deref(), "puppy is the item"); - assert!(verify_proof::(&root, &proof, b"dog", Some(b"puppy")).is_ok(), "verifying dog"); -} - -test_layouts!(trie_proof_works, trie_proof_works_internal); -fn trie_proof_works_internal() { - let (root, proof, item) = test_generate_proof::(test_entries(), b"do"); - assert_eq!(Some(b"verb".as_ref()), item.as_deref(), "verb is the item"); - assert!(verify_proof::(&root, &proof, b"do", Some(b"verb")).is_ok(), "verifying do"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"dog"); - assert_eq!(Some(b"puppy".as_ref()), item.as_deref(), "puppy is the item"); - assert!(verify_proof::(&root, &proof, b"dog", Some(b"puppy")).is_ok(), "verifying dog"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"doge"); - assert_eq!(Some([0; 32].as_ref()), item.as_deref(), "[0;32] is the item"); - assert!(verify_proof::(&root, &proof, b"doge", Some(&[0; 32])).is_ok(), "verifying doge"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"bravo"); - assert_eq!(Some(b"bravo".as_ref()), item.as_deref(), "bravo is the item"); - assert!(verify_proof::(&root, &proof, b"bravo", Some(b"bravo")).is_ok(), "verifying bravo"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"alfabet"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"alfabet", None).is_ok(), "verifying alfabet"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"d"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"d", None).is_ok(), "verifying d"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"do\x10"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"do\x10", None).is_ok(), "verifying do\x10"); - - let (root, proof, item) = test_generate_proof::(test_entries(), b"halp"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"halp", None).is_ok(), "verifying halp"); -} - -test_layouts!(trie_proof_works_for_empty_trie, trie_proof_works_for_empty_trie_internal); -fn trie_proof_works_for_empty_trie_internal() { - let (root, proof, item) = test_generate_proof::(vec![], b"alpha"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"alpha", None).is_ok(), "verifying alpha"); - let (root, proof, item) = test_generate_proof::(vec![], b"bravo"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"bravo", None).is_ok(), "verifying bravo"); - let (root, proof, item) = test_generate_proof::(vec![], b"\x42\x42"); - assert!(item.is_none(), "item not found"); - assert!(verify_proof::(&root, &proof, b"\x42\x42", None).is_ok(), "verifying \x42\x42"); -} - -test_layouts!( - test_verify_value_mismatch_some_to_none, - test_verify_value_mismatch_some_to_none_internal -); -fn test_verify_value_mismatch_some_to_none_internal() { - let (root, proof, _) = test_generate_proof::(test_entries(), b"horse"); - let res = verify_proof::(&root, &proof, b"horse", Some(b"stallion")); - assert!(res.is_ok(), "verifying horse"); - - let res = verify_proof::(&root, &proof, b"halp", Some(b"plz")); - assert!(res.is_err(), "verifying halp"); - assert!(matches!(res.err().unwrap(), VerifyError::NonExistingValue(_))); - - let res = verify_proof::(&root, &proof, b"horse", Some(b"rocinante")); - assert!(res.is_err(), "verifying horse"); - //checking for two variants as it depends on the TrieLayout which one occurs - let is_ok = match res { - Err(VerifyError::HashMismatch(_)) | Err(VerifyError::ValueMismatch(_)) => true, - _ => false, - }; - assert!(is_ok); -} - -test_layouts!(test_verify_incomplete_proof, test_verify_incomplete_proof_internal); -fn test_verify_incomplete_proof_internal() { - let (root, mut proof, item) = test_generate_proof::(test_entries(), b"alfa"); - - proof.pop(); - let res = verify_proof::(&root, &proof, b"alfa", item.as_deref()); - assert!(matches!(res, Err(VerifyError::IncompleteProof))); -} - -test_layouts!(test_verify_decode_error, test_verify_decode_error_internal); -fn test_verify_decode_error_internal() { - let (_, mut proof, item) = test_generate_proof::(test_entries(), b"bravo"); - - let fake_node = b"this is not a trie node"; - proof.insert(0, fake_node.to_vec()); - let fake_root = T::Hash::hash(fake_node); - let res = verify_proof::(&fake_root, &proof, b"bravo", item.as_deref()); - assert!(matches!(res, Err(VerifyError::DecodeError(_)))); -} diff --git a/trie-eip1186/test/src/lib.rs b/trie-eip1186/test/src/lib.rs deleted file mode 100644 index 55910011..00000000 --- a/trie-eip1186/test/src/lib.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2021 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Tests for trie-eip1186 crate. - -#[cfg(test)] -mod eip1186; diff --git a/trie-root/CHANGELOG.md b/trie-root/CHANGELOG.md deleted file mode 100644 index a35b5b7b..00000000 --- a/trie-root/CHANGELOG.md +++ /dev/null @@ -1,17 +0,0 @@ -# Changelog - -The format is based on [Keep a Changelog]. - -[Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ - -## [Unreleased] - - -## [0.18.0] - 2023-03-14 -- Update dependencies. [#188](https://github.com/paritytech/trie/pull/188) and [#187](https://github.com/paritytech/trie/pull/187) - -## [0.17.0] - 2021-10-19 -- Support for value nodes. [#142](https://github.com/paritytech/trie/pull/142) - -## [0.16.0] - 2020-02-07 -- Update reference-trie to v0.20.0 [#78](https://github.com/paritytech/trie/pull/78) diff --git a/trie-root/Cargo.toml b/trie-root/Cargo.toml deleted file mode 100644 index 7a39bddd..00000000 --- a/trie-root/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "trie-root" -version = "0.18.0" -authors = ["Parity Technologies "] -description = "In-memory patricia trie operations" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -categories = [ "no-std" ] -edition = "2018" - -[dependencies] -hash-db = { path = "../hash-db", default-features = false, version = "0.16.0" } - -[features] -default = ["std"] -std = [ - "hash-db/std" -] diff --git a/trie-root/README.md b/trie-root/README.md deleted file mode 100644 index 36d38b68..00000000 --- a/trie-root/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This crate provides utility functions to validate and initialize tries using flexible input. -It is used extensively in `substrate` to validate blocks (mostly transactions and receipt roots). diff --git a/trie-root/test/Cargo.toml b/trie-root/test/Cargo.toml deleted file mode 100644 index 3e3b1e26..00000000 --- a/trie-root/test/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "trie-root-test" -version = "0.21.0" -authors = ["Parity Technologies "] -description = "Tests fo trie-root crate" -repository = "https://github.com/paritytech/trie" -license = "Apache-2.0" -categories = [ ] -edition = "2018" - -[dependencies] -trie-root = { path = "..", version = "0.18.0" } -hex-literal = "0.3" -keccak-hasher = { path = "../../test-support/keccak-hasher", version = "0.16.0" } -reference-trie = { path = "../../test-support/reference-trie", version = "0.29.0" } diff --git a/trie-root/test/src/lib.rs b/trie-root/test/src/lib.rs deleted file mode 100644 index 0fef011c..00000000 --- a/trie-root/test/src/lib.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017, 2020 Parity Technologies -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Test for trie-root crate. - -#[cfg(test)] -mod test { - use hex_literal::hex; - use keccak_hasher::KeccakHasher; - use reference_trie::ReferenceTrieStream; - use trie_root::{sec_trie_root, trie_root}; - - #[test] - fn previous_doc_test_1() { - let v = vec![("doe", "reindeer"), ("dog", "puppy"), ("dogglesworth", "cat")]; - - let root = hex!["d6e02b2bd48aa04fd2ad87cfac1144a29ca7f7dc60f4526c7b7040763abe3d43"]; - assert_eq!( - sec_trie_root::(v, Default::default()), - root - ); - } - - #[test] - fn previous_doc_test_2() { - let v = vec![("doe", "reindeer"), ("dog", "puppy"), ("dogglesworth", "cat")]; - - let root = hex!["0807d5393ae7f349481063ebb5dbaf6bda58db282a385ca97f37dccba717cb79"]; - assert_eq!( - trie_root::(v, Default::default()), - root - ); - } -} From 2699e69a7fb119955e81d9f5ee8b7b794c028f81 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Tue, 9 Apr 2024 18:55:25 +0200 Subject: [PATCH 71/90] possibly repro issue on ksm --- test/src/triedbmut.rs | 271 ++++++++++++++++++++++++++---------------- 1 file changed, 167 insertions(+), 104 deletions(-) diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index 951d1319..3b4a6090 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -870,135 +870,198 @@ fn attached_trie_internal>() { // Direct copy if using ref counting and location in db. // Pruning. let mut seed = Default::default(); - let nb_attached_trie = 10; + let nb_attached_trie = 5; // TODO happen between 4 and 5 + let nb_attaching = 2; // let nb_attached_trie = 1; let support_location = DB::support_location(); let mut memdb = DB::default(); - let mut attached_tries: BTreeMap, ATrie> = Default::default(); let mut keyspaced_memdb; let mut main_trie: ATrie = ATrie { root: Default::default(), data: Default::default(), changeset: None }; - for i in 0..nb_attached_trie + 1 { - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 3, - journal_key: 0, - value_mode: ValueMode::Index, - count: 20, - //count: 2, + let mut all_attached_tries: Vec, ATrie>> = Default::default(); + let mut main_root_init = false; + for _ in 0..nb_attaching { + let mut attached_tries: BTreeMap, ATrie> = Default::default(); + for i in 0..nb_attached_trie + 1 { + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 3, + journal_key: 0, + value_mode: ValueMode::Index, + count: 20, + //count: 2, + } + .make_with(&mut seed); + + let data: BTreeMap, Vec> = x.iter().cloned().collect(); + if i == nb_attached_trie { + let mut memtrie = if !main_root_init { + populate_trie::(&memdb, &x) + } else { + TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build() + }; + for (k, c) in attached_tries.iter_mut() { + let key: &[u8] = &k[..]; + let val: &[u8] = c.root.as_ref(); + let changeset = c.changeset.take().unwrap(); + memtrie.insert_with_tree_ref(key, val, Some(changeset)).unwrap(); + } + let changeset = memtrie.commit(); + let root = changeset.commit_to(&mut memdb); + main_trie.root = root; + main_trie.data = data; // TODO default (unused and unmaintained) + } else { + let memtrie = populate_trie::(&memdb, &x); + let attached_trie_root_key = data.iter().next().unwrap().0; + println!("{:x?}", attached_trie_root_key); + let changeset = memtrie.commit_with_keyspace(attached_trie_root_key); + let root = changeset.root_hash(); + attached_tries.insert( + attached_trie_root_key.clone(), + ATrie { root, data, changeset: Some(changeset.into()) }, + ); + } } - .make_with(&mut seed); - - let mut memtrie = populate_trie::(&mut memdb, &x); - let data: BTreeMap, Vec> = x.iter().cloned().collect(); - if i == nb_attached_trie { - for (k, c) in attached_tries.iter_mut() { - let key: &[u8] = &k[..]; - let val: &[u8] = c.root.as_ref(); - let changeset = c.changeset.take().unwrap(); - memtrie.insert_with_tree_ref(key, val, Some(changeset)).unwrap(); + all_attached_tries.push(attached_tries); + // check data + if !main_root_init { + let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); + for (k, v) in main_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); } - let changeset = memtrie.commit(); - let root = changeset.commit_to(&mut memdb); - main_trie.root = root; - main_trie.data = data; - } else { - let attached_trie_root_key = data.iter().next().unwrap().0; - let changeset = memtrie.commit_with_keyspace(attached_trie_root_key); - let root = changeset.root_hash(); - attached_tries.insert( - attached_trie_root_key.clone(), - ATrie { root, data, changeset: Some(changeset.into()) }, - ); + main_root_init = true; } - } - // check data - { - let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); - for (k, v) in main_trie.data.iter() { - assert_eq!(&trie.get(k).unwrap().unwrap(), v); + for attached_tries in all_attached_tries.iter() { + for (root_key, attached_trie) in attached_tries { + let (attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); + + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + assert!(attached_trie_location.is_none()); + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + + let trie = TrieDBBuilder::::new_with_db_location( + child_memdb, + &attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + for (k, v) in attached_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } } - } - for (root_key, attached_trie) in &attached_tries { - let (attached_trie_root, attached_trie_location) = + } + // Modifying an existing child trie. + let attached_tries = all_attached_tries.last_mut().unwrap(); + let (root_key, a_attached_trie) = attached_tries.iter().next().unwrap(); + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); + let (tree_ref_changeset, treeref_root_hash) = { + assert_eq!(a_attached_trie_root, a_attached_trie.root); + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( + child_memdb, + a_attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + attached_trie.remove(a_attached_trie.data.iter().next().unwrap().0).unwrap(); + attached_trie.insert(b"make_sure_it_changes", b"value").unwrap(); + let changeset = attached_trie.commit_with_keyspace(root_key); + let new_root = changeset.root_hash(); + assert!(new_root != a_attached_trie_root); + (changeset, new_root) + }; + let mut main_tree = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); + main_tree + .insert_with_tree_ref( + root_key, + treeref_root_hash.as_ref(), + Some(tree_ref_changeset.into()), + ) + .unwrap(); + let changeset = main_tree.commit(); + main_trie.root = changeset.root_hash(); + changeset.commit_to(&mut memdb); + // checking modification + let (a_attached_trie_root, attached_trie_location) = attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { &memdb } else { - assert!(attached_trie_location.is_none()); keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); &keyspaced_memdb }; - let trie = TrieDBBuilder::::new_with_db_location( child_memdb, - &attached_trie_root, + &a_attached_trie_root, attached_trie_location.unwrap_or_default(), ) .build(); - for (k, v) in attached_trie.data.iter() { - assert_eq!(&trie.get(k).unwrap().unwrap(), v); + trie.get(b"make_sure_it_changes").unwrap().unwrap(); + let mut first = true; + for (k, v) in a_attached_trie.data.iter() { + if first { + assert!(&trie.get(k).unwrap().is_none()); + first = false; + } else { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } } - } - // Modifying an existing child trie. - let (root_key, a_attached_trie) = attached_tries.iter().next().unwrap(); - let (a_attached_trie_root, attached_trie_location) = - attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); - let (tree_ref_changeset, treeref_root_hash) = { - assert_eq!(a_attached_trie_root, a_attached_trie.root); - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { - &memdb - } else { - keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); - &keyspaced_memdb + + //------- + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); + let tree_ref_changeset = { + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( + child_memdb, + a_attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + attached_trie.remove(b"make_sure_it_changes").unwrap(); + let mut first = true; + for (k, _v) in a_attached_trie.data.iter() { + if !first { + attached_trie.remove(k).unwrap(); + } else { + first = false; + } + } + let changeset = attached_trie.commit_with_keyspace(root_key); + //let new_root = changeset.root_hash(); + //assert!(new_root != empty_); + changeset }; - let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( - child_memdb, - a_attached_trie_root, - attached_trie_location.unwrap_or_default(), - ) - .build(); - attached_trie.remove(a_attached_trie.data.iter().next().unwrap().0).unwrap(); - attached_trie.insert(b"make_sure_it_changes", b"value").unwrap(); - let changeset = attached_trie.commit_with_keyspace(root_key); - let new_root = changeset.root_hash(); - assert!(new_root != a_attached_trie_root); - (changeset, new_root) - }; - let mut main_trie = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); - main_trie - .insert_with_tree_ref(root_key, treeref_root_hash.as_ref(), Some(tree_ref_changeset.into())) - .unwrap(); - let changeset = main_trie.commit(); - let main_root = changeset.root_hash(); - changeset.commit_to(&mut memdb); - // checking modification - let (a_attached_trie_root, attached_trie_location) = - attached_trie_root(&memdb, &main_root, root_key).unwrap(); - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { - &memdb - } else { - keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); - &keyspaced_memdb - }; - let trie = TrieDBBuilder::::new_with_db_location( - child_memdb, - &a_attached_trie_root, - attached_trie_location.unwrap_or_default(), - ) - .build(); - trie.get(b"make_sure_it_changes").unwrap().unwrap(); - let mut first = true; - for (k, v) in a_attached_trie.data.iter() { - if first { - assert!(&trie.get(k).unwrap().is_none()); - first = false; - } else { - assert_eq!(&trie.get(k).unwrap().unwrap(), v); - } + + let mut main_tree = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); + main_tree + .remove_with_tree_ref( + root_key, + Some(tree_ref_changeset.into()), + ) + .unwrap(); + let changeset = main_tree.commit(); + main_trie.root = changeset.root_hash(); + changeset.commit_to(&mut memdb); + let root_key = root_key.clone(); + attached_tries.remove(&root_key); } - trie.get(b"make_sure_it_changes").unwrap().unwrap(); } #[cfg(test)] From e00e121c8ab37400f13e776cfdf3dbe7642c4f49 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 10 Apr 2024 18:54:39 +0200 Subject: [PATCH 72/90] fix to bring (warn change test need to be restored and with removal handle). --- subtrie/src/node.rs | 3 ++- subtrie/src/triedbmut.rs | 6 +++--- test/src/triedbmut.rs | 29 +++++++++++++++++++++++------ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/subtrie/src/node.rs b/subtrie/src/node.rs index d0a36804..2ab25a01 100644 --- a/subtrie/src/node.rs +++ b/subtrie/src/node.rs @@ -188,7 +188,8 @@ impl ValueOwned { /// Type of node in the trie and essential information thereof. #[derive(Eq, PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum Node<'a, L> { +pub enum Node<'a, L> { // TODO HERE node need attached content location so when change in triedbmut this content is kept + // see how it is stored in triedbmut `Node` /// Null trie node; could be an empty root or an empty branch entry. Empty, /// Leaf node; has key slice and value. Value may not be empty. diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index 1b1c9c80..195ab6fb 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -760,8 +760,8 @@ pub struct NewChangesetNode { #[derive(Debug)] pub struct ExistingChangesetNode { - pub hash: H, - pub prefix: OwnedPrefix, + pub hash: H, // TODO is it used? + pub prefix: OwnedPrefix, // TODO is it used? pub location: DL, } @@ -980,7 +980,7 @@ where Stored::Cached(node, hash, location) => match inspector(self, node, key)? { Action::Restore(node) => Some((Stored::Cached(node, hash, location), false)), Action::Replace(node) => { - self.death_row.insert((hash, current_key.left_owned())); + self.death_row.insert((hash, current_key.left_owned())); // TODO don't feed deathrow when unused (location support except root) Some((Stored::New(node), true)) }, Action::Delete(tree_ref) => { diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index 3b4a6090..a559844e 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -870,7 +870,7 @@ fn attached_trie_internal>() { // Direct copy if using ref counting and location in db. // Pruning. let mut seed = Default::default(); - let nb_attached_trie = 5; // TODO happen between 4 and 5 + let nb_attached_trie = 4; // TODO happen between 3 and 4 let nb_attaching = 2; // let nb_attached_trie = 1; let support_location = DB::support_location(); @@ -880,7 +880,7 @@ fn attached_trie_internal>() { ATrie { root: Default::default(), data: Default::default(), changeset: None }; let mut all_attached_tries: Vec, ATrie>> = Default::default(); let mut main_root_init = false; - for _ in 0..nb_attaching { + for a in 0..nb_attaching { let mut attached_tries: BTreeMap, ATrie> = Default::default(); for i in 0..nb_attached_trie + 1 { let x = StandardMap { @@ -888,8 +888,8 @@ fn attached_trie_internal>() { min_key: 3, journal_key: 0, value_mode: ValueMode::Index, - count: 20, - //count: 2, + //count: 20, + count: 2, } .make_with(&mut seed); @@ -900,11 +900,16 @@ fn attached_trie_internal>() { } else { TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build() }; + let mut b = 0; for (k, c) in attached_tries.iter_mut() { let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let changeset = c.changeset.take().unwrap(); memtrie.insert_with_tree_ref(key, val, Some(changeset)).unwrap(); + if a == 1 && b == 1 { + break; + } + b+=1; } let changeset = memtrie.commit(); let root = changeset.commit_to(&mut memdb); @@ -926,13 +931,20 @@ fn attached_trie_internal>() { // check data if !main_root_init { let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); + println!("{:?}", trie); for (k, v) in main_trie.data.iter() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } main_root_init = true; + } else { + let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); + println!("{:?}", trie); } for attached_tries in all_attached_tries.iter() { + let mut first = 0; for (root_key, attached_trie) in attached_tries { + first+=1; + if first > 0 { let (attached_trie_root, attached_trie_location) = attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); @@ -953,7 +965,8 @@ fn attached_trie_internal>() { for (k, v) in attached_trie.data.iter() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } - } + }} + } // Modifying an existing child trie. let attached_tries = all_attached_tries.last_mut().unwrap(); @@ -1017,7 +1030,10 @@ fn attached_trie_internal>() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } - + //-- + let root_key = root_key.clone(); + attached_tries.remove(&root_key); +/* //------- let (a_attached_trie_root, attached_trie_location) = attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); @@ -1061,6 +1077,7 @@ fn attached_trie_internal>() { changeset.commit_to(&mut memdb); let root_key = root_key.clone(); attached_tries.remove(&root_key); +*/ } } From 96867d4560bf2bf7ebef53c925bde583c6f34a84 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 15 Apr 2024 14:34:08 +0200 Subject: [PATCH 73/90] switch to tree ref changeset enum --- subtrie/src/lib.rs | 4 +- subtrie/src/node.rs | 5 +- subtrie/src/triedbmut.rs | 165 +++++++++++++++++++++++++++------------ test/src/triedbmut.rs | 149 ++++++++++++++++++----------------- 4 files changed, 196 insertions(+), 127 deletions(-) diff --git a/subtrie/src/lib.rs b/subtrie/src/lib.rs index 51421177..a3f45fa6 100644 --- a/subtrie/src/lib.rs +++ b/subtrie/src/lib.rs @@ -79,8 +79,8 @@ pub use self::{ recorder::Recorder, triedb::{TrieDB, TrieDBBuilder, TrieDBIterator, TrieDBKeyIterator}, triedbmut::{ - Changeset, ChildReference, ExistingChangesetNode, NewChangesetNode, OwnedPrefix, TrieDBMut, - TrieDBMutBuilder, Value, + Changeset, ChildReference, ExistingChangesetNode, NewChangesetNode, OwnedPrefix, + TreeRefChangeset, TrieDBMut, TrieDBMutBuilder, Value, }, }; use crate::node_db::Hasher; diff --git a/subtrie/src/node.rs b/subtrie/src/node.rs index 2ab25a01..e173c361 100644 --- a/subtrie/src/node.rs +++ b/subtrie/src/node.rs @@ -188,8 +188,9 @@ impl ValueOwned { /// Type of node in the trie and essential information thereof. #[derive(Eq, PartialEq, Clone)] #[cfg_attr(feature = "std", derive(Debug))] -pub enum Node<'a, L> { // TODO HERE node need attached content location so when change in triedbmut this content is kept - // see how it is stored in triedbmut `Node` +pub enum Node<'a, L> { + // TODO HERE node need attached content location so when change in triedbmut this content is + // kept see how it is stored in triedbmut `Node` /// Null trie node; could be an empty root or an empty branch entry. Empty, /// Leaf node; has key slice and value. Value may not be empty. diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index 195ab6fb..4e696d0b 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -69,7 +69,67 @@ fn empty_children() -> Box<[Option>; nibble_ops::NIBBLE_L ]) } -pub type TreeRefChangeset = Box, ::Location>>; +/// Attached db level non merklized data. +pub enum TreeRefChangeset { + None, + Existing(::Location), + Changed(Box, ::Location>>), +} + +impl Default for TreeRefChangeset { + fn default() -> Self { + TreeRefChangeset::None + } +} + +impl TreeRefChangeset { + fn location(&self) -> Option { + match self { + TreeRefChangeset::None => None, + TreeRefChangeset::Existing(l) => Some(*l), + TreeRefChangeset::Changed(c) => match &**c { + Changeset::New(_) => None, + Changeset::Existing(e) => Some(e.location), + }, + } + } + + fn has_changes(&self) -> bool { + if let TreeRefChangeset::Changed(c) = self { + if let Changeset::New(_) = &**c { + return true + } + } + false + } + + fn changes_ref(&self) -> Option<&NewChangesetNode, ::Location>> { + if let TreeRefChangeset::Changed(c) = self { + if let Changeset::New(c) = &**c { + return Some(c) + } + } + None + } + + fn changes_mut( + &mut self, + ) -> Option<&mut NewChangesetNode, ::Location>> { + if let TreeRefChangeset::Changed(c) = self { + if let Changeset::New(c) = &mut **c { + return Some(c) + } + } + None + } + + fn into_changes(self) -> Option, ::Location>>> { + if let TreeRefChangeset::Changed(c) = self { + return Some(c) + } + None + } +} /// Type alias to indicate the nible covers a full key, /// therefore its left side is a full prefix. @@ -221,7 +281,7 @@ enum Node { /// A leaf node contains the end of a key and a value. /// This key is encoded from a `NibbleSlice`, meaning it contains /// a flag indicating it is a leaf. - Leaf(NodeKey, Value, Option>), + Leaf(NodeKey, Value, TreeRefChangeset), /// An extension contains a shared portion of a key and a child node. /// The shared portion is encoded from a `NibbleSlice` meaning it contains /// a flag indicating it is an extension. @@ -231,14 +291,14 @@ enum Node { Branch( Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, Option>, - Option>, + TreeRefChangeset, ), /// Branch node with support for a nibble (to avoid extension node). NibbledBranch( NodeKey, Box<[Option, L::Location>>; nibble_ops::NIBBLE_LENGTH]>, Option>, - Option>, + TreeRefChangeset, ), } @@ -331,7 +391,7 @@ impl Node { .map_err(|e| Box::new(TrieError::DecoderError(node_hash, e)))?; let node = match encoded_node { EncodedNode::Empty => Node::Empty, - EncodedNode::Leaf(k, v) => Node::Leaf(k.into(), v.into(), None), + EncodedNode::Leaf(k, v) => Node::Leaf(k.into(), v.into(), TreeRefChangeset::None), EncodedNode::Extension(key, cb) => Node::Extension(key.into(), Self::inline_or_hash(node_hash, cb, storage)?), EncodedNode::Branch(encoded_children, val) => { @@ -359,7 +419,7 @@ impl Node { child(15)?, ]); - Node::Branch(children, val.map(Into::into), None) + Node::Branch(children, val.map(Into::into), TreeRefChangeset::None) }, EncodedNode::NibbledBranch(k, encoded_children, val) => { let mut child = |i: usize| match encoded_children[i] { @@ -386,7 +446,7 @@ impl Node { child(15)?, ]); - Node::NibbledBranch(k.into(), children, val.map(Into::into), None) + Node::NibbledBranch(k.into(), children, val.map(Into::into), TreeRefChangeset::None) }, }; Ok(node) @@ -399,7 +459,7 @@ impl Node { ) -> Self { match node_owned { NodeOwned::Empty => Node::Empty, - NodeOwned::Leaf(k, v) => Node::Leaf(k.into(), v.into(), None), + NodeOwned::Leaf(k, v) => Node::Leaf(k.into(), v.into(), TreeRefChangeset::None), NodeOwned::Extension(key, cb) => Node::Extension(key.into(), Self::inline_or_hash_owned(cb, storage)), NodeOwned::Branch(encoded_children, val) => { @@ -428,7 +488,7 @@ impl Node { child(15), ]); - Node::Branch(children, val.as_ref().map(Into::into), None) + Node::Branch(children, val.as_ref().map(Into::into), TreeRefChangeset::None) }, NodeOwned::NibbledBranch(k, encoded_children, val) => { let mut child = |i: usize| { @@ -456,7 +516,12 @@ impl Node { child(15), ]); - Node::NibbledBranch(k.into(), children, val.as_ref().map(Into::into), None) + Node::NibbledBranch( + k.into(), + children, + val.as_ref().map(Into::into), + TreeRefChangeset::None, + ) }, NodeOwned::Value(_, _) => unreachable!("`NodeOwned::Value` can only be returned for the hash of a value."), @@ -466,7 +531,7 @@ impl Node { // TODO: parallelize /// Here `child_cb` should process the first parameter to either insert an external /// node value or to encode and add a new branch child node. - fn into_encoded(self, mut child_cb: F) -> (Vec, Option>) + fn into_encoded(self, mut child_cb: F) -> (Vec, TreeRefChangeset) where F: FnMut( NodeToEncode, L::Location>, @@ -475,7 +540,7 @@ impl Node { ) -> ChildReference, L::Location>, { match self { - Node::Empty => (L::Codec::empty_node().to_vec(), None), + Node::Empty => (L::Codec::empty_node().to_vec(), Default::default()), Node::Leaf(partial, mut value, tree_ref) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let value = value.into_encoded::(Some(&pr), &mut child_cb); @@ -485,7 +550,7 @@ impl Node { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let it = pr.right_iter(); let c = child_cb(NodeToEncode::TrieNode(child), Some(&pr), None); - (L::Codec::extension_node(it, pr.len(), c), None) + (L::Codec::extension_node(it, pr.len(), c), TreeRefChangeset::None) }, Node::Branch(mut children, mut value, tree_ref) => { let value = value.as_mut().map(|v| v.into_encoded::(None, &mut child_cb)); @@ -542,7 +607,7 @@ enum Action { // Restore the original node. This trusts that the node is actually the original. Restore(Node), // if it is a new node, just clears the storage. - Delete(Option>), + Delete(TreeRefChangeset), } // post-insert action. Same as action without delete @@ -760,7 +825,7 @@ pub struct NewChangesetNode { #[derive(Debug)] pub struct ExistingChangesetNode { - pub hash: H, // TODO is it used? + pub hash: H, // TODO is it used? pub prefix: OwnedPrefix, // TODO is it used? pub location: DL, } @@ -973,7 +1038,9 @@ where Action::Restore(node) => Some((Stored::New(node), false)), Action::Replace(node) => Some((Stored::New(node), true)), Action::Delete(tree_ref) => { - tree_ref.map(|c| self.death_row_child.push(c)); + if !matches!(tree_ref, TreeRefChangeset::None) { + self.death_row_child.push(tree_ref); + } None }, }, @@ -985,7 +1052,9 @@ where }, Action::Delete(tree_ref) => { self.death_row.insert((hash, current_key.left_owned())); - tree_ref.map(|c| self.death_row_child.push(c)); + if !matches!(tree_ref, TreeRefChangeset::None) { + self.death_row_child.push(tree_ref); + } None }, }, @@ -1097,7 +1166,7 @@ where key: &mut NibbleFullKey, value: Bytes, old_val: &mut Option>, - tree_ref: Option>, + tree_ref: TreeRefChangeset, ) -> Result<(StorageHandle, bool), TrieHash, CError> { let h = match handle { NodeHandle::InMemory(h) => h, @@ -1141,7 +1210,7 @@ where key: &mut NibbleFullKey, value: Bytes, old_val: &mut Option>, - tree_ref: Option>, + tree_ref: TreeRefChangeset, ) -> Result, TrieHash, CError> { let partial = *key; @@ -1265,7 +1334,7 @@ where existing_key.to_stored_range(common), children, None, - None, + TreeRefChangeset::None, )) } } else { @@ -1354,13 +1423,13 @@ where children[idx] = Some(self.storage.alloc(Stored::New(new_leaf)).into()); if L::USE_EXTENSION { - Node::Branch(children, None, None) + Node::Branch(children, None, TreeRefChangeset::None) } else { Node::NibbledBranch( partial.to_stored_range(common), children, None, - None, + TreeRefChangeset::None, ) } }; @@ -1465,7 +1534,7 @@ where // continue inserting. let branch_action = self .insert_inspector( - Node::Branch(children, None, None), + Node::Branch(children, None, TreeRefChangeset::None), key, value, old_val, @@ -1528,7 +1597,7 @@ where handle: NodeHandle, L::Location>, key: &mut NibbleFullKey, old_val: &mut Option>, - tree_ref: Option>, + tree_ref: TreeRefChangeset, ) -> Result, TrieHash, CError> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), @@ -1551,11 +1620,11 @@ where node: Node, key: &mut NibbleFullKey, old_val: &mut Option>, - tree_ref: Option>, + tree_ref: TreeRefChangeset, ) -> Result, TrieHash, CError> { let partial = *key; Ok(match (node, partial.is_empty()) { - (Node::Empty, _) => Action::Delete(None), + (Node::Empty, _) => Action::Delete(TreeRefChangeset::None), (Node::Branch(c, None, _), true) => Action::Restore(Node::Branch(c, None, tree_ref)), (Node::NibbledBranch(n, c, None, _), true) => Action::Restore(Node::NibbledBranch(n, c, None, tree_ref)), @@ -1718,7 +1787,7 @@ where None => { // the whole branch got deleted. // that means that this extension is useless. - Action::Delete(None) + Action::Delete(TreeRefChangeset::None) }, } } else { @@ -1853,7 +1922,9 @@ where node }, }; - btreerefset.map(|c| self.death_row_child.push(c)); + if !matches!(btreerefset, TreeRefChangeset::None) { + self.death_row_child.push(btreerefset); + } match child_node { Node::Leaf(sub_partial, value, ltreerefset) => { let mut enc_nibble = enc_nibble; @@ -2074,7 +2145,7 @@ where }, } }); - roottreerefset.map(|c| { + roottreerefset.into_changes().map(|c| { children.push(*c); }); #[cfg(feature = "std")] @@ -2196,7 +2267,7 @@ where let result = if encoded.len() >= L::Hash::LENGTH { let hash = self.db.hash(&encoded); self.cache_node(hash); - child_set.map(|c| { + child_set.into_changes().map(|c| { sub_children.push(*c); }); children.push(Changeset::New(NewChangesetNode { @@ -2256,24 +2327,21 @@ where key: &[u8], value: &[u8], ) -> Result>, TrieHash, CError> { - self.insert_with_tree_ref(key, value, None) + self.insert_with_tree_ref(key, value, TreeRefChangeset::None) } pub fn insert_with_tree_ref( &mut self, key: &[u8], value: &[u8], - tree_ref: Option>, + tree_ref: TreeRefChangeset, ) -> Result>, TrieHash, CError> { // expect for the child changes to have a key. - debug_assert!(tree_ref - .as_ref() - .map(|c| if let Changeset::New(set) = c.as_ref() { - set.removed_keys.is_some() - } else { - true - }) - .unwrap_or(true)); + debug_assert!(if let Some(set) = tree_ref.changes_ref() { + set.removed_keys.is_some() + } else { + true + }); if !L::ALLOW_EMPTY && value.is_empty() { return self.remove(key) } @@ -2296,23 +2364,20 @@ where } pub fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { - self.remove_with_tree_ref(key, None) + self.remove_with_tree_ref(key, TreeRefChangeset::None) } pub fn remove_with_tree_ref( &mut self, key: &[u8], - tree_ref: Option>, + tree_ref: TreeRefChangeset, ) -> Result>, TrieHash, CError> { // expect for the child changes to have a key. - debug_assert!(tree_ref - .as_ref() - .map(|c| if let Changeset::New(set) = c.as_ref() { - set.removed_keys.is_some() - } else { - true - }) - .unwrap_or(true)); + debug_assert!(if let Some(set) = tree_ref.changes_ref() { + set.removed_keys.is_some() + } else { + true + }); #[cfg(feature = "std")] trace!(target: "trie", "remove: key={:?}", ToHex(key)); diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index a559844e..dcd93a5d 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -25,8 +25,9 @@ use trie_db::{ memory_db::{HashKey, MemoryDB, PrefixedKey}, node_db::{Hasher, NodeDB, Prefix, EMPTY_PREFIX}, test_utils::*, - CachedValue, Changeset, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, - TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, TrieLayout, Value, + CachedValue, Changeset, DBValue, NodeCodec, Recorder, TreeRefChangeset, Trie, TrieCache, + TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, + TrieLayout, Value, }; use crate::{TestCommit, TestDB}; @@ -905,11 +906,13 @@ fn attached_trie_internal>() { let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let changeset = c.changeset.take().unwrap(); - memtrie.insert_with_tree_ref(key, val, Some(changeset)).unwrap(); + memtrie + .insert_with_tree_ref(key, val, TreeRefChangeset::Changed(changeset)) + .unwrap(); if a == 1 && b == 1 { break; } - b+=1; + b += 1; } let changeset = memtrie.commit(); let root = changeset.commit_to(&mut memdb); @@ -942,31 +945,31 @@ fn attached_trie_internal>() { } for attached_tries in all_attached_tries.iter() { let mut first = 0; - for (root_key, attached_trie) in attached_tries { - first+=1; - if first > 0 { - let (attached_trie_root, attached_trie_location) = - attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); - - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { - &memdb - } else { - assert!(attached_trie_location.is_none()); - keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); - &keyspaced_memdb - }; - - let trie = TrieDBBuilder::::new_with_db_location( - child_memdb, - &attached_trie_root, - attached_trie_location.unwrap_or_default(), - ) - .build(); - for (k, v) in attached_trie.data.iter() { - assert_eq!(&trie.get(k).unwrap().unwrap(), v); + for (root_key, attached_trie) in attached_tries { + first += 1; + if first > 0 { + let (attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); + + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + assert!(attached_trie_location.is_none()); + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + + let trie = TrieDBBuilder::::new_with_db_location( + child_memdb, + &attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + for (k, v) in attached_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } + } } - }} - } // Modifying an existing child trie. let attached_tries = all_attached_tries.last_mut().unwrap(); @@ -999,7 +1002,7 @@ fn attached_trie_internal>() { .insert_with_tree_ref( root_key, treeref_root_hash.as_ref(), - Some(tree_ref_changeset.into()), + TreeRefChangeset::Changed(tree_ref_changeset.into()), ) .unwrap(); let changeset = main_tree.commit(); @@ -1033,51 +1036,51 @@ fn attached_trie_internal>() { //-- let root_key = root_key.clone(); attached_tries.remove(&root_key); -/* - //------- - let (a_attached_trie_root, attached_trie_location) = - attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); - let tree_ref_changeset = { - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { - &memdb - } else { - keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); - &keyspaced_memdb - }; - let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( - child_memdb, - a_attached_trie_root, - attached_trie_location.unwrap_or_default(), - ) - .build(); - attached_trie.remove(b"make_sure_it_changes").unwrap(); - let mut first = true; - for (k, _v) in a_attached_trie.data.iter() { - if !first { - attached_trie.remove(k).unwrap(); - } else { - first = false; - } - } - let changeset = attached_trie.commit_with_keyspace(root_key); - //let new_root = changeset.root_hash(); - //assert!(new_root != empty_); - changeset - }; + /* + //------- + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); + let tree_ref_changeset = { + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( + child_memdb, + a_attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + attached_trie.remove(b"make_sure_it_changes").unwrap(); + let mut first = true; + for (k, _v) in a_attached_trie.data.iter() { + if !first { + attached_trie.remove(k).unwrap(); + } else { + first = false; + } + } + let changeset = attached_trie.commit_with_keyspace(root_key); + //let new_root = changeset.root_hash(); + //assert!(new_root != empty_); + changeset + }; - let mut main_tree = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); - main_tree - .remove_with_tree_ref( - root_key, - Some(tree_ref_changeset.into()), - ) - .unwrap(); - let changeset = main_tree.commit(); - main_trie.root = changeset.root_hash(); - changeset.commit_to(&mut memdb); - let root_key = root_key.clone(); - attached_tries.remove(&root_key); -*/ + let mut main_tree = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); + main_tree + .remove_with_tree_ref( + root_key, + Some(tree_ref_changeset.into()), + ) + .unwrap(); + let changeset = main_tree.commit(); + main_trie.root = changeset.root_hash(); + changeset.commit_to(&mut memdb); + let root_key = root_key.clone(); + attached_tries.remove(&root_key); + */ } } From f46295a7bffdfc8ac3c01a145035956b5209985b Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 15 Apr 2024 17:21:46 +0200 Subject: [PATCH 74/90] change types to contain attached location --- reference-trie/src/lib.rs | 2 +- subtrie/src/iterator.rs | 12 +-- subtrie/src/lib.rs | 10 ++- subtrie/src/lookup.rs | 18 ++-- subtrie/src/node.rs | 139 +++++++++++++++++------------- subtrie/src/node_codec.rs | 4 +- subtrie/src/proof/generate.rs | 18 ++-- subtrie/src/proof/verify.rs | 20 ++--- subtrie/src/trie_codec.rs | 28 +++--- subtrie/src/triedb.rs | 6 +- subtrie/src/triedbmut.rs | 34 ++++---- test/src/double_ended_iterator.rs | 30 +++---- test/src/iterator.rs | 26 +++--- 13 files changed, 189 insertions(+), 158 deletions(-) diff --git a/reference-trie/src/lib.rs b/reference-trie/src/lib.rs index 51e55fd4..691cfe44 100644 --- a/reference-trie/src/lib.rs +++ b/reference-trie/src/lib.rs @@ -1262,7 +1262,7 @@ mod tests { ); let dec = as NodeCodec>::decode(&enc, &[] as &[()]).unwrap(); - let o_sl = if let Node::Leaf(sl, _) = dec { Some(sl) } else { None }; + let o_sl = if let Node::Leaf(sl, _, _) = dec { Some(sl) } else { None }; assert!(o_sl.is_some()); } diff --git a/subtrie/src/iterator.rs b/subtrie/src/iterator.rs index 4a18b322..8a25ece5 100644 --- a/subtrie/src/iterator.rs +++ b/subtrie/src/iterator.rs @@ -557,15 +557,15 @@ impl TrieDBRawIterator { ) -> Option, CError>> { let mut prefix = prefix.clone(); let value = match node.node() { - Node::Leaf(partial, value) => { + Node::Leaf(partial, value, _) => { prefix.append_partial(partial.right()); value }, - Node::Branch(_, value) => match value { + Node::Branch(_, value, _) => match value { Some(value) => value, None => return None, }, - Node::NibbledBranch(partial, _, value) => { + Node::NibbledBranch(partial, _, value, _) => { prefix.append_partial(partial.right()); match value { Some(value) => value, @@ -651,15 +651,15 @@ impl TrieDBRawIterator { let mut prefix = prefix.clone(); let value = match node.node() { - Node::Leaf(partial, value) => { + Node::Leaf(partial, value, _) => { prefix.append_partial(partial.right()); value }, - Node::Branch(_, value) => match value { + Node::Branch(_, value, _) => match value { Some(value) => value, None => return None, }, - Node::NibbledBranch(partial, _, value) => { + Node::NibbledBranch(partial, _, value, _) => { prefix.append_partial(partial.right()); match value { Some(value) => value, diff --git a/subtrie/src/lib.rs b/subtrie/src/lib.rs index a3f45fa6..d97543bd 100644 --- a/subtrie/src/lib.rs +++ b/subtrie/src/lib.rs @@ -378,7 +378,15 @@ pub trait TrieLayout { } /// Trait alias for requirement of location with `TrieLayout`. -pub trait Location: Copy + Default + Eq + PartialEq + MaybeDebug {} +pub trait Location: Copy + Default + Eq + PartialEq + MaybeDebug { + fn into_changes>(self) -> TreeRefChangeset { + if self == Self::default() { + TreeRefChangeset::None + } else { + TreeRefChangeset::Existing(self) + } + } +} impl Location for T {} diff --git a/subtrie/src/lookup.rs b/subtrie/src/lookup.rs index 45accfeb..fb335f67 100644 --- a/subtrie/src/lookup.rs +++ b/subtrie/src/lookup.rs @@ -223,7 +223,7 @@ where let mut is_inline = false; loop { let next_node = match node { - NodeOwned::Leaf(slice, _) => { + NodeOwned::Leaf(slice, _, _) => { // The leaf slice can be longer than remainder of the provided key // (descendent), but not the other way around. if !slice.starts_with_slice(&partial) { @@ -271,7 +271,7 @@ where return Ok(None) } }, - NodeOwned::Branch(children, value) => + NodeOwned::Branch(children, value, _) => if partial.is_empty() { if value.is_none() { self.record(|| TrieAccess::NonExisting { full_key }); @@ -294,7 +294,7 @@ where }, } }, - NodeOwned::NibbledBranch(slice, children, value) => { + NodeOwned::NibbledBranch(slice, children, value, _) => { // Not enough remainder key to continue the search. if partial.len() < slice.len() { self.record(|| TrieAccess::NonExisting { full_key }); @@ -625,7 +625,7 @@ where // without incrementing the depth. loop { let next_node = match node { - NodeOwned::Leaf(slice, value) => + NodeOwned::Leaf(slice, value, _) => return if partial == *slice { let value = (*value).clone(); load_value_owned( @@ -652,7 +652,7 @@ where return Ok(None) }, - NodeOwned::Branch(children, value) => + NodeOwned::Branch(children, value, _) => if partial.is_empty() { return if let Some(value) = value.clone() { load_value_owned( @@ -683,7 +683,7 @@ where }, } }, - NodeOwned::NibbledBranch(slice, children, value) => { + NodeOwned::NibbledBranch(slice, children, value, _) => { if !partial.starts_with_vec(&slice) { self.record(|| TrieAccess::NonExisting { full_key }); @@ -802,7 +802,7 @@ where }; let next_node = match decoded { - Node::Leaf(slice, value) => + Node::Leaf(slice, value, _) => return if slice == partial { load_value( value, @@ -828,7 +828,7 @@ where return Ok(None) }, - Node::Branch(children, value) => + Node::Branch(children, value, _) => if partial.is_empty() { return if let Some(val) = value { load_value( @@ -860,7 +860,7 @@ where }, } }, - Node::NibbledBranch(slice, children, value) => { + Node::NibbledBranch(slice, children, value, _) => { if !partial.starts_with(&slice) { self.record(|| TrieAccess::NonExisting { full_key }); diff --git a/subtrie/src/node.rs b/subtrie/src/node.rs index e173c361..7791dd6b 100644 --- a/subtrie/src/node.rs +++ b/subtrie/src/node.rs @@ -16,7 +16,7 @@ use crate::{ nibble::{self, nibble_ops, NibbleSlice, NibbleVec}, node_codec::NodeCodec, node_db::Hasher, - Bytes, CError, ChildReference, Result, TrieError, TrieHash, TrieLayout, + Bytes, CError, ChildReference, Location, Result, TrieError, TrieHash, TrieLayout, }; #[cfg(not(feature = "std"))] use alloc::{boxed::Box, vec::Vec}; @@ -34,13 +34,13 @@ pub enum NodeHandle<'a, L> { Inline(&'a [u8]), } -impl<'a, L: Copy + Default> NodeHandle<'a, L> { +impl<'a, L: Location> NodeHandle<'a, L> { /// Converts this node handle into a [`NodeHandleOwned`]. - pub fn to_owned_handle( + pub fn to_owned_handle( &self, ) -> Result, TL::Location>, TrieHash, CError> where - TL::Location: From, + TL: TrieLayout, { match self { Self::Hash(h, l) => decode_hash::(h) @@ -65,7 +65,7 @@ pub enum NodeHandleOwned { impl NodeHandleOwned where H: Default + AsRef<[u8]> + AsMut<[u8]> + Copy, - L: Default + Copy, + L: Location, { /// Returns `self` as a [`ChildReference`]. /// @@ -118,7 +118,7 @@ pub enum Value<'a, L> { Node(&'a [u8], L), } -impl<'a, L: Copy + Default> Value<'a, L> { +impl<'a, L: Location> Value<'a, L> { pub(crate) fn new_inline(value: &'a [u8], threshold: Option) -> Option { if let Some(threshold) = threshold { if value.len() >= threshold as usize { @@ -131,9 +131,9 @@ impl<'a, L: Copy + Default> Value<'a, L> { } } - pub fn to_owned_value(&self) -> ValueOwned, TL::Location> + pub fn to_owned_value(&self) -> ValueOwned, TL::Location> where - TL::Location: From, + TL: TrieLayout, { match self { Self::Inline(data) => ValueOwned::Inline(Bytes::from(*data), TL::Hash::hash(data)), @@ -157,7 +157,7 @@ pub enum ValueOwned { Node(H, L), } -impl + Copy, L: Copy + Default> ValueOwned { +impl + Copy, L: Location> ValueOwned { /// Returns self as [`Value`]. pub fn as_value(&self) -> Value { match self { @@ -193,36 +193,38 @@ pub enum Node<'a, L> { // kept see how it is stored in triedbmut `Node` /// Null trie node; could be an empty root or an empty branch entry. Empty, - /// Leaf node; has key slice and value. Value may not be empty. - Leaf(NibbleSlice<'a>, Value<'a, L>), + /// Leaf node; has key slice and value. Value may not be empty. An attached db location + /// reference can exist. + Leaf(NibbleSlice<'a>, Value<'a, L>, L), /// Extension node; has key slice and node data. Data may not be null. Extension(NibbleSlice<'a>, NodeHandle<'a, L>), /// Branch node; has slice of child nodes (each possibly null) /// and an optional immediate node data. - Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), + Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>, L), /// Branch node with support for a nibble (when extension nodes are not used). NibbledBranch( NibbleSlice<'a>, [Option>; nibble_ops::NIBBLE_LENGTH], Option>, + L, ), } impl Node<'_, Location> { /// Converts this node into a [`NodeOwned`]. - pub fn to_owned_node( + pub fn to_owned_node( &self, ) -> Result, L::Location>, TrieHash, CError> where - L::Location: From, - Location: Copy + Default, + L: TrieLayout, + Location: crate::Location, { match self { Self::Empty => Ok(NodeOwned::Empty), - Self::Leaf(n, d) => Ok(NodeOwned::Leaf((*n).into(), d.to_owned_value::())), + Self::Leaf(n, d, a) => Ok(NodeOwned::Leaf((*n).into(), d.to_owned_value::(), *a)), Self::Extension(n, h) => Ok(NodeOwned::Extension((*n).into(), h.to_owned_handle::()?)), - Self::Branch(childs, data) => { + Self::Branch(childs, data, attached) => { let mut childs_owned = [(); nibble_ops::NIBBLE_LENGTH].map(|_| None); childs .iter() @@ -234,9 +236,13 @@ impl Node<'_, Location> { }) .collect::>()?; - Ok(NodeOwned::Branch(childs_owned, data.as_ref().map(|d| d.to_owned_value::()))) + Ok(NodeOwned::Branch( + childs_owned, + data.as_ref().map(|d| d.to_owned_value::()), + *attached, + )) }, - Self::NibbledBranch(n, childs, data) => { + Self::NibbledBranch(n, childs, data, attached) => { let mut childs_owned = [(); nibble_ops::NIBBLE_LENGTH].map(|_| None); childs .iter() @@ -252,6 +258,7 @@ impl Node<'_, Location> { (*n).into(), childs_owned, data.as_ref().map(|d| d.to_owned_value::()), + *attached, )) }, } @@ -265,17 +272,18 @@ pub enum NodeOwned { /// Null trie node; could be an empty root or an empty branch entry. Empty, /// Leaf node; has key slice and value. Value may not be empty. - Leaf(NibbleVec, ValueOwned), + Leaf(NibbleVec, ValueOwned, L), /// Extension node; has key slice and node data. Data may not be null. Extension(NibbleVec, NodeHandleOwned), /// Branch node; has slice of child nodes (each possibly null) /// and an optional immediate node data. - Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), + Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>, L), /// Branch node with support for a nibble (when extension nodes are not used). NibbledBranch( NibbleVec, [Option>; nibble_ops::NIBBLE_LENGTH], Option>, + L, ), /// Node that represents a value. /// @@ -284,7 +292,7 @@ pub enum NodeOwned { Value(Bytes, H), } -impl NodeOwned +impl NodeOwned where H: Default + AsRef<[u8]> + AsMut<[u8]> + Copy, { @@ -295,18 +303,18 @@ where { match self { Self::Empty => C::empty_node().to_vec(), - Self::Leaf(partial, value) => + Self::Leaf(partial, value, _) => C::leaf_node(partial.right_iter(), partial.len(), value.as_value()), Self::Extension(partial, child) => C::extension_node( partial.right_iter(), partial.len(), child.as_child_reference::(), ), - Self::Branch(children, value) => C::branch_node( + Self::Branch(children, value, _) => C::branch_node( children.iter().map(|child| child.as_ref().map(|c| c.as_child_reference::())), value.as_ref().map(|v| v.as_value()), ), - Self::NibbledBranch(partial, children, value) => C::branch_node_nibbled( + Self::NibbledBranch(partial, children, value, _) => C::branch_node_nibbled( partial.right_iter(), partial.len(), children.iter().map(|child| child.as_ref().map(|c| c.as_child_reference::())), @@ -355,9 +363,9 @@ where } match self { - Self::Leaf(_, _) | Self::Empty | Self::Value(_, _) => ChildIter::Empty, + Self::Leaf(..) | Self::Empty | Self::Value(..) => ChildIter::Empty, Self::Extension(_, child) => ChildIter::Single(child, false), - Self::Branch(children, _) | Self::NibbledBranch(_, children, _) => + Self::Branch(children, ..) | Self::NibbledBranch(_, children, ..) => ChildIter::Array(children, 0), } } @@ -366,10 +374,10 @@ where pub fn data_hash(&self) -> Option { match &self { Self::Empty => None, - Self::Leaf(_, value) => value.data_hash(), + Self::Leaf(_, value, _) => value.data_hash(), Self::Extension(_, _) => None, - Self::Branch(_, value) => value.as_ref().and_then(|v| v.data_hash()), - Self::NibbledBranch(_, _, value) => value.as_ref().and_then(|v| v.data_hash()), + Self::Branch(_, value, _) => value.as_ref().and_then(|v| v.data_hash()), + Self::NibbledBranch(_, _, value, _) => value.as_ref().and_then(|v| v.data_hash()), Self::Value(_, hash) => Some(*hash), } } @@ -380,10 +388,10 @@ impl NodeOwned { pub fn data(&self) -> Option<&Bytes> { match &self { Self::Empty => None, - Self::Leaf(_, value) => value.data(), + Self::Leaf(_, value, _) => value.data(), Self::Extension(_, _) => None, - Self::Branch(_, value) => value.as_ref().and_then(|v| v.data()), - Self::NibbledBranch(_, _, value) => value.as_ref().and_then(|v| v.data()), + Self::Branch(_, value, _) => value.as_ref().and_then(|v| v.data()), + Self::NibbledBranch(_, _, value, _) => value.as_ref().and_then(|v| v.data()), Self::Value(data, _) => Some(data), } } @@ -391,10 +399,10 @@ impl NodeOwned { /// Returns the partial key of this node. pub fn partial_key(&self) -> Option<&NibbleVec> { match self { - Self::Branch(_, _) | Self::Value(_, _) | Self::Empty => None, + Self::Branch(_, _, _) | Self::Value(_, _) | Self::Empty => None, Self::Extension(partial, _) | - Self::Leaf(partial, _) | - Self::NibbledBranch(partial, _, _) => Some(partial), + Self::Leaf(partial, _, _) | + Self::NibbledBranch(partial, _, _, _) => Some(partial), } } @@ -419,7 +427,7 @@ impl NodeOwned { // to add the size of any dynamically allocated data. let dynamic_size = match self { Self::Empty => 0, - Self::Leaf(nibbles, value) => + Self::Leaf(nibbles, value, _) => nibbles.inner().len() + value.data().map_or(0, |b| b.len()), Self::Value(bytes, _) => bytes.len(), Self::Extension(nibbles, child) => { @@ -427,10 +435,10 @@ impl NodeOwned { // `self_size`. nibbles.inner().len() + child.as_inline().map_or(0, |n| n.size_in_bytes()) }, - Self::Branch(childs, value) => + Self::Branch(childs, value, _) => childs_size(childs.iter()) + value.as_ref().and_then(|v| v.data()).map_or(0, |b| b.len()), - Self::NibbledBranch(nibbles, childs, value) => + Self::NibbledBranch(nibbles, childs, value, _) => nibbles.inner().len() + childs_size(childs.iter()) + value.as_ref().and_then(|v| v.data()).map_or(0, |b| b.len()), @@ -517,6 +525,24 @@ impl ValuePlan { } } + // TODO is build still used + /// Build a value slice by decoding a byte slice according to the plan. + pub fn build_leaf<'a, 'b, L: Location>( + &'a self, + data: &'b [u8], + location: &[L], + ) -> (Value<'b, L>, L) { + let (value, ix) = match self { + ValuePlan::Inline(range) => (Value::Inline(&data[range.clone()]), 0), + ValuePlan::Node(range) => ( + Value::Node(&data[range.clone()], location.first().copied().unwrap_or_default()), + 1, + ), + }; + + (value, location.get(ix).copied().unwrap_or_default()) + } + /// Check if the value is inline. pub fn is_inline(&self) -> bool { match self { @@ -559,40 +585,36 @@ impl NodePlan { /// Build a node by decoding a byte slice according to the node plan and attaching location /// dats. It is the responsibility of the caller to ensure that the node plan was created for /// the argument data, otherwise the call may decode incorrectly or panic. - pub fn build<'a, 'b, L: Copy + Default>( - &'a self, - data: &'b [u8], - locations: &[L], - ) -> Node<'b, L> { + pub fn build<'a, 'b, L: Location>(&'a self, data: &'b [u8], locations: &[L]) -> Node<'b, L> { match self { NodePlan::Empty => Node::Empty, - NodePlan::Leaf { partial, value } => Node::Leaf( - partial.build(data), - value.build(data, locations.first().copied().unwrap_or_default()), - ), + NodePlan::Leaf { partial, value } => { + let (value, attached) = value.build_leaf(data, locations); + Node::Leaf(partial.build(data), value, attached) + }, NodePlan::Extension { partial, child } => Node::Extension( partial.build(data), child.build(data, locations.first().copied().unwrap_or_default()), ), NodePlan::Branch { value, children } => { - let (value, child_slices) = + let (value, child_slices, attached) = Self::build_value_and_children(value.as_ref(), children, data, locations); - Node::Branch(child_slices, value) + Node::Branch(child_slices, value, attached) }, NodePlan::NibbledBranch { partial, value, children } => { - let (value, child_slices) = + let (value, child_slices, attached) = Self::build_value_and_children(value.as_ref(), children, data, locations); - Node::NibbledBranch(partial.build(data), child_slices, value) + Node::NibbledBranch(partial.build(data), child_slices, value, attached) }, } } - fn build_value_and_children<'a, 'b, L: Copy + Default>( + fn build_value_and_children<'a, 'b, L: Location>( value: Option<&'a ValuePlan>, children: &'a [Option; nibble_ops::NIBBLE_LENGTH], data: &'b [u8], locations: &[L], - ) -> (Option>, [Option>; nibble_ops::NIBBLE_LENGTH]) { + ) -> (Option>, [Option>; nibble_ops::NIBBLE_LENGTH], L) { let mut child_slices = [None; nibble_ops::NIBBLE_LENGTH]; let mut nc = 0; let value = if let Some(v) = value { @@ -617,10 +639,11 @@ impl NodePlan { child_slices[i] = Some(child.build(data, location)); } } - (value, child_slices) + let attached = locations.get(nc).copied().unwrap_or_default(); + (value, child_slices, attached) } - pub(crate) fn build_child<'a, 'b, L: Copy + Default>( + pub(crate) fn build_child<'a, 'b, L: Location>( value: Option<&'a ValuePlan>, children: &'a [Option; nibble_ops::NIBBLE_LENGTH], index: usize, @@ -693,7 +716,7 @@ impl NodePlan { /// Check if an extra location is defined, it can be attached state. /// This method is counting and should be call only when needed. - pub fn additional_ref_location(&self, locations: &[L]) -> Option { + pub fn additional_ref_location(&self, locations: &[L]) -> Option { let offset = if self.has_location_for_value() { 1 } else { 0 } + self.num_children_locations(); if locations.len() > offset { @@ -716,7 +739,7 @@ pub struct OwnedNode, L> { locations: Vec, } -impl, L: Default + Copy> OwnedNode { +impl, L: Location> OwnedNode { /// Construct an `OwnedNode` by decoding an owned data source according to some codec. pub fn new(data: D, locations: Vec) -> core::result::Result { let plan = C::decode_plan(data.borrow())?; diff --git a/subtrie/src/node_codec.rs b/subtrie/src/node_codec.rs index c4aa2635..296e2fdf 100644 --- a/subtrie/src/node_codec.rs +++ b/subtrie/src/node_codec.rs @@ -17,7 +17,7 @@ use crate::{ node::{Node, NodePlan, Value}, - ChildReference, MaybeDebug, + ChildReference, Location, MaybeDebug, }; use crate::rstd::{borrow::Borrow, hash, vec::Vec, Error}; @@ -59,7 +59,7 @@ pub trait NodeCodec: Sized { fn decode_plan(data: &[u8]) -> Result; /// Decode bytes to a `Node`. Returns `Self::E` on failure. - fn decode<'a, L: Copy + Default>( + fn decode<'a, L: Location>( data: &'a [u8], locations: &[L], ) -> Result, Self::Error> { diff --git a/subtrie/src/proof/generate.rs b/subtrie/src/proof/generate.rs index a874e0e2..08ee9b9c 100644 --- a/subtrie/src/proof/generate.rs +++ b/subtrie/src/proof/generate.rs @@ -23,8 +23,8 @@ use crate::{ nibble_ops::NIBBLE_LENGTH, node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, Value, ValuePlan}, recorder::Record, - CError, ChildReference, DBValue, NibbleSlice, NodeCodec, Recorder, Result as TrieResult, Trie, - TrieDBBuilder, TrieError, TrieHash, TrieLayout, + CError, ChildReference, DBValue, Location, NibbleSlice, NodeCodec, Recorder, + Result as TrieResult, Trie, TrieDBBuilder, TrieError, TrieHash, TrieLayout, }; struct StackEntry<'a, C: NodeCodec, L> { @@ -46,7 +46,7 @@ struct StackEntry<'a, C: NodeCodec, L> { _marker: PhantomData, } -impl<'a, C: NodeCodec, L: Copy + Default> StackEntry<'a, C, L> { +impl<'a, C: NodeCodec, L: Location> StackEntry<'a, C, L> { fn new( prefix: LeftNibbleSlice<'a>, node_data: Vec, @@ -393,7 +393,7 @@ fn resolve_value( /// Determine the next algorithmic step to take by matching the current key against the current top /// entry on the stack. -fn match_key_to_node<'a, C: NodeCodec, L: Copy + Default>( +fn match_key_to_node<'a, C: NodeCodec, L: Location>( node: &'a OwnedNode, L>, omit_value: &mut bool, child_index: &mut usize, @@ -405,7 +405,7 @@ fn match_key_to_node<'a, C: NodeCodec, L: Copy + Default>( let node = node.node(); Ok(match node { Node::Empty => Step::FoundValue(None), - Node::Leaf(partial, value) => { + Node::Leaf(partial, value, _) => { if key.contains(&partial, prefix_len) && key.len() == prefix_len + partial.len() { match value { Value::Inline(data) => { @@ -429,7 +429,7 @@ fn match_key_to_node<'a, C: NodeCodec, L: Copy + Default>( } else { Step::FoundValue(None) }, - Node::Branch(child_handles, value) => match_key_to_branch_node::( + Node::Branch(child_handles, value, _) => match_key_to_branch_node::( value.clone(), &child_handles, omit_value, @@ -440,7 +440,7 @@ fn match_key_to_node<'a, C: NodeCodec, L: Copy + Default>( NibbleSlice::new(&[]), recorded_nodes, )?, - Node::NibbledBranch(partial, child_handles, value) => match_key_to_branch_node::( + Node::NibbledBranch(partial, child_handles, value, _) => match_key_to_branch_node::( value.clone(), &child_handles, omit_value, @@ -454,7 +454,7 @@ fn match_key_to_node<'a, C: NodeCodec, L: Copy + Default>( }) } -fn match_key_to_branch_node<'a, 'b, C: NodeCodec, L: Copy + Default>( +fn match_key_to_branch_node<'a, 'b, C: NodeCodec, L: Location>( value: Option>, child_handles: &'b [Option>; NIBBLE_LENGTH], omit_value: &mut bool, @@ -513,7 +513,7 @@ fn match_key_to_branch_node<'a, 'b, C: NodeCodec, L: Copy + Default>( /// Unwind the stack until the given key is prefixed by the entry at the top of the stack. If the /// key is None, unwind the stack completely. As entries are popped from the stack, they are /// encoded into proof nodes and added to the finalized proof. -fn unwind_stack( +fn unwind_stack( stack: &mut Vec>, proof_nodes: &mut Vec>, maybe_key: Option<&LeftNibbleSlice>, diff --git a/subtrie/src/proof/verify.rs b/subtrie/src/proof/verify.rs index d0e0cde1..24f281d7 100644 --- a/subtrie/src/proof/verify.rs +++ b/subtrie/src/proof/verify.rs @@ -117,8 +117,8 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { }; let value = match &node { Node::Empty | Node::Extension(_, _) => None, - Node::Leaf(_, value) => Some(value.clone()), - Node::Branch(_, value) | Node::NibbledBranch(_, _, value) => value.clone(), + Node::Leaf(_, value, _) => Some(value.clone()), + Node::Branch(_, value, _) | Node::NibbledBranch(_, _, value, _) => value.clone(), }; Ok(StackEntry { node, @@ -145,7 +145,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { self.complete_children()?; Ok(match self.node { Node::Empty => L::Codec::empty_node().to_vec(), - Node::Leaf(partial, _) => { + Node::Leaf(partial, _, _) => { let value = self.value().expect( "value is assigned to Some in StackEntry::new; \ value is only ever reassigned in the ValueMatch::MatchesLeaf match \ @@ -158,8 +158,8 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { self.children[0].expect("the child must be completed since child_index is 1"); L::Codec::extension_node(partial.right_iter(), partial.len(), child) }, - Node::Branch(_, _) => L::Codec::branch_node(self.children.iter(), self.value()), - Node::NibbledBranch(partial, _, _) => L::Codec::branch_node_nibbled( + Node::Branch(_, _, _) => L::Codec::branch_node(self.children.iter(), self.value()), + Node::NibbledBranch(partial, _, _, _) => L::Codec::branch_node_nibbled( partial.right_iter(), partial.len(), self.children.iter(), @@ -182,7 +182,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { assert_eq!(self.child_index, 0); Self::make_child_entry(proof_iter, child, child_prefix) }, - Node::Branch(children, _) | Node::NibbledBranch(_, children, _) => { + Node::Branch(children, _, _) | Node::NibbledBranch(_, children, _, _) => { // because this is a branch assert!(child_prefix.len() > 0); let child_index = child_prefix @@ -210,7 +210,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { self.children[self.child_index] = Some(child_ref); self.child_index += 1; }, - Node::Branch(children, _) | Node::NibbledBranch(_, children, _) => { + Node::Branch(children, _, _) | Node::NibbledBranch(_, children, _, _) => { while self.child_index < NIBBLE_LENGTH { if let Some(child) = children[self.child_index] { let child_ref = child.try_into().map_err(Error::InvalidChildReference)?; @@ -327,7 +327,7 @@ fn match_key_to_node<'a>( ) -> ValueMatch<'a> { match node { Node::Empty => ValueMatch::NotFound, - Node::Leaf(partial, value) => { + Node::Leaf(partial, value, _) => { if key.contains(partial, prefix_len) && key.len() == prefix_len + partial.len() { match value { Value::Node(..) => ValueMatch::NotOmitted, @@ -348,9 +348,9 @@ fn match_key_to_node<'a>( } else { ValueMatch::NotFound }, - Node::Branch(children, value) => + Node::Branch(children, value, _) => match_key_to_branch_node(key, prefix_len, children, value.as_ref()), - Node::NibbledBranch(partial, children, value) => + Node::NibbledBranch(partial, children, value, _) => if key.contains(partial, prefix_len) { match_key_to_branch_node(key, prefix_len + partial.len(), children, value.as_ref()) } else { diff --git a/subtrie/src/trie_codec.rs b/subtrie/src/trie_codec.rs index c401f55c..cbe51785 100644 --- a/subtrie/src/trie_codec.rs +++ b/subtrie/src/trie_codec.rs @@ -31,8 +31,8 @@ use crate::{ node::{Node, NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, ValuePlan}, node_db::Prefix, rstd::{boxed::Box, convert::TryInto, marker::PhantomData, result, sync::Arc, vec, vec::Vec}, - CError, ChildReference, DBValue, NibbleVec, NodeCodec, Result, TrieDB, TrieDBRawIterator, - TrieError, TrieHash, TrieLayout, + CError, ChildReference, DBValue, Location, NibbleVec, NodeCodec, Result, TrieDB, + TrieDBRawIterator, TrieError, TrieHash, TrieLayout, }; const OMIT_VALUE_HASH: crate::node::Value<'static, ()> = crate::node::Value::Inline(&[]); @@ -55,7 +55,7 @@ struct EncoderStackEntry { _marker: PhantomData, } -impl EncoderStackEntry { +impl EncoderStackEntry { /// Given the prefix of the next child node, identify its index and advance `child_index` to /// that. For a given entry, this must be called sequentially only with strictly increasing /// child prefixes. Returns an error if the child prefix is not a child of this entry or if @@ -344,7 +344,7 @@ struct DecoderStackEntry<'a, C: NodeCodec, L> { _marker: PhantomData, } -impl<'a, C: NodeCodec, L: Copy + Default> DecoderStackEntry<'a, C, L> { +impl<'a, C: NodeCodec, L: Location> DecoderStackEntry<'a, C, L> { /// Advance the child index until either it exceeds the number of children or the child is /// marked as omitted. Omitted children are indicated by an empty inline reference. For each /// child that is passed over and not omitted, copy over the child reference from the node to @@ -367,7 +367,7 @@ impl<'a, C: NodeCodec, L: Copy + Default> DecoderStackEntry<'a, C, L> { } self.child_index += 1; }, - Node::Branch(children, _) | Node::NibbledBranch(_, children, _) => { + Node::Branch(children, _, _) | Node::NibbledBranch(_, children, _, _) => { while self.child_index < NIBBLE_LENGTH { match children[self.child_index] { Some(NodeHandle::Inline(data)) if data.is_empty() => return Ok(false), @@ -392,13 +392,13 @@ impl<'a, C: NodeCodec, L: Copy + Default> DecoderStackEntry<'a, C, L> { fn push_to_prefix(&self, prefix: &mut NibbleVec) { match self.node { Node::Empty => {}, - Node::Leaf(partial, _) | Node::Extension(partial, _) => { + Node::Leaf(partial, _, _) | Node::Extension(partial, _) => { prefix.append_partial(partial.right()); }, - Node::Branch(_, _) => { + Node::Branch(_, _, _) => { prefix.push(self.child_index as u8); }, - Node::NibbledBranch(partial, _, _) => { + Node::NibbledBranch(partial, _, _, _) => { prefix.append_partial(partial.right()); prefix.push(self.child_index as u8); }, @@ -410,13 +410,13 @@ impl<'a, C: NodeCodec, L: Copy + Default> DecoderStackEntry<'a, C, L> { fn pop_from_prefix(&self, prefix: &mut NibbleVec) { match self.node { Node::Empty => {}, - Node::Leaf(partial, _) | Node::Extension(partial, _) => { + Node::Leaf(partial, _, _) | Node::Extension(partial, _) => { prefix.drop_lasts(partial.len()); }, - Node::Branch(_, _) => { + Node::Branch(_, _, _) => { prefix.pop(); }, - Node::NibbledBranch(partial, _, _) => { + Node::NibbledBranch(partial, _, _, _) => { prefix.pop(); prefix.drop_lasts(partial.len()); }, @@ -431,18 +431,18 @@ impl<'a, C: NodeCodec, L: Copy + Default> DecoderStackEntry<'a, C, L> { let attached_hash = attached_hash.map(|h| crate::node::Value::Node(h, Default::default())); match self.node { Node::Empty => C::empty_node().to_vec(), - Node::Leaf(partial, value) => + Node::Leaf(partial, value, _) => C::leaf_node(partial.right_iter(), partial.len(), attached_hash.unwrap_or(value)), Node::Extension(partial, _) => C::extension_node( partial.right_iter(), partial.len(), self.children[0].expect("required by method precondition; qed"), ), - Node::Branch(_, value) => C::branch_node( + Node::Branch(_, value, _) => C::branch_node( self.children.into_iter(), if attached_hash.is_some() { attached_hash } else { value }, ), - Node::NibbledBranch(partial, _, value) => C::branch_node_nibbled( + Node::NibbledBranch(partial, _, value, _) => C::branch_node_nibbled( partial.right_iter(), partial.len(), self.children.iter(), diff --git a/subtrie/src/triedb.rs b/subtrie/src/triedb.rs index 9d649615..17d4feff 100644 --- a/subtrie/src/triedb.rs +++ b/subtrie/src/triedb.rs @@ -369,7 +369,7 @@ where false, ) { Ok((owned_node, _node_hash)) => match owned_node.node() { - Node::Leaf(slice, value) => { + Node::Leaf(slice, value, _) => { let mut disp = f.debug_struct("Node::Leaf"); if let Some(i) = self.index { disp.field("index", &i); @@ -395,7 +395,7 @@ where ); disp.finish() }, - Node::Branch(ref nodes, ref value) => { + Node::Branch(ref nodes, ref value, _) => { let nodes: Vec> = nodes .into_iter() .enumerate() @@ -416,7 +416,7 @@ where disp.field("nodes", &nodes).field("value", &value); disp.finish() }, - Node::NibbledBranch(slice, nodes, value) => { + Node::NibbledBranch(slice, nodes, value, _) => { let nodes: Vec> = nodes .iter() .enumerate() diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index 4e696d0b..7dc53870 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -23,8 +23,8 @@ use crate::{ }, node_codec::NodeCodec, rstd::{boxed::Box, convert::TryFrom, mem, ops::Index, result, vec::Vec, VecDeque}, - Bytes, CError, DBValue, Result, TrieAccess, TrieCache, TrieError, TrieHash, TrieLayout, - TrieRecorder, + Bytes, CError, DBValue, Location, Result, TrieAccess, TrieCache, TrieError, TrieHash, + TrieLayout, TrieRecorder, }; use crate::node_db::{Hasher, NodeDB, Prefix}; @@ -83,13 +83,13 @@ impl Default for TreeRefChangeset { } impl TreeRefChangeset { - fn location(&self) -> Option { + fn location(&self) -> L::Location { match self { - TreeRefChangeset::None => None, - TreeRefChangeset::Existing(l) => Some(*l), + TreeRefChangeset::None => Default::default(), + TreeRefChangeset::Existing(l) => *l, TreeRefChangeset::Changed(c) => match &**c { - Changeset::New(_) => None, - Changeset::Existing(e) => Some(e.location), + Changeset::New(_) => Default::default(), + Changeset::Existing(e) => e.location, }, } } @@ -391,10 +391,10 @@ impl Node { .map_err(|e| Box::new(TrieError::DecoderError(node_hash, e)))?; let node = match encoded_node { EncodedNode::Empty => Node::Empty, - EncodedNode::Leaf(k, v) => Node::Leaf(k.into(), v.into(), TreeRefChangeset::None), + EncodedNode::Leaf(k, v, a) => Node::Leaf(k.into(), v.into(), a.into_changes()), EncodedNode::Extension(key, cb) => Node::Extension(key.into(), Self::inline_or_hash(node_hash, cb, storage)?), - EncodedNode::Branch(encoded_children, val) => { + EncodedNode::Branch(encoded_children, val, a) => { let mut child = |i: usize| match encoded_children[i] { Some(child) => Self::inline_or_hash(node_hash, child, storage).map(Some), None => Ok(None), @@ -419,9 +419,9 @@ impl Node { child(15)?, ]); - Node::Branch(children, val.map(Into::into), TreeRefChangeset::None) + Node::Branch(children, val.map(Into::into), a.into_changes()) }, - EncodedNode::NibbledBranch(k, encoded_children, val) => { + EncodedNode::NibbledBranch(k, encoded_children, val, a) => { let mut child = |i: usize| match encoded_children[i] { Some(child) => Self::inline_or_hash(node_hash, child, storage).map(Some), None => Ok(None), @@ -446,7 +446,7 @@ impl Node { child(15)?, ]); - Node::NibbledBranch(k.into(), children, val.map(Into::into), TreeRefChangeset::None) + Node::NibbledBranch(k.into(), children, val.map(Into::into), a.into_changes()) }, }; Ok(node) @@ -459,10 +459,10 @@ impl Node { ) -> Self { match node_owned { NodeOwned::Empty => Node::Empty, - NodeOwned::Leaf(k, v) => Node::Leaf(k.into(), v.into(), TreeRefChangeset::None), + NodeOwned::Leaf(k, v, a) => Node::Leaf(k.into(), v.into(), a.into_changes()), NodeOwned::Extension(key, cb) => Node::Extension(key.into(), Self::inline_or_hash_owned(cb, storage)), - NodeOwned::Branch(encoded_children, val) => { + NodeOwned::Branch(encoded_children, val, a) => { let mut child = |i: usize| { encoded_children[i] .as_ref() @@ -488,9 +488,9 @@ impl Node { child(15), ]); - Node::Branch(children, val.as_ref().map(Into::into), TreeRefChangeset::None) + Node::Branch(children, val.as_ref().map(Into::into), a.into_changes()) }, - NodeOwned::NibbledBranch(k, encoded_children, val) => { + NodeOwned::NibbledBranch(k, encoded_children, val, a) => { let mut child = |i: usize| { encoded_children[i] .as_ref() @@ -520,7 +520,7 @@ impl Node { k.into(), children, val.as_ref().map(Into::into), - TreeRefChangeset::None, + a.into_changes(), ) }, NodeOwned::Value(_, _) => diff --git a/test/src/double_ended_iterator.rs b/test/src/double_ended_iterator.rs index 77fa4ce1..cc689d05 100644 --- a/test/src/double_ended_iterator.rs +++ b/test/src/double_ended_iterator.rs @@ -41,7 +41,7 @@ fn node_double_ended_iterator>() { Some(Ok((prefix, Some(_), node))) => { assert_eq!(prefix, nibble_vec(hex!("02"), 2)); match node.node() { - Node::Leaf(partial, _) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), + Node::Leaf(partial, ..) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), _ => panic!("unexpected node"), } }, @@ -52,7 +52,7 @@ fn node_double_ended_iterator>() { Some(Ok((prefix, None, node))) => { assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); match node.node() { - Node::Leaf(partial, _) => + Node::Leaf(partial, ..) => assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)), _ => panic!("unexpected node"), } @@ -64,7 +64,7 @@ fn node_double_ended_iterator>() { Some(Ok((prefix, None, node))) => { assert_eq!(prefix, nibble_vec(hex!("01"), 2)); match node.node() { - Node::Branch(_, _) => {}, + Node::Branch(..) => {}, _ => panic!("unexpected node"), } }, @@ -75,7 +75,7 @@ fn node_double_ended_iterator>() { Some(Ok((prefix, Some(_), node))) => { assert_eq!(prefix, nibble_vec(hex!("00"), 1)); match node.node() { - Node::Branch(_, _) => {}, + Node::Branch(..) => {}, _ => panic!("unexpected node"), } }, @@ -103,7 +103,7 @@ fn node_double_ended_iterator>() { Some(Ok((prefix, Some(_), node))) => { assert_eq!(prefix, nibble_vec(hex!("02"), 2)); match node.node() { - Node::Leaf(partial, _) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), + Node::Leaf(partial, ..) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), _ => panic!("unexpected node"), } }, @@ -117,7 +117,7 @@ fn node_double_ended_iterator>() { } assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); match node.node() { - Node::Leaf(partial, _) => + Node::Leaf(partial, ..) => assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)), _ => panic!("unexpected node"), } @@ -132,7 +132,7 @@ fn node_double_ended_iterator>() { } assert_eq!(prefix, nibble_vec(hex!("01"), 2)); match node.node() { - Node::NibbledBranch(partial, _, _) => + Node::NibbledBranch(partial, ..) => assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)), _ => panic!("unexpected node"), } @@ -144,7 +144,7 @@ fn node_double_ended_iterator>() { Some(Ok((prefix, Some(_), node))) => { assert_eq!(prefix, nibble_vec(hex!(""), 0)); match node.node() { - Node::NibbledBranch(partial, _, _) => + Node::NibbledBranch(partial, ..) => assert_eq!(partial, NibbleSlice::new_offset(&hex!("00")[..], 1)), _ => panic!("unexpected node"), } @@ -301,7 +301,7 @@ fn prefix_back_works_internal>() { Some(Ok((prefix, None, node))) => { assert_eq!(prefix, nibble_vec(hex!("0123"), 4)); match node.node() { - Node::Leaf(partial, _) => { + Node::Leaf(partial, ..) => { assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)) }, _ => panic!("unexpected node"), @@ -317,7 +317,7 @@ fn prefix_back_works_internal>() { } assert_eq!(prefix, nibble_vec(hex!("0123"), 4)); match node.node() { - Node::Leaf(partial, _) => { + Node::Leaf(partial, ..) => { assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)) }, _ => panic!("unexpected node"), @@ -334,7 +334,7 @@ fn prefix_back_works_internal>() { } assert_eq!(prefix, nibble_vec(hex!("0122"), 4)); match node.node() { - Node::Leaf(partial, _) => { + Node::Leaf(partial, ..) => { assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)) }, _ => panic!("unexpected node"), @@ -350,9 +350,9 @@ fn prefix_back_works_internal>() { } assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); match node.node() { - Node::NibbledBranch(partial, _, _) => + Node::NibbledBranch(partial, ..) => assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)), - Node::Branch(_, _) => {}, + Node::Branch(..) => {}, _ => panic!("unexpected node"), } }, @@ -366,9 +366,9 @@ fn prefix_back_works_internal>() { } assert_eq!(prefix, nibble_vec(hex!("01"), 2)); match node.node() { - Node::NibbledBranch(partial, _, _) => + Node::NibbledBranch(partial, ..) => assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)), - Node::Branch(_, _) => {}, + Node::Branch(..) => {}, _ => panic!("unexpected node"), } }, diff --git a/test/src/iterator.rs b/test/src/iterator.rs index 4225e5b0..08da07cb 100644 --- a/test/src/iterator.rs +++ b/test/src/iterator.rs @@ -76,7 +76,7 @@ fn iterator_works_internal>() { Some(Ok((prefix, Some(_), node))) => { assert_eq!(prefix, nibble_vec(hex!("00"), 1)); match node.node() { - Node::Branch(_, _) => {}, + Node::Branch(..) => {}, _ => panic!("unexpected node"), } }, @@ -87,7 +87,7 @@ fn iterator_works_internal>() { Some(Ok((prefix, None, node))) => { assert_eq!(prefix, nibble_vec(hex!("01"), 2)); match node.node() { - Node::Branch(_, _) => {}, + Node::Branch(..) => {}, _ => panic!("unexpected node"), } }, @@ -98,7 +98,7 @@ fn iterator_works_internal>() { Some(Ok((prefix, None, node))) => { assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); match node.node() { - Node::Leaf(partial, _) => + Node::Leaf(partial, ..) => assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)), _ => panic!("unexpected node"), } @@ -110,7 +110,7 @@ fn iterator_works_internal>() { Some(Ok((prefix, Some(_), node))) => { assert_eq!(prefix, nibble_vec(hex!("02"), 2)); match node.node() { - Node::Leaf(partial, _) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), + Node::Leaf(partial, ..) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), _ => panic!("unexpected node"), } }, @@ -125,7 +125,7 @@ fn iterator_works_internal>() { Some(Ok((prefix, Some(_), node))) => { assert_eq!(prefix, nibble_vec(hex!(""), 0)); match node.node() { - Node::NibbledBranch(partial, _, _) => + Node::NibbledBranch(partial, ..) => assert_eq!(partial, NibbleSlice::new_offset(&hex!("00")[..], 1)), _ => panic!("unexpected node"), } @@ -140,7 +140,7 @@ fn iterator_works_internal>() { } assert_eq!(prefix, nibble_vec(hex!("01"), 2)); match node.node() { - Node::NibbledBranch(partial, _, _) => + Node::NibbledBranch(partial, ..) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), _ => panic!("unexpected node"), } @@ -155,7 +155,7 @@ fn iterator_works_internal>() { } assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); match node.node() { - Node::Leaf(partial, _) => + Node::Leaf(partial, ..) => assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)), _ => panic!("unexpected node"), } @@ -168,7 +168,7 @@ fn iterator_works_internal>() { Some(Ok((prefix, Some(_), node))) => { assert_eq!(prefix, nibble_vec(hex!("02"), 2)); match node.node() { - Node::Leaf(partial, _) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), + Node::Leaf(partial, ..) => assert_eq!(partial, NibbleSlice::new(&hex!("")[..])), _ => panic!("unexpected node"), } }, @@ -283,7 +283,7 @@ where TrieIterator::seek(&mut iter, &hex!("02")[..]).unwrap(); match iter.next() { Some(Ok((_, Some(hash), node))) => match node.node() { - Node::Leaf(_, _) => hash, + Node::Leaf(..) => hash, _ => panic!("unexpected node"), }, _ => panic!("unexpected item"), @@ -323,7 +323,7 @@ where } match iter.next() { Some(Ok((_, _, node))) => match node.node() { - Node::Leaf(_, v) => + Node::Leaf(_, v, ..) => if !matches!(v, Value::Node(..)) { assert_eq!(v, Value::Inline(&vec![2; 32][..])); }, @@ -356,7 +356,7 @@ fn prefix_works_internal>() { Some(Ok((prefix, None, node))) => { assert_eq!(prefix, nibble_vec(hex!("01"), 2)); match node.node() { - Node::Branch(_, _) => {}, + Node::Branch(..) => {}, _ => panic!("unexpected node"), } }, @@ -370,7 +370,7 @@ fn prefix_works_internal>() { } assert_eq!(prefix, nibble_vec(hex!("01"), 2)); match node.node() { - Node::NibbledBranch(partial, _, _) => + Node::NibbledBranch(partial, ..) => assert_eq!(partial, NibbleSlice::new_offset(&hex!("")[..], 0)), _ => panic!("unexpected node"), } @@ -386,7 +386,7 @@ fn prefix_works_internal>() { } assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); match node.node() { - Node::Leaf(partial, _) => { + Node::Leaf(partial, ..) => { assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)) }, _ => panic!("unexpected node"), From 52c9f524e277c66b149a413e7b8d63c67a8d7b42 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 15 Apr 2024 19:35:40 +0200 Subject: [PATCH 75/90] rem unuse prefix --- subtrie/src/mem_tree_db.rs | 1 - subtrie/src/triedbmut.rs | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/subtrie/src/mem_tree_db.rs b/subtrie/src/mem_tree_db.rs index b7c17f0e..51f09b73 100644 --- a/subtrie/src/mem_tree_db.rs +++ b/subtrie/src/mem_tree_db.rs @@ -265,7 +265,6 @@ mod tests { let existing_node = Changeset::Existing(ExistingChangesetNode { hash: hash(1), location: Some(new_location), - prefix: Default::default(), }); let existing_location = db.apply(&existing_node); diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index 7dc53870..1d802d17 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -825,8 +825,7 @@ pub struct NewChangesetNode { #[derive(Debug)] pub struct ExistingChangesetNode { - pub hash: H, // TODO is it used? - pub prefix: OwnedPrefix, // TODO is it used? + pub hash: H, // TODO take lot of mem when only use for root pub location: DL, } @@ -920,7 +919,6 @@ impl Changeset { pub fn unchanged(root: H) -> Self { Changeset::Existing(ExistingChangesetNode { hash: root, - prefix: (BackingByteVec::new(), None), location: Default::default(), }) } @@ -2109,7 +2107,6 @@ where debug_assert!(removed.is_empty()); return Changeset::Existing(ExistingChangesetNode { hash, - prefix: Default::default(), location, }); }, // no changes necessary. @@ -2177,7 +2174,6 @@ where debug_assert!(removed.is_empty()); Changeset::Existing(ExistingChangesetNode { hash, - prefix: Default::default(), location, }) }, @@ -2213,7 +2209,6 @@ where NodeHandle::Hash(hash, location) => { children.push(Changeset::Existing(ExistingChangesetNode { hash, - prefix: prefix.as_owned_prefix(), location, })); ChildReference::Hash(hash, location) @@ -2223,7 +2218,6 @@ where Stored::Cached(_, hash, location) => { children.push(Changeset::Existing(ExistingChangesetNode { hash, - prefix: prefix.as_owned_prefix(), location, })); ChildReference::Hash(hash, location) From 8a63867630074ca1dc2e5dc73c510880df6db948 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 15 Apr 2024 20:05:07 +0200 Subject: [PATCH 76/90] direction TODO --- subtrie/src/triedbmut.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index 1d802d17..0b965d4f 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -830,21 +830,24 @@ pub struct ExistingChangesetNode { } #[derive(Debug)] -pub enum Changeset { - New(NewChangesetNode), - Existing(ExistingChangesetNode), +pub struct Changeset { + pub old_root: H, + pub change: Changenode, +} + +#[derive(Debug)] +pub enum Changenode { + New(Box>), + Existing(DL), } impl Changeset { pub fn hash(&self) -> &H { - match self { - Changeset::New(node) => &node.hash, - Changeset::Existing(node) => &node.hash, - } + &self.old_root } } -impl Changeset { +impl Changenode { /// In case the underlying db do not /// do empty node optimization, it can /// make sense to insert the empty node. @@ -858,6 +861,7 @@ impl Changeset { }) } } + pub fn prefix_prefix(ks: &[u8], prefix: Prefix) -> (Vec, Option) { let mut result = Vec::with_capacity(ks.len() + prefix.0.len()); result.extend_from_slice(ks); @@ -872,7 +876,7 @@ impl Changeset { MH: Hasher + Send + Sync, { fn apply_node<'a, H, DL, MH, K>( - node: &'a Changeset, + node: &'a Changenode, mem_db: &mut MemoryDB, mut ks: Option<&'a [u8]>, ) where From 95c5917cf8ebde9945d7f9fd53bbaf3e52a03aaa Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 16 Apr 2024 14:05:05 +0200 Subject: [PATCH 77/90] refact payload --- reference-trie/src/lib.rs | 6 +- subtrie/src/bench.rs | 2 +- subtrie/src/lib.rs | 8 +- subtrie/src/mem_tree_db.rs | 95 +++++++++---------- subtrie/src/triedbmut.rs | 181 ++++++++++++++++++------------------- test/src/iter_build.rs | 3 +- test/src/recorder.rs | 2 +- test/src/triedbmut.rs | 19 ++-- 8 files changed, 153 insertions(+), 163 deletions(-) diff --git a/reference-trie/src/lib.rs b/reference-trie/src/lib.rs index 691cfe44..fa5abe1d 100644 --- a/reference-trie/src/lib.rs +++ b/reference-trie/src/lib.rs @@ -999,7 +999,7 @@ pub fn compare_root::new(&mut mem_db2); @@ -1120,7 +1120,7 @@ where let mut memdb = MemoryDB::::default(); let mut root = { let t = TrieDBMutBuilder::::new(&memdb).build(); - *t.commit().hash() + t.commit().root_hash() }; while a < data.len() { // new triemut every 3 element diff --git a/subtrie/src/bench.rs b/subtrie/src/bench.rs index 870b945e..3372f0b2 100644 --- a/subtrie/src/bench.rs +++ b/subtrie/src/bench.rs @@ -81,7 +81,7 @@ fn benchmark( } t.commit() }; - let root = commit.hash(); + let root = commit.root_hash(); commit.apply_to(&mut memdb); b.iter(&mut || { let t = TrieDBBuilder::::new(&memdb, &root).build(); diff --git a/subtrie/src/lib.rs b/subtrie/src/lib.rs index d97543bd..40f49c3b 100644 --- a/subtrie/src/lib.rs +++ b/subtrie/src/lib.rs @@ -79,8 +79,8 @@ pub use self::{ recorder::Recorder, triedb::{TrieDB, TrieDBBuilder, TrieDBIterator, TrieDBKeyIterator}, triedbmut::{ - Changeset, ChildReference, ExistingChangesetNode, NewChangesetNode, OwnedPrefix, - TreeRefChangeset, TrieDBMut, TrieDBMutBuilder, Value, + Changenode, Changeset, ChildReference, ExistingChangesetNode, NewChangesetNode, + OwnedPrefix, TreeRefChangeset, TrieDBMut, TrieDBMutBuilder, Value, }, }; use crate::node_db::Hasher; @@ -381,9 +381,9 @@ pub trait TrieLayout { pub trait Location: Copy + Default + Eq + PartialEq + MaybeDebug { fn into_changes>(self) -> TreeRefChangeset { if self == Self::default() { - TreeRefChangeset::None + None } else { - TreeRefChangeset::Existing(self) + Some(Changenode::Existing(self)) } } } diff --git a/subtrie/src/mem_tree_db.rs b/subtrie/src/mem_tree_db.rs index 51f09b73..c0bdf435 100644 --- a/subtrie/src/mem_tree_db.rs +++ b/subtrie/src/mem_tree_db.rs @@ -18,7 +18,7 @@ use std::collections::HashMap; use crate::{ node_db::{Hasher, NodeDB, NodeDBMut, Prefix}, - Changeset, NewChangesetNode, + Changenode, Changeset, }; /// Node location which is just an index into the `nodes` vector. @@ -135,10 +135,14 @@ where self.roots.is_empty() } - fn apply(&mut self, c: &Changeset) -> usize { - match c { - Changeset::Existing(e) => { - let location = e.location.unwrap_or_else(|| *self.roots.get(&e.hash).unwrap()); + fn apply(&mut self, c: &Changenode, is_root: Option) -> usize { + match &c { + Changenode::Existing(e) => { + let location = if e == &Location::default() { + *self.roots.get(&is_root.unwrap()).unwrap() + } else { + e.unwrap() + }; let entry = self.nodes.get_mut(location).unwrap(); match entry { NodeEntry::Live { rc, .. } => { @@ -150,8 +154,8 @@ where }; location }, - Changeset::New(n) => { - let children = n.children.iter().map(|c| self.apply(c)).collect(); + Changenode::New(n) => { + let children = n.children.iter().map(|c| self.apply(c, None)).collect(); self.nodes.push(NodeEntry::Live { key: n.hash, data: n.data.clone(), @@ -166,16 +170,12 @@ where pub fn apply_commit(&mut self, commit: Changeset) -> H::Out { let root = commit.root_hash(); if root != self.hashed_null_node { - let root = self.apply(&commit); - let key = commit.hash(); - self.roots.insert(*key, root); + let root = self.apply(&commit.change, Some(commit.old_root)); + let key = commit.root_hash(); + self.roots.insert(key, root); } - // In non test use, the root should be store before calling commit (known - // from tree where commit was build from). - if let Changeset::New(NewChangesetNode { removed_keys: Some((_, removed)), .. }) = &commit { - for (k, _) in removed { - self.remove_root(&k); - } + if let Changenode::New(_) = &commit.change { + self.remove_root(&commit.old_root); } root } @@ -240,7 +240,7 @@ mod tests { use crate::{ keccak_hasher::{KeccakHash, KeccakHasher}, node_db::{Hasher, NodeDB}, - Changeset, ExistingChangesetNode, NewChangesetNode, + Changenode, Changeset, NewChangesetNode, }; fn hash(i: u32) -> KeccakHash { @@ -252,21 +252,18 @@ mod tests { let mut db = MemTreeDB::::default(); // First, apply a new node - let new_node = Changeset::New(NewChangesetNode { + let new_node = Changenode::New(Box::new(NewChangesetNode { hash: hash(1), prefix: Default::default(), data: vec![1, 2, 3], children: vec![], removed_keys: None, - }); - let new_location = db.apply(&new_node); + })); + let new_location = db.apply(&new_node, None); // Then, apply an existing node that refers to the new node - let existing_node = Changeset::Existing(ExistingChangesetNode { - hash: hash(1), - location: Some(new_location), - }); - let existing_location = db.apply(&existing_node); + let existing_node = Changenode::Existing(Some(new_location)); + let existing_location = db.apply(&existing_node, None); assert_eq!(existing_location, new_location); } @@ -274,27 +271,30 @@ mod tests { #[test] fn test_apply_new_node() { let mut db = MemTreeDB::::default(); - let node = Changeset::New(NewChangesetNode { + let node = Changenode::New(Box::new(NewChangesetNode { hash: KeccakHash::default(), prefix: Default::default(), data: vec![1, 2, 3], children: vec![], removed_keys: None, - }); - let location = db.apply(&node); + })); + let location = db.apply(&node, None); assert_eq!(location, db.nodes.len() - 1); } #[test] fn test_apply_commit() { let mut db = MemTreeDB::::default(); - let commit = Changeset::New(NewChangesetNode { - hash: KeccakHash::default(), - prefix: Default::default(), - data: vec![1, 2, 3], - children: vec![], - removed_keys: None, - }); + let commit = Changeset { + old_root: hash(0), + change: Changenode::New(Box::new(NewChangesetNode { + hash: KeccakHash::default(), + prefix: Default::default(), + data: vec![1, 2, 3], + children: vec![], + removed_keys: None, + })), + }; db.apply_commit(commit); assert_eq!(db.roots.len(), 1); } @@ -304,29 +304,32 @@ mod tests { let mut db = MemTreeDB::::default(); // Create two child nodes - let child1 = Changeset::New(NewChangesetNode { + let child1 = Changenode::New(Box::new(NewChangesetNode { hash: hash(1), prefix: Default::default(), data: vec![1, 2, 3], children: vec![], removed_keys: None, - }); - let child2 = Changeset::New(NewChangesetNode { + })); + let child2 = Changenode::New(Box::new(NewChangesetNode { hash: hash(2), prefix: Default::default(), data: vec![4, 5, 6], children: vec![], removed_keys: None, - }); + })); // Create a root node that refers to the child nodes - let commit = Changeset::New(NewChangesetNode { - hash: hash(0), - prefix: Default::default(), - data: vec![7, 8, 9], - children: vec![child1, child2], - removed_keys: None, - }); + let commit = Changeset { + old_root: hash(999), + change: Changenode::New(Box::new(NewChangesetNode { + hash: hash(0), + prefix: Default::default(), + data: vec![7, 8, 9], + children: vec![child1, child2], + removed_keys: None, + })), + }; db.apply_commit(commit); diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index 0b965d4f..cc13c881 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -69,20 +69,11 @@ fn empty_children() -> Box<[Option>; nibble_ops::NIBBLE_L ]) } -/// Attached db level non merklized data. -pub enum TreeRefChangeset { - None, - Existing(::Location), - Changed(Box, ::Location>>), -} - -impl Default for TreeRefChangeset { - fn default() -> Self { - TreeRefChangeset::None - } -} - -impl TreeRefChangeset { +/// Alias on changenode over trie layout. +/// TODO rename (is change node) +pub type TreeRefChangeset = Option, ::Location>>; +/* +impl Changenode, ::Location> { fn location(&self) -> L::Location { match self { TreeRefChangeset::None => Default::default(), @@ -130,6 +121,7 @@ impl TreeRefChangeset { None } } + */ /// Type alias to indicate the nible covers a full key, /// therefore its left side is a full prefix. @@ -391,7 +383,7 @@ impl Node { .map_err(|e| Box::new(TrieError::DecoderError(node_hash, e)))?; let node = match encoded_node { EncodedNode::Empty => Node::Empty, - EncodedNode::Leaf(k, v, a) => Node::Leaf(k.into(), v.into(), a.into_changes()), + EncodedNode::Leaf(k, v, a) => Node::Leaf(k.into(), v.into(), a.into_changes::()), EncodedNode::Extension(key, cb) => Node::Extension(key.into(), Self::inline_or_hash(node_hash, cb, storage)?), EncodedNode::Branch(encoded_children, val, a) => { @@ -419,7 +411,7 @@ impl Node { child(15)?, ]); - Node::Branch(children, val.map(Into::into), a.into_changes()) + Node::Branch(children, val.map(Into::into), a.into_changes::()) }, EncodedNode::NibbledBranch(k, encoded_children, val, a) => { let mut child = |i: usize| match encoded_children[i] { @@ -446,7 +438,7 @@ impl Node { child(15)?, ]); - Node::NibbledBranch(k.into(), children, val.map(Into::into), a.into_changes()) + Node::NibbledBranch(k.into(), children, val.map(Into::into), a.into_changes::()) }, }; Ok(node) @@ -459,7 +451,7 @@ impl Node { ) -> Self { match node_owned { NodeOwned::Empty => Node::Empty, - NodeOwned::Leaf(k, v, a) => Node::Leaf(k.into(), v.into(), a.into_changes()), + NodeOwned::Leaf(k, v, a) => Node::Leaf(k.into(), v.into(), a.into_changes::()), NodeOwned::Extension(key, cb) => Node::Extension(key.into(), Self::inline_or_hash_owned(cb, storage)), NodeOwned::Branch(encoded_children, val, a) => { @@ -488,7 +480,7 @@ impl Node { child(15), ]); - Node::Branch(children, val.as_ref().map(Into::into), a.into_changes()) + Node::Branch(children, val.as_ref().map(Into::into), a.into_changes::()) }, NodeOwned::NibbledBranch(k, encoded_children, val, a) => { let mut child = |i: usize| { @@ -520,7 +512,7 @@ impl Node { k.into(), children, val.as_ref().map(Into::into), - a.into_changes(), + a.into_changes::(), ) }, NodeOwned::Value(_, _) => @@ -540,7 +532,7 @@ impl Node { ) -> ChildReference, L::Location>, { match self { - Node::Empty => (L::Codec::empty_node().to_vec(), Default::default()), + Node::Empty => (L::Codec::empty_node().to_vec(), None), Node::Leaf(partial, mut value, tree_ref) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let value = value.into_encoded::(Some(&pr), &mut child_cb); @@ -550,7 +542,7 @@ impl Node { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let it = pr.right_iter(); let c = child_cb(NodeToEncode::TrieNode(child), Some(&pr), None); - (L::Codec::extension_node(it, pr.len(), c), TreeRefChangeset::None) + (L::Codec::extension_node(it, pr.len(), c), None) }, Node::Branch(mut children, mut value, tree_ref) => { let value = value.as_mut().map(|v| v.into_encoded::(None, &mut child_cb)); @@ -817,7 +809,7 @@ pub struct NewChangesetNode { pub hash: H, pub prefix: OwnedPrefix, pub data: Vec, - pub children: Vec>, + pub children: Vec>, // Storing the key and removed nodes related // to this change set node (only needed for old trie). pub removed_keys: Option<(Option>, Vec<(H, OwnedPrefix)>)>, @@ -825,7 +817,7 @@ pub struct NewChangesetNode { #[derive(Debug)] pub struct ExistingChangesetNode { - pub hash: H, // TODO take lot of mem when only use for root + pub hash: H, // TODO take lot of mem when only use for root pub location: DL, } @@ -841,9 +833,17 @@ pub enum Changenode { Existing(DL), } -impl Changeset { - pub fn hash(&self) -> &H { - &self.old_root +impl Default for Changenode { + fn default() -> Self { + println!("d"); + Self::Existing(Default::default()) + } +} + +impl From
for Changenode { + fn from(l: DL) -> Self { + println!("d"); + Changenode::Existing(l) } } @@ -851,14 +851,15 @@ impl Changenode { /// In case the underlying db do not /// do empty node optimization, it can /// make sense to insert the empty node. + /// TODO used?? pub fn new_empty>() -> Self { - Self::New(NewChangesetNode { + Self::New(Box::new(NewChangesetNode { hash: C::hashed_null_node(), prefix: Default::default(), data: C::empty_node().to_vec(), children: Default::default(), removed_keys: None, - }) + })) } } @@ -884,7 +885,7 @@ impl Changeset { MH: Hasher + Send + Sync, { match node { - Changeset::New(node) => { + Changenode::New(node) => { if let Some((k, removed)) = node.removed_keys.as_ref() { for (hash, p) in removed.iter() { if let Some(k) = k { @@ -906,25 +907,23 @@ impl Changeset { mem_db.insert((node.prefix.0.as_slice(), node.prefix.1), &node.data); } }, - Changeset::Existing(_) => {}, + Changenode::Existing(_) => {}, } } - apply_node::(&self, mem_db, None); + apply_node::(&self.change, mem_db, None); self.root_hash() } pub fn root_hash(&self) -> H { - match &self { - Changeset::New(node) => node.hash, - Changeset::Existing(node) => node.hash, + match &self.change { + Changenode::New(node) => node.hash, + Changenode::Existing(_) => self.old_root, } } - pub fn unchanged(root: H) -> Self { - Changeset::Existing(ExistingChangesetNode { - hash: root, - location: Default::default(), - }) + pub fn unchanged(root: H, root_location: DL) -> Self { + println!("de"); + Changeset { old_root: root, change: Changenode::Existing(root_location) } } } @@ -1040,7 +1039,7 @@ where Action::Restore(node) => Some((Stored::New(node), false)), Action::Replace(node) => Some((Stored::New(node), true)), Action::Delete(tree_ref) => { - if !matches!(tree_ref, TreeRefChangeset::None) { + if tree_ref.is_some() { self.death_row_child.push(tree_ref); } None @@ -1054,7 +1053,7 @@ where }, Action::Delete(tree_ref) => { self.death_row.insert((hash, current_key.left_owned())); - if !matches!(tree_ref, TreeRefChangeset::None) { + if tree_ref.is_some() { self.death_row_child.push(tree_ref); } None @@ -1336,7 +1335,7 @@ where existing_key.to_stored_range(common), children, None, - TreeRefChangeset::None, + None, )) } } else { @@ -1425,13 +1424,13 @@ where children[idx] = Some(self.storage.alloc(Stored::New(new_leaf)).into()); if L::USE_EXTENSION { - Node::Branch(children, None, TreeRefChangeset::None) + Node::Branch(children, None, None) } else { Node::NibbledBranch( partial.to_stored_range(common), children, None, - TreeRefChangeset::None, + None, ) } }; @@ -1536,7 +1535,7 @@ where // continue inserting. let branch_action = self .insert_inspector( - Node::Branch(children, None, TreeRefChangeset::None), + Node::Branch(children, None, None), key, value, old_val, @@ -1626,7 +1625,7 @@ where ) -> Result, TrieHash, CError> { let partial = *key; Ok(match (node, partial.is_empty()) { - (Node::Empty, _) => Action::Delete(TreeRefChangeset::None), + (Node::Empty, _) => Action::Delete(None), (Node::Branch(c, None, _), true) => Action::Restore(Node::Branch(c, None, tree_ref)), (Node::NibbledBranch(n, c, None, _), true) => Action::Restore(Node::NibbledBranch(n, c, None, tree_ref)), @@ -1789,7 +1788,7 @@ where None => { // the whole branch got deleted. // that means that this extension is useless. - Action::Delete(TreeRefChangeset::None) + Action::Delete(None) }, } } else { @@ -1924,7 +1923,7 @@ where node }, }; - if !matches!(btreerefset, TreeRefChangeset::None) { + if btreerefset.is_some() { self.death_row_child.push(btreerefset); } match child_node { @@ -2109,10 +2108,7 @@ where let handle = match self.root_handle() { NodeHandle::Hash(hash, location) => { debug_assert!(removed.is_empty()); - return Changeset::Existing(ExistingChangesetNode { - hash, - location, - }); + return Changeset { old_root: hash, change: Changenode::Existing(location) }; }, // no changes necessary. NodeHandle::InMemory(h) => h, }; @@ -2128,13 +2124,13 @@ where NodeToEncode::Node(value) => { let value_hash = self.db.hash(value); self.cache_value(k.inner(), value, value_hash); - children.push(Changeset::New(NewChangesetNode { + children.push(Changenode::New(Box::new(NewChangesetNode { hash: value_hash, prefix: k.as_owned_prefix(), data: value.to_vec(), //TODO: avoid allocation children: Default::default(), removed_keys: None, - })); + }))); k.drop_lasts(mov); ChildReference::Hash(value_hash, Default::default()) @@ -2146,24 +2142,28 @@ where }, } }); - roottreerefset.into_changes().map(|c| { - children.push(*c); - }); + if let Some(c) = roottreerefset { + children.push(c); + } #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:?}", ToHex(&encoded_root[..])); + let old_root = self.root; self.root = self.db.hash(&encoded_root); self.cache_node(self.root); self.root_handle = NodeHandle::Hash(self.root, Default::default()); - Changeset::New(NewChangesetNode { - hash: self.root.clone(), - prefix: Default::default(), - data: encoded_root, - children, - removed_keys: Some((keyspace.map(|s| s.to_vec()), removed)), - }) + Changeset { + old_root, + change: Changenode::New(Box::new(NewChangesetNode { + hash: self.root.clone(), + prefix: Default::default(), + data: encoded_root, + children, + removed_keys: Some((keyspace.map(|s| s.to_vec()), removed)), + })), + } }, Stored::Cached(node, hash, location) => { // probably won't happen, but update the root and move on. @@ -2176,10 +2176,7 @@ where Default::default(), ))); debug_assert!(removed.is_empty()); - Changeset::Existing(ExistingChangesetNode { - hash, - location, - }) + Changeset { old_root: hash, change: Changenode::Existing(location) } }, } } @@ -2207,23 +2204,17 @@ where &mut self, handle: NodeHandle, L::Location>, prefix: &mut NibbleVec, - children: &mut Vec, L::Location>>, + children: &mut Vec, L::Location>>, ) -> ChildReference, L::Location> { match handle { NodeHandle::Hash(hash, location) => { - children.push(Changeset::Existing(ExistingChangesetNode { - hash, - location, - })); + children.push(Changenode::Existing(location)); ChildReference::Hash(hash, location) }, NodeHandle::InMemory(storage_handle) => { match self.storage.destroy(storage_handle) { Stored::Cached(_, hash, location) => { - children.push(Changeset::Existing(ExistingChangesetNode { - hash, - location, - })); + children.push(Changenode::Existing(location)); ChildReference::Hash(hash, location) }, Stored::New(node) => { @@ -2236,13 +2227,15 @@ where match node { NodeToEncode::Node(value) => { let value_hash = self.db.hash(value); - sub_children.push(Changeset::New(NewChangesetNode { - hash: value_hash, - prefix: prefix.as_owned_prefix(), - data: value.to_vec(), //TODO: avoid allocation - children: Default::default(), - removed_keys: None, - })); + sub_children.push(Changenode::New(Box::new( + NewChangesetNode { + hash: value_hash, + prefix: prefix.as_owned_prefix(), + data: value.to_vec(), //TODO: avoid allocation + children: Default::default(), + removed_keys: None, + }, + ))); self.cache_value(prefix.inner(), value, value_hash); @@ -2265,16 +2258,16 @@ where let result = if encoded.len() >= L::Hash::LENGTH { let hash = self.db.hash(&encoded); self.cache_node(hash); - child_set.into_changes().map(|c| { - sub_children.push(*c); - }); - children.push(Changeset::New(NewChangesetNode { + if let Some(c) = child_set { + sub_children.push(c); + }; + children.push(Changenode::New(Box::new(NewChangesetNode { hash, prefix: prefix.as_owned_prefix(), data: encoded, children: sub_children, removed_keys: None, - })); + }))); ChildReference::Hash(hash, Default::default()) } else { // it's a small value, so we cram it into a `TrieHash` @@ -2325,7 +2318,7 @@ where key: &[u8], value: &[u8], ) -> Result>, TrieHash, CError> { - self.insert_with_tree_ref(key, value, TreeRefChangeset::None) + self.insert_with_tree_ref(key, value, None) } pub fn insert_with_tree_ref( @@ -2335,7 +2328,7 @@ where tree_ref: TreeRefChangeset, ) -> Result>, TrieHash, CError> { // expect for the child changes to have a key. - debug_assert!(if let Some(set) = tree_ref.changes_ref() { + debug_assert!(if let Some(Changenode::New(set)) = tree_ref.as_ref() { set.removed_keys.is_some() } else { true @@ -2362,7 +2355,7 @@ where } pub fn remove(&mut self, key: &[u8]) -> Result>, TrieHash, CError> { - self.remove_with_tree_ref(key, TreeRefChangeset::None) + self.remove_with_tree_ref(key, None) } pub fn remove_with_tree_ref( @@ -2371,7 +2364,7 @@ where tree_ref: TreeRefChangeset, ) -> Result>, TrieHash, CError> { // expect for the child changes to have a key. - debug_assert!(if let Some(set) = tree_ref.changes_ref() { + debug_assert!(if let Some(Changenode::New(set)) = tree_ref.as_ref() { set.removed_keys.is_some() } else { true diff --git a/test/src/iter_build.rs b/test/src/iter_build.rs index 546f6ed9..11e9ca1e 100644 --- a/test/src/iter_build.rs +++ b/test/src/iter_build.rs @@ -52,7 +52,8 @@ fn test_iter(data: Vec<(Vec, Vec)>) { t.commit() }; changeset.apply_to(&mut db); - let t = TrieDBBuilder::::new(&db, changeset.hash()).build(); + let r = changeset.root_hash(); + let t = TrieDBBuilder::::new(&db, &r).build(); for (i, kv) in t.iter().unwrap().enumerate() { let (k, v) = kv.unwrap(); let key: &[u8] = &data[i].0; diff --git a/test/src/recorder.rs b/test/src/recorder.rs index baa39ad8..285b2169 100644 --- a/test/src/recorder.rs +++ b/test/src/recorder.rs @@ -34,7 +34,7 @@ fn trie_record() { x.insert(b"pirate", b"aargh!").unwrap(); x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); let commit = x.commit(); - let root = *commit.hash(); + let root = commit.root_hash(); commit.apply_to(&mut db); { diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index dcd93a5d..219c9f5d 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -25,9 +25,8 @@ use trie_db::{ memory_db::{HashKey, MemoryDB, PrefixedKey}, node_db::{Hasher, NodeDB, Prefix, EMPTY_PREFIX}, test_utils::*, - CachedValue, Changeset, DBValue, NodeCodec, Recorder, TreeRefChangeset, Trie, TrieCache, - TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, - TrieLayout, Value, + CachedValue, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, + TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, TrieLayout, Value, TreeRefChangeset, }; use crate::{TestCommit, TestDB}; @@ -863,7 +862,7 @@ fn attached_trie_internal>() { struct ATrie { root: TrieHash, data: BTreeMap, Vec>, - changeset: Option, T::Location>>>, + changeset: TreeRefChangeset, } // Running a typical attached trie scenario (childtrie on substrate): // different trie, attached trie root written all @@ -906,9 +905,7 @@ fn attached_trie_internal>() { let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let changeset = c.changeset.take().unwrap(); - memtrie - .insert_with_tree_ref(key, val, TreeRefChangeset::Changed(changeset)) - .unwrap(); + memtrie.insert_with_tree_ref(key, val, Some(changeset)).unwrap(); if a == 1 && b == 1 { break; } @@ -926,7 +923,7 @@ fn attached_trie_internal>() { let root = changeset.root_hash(); attached_tries.insert( attached_trie_root_key.clone(), - ATrie { root, data, changeset: Some(changeset.into()) }, + ATrie { root, data, changeset: Some(changeset.change) }, ); } } @@ -999,11 +996,7 @@ fn attached_trie_internal>() { }; let mut main_tree = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); main_tree - .insert_with_tree_ref( - root_key, - treeref_root_hash.as_ref(), - TreeRefChangeset::Changed(tree_ref_changeset.into()), - ) + .insert_with_tree_ref(root_key, treeref_root_hash.as_ref(), Some(tree_ref_changeset.change)) .unwrap(); let changeset = main_tree.commit(); main_trie.root = changeset.root_hash(); From 3d6b21f0df08727c7ea553f3b6e5821563e0845d Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 16 Apr 2024 14:45:24 +0200 Subject: [PATCH 78/90] insert only looks ok (except key collision but out of scope). --- test/src/triedbmut.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index 219c9f5d..04e5e13a 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -870,8 +870,8 @@ fn attached_trie_internal>() { // Direct copy if using ref counting and location in db. // Pruning. let mut seed = Default::default(); - let nb_attached_trie = 4; // TODO happen between 3 and 4 - let nb_attaching = 2; + let nb_attached_trie = 10; + let nb_attaching = 4; // let nb_attached_trie = 1; let support_location = DB::support_location(); let mut memdb = DB::default(); @@ -880,7 +880,7 @@ fn attached_trie_internal>() { ATrie { root: Default::default(), data: Default::default(), changeset: None }; let mut all_attached_tries: Vec, ATrie>> = Default::default(); let mut main_root_init = false; - for a in 0..nb_attaching { + for _ in 0..nb_attaching { let mut attached_tries: BTreeMap, ATrie> = Default::default(); for i in 0..nb_attached_trie + 1 { let x = StandardMap { @@ -900,16 +900,11 @@ fn attached_trie_internal>() { } else { TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build() }; - let mut b = 0; for (k, c) in attached_tries.iter_mut() { let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let changeset = c.changeset.take().unwrap(); memtrie.insert_with_tree_ref(key, val, Some(changeset)).unwrap(); - if a == 1 && b == 1 { - break; - } - b += 1; } let changeset = memtrie.commit(); let root = changeset.commit_to(&mut memdb); @@ -947,7 +942,6 @@ fn attached_trie_internal>() { if first > 0 { let (attached_trie_root, attached_trie_location) = attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { &memdb } else { From 36980967da09f2172e22ca7e7b4aa28b16b387a0 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 16 Apr 2024 16:33:30 +0200 Subject: [PATCH 79/90] rem print --- subtrie/src/triedbmut.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index cc13c881..cc6f3997 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -835,14 +835,12 @@ pub enum Changenode { impl Default for Changenode { fn default() -> Self { - println!("d"); Self::Existing(Default::default()) } } impl From
for Changenode { fn from(l: DL) -> Self { - println!("d"); Changenode::Existing(l) } } @@ -922,7 +920,7 @@ impl Changeset { } pub fn unchanged(root: H, root_location: DL) -> Self { - println!("de"); + // TODO consider Changenode::None Changeset { old_root: root, change: Changenode::Existing(root_location) } } } From f240f4a74cbad1751f66206ab86e06561a1f4209 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 16 Apr 2024 17:11:58 +0200 Subject: [PATCH 80/90] default to old_root on empty --- subtrie/src/mem_tree_db.rs | 6 +++--- subtrie/src/triedbmut.rs | 19 +++++++++++-------- test/src/triedbmut.rs | 10 +++++++--- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/subtrie/src/mem_tree_db.rs b/subtrie/src/mem_tree_db.rs index c0bdf435..926266f2 100644 --- a/subtrie/src/mem_tree_db.rs +++ b/subtrie/src/mem_tree_db.rs @@ -169,14 +169,14 @@ where pub fn apply_commit(&mut self, commit: Changeset) -> H::Out { let root = commit.root_hash(); + if let Changenode::New(_) = &commit.change { + self.remove_root(&commit.old_root); + } if root != self.hashed_null_node { let root = self.apply(&commit.change, Some(commit.old_root)); let key = commit.root_hash(); self.roots.insert(key, root); } - if let Changenode::New(_) = &commit.change { - self.remove_root(&commit.old_root); - } root } } diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index cc6f3997..de7b1e31 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -845,19 +845,22 @@ impl From
for Changenode { } } -impl Changenode { +impl Changeset { /// In case the underlying db do not /// do empty node optimization, it can /// make sense to insert the empty node. /// TODO used?? pub fn new_empty>() -> Self { - Self::New(Box::new(NewChangesetNode { - hash: C::hashed_null_node(), - prefix: Default::default(), - data: C::empty_node().to_vec(), - children: Default::default(), - removed_keys: None, - })) + Changeset { + old_root: Default::default(), + change: Changenode::New(Box::new(NewChangesetNode { + hash: C::hashed_null_node(), + prefix: Default::default(), + data: C::empty_node().to_vec(), + children: Default::default(), + removed_keys: None, + })), + } } } diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index 04e5e13a..995b52ae 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -25,8 +25,8 @@ use trie_db::{ memory_db::{HashKey, MemoryDB, PrefixedKey}, node_db::{Hasher, NodeDB, Prefix, EMPTY_PREFIX}, test_utils::*, - CachedValue, DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, - TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, TrieLayout, Value, TreeRefChangeset, + CachedValue, DBValue, NodeCodec, Recorder, TreeRefChangeset, Trie, TrieCache, TrieDBBuilder, + TrieDBMut, TrieDBMutBuilder, TrieDBNodeIterator, TrieError, TrieHash, TrieLayout, Value, }; use crate::{TestCommit, TestDB}; @@ -990,7 +990,11 @@ fn attached_trie_internal>() { }; let mut main_tree = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); main_tree - .insert_with_tree_ref(root_key, treeref_root_hash.as_ref(), Some(tree_ref_changeset.change)) + .insert_with_tree_ref( + root_key, + treeref_root_hash.as_ref(), + Some(tree_ref_changeset.change), + ) .unwrap(); let changeset = main_tree.commit(); main_trie.root = changeset.root_hash(); From 95b443b8c5d41f5ea79b0d4f594149571f9d9028 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 17 Apr 2024 09:11:43 +0200 Subject: [PATCH 81/90] fix --- subtrie/src/mem_tree_db.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/subtrie/src/mem_tree_db.rs b/subtrie/src/mem_tree_db.rs index 926266f2..a8b1a8ac 100644 --- a/subtrie/src/mem_tree_db.rs +++ b/subtrie/src/mem_tree_db.rs @@ -169,14 +169,16 @@ where pub fn apply_commit(&mut self, commit: Changeset) -> H::Out { let root = commit.root_hash(); - if let Changenode::New(_) = &commit.change { - self.remove_root(&commit.old_root); - } if root != self.hashed_null_node { let root = self.apply(&commit.change, Some(commit.old_root)); let key = commit.root_hash(); self.roots.insert(key, root); } + if let Changenode::New(_) = &commit.change { + if commit.old_root != root { + self.remove_root(&commit.old_root); + } + } root } } From 62cef480f0ee012f612130dbe6a2cc7eced3141f Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 17 Apr 2024 11:42:08 +0200 Subject: [PATCH 82/90] use death_row_child --- subtrie/src/mem_tree_db.rs | 2 + subtrie/src/triedbmut.rs | 42 ++++++++++++++----- test/src/triedbmut.rs | 86 ++++++++++++++++++-------------------- 3 files changed, 74 insertions(+), 56 deletions(-) diff --git a/subtrie/src/mem_tree_db.rs b/subtrie/src/mem_tree_db.rs index a8b1a8ac..2d804b20 100644 --- a/subtrie/src/mem_tree_db.rs +++ b/subtrie/src/mem_tree_db.rs @@ -289,6 +289,7 @@ mod tests { let mut db = MemTreeDB::::default(); let commit = Changeset { old_root: hash(0), + death_row_child: Vec::new(), change: Changenode::New(Box::new(NewChangesetNode { hash: KeccakHash::default(), prefix: Default::default(), @@ -324,6 +325,7 @@ mod tests { // Create a root node that refers to the child nodes let commit = Changeset { old_root: hash(999), + death_row_child: Vec::new(), change: Changenode::New(Box::new(NewChangesetNode { hash: hash(0), prefix: Default::default(), diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index de7b1e31..16e5baf8 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -812,6 +812,8 @@ pub struct NewChangesetNode { pub children: Vec>, // Storing the key and removed nodes related // to this change set node (only needed for old trie). + // Warning this is also here that the contextual keyspace + // is stored (so it should always be define). pub removed_keys: Option<(Option>, Vec<(H, OwnedPrefix)>)>, } @@ -824,6 +826,7 @@ pub struct ExistingChangesetNode { #[derive(Debug)] pub struct Changeset { pub old_root: H, + pub death_row_child: Vec>, pub change: Changenode, } @@ -853,6 +856,7 @@ impl Changeset { pub fn new_empty>() -> Self { Changeset { old_root: Default::default(), + death_row_child: Default::default(), change: Changenode::New(Box::new(NewChangesetNode { hash: C::hashed_null_node(), prefix: Default::default(), @@ -911,6 +915,9 @@ impl Changeset { Changenode::Existing(_) => {}, } } + for c in &self.death_row_child { + apply_node::(c, mem_db, None); + } apply_node::(&self.change, mem_db, None); self.root_hash() } @@ -924,7 +931,11 @@ impl Changeset { pub fn unchanged(root: H, root_location: DL) -> Self { // TODO consider Changenode::None - Changeset { old_root: root, change: Changenode::Existing(root_location) } + Changeset { + old_root: root, + death_row_child: Default::default(), + change: Changenode::Existing(root_location), + } } } @@ -964,7 +975,7 @@ where root: TrieHash, root_handle: NodeHandle, L::Location>, death_row: Set<(TrieHash, OwnedPrefix)>, - death_row_child: Vec>, + death_row_child: Vec, L::Location>>, /// Optional cache for speeding up the lookup of nodes. cache: Option<&'a mut dyn TrieCache>, /// Optional trie recorder for recording trie accesses. @@ -1040,8 +1051,8 @@ where Action::Restore(node) => Some((Stored::New(node), false)), Action::Replace(node) => Some((Stored::New(node), true)), Action::Delete(tree_ref) => { - if tree_ref.is_some() { - self.death_row_child.push(tree_ref); + if let Some(c) = tree_ref { + self.death_row_child.push(c); } None }, @@ -1054,8 +1065,8 @@ where }, Action::Delete(tree_ref) => { self.death_row.insert((hash, current_key.left_owned())); - if tree_ref.is_some() { - self.death_row_child.push(tree_ref); + if let Some(c) = tree_ref { + self.death_row_child.push(c); } None }, @@ -1750,6 +1761,7 @@ where let mut key_val = key.clone(); key_val.advance(existing_key.len()); self.replace_old_value(old_val, Some(value), key_val.left()); + // Note that ltreerefset is also drop here, same for update of attached. Action::Delete(tree_ref) } else { // leaf the node alone. @@ -1924,8 +1936,8 @@ where node }, }; - if btreerefset.is_some() { - self.death_row_child.push(btreerefset); + if let Some(c) = btreerefset { + self.death_row_child.push(c); } match child_node { Node::Leaf(sub_partial, value, ltreerefset) => { @@ -2088,6 +2100,7 @@ where /// stored date. /// `keyspace` only apply for hash base storage to avoid key collision /// between composed tree states. + /// See apply_to using removed key store keyspace to feed memory db with prefix. pub fn commit_with_keyspace(self, keyspace: &[u8]) -> Changeset, L::Location> { self.commit_inner(Some(keyspace)) } @@ -2109,7 +2122,11 @@ where let handle = match self.root_handle() { NodeHandle::Hash(hash, location) => { debug_assert!(removed.is_empty()); - return Changeset { old_root: hash, change: Changenode::Existing(location) }; + return Changeset { + old_root: hash, + death_row_child: self.death_row_child, + change: Changenode::Existing(location), + }; }, // no changes necessary. NodeHandle::InMemory(h) => h, }; @@ -2157,6 +2174,7 @@ where self.root_handle = NodeHandle::Hash(self.root, Default::default()); Changeset { old_root, + death_row_child: self.death_row_child, change: Changenode::New(Box::new(NewChangesetNode { hash: self.root.clone(), prefix: Default::default(), @@ -2177,7 +2195,11 @@ where Default::default(), ))); debug_assert!(removed.is_empty()); - Changeset { old_root: hash, change: Changenode::Existing(location) } + Changeset { + old_root: hash, + death_row_child: self.death_row_child, + change: Changenode::Existing(location), + } }, } } diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index 995b52ae..99b4a8a1 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -1024,54 +1024,48 @@ fn attached_trie_internal>() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } - //-- + // let root_key = root_key.clone(); + // attached_tries.remove(&root_key); + //------- + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); + let tree_ref_changeset = { + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( + child_memdb, + a_attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + attached_trie.remove(b"make_sure_it_changes").unwrap(); + let mut first = true; + for (k, _v) in a_attached_trie.data.iter() { + if !first { + attached_trie.remove(k).unwrap(); + } else { + first = false; + } + } + let changeset = attached_trie.commit_with_keyspace(root_key); + //let new_root = changeset.root_hash(); + //assert!(new_root != empty_); + changeset + }; + + let mut main_tree = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); + main_tree + .remove_with_tree_ref(root_key, Some(tree_ref_changeset.change)) + .unwrap(); + let changeset = main_tree.commit(); + main_trie.root = changeset.root_hash(); + changeset.commit_to(&mut memdb); let root_key = root_key.clone(); attached_tries.remove(&root_key); - /* - //------- - let (a_attached_trie_root, attached_trie_location) = - attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); - let tree_ref_changeset = { - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { - &memdb - } else { - keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); - &keyspaced_memdb - }; - let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( - child_memdb, - a_attached_trie_root, - attached_trie_location.unwrap_or_default(), - ) - .build(); - attached_trie.remove(b"make_sure_it_changes").unwrap(); - let mut first = true; - for (k, _v) in a_attached_trie.data.iter() { - if !first { - attached_trie.remove(k).unwrap(); - } else { - first = false; - } - } - let changeset = attached_trie.commit_with_keyspace(root_key); - //let new_root = changeset.root_hash(); - //assert!(new_root != empty_); - changeset - }; - - let mut main_tree = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); - main_tree - .remove_with_tree_ref( - root_key, - Some(tree_ref_changeset.into()), - ) - .unwrap(); - let changeset = main_tree.commit(); - main_trie.root = changeset.root_hash(); - changeset.commit_to(&mut memdb); - let root_key = root_key.clone(); - attached_tries.remove(&root_key); - */ } } From a6916c0ca5a29902baaf00001993413765dd81b0 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 19 Apr 2024 18:38:39 +0200 Subject: [PATCH 83/90] fix test --- subtrie/src/triedbmut.rs | 16 ++++----- test/src/triedbmut.rs | 71 +++++++++++++++++++++------------------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index 16e5baf8..9aa75b8e 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -1197,7 +1197,7 @@ where Ok((self.storage.alloc(new_stored), changed)) } - fn replace_old_value( + fn return_old_value( &mut self, old_value: &mut Option>, stored_value: Option>, @@ -1247,7 +1247,7 @@ where let unchanged = stored_value == value; let branch = Node::Branch(children, value, tree_ref); - self.replace_old_value(old_val, stored_value, key.left()); + self.return_old_value(old_val, stored_value, key.left()); if unchanged { InsertAction::Restore(branch) @@ -1300,7 +1300,7 @@ where let mut key_val = key.clone(); key_val.advance(existing_key.len()); - self.replace_old_value(old_val, stored_value, key_val.left()); + self.return_old_value(old_val, stored_value, key_val.left()); if unchanged { InsertAction::Restore(branch) @@ -1402,7 +1402,7 @@ where let unchanged = stored_value == value; let mut key_val = key.clone(); key_val.advance(existing_key.len()); - self.replace_old_value(old_val, Some(stored_value), key_val.left()); + self.return_old_value(old_val, Some(stored_value), key_val.left()); if unchanged { // unchanged. restore InsertAction::Restore(Node::Leaf(encoded.clone(), value, l_tree_ref)) @@ -1642,12 +1642,12 @@ where (Node::NibbledBranch(n, c, None, _), true) => Action::Restore(Node::NibbledBranch(n, c, None, tree_ref)), (Node::Branch(children, val, _), true) => { - self.replace_old_value(old_val, val, key.left()); + self.return_old_value(old_val, val, key.left()); // always replace since we took the value out. Action::Replace(self.fix(Node::Branch(children, None, tree_ref), *key)?) }, (Node::NibbledBranch(n, children, val, _), true) => { - self.replace_old_value(old_val, val, key.left()); + self.return_old_value(old_val, val, key.left()); // always replace since we took the value out. Action::Replace(self.fix(Node::NibbledBranch(n, children, None, tree_ref), *key)?) }, @@ -1698,7 +1698,7 @@ where if let Some(value) = value { let mut key_val = key.clone(); key_val.advance(existing_length); - self.replace_old_value(old_val, Some(value), key_val.left()); + self.return_old_value(old_val, Some(value), key_val.left()); let f = self.fix(Node::NibbledBranch(encoded, children, None, tree_ref), *key); Action::Replace(f?) @@ -1760,7 +1760,7 @@ where // this is the node we were looking for. Let's delete it. let mut key_val = key.clone(); key_val.advance(existing_key.len()); - self.replace_old_value(old_val, Some(value), key_val.left()); + self.return_old_value(old_val, Some(value), key_val.left()); // Note that ltreerefset is also drop here, same for update of attached. Action::Delete(tree_ref) } else { diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index 99b4a8a1..173b92d9 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -870,19 +870,22 @@ fn attached_trie_internal>() { // Direct copy if using ref counting and location in db. // Pruning. let mut seed = Default::default(); - let nb_attached_trie = 10; - let nb_attaching = 4; + let nb_attached_trie = 200; // let nb_attached_trie = 1; + let nb_attaching = 200; +// let nb_attaching = 200; let support_location = DB::support_location(); let mut memdb = DB::default(); let mut keyspaced_memdb; let mut main_trie: ATrie = ATrie { root: Default::default(), data: Default::default(), changeset: None }; - let mut all_attached_tries: Vec, ATrie>> = Default::default(); + let mut all_attached_tries: BTreeMap, ATrie> = Default::default(); let mut main_root_init = false; for _ in 0..nb_attaching { let mut attached_tries: BTreeMap, ATrie> = Default::default(); - for i in 0..nb_attached_trie + 1 { + let mut work_key = None; + let mut i = 0; + loop { let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), min_key: 3, @@ -900,19 +903,25 @@ fn attached_trie_internal>() { } else { TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build() }; - for (k, c) in attached_tries.iter_mut() { + for (k, mut c) in attached_tries.into_iter() { + work_key = Some(k.clone()); let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let changeset = c.changeset.take().unwrap(); memtrie.insert_with_tree_ref(key, val, Some(changeset)).unwrap(); + all_attached_tries.insert(k, c); } let changeset = memtrie.commit(); let root = changeset.commit_to(&mut memdb); main_trie.root = root; main_trie.data = data; // TODO default (unused and unmaintained) + break; } else { let memtrie = populate_trie::(&memdb, &x); let attached_trie_root_key = data.iter().next().unwrap().0; + if all_attached_tries.contains_key(attached_trie_root_key) || attached_tries.contains_key(attached_trie_root_key) { + continue; + } println!("{:x?}", attached_trie_root_key); let changeset = memtrie.commit_with_keyspace(attached_trie_root_key); let root = changeset.root_hash(); @@ -920,9 +929,9 @@ fn attached_trie_internal>() { attached_trie_root_key.clone(), ATrie { root, data, changeset: Some(changeset.change) }, ); + i += 1; } } - all_attached_tries.push(attached_tries); // check data if !main_root_init { let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); @@ -935,36 +944,30 @@ fn attached_trie_internal>() { let trie = TrieDBBuilder::::new(&memdb, &main_trie.root).build(); println!("{:?}", trie); } - for attached_tries in all_attached_tries.iter() { - let mut first = 0; - for (root_key, attached_trie) in attached_tries { - first += 1; - if first > 0 { - let (attached_trie_root, attached_trie_location) = - attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { - &memdb - } else { - assert!(attached_trie_location.is_none()); - keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); - &keyspaced_memdb - }; - - let trie = TrieDBBuilder::::new_with_db_location( - child_memdb, - &attached_trie_root, - attached_trie_location.unwrap_or_default(), - ) - .build(); - for (k, v) in attached_trie.data.iter() { - assert_eq!(&trie.get(k).unwrap().unwrap(), v); - } - } + for (root_key, attached_trie) in all_attached_tries.iter() { + let (attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + assert!(attached_trie_location.is_none()); + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + + let trie = TrieDBBuilder::::new_with_db_location( + child_memdb, + &attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + for (k, v) in attached_trie.data.iter() { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } // Modifying an existing child trie. - let attached_tries = all_attached_tries.last_mut().unwrap(); - let (root_key, a_attached_trie) = attached_tries.iter().next().unwrap(); + let root_key = work_key.as_ref().unwrap(); + let a_attached_trie = all_attached_tries.get(root_key).unwrap(); let (a_attached_trie_root, attached_trie_location) = attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); let (tree_ref_changeset, treeref_root_hash) = { @@ -1065,7 +1068,7 @@ fn attached_trie_internal>() { main_trie.root = changeset.root_hash(); changeset.commit_to(&mut memdb); let root_key = root_key.clone(); - attached_tries.remove(&root_key); + all_attached_tries.remove(&root_key); } } From 2f13772e7d3453646701957c6baf166e0746b9ae Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 19 Apr 2024 18:49:50 +0200 Subject: [PATCH 84/90] removing half attached --- test/src/triedbmut.rs | 223 ++++++++++++++++++++++-------------------- 1 file changed, 116 insertions(+), 107 deletions(-) diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index 173b92d9..20400552 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -870,10 +870,11 @@ fn attached_trie_internal>() { // Direct copy if using ref counting and location in db. // Pruning. let mut seed = Default::default(); - let nb_attached_trie = 200; + let nb_attached_trie = 20; + let nb_removed_trie = 10; // let nb_attached_trie = 1; - let nb_attaching = 200; -// let nb_attaching = 200; + let nb_attaching = 10; + // let nb_attaching = 200; let support_location = DB::support_location(); let mut memdb = DB::default(); let mut keyspaced_memdb; @@ -883,7 +884,7 @@ fn attached_trie_internal>() { let mut main_root_init = false; for _ in 0..nb_attaching { let mut attached_tries: BTreeMap, ATrie> = Default::default(); - let mut work_key = None; + let mut remove_keys = Vec::new(); let mut i = 0; loop { let x = StandardMap { @@ -904,7 +905,9 @@ fn attached_trie_internal>() { TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build() }; for (k, mut c) in attached_tries.into_iter() { - work_key = Some(k.clone()); + if remove_keys.len() < nb_removed_trie { + remove_keys.push(k.clone()); + } let key: &[u8] = &k[..]; let val: &[u8] = c.root.as_ref(); let changeset = c.changeset.take().unwrap(); @@ -919,7 +922,9 @@ fn attached_trie_internal>() { } else { let memtrie = populate_trie::(&memdb, &x); let attached_trie_root_key = data.iter().next().unwrap().0; - if all_attached_tries.contains_key(attached_trie_root_key) || attached_tries.contains_key(attached_trie_root_key) { + if all_attached_tries.contains_key(attached_trie_root_key) || + attached_tries.contains_key(attached_trie_root_key) + { continue; } println!("{:x?}", attached_trie_root_key); @@ -965,110 +970,114 @@ fn attached_trie_internal>() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } } - // Modifying an existing child trie. - let root_key = work_key.as_ref().unwrap(); - let a_attached_trie = all_attached_tries.get(root_key).unwrap(); - let (a_attached_trie_root, attached_trie_location) = - attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); - let (tree_ref_changeset, treeref_root_hash) = { - assert_eq!(a_attached_trie_root, a_attached_trie.root); - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { - &memdb - } else { - keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); - &keyspaced_memdb - }; - let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( - child_memdb, - a_attached_trie_root, - attached_trie_location.unwrap_or_default(), - ) - .build(); - attached_trie.remove(a_attached_trie.data.iter().next().unwrap().0).unwrap(); - attached_trie.insert(b"make_sure_it_changes", b"value").unwrap(); - let changeset = attached_trie.commit_with_keyspace(root_key); - let new_root = changeset.root_hash(); - assert!(new_root != a_attached_trie_root); - (changeset, new_root) - }; - let mut main_tree = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); - main_tree - .insert_with_tree_ref( - root_key, - treeref_root_hash.as_ref(), - Some(tree_ref_changeset.change), - ) - .unwrap(); - let changeset = main_tree.commit(); - main_trie.root = changeset.root_hash(); - changeset.commit_to(&mut memdb); - // checking modification - let (a_attached_trie_root, attached_trie_location) = - attached_trie_root(&memdb, &main_trie.root, root_key).unwrap(); - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { - &memdb - } else { - keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); - &keyspaced_memdb - }; - let trie = TrieDBBuilder::::new_with_db_location( - child_memdb, - &a_attached_trie_root, - attached_trie_location.unwrap_or_default(), - ) - .build(); - trie.get(b"make_sure_it_changes").unwrap().unwrap(); - let mut first = true; - for (k, v) in a_attached_trie.data.iter() { - if first { - assert!(&trie.get(k).unwrap().is_none()); - first = false; - } else { - assert_eq!(&trie.get(k).unwrap().unwrap(), v); - } - } - // let root_key = root_key.clone(); - // attached_tries.remove(&root_key); - //------- - let (a_attached_trie_root, attached_trie_location) = - attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); - let tree_ref_changeset = { - let child_memdb: &dyn NodeDB<_, _, _> = if support_location { - &memdb - } else { - keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); - &keyspaced_memdb - }; - let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( - child_memdb, - a_attached_trie_root, - attached_trie_location.unwrap_or_default(), - ) - .build(); - attached_trie.remove(b"make_sure_it_changes").unwrap(); - let mut first = true; - for (k, _v) in a_attached_trie.data.iter() { - if !first { - attached_trie.remove(k).unwrap(); + for (i, root_key) in remove_keys.into_iter().enumerate() { + let a_attached_trie = all_attached_tries.get(&root_key).unwrap(); + if i == 0 { + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); + let (tree_ref_changeset, treeref_root_hash) = { + assert_eq!(a_attached_trie_root, a_attached_trie.root); + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( + child_memdb, + a_attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + attached_trie.remove(a_attached_trie.data.iter().next().unwrap().0).unwrap(); + attached_trie.insert(b"make_sure_it_changes", b"value").unwrap(); + let changeset = attached_trie.commit_with_keyspace(&root_key); + let new_root = changeset.root_hash(); + assert!(new_root != a_attached_trie_root); + (changeset, new_root) + }; + let mut main_tree = + TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); + main_tree + .insert_with_tree_ref( + &root_key, + treeref_root_hash.as_ref(), + Some(tree_ref_changeset.change), + ) + .unwrap(); + let changeset = main_tree.commit(); + main_trie.root = changeset.root_hash(); + changeset.commit_to(&mut memdb); + // checking modification + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb } else { - first = false; + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let trie = TrieDBBuilder::::new_with_db_location( + child_memdb, + &a_attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + trie.get(b"make_sure_it_changes").unwrap().unwrap(); + let mut first = true; + for (k, v) in a_attached_trie.data.iter() { + if first { + assert!(&trie.get(k).unwrap().is_none()); + first = false; + } else { + assert_eq!(&trie.get(k).unwrap().unwrap(), v); + } } } - let changeset = attached_trie.commit_with_keyspace(root_key); - //let new_root = changeset.root_hash(); - //assert!(new_root != empty_); - changeset - }; - - let mut main_tree = TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); - main_tree - .remove_with_tree_ref(root_key, Some(tree_ref_changeset.change)) - .unwrap(); - let changeset = main_tree.commit(); - main_trie.root = changeset.root_hash(); - changeset.commit_to(&mut memdb); - let root_key = root_key.clone(); - all_attached_tries.remove(&root_key); + // let root_key = root_key.clone(); + // attached_tries.remove(&root_key); + //------- + let (a_attached_trie_root, attached_trie_location) = + attached_trie_root(&memdb, &main_trie.root, &root_key).unwrap(); + let tree_ref_changeset = { + let child_memdb: &dyn NodeDB<_, _, _> = if support_location { + &memdb + } else { + keyspaced_memdb = KeySpacedDB::new(&memdb, &root_key[..]); + &keyspaced_memdb + }; + let mut attached_trie = TrieDBMutBuilder::::from_existing_with_db_location( + child_memdb, + a_attached_trie_root, + attached_trie_location.unwrap_or_default(), + ) + .build(); + attached_trie.remove(b"make_sure_it_changes").unwrap(); + let mut first = true; + for (k, _v) in a_attached_trie.data.iter() { + if !first { + attached_trie.remove(k).unwrap(); + } else { + first = false; + } + } + let changeset = attached_trie.commit_with_keyspace(&root_key); + //let new_root = changeset.root_hash(); + //assert!(new_root != empty_); + changeset + }; + + let mut main_tree = + TrieDBMutBuilder::::from_existing(&memdb, main_trie.root).build(); + main_tree + .remove_with_tree_ref(&root_key, Some(tree_ref_changeset.change)) + .unwrap(); + let changeset = main_tree.commit(); + main_trie.root = changeset.root_hash(); + changeset.commit_to(&mut memdb); + let root_key = root_key.clone(); + all_attached_tries.remove(&root_key); + } } } From 4bb871764d406f1ab6fbab1f4394dc394e27ab96 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 19 Apr 2024 19:07:39 +0200 Subject: [PATCH 85/90] check iter --- test/src/triedbmut.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index 20400552..4f398cb3 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -969,6 +969,10 @@ fn attached_trie_internal>() { for (k, v) in attached_trie.data.iter() { assert_eq!(&trie.get(k).unwrap().unwrap(), v); } + let mut key_iter = trie.key_iter().unwrap(); + for (k, _) in attached_trie.data.iter() { + assert_eq!(&key_iter.next().unwrap().unwrap(), k); + } } for (i, root_key) in remove_keys.into_iter().enumerate() { let a_attached_trie = all_attached_tries.get(&root_key).unwrap(); From 51e97f4a1d8085793a7a630fa506bcdf6723715a Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 19 Apr 2024 19:24:51 +0200 Subject: [PATCH 86/90] Found issue --- subtrie/src/iterator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subtrie/src/iterator.rs b/subtrie/src/iterator.rs index 8a25ece5..c9a994a1 100644 --- a/subtrie/src/iterator.rs +++ b/subtrie/src/iterator.rs @@ -155,7 +155,7 @@ impl TrieDBRawIterator { let (mut node, mut node_hash) = db.get_raw_or_lookup( >::default(), - NodeHandle::Hash(db.root().as_ref(), Default::default()), + NodeHandle::Hash(db.root().as_ref(), db.root_location()), EMPTY_PREFIX, true, )?; From ecc52d3ccb9261d4f39fe46da74b726bbf7a00ab Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 19 Apr 2024 19:29:24 +0200 Subject: [PATCH 87/90] test it --- test/src/triedbmut.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index 4f398cb3..b5bf2d3e 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -973,6 +973,17 @@ fn attached_trie_internal>() { for (k, _) in attached_trie.data.iter() { assert_eq!(&key_iter.next().unwrap().unwrap(), k); } + + let mut key_iter = trie.key_iter().unwrap(); + key_iter.seek(attached_trie.data.iter().next().unwrap().0).unwrap(); + let mut first = true; + for (k, _) in attached_trie.data.iter() { + if first { + first = false; + } + assert_eq!(&key_iter.next().unwrap().unwrap(), k); + } + } for (i, root_key) in remove_keys.into_iter().enumerate() { let a_attached_trie = all_attached_tries.get(&root_key).unwrap(); From ed7c5f02401125715f5f953d3af447bb374e5403 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 May 2024 10:07:27 +0200 Subject: [PATCH 88/90] export --- subtrie/src/lib.rs | 3 +-- test/src/triedbmut.rs | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/subtrie/src/lib.rs b/subtrie/src/lib.rs index 40f49c3b..84e9bf50 100644 --- a/subtrie/src/lib.rs +++ b/subtrie/src/lib.rs @@ -45,7 +45,6 @@ mod rstd { use self::rstd::{fmt, Error}; use self::rstd::{boxed::Box, vec::Vec}; -pub use iterator::TrieDBNodeDoubleEndedIterator; use node::NodeOwned; use node_db::MaybeDebug; @@ -86,7 +85,7 @@ pub use self::{ use crate::node_db::Hasher; pub use crate::{ iter_build::{trie_visit, ProcessEncodedNode, TrieBuilder, TrieRoot, TrieRootUnhashed}, - iterator::{TrieDBNodeIterator, TrieDBRawIterator}, + iterator::{TrieDBNodeDoubleEndedIterator, TrieDBNodeIterator, TrieDBRawIterator}, node_codec::{NodeCodec, Partial}, trie_codec::{decode_compact, decode_compact_from_iter, encode_compact}, }; diff --git a/test/src/triedbmut.rs b/test/src/triedbmut.rs index b5bf2d3e..70776ebb 100644 --- a/test/src/triedbmut.rs +++ b/test/src/triedbmut.rs @@ -983,7 +983,6 @@ fn attached_trie_internal>() { } assert_eq!(&key_iter.next().unwrap().unwrap(), k); } - } for (i, root_key) in remove_keys.into_iter().enumerate() { let a_attached_trie = all_attached_tries.get(&root_key).unwrap(); From 2c06f2ede1e981176ff0288048a7f7eda5d79d8e Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 May 2024 12:09:07 +0200 Subject: [PATCH 89/90] clean unused and clone on changeset --- subtrie/src/lib.rs | 2 +- subtrie/src/triedbmut.rs | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/subtrie/src/lib.rs b/subtrie/src/lib.rs index 84e9bf50..80e393d1 100644 --- a/subtrie/src/lib.rs +++ b/subtrie/src/lib.rs @@ -78,7 +78,7 @@ pub use self::{ recorder::Recorder, triedb::{TrieDB, TrieDBBuilder, TrieDBIterator, TrieDBKeyIterator}, triedbmut::{ - Changenode, Changeset, ChildReference, ExistingChangesetNode, NewChangesetNode, + Changenode, Changeset, ChildReference, NewChangesetNode, OwnedPrefix, TreeRefChangeset, TrieDBMut, TrieDBMutBuilder, Value, }, }; diff --git a/subtrie/src/triedbmut.rs b/subtrie/src/triedbmut.rs index 9aa75b8e..6b99f508 100644 --- a/subtrie/src/triedbmut.rs +++ b/subtrie/src/triedbmut.rs @@ -804,7 +804,7 @@ impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct NewChangesetNode { pub hash: H, pub prefix: OwnedPrefix, @@ -817,20 +817,14 @@ pub struct NewChangesetNode { pub removed_keys: Option<(Option>, Vec<(H, OwnedPrefix)>)>, } -#[derive(Debug)] -pub struct ExistingChangesetNode { - pub hash: H, // TODO take lot of mem when only use for root - pub location: DL, -} - -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Changeset { pub old_root: H, pub death_row_child: Vec>, pub change: Changenode, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum Changenode { New(Box>), Existing(DL), From 620d6761d13d82638638c7429e88955f94a1091c Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 May 2024 12:37:53 +0200 Subject: [PATCH 90/90] fmt --- subtrie/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subtrie/src/lib.rs b/subtrie/src/lib.rs index 80e393d1..8e3aad66 100644 --- a/subtrie/src/lib.rs +++ b/subtrie/src/lib.rs @@ -78,8 +78,8 @@ pub use self::{ recorder::Recorder, triedb::{TrieDB, TrieDBBuilder, TrieDBIterator, TrieDBKeyIterator}, triedbmut::{ - Changenode, Changeset, ChildReference, NewChangesetNode, - OwnedPrefix, TreeRefChangeset, TrieDBMut, TrieDBMutBuilder, Value, + Changenode, Changeset, ChildReference, NewChangesetNode, OwnedPrefix, TreeRefChangeset, + TrieDBMut, TrieDBMutBuilder, Value, }, }; use crate::node_db::Hasher;