trie-db: Fetch the closest merkle value (#199)

* trie-db: Add `get_closest_merkle_value` to Trie trait Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Extract the merkle value Signed-off-by: Alexandru Vasile <[email protected]> * trie-db/test: Check merkle value on update key Signed-off-by: Alexandru Vasile <[email protected]> * Update trie-db/src/lookup.rs Co-authored-by: Arkadiy Paronyan <[email protected]> * Update trie-db/src/lookup.rs Co-authored-by: Arkadiy Paronyan <[email protected]> * trie-db: Rename look_up_merkle_without_cache function Signed-off-by: Alexandru Vasile <[email protected]> * trie-db/tests: Check closest descendant of partial keys Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Adjust lookups for partial keys Signed-off-by: Alexandru Vasile <[email protected]> * trie-db/tests: Check non-existent key and branch nodes Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Ensure recording of `NonExisting` for leaves Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Ensure the merkle descedent hash is returned Signed-off-by: Alexandru Vasile <[email protected]> * trie-db/tests: Extend tests with branch nodes and single key db Signed-off-by: Alexandru Vasile <[email protected]> * trie-db/tests: Check trie modification and merkle propagation Signed-off-by: Alexandru Vasile <[email protected]> * trie-db/tests: Use `PrefixedKey` instead of `HashKey` Signed-off-by: Alexandru Vasile <[email protected]> * trie-db/tests: Test extra keys for `test_merkle_value` Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Return the extension node hash Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Use `starts_with` method instead of common prefix Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Return no merkle value on empty node Signed-off-by: Alexandru Vasile <[email protected]> * trie-db/tests: Ensure inline nodes Signed-off-by: Alexandru Vasile <[email protected]> * trie-db/tests: Check empty trie with empty keys Signed-off-by: Alexandru Vasile <[email protected]> * trie-db/tests: Add extra keys to check Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Use `starts_with` for extension nodes Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Rename merkle lookups to lookup_first_descendant Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Return inline hashes properly Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Implement starts_with_slice for NibbleVec Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Use cache for finding first descendent hash Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Introduce caching for descedent node access Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Use rstd::vec::Vec Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Forward merkle value for fatdb and sectriedb Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Rename `get_closest_merkle_value` to `lookup_first_descendant` Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Introduce MerkleValue to return inline nodes and hashes Signed-off-by: Alexandru Vasile <[email protected]> * trie-db: Remove inner function for merkle value lookups Signed-off-by: Alexandru Vasile <[email protected]> * Update trie-db/src/lib.rs Co-authored-by: Bastian Köcher <[email protected]> * Update trie-db/src/lib.rs Co-authored-by: Bastian Köcher <[email protected]> * Apply fmt Signed-off-by: Alexandru Vasile <[email protected]> --------- Signed-off-by: Alexandru Vasile <[email protected]> Co-authored-by: Arkadiy Paronyan <[email protected]> Co-authored-by: Bastian Köcher <[email protected]>
paritytech · Sep 11, 2023 · 08a2305 · 08a2305
1 parent c4be095
commit 08a2305
Show file tree

Hide file tree

Showing 7 changed files with 517 additions and 8 deletions.
diff --git a/trie-db/src/fatdb.rs b/trie-db/src/fatdb.rs
@@ -18,7 +18,7 @@ use super::{
 };
 use hash_db::{HashDBRef, Hasher};
 
-use crate::{rstd::boxed::Box, TrieDBBuilder};
+use crate::{rstd::boxed::Box, MerkleValue, TrieDBBuilder};
 
 /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
 /// Additionaly it stores inserted hash-key mappings for later retrieval.
@@ -72,6 +72,13 @@ where
 		self.raw.get_with(L::Hash::hash(key).as_ref(), query)
 	}
 
+	fn lookup_first_descendant(
+		&self,
+		key: &[u8],
+	) -> Result<Option<MerkleValue<TrieHash<L>>>, TrieHash<L>, CError<L>> {
+		self.raw.lookup_first_descendant(key)
+	}
+
 	fn iter<'a>(
 		&'a self,
 	) -> Result<

diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs
@@ -277,6 +277,17 @@ pub trait Trie<L: TrieLayout> {
 		query: Q,
 	) -> Result<Option<Q::Item>, TrieHash<L>, CError<L>>;
 
+	/// Look up the [`MerkleValue`] of the node that is the closest descendant for the provided
+	/// key.
+	///
+	/// When the provided key leads to a node, then the merkle value of that node
+	/// is returned. However, if the key does not lead to a node, then the merkle value
+	/// of the closest descendant is returned. `None` if no such descendant exists.
+	fn lookup_first_descendant(
+		&self,
+		key: &[u8],
+	) -> Result<Option<MerkleValue<TrieHash<L>>>, TrieHash<L>, CError<L>>;
+
 	/// Returns a depth-first iterator over the elements of trie.
 	fn iter<'a>(
 		&'a self,
@@ -404,6 +415,13 @@ impl<'db, 'cache, L: TrieLayout> Trie<L> for TrieKinds<'db, 'cache, L> {
 		wrapper!(self, get_with, key, query)
 	}
 
+	fn lookup_first_descendant(
+		&self,
+		key: &[u8],
+	) -> Result<Option<MerkleValue<TrieHash<L>>>, TrieHash<L>, CError<L>> {
+		wrapper!(self, lookup_first_descendant, key)
+	}
+
 	fn iter<'a>(
 		&'a self,
 	) -> Result<
@@ -738,3 +756,18 @@ impl From<Bytes> for BytesWeak {
 		Self(rstd::sync::Arc::downgrade(&bytes.0))
 	}
 }
+
+/// Either the `hash` or `value` of a node depending on its size.
+///
+/// If the size of the node `value` is bigger or equal than `MAX_INLINE_VALUE` the `hash` is
+/// returned.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum MerkleValue<H> {
+	/// The merkle value is the node data itself when the
+	/// node data is smaller than `MAX_INLINE_VALUE`.
+	///
+	/// Note: The case of inline nodes.
+	Node(Vec<u8>),
+	/// The merkle value is the hash of the node.
+	Hash(H),
+}
diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs
@@ -18,9 +18,9 @@ use crate::{
 	nibble::NibbleSlice,
 	node::{decode_hash, Node, NodeHandle, NodeHandleOwned, NodeOwned, Value, ValueOwned},
 	node_codec::NodeCodec,
-	rstd::boxed::Box,
-	Bytes, CError, CachedValue, DBValue, Query, RecordedForKey, Result, TrieAccess, TrieCache,
-	TrieError, TrieHash, TrieLayout, TrieRecorder,
+	rstd::{boxed::Box, vec::Vec},
+	Bytes, CError, CachedValue, DBValue, MerkleValue, Query, RecordedForKey, Result, TrieAccess,
+	TrieCache, TrieError, TrieHash, TrieLayout, TrieRecorder,
 };
 use hash_db::{HashDBRef, Hasher, Prefix};
 
@@ -133,6 +133,227 @@ where
 			recorder.record(get_access());
 		}
 	}
+	/// Look up the merkle value (hash) of the node that is the closest descendant for the provided
+	/// key.
+	///
+	/// When the provided key leads to a node, then the merkle value (hash) of that node
+	/// is returned. However, if the key does not lead to a node, then the merkle value
+	/// of the closest descendant is returned. `None` if no such descendant exists.
+	pub fn lookup_first_descendant(
+		mut self,
+		full_key: &[u8],
+		nibble_key: NibbleSlice,
+	) -> Result<Option<MerkleValue<TrieHash<L>>>, TrieHash<L>, CError<L>> {
+		let mut partial = nibble_key;
+		let mut hash = self.hash;
+		let mut key_nibbles = 0;
+
+		let mut cache = self.cache.take();
+
+		// this loop iterates through non-inline nodes.
+		for depth in 0.. {
+			// Ensure the owned node reference lives long enough.
+			// Value is never read, but the reference is.
+			let mut _owned_node = NodeOwned::Empty;
+
+			// The binary encoded data of the node fetched from the database.
+			//
+			// Populated by `get_owned_node` to avoid one extra allocation by not
+			// calling `NodeOwned::to_encoded` when computing the hash of inlined nodes.
+			let mut node_data = Vec::new();
+
+			// Get the owned node representation from the database.
+			let mut get_owned_node = |depth: i32| {
+				let data = match self.db.get(&hash, nibble_key.mid(key_nibbles).left()) {
+					Some(value) => value,
+					None =>
+						return Err(Box::new(match depth {
+							0 => TrieError::InvalidStateRoot(hash),
+							_ => TrieError::IncompleteDatabase(hash),
+						})),
+				};
+
+				let decoded = match L::Codec::decode(&data[..]) {
+					Ok(node) => node,
+					Err(e) => return Err(Box::new(TrieError::DecoderError(hash, e))),
+				};
+
+				let owned = decoded.to_owned_node::<L>()?;
+				node_data = data;
+				Ok(owned)
+			};
+
+			let mut node = if let Some(cache) = &mut cache {
+				let node = cache.get_or_insert_node(hash, &mut || get_owned_node(depth))?;
+
+				self.record(|| TrieAccess::NodeOwned { hash, node_owned: node });
+
+				node
+			} else {
+				_owned_node = get_owned_node(depth)?;
+
+				self.record(|| TrieAccess::EncodedNode {
+					hash,
+					encoded_node: node_data.as_slice().into(),
+				});
+
+				&_owned_node
+			};
+
+			// this loop iterates through all inline children (usually max 1)
+			// without incrementing the depth.
+			let mut is_inline = false;
+			loop {
+				let next_node = match node {
+					NodeOwned::Leaf(slice, _) => {
+						// The leaf slice can be longer than remainder of the provided key
+						// (descendent), but not the other way around.
+						if !slice.starts_with_slice(&partial) {
+							self.record(|| TrieAccess::NonExisting { full_key });
+							return Ok(None)
+						}
+
+						if partial.len() != slice.len() {
+							self.record(|| TrieAccess::NonExisting { full_key });
+						}
+
+						let res = is_inline
+							.then(|| MerkleValue::Node(node_data))
+							.unwrap_or_else(|| MerkleValue::Hash(hash));
+						return Ok(Some(res))
+					},
+					NodeOwned::Extension(slice, item) => {
+						if partial.len() < slice.len() {
+							self.record(|| TrieAccess::NonExisting { full_key });
+
+							// Extension slice can be longer than remainder of the provided key
+							// (descendent), ensure the extension slice starts with the remainder
+							// of the provided key.
+							return if slice.starts_with_slice(&partial) {
+								let res = is_inline
+									.then(|| MerkleValue::Node(node_data))
+									.unwrap_or_else(|| MerkleValue::Hash(hash));
+								Ok(Some(res))
+							} else {
+								Ok(None)
+							}
+						}
+
+						// Remainder of the provided key is longer than the extension slice,
+						// must advance the node iteration if and only if keys share
+						// a common prefix.
+						if partial.starts_with_vec(&slice) {
+							// Empties the partial key if the extension slice is longer.
+							partial = partial.mid(slice.len());
+							key_nibbles += slice.len();
+							item
+						} else {
+							self.record(|| TrieAccess::NonExisting { full_key });
+
+							return Ok(None)
+						}
+					},
+					NodeOwned::Branch(children, value) =>
+						if partial.is_empty() {
+							if value.is_none() {
+								self.record(|| TrieAccess::NonExisting { full_key });
+							}
+							let res = is_inline
+								.then(|| MerkleValue::Node(node_data))
+								.unwrap_or_else(|| MerkleValue::Hash(hash));
+							return Ok(Some(res))
+						} else {
+							match &children[partial.at(0) as usize] {
+								Some(x) => {
+									partial = partial.mid(1);
+									key_nibbles += 1;
+									x
+								},
+								None => {
+									self.record(|| TrieAccess::NonExisting { full_key });
+
+									return Ok(None)
+								},
+							}
+						},
+					NodeOwned::NibbledBranch(slice, children, value) => {
+						// Not enough remainder key to continue the search.
+						if partial.len() < slice.len() {
+							self.record(|| TrieAccess::NonExisting { full_key });
+
+							// Branch slice starts with the remainder key, there's nothing to
+							// advance.
+							return if slice.starts_with_slice(&partial) {
+								let res = is_inline
+									.then(|| MerkleValue::Node(node_data))
+									.unwrap_or_else(|| MerkleValue::Hash(hash));
+								Ok(Some(res))
+							} else {
+								Ok(None)
+							}
+						}
+
+						// Partial key is longer or equal than the branch slice.
+						// Ensure partial key starts with the branch slice.
+						if !partial.starts_with_vec(&slice) {
+							self.record(|| TrieAccess::NonExisting { full_key });
+							return Ok(None)
+						}
+
+						// Partial key starts with the branch slice.
+						if partial.len() == slice.len() {
+							if value.is_none() {
+								self.record(|| TrieAccess::NonExisting { full_key });
+							}
+
+							let res = is_inline
+								.then(|| MerkleValue::Node(node_data))
+								.unwrap_or_else(|| MerkleValue::Hash(hash));
+							return Ok(Some(res))
+						} else {
+							match &children[partial.at(slice.len()) as usize] {
+								Some(x) => {
+									partial = partial.mid(slice.len() + 1);
+									key_nibbles += slice.len() + 1;
+									x
+								},
+								None => {
+									self.record(|| TrieAccess::NonExisting { full_key });
+
+									return Ok(None)
+								},
+							}
+						}
+					},
+					NodeOwned::Empty => {
+						self.record(|| TrieAccess::NonExisting { full_key });
+
+						return Ok(None)
+					},
+					NodeOwned::Value(_, _) => {
+						unreachable!(
+							"`NodeOwned::Value` can not be reached by using the hash of a node. \
+							 `NodeOwned::Value` is only constructed when loading a value into memory, \
+							 which needs to have a different hash than any node; qed",
+						)
+					},
+				};
+
+				// check if new node data is inline or hash.
+				match next_node {
+					NodeHandleOwned::Hash(new_hash) => {
+						hash = *new_hash;
+						break
+					},
+					NodeHandleOwned::Inline(inline_node) => {
+						node = &inline_node;
+						is_inline = true;
+					},
+				}
+			}
+		}
+		Ok(None)
+	}
 
 	/// Look up the given `nibble_key`.
 	///

diff --git a/trie-db/src/nibble/nibblevec.rs b/trie-db/src/nibble/nibblevec.rs
@@ -235,6 +235,25 @@ impl NibbleVec {
 		true
 	}
 
+	/// Same as [`Self::starts_with`] but using [`NibbleSlice`].
+	pub fn starts_with_slice(&self, other: &NibbleSlice) -> bool {
+		if self.len() < other.len() {
+			return false
+		}
+
+		match self.as_nibbleslice() {
+			Some(slice) => slice.starts_with(&other),
+			None => {
+				for i in 0..other.len() {
+					if self.at(i) != other.at(i) {
+						return false
+					}
+				}
+				true
+			},
+		}
+	}
+
 	/// Return an iterator over `Partial` bytes representation.
 	pub fn right_iter<'a>(&'a self) -> impl Iterator<Item = u8> + 'a {
 		let require_padding = self.len % nibble_ops::NIBBLE_PER_BYTE != 0;

diff --git a/trie-db/src/sectriedb.rs b/trie-db/src/sectriedb.rs
@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use crate::{
-	rstd::boxed::Box, triedb::TrieDB, CError, DBValue, Query, Result, Trie, TrieDBBuilder,
-	TrieHash, TrieItem, TrieIterator, TrieKeyItem, TrieLayout,
+	rstd::boxed::Box, triedb::TrieDB, CError, DBValue, MerkleValue, Query, Result, Trie,
+	TrieDBBuilder, TrieHash, TrieItem, TrieIterator, TrieKeyItem, TrieLayout,
 };
 use hash_db::{HashDBRef, Hasher};
 
@@ -75,6 +75,13 @@ where
 		self.raw.get_with(L::Hash::hash(key).as_ref(), query)
 	}
 
+	fn lookup_first_descendant(
+		&self,
+		key: &[u8],
+	) -> Result<Option<MerkleValue<TrieHash<L>>>, TrieHash<L>, CError<L>> {
+		self.raw.lookup_first_descendant(key)
+	}
+
 	fn iter<'a>(
 		&'a self,
 	) -> Result<