bulk child trie deletion

Cargo.toml

@@ -168,3 +168,12 @@ members = [
 [profile.release]
 # Substrate runtime requires unwinding.
 panic = "unwind"
+
+[patch.crates-io]
+kvdb = { git = "https://github.com/cheme/parity-common", branch="upgraded_del_range" }
+kvdb-rocksdb = { git = "https://github.com/cheme/parity-common", branch="upgraded_del_range" }
+kvdb-web = { git = "https://github.com/cheme/parity-common", branch="upgraded_del_range" }
+kvdb-memorydb = { git = "https://github.com/cheme/parity-common", branch="upgraded_del_range" }
+#parity-util-mem = { git = "https://github.com/cheme/parity-common", branch="upgraded_del_range" }
+#fixed-hash = { git = "https://github.com/cheme/parity-common", branch="upgraded_del_range" }
+#primitive-types = { git = "https://github.com/cheme/parity-common", branch="upgraded_del_range" }

bin/node/executor/tests/basic.rs

@@ -170,7 +170,7 @@ fn panic_execution_with_foreign_code_gives_error() {
 				vec![0u8; 32]
 			}
 		],
-		children: map![],
+		children_default: map![],
 	});
 
 	let r = executor_call::<NeverNativeValue, fn() -> _>(
@@ -206,7 +206,7 @@ fn bad_extrinsic_with_native_equivalent_code_gives_error() {
 				vec![0u8; 32]
 			}
 		],
-		children: map![],
+		children_default: map![],
 	});
 
 	let r = executor_call::<NeverNativeValue, fn() -> _>(
@@ -240,7 +240,7 @@ fn successful_execution_with_native_equivalent_code_gives_ok() {
 			},
 			<frame_system::BlockHash<Runtime>>::hashed_key_for(0) => vec![0u8; 32]
 		],
-		children: map![],
+		children_default: map![],
 	});
 
 	let r = executor_call::<NeverNativeValue, fn() -> _>(
@@ -282,7 +282,7 @@ fn successful_execution_with_foreign_code_gives_ok() {
 			},
 			<frame_system::BlockHash<Runtime>>::hashed_key_for(0) => vec![0u8; 32]
 		],
-		children: map![],
+		children_default: map![],
 	});
 
 	let r = executor_call::<NeverNativeValue, fn() -> _>(
@@ -704,7 +704,7 @@ fn panic_execution_gives_error() {
 			},
 			<frame_system::BlockHash<Runtime>>::hashed_key_for(0) => vec![0u8; 32]
 		],
-		children: map![],
+		children_default: map![],
 	});
 
 	let r = executor_call::<NeverNativeValue, fn() -> _>(
@@ -738,7 +738,7 @@ fn successful_execution_gives_ok() {
 			},
 			<frame_system::BlockHash<Runtime>>::hashed_key_for(0) => vec![0u8; 32]
 		],
-		children: map![],
+		children_default: map![],
 	});
 
 	let r = executor_call::<NeverNativeValue, fn() -> _>(

bin/node/executor/tests/fees.rs

@@ -143,7 +143,7 @@ fn transaction_fee_is_correct_ultimate() {
 			},
 			<frame_system::BlockHash<Runtime>>::hashed_key_for(0) => vec![0u8; 32]
 		],
-		children: map![],
+		children_default: map![],
 	});
 
 	let tip = 1_000_000;

bin/node/runtime/src/lib.rs

@@ -82,7 +82,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	// and set impl_version to 0. If only runtime
 	// implementation changes and behavior does not, then leave spec_version as
 	// is and increment impl_version.
-	spec_version: 237,
+	spec_version: 238,
 	impl_version: 0,
 	apis: RUNTIME_API_VERSIONS,
 };

client/api/src/backend.rs

@@ -24,7 +24,7 @@ use sp_runtime::{generic::BlockId, Justification, Storage};
 use sp_runtime::traits::{Block as BlockT, NumberFor, HashFor};
 use sp_state_machine::{
 	ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction,
-	StorageCollection, ChildStorageCollection,
+	ChildStorageCollection, StorageCollection,
 };
 use sp_storage::{StorageData, StorageKey, ChildInfo};
 use crate::{
@@ -280,6 +280,7 @@ impl<'a, State, Block> Iterator for KeyIterator<'a, State, Block> where
 		Some(StorageKey(next_key))
 	}
 }
+
 /// Provides acess to storage primitives
 pub trait StorageProvider<Block: BlockT, B: Backend<Block>> {
 	/// Given a `BlockId` and a key, return the value under the key in that block.
@@ -310,26 +311,23 @@ pub trait StorageProvider<Block: BlockT, B: Backend<Block>> {
 	fn child_storage(
 		&self,
 		id: &BlockId<Block>,
-		storage_key: &StorageKey,
-		child_info: ChildInfo,
+		child_info: &ChildInfo,
 		key: &StorageKey
 	) -> sp_blockchain::Result<Option<StorageData>>;
 
 	/// Given a `BlockId`, a key prefix, and a child storage key, return the matching child storage keys.
 	fn child_storage_keys(
 		&self,
 		id: &BlockId<Block>,
-		child_storage_key: &StorageKey,
-		child_info: ChildInfo,
+		child_info: &ChildInfo,
 		key_prefix: &StorageKey
 	) -> sp_blockchain::Result<Vec<StorageKey>>;
 
 	/// Given a `BlockId`, a key and a child storage key, return the hash under the key in that block.
 	fn child_storage_hash(
 		&self,
 		id: &BlockId<Block>,
-		storage_key: &StorageKey,
-		child_info: ChildInfo,
+		child_info: &ChildInfo,
 		key: &StorageKey
 	) -> sp_blockchain::Result<Option<Block::Hash>>;
 

client/api/src/light.rs

@@ -82,11 +82,6 @@ pub struct RemoteReadChildRequest<Header: HeaderT> {
 	pub header: Header,
 	/// Storage key for child.
 	pub storage_key: Vec<u8>,
-	/// Child trie source information.
-	pub child_info: Vec<u8>,
-	/// Child type, its required to resolve `child_info`
-	/// content and choose child implementation.
-	pub child_type: u32,
 	/// Child storage key to read.
 	pub keys: Vec<Vec<u8>>,
 	/// Number of times to retry request. None means that default RETRY_COUNT is used.

client/api/src/notifications.rs

@@ -23,14 +23,14 @@ use std::{
 
 use fnv::{FnvHashSet, FnvHashMap};
 use futures::channel::mpsc;
-use sp_core::storage::{StorageKey, StorageData};
+use sp_core::storage::{StorageKey, StorageData, ChildChange};
 use sp_runtime::traits::Block as BlockT;
 
 /// Storage change set
 #[derive(Debug)]
 pub struct StorageChangeSet {
 	changes: Arc<Vec<(StorageKey, Option<StorageData>)>>,
-	child_changes: Arc<Vec<(StorageKey, Vec<(StorageKey, Option<StorageData>)>)>>,
+	child_changes: Arc<Vec<(StorageKey, ChildChange, Vec<(StorageKey, Option<StorageData>)>)>>,
 	filter: Option<HashSet<StorageKey>>,
 	child_filters: Option<HashMap<StorageKey, Option<HashSet<StorageKey>>>>,
 }
@@ -48,16 +48,25 @@ impl StorageChangeSet {
 			.map(move |(k,v)| (None, k, v.as_ref()));
 		let children = self.child_changes
 			.iter()
-			.filter_map(move |(sk, changes)| {
+			.filter_map(move |(sk, change, changes)| {
 				if let Some(cf) = self.child_filters.as_ref() {
 					if let Some(filter) = cf.get(sk) {
-						Some(changes
-							.iter()
-							.filter(move |&(key, _)| match filter {
-								Some(ref filter) => filter.contains(key),
-								None => true,
-							})
-							.map(move |(k,v)| (Some(sk), k, v.as_ref())))
+						let bulk_delete = change == &ChildChange::BulkDeleteByKeyspace;
+							Some(changes
+								.iter()
+								.filter(move |&(key, _)| if bulk_delete {
+									true
+								} else {
+									match filter {
+										Some(ref filter) => filter.contains(key),
+										None => true,
+									}
+								})
+								.map(move |(k,v)| (Some(sk), k, if bulk_delete {
+									None
+								} else {
+									v.as_ref()
+								})))
 					} else { None }
 				} else { None	}
 			})
@@ -110,7 +119,7 @@ impl<Block: BlockT> StorageNotifications<Block> {
 		hash: &Block::Hash,
 		changeset: impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
 		child_changeset: impl Iterator<
-			Item=(Vec<u8>, impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>)
+			Item=(Vec<u8>, ChildChange, impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>)
 		>,
 	) {
 		let has_wildcard = !self.wildcard_listeners.is_empty();
@@ -137,7 +146,7 @@ impl<Block: BlockT> StorageNotifications<Block> {
 				changes.push((k, v.map(StorageData)));
 			}
 		}
-		for (sk, changeset) in child_changeset {
+		for (sk, change, changeset) in child_changeset {
 			let sk = StorageKey(sk);
 			if let Some((cl, cw)) = self.child_listeners.get(&sk) {
 				let mut changes = Vec::new();
@@ -156,7 +165,7 @@ impl<Block: BlockT> StorageNotifications<Block> {
 					}
 				}
 				if !changes.is_empty() {
-					child_changes.push((sk, changes));
+					child_changes.push((sk, change, changes));
 				}
 			}
 		}
@@ -313,7 +322,7 @@ mod tests {
 
 	type TestChangeSet = (
 		Vec<(StorageKey, Option<StorageData>)>,
-		Vec<(StorageKey, Vec<(StorageKey, Option<StorageData>)>)>,
+		Vec<(StorageKey, ChildChange, Vec<(StorageKey, Option<StorageData>)>)>,
 	);
 
 	#[cfg(test)]
@@ -360,18 +369,18 @@ mod tests {
 			(vec![5], Some(vec![4])),
 			(vec![6], None),
 		];
-		let c_changeset = vec![(vec![4], c_changeset_1)];
+		let c_changeset = vec![(vec![4], ChildChange::Update, c_changeset_1)];
 		notifications.trigger(
 			&Hash::from_low_u64_be(1),
 			changeset.into_iter(),
-			c_changeset.into_iter().map(|(a,b)| (a, b.into_iter())),
+			c_changeset.into_iter().map(|(a, b, c)| (a, b, c.into_iter())),
 		);
 
 		// then
 		assert_eq!(recv.next().unwrap(), (Hash::from_low_u64_be(1), (vec![
 			(StorageKey(vec![2]), Some(StorageData(vec![3]))),
 			(StorageKey(vec![3]), None),
-		], vec![(StorageKey(vec![4]), vec![
+		], vec![(StorageKey(vec![4]), ChildChange::Update, vec![
 			(StorageKey(vec![5]), Some(StorageData(vec![4]))),
 			(StorageKey(vec![6]), None),
 		])]).into()));
@@ -402,11 +411,11 @@ mod tests {
 			(vec![6], None),
 		];
 
-		let c_changeset = vec![(vec![4], c_changeset_1)];
+		let c_changeset = vec![(vec![4], ChildChange::Update, c_changeset_1)];
 		notifications.trigger(
 			&Hash::from_low_u64_be(1),
 			changeset.into_iter(),
-			c_changeset.into_iter().map(|(a,b)| (a, b.into_iter())),
+			c_changeset.into_iter().map(|(a, b, c)| (a, b, c.into_iter())),
 		);
 
 		// then
@@ -417,9 +426,11 @@ mod tests {
 			(StorageKey(vec![2]), Some(StorageData(vec![3]))),
 		], vec![]).into()));
 		assert_eq!(recv3.next().unwrap(), (Hash::from_low_u64_be(1), (vec![],
-		vec![
-			(StorageKey(vec![4]), vec![(StorageKey(vec![5]), Some(StorageData(vec![4])))]),
-		]).into()));
+		vec![(
+			StorageKey(vec![4]),
+			ChildChange::Update,
+			vec![(StorageKey(vec![5]), Some(StorageData(vec![4])))],
+		)]).into()));
 
 	}
 
@@ -451,7 +462,7 @@ mod tests {
 			(vec![2], Some(vec![3])),
 			(vec![1], None),
 		];
-		let c_changeset = empty::<(_, Empty<_>)>();
+		let c_changeset = empty::<(_, _, Empty<_>)>();
 		notifications.trigger(&Hash::from_low_u64_be(1), changeset.into_iter(), c_changeset);
 
 		// then
@@ -469,7 +480,7 @@ mod tests {
 
 			// when
 			let changeset = vec![];
-			let c_changeset = empty::<(_, Empty<_>)>();
+			let c_changeset = empty::<(_, _, Empty<_>)>();
 			notifications.trigger(&Hash::from_low_u64_be(1), changeset.into_iter(), c_changeset);
 			recv
 		};

client/api/src/proof_provider.rs

@@ -35,8 +35,7 @@ pub trait ProofProvider<Block: BlockT> {
 	fn read_child_proof(
 		&self,
 		id: &BlockId<Block>,
-		storage_key: &[u8],
-		child_info: ChildInfo,
+		child_info: &ChildInfo,
 		keys: &mut dyn Iterator<Item=&[u8]>,
 	) -> sp_blockchain::Result<StorageProof>;