From d513c8b9ad373432b671a46f28b1835c1f879923 Mon Sep 17 00:00:00 2001 From: Arkadiy Paronyan Date: Tue, 12 Jul 2022 13:14:42 +0200 Subject: [PATCH 01/46] sync: Fixed clearing subsequent ranges (#11815) * sync: Fixed clearing subsequent ranges * Warp sync fix * Better documentation --- client/network/sync/src/blocks.rs | 48 +++++++++++++++++++++++++------ client/network/sync/src/lib.rs | 3 ++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/client/network/sync/src/blocks.rs b/client/network/sync/src/blocks.rs index 8d9b039b14fba..26753f120a170 100644 --- a/client/network/sync/src/blocks.rs +++ b/client/network/sync/src/blocks.rs @@ -180,6 +180,9 @@ impl BlockCollection { /// Get a valid chain of blocks ordered in descending order and ready for importing into /// the blockchain. + /// `from` is the maximum block number for the start of the range that we are interested in. + /// The function will return empty Vec if the first block ready is higher than `from`. + /// For each returned block hash `clear_queued` must be called at some later stage. pub fn ready_blocks(&mut self, from: NumberFor) -> Vec> { let mut ready = Vec::new(); @@ -192,6 +195,10 @@ impl BlockCollection { BlockRangeState::Complete(blocks) => { let len = (blocks.len() as u32).into(); prev = start + len; + if let Some(BlockData { block, .. }) = blocks.first() { + self.queued_blocks + .insert(block.hash, (start, start + (blocks.len() as u32).into())); + } // Remove all elements from `blocks` and add them to `ready` ready.append(blocks); len @@ -201,18 +208,12 @@ impl BlockCollection { }; *range_data = BlockRangeState::Queued { len }; } - - if let Some(BlockData { block, .. }) = ready.first() { - self.queued_blocks - .insert(block.hash, (from, from + (ready.len() as u32).into())); - } - trace!(target: "sync", "{} blocks ready for import", ready.len()); ready } - pub fn clear_queued(&mut self, from_hash: &B::Hash) { - if let Some((from, to)) = self.queued_blocks.remove(from_hash) { + pub fn clear_queued(&mut self, hash: &B::Hash) { + if let Some((from, to)) = self.queued_blocks.remove(hash) { let mut block_num = from; while block_num < to { self.blocks.remove(&block_num); @@ -399,4 +400,35 @@ mod test { assert_eq!(bc.needed_blocks(peer.clone(), 5, 50, 39, 0, 200), Some(45..50)); } + + #[test] + fn clear_queued_subsequent_ranges() { + let mut bc = BlockCollection::new(); + assert!(is_empty(&bc)); + let peer = PeerId::random(); + + let blocks = generate_blocks(10); + + // Request 2 ranges + assert_eq!(bc.needed_blocks(peer.clone(), 5, 50, 39, 0, 200), Some(40..45)); + assert_eq!(bc.needed_blocks(peer.clone(), 5, 50, 39, 0, 200), Some(45..50)); + + // got a response on the request for `40..50` + bc.clear_peer_download(&peer); + bc.insert(40, blocks.to_vec(), peer.clone()); + + // request any blocks starting from 1000 or lower. + let ready = bc.ready_blocks(1000); + assert_eq!( + ready, + blocks + .iter() + .map(|b| BlockData { block: b.clone(), origin: Some(peer.clone()) }) + .collect::>() + ); + + bc.clear_queued(&blocks[0].hash); + assert!(bc.blocks.is_empty()); + assert!(bc.queued_blocks.is_empty()); + } } diff --git a/client/network/sync/src/lib.rs b/client/network/sync/src/lib.rs index 5b5216c745c98..1ce69b6dc816f 100644 --- a/client/network/sync/src/lib.rs +++ b/client/network/sync/src/lib.rs @@ -1521,6 +1521,9 @@ where for (_, hash) in &results { self.queue_blocks.remove(hash); self.blocks.clear_queued(hash); + if let Some(gap_sync) = &mut self.gap_sync { + gap_sync.blocks.clear_queued(hash); + } } for (result, hash) in results { if has_error { From 43fd9a9dc2ba2644f88531b52a196a4f0800c2bf Mon Sep 17 00:00:00 2001 From: Xiliang Chen Date: Tue, 12 Jul 2022 23:19:13 +1200 Subject: [PATCH 02/46] CLI flag to configure tx ban duration (#11786) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add tx-ban-seconds * fix * trigger CI * trigger CI * remove test print * Update client/cli/src/params/transaction_pool_params.rs Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com> Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com> --- bin/node/cli/benches/transaction_pool.rs | 3 +++ client/cli/src/commands/run_cmd.rs | 4 ++-- client/cli/src/config.rs | 4 ++-- client/cli/src/params/transaction_pool_params.rs | 14 +++++++++++++- client/transaction-pool/src/graph/pool.rs | 5 ++++- client/transaction-pool/src/graph/rotator.rs | 5 +++++ .../transaction-pool/src/graph/validated_pool.rs | 3 ++- 7 files changed, 31 insertions(+), 7 deletions(-) diff --git a/bin/node/cli/benches/transaction_pool.rs b/bin/node/cli/benches/transaction_pool.rs index f1fce16d8c1b3..5b96355b9ca70 100644 --- a/bin/node/cli/benches/transaction_pool.rs +++ b/bin/node/cli/benches/transaction_pool.rs @@ -16,6 +16,8 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +use std::time::Duration; + use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; use futures::{future, StreamExt}; use node_cli::service::{create_extrinsic, fetch_nonce, FullClient, TransactionPool}; @@ -58,6 +60,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase { ready: PoolLimit { count: 100_000, total_bytes: 100 * 1024 * 1024 }, future: PoolLimit { count: 100_000, total_bytes: 100 * 1024 * 1024 }, reject_future_transactions: false, + ban_time: Duration::from_secs(30 * 60), }, network: network_config, keystore: KeystoreConfig::InMemory, diff --git a/client/cli/src/commands/run_cmd.rs b/client/cli/src/commands/run_cmd.rs index b4a3885e39906..12774eecc4174 100644 --- a/client/cli/src/commands/run_cmd.rs +++ b/client/cli/src/commands/run_cmd.rs @@ -480,8 +480,8 @@ impl CliConfiguration for RunCmd { Ok(self.ws_max_out_buffer_capacity) } - fn transaction_pool(&self) -> Result { - Ok(self.pool_config.transaction_pool()) + fn transaction_pool(&self, is_dev: bool) -> Result { + Ok(self.pool_config.transaction_pool(is_dev)) } fn max_runtime_instances(&self) -> Result> { diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index 6e1317c11fbc4..01c541bf75255 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -145,7 +145,7 @@ pub trait CliConfiguration: Sized { /// Get the transaction pool options /// /// By default this is `TransactionPoolOptions::default()`. - fn transaction_pool(&self) -> Result { + fn transaction_pool(&self, _is_dev: bool) -> Result { Ok(Default::default()) } @@ -523,7 +523,7 @@ pub trait CliConfiguration: Sized { impl_name: C::impl_name(), impl_version: C::impl_version(), tokio_handle, - transaction_pool: self.transaction_pool()?, + transaction_pool: self.transaction_pool(is_dev)?, network: self.network_config( &chain_spec, is_dev, diff --git a/client/cli/src/params/transaction_pool_params.rs b/client/cli/src/params/transaction_pool_params.rs index efb78430ced55..6429dfec3f908 100644 --- a/client/cli/src/params/transaction_pool_params.rs +++ b/client/cli/src/params/transaction_pool_params.rs @@ -29,11 +29,15 @@ pub struct TransactionPoolParams { /// Maximum number of kilobytes of all transactions stored in the pool. #[clap(long, value_name = "COUNT", default_value = "20480")] pub pool_kbytes: usize, + + /// How long a transaction is banned for, if it is considered invalid. Defaults to 1800s. + #[clap(long, value_name = "SECONDS")] + pub tx_ban_seconds: Option, } impl TransactionPoolParams { /// Fill the given `PoolConfiguration` by looking at the cli parameters. - pub fn transaction_pool(&self) -> TransactionPoolOptions { + pub fn transaction_pool(&self, is_dev: bool) -> TransactionPoolOptions { let mut opts = TransactionPoolOptions::default(); // ready queue @@ -45,6 +49,14 @@ impl TransactionPoolParams { opts.future.count = self.pool_limit / factor; opts.future.total_bytes = self.pool_kbytes * 1024 / factor; + opts.ban_time = if let Some(ban_seconds) = self.tx_ban_seconds { + std::time::Duration::from_secs(ban_seconds) + } else if is_dev { + std::time::Duration::from_secs(0) + } else { + std::time::Duration::from_secs(30 * 60) + }; + opts } } diff --git a/client/transaction-pool/src/graph/pool.rs b/client/transaction-pool/src/graph/pool.rs index 618ba8ccf24d5..4ce7954f8d479 100644 --- a/client/transaction-pool/src/graph/pool.rs +++ b/client/transaction-pool/src/graph/pool.rs @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, sync::Arc, time::Duration}; use futures::{channel::mpsc::Receiver, Future}; use sc_transaction_pool_api::error; @@ -108,6 +108,8 @@ pub struct Options { pub future: base::Limit, /// Reject future transactions. pub reject_future_transactions: bool, + /// How long the extrinsic is banned for. + pub ban_time: Duration, } impl Default for Options { @@ -116,6 +118,7 @@ impl Default for Options { ready: base::Limit { count: 8192, total_bytes: 20 * 1024 * 1024 }, future: base::Limit { count: 512, total_bytes: 1 * 1024 * 1024 }, reject_future_transactions: false, + ban_time: Duration::from_secs(60 * 30), } } } diff --git a/client/transaction-pool/src/graph/rotator.rs b/client/transaction-pool/src/graph/rotator.rs index b897fe7885033..c91c8e407bc0f 100644 --- a/client/transaction-pool/src/graph/rotator.rs +++ b/client/transaction-pool/src/graph/rotator.rs @@ -51,6 +51,11 @@ impl Default for PoolRotator { } impl PoolRotator { + /// New rotator instance with specified ban time. + pub fn new(ban_time: Duration) -> Self { + Self { ban_time, banned_until: Default::default() } + } + /// Returns `true` if extrinsic hash is currently banned. pub fn is_banned(&self, hash: &Hash) -> bool { self.banned_until.read().contains_key(hash) diff --git a/client/transaction-pool/src/graph/validated_pool.rs b/client/transaction-pool/src/graph/validated_pool.rs index 084b04842ee90..e59c5afbdc512 100644 --- a/client/transaction-pool/src/graph/validated_pool.rs +++ b/client/transaction-pool/src/graph/validated_pool.rs @@ -125,6 +125,7 @@ impl ValidatedPool { /// Create a new transaction pool. pub fn new(options: Options, is_validator: IsValidator, api: Arc) -> Self { let base_pool = base::BasePool::new(options.reject_future_transactions); + let ban_time = options.ban_time; Self { is_validator, options, @@ -132,7 +133,7 @@ impl ValidatedPool { api, pool: RwLock::new(base_pool), import_notification_sinks: Default::default(), - rotator: Default::default(), + rotator: PoolRotator::new(ban_time), } } From 5677b91eb6ad8e280cf7dcbdb8990d31bd2d74b1 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 12 Jul 2022 21:00:00 +0400 Subject: [PATCH 03/46] Upgrade libp2p to 0.46.1 (#11787) * Update libp2p to 0.46.0 * Update libp2p to 0.46.1 * Fix telemetry initialization * Fix tests --- Cargo.lock | 362 ++++++------------ client/authority-discovery/Cargo.toml | 2 +- client/cli/Cargo.toml | 2 +- client/consensus/common/Cargo.toml | 2 +- client/network-gossip/Cargo.toml | 2 +- client/network/Cargo.toml | 2 +- client/network/common/Cargo.toml | 2 +- client/network/light/Cargo.toml | 2 +- client/network/src/discovery.rs | 6 +- client/network/src/peer_info.rs | 5 +- client/network/src/protocol.rs | 5 +- .../src/protocol/notifications/tests.rs | 6 +- client/network/src/request_responses.rs | 7 +- client/network/src/transport.rs | 9 +- client/network/sync/Cargo.toml | 2 +- client/network/test/Cargo.toml | 2 +- client/peerset/Cargo.toml | 2 +- client/telemetry/Cargo.toml | 2 +- client/telemetry/src/transport.rs | 3 +- 19 files changed, 151 insertions(+), 274 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d68330cfb15c2..26f43fa4ae0ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3207,7 +3207,7 @@ dependencies = [ "http", "jsonrpsee-core", "jsonrpsee-types", - "pin-project 1.0.10", + "pin-project", "rustls-native-certs", "soketto", "thiserror", @@ -3461,9 +3461,9 @@ checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" [[package]] name = "libp2p" -version = "0.45.1" +version = "0.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41726ee8f662563fafba2d2d484b14037cc8ecb8c953fbfc8439d4ce3a0a9029" +checksum = "81327106887e42d004fbdab1fef93675be2e2e07c1b95fce45e2cc813485611d" dependencies = [ "bytes", "futures", @@ -3472,7 +3472,7 @@ dependencies = [ "instant", "lazy_static", "libp2p-autonat", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-deflate", "libp2p-dns", "libp2p-floodsub", @@ -3498,69 +3498,35 @@ dependencies = [ "libp2p-yamux", "multiaddr", "parking_lot 0.12.0", - "pin-project 1.0.10", + "pin-project", "rand 0.7.3", "smallvec", ] [[package]] name = "libp2p-autonat" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50de7c1d5c3f040fccb469e8a2d189e068b7627d760dd74ef914071c16bbe905" +checksum = "4decc51f3573653a9f4ecacb31b1b922dd20c25a6322bb15318ec04287ec46f9" dependencies = [ "async-trait", "futures", "futures-timer", "instant", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-request-response", "libp2p-swarm", "log", - "prost 0.10.3", - "prost-build 0.10.4", - "rand 0.8.4", -] - -[[package]] -name = "libp2p-core" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db5b02602099fb75cb2d16f9ea860a320d6eb82ce41e95ab680912c454805cd5" -dependencies = [ - "asn1_der", - "bs58", - "ed25519-dalek", - "either", - "fnv", - "futures", - "futures-timer", - "instant", - "lazy_static", - "log", - "multiaddr", - "multihash", - "multistream-select", - "parking_lot 0.12.0", - "pin-project 1.0.10", - "prost 0.9.0", - "prost-build 0.9.0", + "prost", + "prost-build", "rand 0.8.4", - "ring", - "rw-stream-sink 0.2.1", - "sha2 0.10.2", - "smallvec", - "thiserror", - "unsigned-varint", - "void", - "zeroize", ] [[package]] name = "libp2p-core" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d46fca305dee6757022e2f5a4f6c023315084d0ed7441c3ab244e76666d979" +checksum = "fbf9b94cefab7599b2d3dff2f93bee218c6621d68590b23ede4485813cbcece6" dependencies = [ "asn1_der", "bs58", @@ -3577,12 +3543,12 @@ dependencies = [ "multihash", "multistream-select", "parking_lot 0.12.0", - "pin-project 1.0.10", - "prost 0.10.3", - "prost-build 0.10.4", + "pin-project", + "prost", + "prost-build", "rand 0.8.4", "ring", - "rw-stream-sink 0.3.0", + "rw-stream-sink", "sha2 0.10.2", "smallvec", "thiserror", @@ -3593,24 +3559,24 @@ dependencies = [ [[package]] name = "libp2p-deflate" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86adefc55ea4ed8201149f052fb441210727481dff1fb0b8318460206a79f5fb" +checksum = "d0183dc2a3da1fbbf85e5b6cf51217f55b14f5daea0c455a9536eef646bfec71" dependencies = [ "flate2", "futures", - "libp2p-core 0.33.0", + "libp2p-core", ] [[package]] name = "libp2p-dns" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb462ec3a51fab457b4b44ac295e8b0a4b04dc175127e615cf996b1f0f1a268" +checksum = "6cbf54723250fa5d521383be789bf60efdabe6bacfb443f87da261019a49b4b5" dependencies = [ "async-std-resolver", "futures", - "libp2p-core 0.33.0", + "libp2p-core", "log", "parking_lot 0.12.0", "smallvec", @@ -3619,27 +3585,27 @@ dependencies = [ [[package]] name = "libp2p-floodsub" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a505d0c6f851cbf2919535150198e530825def8bd3757477f13dc3a57f46cbcc" +checksum = "98a4b6ffd53e355775d24b76f583fdda54b3284806f678499b57913adb94f231" dependencies = [ "cuckoofilter", "fnv", "futures", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-swarm", "log", - "prost 0.10.3", - "prost-build 0.10.4", + "prost", + "prost-build", "rand 0.7.3", "smallvec", ] [[package]] name = "libp2p-gossipsub" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be947d8cea8e6b469201314619395826896d2c051053c3723910ba98e68e04" +checksum = "74b4b888cfbeb1f5551acd3aa1366e01bf88ede26cc3c4645d0d2d004d5ca7b0" dependencies = [ "asynchronous-codec", "base64", @@ -3649,12 +3615,12 @@ dependencies = [ "futures", "hex_fmt", "instant", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-swarm", "log", "prometheus-client", - "prost 0.10.3", - "prost-build 0.10.4", + "prost", + "prost-build", "rand 0.7.3", "regex", "sha2 0.10.2", @@ -3665,19 +3631,19 @@ dependencies = [ [[package]] name = "libp2p-identify" -version = "0.36.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84b53490442d086db1fa5375670c9666e79143dccadef3f7c74a4346899a984" +checksum = "c50b585518f8efd06f93ac2f976bd672e17cdac794644b3117edd078e96bda06" dependencies = [ "asynchronous-codec", "futures", "futures-timer", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-swarm", "log", "lru", - "prost 0.10.3", - "prost-build 0.10.4", + "prost", + "prost-build", "prost-codec", "smallvec", "thiserror", @@ -3686,9 +3652,9 @@ dependencies = [ [[package]] name = "libp2p-kad" -version = "0.37.1" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6b5d4de90fcd35feb65ea6223fd78f3b747a64ca4b65e0813fbe66a27d56aa" +checksum = "740862893bb5f06ac24acc9d49bdeadc3a5e52e51818a30a25c1f3519da2c851" dependencies = [ "arrayvec 0.7.2", "asynchronous-codec", @@ -3698,11 +3664,11 @@ dependencies = [ "futures", "futures-timer", "instant", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-swarm", "log", - "prost 0.10.3", - "prost-build 0.10.4", + "prost", + "prost-build", "rand 0.7.3", "sha2 0.10.2", "smallvec", @@ -3714,9 +3680,9 @@ dependencies = [ [[package]] name = "libp2p-mdns" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4783f8cf00c7b6c1ff0f1870b4fcf50b042b45533d2e13b6fb464caf447a6951" +checksum = "66e5e5919509603281033fd16306c61df7a4428ce274b67af5e14b07de5cdcb2" dependencies = [ "async-io", "data-encoding", @@ -3724,7 +3690,7 @@ dependencies = [ "futures", "if-watch", "lazy_static", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-swarm", "log", "rand 0.8.4", @@ -3735,11 +3701,11 @@ dependencies = [ [[package]] name = "libp2p-metrics" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4357140141ba9739eee71b20aa735351c0fc642635b2bffc7f57a6b5c1090" +checksum = "ef8aff4a1abef42328fbb30b17c853fff9be986dc39af17ee39f9c5f755c5e0c" dependencies = [ - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-gossipsub", "libp2p-identify", "libp2p-kad", @@ -3751,14 +3717,14 @@ dependencies = [ [[package]] name = "libp2p-mplex" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ff9c893f2367631a711301d703c47432af898c9bb8253bea0e2c051a13f7640" +checksum = "61fd1b20638ec209c5075dfb2e8ce6a7ea4ec3cd3ad7b77f7a477c06d53322e2" dependencies = [ "asynchronous-codec", "bytes", "futures", - "libp2p-core 0.33.0", + "libp2p-core", "log", "nohash-hasher", "parking_lot 0.12.0", @@ -3769,18 +3735,18 @@ dependencies = [ [[package]] name = "libp2p-noise" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2cee1dad1c83325bbd182a8e94555778699cec8a9da00086efb7522c4c15ad" +checksum = "762408cb5d84b49a600422d7f9a42c18012d8da6ebcd570f9a4a4290ba41fb6f" dependencies = [ "bytes", "curve25519-dalek 3.0.2", "futures", "lazy_static", - "libp2p-core 0.33.0", + "libp2p-core", "log", - "prost 0.10.3", - "prost-build 0.10.4", + "prost", + "prost-build", "rand 0.8.4", "sha2 0.10.2", "snow", @@ -3791,14 +3757,14 @@ dependencies = [ [[package]] name = "libp2p-ping" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d41516c82fe8dd148ec925eead0c5ec08a0628f7913597e93e126e4dfb4e0787" +checksum = "100a6934ae1dbf8a693a4e7dd1d730fd60b774dafc45688ed63b554497c6c925" dependencies = [ "futures", "futures-timer", "instant", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-swarm", "log", "rand 0.7.3", @@ -3807,17 +3773,17 @@ dependencies = [ [[package]] name = "libp2p-plaintext" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db007e737adc5d28b2e03223b0210164928ad742591127130796a72aa8eaf54f" +checksum = "be27bf0820a6238a4e06365b096d428271cce85a129cf16f2fe9eb1610c4df86" dependencies = [ "asynchronous-codec", "bytes", "futures", - "libp2p-core 0.33.0", + "libp2p-core", "log", - "prost 0.10.3", - "prost-build 0.10.4", + "prost", + "prost-build", "unsigned-varint", "void", ] @@ -3830,7 +3796,7 @@ checksum = "0f1a458bbda880107b5b36fcb9b5a1ef0c329685da0e203ed692a8ebe64cc92c" dependencies = [ "futures", "log", - "pin-project 1.0.10", + "pin-project", "rand 0.7.3", "salsa20", "sha3 0.9.1", @@ -3838,9 +3804,9 @@ dependencies = [ [[package]] name = "libp2p-relay" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624ead3406f64437a0d4567c31bd128a9a0b8226d5f16c074038f5d0fc32f650" +checksum = "4931547ee0cce03971ccc1733ff05bb0c4349fd89120a39e9861e2bbe18843c3" dependencies = [ "asynchronous-codec", "bytes", @@ -3848,12 +3814,12 @@ dependencies = [ "futures", "futures-timer", "instant", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-swarm", "log", - "pin-project 1.0.10", - "prost 0.10.3", - "prost-build 0.10.4", + "pin-project", + "prost", + "prost-build", "prost-codec", "rand 0.8.4", "smallvec", @@ -3864,20 +3830,20 @@ dependencies = [ [[package]] name = "libp2p-rendezvous" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59967ea2db2c7560f641aa58ac05982d42131863fcd3dd6dcf0dd1daf81c60c" +checksum = "9511c9672ba33284838e349623319c8cad2d18cfad243ae46c6b7e8a2982ea4e" dependencies = [ "asynchronous-codec", "bimap", "futures", "futures-timer", "instant", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-swarm", "log", - "prost 0.10.3", - "prost-build 0.10.4", + "prost", + "prost-build", "rand 0.8.4", "sha2 0.10.2", "thiserror", @@ -3887,15 +3853,15 @@ dependencies = [ [[package]] name = "libp2p-request-response" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b02e0acb725e5a757d77c96b95298fd73a7394fe82ba7b8bbeea510719cbe441" +checksum = "508a189e2795d892c8f5c1fa1e9e0b1845d32d7b0b249dbf7b05b18811361843" dependencies = [ "async-trait", "bytes", "futures", "instant", - "libp2p-core 0.33.0", + "libp2p-core", "libp2p-swarm", "log", "rand 0.7.3", @@ -3905,18 +3871,18 @@ dependencies = [ [[package]] name = "libp2p-swarm" -version = "0.36.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4bb21c5abadbf00360c734f16bf87f1712ed4f23cd46148f625d2ddb867346" +checksum = "95ac5be6c2de2d1ff3f7693fda6faf8a827b1f3e808202277783fea9f527d114" dependencies = [ "either", "fnv", "futures", "futures-timer", "instant", - "libp2p-core 0.33.0", + "libp2p-core", "log", - "pin-project 1.0.10", + "pin-project", "rand 0.7.3", "smallvec", "thiserror", @@ -3925,9 +3891,9 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf2fe8c80b43561355f4d51875273b5b6dfbac37952e8f64b1270769305c9d7" +checksum = "9f54a64b6957249e0ce782f8abf41d97f69330d02bf229f0672d864f0650cc76" dependencies = [ "quote", "syn", @@ -3935,9 +3901,9 @@ dependencies = [ [[package]] name = "libp2p-tcp" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4933e38ef21b50698aefc87799c24f2a365c9d3f6cf50471f3f6a0bc410892" +checksum = "8a6771dc19aa3c65d6af9a8c65222bfc8fcd446630ddca487acd161fa6096f3b" dependencies = [ "async-io", "futures", @@ -3945,32 +3911,32 @@ dependencies = [ "if-watch", "ipnet", "libc", - "libp2p-core 0.33.0", + "libp2p-core", "log", "socket2 0.4.4", ] [[package]] name = "libp2p-uds" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24bdab114f7f2701757d6541266e1131b429bbae382008f207f2114ee4222dcb" +checksum = "d125e3e5f0d58f3c6ac21815b20cf4b6a88b8db9dc26368ea821838f4161fd4d" dependencies = [ "async-std", "futures", - "libp2p-core 0.32.1", + "libp2p-core", "log", ] [[package]] name = "libp2p-wasm-ext" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f066f2b8b1a1d64793f05da2256e6842ecd0293d6735ca2e9bda89831a1bdc06" +checksum = "ec894790eec3c1608f8d1a8a0bdf0dbeb79ed4de2dce964222011c2896dfa05a" dependencies = [ "futures", "js-sys", - "libp2p-core 0.33.0", + "libp2p-core", "parity-send-wrapper", "wasm-bindgen", "wasm-bindgen-futures", @@ -3978,18 +3944,18 @@ dependencies = [ [[package]] name = "libp2p-websocket" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39d398fbb29f432c4128fabdaac2ed155c3bcaf1b9bd40eeeb10a471eefacbf5" +checksum = "9808e57e81be76ff841c106b4c5974fb4d41a233a7bdd2afbf1687ac6def3818" dependencies = [ "either", "futures", "futures-rustls", - "libp2p-core 0.33.0", + "libp2p-core", "log", "parking_lot 0.12.0", "quicksink", - "rw-stream-sink 0.3.0", + "rw-stream-sink", "soketto", "url", "webpki-roots", @@ -3997,12 +3963,12 @@ dependencies = [ [[package]] name = "libp2p-yamux" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fe653639ad74877c759720febb0cbcbf4caa221adde4eed2d3126ce5c6f381f" +checksum = "c6dea686217a06072033dc025631932810e2f6ad784e4fafa42e27d311c7a81c" dependencies = [ "futures", - "libp2p-core 0.33.0", + "libp2p-core", "parking_lot 0.12.0", "thiserror", "yamux", @@ -4462,7 +4428,7 @@ dependencies = [ "bytes", "futures", "log", - "pin-project 1.0.10", + "pin-project", "smallvec", "unsigned-varint", ] @@ -6836,33 +6802,13 @@ dependencies = [ "indexmap", ] -[[package]] -name = "pin-project" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9615c18d31137579e9ff063499264ddc1278e7b1982757ebc111028c4d1dc909" -dependencies = [ - "pin-project-internal 0.4.29", -] - [[package]] name = "pin-project" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" dependencies = [ - "pin-project-internal 1.0.10", -] - -[[package]] -name = "pin-project-internal" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "044964427019eed9d49d9d5bbce6047ef18f37100ea400912a9fa4a3523ab12a" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "pin-project-internal", ] [[package]] @@ -7114,16 +7060,6 @@ dependencies = [ "syn", ] -[[package]] -name = "prost" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" -dependencies = [ - "bytes", - "prost-derive 0.9.0", -] - [[package]] name = "prost" version = "0.10.3" @@ -7131,27 +7067,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc03e116981ff7d8da8e5c220e374587b98d294af7ba7dd7fda761158f00086f" dependencies = [ "bytes", - "prost-derive 0.10.1", -] - -[[package]] -name = "prost-build" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" -dependencies = [ - "bytes", - "heck 0.3.2", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prost 0.9.0", - "prost-types 0.9.0", - "regex", - "tempfile", - "which", + "prost-derive", ] [[package]] @@ -7169,8 +7085,8 @@ dependencies = [ "log", "multimap", "petgraph", - "prost 0.10.3", - "prost-types 0.10.1", + "prost", + "prost-types", "regex", "tempfile", "which", @@ -7184,24 +7100,11 @@ checksum = "00af1e92c33b4813cc79fda3f2dbf56af5169709be0202df730e9ebc3e4cd007" dependencies = [ "asynchronous-codec", "bytes", - "prost 0.10.3", + "prost", "thiserror", "unsigned-varint", ] -[[package]] -name = "prost-derive" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "prost-derive" version = "0.10.1" @@ -7215,16 +7118,6 @@ dependencies = [ "syn", ] -[[package]] -name = "prost-types" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" -dependencies = [ - "bytes", - "prost 0.9.0", -] - [[package]] name = "prost-types" version = "0.10.1" @@ -7232,7 +7125,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" dependencies = [ "bytes", - "prost 0.10.3", + "prost", ] [[package]] @@ -7922,17 +7815,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" -[[package]] -name = "rw-stream-sink" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4da5fcb054c46f5a5dff833b129285a93d3f0179531735e6c866e8cc307d2020" -dependencies = [ - "futures", - "pin-project 0.4.29", - "static_assertions", -] - [[package]] name = "rw-stream-sink" version = "0.3.0" @@ -7940,7 +7822,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26338f5e09bb721b85b135ea05af7767c90b52f6de4f087d4f4a3a9d64e7dc04" dependencies = [ "futures", - "pin-project 1.0.10", + "pin-project", "static_assertions", ] @@ -7998,8 +7880,8 @@ dependencies = [ "libp2p", "log", "parity-scale-codec", - "prost 0.10.3", - "prost-build 0.10.4", + "prost", + "prost-build", "quickcheck", "rand 0.7.3", "sc-client-api", @@ -8651,9 +8533,9 @@ dependencies = [ "lru", "parity-scale-codec", "parking_lot 0.12.0", - "pin-project 1.0.10", - "prost 0.10.3", - "prost-build 0.10.4", + "pin-project", + "prost", + "prost-build", "rand 0.7.3", "sc-block-builder", "sc-client-api", @@ -8691,7 +8573,7 @@ dependencies = [ "futures", "libp2p", "parity-scale-codec", - "prost-build 0.10.4", + "prost-build", "sc-peerset", "smallvec", ] @@ -8723,8 +8605,8 @@ dependencies = [ "libp2p", "log", "parity-scale-codec", - "prost 0.10.3", - "prost-build 0.10.4", + "prost", + "prost-build", "sc-client-api", "sc-network-common", "sc-peerset", @@ -8746,8 +8628,8 @@ dependencies = [ "log", "lru", "parity-scale-codec", - "prost 0.10.3", - "prost-build 0.10.4", + "prost", + "prost-build", "quickcheck", "sc-block-builder", "sc-client-api", @@ -8954,7 +8836,7 @@ dependencies = [ "parity-scale-codec", "parity-util-mem", "parking_lot 0.12.0", - "pin-project 1.0.10", + "pin-project", "rand 0.7.3", "sc-block-builder", "sc-chain-spec", @@ -9099,7 +8981,7 @@ dependencies = [ "libp2p", "log", "parking_lot 0.12.0", - "pin-project 1.0.10", + "pin-project", "rand 0.7.3", "serde", "serde_json", @@ -11189,7 +11071,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 1.0.10", + "pin-project", "tracing", ] diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml index 32a5e23bd49d9..012e6096aab80 100644 --- a/client/authority-discovery/Cargo.toml +++ b/client/authority-discovery/Cargo.toml @@ -22,7 +22,7 @@ codec = { package = "parity-scale-codec", version = "3.0.0", default-features = futures = "0.3.21" futures-timer = "3.0.1" ip_network = "0.4.1" -libp2p = { version = "0.45.1", default-features = false, features = ["kad"] } +libp2p = { version = "0.46.1", default-features = false, features = ["kad"] } log = "0.4.17" prost = "0.10" rand = "0.7.2" diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index 72b4efde077ff..ea60e4c9f87e5 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -18,7 +18,7 @@ clap = { version = "3.1.18", features = ["derive"] } fdlimit = "0.2.1" futures = "0.3.21" hex = "0.4.2" -libp2p = "0.45.1" +libp2p = "0.46.1" log = "0.4.17" names = { version = "0.13.0", default-features = false } parity-scale-codec = "3.0.0" diff --git a/client/consensus/common/Cargo.toml b/client/consensus/common/Cargo.toml index 6d76eba0935ff..12b630f36b89b 100644 --- a/client/consensus/common/Cargo.toml +++ b/client/consensus/common/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] async-trait = "0.1.42" futures = { version = "0.3.21", features = ["thread-pool"] } futures-timer = "3.0.1" -libp2p = { version = "0.45.1", default-features = false } +libp2p = { version = "0.46.1", default-features = false } log = "0.4.17" parking_lot = "0.12.0" serde = { version = "1.0", features = ["derive"] } diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml index 0fac96b0fdde0..144c5ad168996 100644 --- a/client/network-gossip/Cargo.toml +++ b/client/network-gossip/Cargo.toml @@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] ahash = "0.7.6" futures = "0.3.21" futures-timer = "3.0.1" -libp2p = { version = "0.45.1", default-features = false } +libp2p = { version = "0.46.1", default-features = false } log = "0.4.17" lru = "0.7.5" tracing = "0.1.29" diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index 1b525844395b5..dfd2db2e6b844 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -29,7 +29,7 @@ futures = "0.3.21" futures-timer = "3.0.2" hex = "0.4.0" ip_network = "0.4.1" -libp2p = "0.45.1" +libp2p = "0.46.1" linked_hash_set = "0.1.3" linked-hash-map = "0.5.4" log = "0.4.17" diff --git a/client/network/common/Cargo.toml b/client/network/common/Cargo.toml index 553eb57958a5d..e69787d7aff77 100644 --- a/client/network/common/Cargo.toml +++ b/client/network/common/Cargo.toml @@ -21,6 +21,6 @@ codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive", ] } futures = "0.3.21" -libp2p = "0.45.1" +libp2p = "0.46.1" smallvec = "1.8.0" sc-peerset = { version = "4.0.0-dev", path = "../../peerset" } diff --git a/client/network/light/Cargo.toml b/client/network/light/Cargo.toml index 924b8ed06ab0a..0037177fb4046 100644 --- a/client/network/light/Cargo.toml +++ b/client/network/light/Cargo.toml @@ -21,7 +21,7 @@ codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive", ] } futures = "0.3.21" -libp2p = "0.45.1" +libp2p = "0.46.1" log = "0.4.16" prost = "0.10" sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index 2bae2fb807f7e..f3d1588c0280e 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -52,8 +52,8 @@ use futures_timer::Delay; use ip_network::IpNetwork; use libp2p::{ core::{ - connection::{ConnectionId, ListenerId}, - ConnectedPoint, Multiaddr, PeerId, PublicKey, + connection::ConnectionId, transport::ListenerId, ConnectedPoint, Multiaddr, PeerId, + PublicKey, }, kad::{ handler::KademliaHandlerProto, @@ -1030,7 +1030,7 @@ mod tests { let noise_keys = noise::Keypair::::new().into_authentic(&keypair).unwrap(); - let transport = MemoryTransport + let transport = MemoryTransport::new() .upgrade(upgrade::Version::V1) .authenticate(noise::NoiseConfig::xx(noise_keys).into_authenticated()) .multiplex(yamux::YamuxConfig::default()) diff --git a/client/network/src/peer_info.rs b/client/network/src/peer_info.rs index c3e79437bdd06..d668cb25ea455 100644 --- a/client/network/src/peer_info.rs +++ b/client/network/src/peer_info.rs @@ -21,9 +21,8 @@ use fnv::FnvHashMap; use futures::prelude::*; use libp2p::{ core::{ - connection::{ConnectionId, ListenerId}, - either::EitherOutput, - ConnectedPoint, PeerId, PublicKey, + connection::ConnectionId, either::EitherOutput, transport::ListenerId, ConnectedPoint, + PeerId, PublicKey, }, identify::{Identify, IdentifyConfig, IdentifyEvent, IdentifyInfo}, ping::{Ping, PingConfig, PingEvent, PingSuccess}, diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 387a7b3fdde90..348d2d0bf8877 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -27,10 +27,7 @@ use bytes::Bytes; use codec::{Decode, DecodeAll, Encode}; use futures::{channel::oneshot, prelude::*}; use libp2p::{ - core::{ - connection::{ConnectionId, ListenerId}, - ConnectedPoint, - }, + core::{connection::ConnectionId, transport::ListenerId, ConnectedPoint}, request_response::OutboundFailure, swarm::{ ConnectionHandler, IntoConnectionHandler, NetworkBehaviour, NetworkBehaviourAction, diff --git a/client/network/src/protocol/notifications/tests.rs b/client/network/src/protocol/notifications/tests.rs index da12dbf216c4c..fa79366d20283 100644 --- a/client/network/src/protocol/notifications/tests.rs +++ b/client/network/src/protocol/notifications/tests.rs @@ -23,8 +23,8 @@ use crate::protocol::notifications::{Notifications, NotificationsOut, ProtocolCo use futures::prelude::*; use libp2p::{ core::{ - connection::{ConnectionId, ListenerId}, - transport::MemoryTransport, + connection::ConnectionId, + transport::{ListenerId, MemoryTransport}, upgrade, ConnectedPoint, }, identity, noise, @@ -56,7 +56,7 @@ fn build_nodes() -> (Swarm, Swarm) { let noise_keys = noise::Keypair::::new().into_authentic(&keypair).unwrap(); - let transport = MemoryTransport + let transport = MemoryTransport::new() .upgrade(upgrade::Version::V1) .authenticate(noise::NoiseConfig::xx(noise_keys).into_authenticated()) .multiplex(yamux::YamuxConfig::default()) diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index 6c7b0f3fcdfc8..cec4aa2a07fba 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -40,10 +40,7 @@ use futures::{ prelude::*, }; use libp2p::{ - core::{ - connection::{ConnectionId, ListenerId}, - ConnectedPoint, Multiaddr, PeerId, - }, + core::{connection::ConnectionId, transport::ListenerId, ConnectedPoint, Multiaddr, PeerId}, request_response::{ handler::RequestResponseHandler, ProtocolSupport, RequestResponse, RequestResponseCodec, RequestResponseConfig, RequestResponseEvent, RequestResponseMessage, ResponseChannel, @@ -968,7 +965,7 @@ mod tests { let noise_keys = noise::Keypair::::new().into_authentic(&keypair).unwrap(); - let transport = MemoryTransport + let transport = MemoryTransport::new() .upgrade(upgrade::Version::V1) .authenticate(noise::NoiseConfig::xx(noise_keys).into_authenticated()) .multiplex(libp2p::yamux::YamuxConfig::default()) diff --git a/client/network/src/transport.rs b/client/network/src/transport.rs index 883f828e7db51..23645b11795c3 100644 --- a/client/network/src/transport.rs +++ b/client/network/src/transport.rs @@ -54,16 +54,17 @@ pub fn build_transport( ) -> (Boxed<(PeerId, StreamMuxerBox)>, Arc) { // Build the base layer of the transport. let transport = if !memory_only { - let desktop_trans = tcp::TcpConfig::new().nodelay(true); + let tcp_config = tcp::GenTcpConfig::new().nodelay(true); + let desktop_trans = tcp::TcpTransport::new(tcp_config.clone()); let desktop_trans = websocket::WsConfig::new(desktop_trans) - .or_transport(tcp::TcpConfig::new().nodelay(true)); + .or_transport(tcp::TcpTransport::new(tcp_config.clone())); let dns_init = futures::executor::block_on(dns::DnsConfig::system(desktop_trans)); EitherTransport::Left(if let Ok(dns) = dns_init { EitherTransport::Left(dns) } else { - let desktop_trans = tcp::TcpConfig::new().nodelay(true); + let desktop_trans = tcp::TcpTransport::new(tcp_config.clone()); let desktop_trans = websocket::WsConfig::new(desktop_trans) - .or_transport(tcp::TcpConfig::new().nodelay(true)); + .or_transport(tcp::TcpTransport::new(tcp_config)); EitherTransport::Right(desktop_trans.map_err(dns::DnsErr::Transport)) }) } else { diff --git a/client/network/sync/Cargo.toml b/client/network/sync/Cargo.toml index 9185e812bc6cd..a61f780fd56ad 100644 --- a/client/network/sync/Cargo.toml +++ b/client/network/sync/Cargo.toml @@ -23,7 +23,7 @@ codec = { package = "parity-scale-codec", version = "3.0.0", features = [ ] } either = "1.5.3" futures = "0.3.21" -libp2p = "0.45.1" +libp2p = "0.46.1" log = "0.4.17" lru = "0.7.5" prost = "0.10" diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index 2af1dd8cb9b54..fdb9befc41cb1 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -17,7 +17,7 @@ async-std = "1.11.0" async-trait = "0.1.50" futures = "0.3.21" futures-timer = "3.0.1" -libp2p = { version = "0.45.1", default-features = false } +libp2p = { version = "0.46.1", default-features = false } log = "0.4.17" parking_lot = "0.12.0" rand = "0.7.2" diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index f150c728613ba..510ebf7006155 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = "0.3.21" -libp2p = { version = "0.45.1", default-features = false } +libp2p = { version = "0.46.1", default-features = false } log = "0.4.17" serde_json = "1.0.79" wasm-timer = "0.2" diff --git a/client/telemetry/Cargo.toml b/client/telemetry/Cargo.toml index 9682dc824f930..0d966dacfc2c8 100644 --- a/client/telemetry/Cargo.toml +++ b/client/telemetry/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] chrono = "0.4.19" futures = "0.3.21" -libp2p = { version = "0.45.1", default-features = false, features = ["dns-async-std", "tcp-async-io", "wasm-ext", "websocket"] } +libp2p = { version = "0.46.1", default-features = false, features = ["dns-async-std", "tcp-async-io", "wasm-ext", "websocket"] } log = "0.4.17" parking_lot = "0.12.0" pin-project = "1.0.10" diff --git a/client/telemetry/src/transport.rs b/client/telemetry/src/transport.rs index e21a2380be255..d64da44a83b6b 100644 --- a/client/telemetry/src/transport.rs +++ b/client/telemetry/src/transport.rs @@ -31,7 +31,8 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(20); pub(crate) fn initialize_transport() -> Result { let transport = { - let inner = block_on(libp2p::dns::DnsConfig::system(libp2p::tcp::TcpConfig::new()))?; + let tcp_transport = libp2p::tcp::TcpTransport::new(libp2p::tcp::GenTcpConfig::new()); + let inner = block_on(libp2p::dns::DnsConfig::system(tcp_transport))?; libp2p::websocket::framed::WsConfig::new(inner).and_then(|connec, _| { let connec = connec .with(|item| { From 3c48c61ac6a088901668820a825e2d5823bdea82 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Tue, 12 Jul 2022 18:08:54 +0100 Subject: [PATCH 04/46] prep council election pallet for being dissolved (#11790) * prep council election pallet for being dissolved + make it temporarily bounded * fix tests * fix * Update frame/elections-phragmen/src/lib.rs * fix bench? --- Cargo.lock | 1 + frame/elections-phragmen/Cargo.toml | 1 + frame/elections-phragmen/src/benchmarking.rs | 86 ++----------- frame/elections-phragmen/src/lib.rs | 129 ++++++++----------- 4 files changed, 66 insertions(+), 151 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26f43fa4ae0ba..f98d98a3c571f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5673,6 +5673,7 @@ dependencies = [ "sp-npos-elections", "sp-runtime", "sp-std", + "sp-tracing", "substrate-test-utils", ] diff --git a/frame/elections-phragmen/Cargo.toml b/frame/elections-phragmen/Cargo.toml index 0f195f12cce3c..d71a74f76a114 100644 --- a/frame/elections-phragmen/Cargo.toml +++ b/frame/elections-phragmen/Cargo.toml @@ -30,6 +30,7 @@ sp-std = { version = "4.0.0", default-features = false, path = "../../primitives [dev-dependencies] pallet-balances = { version = "4.0.0-dev", path = "../balances" } sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-tracing = { path = "../../primitives/tracing" } substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } [features] diff --git a/frame/elections-phragmen/src/benchmarking.rs b/frame/elections-phragmen/src/benchmarking.rs index 44dd6aff09f0c..9a9d448449eca 100644 --- a/frame/elections-phragmen/src/benchmarking.rs +++ b/frame/elections-phragmen/src/benchmarking.rs @@ -22,17 +22,12 @@ use super::*; use frame_benchmarking::{account, benchmarks, whitelist, BenchmarkError, BenchmarkResult}; -use frame_support::{ - dispatch::{DispatchResultWithPostInfo, UnfilteredDispatchable}, - traits::OnInitialize, -}; +use frame_support::{dispatch::DispatchResultWithPostInfo, traits::OnInitialize}; use frame_system::RawOrigin; use crate::Pallet as Elections; const BALANCE_FACTOR: u32 = 250; -const MAX_VOTERS: u32 = 500; -const MAX_CANDIDATES: u32 = 200; type Lookup = <::Lookup as StaticLookup>::Source; @@ -345,38 +340,6 @@ benchmarks! { ))?; } - // -- Root ones - #[extra] // this calls into phragmen and consumes a full block for now. - remove_member_without_replacement_extra { - // worse case is when we remove a member and we have no runner as a replacement. This - // triggers phragmen again. The only parameter is how many candidates will compete for the - // new slot. - let c in 1 .. MAX_CANDIDATES; - clean::(); - - // fill only desired members. no runners-up. - let all_members = fill_seats_up_to::(T::DesiredMembers::get())?; - assert_eq!(>::members().len() as u32, T::DesiredMembers::get()); - - // submit a new one to compensate, with self-vote. - let replacements = submit_candidates_with_self_vote::(c, "new_candidate")?; - - // create some voters for these replacements. - distribute_voters::(replacements, MAX_VOTERS, MAXIMUM_VOTE)?; - - let to_remove = as_lookup::(all_members[0].clone()); - }: remove_member(RawOrigin::Root, to_remove, false) - verify { - // must still have the desired number of members members. - assert_eq!(>::members().len() as u32, T::DesiredMembers::get()); - #[cfg(test)] - { - // reset members in between benchmark tests. - use crate::tests::MEMBERS; - MEMBERS.with(|m| *m.borrow_mut() = vec![]); - } - } - remove_member_with_replacement { // easy case. We have a runner up. Nothing will have that much of an impact. m will be // number of members and runners. There is always at least one runner. @@ -385,7 +348,7 @@ benchmarks! { let _ = fill_seats_up_to::(m)?; let removing = as_lookup::(>::members_ids()[0].clone()); - }: remove_member(RawOrigin::Root, removing, true) + }: remove_member(RawOrigin::Root, removing, true, false) verify { // must still have enough members. assert_eq!(>::members().len() as u32, T::DesiredMembers::get()); @@ -397,39 +360,6 @@ benchmarks! { } } - remove_member_wrong_refund { - // The root call by mistake indicated that this will have no replacement, while it has! - // this has now consumed a lot of weight and need to refund. - let m = T::DesiredMembers::get() + T::DesiredRunnersUp::get(); - clean::(); - - let _ = fill_seats_up_to::(m)?; - let removing = as_lookup::(>::members_ids()[0].clone()); - let who = T::Lookup::lookup(removing.clone()).expect("member was added above"); - let call = Call::::remove_member { who: removing, has_replacement: false }.encode(); - }: { - assert_eq!( - as Decode>::decode(&mut &*call) - .expect("call is encoded above, encoding must be correct") - .dispatch_bypass_filter(RawOrigin::Root.into()) - .unwrap_err() - .error, - Error::::InvalidReplacement.into(), - ); - } - verify { - // must still have enough members. - assert_eq!(>::members().len() as u32, T::DesiredMembers::get()); - // on fail, `who` must still be a member - assert!(>::members_ids().contains(&who)); - #[cfg(test)] - { - // reset members in between benchmark tests. - use crate::tests::MEMBERS; - MEMBERS.with(|m| *m.borrow_mut() = vec![]); - } - } - clean_defunct_voters { // total number of voters. let v in (MAX_VOTERS / 2) .. MAX_VOTERS; @@ -439,7 +369,7 @@ benchmarks! { // remove any previous stuff. clean::(); - let all_candidates = submit_candidates::(v, "candidates")?; + let all_candidates = submit_candidates::(MAX_CANDIDATES, "candidates")?; distribute_voters::(all_candidates, v, MAXIMUM_VOTE)?; // all candidates leave. @@ -474,7 +404,7 @@ benchmarks! { let votes_per_voter = (e / v).min(MAXIMUM_VOTE as u32); let all_candidates = submit_candidates_with_self_vote::(c, "candidates")?; - let _ = distribute_voters::(all_candidates, v, votes_per_voter as usize)?; + let _ = distribute_voters::(all_candidates, v.saturating_sub(c), votes_per_voter as usize)?; }: { >::on_initialize(T::TermDuration::get()); } @@ -503,7 +433,11 @@ benchmarks! { let votes_per_voter = e / fixed_v; let all_candidates = submit_candidates_with_self_vote::(c, "candidates")?; - let _ = distribute_voters::(all_candidates, fixed_v, votes_per_voter as usize)?; + let _ = distribute_voters::( + all_candidates, + fixed_v - c, + votes_per_voter as usize, + )?; }: { >::on_initialize(T::TermDuration::get()); } @@ -525,7 +459,7 @@ benchmarks! { #[extra] election_phragmen_v { let v in 4 .. 16; - let fixed_c = MAX_CANDIDATES; + let fixed_c = MAX_CANDIDATES / 10; let fixed_e = 64; clean::(); diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index ec2234cde5a6e..28fed28f18e5c 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -100,7 +100,6 @@ use codec::{Decode, Encode}; use frame_support::{ - dispatch::WithPostDispatchInfo, traits::{ defensive_prelude::*, ChangeMembers, Contains, ContainsLengthBound, Currency, CurrencyToVote, Get, InitializeMembers, LockIdentifier, LockableCurrency, OnUnbalanced, @@ -126,6 +125,17 @@ pub mod migrations; /// The maximum votes allowed per voter. pub const MAXIMUM_VOTE: usize = 16; +// Some safe temp values to make the wasm execution sane while we still use this pallet. +#[cfg(test)] +pub(crate) const MAX_CANDIDATES: u32 = 100; +#[cfg(not(test))] +pub(crate) const MAX_CANDIDATES: u32 = 1000; + +#[cfg(test)] +pub(crate) const MAX_VOTERS: u32 = 1000; +#[cfg(not(test))] +pub(crate) const MAX_VOTERS: u32 = 10 * 1000; + type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; type NegativeImbalanceOf = <::Currency as Currency< @@ -385,8 +395,9 @@ pub mod pallet { ) -> DispatchResultWithPostInfo { let who = ensure_signed(origin)?; - let actual_count = >::decode_len().unwrap_or(0); - ensure!(actual_count as u32 <= candidate_count, Error::::InvalidWitnessData); + let actual_count = >::decode_len().unwrap_or(0) as u32; + ensure!(actual_count <= candidate_count, Error::::InvalidWitnessData); + ensure!(actual_count <= MAX_CANDIDATES, Error::::TooManyCandidates); let index = Self::is_candidate(&who).err().ok_or(Error::::DuplicatedCandidate)?; @@ -469,7 +480,11 @@ pub mod pallet { /// the outgoing member is slashed. /// /// If a runner-up is available, then the best runner-up will be removed and replaces the - /// outgoing member. Otherwise, a new phragmen election is started. + /// outgoing member. Otherwise, if `rerun_election` is `true`, a new phragmen election is + /// started, else, nothing happens. + /// + /// If `slash_bond` is set to true, the bond of the member being removed is slashed. Else, + /// it is returned. /// /// The dispatch origin of this call must be root. /// @@ -479,33 +494,24 @@ pub mod pallet { /// If we have a replacement, we use a small weight. Else, since this is a root call and /// will go into phragmen, we assume full block for now. /// # - #[pallet::weight(if *has_replacement { - T::WeightInfo::remove_member_with_replacement() - } else { + #[pallet::weight(if *rerun_election { T::WeightInfo::remove_member_without_replacement() + } else { + T::WeightInfo::remove_member_with_replacement() })] pub fn remove_member( origin: OriginFor, who: ::Source, - has_replacement: bool, + slash_bond: bool, + rerun_election: bool, ) -> DispatchResultWithPostInfo { ensure_root(origin)?; let who = T::Lookup::lookup(who)?; - let will_have_replacement = >::decode_len().map_or(false, |l| l > 0); - if will_have_replacement != has_replacement { - // In both cases, we will change more weight than need. Refund and abort. - return Err(Error::::InvalidReplacement.with_weight( - // refund. The weight value comes from a benchmark which is special to this. - T::WeightInfo::remove_member_wrong_refund(), - )) - } - - let had_replacement = Self::remove_and_replace_member(&who, true)?; - debug_assert_eq!(has_replacement, had_replacement); + let _ = Self::remove_and_replace_member(&who, slash_bond)?; Self::deposit_event(Event::MemberKicked { member: who }); - if !had_replacement { + if rerun_election { Self::do_phragmen(); } @@ -585,10 +591,10 @@ pub mod pallet { UnableToPayBond, /// Must be a voter. MustBeVoter, - /// Cannot report self. - ReportSelf, /// Duplicated candidate submission. DuplicatedCandidate, + /// Too many candidates have been created. + TooManyCandidates, /// Member cannot re-submit candidacy. MemberSubmit, /// Runner cannot re-submit candidacy. @@ -893,7 +899,7 @@ impl Pallet { if candidates_and_deposit.len().is_zero() { Self::deposit_event(Event::EmptyTerm); - return T::DbWeight::get().reads(5) + return T::DbWeight::get().reads(3) } // All of the new winners that come out of phragmen will thus have a deposit recorded. @@ -906,10 +912,28 @@ impl Pallet { let to_balance = |e: ExtendedBalance| T::CurrencyToVote::to_currency(e, total_issuance); let mut num_edges: u32 = 0; + // used for prime election. - let voters_and_stakes = Voting::::iter() - .map(|(voter, Voter { stake, votes, .. })| (voter, stake, votes)) - .collect::>(); + let mut voters_and_stakes = Vec::new(); + match Voting::::iter().try_for_each(|(voter, Voter { stake, votes, .. })| { + if voters_and_stakes.len() < MAX_VOTERS as usize { + voters_and_stakes.push((voter, stake, votes)); + Ok(()) + } else { + Err(()) + } + }) { + Ok(_) => (), + Err(_) => { + log::error!( + target: "runtime::elections-phragmen", + "Failed to run election. Number of voters exceeded", + ); + Self::deposit_event(Event::ElectionError); + return T::DbWeight::get().reads(3 + MAX_VOTERS as u64) + }, + } + // used for phragmen. let voters_and_votes = voters_and_stakes .iter() @@ -1137,7 +1161,7 @@ mod tests { use sp_runtime::{ testing::Header, traits::{BlakeTwo256, IdentityLookup}, - BuildStorage, ModuleError, + BuildStorage, }; use substrate_test_utils::assert_eq_uvec; @@ -1321,6 +1345,7 @@ mod tests { self } pub fn build_and_execute(self, test: impl FnOnce() -> ()) { + sp_tracing::try_init_simple(); MEMBERS.with(|m| { *m.borrow_mut() = self.genesis_members.iter().map(|(m, _)| m.clone()).collect::>() @@ -2494,7 +2519,7 @@ mod tests { assert_ok!(submit_candidacy(Origin::signed(3))); assert_ok!(vote(Origin::signed(3), vec![3], 30)); - assert_ok!(Elections::remove_member(Origin::root(), 4, false)); + assert_ok!(Elections::remove_member(Origin::root(), 4, true, true)); assert_eq!(balances(&4), (35, 2)); // slashed assert_eq!(Elections::election_rounds(), 2); // new election round @@ -2502,52 +2527,6 @@ mod tests { }); } - #[test] - fn remove_member_should_indicate_replacement() { - ExtBuilder::default().build_and_execute(|| { - assert_ok!(submit_candidacy(Origin::signed(5))); - assert_ok!(submit_candidacy(Origin::signed(4))); - - assert_ok!(vote(Origin::signed(4), vec![4], 40)); - assert_ok!(vote(Origin::signed(5), vec![5], 50)); - - System::set_block_number(5); - Elections::on_initialize(System::block_number()); - assert_eq!(members_ids(), vec![4, 5]); - - // no replacement yet. - let unwrapped_error = Elections::remove_member(Origin::root(), 4, true).unwrap_err(); - assert!(matches!( - unwrapped_error.error, - DispatchError::Module(ModuleError { message: Some("InvalidReplacement"), .. }) - )); - assert!(unwrapped_error.post_info.actual_weight.is_some()); - }); - - ExtBuilder::default().desired_runners_up(1).build_and_execute(|| { - assert_ok!(submit_candidacy(Origin::signed(5))); - assert_ok!(submit_candidacy(Origin::signed(4))); - assert_ok!(submit_candidacy(Origin::signed(3))); - - assert_ok!(vote(Origin::signed(3), vec![3], 30)); - assert_ok!(vote(Origin::signed(4), vec![4], 40)); - assert_ok!(vote(Origin::signed(5), vec![5], 50)); - - System::set_block_number(5); - Elections::on_initialize(System::block_number()); - assert_eq!(members_ids(), vec![4, 5]); - assert_eq!(runners_up_ids(), vec![3]); - - // there is a replacement! and this one needs a weight refund. - let unwrapped_error = Elections::remove_member(Origin::root(), 4, false).unwrap_err(); - assert!(matches!( - unwrapped_error.error, - DispatchError::Module(ModuleError { message: Some("InvalidReplacement"), .. }) - )); - assert!(unwrapped_error.post_info.actual_weight.is_some()); - }); - } - #[test] fn seats_should_be_released_when_no_vote() { ExtBuilder::default().build_and_execute(|| { @@ -2684,7 +2663,7 @@ mod tests { Elections::on_initialize(System::block_number()); assert_eq!(members_ids(), vec![2, 4]); - assert_ok!(Elections::remove_member(Origin::root(), 2, true)); + assert_ok!(Elections::remove_member(Origin::root(), 2, true, false)); assert_eq!(members_ids(), vec![4, 5]); }); } From b0777b4c7f7d8d3c635319d0153953d0a22aa58e Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Tue, 12 Jul 2022 23:34:17 +0300 Subject: [PATCH 05/46] Network sync refactoring (part 4) (#11412) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove direct dependency of `sc-network` on `sc-network-light` * Move `WarpSyncProvider` trait and surrounding data structures into `sc-network-common` * Move `WarpSyncProvider` trait and surrounding data structures into `sc-network-common` * Create `sync` module in `sc-network-common`, create `ChainSync` trait there (not used yet), move a bunch of associated data structures from `sc-network-sync` * Switch from concrete implementation to `ChainSync` trait from `sc-network-common` * Introduce `OpaqueStateRequest`/`OpaqueStateResponse` to remove generics from `StateSync` trait * Introduce `OpaqueBlockRequest`/`OpaqueBlockResponse`, make `scheme` module of `sc-network-sync` private * Surface `sc-network-sync` into `sc-service` and make `sc-network` not depend on it anymore * Remove now unnecessary dependency from `sc-network` * Replace crate links with just text since dependencies are gone now * Remove `warp_sync` re-export from `sc-network-common` * Update copyright in network-related files * Address review comments about documentation * Apply review suggestion * Rename `extra_requests` module to `metrics` Co-authored-by: Bastian Köcher --- Cargo.lock | 13 +- client/beefy/src/tests.rs | 14 +- client/consensus/aura/src/lib.rs | 14 +- client/consensus/babe/src/tests.rs | 15 +- client/finality-grandpa/Cargo.toml | 1 + client/finality-grandpa/src/tests.rs | 20 +- client/finality-grandpa/src/warp_proof.rs | 2 +- client/network/Cargo.toml | 5 +- client/network/common/Cargo.toml | 5 + client/network/common/src/lib.rs | 1 + client/network/common/src/sync.rs | 394 ++++++ .../{sync/src => common/src/sync}/message.rs | 2 +- client/network/common/src/sync/metrics.rs | 25 + client/network/common/src/sync/warp.rs | 94 ++ client/network/src/behaviour.rs | 68 +- client/network/src/bitswap.rs | 5 +- client/network/src/config.rs | 33 +- client/network/src/lib.rs | 8 +- client/network/src/protocol.rs | 310 ++--- client/network/src/protocol/message.rs | 10 +- client/network/src/service.rs | 27 +- client/network/src/service/tests.rs | 24 +- client/network/sync/Cargo.toml | 2 - .../network/sync/src/block_request_handler.rs | 6 +- client/network/sync/src/blocks.rs | 4 +- client/network/sync/src/extra_requests.rs | 11 +- client/network/sync/src/lib.rs | 1112 ++++++++--------- client/network/sync/src/schema.rs | 2 +- client/network/sync/src/state.rs | 10 +- client/network/sync/src/warp.rs | 45 +- .../network/sync/src/warp_request_handler.rs | 47 +- client/network/test/Cargo.toml | 2 + client/network/test/src/lib.rs | 110 +- client/service/Cargo.toml | 4 +- client/service/src/builder.rs | 28 +- 35 files changed, 1359 insertions(+), 1114 deletions(-) create mode 100644 client/network/common/src/sync.rs rename client/network/{sync/src => common/src/sync}/message.rs (99%) create mode 100644 client/network/common/src/sync/metrics.rs create mode 100644 client/network/common/src/sync/warp.rs diff --git a/Cargo.lock b/Cargo.lock index f98d98a3c571f..3b7c315be4390 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8429,6 +8429,7 @@ dependencies = [ "sc-consensus", "sc-keystore", "sc-network", + "sc-network-common", "sc-network-gossip", "sc-network-test", "sc-telemetry", @@ -8553,7 +8554,6 @@ dependencies = [ "sp-blockchain", "sp-consensus", "sp-core", - "sp-finality-grandpa", "sp-runtime", "sp-test-primitives", "sp-tracing", @@ -8571,12 +8571,17 @@ dependencies = [ name = "sc-network-common" version = "0.10.0-dev" dependencies = [ + "bitflags", "futures", "libp2p", "parity-scale-codec", "prost-build", + "sc-consensus", "sc-peerset", "smallvec", + "sp-consensus", + "sp-finality-grandpa", + "sp-runtime", ] [[package]] @@ -8621,8 +8626,6 @@ dependencies = [ name = "sc-network-sync" version = "0.10.0-dev" dependencies = [ - "bitflags", - "either", "fork-tree", "futures", "libp2p", @@ -8667,6 +8670,8 @@ dependencies = [ "sc-consensus", "sc-network", "sc-network-common", + "sc-network-light", + "sc-network-sync", "sc-service", "sp-blockchain", "sp-consensus", @@ -8849,6 +8854,8 @@ dependencies = [ "sc-keystore", "sc-network", "sc-network-common", + "sc-network-light", + "sc-network-sync", "sc-offchain", "sc-rpc", "sc-rpc-server", diff --git a/client/beefy/src/tests.rs b/client/beefy/src/tests.rs index b5ff27c808908..8090c425e71db 100644 --- a/client/beefy/src/tests.rs +++ b/client/beefy/src/tests.rs @@ -28,7 +28,6 @@ use sc_chain_spec::{ChainSpec, GenericChainSpec}; use sc_client_api::HeaderBackend; use sc_consensus::BoxJustificationImport; use sc_keystore::LocalKeystore; -use sc_network::config::ProtocolConfig; use sc_network_test::{ Block, BlockImportAdapter, FullPeerConfig, PassThroughVerifier, Peer, PeersClient, TestNetFactory, @@ -111,6 +110,7 @@ pub(crate) struct PeerData { pub(crate) beefy_link_half: Mutex>, } +#[derive(Default)] pub(crate) struct BeefyTestNet { peers: Vec, } @@ -166,17 +166,7 @@ impl TestNetFactory for BeefyTestNet { type BlockImport = PeersClient; type PeerData = PeerData; - /// Create new test network with peers and given config. - fn from_config(_config: &ProtocolConfig) -> Self { - BeefyTestNet { peers: Vec::new() } - } - - fn make_verifier( - &self, - _client: PeersClient, - _cfg: &ProtocolConfig, - _: &PeerData, - ) -> Self::Verifier { + fn make_verifier(&self, _client: PeersClient, _: &PeerData) -> Self::Verifier { PassThroughVerifier::new(false) // use non-instant finality. } diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs index ac3b89f2ff9a2..ee8be727dcdac 100644 --- a/client/consensus/aura/src/lib.rs +++ b/client/consensus/aura/src/lib.rs @@ -566,7 +566,6 @@ mod tests { use sc_consensus::BoxJustificationImport; use sc_consensus_slots::{BackoffAuthoringOnFinalizedHeadLagging, SimpleSlotWorker}; use sc_keystore::LocalKeystore; - use sc_network::config::ProtocolConfig; use sc_network_test::{Block as TestBlock, *}; use sp_application_crypto::key_types::AURA; use sp_consensus::{ @@ -645,6 +644,7 @@ mod tests { >; type AuraPeer = Peer<(), PeersClient>; + #[derive(Default)] pub struct AuraTestNet { peers: Vec, } @@ -654,17 +654,7 @@ mod tests { type PeerData = (); type BlockImport = PeersClient; - /// Create new test network with peers and given config. - fn from_config(_config: &ProtocolConfig) -> Self { - AuraTestNet { peers: Vec::new() } - } - - fn make_verifier( - &self, - client: PeersClient, - _cfg: &ProtocolConfig, - _peer_data: &(), - ) -> Self::Verifier { + fn make_verifier(&self, client: PeersClient, _peer_data: &()) -> Self::Verifier { let client = client.as_client(); let slot_duration = slot_duration(&*client).expect("slot duration available"); diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index e0590fc0cd86e..c0a7a8c6c013a 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -29,7 +29,6 @@ use sc_client_api::{backend::TransactionFor, BlockchainEvents, Finalizer}; use sc_consensus::{BoxBlockImport, BoxJustificationImport}; use sc_consensus_slots::BackoffAuthoringOnFinalizedHeadLagging; use sc_keystore::LocalKeystore; -use sc_network::config::ProtocolConfig; use sc_network_test::{Block as TestBlock, *}; use sp_application_crypto::key_types::BABE; use sp_consensus::{AlwaysCanAuthor, DisableProofRecording, NoNetwork as DummyOracle, Proposal}; @@ -220,6 +219,7 @@ where type BabePeer = Peer, BabeBlockImport>; +#[derive(Default)] pub struct BabeTestNet { peers: Vec, } @@ -278,12 +278,6 @@ impl TestNetFactory for BabeTestNet { type PeerData = Option; type BlockImport = BabeBlockImport; - /// Create new test network with peers and given config. - fn from_config(_config: &ProtocolConfig) -> Self { - debug!(target: "babe", "Creating test network from config"); - BabeTestNet { peers: Vec::new() } - } - fn make_block_import( &self, client: PeersClient, @@ -309,12 +303,7 @@ impl TestNetFactory for BabeTestNet { ) } - fn make_verifier( - &self, - client: PeersClient, - _cfg: &ProtocolConfig, - maybe_link: &Option, - ) -> Self::Verifier { + fn make_verifier(&self, client: PeersClient, maybe_link: &Option) -> Self::Verifier { use substrate_test_runtime_client::DefaultTestClientBuilderExt; let client = client.as_client(); diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index 77cd847d48169..a5f20b9f3261d 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -36,6 +36,7 @@ sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" } sc-keystore = { version = "4.0.0-dev", path = "../keystore" } sc-network = { version = "0.10.0-dev", path = "../network" } sc-network-gossip = { version = "0.10.0-dev", path = "../network-gossip" } +sc-network-common = { version = "0.10.0-dev", path = "../network/common" } sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" } sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 2d12232b04f15..623ac577c5579 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -28,7 +28,7 @@ use sc_consensus::{ BlockImport, BlockImportParams, BoxJustificationImport, ForkChoiceStrategy, ImportResult, ImportedAux, }; -use sc_network::config::{ProtocolConfig, Role}; +use sc_network::config::Role; use sc_network_test::{ Block, BlockImportAdapter, FullPeerConfig, Hash, PassThroughVerifier, Peer, PeersClient, PeersFullClient, TestClient, TestNetFactory, @@ -73,6 +73,7 @@ type GrandpaBlockImport = crate::GrandpaBlockImport< LongestChain, >; +#[derive(Default)] struct GrandpaTestNet { peers: Vec, test_config: TestApi, @@ -110,16 +111,6 @@ impl TestNetFactory for GrandpaTestNet { type PeerData = PeerData; type BlockImport = GrandpaBlockImport; - /// Create new test network with peers and given config. - fn from_config(_config: &ProtocolConfig) -> Self { - GrandpaTestNet { peers: Vec::new(), test_config: Default::default() } - } - - fn default_config() -> ProtocolConfig { - // This is unused. - ProtocolConfig::default() - } - fn add_full_peer(&mut self) { self.add_full_peer_with_config(FullPeerConfig { notifications_protocols: vec![grandpa_protocol_name::NAME.into()], @@ -128,12 +119,7 @@ impl TestNetFactory for GrandpaTestNet { }) } - fn make_verifier( - &self, - _client: PeersClient, - _cfg: &ProtocolConfig, - _: &PeerData, - ) -> Self::Verifier { + fn make_verifier(&self, _client: PeersClient, _: &PeerData) -> Self::Verifier { PassThroughVerifier::new(false) // use non-instant finality. } diff --git a/client/finality-grandpa/src/warp_proof.rs b/client/finality-grandpa/src/warp_proof.rs index 90f6828a1105d..a31a0a8b91908 100644 --- a/client/finality-grandpa/src/warp_proof.rs +++ b/client/finality-grandpa/src/warp_proof.rs @@ -23,7 +23,7 @@ use crate::{ BlockNumberOps, GrandpaJustification, SharedAuthoritySet, }; use sc_client_api::Backend as ClientBackend; -use sc_network::warp_request_handler::{EncodedProof, VerificationResult, WarpSyncProvider}; +use sc_network_common::sync::warp::{EncodedProof, VerificationResult, WarpSyncProvider}; use sp_blockchain::{Backend as BlockchainBackend, HeaderBackend}; use sp_finality_grandpa::{AuthorityList, SetId, GRANDPA_ENGINE_ID}; use sp_runtime::{ diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index dfd2db2e6b844..2742262b57e40 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -51,15 +51,12 @@ sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" } sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" } sc-network-common = { version = "0.10.0-dev", path = "./common" } -sc-network-light = { version = "0.10.0-dev", path = "./light" } -sc-network-sync = { version = "0.10.0-dev", path = "./sync" } sc-peerset = { version = "4.0.0-dev", path = "../peerset" } sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-arithmetic = { version = "5.0.0", path = "../../primitives/arithmetic" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } sp-core = { version = "6.0.0", path = "../../primitives/core" } -sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" } sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } [dev-dependencies] @@ -67,6 +64,8 @@ assert_matches = "1.3" async-std = "1.11.0" rand = "0.7.2" tempfile = "3.1.0" +sc-network-light = { version = "0.10.0-dev", path = "./light" } +sc-network-sync = { version = "0.10.0-dev", path = "./sync" } sp-test-primitives = { version = "2.0.0", path = "../../primitives/test-primitives" } sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } diff --git a/client/network/common/Cargo.toml b/client/network/common/Cargo.toml index e69787d7aff77..b0e3a8fe42a83 100644 --- a/client/network/common/Cargo.toml +++ b/client/network/common/Cargo.toml @@ -17,10 +17,15 @@ targets = ["x86_64-unknown-linux-gnu"] prost-build = "0.10" [dependencies] +bitflags = "1.3.2" codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive", ] } futures = "0.3.21" libp2p = "0.46.1" smallvec = "1.8.0" +sc-consensus = { version = "0.10.0-dev", path = "../../consensus/common" } sc-peerset = { version = "4.0.0-dev", path = "../../peerset" } +sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } +sp-finality-grandpa = { version = "4.0.0-dev", path = "../../../primitives/finality-grandpa" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } diff --git a/client/network/common/src/lib.rs b/client/network/common/src/lib.rs index 81769e23debbb..9fbedc542c123 100644 --- a/client/network/common/src/lib.rs +++ b/client/network/common/src/lib.rs @@ -21,3 +21,4 @@ pub mod config; pub mod message; pub mod request_responses; +pub mod sync; diff --git a/client/network/common/src/sync.rs b/client/network/common/src/sync.rs new file mode 100644 index 0000000000000..2ee8f8c51814b --- /dev/null +++ b/client/network/common/src/sync.rs @@ -0,0 +1,394 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Abstract interfaces and data structures related to network sync. + +pub mod message; +pub mod metrics; +pub mod warp; + +use libp2p::PeerId; +use message::{BlockAnnounce, BlockData, BlockRequest, BlockResponse}; +use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; +use sp_consensus::BlockOrigin; +use sp_runtime::{ + traits::{Block as BlockT, NumberFor}, + Justifications, +}; +use std::{any::Any, fmt, fmt::Formatter, task::Poll}; +use warp::{EncodedProof, WarpProofRequest, WarpSyncProgress}; + +/// The sync status of a peer we are trying to sync with +#[derive(Debug)] +pub struct PeerInfo { + /// Their best block hash. + pub best_hash: Block::Hash, + /// Their best block number. + pub best_number: NumberFor, +} + +/// Reported sync state. +#[derive(Clone, Eq, PartialEq, Debug)] +pub enum SyncState { + /// Initial sync is complete, keep-up sync is active. + Idle, + /// Actively catching up with the chain. + Downloading, +} + +/// Reported state download progress. +#[derive(Clone, Eq, PartialEq, Debug)] +pub struct StateDownloadProgress { + /// Estimated download percentage. + pub percentage: u32, + /// Total state size in bytes downloaded so far. + pub size: u64, +} + +/// Syncing status and statistics. +#[derive(Clone)] +pub struct SyncStatus { + /// Current global sync state. + pub state: SyncState, + /// Target sync block number. + pub best_seen_block: Option>, + /// Number of peers participating in syncing. + pub num_peers: u32, + /// Number of blocks queued for import + pub queued_blocks: u32, + /// State sync status in progress, if any. + pub state_sync: Option, + /// Warp sync in progress, if any. + pub warp_sync: Option>, +} + +/// A peer did not behave as expected and should be reported. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BadPeer(pub PeerId, pub sc_peerset::ReputationChange); + +impl fmt::Display for BadPeer { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Bad peer {}; Reputation change: {:?}", self.0, self.1) + } +} + +impl std::error::Error for BadPeer {} + +/// Result of [`ChainSync::on_block_data`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OnBlockData { + /// The block should be imported. + Import(BlockOrigin, Vec>), + /// A new block request needs to be made to the given peer. + Request(PeerId, BlockRequest), +} + +/// Result of [`ChainSync::on_block_justification`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OnBlockJustification { + /// The justification needs no further handling. + Nothing, + /// The justification should be imported. + Import { + peer: PeerId, + hash: Block::Hash, + number: NumberFor, + justifications: Justifications, + }, +} + +/// Result of [`ChainSync::on_state_data`]. +#[derive(Debug)] +pub enum OnStateData { + /// The block and state that should be imported. + Import(BlockOrigin, IncomingBlock), + /// A new state request needs to be made to the given peer. + Continue, +} + +/// Result of [`ChainSync::poll_block_announce_validation`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PollBlockAnnounceValidation { + /// The announcement failed at validation. + /// + /// The peer reputation should be decreased. + Failure { + /// Who sent the processed block announcement? + who: PeerId, + /// Should the peer be disconnected? + disconnect: bool, + }, + /// The announcement does not require further handling. + Nothing { + /// Who sent the processed block announcement? + who: PeerId, + /// Was this their new best block? + is_best: bool, + /// The announcement. + announce: BlockAnnounce, + }, + /// The announcement header should be imported. + ImportHeader { + /// Who sent the processed block announcement? + who: PeerId, + /// Was this their new best block? + is_best: bool, + /// The announcement. + announce: BlockAnnounce, + }, + /// The block announcement should be skipped. + Skip, +} + +/// Operation mode. +#[derive(Debug, PartialEq, Eq)] +pub enum SyncMode { + // Sync headers only + Light, + // Sync headers and block bodies + Full, + // Sync headers and the last finalied state + LightState { storage_chain_mode: bool, skip_proofs: bool }, + // Warp sync mode. + Warp, +} + +#[derive(Debug)] +pub struct Metrics { + pub queued_blocks: u32, + pub fork_targets: u32, + pub justifications: metrics::Metrics, +} + +/// Wrapper for implementation-specific state request. +/// +/// NOTE: Implementation must be able to encode and decode it for network purposes. +pub struct OpaqueStateRequest(pub Box); + +impl fmt::Debug for OpaqueStateRequest { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("OpaqueStateRequest").finish() + } +} + +/// Wrapper for implementation-specific state response. +/// +/// NOTE: Implementation must be able to encode and decode it for network purposes. +pub struct OpaqueStateResponse(pub Box); + +impl fmt::Debug for OpaqueStateResponse { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("OpaqueStateResponse").finish() + } +} + +/// Wrapper for implementation-specific block request. +/// +/// NOTE: Implementation must be able to encode and decode it for network purposes. +pub struct OpaqueBlockRequest(pub Box); + +impl fmt::Debug for OpaqueBlockRequest { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("OpaqueBlockRequest").finish() + } +} + +/// Wrapper for implementation-specific block response. +/// +/// NOTE: Implementation must be able to encode and decode it for network purposes. +pub struct OpaqueBlockResponse(pub Box); + +impl fmt::Debug for OpaqueBlockResponse { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("OpaqueBlockResponse").finish() + } +} + +/// Something that represents the syncing strategy to download past and future blocks of the chain. +pub trait ChainSync: Send { + /// Returns the state of the sync of the given peer. + /// + /// Returns `None` if the peer is unknown. + fn peer_info(&self, who: &PeerId) -> Option>; + + /// Returns the current sync status. + fn status(&self) -> SyncStatus; + + /// Number of active forks requests. This includes + /// requests that are pending or could be issued right away. + fn num_sync_requests(&self) -> usize; + + /// Number of downloaded blocks. + fn num_downloaded_blocks(&self) -> usize; + + /// Returns the current number of peers stored within this state machine. + fn num_peers(&self) -> usize; + + /// Handle a new connected peer. + /// + /// Call this method whenever we connect to a new peer. + fn new_peer( + &mut self, + who: PeerId, + best_hash: Block::Hash, + best_number: NumberFor, + ) -> Result>, BadPeer>; + + /// Signal that a new best block has been imported. + fn update_chain_info(&mut self, best_hash: &Block::Hash, best_number: NumberFor); + + /// Schedule a justification request for the given block. + fn request_justification(&mut self, hash: &Block::Hash, number: NumberFor); + + /// Clear all pending justification requests. + fn clear_justification_requests(&mut self); + + /// Request syncing for the given block from given set of peers. + fn set_sync_fork_request( + &mut self, + peers: Vec, + hash: &Block::Hash, + number: NumberFor, + ); + + /// Get an iterator over all scheduled justification requests. + fn justification_requests( + &mut self, + ) -> Box)> + '_>; + + /// Get an iterator over all block requests of all peers. + fn block_requests(&mut self) -> Box)> + '_>; + + /// Get a state request, if any. + fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)>; + + /// Get a warp sync request, if any. + fn warp_sync_request(&mut self) -> Option<(PeerId, WarpProofRequest)>; + + /// Handle a response from the remote to a block request that we made. + /// + /// `request` must be the original request that triggered `response`. + /// or `None` if data comes from the block announcement. + /// + /// If this corresponds to a valid block, this outputs the block that + /// must be imported in the import queue. + fn on_block_data( + &mut self, + who: &PeerId, + request: Option>, + response: BlockResponse, + ) -> Result, BadPeer>; + + /// Handle a response from the remote to a state request that we made. + fn on_state_data( + &mut self, + who: &PeerId, + response: OpaqueStateResponse, + ) -> Result, BadPeer>; + + /// Handle a response from the remote to a warp proof request that we made. + fn on_warp_sync_data(&mut self, who: &PeerId, response: EncodedProof) -> Result<(), BadPeer>; + + /// Handle a response from the remote to a justification request that we made. + /// + /// `request` must be the original request that triggered `response`. + fn on_block_justification( + &mut self, + who: PeerId, + response: BlockResponse, + ) -> Result, BadPeer>; + + /// A batch of blocks have been processed, with or without errors. + /// + /// Call this when a batch of blocks have been processed by the import + /// queue, with or without errors. + fn on_blocks_processed( + &mut self, + imported: usize, + count: usize, + results: Vec<(Result>, BlockImportError>, Block::Hash)>, + ) -> Box), BadPeer>>>; + + /// Call this when a justification has been processed by the import queue, + /// with or without errors. + fn on_justification_import( + &mut self, + hash: Block::Hash, + number: NumberFor, + success: bool, + ); + + /// Notify about finalization of the given block. + fn on_block_finalized(&mut self, hash: &Block::Hash, number: NumberFor); + + /// Push a block announce validation. + /// + /// It is required that [`ChainSync::poll_block_announce_validation`] is called + /// to check for finished block announce validations. + fn push_block_announce_validation( + &mut self, + who: PeerId, + hash: Block::Hash, + announce: BlockAnnounce, + is_best: bool, + ); + + /// Poll block announce validation. + /// + /// Block announce validations can be pushed by using + /// [`ChainSync::push_block_announce_validation`]. + /// + /// This should be polled until it returns [`Poll::Pending`]. + /// + /// If [`PollBlockAnnounceValidation::ImportHeader`] is returned, then the caller MUST try to + /// import passed header (call `on_block_data`). The network request isn't sent in this case. + fn poll_block_announce_validation( + &mut self, + cx: &mut std::task::Context, + ) -> Poll>; + + /// Call when a peer has disconnected. + /// Canceled obsolete block request may result in some blocks being ready for + /// import, so this functions checks for such blocks and returns them. + fn peer_disconnected(&mut self, who: &PeerId) -> Option>; + + /// Return some key metrics. + fn metrics(&self) -> Metrics; + + /// Create implementation-specific block request. + fn create_opaque_block_request(&self, request: &BlockRequest) -> OpaqueBlockRequest; + + /// Encode implementation-specific block request. + fn encode_block_request(&self, request: &OpaqueBlockRequest) -> Result, String>; + + /// Decode implementation-specific block response. + fn decode_block_response(&self, response: &[u8]) -> Result; + + /// Access blocks from implementation-specific block response. + fn block_response_into_blocks( + &self, + request: &BlockRequest, + response: OpaqueBlockResponse, + ) -> Result>, String>; + + /// Encode implementation-specific state request. + fn encode_state_request(&self, request: &OpaqueStateRequest) -> Result, String>; + + /// Decode implementation-specific state response. + fn decode_state_response(&self, response: &[u8]) -> Result; +} diff --git a/client/network/sync/src/message.rs b/client/network/common/src/sync/message.rs similarity index 99% rename from client/network/sync/src/message.rs rename to client/network/common/src/sync/message.rs index 996ee5231cf2e..27ab2704e6471 100644 --- a/client/network/sync/src/message.rs +++ b/client/network/common/src/sync/message.rs @@ -124,8 +124,8 @@ impl BlockAnnounce { /// Generic types. pub mod generic { use super::{BlockAttributes, BlockState, Direction}; + use crate::message::RequestId; use codec::{Decode, Encode, Input, Output}; - use sc_network_common::message::RequestId; use sp_runtime::{EncodedJustification, Justifications}; /// Block data sent in the response. diff --git a/client/network/common/src/sync/metrics.rs b/client/network/common/src/sync/metrics.rs new file mode 100644 index 0000000000000..15ff090a8ccac --- /dev/null +++ b/client/network/common/src/sync/metrics.rs @@ -0,0 +1,25 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#[derive(Debug)] +pub struct Metrics { + pub pending_requests: u32, + pub active_requests: u32, + pub importing_requests: u32, + pub failed_requests: u32, +} diff --git a/client/network/common/src/sync/warp.rs b/client/network/common/src/sync/warp.rs new file mode 100644 index 0000000000000..339a4c33a7eeb --- /dev/null +++ b/client/network/common/src/sync/warp.rs @@ -0,0 +1,94 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use codec::{Decode, Encode}; +pub use sp_finality_grandpa::{AuthorityList, SetId}; +use sp_runtime::traits::{Block as BlockT, NumberFor}; +use std::fmt; + +/// Scale-encoded warp sync proof response. +pub struct EncodedProof(pub Vec); + +/// Warp sync request +#[derive(Encode, Decode, Debug)] +pub struct WarpProofRequest { + /// Start collecting proofs from this block. + pub begin: B::Hash, +} + +/// Proof verification result. +pub enum VerificationResult { + /// Proof is valid, but the target was not reached. + Partial(SetId, AuthorityList, Block::Hash), + /// Target finality is proved. + Complete(SetId, AuthorityList, Block::Header), +} + +/// Warp sync backend. Handles retrieveing and verifying warp sync proofs. +pub trait WarpSyncProvider: Send + Sync { + /// Generate proof starting at given block hash. The proof is accumulated until maximum proof + /// size is reached. + fn generate( + &self, + start: Block::Hash, + ) -> Result>; + /// Verify warp proof against current set of authorities. + fn verify( + &self, + proof: &EncodedProof, + set_id: SetId, + authorities: AuthorityList, + ) -> Result, Box>; + /// Get current list of authorities. This is supposed to be genesis authorities when starting + /// sync. + fn current_authorities(&self) -> AuthorityList; +} + +/// Reported warp sync phase. +#[derive(Clone, Eq, PartialEq, Debug)] +pub enum WarpSyncPhase { + /// Waiting for peers to connect. + AwaitingPeers, + /// Downloading and verifying grandpa warp proofs. + DownloadingWarpProofs, + /// Downloading state data. + DownloadingState, + /// Importing state. + ImportingState, + /// Downloading block history. + DownloadingBlocks(NumberFor), +} + +impl fmt::Display for WarpSyncPhase { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::AwaitingPeers => write!(f, "Waiting for peers"), + Self::DownloadingWarpProofs => write!(f, "Downloading finality proofs"), + Self::DownloadingState => write!(f, "Downloading state"), + Self::ImportingState => write!(f, "Importing state"), + Self::DownloadingBlocks(n) => write!(f, "Downloading block history (#{})", n), + } + } +} + +/// Reported warp sync progress. +#[derive(Clone, Eq, PartialEq, Debug)] +pub struct WarpSyncProgress { + /// Estimated download percentage. + pub phase: WarpSyncPhase, + /// Total bytes downloaded so far. + pub total_bytes: u64, +} diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 091dd116e4c9c..515608df13d0f 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -38,7 +38,7 @@ use libp2p::{ NetworkBehaviour, }; use log::debug; -use prost::Message; + use sc_client_api::{BlockBackend, ProofProvider}; use sc_consensus::import_queue::{IncomingBlock, Origin}; use sc_network_common::{config::ProtocolId, request_responses::ProtocolConfig}; @@ -382,42 +382,44 @@ where .events .push_back(BehaviourOut::JustificationImport(origin, hash, nb, justification)), CustomMessageOutcome::BlockRequest { target, request, pending_response } => { - let mut buf = Vec::with_capacity(request.encoded_len()); - if let Err(err) = request.encode(&mut buf) { - log::warn!( - target: "sync", - "Failed to encode block request {:?}: {:?}", - request, err - ); - return + match self.substrate.encode_block_request(&request) { + Ok(data) => { + self.request_responses.send_request( + &target, + &self.block_request_protocol_name, + data, + pending_response, + IfDisconnected::ImmediateError, + ); + }, + Err(err) => { + log::warn!( + target: "sync", + "Failed to encode block request {:?}: {:?}", + request, err + ); + }, } - - self.request_responses.send_request( - &target, - &self.block_request_protocol_name, - buf, - pending_response, - IfDisconnected::ImmediateError, - ); }, CustomMessageOutcome::StateRequest { target, request, pending_response } => { - let mut buf = Vec::with_capacity(request.encoded_len()); - if let Err(err) = request.encode(&mut buf) { - log::warn!( - target: "sync", - "Failed to encode state request {:?}: {:?}", - request, err - ); - return + match self.substrate.encode_state_request(&request) { + Ok(data) => { + self.request_responses.send_request( + &target, + &self.state_request_protocol_name, + data, + pending_response, + IfDisconnected::ImmediateError, + ); + }, + Err(err) => { + log::warn!( + target: "sync", + "Failed to encode state request {:?}: {:?}", + request, err + ); + }, } - - self.request_responses.send_request( - &target, - &self.state_request_protocol_name, - buf, - pending_response, - IfDisconnected::ImmediateError, - ); }, CustomMessageOutcome::WarpSyncRequest { target, request, pending_response } => match &self.warp_sync_protocol_name { diff --git a/client/network/src/bitswap.rs b/client/network/src/bitswap.rs index d5039faaca113..2dab45adc5618 100644 --- a/client/network/src/bitswap.rs +++ b/client/network/src/bitswap.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. +// Copyright 2022 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify @@ -118,8 +118,7 @@ where fn upgrade_outbound(self, mut socket: TSocket, _info: Self::Info) -> Self::Future { Box::pin(async move { - let mut data = Vec::with_capacity(self.encoded_len()); - self.encode(&mut data)?; + let data = self.encode_to_vec(); upgrade::write_length_prefixed(&mut socket, data).await }) } diff --git a/client/network/src/config.rs b/client/network/src/config.rs index e44977e5be6b3..430efd697a18c 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -26,16 +26,11 @@ pub use sc_network_common::{ request_responses::{ IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig, }, + sync::warp::WarpSyncProvider, }; -pub use sc_network_sync::warp_request_handler::WarpSyncProvider; pub use libp2p::{build_multiaddr, core::PublicKey, identity}; -// Note: this re-export shouldn't be part of the public API of the crate and will be removed in -// the future. -#[doc(hidden)] -pub use crate::protocol::ProtocolConfig; - use crate::ExHashT; use core::{fmt, iter}; @@ -46,7 +41,7 @@ use libp2p::{ }; use prometheus_endpoint::Registry; use sc_consensus::ImportQueue; -use sp_consensus::block_validation::BlockAnnounceValidator; +use sc_network_common::sync::ChainSync; use sp_runtime::traits::Block as BlockT; use std::{ borrow::Cow, @@ -101,8 +96,14 @@ where /// valid. pub import_queue: Box>, - /// Type to check incoming block announcements. - pub block_announce_validator: Box + Send>, + /// Factory function that creates a new instance of chain sync. + pub create_chain_sync: Box< + dyn FnOnce( + sc_network_common::sync::SyncMode, + Arc, + Option>>, + ) -> crate::error::Result>>, + >, /// Registry for recording prometheus metrics to. pub metrics_registry: Option, @@ -114,26 +115,26 @@ where /// block requests, if enabled. /// /// Can be constructed either via - /// [`sc_network_sync::block_request_handler::generate_protocol_config`] allowing outgoing but - /// not incoming requests, or constructed via [`sc_network_sync::block_request_handler:: - /// BlockRequestHandler::new`] allowing both outgoing and incoming requests. + /// `sc_network_sync::block_request_handler::generate_protocol_config` allowing outgoing but + /// not incoming requests, or constructed via `sc_network_sync::block_request_handler:: + /// BlockRequestHandler::new` allowing both outgoing and incoming requests. pub block_request_protocol_config: RequestResponseConfig, /// Request response configuration for the light client request protocol. /// /// Can be constructed either via - /// [`sc_network_light::light_client_requests::generate_protocol_config`] allowing outgoing but + /// `sc_network_light::light_client_requests::generate_protocol_config` allowing outgoing but /// not incoming requests, or constructed via - /// [`sc_network_light::light_client_requests::handler::LightClientRequestHandler::new`] + /// `sc_network_light::light_client_requests::handler::LightClientRequestHandler::new` /// allowing both outgoing and incoming requests. pub light_client_request_protocol_config: RequestResponseConfig, /// Request response configuration for the state request protocol. /// /// Can be constructed either via - /// [`sc_network_sync::block_request_handler::generate_protocol_config`] allowing outgoing but + /// `sc_network_sync::state_request_handler::generate_protocol_config` allowing outgoing but /// not incoming requests, or constructed via - /// [`crate::state_request_handler::StateRequestHandler::new`] allowing + /// `sc_network_sync::state_request_handler::StateRequestHandler::new` allowing /// both outgoing and incoming requests. pub state_request_protocol_config: RequestResponseConfig, diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index fff30550eb8c9..83bc1075b8bad 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -266,13 +266,9 @@ pub use protocol::{ event::{DhtEvent, Event, ObservedRole}, PeerInfo, }; -pub use sc_network_light::light_client_requests; -pub use sc_network_sync::{ - block_request_handler, - state::StateDownloadProgress, - state_request_handler, +pub use sc_network_common::sync::{ warp::{WarpSyncPhase, WarpSyncProgress}, - warp_request_handler, SyncState, + StateDownloadProgress, SyncState, }; pub use service::{ DecodingError, IfDisconnected, KademliaKey, Keypair, NetworkService, NetworkWorker, diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 348d2d0bf8877..3698a6b936ed5 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -20,7 +20,6 @@ use crate::{ config, error, request_responses::RequestFailure, utils::{interval, LruHashSet}, - warp_request_handler::{EncodedProof, WarpSyncProvider}, }; use bytes::Bytes; @@ -42,21 +41,23 @@ use message::{ }; use notifications::{Notifications, NotificationsOut}; use prometheus_endpoint::{register, Gauge, GaugeVec, Opts, PrometheusError, Registry, U64}; -use prost::Message as _; use sc_client_api::{BlockBackend, HeaderBackend, ProofProvider}; use sc_consensus::import_queue::{BlockImportError, BlockImportStatus, IncomingBlock, Origin}; -use sc_network_common::config::ProtocolId; -use sc_network_sync::{ - message::{ - BlockAnnounce, BlockAttributes, BlockData, BlockRequest, BlockResponse, BlockState, - FromBlock, +use sc_network_common::{ + config::ProtocolId, + sync::{ + message::{ + BlockAnnounce, BlockAttributes, BlockData, BlockRequest, BlockResponse, BlockState, + }, + warp::{EncodedProof, WarpProofRequest}, + BadPeer, ChainSync, OnBlockData, OnBlockJustification, OnStateData, OpaqueBlockRequest, + OpaqueBlockResponse, OpaqueStateRequest, OpaqueStateResponse, PollBlockAnnounceValidation, + SyncStatus, }, - schema::v1::StateResponse, - BadPeer, ChainSync, OnBlockData, OnBlockJustification, OnStateData, - PollBlockAnnounceValidation, Status as SyncStatus, }; use sp_arithmetic::traits::SaturatedConversion; -use sp_consensus::{block_validation::BlockAnnounceValidator, BlockOrigin}; +use sp_blockchain::HeaderMetadata; +use sp_consensus::BlockOrigin; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, CheckedSub, Header as HeaderT, NumberFor, Zero}, @@ -79,7 +80,6 @@ pub mod event; pub mod message; pub use notifications::{NotificationsSink, NotifsHandlerError, Ready}; -use sp_blockchain::HeaderMetadata; /// Interval at which we perform time based maintenance const TICK_TIMEOUT: time::Duration = time::Duration::from_millis(1100); @@ -167,11 +167,12 @@ pub struct Protocol { tick_timeout: Pin + Send>>, /// Pending list of messages to return from `poll` as a priority. pending_messages: VecDeque>, - config: ProtocolConfig, + /// Assigned roles. + roles: Roles, genesis_hash: B::Hash, /// State machine that handles the list of in-progress requests. Only full node peers are /// registered. - sync: ChainSync, + chain_sync: Box>, // All connected peers. Contains both full and light node peers. peers: HashMap>, chain: Arc, @@ -231,38 +232,6 @@ pub struct PeerInfo { pub best_number: ::Number, } -/// Configuration for the Substrate-specific part of the networking layer. -#[derive(Clone)] -pub struct ProtocolConfig { - /// Assigned roles. - pub roles: Roles, - /// Maximum number of peers to ask the same blocks in parallel. - pub max_parallel_downloads: u32, - /// Enable state sync. - pub sync_mode: config::SyncMode, -} - -impl ProtocolConfig { - fn sync_mode(&self) -> sc_network_sync::SyncMode { - if self.roles.is_light() { - sc_network_sync::SyncMode::Light - } else { - match self.sync_mode { - config::SyncMode::Full => sc_network_sync::SyncMode::Full, - config::SyncMode::Fast { skip_proofs, storage_chain_mode } => - sc_network_sync::SyncMode::LightState { skip_proofs, storage_chain_mode }, - config::SyncMode::Warp => sc_network_sync::SyncMode::Warp, - } - } - } -} - -impl Default for ProtocolConfig { - fn default() -> ProtocolConfig { - Self { roles: Roles::FULL, max_parallel_downloads: 5, sync_mode: config::SyncMode::Full } - } -} - /// Handshake sent when we open a block announces substream. #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] struct BlockAnnouncesHandshake { @@ -278,12 +247,12 @@ struct BlockAnnouncesHandshake { impl BlockAnnouncesHandshake { fn build( - protocol_config: &ProtocolConfig, + roles: Roles, best_number: NumberFor, best_hash: B::Hash, genesis_hash: B::Hash, ) -> Self { - Self { genesis_hash, roles: protocol_config.roles, best_number, best_hash } + Self { genesis_hash, roles, best_number, best_hash } } } @@ -300,24 +269,15 @@ where { /// Create a new instance. pub fn new( - config: ProtocolConfig, + roles: Roles, chain: Arc, protocol_id: ProtocolId, network_config: &config::NetworkConfiguration, notifications_protocols_handshakes: Vec>, - block_announce_validator: Box + Send>, metrics_registry: Option<&Registry>, - warp_sync_provider: Option>>, - ) -> error::Result<(Protocol, sc_peerset::PeersetHandle, Vec<(PeerId, Multiaddr)>)> { + chain_sync: Box>, + ) -> error::Result<(Self, sc_peerset::PeersetHandle, Vec<(PeerId, Multiaddr)>)> { let info = chain.info(); - let sync = ChainSync::new( - config.sync_mode(), - chain.clone(), - block_announce_validator, - config.max_parallel_downloads, - warp_sync_provider, - ) - .map_err(Box::new)?; let boot_node_ids = { let mut list = HashSet::new(); @@ -405,7 +365,7 @@ where let genesis_hash = info.genesis_hash; let block_announces_handshake = - BlockAnnouncesHandshake::::build(&config, best_number, best_hash, genesis_hash) + BlockAnnouncesHandshake::::build(roles, best_number, best_hash, genesis_hash) .encode(); let sync_protocol_config = notifications::ProtocolConfig { @@ -438,11 +398,11 @@ where let protocol = Self { tick_timeout: Box::pin(interval(TICK_TIMEOUT)), pending_messages: VecDeque::new(), - config, + roles, peers: HashMap::new(), chain, genesis_hash: info.genesis_hash, - sync, + chain_sync, important_peers, default_peers_set_num_full: network_config.default_peers_set_num_full as usize, default_peers_set_num_light: { @@ -510,49 +470,49 @@ where /// Current global sync state. pub fn sync_state(&self) -> SyncStatus { - self.sync.status() + self.chain_sync.status() } /// Target sync block number. pub fn best_seen_block(&self) -> Option> { - self.sync.status().best_seen_block + self.chain_sync.status().best_seen_block } /// Number of peers participating in syncing. pub fn num_sync_peers(&self) -> u32 { - self.sync.status().num_peers + self.chain_sync.status().num_peers } /// Number of blocks in the import queue. pub fn num_queued_blocks(&self) -> u32 { - self.sync.status().queued_blocks + self.chain_sync.status().queued_blocks } /// Number of downloaded blocks. pub fn num_downloaded_blocks(&self) -> usize { - self.sync.num_downloaded_blocks() + self.chain_sync.num_downloaded_blocks() } /// Number of active sync requests. pub fn num_sync_requests(&self) -> usize { - self.sync.num_sync_requests() + self.chain_sync.num_sync_requests() } /// Inform sync about new best imported block. pub fn new_best_block_imported(&mut self, hash: B::Hash, number: NumberFor) { debug!(target: "sync", "New best block imported {:?}/#{}", hash, number); - self.sync.update_chain_info(&hash, number); + self.chain_sync.update_chain_info(&hash, number); self.behaviour.set_notif_protocol_handshake( HARDCODED_PEERSETS_SYNC, - BlockAnnouncesHandshake::::build(&self.config, number, hash, self.genesis_hash) + BlockAnnouncesHandshake::::build(self.roles, number, hash, self.genesis_hash) .encode(), ); } fn update_peer_info(&mut self, who: &PeerId) { - if let Some(info) = self.sync.peer_info(who) { + if let Some(info) = self.chain_sync.peer_info(who) { if let Some(ref mut peer) = self.peers.get_mut(who) { peer.info.best_hash = info.best_hash; peer.info.best_number = info.best_number; @@ -565,14 +525,6 @@ where self.peers.iter().map(|(id, peer)| (id, &peer.info)) } - fn prepare_block_request( - &mut self, - who: PeerId, - request: BlockRequest, - ) -> CustomMessageOutcome { - prepare_block_request::(&mut self.peers, who, request) - } - /// Called by peer when it is disconnecting. /// /// Returns a result if the handshake of this peer was indeed accepted. @@ -584,7 +536,9 @@ where } if let Some(_peer_data) = self.peers.remove(&peer) { - if let Some(OnBlockData::Import(origin, blocks)) = self.sync.peer_disconnected(&peer) { + if let Some(OnBlockData::Import(origin, blocks)) = + self.chain_sync.peer_disconnected(&peer) + { self.pending_messages .push_back(CustomMessageOutcome::BlockImport(origin, blocks)); } @@ -605,62 +559,9 @@ where &mut self, peer_id: PeerId, request: BlockRequest, - response: sc_network_sync::schema::v1::BlockResponse, + response: OpaqueBlockResponse, ) -> CustomMessageOutcome { - let blocks = response - .blocks - .into_iter() - .map(|block_data| { - Ok(BlockData:: { - hash: Decode::decode(&mut block_data.hash.as_ref())?, - header: if !block_data.header.is_empty() { - Some(Decode::decode(&mut block_data.header.as_ref())?) - } else { - None - }, - body: if request.fields.contains(BlockAttributes::BODY) { - Some( - block_data - .body - .iter() - .map(|body| Decode::decode(&mut body.as_ref())) - .collect::, _>>()?, - ) - } else { - None - }, - indexed_body: if request.fields.contains(BlockAttributes::INDEXED_BODY) { - Some(block_data.indexed_body) - } else { - None - }, - receipt: if !block_data.receipt.is_empty() { - Some(block_data.receipt) - } else { - None - }, - message_queue: if !block_data.message_queue.is_empty() { - Some(block_data.message_queue) - } else { - None - }, - justification: if !block_data.justification.is_empty() { - Some(block_data.justification) - } else if block_data.is_empty_justification { - Some(Vec::new()) - } else { - None - }, - justifications: if !block_data.justifications.is_empty() { - Some(DecodeAll::decode_all(&mut block_data.justifications.as_ref())?) - } else { - None - }, - }) - }) - .collect::, codec::Error>>(); - - let blocks = match blocks { + let blocks = match self.chain_sync.block_response_into_blocks(&request, response) { Ok(blocks) => blocks, Err(err) => { debug!(target: "sync", "Failed to decode block response from {}: {}", peer_id, err); @@ -690,7 +591,7 @@ where ); if request.fields == BlockAttributes::JUSTIFICATION { - match self.sync.on_block_justification(peer_id, block_response) { + match self.chain_sync.on_block_justification(peer_id, block_response) { Ok(OnBlockJustification::Nothing) => CustomMessageOutcome::None, Ok(OnBlockJustification::Import { peer, hash, number, justifications }) => CustomMessageOutcome::JustificationImport(peer, hash, number, justifications), @@ -701,10 +602,11 @@ where }, } } else { - match self.sync.on_block_data(&peer_id, Some(request), block_response) { + match self.chain_sync.on_block_data(&peer_id, Some(request), block_response) { Ok(OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), - Ok(OnBlockData::Request(peer, req)) => self.prepare_block_request(peer, req), + Ok(OnBlockData::Request(peer, req)) => + prepare_block_request(self.chain_sync.as_ref(), &mut self.peers, peer, req), Err(BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC); self.peerset_handle.report_peer(id, repu); @@ -719,9 +621,9 @@ where pub fn on_state_response( &mut self, peer_id: PeerId, - response: StateResponse, + response: OpaqueStateResponse, ) -> CustomMessageOutcome { - match self.sync.on_state_data(&peer_id, response) { + match self.chain_sync.on_state_data(&peer_id, response) { Ok(OnStateData::Import(origin, block)) => CustomMessageOutcome::BlockImport(origin, vec![block]), Ok(OnStateData::Continue) => CustomMessageOutcome::None, @@ -738,9 +640,9 @@ where pub fn on_warp_sync_response( &mut self, peer_id: PeerId, - response: crate::warp_request_handler::EncodedProof, + response: EncodedProof, ) -> CustomMessageOutcome { - match self.sync.on_warp_sync_data(&peer_id, response) { + match self.chain_sync.on_warp_sync_data(&peer_id, response) { Ok(()) => CustomMessageOutcome::None, Err(BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC); @@ -798,7 +700,7 @@ where return Err(()) } - if self.config.roles.is_light() { + if self.roles.is_light() { // we're not interested in light peers if status.roles.is_light() { debug!(target: "sync", "Peer {} is unable to serve light requests", who); @@ -821,14 +723,15 @@ where } } - if status.roles.is_full() && self.sync.num_peers() >= self.default_peers_set_num_full { + if status.roles.is_full() && self.chain_sync.num_peers() >= self.default_peers_set_num_full + { debug!(target: "sync", "Too many full nodes, rejecting {}", who); self.behaviour.disconnect_peer(&who, HARDCODED_PEERSETS_SYNC); return Err(()) } if status.roles.is_light() && - (self.peers.len() - self.sync.num_peers()) >= self.default_peers_set_num_light + (self.peers.len() - self.chain_sync.num_peers()) >= self.default_peers_set_num_light { // Make sure that not all slots are occupied by light clients. debug!(target: "sync", "Too many light nodes, rejecting {}", who); @@ -849,7 +752,7 @@ where }; let req = if peer.info.roles.is_full() { - match self.sync.new_peer(who, peer.info.best_hash, peer.info.best_number) { + match self.chain_sync.new_peer(who, peer.info.best_hash, peer.info.best_number) { Ok(req) => req, Err(BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC); @@ -868,8 +771,12 @@ where .push_back(CustomMessageOutcome::PeerNewBest(who, status.best_number)); if let Some(req) = req { - let event = self.prepare_block_request(who, req); - self.pending_messages.push_back(event); + self.pending_messages.push_back(prepare_block_request( + self.chain_sync.as_ref(), + &mut self.peers, + who, + req, + )); } Ok(()) @@ -953,7 +860,7 @@ where }; if peer.info.roles.is_full() { - self.sync.push_block_announce_validation(who, hash, announce, is_best); + self.chain_sync.push_block_announce_validation(who, hash, announce, is_best); } } @@ -1010,7 +917,7 @@ where // to import header from announced block let's construct response to request that normally // would have been sent over network (but it is not in our case) - let blocks_to_import = self.sync.on_block_data( + let blocks_to_import = self.chain_sync.on_block_data( &who, None, BlockResponse:: { @@ -1035,7 +942,8 @@ where match blocks_to_import { Ok(OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), - Ok(OnBlockData::Request(peer, req)) => self.prepare_block_request(peer, req), + Ok(OnBlockData::Request(peer, req)) => + prepare_block_request(self.chain_sync.as_ref(), &mut self.peers, peer, req), Err(BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC); self.peerset_handle.report_peer(id, repu); @@ -1047,7 +955,7 @@ where /// Call this when a block has been finalized. The sync layer may have some additional /// requesting to perform. pub fn on_block_finalized(&mut self, hash: B::Hash, header: &B::Header) { - self.sync.on_block_finalized(&hash, *header.number()) + self.chain_sync.on_block_finalized(&hash, *header.number()) } /// Request a justification for the given block. @@ -1055,12 +963,12 @@ where /// Uses `protocol` to queue a new justification request and tries to dispatch all pending /// requests. pub fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { - self.sync.request_justification(hash, number) + self.chain_sync.request_justification(hash, number) } /// Clear all pending justification requests. pub fn clear_justification_requests(&mut self) { - self.sync.clear_justification_requests(); + self.chain_sync.clear_justification_requests(); } /// Request syncing for the given block from given set of peers. @@ -1072,7 +980,7 @@ where hash: &B::Hash, number: NumberFor, ) { - self.sync.set_sync_fork_request(peers, hash, number) + self.chain_sync.set_sync_fork_request(peers, hash, number) } /// A batch of blocks have been processed, with or without errors. @@ -1084,11 +992,12 @@ where count: usize, results: Vec<(Result>, BlockImportError>, B::Hash)>, ) { - let results = self.sync.on_blocks_processed(imported, count, results); + let results = self.chain_sync.on_blocks_processed(imported, count, results); for result in results { match result { Ok((id, req)) => { self.pending_messages.push_back(prepare_block_request( + self.chain_sync.as_ref(), &mut self.peers, id, req, @@ -1111,7 +1020,7 @@ where number: NumberFor, success: bool, ) { - self.sync.on_justification_import(hash, number, success); + self.chain_sync.on_justification_import(hash, number, success); if !success { info!("💔 Invalid justification provided by {} for #{}", who, hash); self.behaviour.disconnect_peer(&who, HARDCODED_PEERSETS_SYNC); @@ -1228,12 +1137,22 @@ where } } + /// Encode implementation-specific block request. + pub fn encode_block_request(&self, request: &OpaqueBlockRequest) -> Result, String> { + self.chain_sync.encode_block_request(request) + } + + /// Encode implementation-specific state request. + pub fn encode_state_request(&self, request: &OpaqueStateRequest) -> Result, String> { + self.chain_sync.encode_state_request(request) + } + fn report_metrics(&self) { if let Some(metrics) = &self.metrics { let n = u64::try_from(self.peers.len()).unwrap_or(std::u64::MAX); metrics.peers.set(n); - let m = self.sync.metrics(); + let m = self.chain_sync.metrics(); metrics.fork_targets.set(m.fork_targets.into()); metrics.queued_blocks.set(m.queued_blocks.into()); @@ -1259,6 +1178,7 @@ where } fn prepare_block_request( + chain_sync: &dyn ChainSync, peers: &mut HashMap>, who: PeerId, request: BlockRequest, @@ -1269,19 +1189,7 @@ fn prepare_block_request( peer.request = Some((PeerRequest::Block(request.clone()), rx)); } - let request = sc_network_sync::schema::v1::BlockRequest { - fields: request.fields.to_be_u32(), - from_block: match request.from { - FromBlock::Hash(h) => - Some(sc_network_sync::schema::v1::block_request::FromBlock::Hash(h.encode())), - FromBlock::Number(n) => - Some(sc_network_sync::schema::v1::block_request::FromBlock::Number(n.encode())), - }, - to_block: request.to.map(|h| h.encode()).unwrap_or_default(), - direction: request.direction as i32, - max_blocks: request.max.unwrap_or(0), - support_multiple_justifications: true, - }; + let request = chain_sync.create_opaque_block_request(&request); CustomMessageOutcome::BlockRequest { target: who, request, pending_response: tx } } @@ -1289,7 +1197,7 @@ fn prepare_block_request( fn prepare_state_request( peers: &mut HashMap>, who: PeerId, - request: sc_network_sync::schema::v1::StateRequest, + request: OpaqueStateRequest, ) -> CustomMessageOutcome { let (tx, rx) = oneshot::channel(); @@ -1302,7 +1210,7 @@ fn prepare_state_request( fn prepare_warp_sync_request( peers: &mut HashMap>, who: PeerId, - request: crate::warp_request_handler::Request, + request: WarpProofRequest, ) -> CustomMessageOutcome { let (tx, rx) = oneshot::channel(); @@ -1346,19 +1254,19 @@ pub enum CustomMessageOutcome { /// A new block request must be emitted. BlockRequest { target: PeerId, - request: sc_network_sync::schema::v1::BlockRequest, + request: OpaqueBlockRequest, pending_response: oneshot::Sender, RequestFailure>>, }, /// A new storage request must be emitted. StateRequest { target: PeerId, - request: sc_network_sync::schema::v1::StateRequest, + request: OpaqueStateRequest, pending_response: oneshot::Sender, RequestFailure>>, }, /// A new warp sync request must be emitted. WarpSyncRequest { target: PeerId, - request: crate::warp_request_handler::Request, + request: WarpProofRequest, pending_response: oneshot::Sender, RequestFailure>>, }, /// Peer has a reported a new head of chain. @@ -1455,10 +1363,8 @@ where let (req, _) = peer.request.take().unwrap(); match req { PeerRequest::Block(req) => { - let protobuf_response = - match sc_network_sync::schema::v1::BlockResponse::decode( - &resp[..], - ) { + let response = + match self.chain_sync.decode_block_response(&resp[..]) { Ok(proto) => proto, Err(e) => { debug!( @@ -1474,13 +1380,11 @@ where }, }; - finished_block_requests.push((*id, req, protobuf_response)); + finished_block_requests.push((*id, req, response)); }, PeerRequest::State => { - let protobuf_response = - match sc_network_sync::schema::v1::StateResponse::decode( - &resp[..], - ) { + let response = + match self.chain_sync.decode_state_response(&resp[..]) { Ok(proto) => proto, Err(e) => { debug!( @@ -1496,7 +1400,7 @@ where }, }; - finished_state_requests.push((*id, protobuf_response)); + finished_state_requests.push((*id, response)); }, PeerRequest::WarpProof => { finished_warp_sync_requests.push((*id, resp)); @@ -1555,12 +1459,12 @@ where } } } - for (id, req, protobuf_response) in finished_block_requests { - let ev = self.on_block_response(id, req, protobuf_response); + for (id, req, response) in finished_block_requests { + let ev = self.on_block_response(id, req, response); self.pending_messages.push_back(ev); } - for (id, protobuf_response) in finished_state_requests { - let ev = self.on_state_response(id, protobuf_response); + for (id, response) in finished_state_requests { + let ev = self.on_state_response(id, response); self.pending_messages.push_back(ev); } for (id, response) in finished_warp_sync_requests { @@ -1572,25 +1476,32 @@ where self.tick(); } - for (id, request) in self.sync.block_requests() { - let event = prepare_block_request(&mut self.peers, *id, request); + for (id, request) in self + .chain_sync + .block_requests() + .map(|(peer_id, request)| (*peer_id, request)) + .collect::>() + { + let event = + prepare_block_request(self.chain_sync.as_ref(), &mut self.peers, id, request); self.pending_messages.push_back(event); } - if let Some((id, request)) = self.sync.state_request() { + if let Some((id, request)) = self.chain_sync.state_request() { let event = prepare_state_request(&mut self.peers, id, request); self.pending_messages.push_back(event); } - for (id, request) in self.sync.justification_requests() { - let event = prepare_block_request(&mut self.peers, id, request); + for (id, request) in self.chain_sync.justification_requests().collect::>() { + let event = + prepare_block_request(self.chain_sync.as_ref(), &mut self.peers, id, request); self.pending_messages.push_back(event); } - if let Some((id, request)) = self.sync.warp_sync_request() { + if let Some((id, request)) = self.chain_sync.warp_sync_request() { let event = prepare_warp_sync_request(&mut self.peers, id, request); self.pending_messages.push_back(event); } // Check if there is any block announcement validation finished. - while let Poll::Ready(result) = self.sync.poll_block_announce_validation(cx) { + while let Poll::Ready(result) = self.chain_sync.poll_block_announce_validation(cx) { match self.process_block_announce_validation_result(result) { CustomMessageOutcome::None => {}, outcome => self.pending_messages.push_back(outcome), @@ -1771,7 +1682,8 @@ where // Make sure that the newly added block announce validation future was // polled once to be registered in the task. - if let Poll::Ready(res) = self.sync.poll_block_announce_validation(cx) { + if let Poll::Ready(res) = self.chain_sync.poll_block_announce_validation(cx) + { self.process_block_announce_validation_result(res) } else { CustomMessageOutcome::None diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index a57740ec2746b..c9512f82e23bb 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -63,10 +63,12 @@ pub mod generic { use bitflags::bitflags; use codec::{Decode, Encode, Input, Output}; use sc_client_api::StorageProof; - use sc_network_common::message::RequestId; - use sc_network_sync::message::{ - generic::{BlockRequest, BlockResponse}, - BlockAnnounce, + use sc_network_common::{ + message::RequestId, + sync::message::{ + generic::{BlockRequest, BlockResponse}, + BlockAnnounce, + }, }; use sp_runtime::ConsensusEngineId; diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 1cc717a50d039..ef7ef2f5a2deb 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -30,7 +30,7 @@ use crate::{ behaviour::{self, Behaviour, BehaviourOut}, bitswap::Bitswap, - config::{parse_str_addr, Params, TransportConfig}, + config::{self, parse_str_addr, Params, TransportConfig}, discovery::DiscoveryConfig, error::Error, network_state::{ @@ -60,7 +60,7 @@ use metrics::{Histogram, HistogramVec, MetricSources, Metrics}; use parking_lot::Mutex; use sc_client_api::{BlockBackend, ProofProvider}; use sc_consensus::{BlockImportError, BlockImportStatus, ImportQueue, Link}; -use sc_network_sync::{Status as SyncStatus, SyncState}; +use sc_network_common::sync::{SyncMode, SyncState, SyncStatus}; use sc_peerset::PeersetHandle; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_blockchain::{HeaderBackend, HeaderMetadata}; @@ -207,13 +207,23 @@ where None => (None, None), }; - let (protocol, peerset_handle, mut known_addresses) = Protocol::new( - protocol::ProtocolConfig { - roles: From::from(¶ms.role), - max_parallel_downloads: params.network_config.max_parallel_downloads, - sync_mode: params.network_config.sync_mode.clone(), + let chain_sync = (params.create_chain_sync)( + if params.role.is_light() { + SyncMode::Light + } else { + match params.network_config.sync_mode { + config::SyncMode::Full => SyncMode::Full, + config::SyncMode::Fast { skip_proofs, storage_chain_mode } => + SyncMode::LightState { skip_proofs, storage_chain_mode }, + config::SyncMode::Warp => SyncMode::Warp, + } }, params.chain.clone(), + warp_sync_provider, + )?; + let (protocol, peerset_handle, mut known_addresses) = Protocol::new( + From::from(¶ms.role), + params.chain.clone(), params.protocol_id.clone(), ¶ms.network_config, iter::once(Vec::new()) @@ -222,9 +232,8 @@ where .map(|_| default_notif_handshake_message.clone()), ) .collect(), - params.block_announce_validator, params.metrics_registry.as_ref(), - warp_sync_provider, + chain_sync, )?; // List of multiaddresses that we know in the network. diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index 808546d67fc7c..181d58130aa6b 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -16,15 +16,17 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::{ - config, state_request_handler::StateRequestHandler, Event, NetworkService, NetworkWorker, -}; +use crate::{config, Event, NetworkService, NetworkWorker}; use futures::prelude::*; use libp2p::PeerId; use sc_network_common::config::ProtocolId; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; -use sc_network_sync::block_request_handler::BlockRequestHandler; +use sc_network_sync::{ + block_request_handler::BlockRequestHandler, state_request_handler::StateRequestHandler, + ChainSync, +}; +use sp_consensus::block_validation::DefaultBlockAnnounceValidator; use sp_runtime::traits::{Block as BlockT, Header as _}; use std::{borrow::Cow, sync::Arc, time::Duration}; use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt as _}; @@ -109,6 +111,7 @@ fn build_test_full_node( protocol_config }; + let max_parallel_downloads = config.max_parallel_downloads; let worker = NetworkWorker::new(config::Params { role: config::Role::Full, executor: None, @@ -120,8 +123,17 @@ fn build_test_full_node( transaction_pool: Arc::new(crate::config::EmptyTransactionPool), protocol_id, import_queue, - block_announce_validator: Box::new( - sp_consensus::block_validation::DefaultBlockAnnounceValidator, + create_chain_sync: Box::new( + move |sync_mode, chain, warp_sync_provider| match ChainSync::new( + sync_mode, + chain, + Box::new(DefaultBlockAnnounceValidator), + max_parallel_downloads, + warp_sync_provider, + ) { + Ok(chain_sync) => Ok(Box::new(chain_sync)), + Err(error) => Err(Box::new(error).into()), + }, ), metrics_registry: None, block_request_protocol_config, diff --git a/client/network/sync/Cargo.toml b/client/network/sync/Cargo.toml index a61f780fd56ad..3e3526146400a 100644 --- a/client/network/sync/Cargo.toml +++ b/client/network/sync/Cargo.toml @@ -17,11 +17,9 @@ targets = ["x86_64-unknown-linux-gnu"] prost-build = "0.10" [dependencies] -bitflags = "1.3.2" codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive", ] } -either = "1.5.3" futures = "0.3.21" libp2p = "0.46.1" log = "0.4.17" diff --git a/client/network/sync/src/block_request_handler.rs b/client/network/sync/src/block_request_handler.rs index 4cd69d1fac4f5..2a847a8bf36ec 100644 --- a/client/network/sync/src/block_request_handler.rs +++ b/client/network/sync/src/block_request_handler.rs @@ -17,10 +17,7 @@ //! Helper for handling (i.e. answering) block requests from a remote peer via the //! `crate::request_responses::RequestResponsesBehaviour`. -use crate::{ - message::BlockAttributes, - schema::v1::{block_request::FromBlock, BlockResponse, Direction}, -}; +use crate::schema::v1::{block_request::FromBlock, BlockResponse, Direction}; use codec::{Decode, Encode}; use futures::{ channel::{mpsc, oneshot}, @@ -34,6 +31,7 @@ use sc_client_api::BlockBackend; use sc_network_common::{ config::ProtocolId, request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig}, + sync::message::BlockAttributes, }; use sp_blockchain::HeaderBackend; use sp_runtime::{ diff --git a/client/network/sync/src/blocks.rs b/client/network/sync/src/blocks.rs index 26753f120a170..5fb1484675071 100644 --- a/client/network/sync/src/blocks.rs +++ b/client/network/sync/src/blocks.rs @@ -16,9 +16,9 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::message; use libp2p::PeerId; use log::trace; +use sc_network_common::sync::message; use sp_runtime::traits::{Block as BlockT, NumberFor, One}; use std::{ cmp, @@ -245,8 +245,8 @@ impl BlockCollection { #[cfg(test)] mod test { use super::{BlockCollection, BlockData, BlockRangeState}; - use crate::message; use libp2p::PeerId; + use sc_network_common::sync::message; use sp_core::H256; use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper}; diff --git a/client/network/sync/src/extra_requests.rs b/client/network/sync/src/extra_requests.rs index c684d8e72783e..6206f8a61bcf4 100644 --- a/client/network/sync/src/extra_requests.rs +++ b/client/network/sync/src/extra_requests.rs @@ -20,6 +20,7 @@ use crate::{PeerSync, PeerSyncState}; use fork_tree::ForkTree; use libp2p::PeerId; use log::{debug, trace, warn}; +use sc_network_common::sync::metrics::Metrics; use sp_blockchain::Error as ClientError; use sp_runtime::traits::{Block as BlockT, NumberFor, Zero}; use std::{ @@ -56,15 +57,6 @@ pub(crate) struct ExtraRequests { request_type_name: &'static str, } -#[derive(Debug)] -pub struct Metrics { - pub pending_requests: u32, - pub active_requests: u32, - pub importing_requests: u32, - pub failed_requests: u32, - _priv: (), -} - impl ExtraRequests { pub(crate) fn new(request_type_name: &'static str) -> Self { Self { @@ -258,7 +250,6 @@ impl ExtraRequests { active_requests: self.active_requests.len().try_into().unwrap_or(std::u32::MAX), failed_requests: self.failed_requests.len().try_into().unwrap_or(std::u32::MAX), importing_requests: self.importing_requests.len().try_into().unwrap_or(std::u32::MAX), - _priv: (), } } } diff --git a/client/network/sync/src/lib.rs b/client/network/sync/src/lib.rs index 1ce69b6dc816f..aff773bd12ed6 100644 --- a/client/network/sync/src/lib.rs +++ b/client/network/sync/src/lib.rs @@ -30,8 +30,7 @@ pub mod block_request_handler; pub mod blocks; -pub mod message; -pub mod schema; +mod schema; pub mod state; pub mod state_request_handler; pub mod warp; @@ -39,22 +38,28 @@ pub mod warp_request_handler; use crate::{ blocks::BlockCollection, - message::{BlockAnnounce, BlockAttributes, BlockRequest, BlockResponse}, schema::v1::{StateRequest, StateResponse}, - state::{StateDownloadProgress, StateSync}, - warp::{ - EncodedProof, WarpProofImportResult, WarpProofRequest, WarpSync, WarpSyncPhase, - WarpSyncProgress, WarpSyncProvider, - }, + state::StateSync, + warp::{WarpProofImportResult, WarpSync}, }; -use codec::Encode; -use either::Either; +use codec::{Decode, DecodeAll, Encode}; use extra_requests::ExtraRequests; use futures::{stream::FuturesUnordered, task::Poll, Future, FutureExt, StreamExt}; use libp2p::PeerId; use log::{debug, error, info, trace, warn}; +use prost::Message; use sc_client_api::{BlockBackend, ProofProvider}; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; +use sc_network_common::sync::{ + message::{ + BlockAnnounce, BlockAttributes, BlockData, BlockRequest, BlockResponse, Direction, + FromBlock, + }, + warp::{EncodedProof, WarpProofRequest, WarpSyncPhase, WarpSyncProgress, WarpSyncProvider}, + BadPeer, ChainSync as ChainSyncT, Metrics, OnBlockData, OnBlockJustification, OnStateData, + OpaqueBlockRequest, OpaqueBlockResponse, OpaqueStateRequest, OpaqueStateResponse, PeerInfo, + PollBlockAnnounceValidation, SyncMode, SyncState, SyncStatus, +}; use sp_arithmetic::traits::Saturating; use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; use sp_consensus::{ @@ -71,7 +76,6 @@ use sp_runtime::{ }; use std::{ collections::{hash_map::Entry, HashMap, HashSet}, - fmt, ops::Range, pin::Pin, sync::Arc, @@ -283,15 +287,6 @@ impl PeerSync { } } -/// The sync status of a peer we are trying to sync with -#[derive(Debug)] -pub struct PeerInfo { - /// Their best block hash. - pub best_hash: B::Hash, - /// Their best block number. - pub best_number: NumberFor, -} - struct ForkTarget { number: NumberFor, parent_hash: Option, @@ -330,108 +325,6 @@ impl PeerSyncState { } } -/// Reported sync state. -#[derive(Clone, Eq, PartialEq, Debug)] -pub enum SyncState { - /// Initial sync is complete, keep-up sync is active. - Idle, - /// Actively catching up with the chain. - Downloading, -} - -/// Syncing status and statistics. -#[derive(Clone)] -pub struct Status { - /// Current global sync state. - pub state: SyncState, - /// Target sync block number. - pub best_seen_block: Option>, - /// Number of peers participating in syncing. - pub num_peers: u32, - /// Number of blocks queued for import - pub queued_blocks: u32, - /// State sync status in progress, if any. - pub state_sync: Option, - /// Warp sync in progress, if any. - pub warp_sync: Option>, -} - -/// A peer did not behave as expected and should be reported. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BadPeer(pub PeerId, pub sc_peerset::ReputationChange); - -impl fmt::Display for BadPeer { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Bad peer {}; Reputation change: {:?}", self.0, self.1) - } -} - -impl std::error::Error for BadPeer {} - -/// Result of [`ChainSync::on_block_data`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum OnBlockData { - /// The block should be imported. - Import(BlockOrigin, Vec>), - /// A new block request needs to be made to the given peer. - Request(PeerId, BlockRequest), -} - -impl OnBlockData { - /// Returns `self` as request. - #[cfg(test)] - fn into_request(self) -> Option<(PeerId, BlockRequest)> { - if let Self::Request(peer, req) = self { - Some((peer, req)) - } else { - None - } - } -} - -/// Result of [`ChainSync::on_state_data`]. -#[derive(Debug)] -pub enum OnStateData { - /// The block and state that should be imported. - Import(BlockOrigin, IncomingBlock), - /// A new state request needs to be made to the given peer. - Continue, -} - -/// Result of [`ChainSync::poll_block_announce_validation`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum PollBlockAnnounceValidation { - /// The announcement failed at validation. - /// - /// The peer reputation should be decreased. - Failure { - /// Who sent the processed block announcement? - who: PeerId, - /// Should the peer be disconnected? - disconnect: bool, - }, - /// The announcement does not require further handling. - Nothing { - /// Who sent the processed block announcement? - who: PeerId, - /// Was this their new best block? - is_best: bool, - /// The announcement. - announce: BlockAnnounce, - }, - /// The announcement header should be imported. - ImportHeader { - /// Who sent the processed block announcement? - who: PeerId, - /// Was this their new best block? - is_best: bool, - /// The announcement. - announce: BlockAnnounce, - }, - /// The block announcement should be skipped. - Skip, -} - /// Result of [`ChainSync::block_announce_validation`]. #[derive(Debug, Clone, PartialEq, Eq)] enum PreValidateBlockAnnounce { @@ -467,28 +360,6 @@ enum PreValidateBlockAnnounce { Skip, } -/// Result of [`ChainSync::on_block_justification`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum OnBlockJustification { - /// The justification needs no further handling. - Nothing, - /// The justification should be imported. - Import { peer: PeerId, hash: B::Hash, number: NumberFor, justifications: Justifications }, -} - -/// Operation mode. -#[derive(Debug, PartialEq, Eq)] -pub enum SyncMode { - // Sync headers only - Light, - // Sync headers and block bodies - Full, - // Sync headers and the last finalied state - LightState { storage_chain_mode: bool, skip_proofs: bool }, - // Warp sync mode. - Warp, -} - /// Result of [`ChainSync::has_slot_for_block_announce_validation`]. enum HasSlotForBlockAnnounceValidation { /// Yes, there is a slot for the block announce validation. @@ -499,7 +370,7 @@ enum HasSlotForBlockAnnounceValidation { MaximumPeerSlotsReached, } -impl ChainSync +impl ChainSyncT for ChainSync where B: BlockT, Client: HeaderBackend @@ -510,88 +381,14 @@ where + Sync + 'static, { - /// Create a new instance. - pub fn new( - mode: SyncMode, - client: Arc, - block_announce_validator: Box + Send>, - max_parallel_downloads: u32, - warp_sync_provider: Option>>, - ) -> Result { - let mut sync = Self { - client, - peers: HashMap::new(), - blocks: BlockCollection::new(), - best_queued_hash: Default::default(), - best_queued_number: Zero::zero(), - extra_justifications: ExtraRequests::new("justification"), - mode, - queue_blocks: Default::default(), - fork_targets: Default::default(), - allowed_requests: Default::default(), - block_announce_validator, - max_parallel_downloads, - downloaded_blocks: 0, - block_announce_validation: Default::default(), - block_announce_validation_per_peer_stats: Default::default(), - state_sync: None, - warp_sync: None, - warp_sync_provider, - import_existing: false, - gap_sync: None, - }; - sync.reset_sync_start_point()?; - Ok(sync) - } - - fn required_block_attributes(&self) -> BlockAttributes { - match self.mode { - SyncMode::Full => - BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::BODY, - SyncMode::Light => BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION, - SyncMode::LightState { storage_chain_mode: false, .. } | SyncMode::Warp => - BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::BODY, - SyncMode::LightState { storage_chain_mode: true, .. } => - BlockAttributes::HEADER | - BlockAttributes::JUSTIFICATION | - BlockAttributes::INDEXED_BODY, - } - } - - fn skip_execution(&self) -> bool { - match self.mode { - SyncMode::Full => false, - SyncMode::Light => true, - SyncMode::LightState { .. } => true, - SyncMode::Warp => true, - } - } - - /// Returns the state of the sync of the given peer. - /// - /// Returns `None` if the peer is unknown. - pub fn peer_info(&self, who: &PeerId) -> Option> { + fn peer_info(&self, who: &PeerId) -> Option> { self.peers .get(who) .map(|p| PeerInfo { best_hash: p.best_hash, best_number: p.best_number }) } - /// Returns the best seen block. - fn best_seen(&self) -> Option> { - let mut best_seens = self.peers.values().map(|p| p.best_number).collect::>(); - - if best_seens.is_empty() { - None - } else { - let middle = best_seens.len() / 2; - - // Not the "perfect median" when we have an even number of peers. - Some(*best_seens.select_nth_unstable(middle).1) - } - } - /// Returns the current sync status. - pub fn status(&self) -> Status { + fn status(&self) -> SyncStatus { let best_seen = self.best_seen(); let sync_state = if let Some(n) = best_seen { // A chain is classified as downloading if the provided best block is @@ -617,7 +414,7 @@ where _ => None, }; - Status { + SyncStatus { state: sync_state, best_seen_block: best_seen, num_peers: self.peers.len() as u32, @@ -627,29 +424,22 @@ where } } - /// Number of active forks requests. This includes - /// requests that are pending or could be issued right away. - pub fn num_sync_requests(&self) -> usize { + fn num_sync_requests(&self) -> usize { self.fork_targets .values() .filter(|f| f.number <= self.best_queued_number) .count() } - /// Number of downloaded blocks. - pub fn num_downloaded_blocks(&self) -> usize { + fn num_downloaded_blocks(&self) -> usize { self.downloaded_blocks } - /// Returns the current number of peers stored within this state machine. - pub fn num_peers(&self) -> usize { + fn num_peers(&self) -> usize { self.peers.len() } - /// Handle a new connected peer. - /// - /// Call this method whenever we connect to a new peer. - pub fn new_peer( + fn new_peer( &mut self, who: PeerId, best_hash: B::Hash, @@ -773,27 +563,22 @@ where } } - /// Signal that a new best block has been imported. - /// `ChainSync` state with that information. - pub fn update_chain_info(&mut self, best_hash: &B::Hash, best_number: NumberFor) { + fn update_chain_info(&mut self, best_hash: &B::Hash, best_number: NumberFor) { self.on_block_queued(best_hash, best_number); } - /// Schedule a justification request for the given block. - pub fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { + fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { let client = &self.client; self.extra_justifications .schedule((*hash, number), |base, block| is_descendent_of(&**client, base, block)) } - /// Clear all pending justification requests. - pub fn clear_justification_requests(&mut self) { + fn clear_justification_requests(&mut self) { self.extra_justifications.reset(); } - /// Request syncing for the given block from given set of peers. // The implementation is similar to on_block_announce with unknown parent hash. - pub fn set_sync_fork_request( + fn set_sync_fork_request( &mut self, mut peers: Vec, hash: &B::Hash, @@ -845,13 +630,12 @@ where .extend(peers); } - /// Get an iterator over all scheduled justification requests. - pub fn justification_requests( + fn justification_requests( &mut self, - ) -> impl Iterator)> + '_ { + ) -> Box)> + '_> { let peers = &mut self.peers; let mut matcher = self.extra_justifications.matcher(); - std::iter::from_fn(move || { + Box::new(std::iter::from_fn(move || { if let Some((peer, request)) = matcher.next(peers) { peers .get_mut(&peer) @@ -859,33 +643,32 @@ where "`Matcher::next` guarantees the `PeerId` comes from the given peers; qed", ) .state = PeerSyncState::DownloadingJustification(request.0); - let req = message::generic::BlockRequest { + let req = BlockRequest:: { id: 0, fields: BlockAttributes::JUSTIFICATION, - from: message::FromBlock::Hash(request.0), + from: FromBlock::Hash(request.0), to: None, - direction: message::Direction::Ascending, + direction: Direction::Ascending, max: Some(1), }; Some((peer, req)) } else { None } - }) + })) } - /// Get an iterator over all block requests of all peers. - pub fn block_requests(&mut self) -> impl Iterator)> + '_ { + fn block_requests(&mut self) -> Box)> + '_> { if self.allowed_requests.is_empty() || self.state_sync.is_some() || self.mode == SyncMode::Warp { - return Either::Left(std::iter::empty()) + return Box::new(std::iter::empty()) } if self.queue_blocks.len() > MAX_IMPORTING_BLOCKS { trace!(target: "sync", "Too many blocks in the queue."); - return Either::Left(std::iter::empty()) + return Box::new(std::iter::empty()) } let major_sync = self.status().state == SyncState::Downloading; let attrs = self.required_block_attributes(); @@ -982,11 +765,10 @@ where None } }); - Either::Right(iter) + Box::new(iter) } - /// Get a state request, if any. - pub fn state_request(&mut self) -> Option<(PeerId, StateRequest)> { + fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)> { if self.allowed_requests.is_empty() { return None } @@ -1007,7 +789,7 @@ where let request = sync.next_request(); trace!(target: "sync", "New StateRequest for {}: {:?}", id, request); self.allowed_requests.clear(); - return Some((*id, request)) + return Some((*id, OpaqueStateRequest(Box::new(request)))) } } } @@ -1023,7 +805,7 @@ where trace!(target: "sync", "New StateRequest for {}: {:?}", id, request); peer.state = PeerSyncState::DownloadingState; self.allowed_requests.clear(); - return Some((*id, request)) + return Some((*id, OpaqueStateRequest(Box::new(request)))) } } } @@ -1031,8 +813,7 @@ where None } - /// Get a warp sync request, if any. - pub fn warp_sync_request(&mut self) -> Option<(PeerId, WarpProofRequest)> { + fn warp_sync_request(&mut self) -> Option<(PeerId, WarpProofRequest)> { if let Some(sync) = &self.warp_sync { if self.allowed_requests.is_empty() || sync.is_complete() || @@ -1063,14 +844,7 @@ where None } - /// Handle a response from the remote to a block request that we made. - /// - /// `request` must be the original request that triggered `response`. - /// or `None` if data comes from the block announcement. - /// - /// If this corresponds to a valid block, this outputs the block that - /// must be imported in the import queue. - pub fn on_block_data( + fn on_block_data( &mut self, who: &PeerId, request: Option>, @@ -1080,10 +854,7 @@ where let mut gap = false; let new_blocks: Vec> = if let Some(peer) = self.peers.get_mut(who) { let mut blocks = response.blocks; - if request - .as_ref() - .map_or(false, |r| r.direction == message::Direction::Descending) - { + if request.as_ref().map_or(false, |r| r.direction == Direction::Descending) { trace!(target: "sync", "Reversing incoming block list"); blocks.reverse() } @@ -1297,14 +1068,20 @@ where Ok(self.validate_and_queue_blocks(new_blocks, gap)) } - /// Handle a response from the remote to a state request that we made. - /// - /// Returns next request if any. - pub fn on_state_data( + fn on_state_data( &mut self, who: &PeerId, - response: StateResponse, + response: OpaqueStateResponse, ) -> Result, BadPeer> { + let response: Box = response.0.downcast().map_err(|_error| { + error!( + target: "sync", + "Failed to downcast opaque state response, this is an implementation bug." + ); + + BadPeer(*who, rep::BAD_RESPONSE) + })?; + if let Some(peer) = self.peers.get_mut(who) { if let PeerSyncState::DownloadingState = peer.state { peer.state = PeerSyncState::Available; @@ -1319,7 +1096,7 @@ where response.entries.len(), response.proof.len(), ); - sync.import(response) + sync.import(*response) } else if let Some(sync) = &mut self.warp_sync { debug!( target: "sync", @@ -1328,7 +1105,7 @@ where response.entries.len(), response.proof.len(), ); - sync.import_state(response) + sync.import_state(*response) } else { debug!(target: "sync", "Ignored obsolete state response from {}", who); return Err(BadPeer(*who, rep::NOT_REQUESTED)) @@ -1360,14 +1137,7 @@ where } } - /// Handle a response from the remote to a warp proof request that we made. - /// - /// Returns next request. - pub fn on_warp_sync_data( - &mut self, - who: &PeerId, - response: EncodedProof, - ) -> Result<(), BadPeer> { + fn on_warp_sync_data(&mut self, who: &PeerId, response: EncodedProof) -> Result<(), BadPeer> { if let Some(peer) = self.peers.get_mut(who) { if let PeerSyncState::DownloadingWarpProof = peer.state { peer.state = PeerSyncState::Available; @@ -1396,57 +1166,7 @@ where } } - fn validate_and_queue_blocks( - &mut self, - mut new_blocks: Vec>, - gap: bool, - ) -> OnBlockData { - let orig_len = new_blocks.len(); - new_blocks.retain(|b| !self.queue_blocks.contains(&b.hash)); - if new_blocks.len() != orig_len { - debug!( - target: "sync", - "Ignoring {} blocks that are already queued", - orig_len - new_blocks.len(), - ); - } - - let origin = if !gap && self.status().state != SyncState::Downloading { - BlockOrigin::NetworkBroadcast - } else { - BlockOrigin::NetworkInitialSync - }; - - if let Some((h, n)) = new_blocks - .last() - .and_then(|b| b.header.as_ref().map(|h| (&b.hash, *h.number()))) - { - trace!( - target:"sync", - "Accepted {} blocks ({:?}) with origin {:?}", - new_blocks.len(), - h, - origin, - ); - self.on_block_queued(h, n) - } - self.queue_blocks.extend(new_blocks.iter().map(|b| b.hash)); - OnBlockData::Import(origin, new_blocks) - } - - fn update_peer_common_number(&mut self, peer_id: &PeerId, new_common: NumberFor) { - if let Some(peer) = self.peers.get_mut(peer_id) { - peer.update_common_number(new_common); - } - } - - /// Handle a response from the remote to a justification request that we made. - /// - /// `request` must be the original request that triggered `response`. - /// - /// Returns `Some` if this produces a justification that must be imported - /// into the import queue. - pub fn on_block_justification( + fn on_block_justification( &mut self, who: PeerId, response: BlockResponse, @@ -1501,18 +1221,12 @@ where Ok(OnBlockJustification::Nothing) } - /// A batch of blocks have been processed, with or without errors. - /// - /// Call this when a batch of blocks have been processed by the import - /// queue, with or without errors. - /// - /// `peer_info` is passed in case of a restart. - pub fn on_blocks_processed( + fn on_blocks_processed( &mut self, imported: usize, count: usize, results: Vec<(Result>, BlockImportError>, B::Hash)>, - ) -> impl Iterator), BadPeer>> { + ) -> Box), BadPeer>>> { trace!(target: "sync", "Imported {} of {}", imported, count); let mut output = Vec::new(); @@ -1654,54 +1368,435 @@ where } self.allowed_requests.set_all(); - output.into_iter() + Box::new(output.into_iter()) + } + + fn on_justification_import(&mut self, hash: B::Hash, number: NumberFor, success: bool) { + let finalization_result = if success { Ok((hash, number)) } else { Err(()) }; + self.extra_justifications + .try_finalize_root((hash, number), finalization_result, true); + self.allowed_requests.set_all(); + } + + fn on_block_finalized(&mut self, hash: &B::Hash, number: NumberFor) { + let client = &self.client; + let r = self.extra_justifications.on_block_finalized(hash, number, |base, block| { + is_descendent_of(&**client, base, block) + }); + + if let SyncMode::LightState { skip_proofs, .. } = &self.mode { + if self.state_sync.is_none() && !self.peers.is_empty() && self.queue_blocks.is_empty() { + // Finalized a recent block. + let mut heads: Vec<_> = + self.peers.iter().map(|(_, peer)| peer.best_number).collect(); + heads.sort(); + let median = heads[heads.len() / 2]; + if number + STATE_SYNC_FINALITY_THRESHOLD.saturated_into() >= median { + if let Ok(Some(header)) = self.client.header(BlockId::hash(*hash)) { + log::debug!( + target: "sync", + "Starting state sync for #{} ({})", + number, + hash, + ); + self.state_sync = + Some(StateSync::new(self.client.clone(), header, *skip_proofs)); + self.allowed_requests.set_all(); + } + } + } + } + + if let Err(err) = r { + warn!( + target: "sync", + "💔 Error cleaning up pending extra justification data requests: {}", + err, + ); + } + } + + fn push_block_announce_validation( + &mut self, + who: PeerId, + hash: B::Hash, + announce: BlockAnnounce, + is_best: bool, + ) { + let header = &announce.header; + let number = *header.number(); + debug!( + target: "sync", + "Pre-validating received block announcement {:?} with number {:?} from {}", + hash, + number, + who, + ); + + if number.is_zero() { + self.block_announce_validation.push( + async move { + warn!( + target: "sync", + "💔 Ignored genesis block (#0) announcement from {}: {}", + who, + hash, + ); + PreValidateBlockAnnounce::Skip + } + .boxed(), + ); + return + } + + // Check if there is a slot for this block announce validation. + match self.has_slot_for_block_announce_validation(&who) { + HasSlotForBlockAnnounceValidation::Yes => {}, + HasSlotForBlockAnnounceValidation::TotalMaximumSlotsReached => { + self.block_announce_validation.push( + async move { + warn!( + target: "sync", + "💔 Ignored block (#{} -- {}) announcement from {} because all validation slots are occupied.", + number, + hash, + who, + ); + PreValidateBlockAnnounce::Skip + } + .boxed(), + ); + return + }, + HasSlotForBlockAnnounceValidation::MaximumPeerSlotsReached => { + self.block_announce_validation.push(async move { + warn!( + target: "sync", + "💔 Ignored block (#{} -- {}) announcement from {} because all validation slots for this peer are occupied.", + number, + hash, + who, + ); + PreValidateBlockAnnounce::Skip + }.boxed()); + return + }, + } + + // Let external validator check the block announcement. + let assoc_data = announce.data.as_ref().map_or(&[][..], |v| v.as_slice()); + let future = self.block_announce_validator.validate(header, assoc_data); + + self.block_announce_validation.push( + async move { + match future.await { + Ok(Validation::Success { is_new_best }) => PreValidateBlockAnnounce::Process { + is_new_best: is_new_best || is_best, + announce, + who, + }, + Ok(Validation::Failure { disconnect }) => { + debug!( + target: "sync", + "Block announcement validation of block {:?} from {} failed", + hash, + who, + ); + PreValidateBlockAnnounce::Failure { who, disconnect } + }, + Err(e) => { + debug!( + target: "sync", + "💔 Block announcement validation of block {:?} errored: {}", + hash, + e, + ); + PreValidateBlockAnnounce::Error { who } + }, + } + } + .boxed(), + ); + } + + fn poll_block_announce_validation( + &mut self, + cx: &mut std::task::Context, + ) -> Poll> { + match self.block_announce_validation.poll_next_unpin(cx) { + Poll::Ready(Some(res)) => { + self.peer_block_announce_validation_finished(&res); + Poll::Ready(self.finish_block_announce_validation(res)) + }, + _ => Poll::Pending, + } + } + + fn peer_disconnected(&mut self, who: &PeerId) -> Option> { + self.blocks.clear_peer_download(who); + if let Some(gap_sync) = &mut self.gap_sync { + gap_sync.blocks.clear_peer_download(who) + } + self.peers.remove(who); + self.extra_justifications.peer_disconnected(who); + self.allowed_requests.set_all(); + self.fork_targets.retain(|_, target| { + target.peers.remove(who); + !target.peers.is_empty() + }); + let blocks = self.ready_blocks(); + (!blocks.is_empty()).then(|| self.validate_and_queue_blocks(blocks, false)) + } + + fn metrics(&self) -> Metrics { + Metrics { + queued_blocks: self.queue_blocks.len().try_into().unwrap_or(std::u32::MAX), + fork_targets: self.fork_targets.len().try_into().unwrap_or(std::u32::MAX), + justifications: self.extra_justifications.metrics(), + } + } + + /// Create implementation-specific block request. + fn create_opaque_block_request(&self, request: &BlockRequest) -> OpaqueBlockRequest { + OpaqueBlockRequest(Box::new(schema::v1::BlockRequest { + fields: request.fields.to_be_u32(), + from_block: match request.from { + FromBlock::Hash(h) => Some(schema::v1::block_request::FromBlock::Hash(h.encode())), + FromBlock::Number(n) => + Some(schema::v1::block_request::FromBlock::Number(n.encode())), + }, + to_block: request.to.map(|h| h.encode()).unwrap_or_default(), + direction: request.direction as i32, + max_blocks: request.max.unwrap_or(0), + support_multiple_justifications: true, + })) + } + + fn encode_block_request(&self, request: &OpaqueBlockRequest) -> Result, String> { + let request: &schema::v1::BlockRequest = request.0.downcast_ref().ok_or_else(|| { + "Failed to downcast opaque block response during encoding, this is an \ + implementation bug." + .to_string() + })?; + + Ok(request.encode_to_vec()) + } + + fn decode_block_response(&self, response: &[u8]) -> Result { + let response = schema::v1::BlockResponse::decode(response) + .map_err(|error| format!("Failed to decode block response: {error}"))?; + + Ok(OpaqueBlockResponse(Box::new(response))) + } + + fn block_response_into_blocks( + &self, + request: &BlockRequest, + response: OpaqueBlockResponse, + ) -> Result>, String> { + let response: Box = response.0.downcast().map_err(|_error| { + "Failed to downcast opaque block response during encoding, this is an \ + implementation bug." + .to_string() + })?; + + response + .blocks + .into_iter() + .map(|block_data| { + Ok(BlockData:: { + hash: Decode::decode(&mut block_data.hash.as_ref())?, + header: if !block_data.header.is_empty() { + Some(Decode::decode(&mut block_data.header.as_ref())?) + } else { + None + }, + body: if request.fields.contains(BlockAttributes::BODY) { + Some( + block_data + .body + .iter() + .map(|body| Decode::decode(&mut body.as_ref())) + .collect::, _>>()?, + ) + } else { + None + }, + indexed_body: if request.fields.contains(BlockAttributes::INDEXED_BODY) { + Some(block_data.indexed_body) + } else { + None + }, + receipt: if !block_data.receipt.is_empty() { + Some(block_data.receipt) + } else { + None + }, + message_queue: if !block_data.message_queue.is_empty() { + Some(block_data.message_queue) + } else { + None + }, + justification: if !block_data.justification.is_empty() { + Some(block_data.justification) + } else if block_data.is_empty_justification { + Some(Vec::new()) + } else { + None + }, + justifications: if !block_data.justifications.is_empty() { + Some(DecodeAll::decode_all(&mut block_data.justifications.as_ref())?) + } else { + None + }, + }) + }) + .collect::>() + .map_err(|error: codec::Error| error.to_string()) + } + + fn encode_state_request(&self, request: &OpaqueStateRequest) -> Result, String> { + let request: &StateRequest = request.0.downcast_ref().ok_or_else(|| { + "Failed to downcast opaque state response during encoding, this is an \ + implementation bug." + .to_string() + })?; + + Ok(request.encode_to_vec()) + } + + fn decode_state_response(&self, response: &[u8]) -> Result { + let response = StateResponse::decode(response) + .map_err(|error| format!("Failed to decode state response: {error}"))?; + + Ok(OpaqueStateResponse(Box::new(response))) + } +} + +impl ChainSync +where + Self: ChainSyncT, + B: BlockT, + Client: HeaderBackend + + BlockBackend + + HeaderMetadata + + ProofProvider + + Send + + Sync + + 'static, +{ + /// Create a new instance. + pub fn new( + mode: SyncMode, + client: Arc, + block_announce_validator: Box + Send>, + max_parallel_downloads: u32, + warp_sync_provider: Option>>, + ) -> Result { + let mut sync = Self { + client, + peers: HashMap::new(), + blocks: BlockCollection::new(), + best_queued_hash: Default::default(), + best_queued_number: Zero::zero(), + extra_justifications: ExtraRequests::new("justification"), + mode, + queue_blocks: Default::default(), + fork_targets: Default::default(), + allowed_requests: Default::default(), + block_announce_validator, + max_parallel_downloads, + downloaded_blocks: 0, + block_announce_validation: Default::default(), + block_announce_validation_per_peer_stats: Default::default(), + state_sync: None, + warp_sync: None, + warp_sync_provider, + import_existing: false, + gap_sync: None, + }; + sync.reset_sync_start_point()?; + Ok(sync) } - /// Call this when a justification has been processed by the import queue, - /// with or without errors. - pub fn on_justification_import(&mut self, hash: B::Hash, number: NumberFor, success: bool) { - let finalization_result = if success { Ok((hash, number)) } else { Err(()) }; - self.extra_justifications - .try_finalize_root((hash, number), finalization_result, true); - self.allowed_requests.set_all(); + /// Returns the best seen block. + fn best_seen(&self) -> Option> { + let mut best_seens = self.peers.values().map(|p| p.best_number).collect::>(); + + if best_seens.is_empty() { + None + } else { + let middle = best_seens.len() / 2; + + // Not the "perfect median" when we have an even number of peers. + Some(*best_seens.select_nth_unstable(middle).1) + } } - /// Notify about finalization of the given block. - pub fn on_block_finalized(&mut self, hash: &B::Hash, number: NumberFor) { - let client = &self.client; - let r = self.extra_justifications.on_block_finalized(hash, number, |base, block| { - is_descendent_of(&**client, base, block) - }); + fn required_block_attributes(&self) -> BlockAttributes { + match self.mode { + SyncMode::Full => + BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::BODY, + SyncMode::Light => BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION, + SyncMode::LightState { storage_chain_mode: false, .. } | SyncMode::Warp => + BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::BODY, + SyncMode::LightState { storage_chain_mode: true, .. } => + BlockAttributes::HEADER | + BlockAttributes::JUSTIFICATION | + BlockAttributes::INDEXED_BODY, + } + } - if let SyncMode::LightState { skip_proofs, .. } = &self.mode { - if self.state_sync.is_none() && !self.peers.is_empty() && self.queue_blocks.is_empty() { - // Finalized a recent block. - let mut heads: Vec<_> = - self.peers.iter().map(|(_, peer)| peer.best_number).collect(); - heads.sort(); - let median = heads[heads.len() / 2]; - if number + STATE_SYNC_FINALITY_THRESHOLD.saturated_into() >= median { - if let Ok(Some(header)) = self.client.header(BlockId::hash(*hash)) { - log::debug!( - target: "sync", - "Starting state sync for #{} ({})", - number, - hash, - ); - self.state_sync = - Some(StateSync::new(self.client.clone(), header, *skip_proofs)); - self.allowed_requests.set_all(); - } - } - } + fn skip_execution(&self) -> bool { + match self.mode { + SyncMode::Full => false, + SyncMode::Light => true, + SyncMode::LightState { .. } => true, + SyncMode::Warp => true, } + } - if let Err(err) = r { - warn!( + fn validate_and_queue_blocks( + &mut self, + mut new_blocks: Vec>, + gap: bool, + ) -> OnBlockData { + let orig_len = new_blocks.len(); + new_blocks.retain(|b| !self.queue_blocks.contains(&b.hash)); + if new_blocks.len() != orig_len { + debug!( target: "sync", - "💔 Error cleaning up pending extra justification data requests: {}", - err, + "Ignoring {} blocks that are already queued", + orig_len - new_blocks.len(), + ); + } + + let origin = if !gap && self.status().state != SyncState::Downloading { + BlockOrigin::NetworkBroadcast + } else { + BlockOrigin::NetworkInitialSync + }; + + if let Some((h, n)) = new_blocks + .last() + .and_then(|b| b.header.as_ref().map(|h| (&b.hash, *h.number()))) + { + trace!( + target:"sync", + "Accepted {} blocks ({:?}) with origin {:?}", + new_blocks.len(), + h, + origin, ); + self.on_block_queued(h, n) + } + self.queue_blocks.extend(new_blocks.iter().map(|b| b.hash)); + OnBlockData::Import(origin, new_blocks) + } + + fn update_peer_common_number(&mut self, peer_id: &PeerId, new_common: NumberFor) { + if let Some(peer) = self.peers.get_mut(peer_id) { + peer.update_common_number(new_common); } } @@ -1779,135 +1874,6 @@ where } } - /// Push a block announce validation. - /// - /// It is required that [`ChainSync::poll_block_announce_validation`] is called - /// to check for finished block announce validations. - pub fn push_block_announce_validation( - &mut self, - who: PeerId, - hash: B::Hash, - announce: BlockAnnounce, - is_best: bool, - ) { - let header = &announce.header; - let number = *header.number(); - debug!( - target: "sync", - "Pre-validating received block announcement {:?} with number {:?} from {}", - hash, - number, - who, - ); - - if number.is_zero() { - self.block_announce_validation.push( - async move { - warn!( - target: "sync", - "💔 Ignored genesis block (#0) announcement from {}: {}", - who, - hash, - ); - PreValidateBlockAnnounce::Skip - } - .boxed(), - ); - return - } - - // Check if there is a slot for this block announce validation. - match self.has_slot_for_block_announce_validation(&who) { - HasSlotForBlockAnnounceValidation::Yes => {}, - HasSlotForBlockAnnounceValidation::TotalMaximumSlotsReached => { - self.block_announce_validation.push( - async move { - warn!( - target: "sync", - "💔 Ignored block (#{} -- {}) announcement from {} because all validation slots are occupied.", - number, - hash, - who, - ); - PreValidateBlockAnnounce::Skip - } - .boxed(), - ); - return - }, - HasSlotForBlockAnnounceValidation::MaximumPeerSlotsReached => { - self.block_announce_validation.push(async move { - warn!( - target: "sync", - "💔 Ignored block (#{} -- {}) announcement from {} because all validation slots for this peer are occupied.", - number, - hash, - who, - ); - PreValidateBlockAnnounce::Skip - }.boxed()); - return - }, - } - - // Let external validator check the block announcement. - let assoc_data = announce.data.as_ref().map_or(&[][..], |v| v.as_slice()); - let future = self.block_announce_validator.validate(header, assoc_data); - - self.block_announce_validation.push( - async move { - match future.await { - Ok(Validation::Success { is_new_best }) => PreValidateBlockAnnounce::Process { - is_new_best: is_new_best || is_best, - announce, - who, - }, - Ok(Validation::Failure { disconnect }) => { - debug!( - target: "sync", - "Block announcement validation of block {:?} from {} failed", - hash, - who, - ); - PreValidateBlockAnnounce::Failure { who, disconnect } - }, - Err(e) => { - debug!( - target: "sync", - "💔 Block announcement validation of block {:?} errored: {}", - hash, - e, - ); - PreValidateBlockAnnounce::Error { who } - }, - } - } - .boxed(), - ); - } - - /// Poll block announce validation. - /// - /// Block announce validations can be pushed by using - /// [`ChainSync::push_block_announce_validation`]. - /// - /// This should be polled until it returns [`Poll::Pending`]. - /// - /// If [`PollBlockAnnounceValidation::ImportHeader`] is returned, then the caller MUST try to - /// import passed header (call `on_block_data`). The network request isn't sent in this case. - pub fn poll_block_announce_validation( - &mut self, - cx: &mut std::task::Context, - ) -> Poll> { - match self.block_announce_validation.poll_next_unpin(cx) { - Poll::Ready(Some(res)) => { - self.peer_block_announce_validation_finished(&res); - Poll::Ready(self.finish_block_announce_validation(res)) - }, - _ => Poll::Pending, - } - } - /// Should be called when a block announce validation is finished, to update the slots /// of the peer that send the block announce. fn peer_block_announce_validation_finished( @@ -2065,29 +2031,6 @@ where PollBlockAnnounceValidation::Nothing { is_best, who, announce } } - /// Call when a peer has disconnected. - /// Canceled obsolete block request may result in some blocks being ready for - /// import, so this functions checks for such blocks and returns them. - pub fn peer_disconnected(&mut self, who: &PeerId) -> Option> { - self.blocks.clear_peer_download(who); - if let Some(gap_sync) = &mut self.gap_sync { - gap_sync.blocks.clear_peer_download(who) - } - self.peers.remove(who); - self.extra_justifications.peer_disconnected(who); - self.allowed_requests.set_all(); - self.fork_targets.retain(|_, target| { - target.peers.remove(who); - !target.peers.is_empty() - }); - let blocks = self.ready_blocks(); - if !blocks.is_empty() { - Some(self.validate_and_queue_blocks(blocks, false)) - } else { - None - } - } - /// Restart the sync process. This will reset all pending block requests and return an iterator /// of new block requests to make to peers. Peers that were downloading finality data (i.e. /// their state was `DownloadingJustification`) are unaffected and will stay in the same state. @@ -2190,16 +2133,6 @@ where .any(|(_, p)| p.state == PeerSyncState::DownloadingStale(*hash)) } - /// Return some key metrics. - pub fn metrics(&self) -> Metrics { - Metrics { - queued_blocks: self.queue_blocks.len().try_into().unwrap_or(std::u32::MAX), - fork_targets: self.fork_targets.len().try_into().unwrap_or(std::u32::MAX), - justifications: self.extra_justifications.metrics(), - _priv: (), - } - } - /// Get the set of downloaded blocks that are ready to be queued for import. fn ready_blocks(&mut self) -> Vec> { self.blocks @@ -2237,23 +2170,15 @@ fn legacy_justification_mapping( justification.map(|just| (*b"FRNK", just).into()) } -#[derive(Debug)] -pub struct Metrics { - pub queued_blocks: u32, - pub fork_targets: u32, - pub justifications: extra_requests::Metrics, - _priv: (), -} - /// Request the ancestry for a block. Sends a request for header and justification for the given /// block number. Used during ancestry search. fn ancestry_request(block: NumberFor) -> BlockRequest { - message::generic::BlockRequest { + BlockRequest:: { id: 0, fields: BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION, - from: message::FromBlock::Number(block), + from: FromBlock::Number(block), to: None, - direction: message::Direction::Ascending, + direction: Direction::Ascending, max: Some(1), } } @@ -2331,7 +2256,7 @@ fn peer_block_request( id: &PeerId, peer: &PeerSync, blocks: &mut BlockCollection, - attrs: message::BlockAttributes, + attrs: BlockAttributes, max_parallel_downloads: u32, finalized: NumberFor, best_num: NumberFor, @@ -2359,17 +2284,17 @@ fn peer_block_request( let last = range.end.saturating_sub(One::one()); let from = if peer.best_number == last { - message::FromBlock::Hash(peer.best_hash) + FromBlock::Hash(peer.best_hash) } else { - message::FromBlock::Number(last) + FromBlock::Number(last) }; - let request = message::generic::BlockRequest { + let request = BlockRequest:: { id: 0, fields: attrs, from, to: None, - direction: message::Direction::Descending, + direction: Direction::Descending, max: Some((range.end - range.start).saturated_into::()), }; @@ -2381,7 +2306,7 @@ fn peer_gap_block_request( id: &PeerId, peer: &PeerSync, blocks: &mut BlockCollection, - attrs: message::BlockAttributes, + attrs: BlockAttributes, target: NumberFor, common_number: NumberFor, ) -> Option<(Range>, BlockRequest)> { @@ -2396,14 +2321,14 @@ fn peer_gap_block_request( // The end is not part of the range. let last = range.end.saturating_sub(One::one()); - let from = message::FromBlock::Number(last); + let from = FromBlock::Number(last); - let request = message::generic::BlockRequest { + let request = BlockRequest:: { id: 0, fields: attrs, from, to: None, - direction: message::Direction::Descending, + direction: Direction::Descending, max: Some((range.end - range.start).saturated_into::()), }; Some((range, request)) @@ -2415,7 +2340,7 @@ fn fork_sync_request( targets: &mut HashMap>, best_num: NumberFor, finalized: NumberFor, - attributes: message::BlockAttributes, + attributes: BlockAttributes, check_block: impl Fn(&B::Hash) -> BlockStatus, ) -> Option<(B::Hash, BlockRequest)> { targets.retain(|hash, r| { @@ -2448,12 +2373,12 @@ fn fork_sync_request( trace!(target: "sync", "Downloading requested fork {:?} from {}, {} blocks", hash, id, count); return Some(( *hash, - message::generic::BlockRequest { + BlockRequest:: { id: 0, fields: attributes, - from: message::FromBlock::Hash(*hash), + from: FromBlock::Hash(*hash), to: None, - direction: message::Direction::Descending, + direction: Direction::Descending, max: Some(count), }, )) @@ -2488,7 +2413,7 @@ where /// /// It is expected that `blocks` are in ascending order. fn validate_blocks( - blocks: &Vec>, + blocks: &Vec>, who: &PeerId, request: Option>, ) -> Result>, BadPeer> { @@ -2505,16 +2430,13 @@ fn validate_blocks( return Err(BadPeer(*who, rep::NOT_REQUESTED)) } - let block_header = if request.direction == message::Direction::Descending { - blocks.last() - } else { - blocks.first() - } - .and_then(|b| b.header.as_ref()); + let block_header = + if request.direction == Direction::Descending { blocks.last() } else { blocks.first() } + .and_then(|b| b.header.as_ref()); let expected_block = block_header.as_ref().map_or(false, |h| match request.from { - message::FromBlock::Hash(hash) => h.hash() == hash, - message::FromBlock::Number(n) => h.number() == &n, + FromBlock::Hash(hash) => h.hash() == hash, + FromBlock::Number(n) => h.number() == &n, }); if !expected_block { @@ -2528,7 +2450,7 @@ fn validate_blocks( return Err(BadPeer(*who, rep::NOT_REQUESTED)) } - if request.fields.contains(message::BlockAttributes::HEADER) && + if request.fields.contains(BlockAttributes::HEADER) && blocks.iter().any(|b| b.header.is_none()) { trace!( @@ -2540,8 +2462,7 @@ fn validate_blocks( return Err(BadPeer(*who, rep::BAD_RESPONSE)) } - if request.fields.contains(message::BlockAttributes::BODY) && - blocks.iter().any(|b| b.body.is_none()) + if request.fields.contains(BlockAttributes::BODY) && blocks.iter().any(|b| b.body.is_none()) { trace!( target: "sync", @@ -2592,13 +2513,10 @@ fn validate_blocks( #[cfg(test)] mod test { - use super::{ - message::{BlockState, FromBlock}, - *, - }; - use crate::message::BlockData; + use super::*; use futures::{executor::block_on, future::poll_fn}; use sc_block_builder::BlockBuilderProvider; + use sc_network_common::sync::message::{BlockData, BlockState, FromBlock}; use sp_blockchain::HeaderBackend; use sp_consensus::block_validation::DefaultBlockAnnounceValidator; use substrate_test_runtime_client::{ @@ -2709,7 +2627,7 @@ mod test { assert!(sync.justification_requests().any(|(p, r)| { p == peer_id3 && r.fields == BlockAttributes::JUSTIFICATION && - r.from == message::FromBlock::Hash(b1_hash) && + r.from == FromBlock::Hash(b1_hash) && r.to == None })); @@ -3125,10 +3043,11 @@ mod test { let response = create_block_response(vec![block.clone()]); let on_block_data = sync.on_block_data(&peer_id1, Some(request), response).unwrap(); - request = match on_block_data.into_request() { - Some(req) => req.1, + request = if let OnBlockData::Request(_peer, request) = on_block_data { + request + } else { // We found the ancenstor - None => break, + break }; log::trace!(target: "sync", "Request: {:?}", request); @@ -3255,10 +3174,11 @@ mod test { let response = create_block_response(vec![block.clone()]); let on_block_data = sync.on_block_data(&peer_id1, Some(request), response).unwrap(); - request = match on_block_data.into_request() { - Some(req) => req.1, + request = if let OnBlockData::Request(_peer, request) = on_block_data { + request + } else { // We found the ancenstor - None => break, + break }; log::trace!(target: "sync", "Request: {:?}", request); diff --git a/client/network/sync/src/schema.rs b/client/network/sync/src/schema.rs index aa3eb84621d8f..b31005360d023 100644 --- a/client/network/sync/src/schema.rs +++ b/client/network/sync/src/schema.rs @@ -18,6 +18,6 @@ //! Include sources generated from protobuf definitions. -pub mod v1 { +pub(crate) mod v1 { include!(concat!(env!("OUT_DIR"), "/api.v1.rs")); } diff --git a/client/network/sync/src/state.rs b/client/network/sync/src/state.rs index 4041c28af0eba..e70d3b6b33a28 100644 --- a/client/network/sync/src/state.rs +++ b/client/network/sync/src/state.rs @@ -23,6 +23,7 @@ use codec::{Decode, Encode}; use log::debug; use sc_client_api::{CompactProof, ProofProvider}; use sc_consensus::ImportedState; +use sc_network_common::sync::StateDownloadProgress; use smallvec::SmallVec; use sp_core::storage::well_known_keys; use sp_runtime::traits::{Block as BlockT, Header, NumberFor}; @@ -42,15 +43,6 @@ pub struct StateSync { skip_proof: bool, } -/// Reported state download progress. -#[derive(Clone, Eq, PartialEq, Debug)] -pub struct StateDownloadProgress { - /// Estimated download percentage. - pub percentage: u32, - /// Total state size in bytes downloaded so far. - pub size: u64, -} - /// Import state chunk result. pub enum ImportResult { /// State is complete and ready for import. diff --git a/client/network/sync/src/warp.rs b/client/network/sync/src/warp.rs index d3d9d7d244153..f3fad6c1b7fdb 100644 --- a/client/network/sync/src/warp.rs +++ b/client/network/sync/src/warp.rs @@ -18,60 +18,25 @@ //! Warp sync support. -pub use crate::warp_request_handler::{ - EncodedProof, Request as WarpProofRequest, VerificationResult, WarpSyncProvider, -}; use crate::{ schema::v1::{StateRequest, StateResponse}, state::{ImportResult, StateSync}, }; use sc_client_api::ProofProvider; +use sc_network_common::sync::warp::{ + EncodedProof, VerificationResult, WarpProofRequest, WarpSyncPhase, WarpSyncProgress, + WarpSyncProvider, +}; use sp_blockchain::HeaderBackend; use sp_finality_grandpa::{AuthorityList, SetId}; use sp_runtime::traits::{Block as BlockT, NumberFor, Zero}; -use std::{fmt, sync::Arc}; +use std::sync::Arc; enum Phase { WarpProof { set_id: SetId, authorities: AuthorityList, last_hash: B::Hash }, State(StateSync), } -/// Reported warp sync phase. -#[derive(Clone, Eq, PartialEq, Debug)] -pub enum WarpSyncPhase { - /// Waiting for peers to connect. - AwaitingPeers, - /// Downloading and verifying grandpa warp proofs. - DownloadingWarpProofs, - /// Downloading state data. - DownloadingState, - /// Importing state. - ImportingState, - /// Downloading block history. - DownloadingBlocks(NumberFor), -} - -impl fmt::Display for WarpSyncPhase { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::AwaitingPeers => write!(f, "Waiting for peers"), - Self::DownloadingWarpProofs => write!(f, "Downloading finality proofs"), - Self::DownloadingState => write!(f, "Downloading state"), - Self::ImportingState => write!(f, "Importing state"), - Self::DownloadingBlocks(n) => write!(f, "Downloading block history (#{})", n), - } - } -} - -/// Reported warp sync progress. -#[derive(Clone, Eq, PartialEq, Debug)] -pub struct WarpSyncProgress { - /// Estimated download percentage. - pub phase: WarpSyncPhase, - /// Total bytes downloaded so far. - pub total_bytes: u64, -} - /// Import warp proof result. pub enum WarpProofImportResult { /// Import was successful. diff --git a/client/network/sync/src/warp_request_handler.rs b/client/network/sync/src/warp_request_handler.rs index 4f66e0a6daf17..53ec216a1e668 100644 --- a/client/network/sync/src/warp_request_handler.rs +++ b/client/network/sync/src/warp_request_handler.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. +// Copyright 2022 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify @@ -16,7 +16,7 @@ //! Helper for handling (i.e. answering) grandpa warp sync requests from a remote peer. -use codec::{Decode, Encode}; +use codec::Decode; use futures::{ channel::{mpsc, oneshot}, stream::StreamExt, @@ -27,52 +27,13 @@ use sc_network_common::{ request_responses::{ IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig, }, + sync::warp::{EncodedProof, WarpProofRequest, WarpSyncProvider}, }; use sp_runtime::traits::Block as BlockT; use std::{sync::Arc, time::Duration}; -pub use sp_finality_grandpa::{AuthorityList, SetId}; - -/// Scale-encoded warp sync proof response. -pub struct EncodedProof(pub Vec); - -/// Warp sync request -#[derive(Encode, Decode, Debug)] -pub struct Request { - /// Start collecting proofs from this block. - pub begin: B::Hash, -} - const MAX_RESPONSE_SIZE: u64 = 16 * 1024 * 1024; -/// Proof verification result. -pub enum VerificationResult { - /// Proof is valid, but the target was not reached. - Partial(SetId, AuthorityList, Block::Hash), - /// Target finality is proved. - Complete(SetId, AuthorityList, Block::Header), -} - -/// Warp sync backend. Handles retrieveing and verifying warp sync proofs. -pub trait WarpSyncProvider: Send + Sync { - /// Generate proof starting at given block hash. The proof is accumulated until maximum proof - /// size is reached. - fn generate( - &self, - start: B::Hash, - ) -> Result>; - /// Verify warp proof against current set of authorities. - fn verify( - &self, - proof: &EncodedProof, - set_id: SetId, - authorities: AuthorityList, - ) -> Result, Box>; - /// Get current list of authorities. This is supposed to be genesis authorities when starting - /// sync. - fn current_authorities(&self) -> AuthorityList; -} - /// Generates a [`RequestResponseConfig`] for the grandpa warp sync request protocol, refusing /// incoming requests. pub fn generate_request_response_config(protocol_id: ProtocolId) -> RequestResponseConfig { @@ -115,7 +76,7 @@ impl RequestHandler { payload: Vec, pending_response: oneshot::Sender, ) -> Result<(), HandleRequestError> { - let request = Request::::decode(&mut &payload[..])?; + let request = WarpProofRequest::::decode(&mut &payload[..])?; let EncodedProof(proof) = self .backend diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index fdb9befc41cb1..1aa6ebd8bf357 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -26,6 +26,8 @@ sc-client-api = { version = "4.0.0-dev", path = "../../api" } sc-consensus = { version = "0.10.0-dev", path = "../../consensus/common" } sc-network = { version = "0.10.0-dev", path = "../" } sc-network-common = { version = "0.10.0-dev", path = "../common" } +sc-network-light = { version = "0.10.0-dev", path = "../light" } +sc-network-sync = { version = "0.10.0-dev", path = "../sync" } sc-service = { version = "0.10.0-dev", default-features = false, features = ["test-helpers"], path = "../../service" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 9e752e81a3bbc..d7c83810d521d 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -48,16 +48,21 @@ use sc_consensus::{ }; pub use sc_network::config::EmptyTransactionPool; use sc_network::{ - block_request_handler::BlockRequestHandler, config::{ - MultiaddrWithPeerId, NetworkConfiguration, NonDefaultSetConfig, NonReservedPeerMode, - ProtocolConfig, Role, SyncMode, TransportConfig, + MultiaddrWithPeerId, NetworkConfiguration, NonDefaultSetConfig, NonReservedPeerMode, Role, + SyncMode, TransportConfig, }, - light_client_requests::handler::LightClientRequestHandler, - state_request_handler::StateRequestHandler, - warp_request_handler, Multiaddr, NetworkService, NetworkWorker, + Multiaddr, NetworkService, NetworkWorker, }; pub use sc_network_common::config::ProtocolId; +use sc_network_common::sync::warp::{ + AuthorityList, EncodedProof, SetId, VerificationResult, WarpSyncProvider, +}; +use sc_network_light::light_client_requests::handler::LightClientRequestHandler; +use sc_network_sync::{ + block_request_handler::BlockRequestHandler, state_request_handler::StateRequestHandler, + warp_request_handler, ChainSync, +}; use sc_service::client::Client; use sp_blockchain::{ well_known_cache_keys::{self, Id as CacheKeyId}, @@ -638,27 +643,26 @@ impl VerifierAdapter { struct TestWarpSyncProvider(Arc>); -impl warp_request_handler::WarpSyncProvider for TestWarpSyncProvider { +impl WarpSyncProvider for TestWarpSyncProvider { fn generate( &self, _start: B::Hash, - ) -> Result> { + ) -> Result> { let info = self.0.info(); let best_header = self.0.header(BlockId::hash(info.best_hash)).unwrap().unwrap(); - Ok(warp_request_handler::EncodedProof(best_header.encode())) + Ok(EncodedProof(best_header.encode())) } fn verify( &self, - proof: &warp_request_handler::EncodedProof, - _set_id: warp_request_handler::SetId, - _authorities: warp_request_handler::AuthorityList, - ) -> Result, Box> - { - let warp_request_handler::EncodedProof(encoded) = proof; + proof: &EncodedProof, + _set_id: SetId, + _authorities: AuthorityList, + ) -> Result, Box> { + let EncodedProof(encoded) = proof; let header = B::Header::decode(&mut encoded.as_slice()).unwrap(); - Ok(warp_request_handler::VerificationResult::Complete(0, Default::default(), header)) + Ok(VerificationResult::Complete(0, Default::default(), header)) } - fn current_authorities(&self) -> warp_request_handler::AuthorityList { + fn current_authorities(&self) -> AuthorityList { Default::default() } } @@ -688,7 +692,7 @@ pub struct FullPeerConfig { pub storage_chain: bool, } -pub trait TestNetFactory: Sized +pub trait TestNetFactory: Default + Sized where >::Transaction: Send, { @@ -696,14 +700,8 @@ where type BlockImport: BlockImport + Clone + Send + Sync + 'static; type PeerData: Default; - /// These two need to be implemented! - fn from_config(config: &ProtocolConfig) -> Self; - fn make_verifier( - &self, - client: PeersClient, - config: &ProtocolConfig, - peer_data: &Self::PeerData, - ) -> Self::Verifier; + /// This one needs to be implemented! + fn make_verifier(&self, client: PeersClient, peer_data: &Self::PeerData) -> Self::Verifier; /// Get reference to peer. fn peer(&mut self, i: usize) -> &mut Peer; @@ -723,15 +721,10 @@ where Self::PeerData, ); - fn default_config() -> ProtocolConfig { - ProtocolConfig::default() - } - /// Create new test network with this many peers. fn new(n: usize) -> Self { trace!(target: "test_network", "Creating test network"); - let config = Self::default_config(); - let mut net = Self::from_config(&config); + let mut net = Self::default(); for i in 0..n { trace!(target: "test_network", "Adding peer {}", i); @@ -767,11 +760,8 @@ where let (block_import, justification_import, data) = self .make_block_import(PeersClient { client: client.clone(), backend: backend.clone() }); - let verifier = self.make_verifier( - PeersClient { client: client.clone(), backend: backend.clone() }, - &Default::default(), - &data, - ); + let verifier = self + .make_verifier(PeersClient { client: client.clone(), backend: backend.clone() }, &data); let verifier = VerifierAdapter::new(verifier); let import_queue = Box::new(BasicQueue::new( @@ -845,6 +835,10 @@ where protocol_config }; + let max_parallel_downloads = network_config.max_parallel_downloads; + let block_announce_validator = config + .block_announce_validator + .unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)); let network = NetworkWorker::new(sc_network::config::Params { role: if config.is_authority { Role::Authority } else { Role::Full }, executor: None, @@ -856,9 +850,18 @@ where transaction_pool: Arc::new(EmptyTransactionPool), protocol_id, import_queue, - block_announce_validator: config - .block_announce_validator - .unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)), + create_chain_sync: Box::new(move |sync_mode, chain, warp_sync_provider| { + match ChainSync::new( + sync_mode, + chain, + block_announce_validator, + max_parallel_downloads, + warp_sync_provider, + ) { + Ok(chain_sync) => Ok(Box::new(chain_sync)), + Err(error) => Err(Box::new(error).into()), + } + }), metrics_registry: None, block_request_protocol_config, state_request_protocol_config, @@ -1012,6 +1015,7 @@ where } } +#[derive(Default)] pub struct TestNet { peers: Vec>, } @@ -1021,17 +1025,7 @@ impl TestNetFactory for TestNet { type PeerData = (); type BlockImport = PeersClient; - /// Create new test network with peers and given config. - fn from_config(_config: &ProtocolConfig) -> Self { - TestNet { peers: Vec::new() } - } - - fn make_verifier( - &self, - _client: PeersClient, - _config: &ProtocolConfig, - _peer_data: &(), - ) -> Self::Verifier { + fn make_verifier(&self, _client: PeersClient, _peer_data: &()) -> Self::Verifier { PassThroughVerifier::new(false) } @@ -1081,6 +1075,7 @@ impl JustificationImport for ForceFinalized { } } +#[derive(Default)] pub struct JustificationTestNet(TestNet); impl TestNetFactory for JustificationTestNet { @@ -1088,17 +1083,8 @@ impl TestNetFactory for JustificationTestNet { type PeerData = (); type BlockImport = PeersClient; - fn from_config(config: &ProtocolConfig) -> Self { - JustificationTestNet(TestNet::from_config(config)) - } - - fn make_verifier( - &self, - client: PeersClient, - config: &ProtocolConfig, - peer_data: &(), - ) -> Self::Verifier { - self.0.make_verifier(client, config, peer_data) + fn make_verifier(&self, client: PeersClient, peer_data: &()) -> Self::Verifier { + self.0.make_verifier(client, peer_data) } fn peer(&mut self, i: usize) -> &mut Peer { diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index ff11dd1344d01..31b0c860cf190 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -50,8 +50,10 @@ sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/comm sc-consensus = { version = "0.10.0-dev", path = "../../client/consensus/common" } sp-inherents = { version = "4.0.0-dev", path = "../../primitives/inherents" } sp-storage = { version = "6.0.0", path = "../../primitives/storage" } -sc-network-common = { version = "0.10.0-dev", path = "../network/common" } sc-network = { version = "0.10.0-dev", path = "../network" } +sc-network-common = { version = "0.10.0-dev", path = "../network/common" } +sc-network-light = { version = "0.10.0-dev", path = "../network/light" } +sc-network-sync = { version = "0.10.0-dev", path = "../network/sync" } sc-chain-spec = { version = "4.0.0-dev", path = "../chain-spec" } sc-client-api = { version = "4.0.0-dev", path = "../api" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 5319bf24d5e72..fe0ae5db53e70 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -38,13 +38,17 @@ use sc_consensus::import_queue::ImportQueue; use sc_executor::RuntimeVersionOf; use sc_keystore::LocalKeystore; use sc_network::{ - block_request_handler::{self, BlockRequestHandler}, config::{Role, SyncMode}, - light_client_requests::{self, handler::LightClientRequestHandler}, - state_request_handler::{self, StateRequestHandler}, - warp_request_handler::{self, RequestHandler as WarpSyncRequestHandler, WarpSyncProvider}, NetworkService, }; +use sc_network_common::sync::warp::WarpSyncProvider; +use sc_network_light::light_client_requests::{self, handler::LightClientRequestHandler}; +use sc_network_sync::{ + block_request_handler::{self, BlockRequestHandler}, + state_request_handler::{self, StateRequestHandler}, + warp_request_handler::{self, RequestHandler as WarpSyncRequestHandler}, + ChainSync, +}; use sc_rpc::{ author::AuthorApiServer, chain::ChainApiServer, @@ -801,6 +805,7 @@ where } }; + let max_parallel_downloads = config.network.max_parallel_downloads; let network_params = sc_network::config::Params { role: config.role.clone(), executor: { @@ -818,9 +823,20 @@ where network_config: config.network.clone(), chain: client.clone(), transaction_pool: transaction_pool_adapter as _, - import_queue: Box::new(import_queue), protocol_id, - block_announce_validator, + import_queue: Box::new(import_queue), + create_chain_sync: Box::new( + move |sync_mode, chain, warp_sync_provider| match ChainSync::new( + sync_mode, + chain, + block_announce_validator, + max_parallel_downloads, + warp_sync_provider, + ) { + Ok(chain_sync) => Ok(Box::new(chain_sync)), + Err(error) => Err(Box::new(error).into()), + }, + ), metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()), block_request_protocol_config, state_request_protocol_config, From 42044053b5895991d06ddc776e45da61c6462c78 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:09:28 +0100 Subject: [PATCH 06/46] Fix off by one error in proportional slashing (#11782) * Fix proportional slashing logic * Update frame/nomination-pools/test-staking/src/lib.rs Co-authored-by: David * Update frame/staking/src/lib.rs Co-authored-by: David * Update frame/staking/src/lib.rs Co-authored-by: David * Update frame/staking/src/lib.rs Co-authored-by: David * fmt * Update frame/nomination-pools/test-staking/src/lib.rs * clean * fix * last fixes * doc Co-authored-by: David --- .../nomination-pools/test-staking/src/lib.rs | 256 +++++++++++++++++- .../nomination-pools/test-staking/src/mock.rs | 7 +- frame/staking/src/lib.rs | 95 +++++-- frame/staking/src/tests.rs | 30 +- 4 files changed, 342 insertions(+), 46 deletions(-) diff --git a/frame/nomination-pools/test-staking/src/lib.rs b/frame/nomination-pools/test-staking/src/lib.rs index 2e40e8c6d917d..be7752ce113b4 100644 --- a/frame/nomination-pools/test-staking/src/lib.rs +++ b/frame/nomination-pools/test-staking/src/lib.rs @@ -22,7 +22,8 @@ mod mock; use frame_support::{assert_noop, assert_ok, bounded_btree_map, traits::Currency}; use mock::*; use pallet_nomination_pools::{ - Error as PoolsError, Event as PoolsEvent, LastPoolId, PoolMember, PoolMembers, PoolState, + BondedPools, Error as PoolsError, Event as PoolsEvent, LastPoolId, PoolMember, PoolMembers, + PoolState, }; use pallet_staking::{CurrentEra, Event as StakingEvent, Payee, RewardDestination}; @@ -273,7 +274,7 @@ fn pool_slash_e2e() { 30, &mut Default::default(), &mut Default::default(), - 1, // slash era 1, affects chunks at era 5 onwards. + 2, // slash era 2, affects chunks at era 5 onwards. ); assert_eq!(staking_events_since_last_call(), vec![StakingEvent::Slashed(POOL1_BONDED, 30)]); @@ -371,3 +372,254 @@ fn pool_slash_e2e() { ); }); } + +#[test] +fn pool_slash_proportional() { + // a typical example where 3 pool members unbond in era 99, 100, and 101, and a slash that + // happened in era 100 should only affect the latter two. + new_test_ext().execute_with(|| { + ExistentialDeposit::set(1); + BondingDuration::set(28); + assert_eq!(Balances::minimum_balance(), 1); + assert_eq!(Staking::current_era(), None); + + // create the pool, we know this has id 1. + assert_ok!(Pools::create(Origin::signed(10), 40, 10, 10, 10)); + assert_eq!(LastPoolId::::get(), 1); + + assert_eq!(staking_events_since_last_call(), vec![StakingEvent::Bonded(POOL1_BONDED, 40)]); + assert_eq!( + pool_events_since_last_call(), + vec![ + PoolsEvent::Created { depositor: 10, pool_id: 1 }, + PoolsEvent::Bonded { member: 10, pool_id: 1, bonded: 40, joined: true }, + ] + ); + + // have two members join + let bond = 20; + assert_ok!(Pools::join(Origin::signed(20), bond, 1)); + assert_ok!(Pools::join(Origin::signed(21), bond, 1)); + assert_ok!(Pools::join(Origin::signed(22), bond, 1)); + + assert_eq!( + staking_events_since_last_call(), + vec![ + StakingEvent::Bonded(POOL1_BONDED, bond), + StakingEvent::Bonded(POOL1_BONDED, bond), + StakingEvent::Bonded(POOL1_BONDED, bond), + ] + ); + assert_eq!( + pool_events_since_last_call(), + vec![ + PoolsEvent::Bonded { member: 20, pool_id: 1, bonded: bond, joined: true }, + PoolsEvent::Bonded { member: 21, pool_id: 1, bonded: bond, joined: true }, + PoolsEvent::Bonded { member: 22, pool_id: 1, bonded: bond, joined: true }, + ] + ); + + // now let's progress a lot. + CurrentEra::::set(Some(99)); + + // and unbond + assert_ok!(Pools::unbond(Origin::signed(20), 20, bond)); + + assert_eq!( + staking_events_since_last_call(), + vec![StakingEvent::Unbonded(POOL1_BONDED, bond),] + ); + assert_eq!( + pool_events_since_last_call(), + vec![PoolsEvent::Unbonded { member: 20, pool_id: 1, balance: bond, points: bond }] + ); + + CurrentEra::::set(Some(100)); + assert_ok!(Pools::unbond(Origin::signed(21), 21, bond)); + assert_eq!( + staking_events_since_last_call(), + vec![StakingEvent::Unbonded(POOL1_BONDED, bond),] + ); + assert_eq!( + pool_events_since_last_call(), + vec![PoolsEvent::Unbonded { member: 21, pool_id: 1, balance: bond, points: bond }] + ); + + CurrentEra::::set(Some(101)); + assert_ok!(Pools::unbond(Origin::signed(22), 22, bond)); + assert_eq!( + staking_events_since_last_call(), + vec![StakingEvent::Unbonded(POOL1_BONDED, bond),] + ); + assert_eq!( + pool_events_since_last_call(), + vec![PoolsEvent::Unbonded { member: 22, pool_id: 1, balance: bond, points: bond }] + ); + + // Apply a slash that happened in era 100. This is typically applied with a delay. + // Of the total 100, 50 is slashed. + assert_eq!(BondedPools::::get(1).unwrap().points, 40); + pallet_staking::slashing::do_slash::( + &POOL1_BONDED, + 50, + &mut Default::default(), + &mut Default::default(), + 100, + ); + + assert_eq!(staking_events_since_last_call(), vec![StakingEvent::Slashed(POOL1_BONDED, 50)]); + assert_eq!( + pool_events_since_last_call(), + vec![ + // This last pool got slashed only the leftover dust. Otherwise in principle, this + // chunk/pool should have not been affected. + PoolsEvent::UnbondingPoolSlashed { pool_id: 1, era: 127, balance: 19 }, + // This pool got slashed 12.5, which rounded down to 12. + PoolsEvent::UnbondingPoolSlashed { pool_id: 1, era: 128, balance: 8 }, + // This pool got slashed 12.5, which rounded down to 12. + PoolsEvent::UnbondingPoolSlashed { pool_id: 1, era: 129, balance: 8 }, + // Bonded pool got slashed for 25, remaining 15 in it. + PoolsEvent::PoolSlashed { pool_id: 1, balance: 15 } + ] + ); + }); +} + +#[test] +fn pool_slash_non_proportional_only_bonded_pool() { + // A typical example where a pool member unbonds in era 99, and he can get away with a slash + // that happened in era 100, as long as the pool has enough active bond to cover the slash. If + // everything else in the slashing/staking system works, this should always be the case. + // Nonetheless, `ledger.slash` has been written such that it will slash greedily from any chunk + // if it runs out of chunks that it thinks should be affected by the slash. + new_test_ext().execute_with(|| { + ExistentialDeposit::set(1); + BondingDuration::set(28); + assert_eq!(Balances::minimum_balance(), 1); + assert_eq!(Staking::current_era(), None); + + // create the pool, we know this has id 1. + assert_ok!(Pools::create(Origin::signed(10), 40, 10, 10, 10)); + assert_eq!(staking_events_since_last_call(), vec![StakingEvent::Bonded(POOL1_BONDED, 40)]); + assert_eq!( + pool_events_since_last_call(), + vec![ + PoolsEvent::Created { depositor: 10, pool_id: 1 }, + PoolsEvent::Bonded { member: 10, pool_id: 1, bonded: 40, joined: true }, + ] + ); + + // have two members join + let bond = 20; + assert_ok!(Pools::join(Origin::signed(20), bond, 1)); + assert_eq!( + staking_events_since_last_call(), + vec![StakingEvent::Bonded(POOL1_BONDED, bond)] + ); + assert_eq!( + pool_events_since_last_call(), + vec![PoolsEvent::Bonded { member: 20, pool_id: 1, bonded: bond, joined: true }] + ); + + // progress and unbond. + CurrentEra::::set(Some(99)); + assert_ok!(Pools::unbond(Origin::signed(20), 20, bond)); + assert_eq!( + staking_events_since_last_call(), + vec![StakingEvent::Unbonded(POOL1_BONDED, bond)] + ); + assert_eq!( + pool_events_since_last_call(), + vec![PoolsEvent::Unbonded { member: 20, pool_id: 1, balance: bond, points: bond }] + ); + + // slash for 30. This will be deducted only from the bonded pool. + CurrentEra::::set(Some(100)); + assert_eq!(BondedPools::::get(1).unwrap().points, 40); + pallet_staking::slashing::do_slash::( + &POOL1_BONDED, + 30, + &mut Default::default(), + &mut Default::default(), + 100, + ); + + assert_eq!(staking_events_since_last_call(), vec![StakingEvent::Slashed(POOL1_BONDED, 30)]); + assert_eq!( + pool_events_since_last_call(), + vec![PoolsEvent::PoolSlashed { pool_id: 1, balance: 10 }] + ); + }); +} + +#[test] +fn pool_slash_non_proportional_bonded_pool_and_chunks() { + // An uncommon example where even though some funds are unlocked such that they should not be + // affected by a slash, we still slash out of them. This should not happen at all. If a + // nomination has unbonded, from the next era onwards, their exposure will drop, so if an era + // happens in that era, then their share of that slash should naturally be less, such that only + // their active ledger stake is enough to compensate it. + new_test_ext().execute_with(|| { + ExistentialDeposit::set(1); + BondingDuration::set(28); + assert_eq!(Balances::minimum_balance(), 1); + assert_eq!(Staking::current_era(), None); + + // create the pool, we know this has id 1. + assert_ok!(Pools::create(Origin::signed(10), 40, 10, 10, 10)); + assert_eq!(staking_events_since_last_call(), vec![StakingEvent::Bonded(POOL1_BONDED, 40)]); + assert_eq!( + pool_events_since_last_call(), + vec![ + PoolsEvent::Created { depositor: 10, pool_id: 1 }, + PoolsEvent::Bonded { member: 10, pool_id: 1, bonded: 40, joined: true }, + ] + ); + + // have two members join + let bond = 20; + assert_ok!(Pools::join(Origin::signed(20), bond, 1)); + assert_eq!( + staking_events_since_last_call(), + vec![StakingEvent::Bonded(POOL1_BONDED, bond)] + ); + assert_eq!( + pool_events_since_last_call(), + vec![PoolsEvent::Bonded { member: 20, pool_id: 1, bonded: bond, joined: true }] + ); + + // progress and unbond. + CurrentEra::::set(Some(99)); + assert_ok!(Pools::unbond(Origin::signed(20), 20, bond)); + assert_eq!( + staking_events_since_last_call(), + vec![StakingEvent::Unbonded(POOL1_BONDED, bond)] + ); + assert_eq!( + pool_events_since_last_call(), + vec![PoolsEvent::Unbonded { member: 20, pool_id: 1, balance: bond, points: bond }] + ); + + // slash 50. This will be deducted only from the bonded pool and one of the unbonding pools. + CurrentEra::::set(Some(100)); + assert_eq!(BondedPools::::get(1).unwrap().points, 40); + pallet_staking::slashing::do_slash::( + &POOL1_BONDED, + 50, + &mut Default::default(), + &mut Default::default(), + 100, + ); + + assert_eq!(staking_events_since_last_call(), vec![StakingEvent::Slashed(POOL1_BONDED, 50)]); + assert_eq!( + pool_events_since_last_call(), + vec![ + // out of 20, 10 was taken. + PoolsEvent::UnbondingPoolSlashed { pool_id: 1, era: 127, balance: 10 }, + // out of 40, all was taken. + PoolsEvent::PoolSlashed { pool_id: 1, balance: 0 } + ] + ); + }); +} diff --git a/frame/nomination-pools/test-staking/src/mock.rs b/frame/nomination-pools/test-staking/src/mock.rs index 7b720c009b29b..d13d8aa341c12 100644 --- a/frame/nomination-pools/test-staking/src/mock.rs +++ b/frame/nomination-pools/test-staking/src/mock.rs @@ -24,6 +24,8 @@ type AccountIndex = u32; type BlockNumber = u64; type Balance = u128; +pub(crate) type T = Runtime; + pub(crate) const POOL1_BONDED: AccountId = 20318131474730217858575332831085u128; pub(crate) const POOL1_REWARD: AccountId = 20397359637244482196168876781421u128; @@ -194,13 +196,14 @@ frame_support::construct_runtime!( ); pub fn new_test_ext() -> sp_io::TestExternalities { + sp_tracing::try_init_simple(); let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); let _ = pallet_nomination_pools::GenesisConfig:: { min_join_bond: 2, min_create_bond: 2, max_pools: Some(3), - max_members_per_pool: Some(3), - max_members: Some(3 * 3), + max_members_per_pool: Some(5), + max_members: Some(3 * 5), } .assimilate_storage(&mut storage) .unwrap(); diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 360d5b5efb58f..50c033bdc7e28 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -529,13 +529,26 @@ impl StakingLedger { (self, unlocking_balance) } - /// Slash the staker for a given amount of balance. This can grow the value of the slash in the - /// case that either the active bonded or some unlocking chunks become dust after slashing. - /// Returns the amount of funds actually slashed. + /// Slash the staker for a given amount of balance. /// - /// `slash_era` is the era in which the slash (which is being enacted now) actually happened. + /// This implements a proportional slashing system, whereby we set our preference to slash as + /// such: + /// + /// - If any unlocking chunks exist that are scheduled to be unlocked at `slash_era + + /// bonding_duration` and onwards, the slash is divided equally between the active ledger and + /// the unlocking chunks. + /// - If no such chunks exist, then only the active balance is slashed. + /// + /// Note that the above is only a *preference*. If for any reason the active ledger, with or + /// without some portion of the unlocking chunks that are more justified to be slashed are not + /// enough, then the slashing will continue and will consume as much of the active and unlocking + /// chunks as needed. /// - /// # Note + /// This will never slash more than the given amount. If any of the chunks become dusted, the + /// last chunk is slashed slightly less to compensate. Returns the amount of funds actually + /// slashed. + /// + /// `slash_era` is the era in which the slash (which is being enacted now) actually happened. /// /// This calls `Config::OnStakerSlash::on_slash` with information as to how the slash was /// applied. @@ -545,54 +558,81 @@ impl StakingLedger { minimum_balance: BalanceOf, slash_era: EraIndex, ) -> BalanceOf { - use sp_staking::OnStakerSlash as _; - if slash_amount.is_zero() { return Zero::zero() } + use sp_staking::OnStakerSlash as _; let mut remaining_slash = slash_amount; let pre_slash_total = self.total; - let era_after_slash = slash_era + 1; - let chunk_unlock_era_after_slash = era_after_slash + T::BondingDuration::get(); + // for a `slash_era = x`, any chunk that is scheduled to be unlocked at era `x + 28` + // (assuming 28 is the bonding duration) onwards should be slashed. + let slashable_chunks_start = slash_era + T::BondingDuration::get(); - // Calculate the total balance of active funds and unlocking funds in the affected range. - let (affected_balance, slash_chunks_priority): (_, Box>) = { - if let Some(start_index) = - self.unlocking.iter().position(|c| c.era >= chunk_unlock_era_after_slash) + // `Some(ratio)` if this is proportional, with `ratio`, `None` otherwise. In both cases, we + // slash first the active chunk, and then `slash_chunks_priority`. + let (maybe_proportional, slash_chunks_priority) = { + if let Some(first_slashable_index) = + self.unlocking.iter().position(|c| c.era >= slashable_chunks_start) { + // If there exists a chunk who's after the first_slashable_start, then this is a + // proportional slash, because we want to slash active and these chunks + // proportionally. + // The indices of the first chunk after the slash up through the most recent chunk. // (The most recent chunk is at greatest from this era) - let affected_indices = start_index..self.unlocking.len(); + let affected_indices = first_slashable_index..self.unlocking.len(); let unbonding_affected_balance = affected_indices.clone().fold(BalanceOf::::zero(), |sum, i| { - if let Some(chunk) = self.unlocking.get_mut(i).defensive() { + if let Some(chunk) = self.unlocking.get(i).defensive() { sum.saturating_add(chunk.value) } else { sum } }); + let affected_balance = self.active.saturating_add(unbonding_affected_balance); + let ratio = Perquintill::from_rational(slash_amount, affected_balance); ( - self.active.saturating_add(unbonding_affected_balance), - Box::new(affected_indices.chain((0..start_index).rev())), + Some(ratio), + affected_indices.chain((0..first_slashable_index).rev()).collect::>(), ) } else { - (self.active, Box::new((0..self.unlocking.len()).rev())) + // We just slash from the last chunk to the most recent one, if need be. + (None, (0..self.unlocking.len()).rev().collect::>()) } }; // Helper to update `target` and the ledgers total after accounting for slashing `target`. - let ratio = Perquintill::from_rational(slash_amount, affected_balance); + log!( + debug, + "slashing {:?} for era {:?} out of {:?}, priority: {:?}, proportional = {:?}", + slash_amount, + slash_era, + self, + slash_chunks_priority, + maybe_proportional, + ); + let mut slash_out_of = |target: &mut BalanceOf, slash_remaining: &mut BalanceOf| { - let mut slash_from_target = - if slash_amount < affected_balance { ratio * (*target) } else { *slash_remaining } - .min(*target); + let mut slash_from_target = if let Some(ratio) = maybe_proportional { + ratio * (*target) + } else { + *slash_remaining + } + // this is the total that that the slash target has. We can't slash more than + // this anyhow! + .min(*target) + // this is the total amount that we would have wanted to slash + // non-proportionally, a proportional slash should never exceed this either! + .min(*slash_remaining); // slash out from *target exactly `slash_from_target`. *target = *target - slash_from_target; if *target < minimum_balance { - // Slash the rest of the target if its dust + // Slash the rest of the target if it's dust. This might cause the last chunk to be + // slightly under-slashed, by at most `MaxUnlockingChunks * ED`, which is not a big + // deal. slash_from_target = sp_std::mem::replace(target, Zero::zero()).saturating_add(slash_from_target) } @@ -606,10 +646,11 @@ impl StakingLedger { let mut slashed_unlocking = BTreeMap::<_, _>::new(); for i in slash_chunks_priority { + if remaining_slash.is_zero() { + break + } + if let Some(chunk) = self.unlocking.get_mut(i).defensive() { - if remaining_slash.is_zero() { - break - } slash_out_of(&mut chunk.value, &mut remaining_slash); // write the new slashed value of this chunk to the map. slashed_unlocking.insert(chunk.era, chunk.value); @@ -618,7 +659,9 @@ impl StakingLedger { } } + // clean unlocking chunks that are set to zero. self.unlocking.retain(|c| !c.value.is_zero()); + T::OnStakerSlash::on_slash(&self.stash, self.active, &slashed_unlocking); pre_slash_total.saturating_sub(self.total) } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 9a13a818f4b59..b76126f0c5d04 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -2081,8 +2081,7 @@ fn reward_validator_slashing_validator_does_not_overflow() { let _ = Balances::make_free_balance_be(&11, stake); let _ = Balances::make_free_balance_be(&2, stake); - // only slashes out of bonded stake are applied. without this line, - // it is 0. + // only slashes out of bonded stake are applied. without this line, it is 0. Staking::bond(Origin::signed(2), 20000, stake - 1, RewardDestination::default()).unwrap(); // Override exposure of 11 ErasStakers::::insert( @@ -2104,7 +2103,7 @@ fn reward_validator_slashing_validator_does_not_overflow() { &[Perbill::from_percent(100)], ); - assert_eq!(Balances::total_balance(&11), stake); + assert_eq!(Balances::total_balance(&11), stake - 1); assert_eq!(Balances::total_balance(&2), 1); }) } @@ -4960,7 +4959,6 @@ fn proportional_ledger_slash_works() { unlocking: bounded_vec![], claimed_rewards: vec![], }; - assert_eq!(BondingDuration::get(), 3); // When we slash a ledger with no unlocking chunks @@ -4997,7 +4995,7 @@ fn proportional_ledger_slash_works() { ledger.total = 4 * 100; ledger.active = 0; // When the first 2 chunks don't overlap with the affected range of unlock eras. - assert_eq!(ledger.slash(140, 0, 2), 140); + assert_eq!(ledger.slash(140, 0, 3), 140); // Then assert_eq!(ledger.unlocking, vec![c(4, 100), c(5, 100), c(6, 30), c(7, 30)]); assert_eq!(ledger.total, 4 * 100 - 140); @@ -5039,7 +5037,7 @@ fn proportional_ledger_slash_works() { ledger.active = 500; ledger.total = 40 + 10 + 100 + 250 + 500; // 900 assert_eq!(ledger.total, 900); - // When we have a higher min balance + // When we have a higher min balance assert_eq!( ledger.slash( 900 / 2, @@ -5047,16 +5045,17 @@ fn proportional_ledger_slash_works() { * get swept */ 0 ), - 475 + 450 ); - let dust = (10 / 2) + (40 / 2); assert_eq!(ledger.active, 500 / 2); - assert_eq!(ledger.unlocking, vec![c(5, 100 / 2), c(7, 250 / 2)]); - assert_eq!(ledger.total, 900 / 2 - dust); + // the last chunk was not slashed 50% like all the rest, because some other earlier chunks got + // dusted. + assert_eq!(ledger.unlocking, vec![c(5, 100 / 2), c(7, 150)]); + assert_eq!(ledger.total, 900 / 2); assert_eq!(LedgerSlashPerEra::get().0, 500 / 2); assert_eq!( LedgerSlashPerEra::get().1, - BTreeMap::from([(4, 0), (5, 100 / 2), (6, 0), (7, 250 / 2)]) + BTreeMap::from([(4, 0), (5, 100 / 2), (6, 0), (7, 150)]) ); // Given @@ -5068,7 +5067,7 @@ fn proportional_ledger_slash_works() { ledger.slash( 500 + 10 + 250 + 100 / 2, // active + era 6 + era 7 + era 5 / 2 0, - 2 /* slash era 2+4 first, so the affected parts are era 2+4, era 3+4 and + 3 /* slash era 6 first, so the affected parts are era 6, era 7 and * ledge.active. This will cause the affected to go to zero, and then we will * start slashing older chunks */ ), @@ -5091,7 +5090,7 @@ fn proportional_ledger_slash_works() { ledger.slash( 351, // active + era 6 + era 7 + era 5 / 2 + 1 50, // min balance - everything slashed below 50 will get dusted - 2 /* slash era 2+4 first, so the affected parts are era 2+4, era 3+4 and + 3 /* slash era 3+3 first, so the affected parts are era 6, era 7 and * ledge.active. This will cause the affected to go to zero, and then we will * start slashing older chunks */ ), @@ -5108,9 +5107,8 @@ fn proportional_ledger_slash_works() { // Given let slash = u64::MAX as Balance * 2; - let value = slash - - (9 * 4) // The value of the other parts of ledger that will get slashed - + 1; + // The value of the other parts of ledger that will get slashed + let value = slash - (10 * 4); ledger.active = 10; ledger.unlocking = bounded_vec![c(4, 10), c(5, 10), c(6, 10), c(7, value)]; From 9dad17f77b8a56ccf0d289ae410ee1e61d9b12a1 Mon Sep 17 00:00:00 2001 From: Jegor Sidorenko <5252494+jsidorenko@users.noreply.github.com> Date: Wed, 13 Jul 2022 13:41:20 +0100 Subject: [PATCH 07/46] Buy&Sell methods for Uniques (#11398) * Allow to set item's price * Clean the state when we transfer/burn an item or destroy a collection * Allow to buy an item * Remove redundant checks * Improve events * Cover with tests * Add comments * Apply suggestions * Fmt * Improvements for price validation * Improve validation * Update to use the new terminology * Remove multi-assets support * Chore * Weights + benchmarking * Shield against human error * Test when we pass the higher item's price * fmt fix * Chore * cargo run --quiet --profile=production --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark pallet --chain=dev --steps=50 --repeat=20 --pallet=pallet_uniques --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/uniques/src/weights.rs --template=./.maintain/frame-weight-template.hbs * Remove is_frozen check when setting the price * Try to fix benchmarking * Fix benchmarking * cargo run --quiet --profile=production --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark pallet --chain=dev --steps=50 --repeat=20 --pallet=pallet_uniques --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/uniques/src/weights.rs --template=./.maintain/frame-weight-template.hbs * Add transactional * Add 'allow deprecated' flag for transactional * Remove #[allow(deprecated)] * ".git/.scripts/bench-bot.sh" pallet dev pallet_uniques Co-authored-by: Parity Bot Co-authored-by: command-bot <> --- frame/uniques/src/benchmarking.rs | 36 ++++++ frame/uniques/src/functions.rs | 79 ++++++++++++- frame/uniques/src/lib.rs | 88 ++++++++++++++- frame/uniques/src/tests.rs | 178 +++++++++++++++++++++++++++++- frame/uniques/src/types.rs | 2 + frame/uniques/src/weights.rs | 174 +++++++++++++++++------------ 6 files changed, 485 insertions(+), 72 deletions(-) diff --git a/frame/uniques/src/benchmarking.rs b/frame/uniques/src/benchmarking.rs index 14a3e2c61809a..713393048f7e9 100644 --- a/frame/uniques/src/benchmarking.rs +++ b/frame/uniques/src/benchmarking.rs @@ -408,5 +408,41 @@ benchmarks_instance_pallet! { }.into()); } + set_price { + let (collection, caller, _) = create_collection::(); + let (item, ..) = mint_item::(0); + let delegate: T::AccountId = account("delegate", 0, SEED); + let delegate_lookup = T::Lookup::unlookup(delegate.clone()); + let price = ItemPrice::::from(100u32); + }: _(SystemOrigin::Signed(caller.clone()), collection, item, Some(price), Some(delegate_lookup)) + verify { + assert_last_event::(Event::ItemPriceSet { + collection, + item, + price, + whitelisted_buyer: Some(delegate), + }.into()); + } + + buy_item { + let (collection, seller, _) = create_collection::(); + let (item, ..) = mint_item::(0); + let buyer: T::AccountId = account("buyer", 0, SEED); + let buyer_lookup = T::Lookup::unlookup(buyer.clone()); + let price = ItemPrice::::from(0u32); + let origin = SystemOrigin::Signed(seller.clone()).into(); + Uniques::::set_price(origin, collection, item, Some(price.clone()), Some(buyer_lookup))?; + T::Currency::make_free_balance_be(&buyer, DepositBalanceOf::::max_value()); + }: _(SystemOrigin::Signed(buyer.clone()), collection, item, price.clone()) + verify { + assert_last_event::(Event::ItemBought { + collection, + item, + price, + seller, + buyer, + }.into()); + } + impl_benchmark_test_suite!(Uniques, crate::mock::new_test_ext(), crate::mock::Test); } diff --git a/frame/uniques/src/functions.rs b/frame/uniques/src/functions.rs index 155fb35ef0999..9e8d3301616df 100644 --- a/frame/uniques/src/functions.rs +++ b/frame/uniques/src/functions.rs @@ -18,7 +18,10 @@ //! Various pieces of common functionality. use super::*; -use frame_support::{ensure, traits::Get}; +use frame_support::{ + ensure, + traits::{ExistenceRequirement, Get}, +}; use sp_runtime::{DispatchError, DispatchResult}; impl, I: 'static> Pallet { @@ -46,6 +49,7 @@ impl, I: 'static> Pallet { let origin = details.owner; details.owner = dest; Item::::insert(&collection, &item, &details); + ItemPriceOf::::remove(&collection, &item); Self::deposit_event(Event::Transferred { collection, @@ -112,6 +116,8 @@ impl, I: 'static> Pallet { } #[allow(deprecated)] ItemMetadataOf::::remove_prefix(&collection, None); + #[allow(deprecated)] + ItemPriceOf::::remove_prefix(&collection, None); CollectionMetadataOf::::remove(&collection); #[allow(deprecated)] Attribute::::remove_prefix((&collection,), None); @@ -196,8 +202,79 @@ impl, I: 'static> Pallet { Item::::remove(&collection, &item); Account::::remove((&owner, &collection, &item)); + ItemPriceOf::::remove(&collection, &item); Self::deposit_event(Event::Burned { collection, item, owner }); Ok(()) } + + pub fn do_set_price( + collection: T::CollectionId, + item: T::ItemId, + sender: T::AccountId, + price: Option>, + whitelisted_buyer: Option, + ) -> DispatchResult { + let details = Item::::get(&collection, &item).ok_or(Error::::UnknownItem)?; + ensure!(details.owner == sender, Error::::NoPermission); + + if let Some(ref price) = price { + ItemPriceOf::::insert( + &collection, + &item, + (price.clone(), whitelisted_buyer.clone()), + ); + Self::deposit_event(Event::ItemPriceSet { + collection, + item, + price: price.clone(), + whitelisted_buyer, + }); + } else { + ItemPriceOf::::remove(&collection, &item); + Self::deposit_event(Event::ItemPriceRemoved { collection, item }); + } + + Ok(()) + } + + pub fn do_buy_item( + collection: T::CollectionId, + item: T::ItemId, + buyer: T::AccountId, + bid_price: ItemPrice, + ) -> DispatchResult { + let details = Item::::get(&collection, &item).ok_or(Error::::UnknownItem)?; + ensure!(details.owner != buyer, Error::::NoPermission); + + let price_info = + ItemPriceOf::::get(&collection, &item).ok_or(Error::::NotForSale)?; + + ensure!(bid_price >= price_info.0, Error::::BidTooLow); + + if let Some(only_buyer) = price_info.1 { + ensure!(only_buyer == buyer, Error::::NoPermission); + } + + T::Currency::transfer( + &buyer, + &details.owner, + price_info.0, + ExistenceRequirement::KeepAlive, + )?; + + let old_owner = details.owner.clone(); + + Self::do_transfer(collection, item, buyer.clone(), |_, _| Ok(()))?; + + Self::deposit_event(Event::ItemBought { + collection, + item, + price: price_info.0, + seller: old_owner, + buyer, + }); + + Ok(()) + } } diff --git a/frame/uniques/src/lib.rs b/frame/uniques/src/lib.rs index 5c7adeac0eff2..53f83605da90e 100644 --- a/frame/uniques/src/lib.rs +++ b/frame/uniques/src/lib.rs @@ -24,6 +24,7 @@ //! * [`System`](../frame_system/index.html) //! * [`Support`](../frame_support/index.html) +#![recursion_limit = "256"] // Ensure we're `no_std` when compiling for Wasm. #![cfg_attr(not(feature = "std"), no_std)] @@ -42,8 +43,11 @@ pub mod migration; pub mod weights; use codec::{Decode, Encode}; -use frame_support::traits::{ - tokens::Locker, BalanceStatus::Reserved, Currency, EnsureOriginWithArg, ReservableCurrency, +use frame_support::{ + traits::{ + tokens::Locker, BalanceStatus::Reserved, Currency, EnsureOriginWithArg, ReservableCurrency, + }, + transactional, }; use frame_system::Config as SystemConfig; use sp_runtime::{ @@ -245,6 +249,18 @@ pub mod pallet { OptionQuery, >; + #[pallet::storage] + /// Price of an asset instance. + pub(super) type ItemPriceOf, I: 'static = ()> = StorageDoubleMap< + _, + Blake2_128Concat, + T::CollectionId, + Blake2_128Concat, + T::ItemId, + (ItemPrice, Option), + OptionQuery, + >; + #[pallet::storage] /// Keeps track of the number of items a collection might have. pub(super) type CollectionMaxSupply, I: 'static = ()> = @@ -341,6 +357,23 @@ pub mod pallet { OwnershipAcceptanceChanged { who: T::AccountId, maybe_collection: Option }, /// Max supply has been set for a collection. CollectionMaxSupplySet { collection: T::CollectionId, max_supply: u32 }, + /// The price was set for the instance. + ItemPriceSet { + collection: T::CollectionId, + item: T::ItemId, + price: ItemPrice, + whitelisted_buyer: Option, + }, + /// The price for the instance was removed. + ItemPriceRemoved { collection: T::CollectionId, item: T::ItemId }, + /// An item was bought. + ItemBought { + collection: T::CollectionId, + item: T::ItemId, + price: ItemPrice, + seller: T::AccountId, + buyer: T::AccountId, + }, } #[pallet::error] @@ -375,6 +408,12 @@ pub mod pallet { MaxSupplyAlreadySet, /// The provided max supply is less to the amount of items a collection already has. MaxSupplyTooSmall, + /// The given item ID is unknown. + UnknownItem, + /// Item is not for sale. + NotForSale, + /// The provided bid is too low. + BidTooLow, } impl, I: 'static> Pallet { @@ -1408,5 +1447,50 @@ pub mod pallet { Self::deposit_event(Event::CollectionMaxSupplySet { collection, max_supply }); Ok(()) } + + /// Set (or reset) the price for an item. + /// + /// Origin must be Signed and must be the owner of the asset `item`. + /// + /// - `collection`: The collection of the item. + /// - `item`: The item to set the price for. + /// - `price`: The price for the item. Pass `None`, to reset the price. + /// - `buyer`: Restricts the buy operation to a specific account. + /// + /// Emits `ItemPriceSet` on success if the price is not `None`. + /// Emits `ItemPriceRemoved` on success if the price is `None`. + #[pallet::weight(T::WeightInfo::set_price())] + pub fn set_price( + origin: OriginFor, + collection: T::CollectionId, + item: T::ItemId, + price: Option>, + whitelisted_buyer: Option<::Source>, + ) -> DispatchResult { + let origin = ensure_signed(origin)?; + let whitelisted_buyer = whitelisted_buyer.map(T::Lookup::lookup).transpose()?; + Self::do_set_price(collection, item, origin, price, whitelisted_buyer) + } + + /// Allows to buy an item if it's up for sale. + /// + /// Origin must be Signed and must not be the owner of the `item`. + /// + /// - `collection`: The collection of the item. + /// - `item`: The item the sender wants to buy. + /// - `bid_price`: The price the sender is willing to pay. + /// + /// Emits `ItemBought` on success. + #[pallet::weight(T::WeightInfo::buy_item())] + #[transactional] + pub fn buy_item( + origin: OriginFor, + collection: T::CollectionId, + item: T::ItemId, + bid_price: ItemPrice, + ) -> DispatchResult { + let origin = ensure_signed(origin)?; + Self::do_buy_item(collection, item, origin, bid_price) + } } } diff --git a/frame/uniques/src/tests.rs b/frame/uniques/src/tests.rs index ef28a2143e84a..8b1d00d7ba0c7 100644 --- a/frame/uniques/src/tests.rs +++ b/frame/uniques/src/tests.rs @@ -18,7 +18,7 @@ //! Tests for Uniques pallet. use crate::{mock::*, Event, *}; -use frame_support::{assert_noop, assert_ok, traits::Currency}; +use frame_support::{assert_noop, assert_ok, dispatch::Dispatchable, traits::Currency}; use pallet_balances::Error as BalancesError; use sp_std::prelude::*; @@ -694,3 +694,179 @@ fn max_supply_should_work() { assert!(!CollectionMaxSupply::::contains_key(collection_id)); }); } + +#[test] +fn set_price_should_work() { + new_test_ext().execute_with(|| { + let user_id = 1; + let collection_id = 0; + let item_1 = 1; + let item_2 = 2; + + assert_ok!(Uniques::force_create(Origin::root(), collection_id, user_id, true)); + + assert_ok!(Uniques::mint(Origin::signed(user_id), collection_id, item_1, user_id)); + assert_ok!(Uniques::mint(Origin::signed(user_id), collection_id, item_2, user_id)); + + assert_ok!(Uniques::set_price( + Origin::signed(user_id), + collection_id, + item_1, + Some(1), + None, + )); + + assert_ok!(Uniques::set_price( + Origin::signed(user_id), + collection_id, + item_2, + Some(2), + Some(3) + )); + + let item = ItemPriceOf::::get(collection_id, item_1).unwrap(); + assert_eq!(item.0, 1); + assert_eq!(item.1, None); + + let item = ItemPriceOf::::get(collection_id, item_2).unwrap(); + assert_eq!(item.0, 2); + assert_eq!(item.1, Some(3)); + + assert!(events().contains(&Event::::ItemPriceSet { + collection: collection_id, + item: item_1, + price: 1, + whitelisted_buyer: None, + })); + + // validate we can unset the price + assert_ok!(Uniques::set_price(Origin::signed(user_id), collection_id, item_2, None, None)); + assert!(events().contains(&Event::::ItemPriceRemoved { + collection: collection_id, + item: item_2 + })); + assert!(!ItemPriceOf::::contains_key(collection_id, item_2)); + }); +} + +#[test] +fn buy_item_should_work() { + new_test_ext().execute_with(|| { + let user_1 = 1; + let user_2 = 2; + let user_3 = 3; + let collection_id = 0; + let item_1 = 1; + let item_2 = 2; + let item_3 = 3; + let price_1 = 20; + let price_2 = 30; + let initial_balance = 100; + + Balances::make_free_balance_be(&user_1, initial_balance); + Balances::make_free_balance_be(&user_2, initial_balance); + Balances::make_free_balance_be(&user_3, initial_balance); + + assert_ok!(Uniques::force_create(Origin::root(), collection_id, user_1, true)); + + assert_ok!(Uniques::mint(Origin::signed(user_1), collection_id, item_1, user_1)); + assert_ok!(Uniques::mint(Origin::signed(user_1), collection_id, item_2, user_1)); + assert_ok!(Uniques::mint(Origin::signed(user_1), collection_id, item_3, user_1)); + + assert_ok!(Uniques::set_price( + Origin::signed(user_1), + collection_id, + item_1, + Some(price_1), + None, + )); + + assert_ok!(Uniques::set_price( + Origin::signed(user_1), + collection_id, + item_2, + Some(price_2), + Some(user_3), + )); + + // can't buy for less + assert_noop!( + Uniques::buy_item(Origin::signed(user_2), collection_id, item_1, 1), + Error::::BidTooLow + ); + + // pass the higher price to validate it will still deduct correctly + assert_ok!(Uniques::buy_item(Origin::signed(user_2), collection_id, item_1, price_1 + 1,)); + + // validate the new owner & balances + let item = Item::::get(collection_id, item_1).unwrap(); + assert_eq!(item.owner, user_2); + assert_eq!(Balances::total_balance(&user_1), initial_balance + price_1); + assert_eq!(Balances::total_balance(&user_2), initial_balance - price_1); + + // can't buy from yourself + assert_noop!( + Uniques::buy_item(Origin::signed(user_1), collection_id, item_2, price_2), + Error::::NoPermission + ); + + // can't buy when the item is listed for a specific buyer + assert_noop!( + Uniques::buy_item(Origin::signed(user_2), collection_id, item_2, price_2), + Error::::NoPermission + ); + + // can buy when I'm a whitelisted buyer + assert_ok!(Uniques::buy_item(Origin::signed(user_3), collection_id, item_2, price_2,)); + + assert!(events().contains(&Event::::ItemBought { + collection: collection_id, + item: item_2, + price: price_2, + seller: user_1, + buyer: user_3, + })); + + // ensure we reset the buyer field + assert!(!ItemPriceOf::::contains_key(collection_id, item_2)); + + // can't buy when item is not for sale + assert_noop!( + Uniques::buy_item(Origin::signed(user_2), collection_id, item_3, price_2), + Error::::NotForSale + ); + + // ensure we can't buy an item when the collection or an item is frozen + { + assert_ok!(Uniques::set_price( + Origin::signed(user_1), + collection_id, + item_3, + Some(price_1), + None, + )); + + // freeze collection + assert_ok!(Uniques::freeze_collection(Origin::signed(user_1), collection_id)); + + let buy_item_call = mock::Call::Uniques(crate::Call::::buy_item { + collection: collection_id, + item: item_3, + bid_price: price_1, + }); + assert_noop!(buy_item_call.dispatch(Origin::signed(user_2)), Error::::Frozen); + + assert_ok!(Uniques::thaw_collection(Origin::signed(user_1), collection_id)); + + // freeze item + assert_ok!(Uniques::freeze(Origin::signed(user_1), collection_id, item_3)); + + let buy_item_call = mock::Call::Uniques(crate::Call::::buy_item { + collection: collection_id, + item: item_3, + bid_price: price_1, + }); + assert_noop!(buy_item_call.dispatch(Origin::signed(user_2)), Error::::Frozen); + } + }); +} diff --git a/frame/uniques/src/types.rs b/frame/uniques/src/types.rs index d7706fd7e3a58..98e056163d28d 100644 --- a/frame/uniques/src/types.rs +++ b/frame/uniques/src/types.rs @@ -30,6 +30,8 @@ pub(super) type CollectionDetailsFor = CollectionDetails<::AccountId, DepositBalanceOf>; pub(super) type ItemDetailsFor = ItemDetails<::AccountId, DepositBalanceOf>; +pub(super) type ItemPrice = + <>::Currency as Currency<::AccountId>>::Balance; #[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub struct CollectionDetails { diff --git a/frame/uniques/src/weights.rs b/frame/uniques/src/weights.rs index d885077a8dee9..7c8cb170b1b1d 100644 --- a/frame/uniques/src/weights.rs +++ b/frame/uniques/src/weights.rs @@ -18,22 +18,22 @@ //! Autogenerated weights for pallet_uniques //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2022-06-03, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! HOSTNAME: `bm3`, CPU: `Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz` +//! DATE: 2022-07-13, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! HOSTNAME: `test-bench-bot`, CPU: `Intel(R) Xeon(R) CPU @ 3.10GHz` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: // target/production/substrate // benchmark // pallet -// --chain=dev // --steps=50 // --repeat=20 -// --pallet=pallet_uniques // --extrinsic=* // --execution=wasm // --wasm-execution=compiled // --heap-pages=4096 +// --pallet=pallet_uniques +// --chain=dev // --output=./frame/uniques/src/weights.rs // --template=./.maintain/frame-weight-template.hbs @@ -70,6 +70,8 @@ pub trait WeightInfo { fn cancel_approval() -> Weight; fn set_accept_ownership() -> Weight; fn set_collection_max_supply() -> Weight; + fn set_price() -> Weight; + fn buy_item() -> Weight; } /// Weights for pallet_uniques using the Substrate node and recommended hardware. @@ -78,14 +80,14 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassAccount (r:0 w:1) fn create() -> Weight { - (27_715_000 as Weight) + (33_075_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassAccount (r:0 w:1) fn force_create() -> Weight { - (16_929_000 as Weight) + (19_528_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -102,12 +104,12 @@ impl WeightInfo for SubstrateWeight { /// The range of component `a` is `[0, 1000]`. fn destroy(n: u32, m: u32, a: u32, ) -> Weight { (0 as Weight) - // Standard Error: 16_000 - .saturating_add((10_481_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 16_000 - .saturating_add((1_762_000 as Weight).saturating_mul(m as Weight)) - // Standard Error: 16_000 - .saturating_add((1_590_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 25_000 + .saturating_add((13_639_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 25_000 + .saturating_add((2_393_000 as Weight).saturating_mul(m as Weight)) + // Standard Error: 25_000 + .saturating_add((2_217_000 as Weight).saturating_mul(a as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(4 as Weight)) @@ -120,33 +122,35 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques CollectionMaxSupply (r:1 w:0) // Storage: Uniques Account (r:0 w:1) fn mint() -> Weight { - (36_577_000 as Weight) + (42_146_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Account (r:0 w:1) + // Storage: Uniques ItemPriceOf (r:0 w:1) fn burn() -> Weight { - (36_124_000 as Weight) + (42_960_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) - .saturating_add(T::DbWeight::get().writes(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Account (r:0 w:2) + // Storage: Uniques ItemPriceOf (r:0 w:1) fn transfer() -> Weight { - (27_225_000 as Weight) + (33_025_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) - .saturating_add(T::DbWeight::get().writes(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques Asset (r:100 w:100) /// The range of component `i` is `[0, 5000]`. fn redeposit(i: u32, ) -> Weight { (0 as Weight) - // Standard Error: 14_000 - .saturating_add((12_407_000 as Weight).saturating_mul(i as Weight)) + // Standard Error: 24_000 + .saturating_add((15_540_000 as Weight).saturating_mul(i as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(i as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) @@ -155,26 +159,26 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Class (r:1 w:0) fn freeze() -> Weight { - (21_452_000 as Weight) + (25_194_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Class (r:1 w:0) fn thaw() -> Weight { - (22_155_000 as Weight) + (25_397_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn freeze_collection() -> Weight { - (16_897_000 as Weight) + (19_278_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn thaw_collection() -> Weight { - (16_657_000 as Weight) + (19_304_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -182,20 +186,20 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassAccount (r:0 w:2) fn transfer_ownership() -> Weight { - (25_057_000 as Weight) + (28_615_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn set_team() -> Weight { - (17_253_000 as Weight) + (19_943_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassAccount (r:0 w:1) fn force_item_status() -> Weight { - (20_010_000 as Weight) + (22_583_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -203,7 +207,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques InstanceMetadataOf (r:1 w:0) // Storage: Uniques Attribute (r:1 w:1) fn set_attribute() -> Weight { - (41_159_000 as Weight) + (47_520_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -211,65 +215,81 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques InstanceMetadataOf (r:1 w:0) // Storage: Uniques Attribute (r:1 w:1) fn clear_attribute() -> Weight { - (39_598_000 as Weight) + (45_316_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques InstanceMetadataOf (r:1 w:1) fn set_metadata() -> Weight { - (33_387_000 as Weight) + (38_391_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques InstanceMetadataOf (r:1 w:1) fn clear_metadata() -> Weight { - (33_208_000 as Weight) + (38_023_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassMetadataOf (r:1 w:1) fn set_collection_metadata() -> Weight { - (32_619_000 as Weight) + (37_398_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques ClassMetadataOf (r:1 w:1) fn clear_collection_metadata() -> Weight { - (31_028_000 as Weight) + (35_621_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques Asset (r:1 w:1) fn approve_transfer() -> Weight { - (22_263_000 as Weight) + (25_856_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques Asset (r:1 w:1) fn cancel_approval() -> Weight { - (22_910_000 as Weight) + (26_098_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques OwnershipAcceptance (r:1 w:1) fn set_accept_ownership() -> Weight { - (20_716_000 as Weight) + (24_076_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques CollectionMaxSupply (r:1 w:1) // Storage: Uniques Class (r:1 w:0) fn set_collection_max_supply() -> Weight { - (19_380_000 as Weight) + (22_035_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: Uniques Asset (r:1 w:0) + // Storage: Uniques ItemPriceOf (r:0 w:1) + fn set_price() -> Weight { + (22_534_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Uniques Asset (r:1 w:1) + // Storage: Uniques ItemPriceOf (r:1 w:1) + // Storage: Uniques Class (r:1 w:0) + // Storage: Uniques Account (r:0 w:2) + fn buy_item() -> Weight { + (45_272_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } } // For backwards compatibility and tests @@ -277,14 +297,14 @@ impl WeightInfo for () { // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassAccount (r:0 w:1) fn create() -> Weight { - (27_715_000 as Weight) + (33_075_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassAccount (r:0 w:1) fn force_create() -> Weight { - (16_929_000 as Weight) + (19_528_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -301,12 +321,12 @@ impl WeightInfo for () { /// The range of component `a` is `[0, 1000]`. fn destroy(n: u32, m: u32, a: u32, ) -> Weight { (0 as Weight) - // Standard Error: 16_000 - .saturating_add((10_481_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 16_000 - .saturating_add((1_762_000 as Weight).saturating_mul(m as Weight)) - // Standard Error: 16_000 - .saturating_add((1_590_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 25_000 + .saturating_add((13_639_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 25_000 + .saturating_add((2_393_000 as Weight).saturating_mul(m as Weight)) + // Standard Error: 25_000 + .saturating_add((2_217_000 as Weight).saturating_mul(a as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) @@ -319,33 +339,35 @@ impl WeightInfo for () { // Storage: Uniques CollectionMaxSupply (r:1 w:0) // Storage: Uniques Account (r:0 w:1) fn mint() -> Weight { - (36_577_000 as Weight) + (42_146_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Account (r:0 w:1) + // Storage: Uniques ItemPriceOf (r:0 w:1) fn burn() -> Weight { - (36_124_000 as Weight) + (42_960_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) - .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Account (r:0 w:2) + // Storage: Uniques ItemPriceOf (r:0 w:1) fn transfer() -> Weight { - (27_225_000 as Weight) + (33_025_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) - .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques Asset (r:100 w:100) /// The range of component `i` is `[0, 5000]`. fn redeposit(i: u32, ) -> Weight { (0 as Weight) - // Standard Error: 14_000 - .saturating_add((12_407_000 as Weight).saturating_mul(i as Weight)) + // Standard Error: 24_000 + .saturating_add((15_540_000 as Weight).saturating_mul(i as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(i as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) @@ -354,26 +376,26 @@ impl WeightInfo for () { // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Class (r:1 w:0) fn freeze() -> Weight { - (21_452_000 as Weight) + (25_194_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Class (r:1 w:0) fn thaw() -> Weight { - (22_155_000 as Weight) + (25_397_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn freeze_collection() -> Weight { - (16_897_000 as Weight) + (19_278_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn thaw_collection() -> Weight { - (16_657_000 as Weight) + (19_304_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -381,20 +403,20 @@ impl WeightInfo for () { // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassAccount (r:0 w:2) fn transfer_ownership() -> Weight { - (25_057_000 as Weight) + (28_615_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn set_team() -> Weight { - (17_253_000 as Weight) + (19_943_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassAccount (r:0 w:1) fn force_item_status() -> Weight { - (20_010_000 as Weight) + (22_583_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -402,7 +424,7 @@ impl WeightInfo for () { // Storage: Uniques InstanceMetadataOf (r:1 w:0) // Storage: Uniques Attribute (r:1 w:1) fn set_attribute() -> Weight { - (41_159_000 as Weight) + (47_520_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -410,63 +432,79 @@ impl WeightInfo for () { // Storage: Uniques InstanceMetadataOf (r:1 w:0) // Storage: Uniques Attribute (r:1 w:1) fn clear_attribute() -> Weight { - (39_598_000 as Weight) + (45_316_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques InstanceMetadataOf (r:1 w:1) fn set_metadata() -> Weight { - (33_387_000 as Weight) + (38_391_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques InstanceMetadataOf (r:1 w:1) fn clear_metadata() -> Weight { - (33_208_000 as Weight) + (38_023_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassMetadataOf (r:1 w:1) fn set_collection_metadata() -> Weight { - (32_619_000 as Weight) + (37_398_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques ClassMetadataOf (r:1 w:1) fn clear_collection_metadata() -> Weight { - (31_028_000 as Weight) + (35_621_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques Asset (r:1 w:1) fn approve_transfer() -> Weight { - (22_263_000 as Weight) + (25_856_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques Asset (r:1 w:1) fn cancel_approval() -> Weight { - (22_910_000 as Weight) + (26_098_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques OwnershipAcceptance (r:1 w:1) fn set_accept_ownership() -> Weight { - (20_716_000 as Weight) + (24_076_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques CollectionMaxSupply (r:1 w:1) // Storage: Uniques Class (r:1 w:0) fn set_collection_max_supply() -> Weight { - (19_380_000 as Weight) + (22_035_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: Uniques Asset (r:1 w:0) + // Storage: Uniques ItemPriceOf (r:0 w:1) + fn set_price() -> Weight { + (22_534_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Uniques Asset (r:1 w:1) + // Storage: Uniques ItemPriceOf (r:1 w:1) + // Storage: Uniques Class (r:1 w:0) + // Storage: Uniques Account (r:0 w:2) + fn buy_item() -> Weight { + (45_272_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } } From 367dab0d4bd7fd7b6c222dd15c753169c057dd42 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Wed, 13 Jul 2022 13:49:20 +0100 Subject: [PATCH 08/46] Revamp nomination pool reward scheme (#11669) * make pool roles optional * undo lock file changes? * add migration * add the ability for pools to chill themselves * boilerplate of tests * somewhat stable, but I think I found another bug as well * Fix it all * Add more more sophisticated test + capture one more bug. * Update frame/staking/src/lib.rs * reduce the diff a little bit * add some test for the slashing bug * cleanup * fix lock file? * Fix * fmt * Update frame/nomination-pools/src/lib.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/lib.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/lib.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/mock.rs Co-authored-by: Oliver Tale-Yazdi * Fix build * fix some fishy tests.. * add one last integrity check for MinCreateBond * remove bad assertion -- needs to be dealt with later * nits * fix tests and add benchmarks for chill * remove stuff * fix benchmarks * cargo run --quiet --profile=production --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark pallet --chain=dev --steps=50 --repeat=20 --pallet=pallet_nomination_pools --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/nomination-pools/src/weights.rs --template=./.maintain/frame-weight-template.hbs * remove defensive * first working version * bring back all tests * ALL new tests work now * cleanup * make sure benchmarks and all work * cargo run --quiet --profile=production --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark pallet --chain=dev --steps=50 --repeat=20 --pallet=pallet_nomination_pools --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/nomination-pools/src/weights.rs --template=./.maintain/frame-weight-template.hbs * round of self-review, make arithmetic safe * fix warn * add migration code * Fix doc * add precision notes * make arithmetic fallible * fix node runtime * a lot of precision tests and notes and stuff * document MaxPOintsToBalance better * :round of self-review * fmt * fix some comments * Fix proportional slashing logic * Update frame/nomination-pools/src/tests.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/tests.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/lib.rs Co-authored-by: Oliver Tale-Yazdi * track poinst in migration * fix * fmt * fix migration * remove event read * Apply suggestions from code review * Update frame/staking/src/lib.rs Co-authored-by: Shawn Tabrizi * Update frame/nomination-pools/src/lib.rs Co-authored-by: Shawn Tabrizi * Update frame/nomination-pools/src/lib.rs Co-authored-by: Shawn Tabrizi * update * fmt * fmt * add one last test * fmt Co-authored-by: Oliver Tale-Yazdi Co-authored-by: Shawn Tabrizi Co-authored-by: Parity Bot --- bin/node/runtime/src/lib.rs | 9 +- .../nomination-pools/benchmarking/src/mock.rs | 11 +- frame/nomination-pools/src/lib.rs | 582 +++--- frame/nomination-pools/src/migration.rs | 284 ++- frame/nomination-pools/src/mock.rs | 52 +- frame/nomination-pools/src/tests.rs | 1805 ++++++++++++----- frame/nomination-pools/src/weights.rs | 98 +- .../nomination-pools/test-staking/src/lib.rs | 5 +- .../nomination-pools/test-staking/src/mock.rs | 17 +- utils/frame/try-runtime/cli/src/lib.rs | 9 +- 10 files changed, 1950 insertions(+), 922 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 2d5981339fab3..b2efcb196787d 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -68,7 +68,7 @@ use sp_runtime::{ SaturatedConversion, StaticLookup, }, transaction_validity::{TransactionPriority, TransactionSource, TransactionValidity}, - ApplyExtrinsicResult, FixedPointNumber, Perbill, Percent, Permill, Perquintill, + ApplyExtrinsicResult, FixedPointNumber, FixedU128, Perbill, Percent, Permill, Perquintill, }; use sp_std::prelude::*; #[cfg(any(feature = "std", test))] @@ -730,7 +730,7 @@ impl pallet_bags_list::Config for Runtime { parameter_types! { pub const PostUnbondPoolsWindow: u32 = 4; pub const NominationPoolsPalletId: PalletId = PalletId(*b"py/nopls"); - pub const MinPointsToBalance: u32 = 10; + pub const MaxPointsToBalance: u8 = 10; } use sp_runtime::traits::Convert; @@ -751,6 +751,8 @@ impl pallet_nomination_pools::Config for Runtime { type WeightInfo = (); type Event = Event; type Currency = Balances; + type CurrencyBalance = Balance; + type RewardCounter = FixedU128; type BalanceToU256 = BalanceToU256; type U256ToBalance = U256ToBalance; type StakingInterface = pallet_staking::Pallet; @@ -758,7 +760,7 @@ impl pallet_nomination_pools::Config for Runtime { type MaxMetadataLen = ConstU32<256>; type MaxUnbonding = ConstU32<8>; type PalletId = NominationPoolsPalletId; - type MinPointsToBalance = MinPointsToBalance; + type MaxPointsToBalance = MaxPointsToBalance; } parameter_types! { @@ -1675,6 +1677,7 @@ pub type Executive = frame_executive::Executive< frame_system::ChainContext, Runtime, AllPalletsWithSystem, + pallet_nomination_pools::migration::v2::MigrateToV2, >; /// MMR helper types. diff --git a/frame/nomination-pools/benchmarking/src/mock.rs b/frame/nomination-pools/benchmarking/src/mock.rs index eb884869f6d32..1c74c301e562d 100644 --- a/frame/nomination-pools/benchmarking/src/mock.rs +++ b/frame/nomination-pools/benchmarking/src/mock.rs @@ -17,7 +17,10 @@ use frame_election_provider_support::VoteWeight; use frame_support::{pallet_prelude::*, parameter_types, traits::ConstU64, PalletId}; -use sp_runtime::traits::{Convert, IdentityLookup}; +use sp_runtime::{ + traits::{Convert, IdentityLookup}, + FixedU128, +}; type AccountId = u128; type AccountIndex = u32; @@ -144,13 +147,15 @@ impl Convert for U256ToBalance { parameter_types! { pub static PostUnbondingPoolsWindow: u32 = 10; pub const PoolsPalletId: PalletId = PalletId(*b"py/nopls"); - pub const MinPointsToBalance: u32 = 10; + pub const MaxPointsToBalance: u8 = 10; } impl pallet_nomination_pools::Config for Runtime { type Event = Event; type WeightInfo = (); type Currency = Balances; + type CurrencyBalance = Balance; + type RewardCounter = FixedU128; type BalanceToU256 = BalanceToU256; type U256ToBalance = U256ToBalance; type StakingInterface = Staking; @@ -158,7 +163,7 @@ impl pallet_nomination_pools::Config for Runtime { type MaxMetadataLen = ConstU32<256>; type MaxUnbonding = ConstU32<8>; type PalletId = PoolsPalletId; - type MinPointsToBalance = MinPointsToBalance; + type MaxPointsToBalance = MaxPointsToBalance; } impl crate::Config for Runtime {} diff --git a/frame/nomination-pools/src/lib.rs b/frame/nomination-pools/src/lib.rs index f7ccac3e35411..53e1c48a5e39e 100644 --- a/frame/nomination-pools/src/lib.rs +++ b/frame/nomination-pools/src/lib.rs @@ -17,8 +17,8 @@ //! # Nomination Pools for Staking Delegation //! -//! A pallet that allows members to delegate their stake to nominating pools. A nomination pool -//! acts as nominator and nominates validators on the members behalf. +//! A pallet that allows members to delegate their stake to nominating pools. A nomination pool acts +//! as nominator and nominates validators on the members behalf. //! //! # Index //! @@ -28,16 +28,27 @@ //! //! ## Key terms //! +//! * pool id: A unique identifier of each pool. Set to u12 //! * bonded pool: Tracks the distribution of actively staked funds. See [`BondedPool`] and -//! [`BondedPoolInner`]. Bonded pools are identified via the pools bonded account. +//! [`BondedPoolInner`]. //! * reward pool: Tracks rewards earned by actively staked funds. See [`RewardPool`] and -//! [`RewardPools`]. Reward pools are identified via the pools bonded account. +//! [`RewardPools`]. //! * unbonding sub pools: Collection of pools at different phases of the unbonding lifecycle. See -//! [`SubPools`] and [`SubPoolsStorage`]. Sub pools are identified via the pools bonded account. -//! * members: Accounts that are members of pools. See [`PoolMember`] and [`PoolMembers`]. Pool -//! members are identified via their account. -//! * point: A unit of measure for a members portion of a pool's funds. +//! [`SubPools`] and [`SubPoolsStorage`]. +//! * members: Accounts that are members of pools. See [`PoolMember`] and [`PoolMembers`]. +//! * roles: Administrative roles of each pool, capable of controlling nomination, and the state of +//! the pool. +//! * point: A unit of measure for a members portion of a pool's funds. Points initially have a +//! ratio of 1 (as set by `POINTS_TO_BALANCE_INIT_RATIO`) to balance, but as slashing happens, +//! this can change. //! * kick: The act of a pool administrator forcibly ejecting a member. +//! * bonded account: A key-less account id derived from the pool id that acts as the bonded +//! account. This account registers itself as a nominator in the staking system, and follows +//! exactly the same rules and conditions as a normal staker. Its bond increases or decreases as +//! members join, it can `nominate` or `chill`, and might not even earn staking rewards if it is +//! not nominating proper validators. +//! * reward account: A similar key-less account, that is set as the `Payee` account fo the bonded +//! account for all staking rewards. //! //! ## Usage //! @@ -55,11 +66,13 @@ //! //! In order to leave, a member must take two steps. //! -//! First, they must call [`Call::unbond`]. The unbond other extrinsic will start the -//! unbonding process by unbonding all of the members funds. +//! First, they must call [`Call::unbond`]. The unbond extrinsic will start the unbonding process by +//! unbonding all or a portion of the members funds. //! -//! Second, once [`sp_staking::StakingInterface::bonding_duration`] eras have passed, the member -//! can call [`Call::withdraw_unbonded`] to withdraw all their funds. +//! > A member can have up to [`Config::MaxUnbonding`] distinct active unbonding requests. +//! +//! Second, once [`sp_staking::StakingInterface::bonding_duration`] eras have passed, the member can +//! call [`Call::withdraw_unbonded`] to withdraw any funds that are free. //! //! For design docs see the [bonded pool](#bonded-pool) and [unbonding sub //! pools](#unbonding-sub-pools) sections. @@ -67,9 +80,13 @@ //! ### Slashes //! //! Slashes are distributed evenly across the bonded pool and the unbonding pools from slash era+1 -//! through the slash apply era. Thus, any member who either a) unbonded or b) was actively -//! bonded in the aforementioned range of eras will be affected by the slash. A member is slashed -//! pro-rata based on its stake relative to the total slash amount. +//! through the slash apply era. Thus, any member who either +//! +//! 1. unbonded, or +//! 2. was actively bonded +// +//! in the aforementioned range of eras will be affected by the slash. A member is slashed pro-rata +//! based on its stake relative to the total slash amount. //! //! For design docs see the [slashing](#slashing) section. //! @@ -82,7 +99,9 @@ //! To help facilitate pool administration the pool has one of three states (see [`PoolState`]): //! //! * Open: Anyone can join the pool and no members can be permissionlessly removed. -//! * Blocked: No members can join and some admin roles can kick members. +//! * Blocked: No members can join and some admin roles can kick members. Kicking is not instant, +//! and follows the same process of `unbond` and then `withdraw_unbonded`. In other words, +//! administrators can permissionlessly unbond other members. //! * Destroying: No members can join and all members can be permissionlessly removed with //! [`Call::unbond`] and [`Call::withdraw_unbonded`]. Once a pool is in destroying state, it //! cannot be reverted to another state. @@ -90,12 +109,23 @@ //! A pool has 4 administrative roles (see [`PoolRoles`]): //! //! * Depositor: creates the pool and is the initial member. They can only leave the pool once all -//! other members have left. Once they fully leave the pool is destroyed. +//! other members have left. Once they fully withdraw their funds, the pool is destroyed. //! * Nominator: can select which validators the pool nominates. //! * State-Toggler: can change the pools state and kick members if the pool is blocked. //! * Root: can change the nominator, state-toggler, or itself and can perform any of the actions //! the nominator or state-toggler can. //! +//! ### Dismantling +//! +//! As noted, a pool is destroyed once +//! +//! 1. First, all members need to fully unbond and withdraw. If the pool state is set to +//! `Destroying`, this can happen permissionlessly. +//! 2. The depositor itself fully unbonds and withdraws. Note that at this point, based on the +//! requirements of the staking system, the pool's bonded account's stake might not be able to ge +//! below a certain threshold as a nominator. At this point, the pool should `chill` itself to +//! allow the depositor to leave. +//! //! ## Design //! //! _Notes_: this section uses pseudo code to explain general design and does not necessarily @@ -108,8 +138,8 @@ //! members that where in the pool while it was backing a validator that got slashed. //! * Maximize scalability in terms of member count. //! -//! In order to maintain scalability, all operations are independent of the number of members. To -//! do this, delegation specific information is stored local to the member while the pool data +//! In order to maintain scalability, all operations are independent of the number of members. To do +//! this, delegation specific information is stored local to the member while the pool data //! structures have bounded datum. //! //! ### Bonded pool @@ -118,9 +148,9 @@ //! unbonding. The total points of a bonded pool are always equal to the sum of points of the //! delegation members. A bonded pool tracks its points and reads its bonded balance. //! -//! When a member joins a pool, `amount_transferred` is transferred from the members account -//! to the bonded pools account. Then the pool calls `staking::bond_extra(amount_transferred)` and -//! issues new points which are tracked by the member and added to the bonded pool's points. +//! When a member joins a pool, `amount_transferred` is transferred from the members account to the +//! bonded pools account. Then the pool calls `staking::bond_extra(amount_transferred)` and issues +//! new points which are tracked by the member and added to the bonded pool's points. //! //! When the pool already has some balance, we want the value of a point before the transfer to //! equal the value of a point after the transfer. So, when a member joins a bonded pool with a @@ -148,77 +178,13 @@ //! ### Reward pool //! //! When a pool is first bonded it sets up an deterministic, inaccessible account as its reward -//! destination. To track staking rewards we track how the balance of this reward account changes. -//! -//! The reward pool needs to store: -//! -//! * The pool balance at the time of the last payout: `reward_pool.balance` -//! * The total earnings ever at the time of the last payout: `reward_pool.total_earnings` -//! * The total points in the pool at the time of the last payout: `reward_pool.points` -//! -//! And the member needs to store: -//! -//! * The total payouts at the time of the last payout by that member: -//! `member.reward_pool_total_earnings` -//! -//! Before the first reward claim is initiated for a pool, all the above variables are set to zero. -//! -//! When a member initiates a claim, the following happens: -//! -//! 1) Compute the reward pool's total points and the member's virtual points in the reward pool -//! * First `current_total_earnings` is computed (`current_balance` is the free balance of the -//! reward pool at the beginning of these operations.) -//! ```text -//! current_total_earnings = -//! current_balance - reward_pool.balance + pool.total_earnings; -//! ``` -//! * Then the `current_points` is computed. Every balance unit that was added to the reward -//! pool since last time recorded means that the `pool.points` is increased by -//! `bonding_pool.total_points`. In other words, for every unit of balance that has been -//! earned by the reward pool, the reward pool points are inflated by `bonded_pool.points`. In -//! effect this allows each, single unit of balance (e.g. planck) to be divvied up pro-rata -//! among members based on points. -//! ```text -//! new_earnings = current_total_earnings - reward_pool.total_earnings; -//! current_points = reward_pool.points + bonding_pool.points * new_earnings; -//! ``` -//! * Finally, the`member_virtual_points` are computed: the product of the member's points in -//! the bonding pool and the total inflow of balance units since the last time the member -//! claimed rewards -//! ```text -//! new_earnings_since_last_claim = current_total_earnings - member.reward_pool_total_earnings; -//! member_virtual_points = member.points * new_earnings_since_last_claim; -//! ``` -//! 2) Compute the `member_payout`: -//! ```text -//! member_pool_point_ratio = member_virtual_points / current_points; -//! member_payout = current_balance * member_pool_point_ratio; -//! ``` -//! 3) Transfer `member_payout` to the member -//! 4) For the member set: -//! ```text -//! member.reward_pool_total_earnings = current_total_earnings; -//! ``` -//! 5) For the pool set: -//! ```text -//! reward_pool.points = current_points - member_virtual_points; -//! reward_pool.balance = current_balance - member_payout; -//! reward_pool.total_earnings = current_total_earnings; -//! ``` -//! -//! _Note_: One short coming of this design is that new joiners can claim rewards for the era after -//! they join even though their funds did not contribute to the pools vote weight. When a -//! member joins, it's `reward_pool_total_earnings` field is set equal to the `total_earnings` -//! of the reward pool at that point in time. At best the reward pool has the rewards up through the -//! previous era. If a member joins prior to the election snapshot it will benefit from the -//! rewards for the active era despite not contributing to the pool's vote weight. If it joins -//! after the election snapshot is taken it will benefit from the rewards of the next _2_ eras -//! because it's vote weight will not be counted until the election snapshot in active era + 1. -//! Related: -// _Note to maintainers_: In order to ensure the reward account never falls below the existential -// deposit, at creation the reward account must be endowed with the existential deposit. All logic -// for calculating rewards then does not see that existential deposit as part of the free balance. -// See `RewardPool::current_balance`. +//! destination. +//! +//! The reward pool is not really a pool anymore, as it does not track points anymore. Instead, it +//! tracks, a virtual value called `reward_counter`, among a few other values. +//! +//! See [this link](https://hackmd.io/PFGn6wI5TbCmBYoEA_f2Uw) for an in-depth explanation of the +//! reward pool mechanism. //! //! **Relevant extrinsics:** //! @@ -297,13 +263,8 @@ //! * PoolMembers cannot vote with their staked funds because they are transferred into the pools //! account. In the future this can be overcome by allowing the members to vote with their bonded //! funds via vote splitting. -//! * PoolMembers cannot quickly transfer to another pool if they do not like the nominations, -//! instead they must wait for the unbonding duration. -//! -//! # Runtime builder warnings -//! -//! * Watch out for overflow of [`RewardPoints`] and [`BalanceOf`] types. Consider things like the -//! chains total issuance, staking reward rate, and burn rate. +//! * PoolMembers cannot quickly transfer to another pool if they do no like nominations, instead +//! they must wait for the unbonding duration. #![cfg_attr(not(feature = "std"), no_std)] @@ -320,7 +281,10 @@ use frame_support::{ }; use scale_info::TypeInfo; use sp_core::U256; -use sp_runtime::traits::{AccountIdConversion, Bounded, CheckedSub, Convert, Saturating, Zero}; +use sp_runtime::{ + traits::{AccountIdConversion, Bounded, CheckedAdd, CheckedSub, Convert, Saturating, Zero}, + FixedPointNumber, FixedPointOperand, +}; use sp_staking::{EraIndex, OnStakerSlash, StakingInterface}; use sp_std::{collections::btree_map::BTreeMap, fmt::Debug, ops::Div, vec::Vec}; @@ -352,8 +316,6 @@ pub use weights::WeightInfo; /// The balance type used by the currency system. pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; -/// Type used to track the points of a reward pool. -pub type RewardPoints = U256; /// Type used for unique identifier of each pool. pub type PoolId = u32; @@ -407,26 +369,50 @@ pub struct PoolMember { /// The quantity of points this member has in the bonded pool or in a sub pool if /// `Self::unbonding_era` is some. pub points: BalanceOf, - /// The reward pools total earnings _ever_ the last time this member claimed a payout. - /// Assuming no massive burning events, we expect this value to always be below total issuance. - /// This value lines up with the [`RewardPool::total_earnings`] after a member claims a - /// payout. - pub reward_pool_total_earnings: BalanceOf, + /// The reward counter at the time of this member's last payout claim. + pub last_recorded_reward_counter: T::RewardCounter, /// The eras in which this member is unbonding, mapped from era index to the number of /// points scheduled to unbond in the given era. pub unbonding_eras: BoundedBTreeMap, T::MaxUnbonding>, } impl PoolMember { - fn total_points(&self) -> BalanceOf { - self.active_points().saturating_add(self.unbonding_points()) + /// The pending rewards of this member. + fn pending_rewards( + &self, + current_reward_counter: T::RewardCounter, + ) -> Result, Error> { + // accuracy note: Reward counters are `FixedU128` with base of 10^18. This value is being + // multiplied by a point. The worse case of a point is 10x the granularity of the balance + // (10x is the common configuration of `MaxPointsToBalance`). + // + // Assuming roughly the current issuance of polkadot (12,047,781,394,999,601,455, which is + // 1.2 * 10^9 * 10^10 = 1.2 * 10^19), the worse case point value is around 10^20. + // + // The final multiplication is: + // + // rc * 10^20 / 10^18 = rc * 100 + // + // meaning that as long as reward_counter's value is less than 1/100th of its max capacity + // (u128::MAX_VALUE), `checked_mul_int` won't saturate. + // + // given the nature of reward counter being 'pending_rewards / pool_total_point', the only + // (unrealistic) way that super high values can be achieved is for a pool to suddenly + // receive massive rewards with a very very small amount of stake. In all normal pools, as + // the points increase, so does the rewards. Moreover, as long as rewards are not + // accumulated for astronomically large durations, + // `current_reward_counter.defensive_saturating_sub(self.last_recorded_reward_counter)` + // won't be extremely big. + (current_reward_counter.defensive_saturating_sub(self.last_recorded_reward_counter)) + .checked_mul_int(self.active_points()) + .ok_or(Error::::OverflowRisk) } /// Active balance of the member. /// /// This is derived from the ratio of points in the pool to which the member belongs to. /// Might return different values based on the pool state for the same member and points. - pub(crate) fn active_balance(&self) -> BalanceOf { + fn active_balance(&self) -> BalanceOf { if let Some(pool) = BondedPool::::get(self.pool_id).defensive() { pool.points_to_balance(self.points) } else { @@ -434,13 +420,18 @@ impl PoolMember { } } + /// Total points of this member, both active and unbonding. + fn total_points(&self) -> BalanceOf { + self.active_points().saturating_add(self.unbonding_points()) + } + /// Active points of the member. - pub(crate) fn active_points(&self) -> BalanceOf { + fn active_points(&self) -> BalanceOf { self.points } /// Inactive points of the member, waiting to be withdrawn. - pub(crate) fn unbonding_points(&self) -> BalanceOf { + fn unbonding_points(&self) -> BalanceOf { self.unbonding_eras .as_ref() .iter() @@ -673,7 +664,7 @@ impl BondedPool { MaxPoolMembers::::get().map_or(true, |max| PoolMembers::::count() < max), Error::::MaxPoolMembers ); - self.member_counter = self.member_counter.defensive_saturating_add(1); + self.member_counter = self.member_counter.checked_add(1).ok_or(Error::::OverflowRisk)?; Ok(()) } @@ -750,19 +741,19 @@ impl BondedPool { // We checked for zero above .div(bonded_balance); - let min_points_to_balance = T::MinPointsToBalance::get(); + let max_points_to_balance = T::MaxPointsToBalance::get(); // Pool points can inflate relative to balance, but only if the pool is slashed. // If we cap the ratio of points:balance so one cannot join a pool that has been slashed - // by `min_points_to_balance`%, if not zero. + // by `max_points_to_balance`%, if not zero. ensure!( - points_to_balance_ratio_floor < min_points_to_balance.into(), + points_to_balance_ratio_floor < max_points_to_balance.into(), Error::::OverflowRisk ); - // while restricting the balance to `min_points_to_balance` of max total issuance, + // while restricting the balance to `max_points_to_balance` of max total issuance, let next_bonded_balance = bonded_balance.saturating_add(new_funds); ensure!( - next_bonded_balance < BalanceOf::::max_value().div(min_points_to_balance.into()), + next_bonded_balance < BalanceOf::::max_value().div(max_points_to_balance.into()), Error::::OverflowRisk ); @@ -904,16 +895,6 @@ impl BondedPool { Ok(points_issued) } - /// If `n` saturates at it's upper bound, mark the pool as destroying. This is useful when a - /// number saturating indicates the pool can no longer correctly keep track of state. - fn bound_check(&mut self, n: U256) -> U256 { - if n == U256::max_value() { - self.set_state(PoolState::Destroying) - } - - n - } - // Set the state of `self`, and deposit an event if the state changed. State should never be set // directly in in order to ensure a state change event is always correctly deposited. fn set_state(&mut self, state: PoolState) { @@ -928,45 +909,104 @@ impl BondedPool { } /// A reward pool. +/// +/// A reward pool is not so much a pool anymore, since it does not contain any shares or points. +/// Rather, simply to fit nicely next to bonded pool and unbonding pools in terms of terminology. In +/// reality, a reward pool is just a container for a few pool-dependent data related to the rewards. #[derive(Encode, Decode, MaxEncodedLen, TypeInfo, RuntimeDebugNoBound)] -#[cfg_attr(feature = "std", derive(Clone, PartialEq))] +#[cfg_attr(feature = "std", derive(Clone, PartialEq, DefaultNoBound))] #[codec(mel_bound(T: Config))] #[scale_info(skip_type_params(T))] pub struct RewardPool { - /// The balance of this reward pool after the last claimed payout. - pub balance: BalanceOf, - /// The total earnings _ever_ of this reward pool after the last claimed payout. I.E. the sum - /// of all incoming balance through the pools life. + /// The last recorded value of the reward counter. /// - /// NOTE: We assume this will always be less than total issuance and thus can use the runtimes - /// `Balance` type. However in a chain with a burn rate higher than the rate this increases, - /// this type should be bigger than `Balance`. - pub total_earnings: BalanceOf, - /// The total points of this reward pool after the last claimed payout. - pub points: RewardPoints, + /// This is updated ONLY when the points in the bonded pool change, which means `join`, + /// `bond_extra` and `unbond`, all of which is done through `update_recorded`. + last_recorded_reward_counter: T::RewardCounter, + /// The last recorded total payouts of the reward pool. + /// + /// Payouts is essentially income of the pool. + /// + /// Update criteria is same as that of `last_recorded_reward_counter`. + last_recorded_total_payouts: BalanceOf, + /// Total amount that this pool has paid out so far to the members. + total_rewards_claimed: BalanceOf, } impl RewardPool { - /// Mutate the reward pool by updating the total earnings and current free balance. - fn update_total_earnings_and_balance(&mut self, id: PoolId) { - let current_balance = Self::current_balance(id); - // The earnings since the last time it was updated - let new_earnings = current_balance.saturating_sub(self.balance); - // The lifetime earnings of the of the reward pool - self.total_earnings = new_earnings.saturating_add(self.total_earnings); - self.balance = current_balance; - } - - /// Get a reward pool and update its total earnings and balance - fn get_and_update(id: PoolId) -> Option { - RewardPools::::get(id).map(|mut r| { - r.update_total_earnings_and_balance(id); - r - }) - } - - /// The current balance of the reward pool. Never access the reward pools free balance directly. - /// The existential deposit was not received as a reward, so the reward pool can not use it. + /// Getter for [`RewardPool::last_recorded_reward_counter`]. + fn last_recorded_reward_counter(&self) -> T::RewardCounter { + self.last_recorded_reward_counter + } + + /// Register some rewards that are claimed from the pool by the members. + fn register_claimed_reward(&mut self, reward: BalanceOf) { + self.total_rewards_claimed = self.total_rewards_claimed.saturating_add(reward); + } + + /// Update the recorded values of the pool. + fn update_records(&mut self, id: PoolId, bonded_points: BalanceOf) -> Result<(), Error> { + let balance = Self::current_balance(id); + self.last_recorded_reward_counter = self.current_reward_counter(id, bonded_points)?; + self.last_recorded_total_payouts = balance + .checked_add(&self.total_rewards_claimed) + .ok_or(Error::::OverflowRisk)?; + Ok(()) + } + + /// Get the current reward counter, based on the given `bonded_points` being the state of the + /// bonded pool at this time. + fn current_reward_counter( + &self, + id: PoolId, + bonded_points: BalanceOf, + ) -> Result> { + let balance = Self::current_balance(id); + let payouts_since_last_record = balance + .saturating_add(self.total_rewards_claimed) + .saturating_sub(self.last_recorded_total_payouts); + + // * accuracy notes regarding the multiplication in `checked_from_rational`: + // `payouts_since_last_record` is a subset of the total_issuance at the very + // worse. `bonded_points` are similarly, in a non-slashed pool, have the same granularity as + // balance, and are thus below within the range of total_issuance. In the worse case + // scenario, for `saturating_from_rational`, we have: + // + // dot_total_issuance * 10^18 / `minJoinBond` + // + // assuming `MinJoinBond == ED` + // + // dot_total_issuance * 10^18 / 10^10 = dot_total_issuance * 10^8 + // + // which, with the current numbers, is a miniscule fraction of the u128 capacity. + // + // Thus, adding two values of type reward counter should be safe for ages in a chain like + // Polkadot. The important note here is that `reward_pool.last_recorded_reward_counter` only + // ever accumulates, but its semantics imply that it is less than total_issuance, when + // represented as `FixedU128`, which means it is less than `total_issuance * 10^18`. + // + // * accuracy notes regarding `checked_from_rational` collapsing to zero, meaning that no + // reward can be claimed: + // + // largest `bonded_points`, such that the reward counter is non-zero, with `FixedU128` + // will be when the payout is being computed. This essentially means `payout/bonded_points` + // needs to be more than 1/1^18. Thus, assuming that `bonded_points` will always be less + // than `10 * dot_total_issuance`, if the reward_counter is the smallest possible value, + // the value of the reward being calculated is: + // + // x / 10^20 = 1/ 10^18 + // + // x = 100 + // + // which is basically 10^-8 DOTs. See `smallest_claimable_reward` for an example of this. + T::RewardCounter::checked_from_rational(payouts_since_last_record, bonded_points) + .and_then(|ref r| self.last_recorded_reward_counter.checked_add(r)) + .ok_or(Error::::OverflowRisk) + } + + /// Current free balance of the reward pool. + /// + /// This is sum of all the rewards that are claimable by pool members. fn current_balance(id: PoolId) -> BalanceOf { T::Currency::free_balance(&Pallet::::create_reward_account(id)) .saturating_sub(T::Currency::minimum_balance()) @@ -1098,9 +1138,10 @@ pub mod pallet { use super::*; use frame_support::traits::StorageVersion; use frame_system::{ensure_signed, pallet_prelude::*}; + use sp_runtime::traits::CheckedAdd; /// The current storage version. - const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); + const STORAGE_VERSION: StorageVersion = StorageVersion::new(2); #[pallet::pallet] #[pallet::generate_store(pub(crate) trait Store)] @@ -1116,19 +1157,53 @@ pub mod pallet { type WeightInfo: weights::WeightInfo; /// The nominating balance. - type Currency: Currency; + type Currency: Currency; + + /// Sadly needed to bound it to `FixedPointOperand`. + // The only alternative is to sprinkle a `where BalanceOf: FixedPointOperand` in roughly + // a million places, so we prefer doing this. + type CurrencyBalance: sp_runtime::traits::AtLeast32BitUnsigned + + codec::FullCodec + + MaybeSerializeDeserialize + + sp_std::fmt::Debug + + Default + + FixedPointOperand + + CheckedAdd + + TypeInfo + + MaxEncodedLen; + + /// The type that is used for reward counter. + /// + /// The arithmetic of the reward counter might saturate based on the size of the + /// `Currency::Balance`. If this happens, operations fails. Nonetheless, this type should be + /// chosen such that this failure almost never happens, as if it happens, the pool basically + /// needs to be dismantled (or all pools migrated to a larger `RewardCounter` type, which is + /// a PITA to do). + /// + /// See the inline code docs of `Member::pending_rewards` and `RewardPool::update_recorded` + /// for example analysis. A [`sp_runtime::FixedU128`] should be fine for chains with balance + /// types similar to that of Polkadot and Kusama, in the absence of severe slashing (or + /// prevented via a reasonable `MaxPointsToBalance`), for many many years to come. + type RewardCounter: FixedPointNumber + MaxEncodedLen + TypeInfo + Default + codec::FullCodec; /// The nomination pool's pallet id. #[pallet::constant] type PalletId: Get; - /// The minimum pool points-to-balance ratio that must be maintained for it to be `open`. + /// The maximum pool points-to-balance ratio that an `open` pool can have. + /// /// This is important in the event slashing takes place and the pool's points-to-balance /// ratio becomes disproportional. + /// + /// Moreover, this relates to the `RewardCounter` type as well, as the arithmetic operations + /// are a function of number of points, and by setting this value to e.g. 10, you ensure + /// that the total number of points in the system are at most 10 times the total_issuance of + /// the chain, in the absolute worse case. + /// /// For a value of 10, the threshold would be a pool points-to-balance ratio of 10:1. /// Such a scenario would also be the equivalent of the pool being 90% slashed. #[pallet::constant] - type MinPointsToBalance: Get; + type MaxPointsToBalance: Get; /// Infallible method for converting `Currency::Balance` to `U256`. type BalanceToU256: Convert, U256>; @@ -1427,10 +1502,10 @@ pub mod pallet { let mut bonded_pool = BondedPool::::get(pool_id).ok_or(Error::::PoolNotFound)?; bonded_pool.ok_to_join(amount)?; - // We just need its total earnings at this point in time, but we don't need to write it - // because we are not adjusting its points (all other values can calculated virtual). - let reward_pool = RewardPool::::get_and_update(pool_id) + let mut reward_pool = RewardPools::::get(pool_id) .defensive_ok_or::>(DefensiveError::RewardPoolNotFound.into())?; + // IMPORTANT: reward pool records must be updated with the old points. + reward_pool.update_records(pool_id, bonded_pool.points)?; bonded_pool.try_inc_members()?; let points_issued = bonded_pool.try_bond_funds(&who, amount, BondType::Later)?; @@ -1440,13 +1515,9 @@ pub mod pallet { PoolMember:: { pool_id, points: points_issued, - // At best the reward pool has the rewards up through the previous era. If the - // member joins prior to the snapshot they will benefit from the rewards of - // the active era despite not contributing to the pool's vote weight. If they - // join after the snapshot is taken they will benefit from the rewards of the - // next 2 eras because their vote weight will not be counted until the - // snapshot in active era + 1. - reward_pool_total_earnings: reward_pool.total_earnings, + // we just updated `last_known_reward_counter` to the current one in + // `update_recorded`. + last_recorded_reward_counter: reward_pool.last_recorded_reward_counter(), unbonding_eras: Default::default(), }, ); @@ -1457,7 +1528,9 @@ pub mod pallet { bonded: amount, joined: true, }); + bonded_pool.put(); + RewardPools::::insert(pool_id, reward_pool); Ok(()) } @@ -1466,6 +1539,8 @@ pub mod pallet { /// /// Additional funds can come from either the free balance of the account, of from the /// accumulated rewards, see [`BondExtra`]. + /// + /// Bonding extra funds implies an automatic payout of all pending rewards as well. // NOTE: this transaction is implemented with the sole purpose of readability and // correctness, not optimization. We read/write several storage items multiple times instead // of just once, in the spirit reusing code. @@ -1478,19 +1553,23 @@ pub mod pallet { let who = ensure_signed(origin)?; let (mut member, mut bonded_pool, mut reward_pool) = Self::get_member_with_pools(&who)?; + // payout related stuff: we must claim the payouts, and updated recorded payout data + // before updating the bonded pool points, similar to that of `join` transaction. + reward_pool.update_records(bonded_pool.id, bonded_pool.points)?; + // TODO: optimize this to not touch the free balance of `who ` at all in benchmarks. + // Currently, bonding rewards is like a batch. In the same PR, also make this function + // take a boolean argument that make it either 100% pure (no storage update), or make it + // also emit event and do the transfer. #11671 + let claimed = + Self::do_reward_payout(&who, &mut member, &mut bonded_pool, &mut reward_pool)?; + let (points_issued, bonded) = match extra { BondExtra::FreeBalance(amount) => (bonded_pool.try_bond_funds(&who, amount, BondType::Later)?, amount), - BondExtra::Rewards => { - let claimed = Self::do_reward_payout( - &who, - &mut member, - &mut bonded_pool, - &mut reward_pool, - )?; - (bonded_pool.try_bond_funds(&who, claimed, BondType::Later)?, claimed) - }, + BondExtra::Rewards => + (bonded_pool.try_bond_funds(&who, claimed, BondType::Later)?, claimed), }; + bonded_pool.ok_to_be_open(bonded)?; member.points = member.points.saturating_add(points_issued); @@ -1558,21 +1637,17 @@ pub mod pallet { member_account: T::AccountId, #[pallet::compact] unbonding_points: BalanceOf, ) -> DispatchResult { - let caller = ensure_signed(origin)?; + let who = ensure_signed(origin)?; let (mut member, mut bonded_pool, mut reward_pool) = Self::get_member_with_pools(&member_account)?; - bonded_pool.ok_to_unbond_with(&caller, &member_account, &member, unbonding_points)?; + bonded_pool.ok_to_unbond_with(&who, &member_account, &member, unbonding_points)?; // Claim the the payout prior to unbonding. Once the user is unbonding their points no // longer exist in the bonded pool and thus they can no longer claim their payouts. It // is not strictly necessary to claim the rewards, but we do it here for UX. - Self::do_reward_payout( - &member_account, - &mut member, - &mut bonded_pool, - &mut reward_pool, - )?; + let _ = reward_pool.update_records(bonded_pool.id, bonded_pool.points)?; + let _ = Self::do_reward_payout(&who, &mut member, &mut bonded_pool, &mut reward_pool)?; let current_era = T::StakingInterface::current_era(); let unbond_era = T::StakingInterface::bonding_duration().saturating_add(current_era); @@ -1846,23 +1921,25 @@ pub mod pallet { PoolMember:: { pool_id, points, - reward_pool_total_earnings: Zero::zero(), + last_recorded_reward_counter: Zero::zero(), unbonding_eras: Default::default(), }, ); RewardPools::::insert( pool_id, RewardPool:: { - balance: Zero::zero(), - points: U256::zero(), - total_earnings: Zero::zero(), + last_recorded_reward_counter: Zero::zero(), + last_recorded_total_payouts: Zero::zero(), + total_rewards_claimed: Zero::zero(), }, ); ReversePoolIdLookup::::insert(bonded_pool.bonded_account(), pool_id); + Self::deposit_event(Event::::Created { depositor: who.clone(), pool_id: pool_id.clone(), }); + Self::deposit_event(Event::::Bonded { member: who, pool_id, @@ -1896,8 +1973,14 @@ pub mod pallet { /// Set a new state for the pool. /// - /// The dispatch origin of this call must be signed by the state toggler, or the root role - /// of the pool. + /// If a pool is already in the `Destroying` state, then under no condition can its state + /// change again. + /// + /// The dispatch origin of this call must be either: + /// + /// 1. signed by the state toggler, or the root role of the pool, + /// 2. if the pool conditions to be open are NOT met (as described by `ok_to_be_open`), and + /// then the state of the pool can be permissionlessly changed to `Destroying`. #[pallet::weight(T::WeightInfo::set_state())] #[transactional] pub fn set_state( @@ -2065,14 +2148,9 @@ pub mod pallet { impl Hooks> for Pallet { fn integrity_test() { assert!( - T::MinPointsToBalance::get() > 0, + T::MaxPointsToBalance::get() > 0, "Minimum points to balance ratio must be greater than 0" ); - assert!( - sp_std::mem::size_of::() >= - 2 * sp_std::mem::size_of::>(), - "bit-length of the reward points must be at least twice as much as balance" - ); assert!( T::StakingInterface::bonding_duration() < TotalUnbondingPools::::get(), "There must be more unbonding pools then the bonding duration / @@ -2119,7 +2197,7 @@ impl Pallet { ExistenceRequirement::AllowDeath, ); - // TODO: this is purely defensive. + // NOTE: this is purely defensive. T::Currency::make_free_balance_be(&reward_account, Zero::zero()); T::Currency::make_free_balance_be(&bonded_pool.bonded_account(), Zero::zero()); @@ -2212,75 +2290,6 @@ impl Pallet { .div(current_points) } - /// Calculate the rewards for `member`. - /// - /// Returns the payout amount. - fn calculate_member_payout( - member: &mut PoolMember, - bonded_pool: &mut BondedPool, - reward_pool: &mut RewardPool, - ) -> Result, DispatchError> { - let u256 = |x| T::BalanceToU256::convert(x); - let balance = |x| T::U256ToBalance::convert(x); - - let last_total_earnings = reward_pool.total_earnings; - reward_pool.update_total_earnings_and_balance(bonded_pool.id); - - // Notice there is an edge case where total_earnings have not increased and this is zero - let new_earnings = u256(reward_pool.total_earnings.saturating_sub(last_total_earnings)); - - // The new points that will be added to the pool. For every unit of balance that has been - // earned by the reward pool, we inflate the reward pool points by `bonded_pool.points`. In - // effect this allows each, single unit of balance (e.g. plank) to be divvied up pro rata - // among members based on points. - let new_points = u256(bonded_pool.points).saturating_mul(new_earnings); - - // The points of the reward pool after taking into account the new earnings. Notice that - // this only stays even or increases over time except for when we subtract member virtual - // shares. - let current_points = bonded_pool.bound_check(reward_pool.points.saturating_add(new_points)); - - // The rewards pool's earnings since the last time this member claimed a payout. - let new_earnings_since_last_claim = - reward_pool.total_earnings.saturating_sub(member.reward_pool_total_earnings); - - // The points of the reward pool that belong to the member. - let member_virtual_points = - // The members portion of the reward pool - u256(member.active_points()) - // times the amount the pool has earned since the member last claimed. - .saturating_mul(u256(new_earnings_since_last_claim)); - - let member_payout = if member_virtual_points.is_zero() || - current_points.is_zero() || - reward_pool.balance.is_zero() - { - Zero::zero() - } else { - // Equivalent to `(member_virtual_points / current_points) * reward_pool.balance` - let numerator = { - let numerator = member_virtual_points.saturating_mul(u256(reward_pool.balance)); - bonded_pool.bound_check(numerator) - }; - balance( - numerator - // We check for zero above - .div(current_points), - ) - }; - - // Record updates - if reward_pool.total_earnings == BalanceOf::::max_value() { - bonded_pool.set_state(PoolState::Destroying); - }; - - member.reward_pool_total_earnings = reward_pool.total_earnings; - reward_pool.points = current_points.saturating_sub(member_virtual_points); - reward_pool.balance = reward_pool.balance.saturating_sub(member_payout); - - Ok(member_payout) - } - /// If the member has some rewards, transfer a payout from the reward pool to the member. // Emits events and potentially modifies pool state if any arithmetic saturates, but does // not persist any of the mutable inputs to storage. @@ -2291,38 +2300,37 @@ impl Pallet { reward_pool: &mut RewardPool, ) -> Result, DispatchError> { debug_assert_eq!(member.pool_id, bonded_pool.id); + // a member who has no skin in the game anymore cannot claim any rewards. ensure!(!member.active_points().is_zero(), Error::::FullyUnbonding); - let was_destroying = bonded_pool.is_destroying(); - let member_payout = Self::calculate_member_payout(member, bonded_pool, reward_pool)?; + let current_reward_counter = + reward_pool.current_reward_counter(bonded_pool.id, bonded_pool.points)?; + let pending_rewards = member.pending_rewards(current_reward_counter)?; - if member_payout.is_zero() { - return Ok(member_payout) + if pending_rewards.is_zero() { + return Ok(pending_rewards) } + // IFF the reward is non-zero alter the member and reward pool info. + member.last_recorded_reward_counter = current_reward_counter; + reward_pool.register_claimed_reward(pending_rewards); + // Transfer payout to the member. T::Currency::transfer( &bonded_pool.reward_account(), &member_account, - member_payout, + pending_rewards, ExistenceRequirement::AllowDeath, )?; Self::deposit_event(Event::::PaidOut { member: member_account.clone(), pool_id: member.pool_id, - payout: member_payout, + payout: pending_rewards, }); - if bonded_pool.is_destroying() && !was_destroying { - Self::deposit_event(Event::::StateChanged { - pool_id: member.pool_id, - new_state: PoolState::Destroying, - }); - } - - Ok(member_payout) + Ok(pending_rewards) } /// Ensure the correctness of the state of this pallet. diff --git a/frame/nomination-pools/src/migration.rs b/frame/nomination-pools/src/migration.rs index e23a35fe85602..f2abfe29dfbf7 100644 --- a/frame/nomination-pools/src/migration.rs +++ b/frame/nomination-pools/src/migration.rs @@ -16,11 +16,12 @@ // limitations under the License. use super::*; +use crate::log; +use frame_support::traits::OnRuntimeUpgrade; +use sp_std::collections::btree_map::BTreeMap; pub mod v1 { use super::*; - use crate::log; - use frame_support::traits::OnRuntimeUpgrade; #[derive(Decode)] pub struct OldPoolRoles { @@ -103,3 +104,282 @@ pub mod v1 { } } } + +pub mod v2 { + use super::*; + use sp_runtime::Perbill; + + #[test] + fn migration_assumption_is_correct() { + // this migrations cleans all the reward accounts to contain exactly ed, and all members + // having no claimable rewards. In this state, all fields of the `RewardPool` and + // `member.last_recorded_reward_counter` are all zero. + use crate::mock::*; + ExtBuilder::default().build_and_execute(|| { + let join = |x| { + Balances::make_free_balance_be(&x, Balances::minimum_balance() + 10); + frame_support::assert_ok!(Pools::join(Origin::signed(x), 10, 1)); + }; + + assert_eq!(BondedPool::::get(1).unwrap().points, 10); + assert_eq!( + RewardPools::::get(1).unwrap(), + RewardPool { ..Default::default() } + ); + assert_eq!( + PoolMembers::::get(10).unwrap().last_recorded_reward_counter, + Zero::zero() + ); + + join(20); + assert_eq!(BondedPool::::get(1).unwrap().points, 20); + assert_eq!( + RewardPools::::get(1).unwrap(), + RewardPool { ..Default::default() } + ); + assert_eq!( + PoolMembers::::get(10).unwrap().last_recorded_reward_counter, + Zero::zero() + ); + assert_eq!( + PoolMembers::::get(20).unwrap().last_recorded_reward_counter, + Zero::zero() + ); + + join(30); + assert_eq!(BondedPool::::get(1).unwrap().points, 30); + assert_eq!( + RewardPools::::get(1).unwrap(), + RewardPool { ..Default::default() } + ); + assert_eq!( + PoolMembers::::get(10).unwrap().last_recorded_reward_counter, + Zero::zero() + ); + assert_eq!( + PoolMembers::::get(20).unwrap().last_recorded_reward_counter, + Zero::zero() + ); + assert_eq!( + PoolMembers::::get(30).unwrap().last_recorded_reward_counter, + Zero::zero() + ); + }); + } + + #[derive(Decode)] + pub struct OldRewardPool { + pub balance: B, + pub total_earnings: B, + pub points: U256, + } + + #[derive(Decode)] + pub struct OldPoolMember { + pub pool_id: PoolId, + pub points: BalanceOf, + pub reward_pool_total_earnings: BalanceOf, + pub unbonding_eras: BoundedBTreeMap, T::MaxUnbonding>, + } + + /// Migrate the pool reward scheme to the new version, as per + /// . + pub struct MigrateToV2(sp_std::marker::PhantomData); + impl MigrateToV2 { + fn run(current: StorageVersion) -> Weight { + let mut reward_pools_translated = 0u64; + let mut members_translated = 0u64; + // just for logging. + let mut total_value_locked = BalanceOf::::zero(); + let mut total_points_locked = BalanceOf::::zero(); + + // store each member of the pool, with their active points. In the process, migrate + // their data as well. + let mut temp_members = BTreeMap::)>>::new(); + PoolMembers::::translate::, _>(|key, old_member| { + let id = old_member.pool_id; + temp_members.entry(id).or_default().push((key, old_member.points)); + + total_points_locked += old_member.points; + members_translated += 1; + Some(PoolMember:: { + last_recorded_reward_counter: Zero::zero(), + pool_id: old_member.pool_id, + points: old_member.points, + unbonding_eras: old_member.unbonding_eras, + }) + }); + + // translate all reward pools. In the process, do the last payout as well. + RewardPools::::translate::>, _>( + |id, _old_reward_pool| { + // each pool should have at least one member. + let members = match temp_members.get(&id) { + Some(x) => x, + None => { + log!(error, "pool {} has no member! deleting it..", id); + return None + }, + }; + let bonded_pool = match BondedPools::::get(id) { + Some(x) => x, + None => { + log!(error, "pool {} has no bonded pool! deleting it..", id); + return None + }, + }; + + let accumulated_reward = RewardPool::::current_balance(id); + let reward_account = Pallet::::create_reward_account(id); + let mut sum_paid_out = BalanceOf::::zero(); + + members + .into_iter() + .filter_map(|(who, points)| { + let bonded_pool = match BondedPool::::get(id) { + Some(x) => x, + None => { + log!(error, "pool {} for member {:?} does not exist!", id, who); + return None + }, + }; + + total_value_locked += bonded_pool.points_to_balance(points.clone()); + let portion = Perbill::from_rational(*points, bonded_pool.points); + let last_claim = portion * accumulated_reward; + + log!( + debug, + "{:?} has {:?} ({:?}) of pool {} with total reward of {:?}", + who, + portion, + last_claim, + id, + accumulated_reward + ); + + if last_claim.is_zero() { + None + } else { + Some((who, last_claim)) + } + }) + .for_each(|(who, last_claim)| { + let outcome = T::Currency::transfer( + &reward_account, + &who, + last_claim, + ExistenceRequirement::KeepAlive, + ); + + if let Err(reason) = outcome { + log!(warn, "last reward claim failed due to {:?}", reason,); + } else { + sum_paid_out = sum_paid_out.saturating_add(last_claim); + } + + Pallet::::deposit_event(Event::::PaidOut { + member: who.clone(), + pool_id: id, + payout: last_claim, + }); + }); + + // this can only be because of rounding down, or because the person we + // wanted to pay their reward to could not accept it (dust). + let leftover = accumulated_reward.saturating_sub(sum_paid_out); + if !leftover.is_zero() { + // pay it all to depositor. + let o = T::Currency::transfer( + &reward_account, + &bonded_pool.roles.depositor, + leftover, + ExistenceRequirement::KeepAlive, + ); + log!(warn, "paying {:?} leftover to the depositor: {:?}", leftover, o); + } + + // finally, migrate the reward pool. + reward_pools_translated += 1; + + Some(RewardPool { + last_recorded_reward_counter: Zero::zero(), + last_recorded_total_payouts: Zero::zero(), + total_rewards_claimed: Zero::zero(), + }) + }, + ); + + log!( + info, + "Upgraded {} members, {} reward pools, TVL {:?} TPL {:?}, storage to version {:?}", + members_translated, + reward_pools_translated, + total_value_locked, + total_points_locked, + current + ); + current.put::>(); + T::DbWeight::get().reads_writes(members_translated + 1, reward_pools_translated + 1) + } + } + + impl OnRuntimeUpgrade for MigrateToV2 { + fn on_runtime_upgrade() -> Weight { + let current = Pallet::::current_storage_version(); + let onchain = Pallet::::on_chain_storage_version(); + + log!( + info, + "Running migration with current storage version {:?} / onchain {:?}", + current, + onchain + ); + + if current == 2 && onchain == 1 { + Self::run(current) + } else { + log!(info, "MigrateToV2 did not executed. This probably should be removed"); + T::DbWeight::get().reads(1) + } + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + // all reward accounts must have more than ED. + RewardPools::::iter().for_each(|(id, _)| { + assert!( + T::Currency::free_balance(&Pallet::::create_reward_account(id)) >= + T::Currency::minimum_balance() + ) + }); + + Ok(()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade() -> Result<(), &'static str> { + // new version must be set. + assert_eq!(Pallet::::on_chain_storage_version(), 2); + + // no reward or bonded pool has been skipped. + assert_eq!(RewardPools::::iter().count() as u32, RewardPools::::count()); + assert_eq!(BondedPools::::iter().count() as u32, BondedPools::::count()); + + // all reward pools must have exactly ED in them. This means no reward can be claimed, + // and that setting reward counters all over the board to zero will work henceforth. + RewardPools::::iter().for_each(|(id, _)| { + assert_eq!( + RewardPool::::current_balance(id), + Zero::zero(), + "reward pool({}) balance is {:?}", + id, + RewardPool::::current_balance(id) + ); + }); + + log!(info, "post upgrade hook for MigrateToV2 executed."); + Ok(()) + } + } +} diff --git a/frame/nomination-pools/src/mock.rs b/frame/nomination-pools/src/mock.rs index 7d71efc3be04d..042a0b666efb1 100644 --- a/frame/nomination-pools/src/mock.rs +++ b/frame/nomination-pools/src/mock.rs @@ -2,9 +2,14 @@ use super::*; use crate::{self as pools}; use frame_support::{assert_ok, parameter_types, PalletId}; use frame_system::RawOrigin; +use sp_runtime::FixedU128; pub type AccountId = u128; pub type Balance = u128; +pub type RewardCounter = FixedU128; +// This sneaky little hack allows us to write code exactly as we would do in the pallet in the tests +// as well, e.g. `StorageItem::::get()`. +pub type T = Runtime; // Ext builder creates a pool with id 1. pub fn default_bonded_account() -> AccountId { @@ -23,6 +28,7 @@ parameter_types! { pub storage UnbondingBalanceMap: BTreeMap = Default::default(); #[derive(Clone, PartialEq)] pub static MaxUnbonding: u32 = 8; + pub static StakingMinBond: Balance = 10; pub storage Nominations: Option> = None; } @@ -40,7 +46,7 @@ impl sp_staking::StakingInterface for StakingMock { type AccountId = AccountId; fn minimum_bond() -> Self::Balance { - 10 + StakingMinBond::get() } fn current_era() -> EraIndex { @@ -184,6 +190,8 @@ impl pools::Config for Runtime { type Event = Event; type WeightInfo = (); type Currency = Balances; + type CurrencyBalance = Balance; + type RewardCounter = RewardCounter; type BalanceToU256 = BalanceToU256; type U256ToBalance = U256ToBalance; type StakingInterface = StakingMock; @@ -191,7 +199,7 @@ impl pools::Config for Runtime { type PalletId = PoolsPalletId; type MaxMetadataLen = MaxMetadataLen; type MaxUnbonding = MaxUnbonding; - type MinPointsToBalance = frame_support::traits::ConstU32<10>; + type MaxPointsToBalance = frame_support::traits::ConstU8<10>; } type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; @@ -208,9 +216,16 @@ frame_support::construct_runtime!( } ); -#[derive(Default)] pub struct ExtBuilder { members: Vec<(AccountId, Balance)>, + max_members: Option, + max_members_per_pool: Option, +} + +impl Default for ExtBuilder { + fn default() -> Self { + Self { members: Default::default(), max_members: Some(4), max_members_per_pool: Some(3) } + } } impl ExtBuilder { @@ -225,11 +240,26 @@ impl ExtBuilder { self } + pub(crate) fn min_bond(self, min: Balance) -> Self { + StakingMinBond::set(min); + self + } + pub(crate) fn with_check(self, level: u8) -> Self { CheckLevel::set(level); self } + pub(crate) fn max_members(mut self, max: Option) -> Self { + self.max_members = max; + self + } + + pub(crate) fn max_members_per_pool(mut self, max: Option) -> Self { + self.max_members_per_pool = max; + self + } + pub(crate) fn build(self) -> sp_io::TestExternalities { let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); @@ -238,8 +268,8 @@ impl ExtBuilder { min_join_bond: 2, min_create_bond: 2, max_pools: Some(2), - max_members_per_pool: Some(3), - max_members: Some(4), + max_members_per_pool: self.max_members_per_pool, + max_members: self.max_members, } .assimilate_storage(&mut storage); @@ -281,8 +311,8 @@ pub(crate) fn unsafe_set_state(pool_id: PoolId, state: PoolState) -> Result<(), } parameter_types! { - static PoolsEvents: usize = 0; - static BalancesEvents: usize = 0; + storage PoolsEvents: u32 = 0; + storage BalancesEvents: u32 = 0; } /// All events of this pallet. @@ -293,8 +323,8 @@ pub(crate) fn pool_events_since_last_call() -> Vec> { .filter_map(|e| if let Event::Pools(inner) = e { Some(inner) } else { None }) .collect::>(); let already_seen = PoolsEvents::get(); - PoolsEvents::set(events.len()); - events.into_iter().skip(already_seen).collect() + PoolsEvents::set(&(events.len() as u32)); + events.into_iter().skip(already_seen as usize).collect() } /// All events of the `Balances` pallet. @@ -305,8 +335,8 @@ pub(crate) fn balances_events_since_last_call() -> Vec>(); let already_seen = BalancesEvents::get(); - BalancesEvents::set(events.len()); - events.into_iter().skip(already_seen).collect() + BalancesEvents::set(&(events.len() as u32)); + events.into_iter().skip(already_seen as usize).collect() } /// Same as `fully_unbond`, in permissioned setting. diff --git a/frame/nomination-pools/src/tests.rs b/frame/nomination-pools/src/tests.rs index 97104423c5910..9989893d86462 100644 --- a/frame/nomination-pools/src/tests.rs +++ b/frame/nomination-pools/src/tests.rs @@ -17,10 +17,7 @@ use super::*; use crate::{mock::*, Event}; -use frame_support::{ - assert_err, assert_noop, assert_ok, assert_storage_noop, bounded_btree_map, - storage::{with_transaction, TransactionOutcome}, -}; +use frame_support::{assert_err, assert_noop, assert_ok, assert_storage_noop, bounded_btree_map}; use pallet_balances::Event as BEvent; use sp_runtime::traits::Dispatchable; @@ -66,7 +63,11 @@ fn test_setup_works() { ); assert_eq!( RewardPools::::get(last_pool).unwrap(), - RewardPool:: { balance: 0, points: 0u32.into(), total_earnings: 0 } + RewardPool:: { + last_recorded_reward_counter: Zero::zero(), + last_recorded_total_payouts: 0, + total_rewards_claimed: 0 + } ); assert_eq!( PoolMembers::::get(10).unwrap(), @@ -206,34 +207,34 @@ mod bonded_pool { }, }; - let min_points_to_balance: u128 = - <::MinPointsToBalance as Get>::get().into(); + let max_points_to_balance: u128 = + <::MaxPointsToBalance as Get>::get().into(); // Simulate a 100% slashed pool StakingMock::set_bonded_balance(pool.bonded_account(), 0); assert_noop!(pool.ok_to_join(0), Error::::OverflowRisk); - // Simulate a slashed pool at `MinPointsToBalance` + 1 slashed pool + // Simulate a slashed pool at `MaxPointsToBalance` + 1 slashed pool StakingMock::set_bonded_balance( pool.bonded_account(), - min_points_to_balance.saturating_add(1).into(), + max_points_to_balance.saturating_add(1).into(), ); assert_ok!(pool.ok_to_join(0)); - // Simulate a slashed pool at `MinPointsToBalance` - StakingMock::set_bonded_balance(pool.bonded_account(), min_points_to_balance); + // Simulate a slashed pool at `MaxPointsToBalance` + StakingMock::set_bonded_balance(pool.bonded_account(), max_points_to_balance); assert_noop!(pool.ok_to_join(0), Error::::OverflowRisk); StakingMock::set_bonded_balance( pool.bonded_account(), - Balance::MAX / min_points_to_balance, + Balance::MAX / max_points_to_balance, ); // New bonded balance would be over threshold of Balance type assert_noop!(pool.ok_to_join(0), Error::::OverflowRisk); // and a sanity check StakingMock::set_bonded_balance( pool.bonded_account(), - Balance::MAX / min_points_to_balance - 1, + Balance::MAX / max_points_to_balance - 1, ); assert_ok!(pool.ok_to_join(0)); }); @@ -515,44 +516,37 @@ mod join { .put(); // and reward pool - RewardPools::::insert( - 123, - RewardPool:: { - balance: Zero::zero(), - total_earnings: Zero::zero(), - points: U256::from(0u32), - }, - ); + RewardPools::::insert(123, RewardPool:: { ..Default::default() }); - // Force the points:balance ratio to `MinPointsToBalance` (100/10) - let min_points_to_balance: u128 = - <::MinPointsToBalance as Get>::get().into(); + // Force the points:balance ratio to `MaxPointsToBalance` (100/10) + let max_points_to_balance: u128 = + <::MaxPointsToBalance as Get>::get().into(); StakingMock::set_bonded_balance( Pools::create_bonded_account(123), - min_points_to_balance, + max_points_to_balance, ); assert_noop!(Pools::join(Origin::signed(11), 420, 123), Error::::OverflowRisk); StakingMock::set_bonded_balance( Pools::create_bonded_account(123), - Balance::MAX / min_points_to_balance, + Balance::MAX / max_points_to_balance, ); - // Balance needs to be gt Balance::MAX / `MinPointsToBalance` + // Balance needs to be gt Balance::MAX / `MaxPointsToBalance` assert_noop!(Pools::join(Origin::signed(11), 5, 123), Error::::OverflowRisk); - StakingMock::set_bonded_balance(Pools::create_bonded_account(1), min_points_to_balance); + StakingMock::set_bonded_balance(Pools::create_bonded_account(1), max_points_to_balance); // Cannot join a pool that isn't open unsafe_set_state(123, PoolState::Blocked).unwrap(); assert_noop!( - Pools::join(Origin::signed(11), min_points_to_balance, 123), + Pools::join(Origin::signed(11), max_points_to_balance, 123), Error::::NotOpen ); unsafe_set_state(123, PoolState::Destroying).unwrap(); assert_noop!( - Pools::join(Origin::signed(11), min_points_to_balance, 123), + Pools::join(Origin::signed(11), max_points_to_balance, 123), Error::::NotOpen ); @@ -649,17 +643,34 @@ mod join { mod claim_payout { use super::*; - fn del(points: Balance, reward_pool_total_earnings: Balance) -> PoolMember { + fn del(points: Balance, last_recorded_reward_counter: u128) -> PoolMember { PoolMember { pool_id: 1, points, - reward_pool_total_earnings, + last_recorded_reward_counter: last_recorded_reward_counter.into(), unbonding_eras: Default::default(), } } - fn rew(balance: Balance, points: u32, total_earnings: Balance) -> RewardPool { - RewardPool { balance, points: points.into(), total_earnings } + fn del_float(points: Balance, last_recorded_reward_counter: f64) -> PoolMember { + PoolMember { + pool_id: 1, + points, + last_recorded_reward_counter: RewardCounter::from_float(last_recorded_reward_counter), + unbonding_eras: Default::default(), + } + } + + fn rew( + last_recorded_reward_counter: u128, + last_recorded_total_payouts: Balance, + total_rewards_claimed: Balance, + ) -> RewardPool { + RewardPool { + last_recorded_reward_counter: last_recorded_reward_counter.into(), + last_recorded_total_payouts, + total_rewards_claimed, + } } #[test] @@ -672,8 +683,12 @@ mod claim_payout { Balances::make_free_balance_be(&40, 0); Balances::make_free_balance_be(&50, 0); let ed = Balances::minimum_balance(); + // and the reward pool has earned 100 in rewards - Balances::make_free_balance_be(&default_reward_account(), ed + 100); + assert_eq!(Balances::free_balance(default_reward_account()), ed); + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 100)); + + let _ = pool_events_since_last_call(); // When assert_ok!(Pools::claim_payout(Origin::signed(10))); @@ -681,22 +696,13 @@ mod claim_payout { // Then assert_eq!( pool_events_since_last_call(), - vec![ - Event::Created { depositor: 10, pool_id: 1 }, - Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, - Event::Bonded { member: 40, pool_id: 1, bonded: 40, joined: true }, - Event::Bonded { member: 50, pool_id: 1, bonded: 50, joined: true }, - Event::PaidOut { member: 10, pool_id: 1, payout: 10 }, - ] - ); - - // Expect a payout of 10: (10 del virtual points / 100 pool points) * 100 pool - // balance - assert_eq!(PoolMembers::::get(10).unwrap(), del(10, 100)); - assert_eq!( - RewardPools::::get(&1).unwrap(), - rew(90, 100 * 100 - 100 * 10, 100) + vec![Event::PaidOut { member: 10, pool_id: 1, payout: 10 },] ); + // last recorded reward counter at the time of this member's payout is 1 + assert_eq!(PoolMembers::::get(10).unwrap(), del(10, 1)); + // pool's 'last_recorded_reward_counter' and 'last_recorded_total_payouts' don't + // really change unless if someone bonds/unbonds. + assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 10)); assert_eq!(Balances::free_balance(&10), 10); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 90); @@ -708,13 +714,8 @@ mod claim_payout { pool_events_since_last_call(), vec![Event::PaidOut { member: 40, pool_id: 1, payout: 40 }] ); - - // Expect payout 40: (400 del virtual points / 900 pool points) * 90 pool balance - assert_eq!(PoolMembers::::get(40).unwrap(), del(40, 100)); - assert_eq!( - RewardPools::::get(&1).unwrap(), - rew(50, 9_000 - 100 * 40, 100) - ); + assert_eq!(PoolMembers::::get(40).unwrap(), del(40, 1)); + assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 50)); assert_eq!(Balances::free_balance(&40), 40); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 50); @@ -726,15 +727,13 @@ mod claim_payout { pool_events_since_last_call(), vec![Event::PaidOut { member: 50, pool_id: 1, payout: 50 }] ); - - // Expect payout 50: (50 del virtual points / 50 pool points) * 50 pool balance - assert_eq!(PoolMembers::::get(50).unwrap(), del(50, 100)); + assert_eq!(PoolMembers::::get(50).unwrap(), del(50, 1)); assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 100)); assert_eq!(Balances::free_balance(&50), 50); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 0); // Given the reward pool has some new rewards - Balances::make_free_balance_be(&default_reward_account(), ed + 50); + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 50)); // When assert_ok!(Pools::claim_payout(Origin::signed(10))); @@ -744,10 +743,8 @@ mod claim_payout { pool_events_since_last_call(), vec![Event::PaidOut { member: 10, pool_id: 1, payout: 5 }] ); - - // Expect payout 5: (500 del virtual points / 5,000 pool points) * 50 pool balance - assert_eq!(PoolMembers::::get(10).unwrap(), del(10, 150)); - assert_eq!(RewardPools::::get(&1).unwrap(), rew(45, 5_000 - 50 * 10, 150)); + assert_eq!(PoolMembers::::get(10).unwrap(), del_float(10, 1.5)); + assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 105)); assert_eq!(Balances::free_balance(&10), 10 + 5); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 45); @@ -759,11 +756,8 @@ mod claim_payout { pool_events_since_last_call(), vec![Event::PaidOut { member: 40, pool_id: 1, payout: 20 }] ); - - // Expect payout 20: (2,000 del virtual points / 4,500 pool points) * 45 pool - // balance - assert_eq!(PoolMembers::::get(40).unwrap(), del(40, 150)); - assert_eq!(RewardPools::::get(&1).unwrap(), rew(25, 4_500 - 50 * 40, 150)); + assert_eq!(PoolMembers::::get(40).unwrap(), del_float(40, 1.5)); + assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 125)); assert_eq!(Balances::free_balance(&40), 40 + 20); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 25); @@ -779,22 +773,8 @@ mod claim_payout { pool_events_since_last_call(), vec![Event::PaidOut { member: 50, pool_id: 1, payout: 50 }] ); - - // We expect a payout of 50: (5,000 del virtual points / 7,5000 pool points) * 75 - // pool balance - assert_eq!(PoolMembers::::get(50).unwrap(), del(50, 200)); - assert_eq!( - RewardPools::::get(&1).unwrap(), - rew( - 25, - // old pool points + points from new earnings - del points. - // - // points from new earnings = new earnings(50) * bonded_pool.points(100) - // del points = member.points(50) * new_earnings_since_last_claim (100) - (2_500 + 50 * 100) - 50 * 100, - 200, - ) - ); + assert_eq!(PoolMembers::::get(50).unwrap(), del_float(50, 2.0)); + assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 175)); assert_eq!(Balances::free_balance(&50), 50 + 50); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 25); @@ -806,10 +786,8 @@ mod claim_payout { pool_events_since_last_call(), vec![Event::PaidOut { member: 10, pool_id: 1, payout: 5 }] ); - - // We expect a payout of 5 - assert_eq!(PoolMembers::::get(10).unwrap(), del(10, 200)); - assert_eq!(RewardPools::::get(&1).unwrap(), rew(20, 2_500 - 10 * 50, 200)); + assert_eq!(PoolMembers::::get(10).unwrap(), del(10, 2)); + assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 180)); assert_eq!(Balances::free_balance(&10), 15 + 5); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 20); @@ -827,19 +805,8 @@ mod claim_payout { ); // We expect a payout of 40 - assert_eq!(PoolMembers::::get(10).unwrap(), del(10, 600)); - assert_eq!( - RewardPools::::get(&1).unwrap(), - rew( - 380, - // old pool points + points from new earnings - del points - // - // points from new earnings = new earnings(400) * bonded_pool.points(100) - // del points = member.points(10) * new_earnings_since_last_claim(400) - (2_000 + 400 * 100) - 10 * 400, - 600 - ) - ); + assert_eq!(PoolMembers::::get(10).unwrap(), del(10, 6)); + assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 220)); assert_eq!(Balances::free_balance(&10), 20 + 40); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 380); @@ -855,14 +822,8 @@ mod claim_payout { pool_events_since_last_call(), vec![Event::PaidOut { member: 10, pool_id: 1, payout: 2 }] ); - - // Expect a payout of 2: (200 del virtual points / 38,000 pool points) * 400 pool - // balance - assert_eq!(PoolMembers::::get(10).unwrap(), del(10, 620)); - assert_eq!( - RewardPools::::get(&1).unwrap(), - rew(398, (38_000 + 20 * 100) - 10 * 20, 620) - ); + assert_eq!(PoolMembers::::get(10).unwrap(), del_float(10, 6.2)); + assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 222)); assert_eq!(Balances::free_balance(&10), 60 + 2); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 398); @@ -874,14 +835,8 @@ mod claim_payout { pool_events_since_last_call(), vec![Event::PaidOut { member: 40, pool_id: 1, payout: 188 }] ); - - // Expect a payout of 188: (18,800 del virtual points / 39,800 pool points) * 399 - // pool balance - assert_eq!(PoolMembers::::get(40).unwrap(), del(40, 620)); - assert_eq!( - RewardPools::::get(&1).unwrap(), - rew(210, 39_800 - 40 * 470, 620) - ); + assert_eq!(PoolMembers::::get(40).unwrap(), del_float(40, 6.2)); + assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 410)); assert_eq!(Balances::free_balance(&40), 60 + 188); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 210); @@ -893,88 +848,20 @@ mod claim_payout { pool_events_since_last_call(), vec![Event::PaidOut { member: 50, pool_id: 1, payout: 210 }] ); - - // Expect payout of 210: (21,000 / 21,000) * 210 - assert_eq!(PoolMembers::::get(50).unwrap(), del(50, 620)); - assert_eq!( - RewardPools::::get(&1).unwrap(), - rew(0, 21_000 - 50 * 420, 620) - ); + assert_eq!(PoolMembers::::get(50).unwrap(), del_float(50, 6.2)); + assert_eq!(RewardPools::::get(&1).unwrap(), rew(0, 0, 620)); assert_eq!(Balances::free_balance(&50), 100 + 210); assert_eq!(Balances::free_balance(&default_reward_account()), ed + 0); }); } - #[test] - fn do_reward_payout_correctly_sets_pool_state_to_destroying() { - ExtBuilder::default().build_and_execute(|| { - let _ = with_transaction(|| -> TransactionOutcome { - let mut bonded_pool = BondedPool::::get(1).unwrap(); - let mut reward_pool = RewardPools::::get(1).unwrap(); - let mut member = PoolMembers::::get(10).unwrap(); - - // -- reward_pool.total_earnings saturates - - // Given - Balances::make_free_balance_be(&default_reward_account(), Balance::MAX); - - // When - assert_ok!(Pools::do_reward_payout( - &10, - &mut member, - &mut bonded_pool, - &mut reward_pool - )); - - // Then - assert!(bonded_pool.is_destroying()); - - storage::TransactionOutcome::Rollback(Ok(())) - }); - - // -- current_points saturates (reward_pool.points + new_earnings * bonded_pool.points) - let _ = with_transaction(|| -> TransactionOutcome { - // Given - let mut bonded_pool = BondedPool::::get(1).unwrap(); - let mut reward_pool = RewardPools::::get(1).unwrap(); - let mut member = PoolMembers::::get(10).unwrap(); - // Force new_earnings * bonded_pool.points == 100 - Balances::make_free_balance_be(&default_reward_account(), 5 + 10); - assert_eq!(bonded_pool.points, 10); - // Force reward_pool.points == U256::MAX - new_earnings * bonded_pool.points - reward_pool.points = U256::MAX - U256::from(100u32); - RewardPools::::insert(1, reward_pool.clone()); - - // When - assert_ok!(Pools::do_reward_payout( - &10, - &mut member, - &mut bonded_pool, - &mut reward_pool - )); - - // Then - assert!(bonded_pool.is_destroying()); - - storage::TransactionOutcome::Rollback(Ok(())) - }); - }); - } - #[test] fn reward_payout_errors_if_a_member_is_fully_unbonding() { ExtBuilder::default().add_members(vec![(11, 11)]).build_and_execute(|| { // fully unbond the member. - assert_ok!(Pools::fully_unbond(Origin::signed(11), 11)); + assert_ok!(fully_unbond_permissioned(11)); - let mut bonded_pool = BondedPool::::get(1).unwrap(); - let mut reward_pool = RewardPools::::get(1).unwrap(); - let mut member = PoolMembers::::get(11).unwrap(); - - assert_noop!( - Pools::do_reward_payout(&11, &mut member, &mut bonded_pool, &mut reward_pool,), - Error::::FullyUnbonding - ); + assert_noop!(Pools::claim_payout(Origin::signed(11)), Error::::FullyUnbonding); assert_eq!( pool_events_since_last_call(), @@ -989,80 +876,87 @@ mod claim_payout { } #[test] - fn calculate_member_payout_works_with_a_pool_of_1() { - let del = |reward_pool_total_earnings| del(10, reward_pool_total_earnings); + fn do_reward_payout_works_with_a_pool_of_1() { + let del = |last_recorded_reward_counter| del_float(10, last_recorded_reward_counter); ExtBuilder::default().build_and_execute(|| { - let mut bonded_pool = BondedPool::::get(1).unwrap(); - let mut reward_pool = RewardPools::::get(1).unwrap(); - let mut member = PoolMembers::::get(10).unwrap(); + let (mut member, mut bonded_pool, mut reward_pool) = + Pools::get_member_with_pools(&10).unwrap(); let ed = Balances::minimum_balance(); - // Given no rewards have been earned - // When let payout = - Pools::calculate_member_payout(&mut member, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&10, &mut member, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then assert_eq!(payout, 0); - assert_eq!(member, del(0)); + assert_eq!(member, del(0.0)); assert_eq!(reward_pool, rew(0, 0, 0)); // Given the pool has earned some rewards for the first time - Balances::make_free_balance_be(&default_reward_account(), ed + 5); + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 5)); // When let payout = - Pools::calculate_member_payout(&mut member, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&10, &mut member, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 5); // (10 * 5 del virtual points / 10 * 5 pool points) * 5 pool balance + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::PaidOut { member: 10, pool_id: 1, payout: 5 } + ] + ); + assert_eq!(payout, 5); assert_eq!(reward_pool, rew(0, 0, 5)); - assert_eq!(member, del(5)); + assert_eq!(member, del(0.5)); // Given the pool has earned rewards again - Balances::make_free_balance_be(&default_reward_account(), ed + 10); + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 10)); // When let payout = - Pools::calculate_member_payout(&mut member, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&10, &mut member, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 10); // (10 * 10 del virtual points / 10 pool points) * 5 pool balance + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 10, pool_id: 1, payout: 10 }] + ); + assert_eq!(payout, 10); assert_eq!(reward_pool, rew(0, 0, 15)); - assert_eq!(member, del(15)); + assert_eq!(member, del(1.5)); // Given the pool has earned no new rewards Balances::make_free_balance_be(&default_reward_account(), ed + 0); // When let payout = - Pools::calculate_member_payout(&mut member, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&10, &mut member, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then + assert_eq!(pool_events_since_last_call(), vec![]); assert_eq!(payout, 0); assert_eq!(reward_pool, rew(0, 0, 15)); - assert_eq!(member, del(15)); + assert_eq!(member, del(1.5)); }); } #[test] - fn calculate_member_payout_works_with_a_pool_of_3() { + fn do_reward_payout_works_with_a_pool_of_3() { ExtBuilder::default() .add_members(vec![(40, 40), (50, 50)]) .build_and_execute(|| { let mut bonded_pool = BondedPool::::get(1).unwrap(); let mut reward_pool = RewardPools::::get(1).unwrap(); - let ed = Balances::minimum_balance(); - // PoolMember with 10 points + let mut del_10 = PoolMembers::::get(10).unwrap(); - // PoolMember with 40 points let mut del_40 = PoolMembers::::get(40).unwrap(); - // PoolMember with 50 points let mut del_50 = PoolMembers::::get(50).unwrap(); assert_eq!( @@ -1078,479 +972,970 @@ mod claim_payout { // Given we have a total of 100 points split among the members assert_eq!(del_50.points + del_40.points + del_10.points, 100); assert_eq!(bonded_pool.points, 100); + // and the reward pool has earned 100 in rewards - Balances::make_free_balance_be(&default_reward_account(), ed + 100); + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 100)); // When let payout = - Pools::calculate_member_payout(&mut del_10, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&10, &mut del_10, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 10); // (10 del virtual points / 100 pool points) * 100 pool balance - assert_eq!(del_10, del(10, 100)); - assert_eq!(reward_pool, rew(90, 100 * 100 - 100 * 10, 100)); - // Mock the reward pool transferring the payout to del_10 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= 10)); + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 10, pool_id: 1, payout: 10 }] + ); + assert_eq!(payout, 10); + assert_eq!(del_10, del(10, 1)); + assert_eq!(reward_pool, rew(0, 0, 10)); // When let payout = - Pools::calculate_member_payout(&mut del_40, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&40, &mut del_40, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 40); // (400 del virtual points / 900 pool points) * 90 pool balance - assert_eq!(del_40, del(40, 100)); assert_eq!( - reward_pool, - rew( - 50, - // old pool points - member virtual points - 9_000 - 100 * 40, - 100 - ) + pool_events_since_last_call(), + vec![Event::PaidOut { member: 40, pool_id: 1, payout: 40 }] ); - // Mock the reward pool transferring the payout to del_40 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= 40)); + assert_eq!(payout, 40); + assert_eq!(del_40, del(40, 1)); + assert_eq!(reward_pool, rew(0, 0, 50)); // When let payout = - Pools::calculate_member_payout(&mut del_50, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&50, &mut del_50, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 50); // (50 del virtual points / 50 pool points) * 50 pool balance - assert_eq!(del_50, del(50, 100)); + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 50, pool_id: 1, payout: 50 }] + ); + assert_eq!(payout, 50); + assert_eq!(del_50, del(50, 1)); assert_eq!(reward_pool, rew(0, 0, 100)); - // Mock the reward pool transferring the payout to del_50 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= 50)); // Given the reward pool has some new rewards - Balances::make_free_balance_be(&default_reward_account(), ed + 50); + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 50)); // When let payout = - Pools::calculate_member_payout(&mut del_10, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&10, &mut del_10, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 5); // (500 del virtual points / 5,000 pool points) * 50 pool balance - assert_eq!(del_10, del(10, 150)); - assert_eq!(reward_pool, rew(45, 5_000 - 50 * 10, 150)); - // Mock the reward pool transferring the payout to del_10 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= 5)); + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 10, pool_id: 1, payout: 5 }] + ); + assert_eq!(payout, 5); + assert_eq!(del_10, del_float(10, 1.5)); + assert_eq!(reward_pool, rew(0, 0, 105)); // When let payout = - Pools::calculate_member_payout(&mut del_40, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&40, &mut del_40, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 20); // (2,000 del virtual points / 4,500 pool points) * 45 pool balance - assert_eq!(del_40, del(40, 150)); - assert_eq!(reward_pool, rew(25, 4_500 - 50 * 40, 150)); - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= 20)); + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 40, pool_id: 1, payout: 20 }] + ); + assert_eq!(payout, 20); + assert_eq!(del_40, del_float(40, 1.5)); + assert_eq!(reward_pool, rew(0, 0, 125)); // Given del_50 hasn't claimed and the reward pools has just earned 50 assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 50)); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 75); // When let payout = - Pools::calculate_member_payout(&mut del_50, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&50, &mut del_50, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 50); // (5,000 del virtual points / 7,5000 pool points) * 75 pool balance - assert_eq!(del_50, del(50, 200)); - assert_eq!( - reward_pool, - rew( - 25, - // old pool points + points from new earnings - del points. - // - // points from new earnings = new earnings(50) * bonded_pool.points(100) - // del points = member.points(50) * new_earnings_since_last_claim (100) - (2_500 + 50 * 100) - 50 * 100, - 200, - ) - ); - // Mock the reward pool transferring the payout to del_50 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= 50)); + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 50, pool_id: 1, payout: 50 }] + ); + assert_eq!(payout, 50); + assert_eq!(del_50, del_float(50, 2.0)); + assert_eq!(reward_pool, rew(0, 0, 175)); // When let payout = - Pools::calculate_member_payout(&mut del_10, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&10, &mut del_10, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 10, pool_id: 1, payout: 5 }] + ); assert_eq!(payout, 5); - assert_eq!(del_10, del(10, 200)); - assert_eq!(reward_pool, rew(20, 2_500 - 10 * 50, 200)); - // Mock the reward pool transferring the payout to del_10 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= 5)); + assert_eq!(del_10, del_float(10, 2.0)); + assert_eq!(reward_pool, rew(0, 0, 180)); // Given del_40 hasn't claimed and the reward pool has just earned 400 assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 400)); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 420); // When let payout = - Pools::calculate_member_payout(&mut del_10, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&10, &mut del_10, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 10, pool_id: 1, payout: 40 }] + ); assert_eq!(payout, 40); - assert_eq!(del_10, del(10, 600)); - assert_eq!( - reward_pool, - rew( - 380, - // old pool points + points from new earnings - del points - // - // points from new earnings = new earnings(400) * bonded_pool.points(100) - // del points = member.points(10) * new_earnings_since_last_claim(400) - (2_000 + 400 * 100) - 10 * 400, - 600 - ) - ); - // Mock the reward pool transferring the payout to del_10 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= 40)); + assert_eq!(del_10, del_float(10, 6.0)); + assert_eq!(reward_pool, rew(0, 0, 220)); // Given del_40 + del_50 haven't claimed and the reward pool has earned 20 assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 20)); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 400); // When let payout = - Pools::calculate_member_payout(&mut del_10, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&10, &mut del_10, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 2); // (200 del virtual points / 38,000 pool points) * 400 pool balance - assert_eq!(del_10, del(10, 620)); - assert_eq!(reward_pool, rew(398, (38_000 + 20 * 100) - 10 * 20, 620)); - // Mock the reward pool transferring the payout to del_10 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= 2)); + assert_eq!(payout, 2); + assert_eq!(del_10, del_float(10, 6.2)); + assert_eq!(reward_pool, rew(0, 0, 222)); // When let payout = - Pools::calculate_member_payout(&mut del_40, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&40, &mut del_40, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 188); // (18,800 del virtual points / 39,800 pool points) * 399 pool balance - assert_eq!(del_40, del(40, 620)); - assert_eq!(reward_pool, rew(210, 39_800 - 40 * 470, 620)); - // Mock the reward pool transferring the payout to del_10 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= 188)); + assert_eq!(payout, 188); // 20 (from the 50) + 160 (from the 400) + 8 (from the 20) + assert_eq!(del_40, del_float(40, 6.2)); + assert_eq!(reward_pool, rew(0, 0, 410)); // When let payout = - Pools::calculate_member_payout(&mut del_50, &mut bonded_pool, &mut reward_pool) + Pools::do_reward_payout(&50, &mut del_50, &mut bonded_pool, &mut reward_pool) .unwrap(); // Then - assert_eq!(payout, 210); // (21,000 / 21,000) * 210 - assert_eq!(del_50, del(50, 620)); - assert_eq!(reward_pool, rew(0, 21_000 - 50 * 420, 620)); + assert_eq!(payout, 210); // 200 (from the 400) + 10 (from the 20) + assert_eq!(del_50, del_float(50, 6.2)); + assert_eq!(reward_pool, rew(0, 0, 620)); }); } #[test] - fn do_reward_payout_works() { - ExtBuilder::default() - .add_members(vec![(40, 40), (50, 50)]) - .build_and_execute(|| { - let mut bonded_pool = BondedPool::::get(1).unwrap(); - let mut reward_pool = RewardPools::::get(1).unwrap(); - let ed = Balances::minimum_balance(); + fn rewards_distribution_is_fair_basic() { + ExtBuilder::default().build_and_execute(|| { + // reward pool by 10. + Balances::mutate_account(&default_reward_account(), |f| f.free += 10).unwrap(); - assert_eq!( - pool_events_since_last_call(), - vec![ - Event::Created { depositor: 10, pool_id: 1 }, - Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, - Event::Bonded { member: 40, pool_id: 1, bonded: 40, joined: true }, - Event::Bonded { member: 50, pool_id: 1, bonded: 50, joined: true } - ] - ); + // 20 joins afterwards. + Balances::make_free_balance_be(&20, Balances::minimum_balance() + 10); + assert_ok!(Pools::join(Origin::signed(20), 10, 1)); - // Given the bonded pool has 100 points - assert_eq!(bonded_pool.points, 100); - // Each member currently has a free balance of - Balances::make_free_balance_be(&10, 0); - Balances::make_free_balance_be(&40, 0); - Balances::make_free_balance_be(&50, 0); - // and the reward pool has earned 100 in rewards - Balances::make_free_balance_be(&default_reward_account(), ed + 100); + // reward by another 20 + Balances::mutate_account(&default_reward_account(), |f| f.free += 20).unwrap(); - let mut del_10 = PoolMembers::get(10).unwrap(); - let mut del_40 = PoolMembers::get(40).unwrap(); - let mut del_50 = PoolMembers::get(50).unwrap(); + // 10 should claim 10 + 10, 20 should claim 20 / 2. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 10, joined: true }, + Event::PaidOut { member: 10, pool_id: 1, payout: 20 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 10 }, + ] + ); - // When - assert_ok!(Pools::do_reward_payout( - &10, - &mut del_10, - &mut bonded_pool, - &mut reward_pool - )); + // any upcoming rewards are shared equally. + Balances::mutate_account(&default_reward_account(), |f| f.free += 20).unwrap(); - // Then - assert_eq!( - pool_events_since_last_call(), - vec![Event::PaidOut { member: 10, pool_id: 1, payout: 10 }] - ); + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); - // Expect a payout of 10: (10 del virtual points / 100 pool points) * 100 pool - // balance - assert_eq!(del_10, del(10, 100)); - assert_eq!(reward_pool, rew(90, 100 * 100 - 100 * 10, 100)); - assert_eq!(Balances::free_balance(&10), 10); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 90); + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::PaidOut { member: 10, pool_id: 1, payout: 10 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 10 }, + ] + ); + }); + } - // When - assert_ok!(Pools::do_reward_payout( - &40, - &mut del_40, - &mut bonded_pool, - &mut reward_pool - )); + #[test] + fn rewards_distribution_is_fair_basic_with_fractions() { + // basically checks the case where the amount of rewards is less than the pool shares. for + // this, we have to rely on fixed point arithmetic. + ExtBuilder::default().build_and_execute(|| { + Balances::mutate_account(&default_reward_account(), |f| f.free += 3).unwrap(); - // Then - assert_eq!( - pool_events_since_last_call(), - vec![Event::PaidOut { member: 40, pool_id: 1, payout: 40 }] - ); + Balances::make_free_balance_be(&20, Balances::minimum_balance() + 10); + assert_ok!(Pools::join(Origin::signed(20), 10, 1)); - // Expect payout 40: (400 del virtual points / 900 pool points) * 90 pool balance - assert_eq!(del_40, del(40, 100)); - assert_eq!(reward_pool, rew(50, 9_000 - 100 * 40, 100)); - assert_eq!(Balances::free_balance(&40), 40); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 50); + Balances::mutate_account(&default_reward_account(), |f| f.free += 6).unwrap(); - // When - assert_ok!(Pools::do_reward_payout( - &50, - &mut del_50, - &mut bonded_pool, - &mut reward_pool - )); + // 10 should claim 3, 20 should claim 3 + 3. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); - // Then - assert_eq!( - pool_events_since_last_call(), - vec![Event::PaidOut { member: 50, pool_id: 1, payout: 50 }] - ); + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 10, joined: true }, + Event::PaidOut { member: 10, pool_id: 1, payout: 3 + 3 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 3 }, + ] + ); - // Expect payout 50: (50 del virtual points / 50 pool points) * 50 pool balance - assert_eq!(del_50, del(50, 100)); - assert_eq!(reward_pool, rew(0, 0, 100)); - assert_eq!(Balances::free_balance(&50), 50); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 0); + // any upcoming rewards are shared equally. + Balances::mutate_account(&default_reward_account(), |f| f.free += 8).unwrap(); - // Given the reward pool has some new rewards - Balances::make_free_balance_be(&default_reward_account(), ed + 50); + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); - // When - assert_ok!(Pools::do_reward_payout( - &10, - &mut del_10, - &mut bonded_pool, - &mut reward_pool - )); + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::PaidOut { member: 10, pool_id: 1, payout: 4 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 4 }, + ] + ); - // Then - assert_eq!( - pool_events_since_last_call(), - vec![Event::PaidOut { member: 10, pool_id: 1, payout: 5 }] - ); + // uneven upcoming rewards are shared equally, rounded down. + Balances::mutate_account(&default_reward_account(), |f| f.free += 7).unwrap(); - // Expect payout 5: (500 del virtual points / 5,000 pool points) * 50 pool balance - assert_eq!(del_10, del(10, 150)); - assert_eq!(reward_pool, rew(45, 5_000 - 50 * 10, 150)); - assert_eq!(Balances::free_balance(&10), 10 + 5); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 45); + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); - // When - assert_ok!(Pools::do_reward_payout( - &40, - &mut del_40, - &mut bonded_pool, - &mut reward_pool - )); + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::PaidOut { member: 10, pool_id: 1, payout: 3 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 3 }, + ] + ); + }); + } - // Then - assert_eq!( - pool_events_since_last_call(), - vec![Event::PaidOut { member: 40, pool_id: 1, payout: 20 }] - ); + #[test] + fn rewards_distribution_is_fair_3() { + ExtBuilder::default().build_and_execute(|| { + let ed = Balances::minimum_balance(); - // Expect payout 20: (2,000 del virtual points / 4,500 pool points) * 45 pool - // balance - assert_eq!(del_40, del(40, 150)); - assert_eq!(reward_pool, rew(25, 4_500 - 50 * 40, 150)); - assert_eq!(Balances::free_balance(&40), 40 + 20); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 25); + Balances::mutate_account(&default_reward_account(), |f| f.free += 30).unwrap(); - // Given del 50 hasn't claimed and the reward pools has just earned 50 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 50)); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 75); + Balances::make_free_balance_be(&20, ed + 10); + assert_ok!(Pools::join(Origin::signed(20), 10, 1)); - // When - assert_ok!(Pools::do_reward_payout( - &50, - &mut del_50, - &mut bonded_pool, - &mut reward_pool - )); + Balances::mutate_account(&default_reward_account(), |f| f.free += 100).unwrap(); - // Then + Balances::make_free_balance_be(&30, ed + 10); + assert_ok!(Pools::join(Origin::signed(30), 10, 1)); + + Balances::mutate_account(&default_reward_account(), |f| f.free += 60).unwrap(); + + // 10 should claim 10, 20 should claim nothing. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + assert_ok!(Pools::claim_payout(Origin::signed(30))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 30, pool_id: 1, bonded: 10, joined: true }, + Event::PaidOut { member: 10, pool_id: 1, payout: 30 + 100 / 2 + 60 / 3 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 100 / 2 + 60 / 3 }, + Event::PaidOut { member: 30, pool_id: 1, payout: 60 / 3 }, + ] + ); + + // any upcoming rewards are shared equally. + Balances::mutate_account(&default_reward_account(), |f| f.free += 30).unwrap(); + + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + assert_ok!(Pools::claim_payout(Origin::signed(30))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::PaidOut { member: 10, pool_id: 1, payout: 10 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 10 }, + Event::PaidOut { member: 30, pool_id: 1, payout: 10 }, + ] + ); + }); + } + + #[test] + fn rewards_distribution_is_fair_bond_extra() { + ExtBuilder::default().build_and_execute(|| { + let ed = Balances::minimum_balance(); + + Balances::make_free_balance_be(&20, ed + 20); + assert_ok!(Pools::join(Origin::signed(20), 20, 1)); + Balances::make_free_balance_be(&30, ed + 20); + assert_ok!(Pools::join(Origin::signed(30), 10, 1)); + + Balances::mutate_account(&default_reward_account(), |f| f.free += 40).unwrap(); + + // everyone claims. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + assert_ok!(Pools::claim_payout(Origin::signed(30))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, + Event::Bonded { member: 30, pool_id: 1, bonded: 10, joined: true }, + Event::PaidOut { member: 10, pool_id: 1, payout: 10 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 20 }, + Event::PaidOut { member: 30, pool_id: 1, payout: 10 } + ] + ); + + // 30 now bumps itself to be like 20. + assert_ok!(Pools::bond_extra(Origin::signed(30), BondExtra::FreeBalance(10))); + + // more rewards come in. + Balances::mutate_account(&default_reward_account(), |f| f.free += 100).unwrap(); + + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + assert_ok!(Pools::claim_payout(Origin::signed(30))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Bonded { member: 30, pool_id: 1, bonded: 10, joined: false }, + Event::PaidOut { member: 10, pool_id: 1, payout: 20 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 40 }, + Event::PaidOut { member: 30, pool_id: 1, payout: 40 } + ] + ); + }); + } + + #[test] + fn rewards_distribution_is_fair_unbond() { + ExtBuilder::default().build_and_execute(|| { + let ed = Balances::minimum_balance(); + + Balances::make_free_balance_be(&20, ed + 20); + assert_ok!(Pools::join(Origin::signed(20), 20, 1)); + + Balances::mutate_account(&default_reward_account(), |f| f.free += 30).unwrap(); + + // everyone claims. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, + Event::PaidOut { member: 10, pool_id: 1, payout: 10 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 20 } + ] + ); + + // 20 unbonds to be equal to 10 (10 points each). + assert_ok!(Pools::unbond(Origin::signed(20), 20, 10)); + + // more rewards come in. + Balances::mutate_account(&default_reward_account(), |f| f.free += 100).unwrap(); + + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Unbonded { member: 20, pool_id: 1, balance: 10, points: 10 }, + Event::PaidOut { member: 10, pool_id: 1, payout: 50 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 50 }, + ] + ); + }); + } + + #[test] + fn unclaimed_reward_is_safe() { + ExtBuilder::default().build_and_execute(|| { + let ed = Balances::minimum_balance(); + + Balances::make_free_balance_be(&20, ed + 20); + assert_ok!(Pools::join(Origin::signed(20), 20, 1)); + Balances::make_free_balance_be(&30, ed + 20); + assert_ok!(Pools::join(Origin::signed(30), 10, 1)); + + // 10 gets 10, 20 gets 20, 30 gets 10 + Balances::mutate_account(&default_reward_account(), |f| f.free += 40).unwrap(); + + // some claim. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, + Event::Bonded { member: 30, pool_id: 1, bonded: 10, joined: true }, + Event::PaidOut { member: 10, pool_id: 1, payout: 10 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 20 } + ] + ); + + // 10 gets 20, 20 gets 40, 30 gets 20 + Balances::mutate_account(&default_reward_account(), |f| f.free += 80).unwrap(); + + // some claim. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::PaidOut { member: 10, pool_id: 1, payout: 20 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 40 } + ] + ); + + // 10 gets 20, 20 gets 40, 30 gets 20 + Balances::mutate_account(&default_reward_account(), |f| f.free += 80).unwrap(); + + // some claim. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::PaidOut { member: 10, pool_id: 1, payout: 20 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 40 } + ] + ); + + // now 30 claims all at once + assert_ok!(Pools::claim_payout(Origin::signed(30))); + + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 30, pool_id: 1, payout: 10 + 20 + 20 }] + ); + }); + } + + #[test] + fn bond_extra_and_delayed_claim() { + ExtBuilder::default().build_and_execute(|| { + let ed = Balances::minimum_balance(); + + Balances::make_free_balance_be(&20, ed + 200); + assert_ok!(Pools::join(Origin::signed(20), 20, 1)); + + // 10 gets 10, 20 gets 20, 30 gets 10 + Balances::mutate_account(&default_reward_account(), |f| f.free += 30).unwrap(); + + // some claim. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, + Event::PaidOut { member: 10, pool_id: 1, payout: 10 } + ] + ); + + // 20 has not claimed yet, more reward comes + Balances::mutate_account(&default_reward_account(), |f| f.free += 60).unwrap(); + + // and 20 bonds more -- they should not have more share of this reward. + assert_ok!(Pools::bond_extra(Origin::signed(20), BondExtra::FreeBalance(10))); + + // everyone claim. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + // 20 + 40, which means the extra amount they bonded did not impact us. + Event::PaidOut { member: 20, pool_id: 1, payout: 60 }, + Event::Bonded { member: 20, pool_id: 1, bonded: 10, joined: false }, + Event::PaidOut { member: 10, pool_id: 1, payout: 20 } + ] + ); + + // but in the next round of rewards, the extra10 they bonded has an impact. + Balances::mutate_account(&default_reward_account(), |f| f.free += 60).unwrap(); + + // everyone claim. + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::PaidOut { member: 10, pool_id: 1, payout: 15 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 45 } + ] + ); + }); + } + + #[test] + fn create_sets_recorded_data() { + ExtBuilder::default().build_and_execute(|| { + MaxPools::::set(None); + // pool 10 has already been created. + let (member_10, _, reward_pool_10) = Pools::get_member_with_pools(&10).unwrap(); + + assert_eq!(reward_pool_10.last_recorded_total_payouts, 0); + assert_eq!(reward_pool_10.total_rewards_claimed, 0); + assert_eq!(reward_pool_10.last_recorded_reward_counter, 0.into()); + + assert_eq!(member_10.last_recorded_reward_counter, 0.into()); + + // transfer some reward to pool 1. + Balances::mutate_account(&default_reward_account(), |f| f.free += 60).unwrap(); + + // create pool 2 + Balances::make_free_balance_be(&20, 100); + assert_ok!(Pools::create(Origin::signed(20), 10, 20, 20, 20)); + + // has no impact -- initial + let (member_20, _, reward_pool_20) = Pools::get_member_with_pools(&20).unwrap(); + + assert_eq!(reward_pool_20.last_recorded_total_payouts, 0); + assert_eq!(reward_pool_20.total_rewards_claimed, 0); + assert_eq!(reward_pool_20.last_recorded_reward_counter, 0.into()); + + assert_eq!(member_20.last_recorded_reward_counter, 0.into()); + + // pre-fund the reward account of pool id 3 with some funds. + Balances::make_free_balance_be(&Pools::create_reward_account(3), 10); + + // create pool 3 + Balances::make_free_balance_be(&30, 100); + assert_ok!(Pools::create(Origin::signed(30), 10, 30, 30, 30)); + + // reward counter is still the same. + let (member_30, _, reward_pool_30) = Pools::get_member_with_pools(&30).unwrap(); + assert_eq!( + Balances::free_balance(&Pools::create_reward_account(3)), + 10 + Balances::minimum_balance() + ); + + assert_eq!(reward_pool_30.last_recorded_total_payouts, 0); + assert_eq!(reward_pool_30.total_rewards_claimed, 0); + assert_eq!(reward_pool_30.last_recorded_reward_counter, 0.into()); + + assert_eq!(member_30.last_recorded_reward_counter, 0.into()); + + // and 30 can claim the reward now. + assert_ok!(Pools::claim_payout(Origin::signed(30))); + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Created { depositor: 20, pool_id: 2 }, + Event::Bonded { member: 20, pool_id: 2, bonded: 10, joined: true }, + Event::Created { depositor: 30, pool_id: 3 }, + Event::Bonded { member: 30, pool_id: 3, bonded: 10, joined: true }, + Event::PaidOut { member: 30, pool_id: 3, payout: 10 } + ] + ); + }) + } + + #[test] + fn join_updates_recorded_data() { + ExtBuilder::default().build_and_execute(|| { + MaxPoolMembers::::set(None); + MaxPoolMembersPerPool::::set(None); + let join = |x, y| { + Balances::make_free_balance_be(&x, y + Balances::minimum_balance()); + assert_ok!(Pools::join(Origin::signed(x), y, 1)); + }; + + { + let (member_10, _, reward_pool_10) = Pools::get_member_with_pools(&10).unwrap(); + + assert_eq!(reward_pool_10.last_recorded_total_payouts, 0); + assert_eq!(reward_pool_10.total_rewards_claimed, 0); + assert_eq!(reward_pool_10.last_recorded_reward_counter, 0.into()); + + assert_eq!(member_10.last_recorded_reward_counter, 0.into()); + } + + // someone joins without any rewards being issued. + { + join(20, 10); + let (member, _, reward_pool) = Pools::get_member_with_pools(&20).unwrap(); + // reward counter is 0 both before.. + assert_eq!(member.last_recorded_reward_counter, 0.into()); + assert_eq!(reward_pool.last_recorded_total_payouts, 0); + assert_eq!(reward_pool.last_recorded_reward_counter, 0.into()); + } + + // transfer some reward to pool 1. + Balances::mutate_account(&default_reward_account(), |f| f.free += 60).unwrap(); + + { + join(30, 10); + let (member, _, reward_pool) = Pools::get_member_with_pools(&30).unwrap(); + assert_eq!(reward_pool.last_recorded_total_payouts, 60); + // explanation: we have a total of 20 points so far (excluding the 10 that just got + // bonded), and 60 unclaimed rewards. each share is then roughly worth of 3 units of + // rewards, thus reward counter is 3. member's reward counter is the same + assert_eq!(member.last_recorded_reward_counter, 3.into()); + assert_eq!(reward_pool.last_recorded_reward_counter, 3.into()); + } + + // someone else joins + { + join(40, 10); + let (member, _, reward_pool) = Pools::get_member_with_pools(&40).unwrap(); + // reward counter does not change since no rewards have came in. + assert_eq!(member.last_recorded_reward_counter, 3.into()); + assert_eq!(reward_pool.last_recorded_reward_counter, 3.into()); + assert_eq!(reward_pool.last_recorded_total_payouts, 60); + } + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 30, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 40, pool_id: 1, bonded: 10, joined: true } + ] + ); + }) + } + + #[test] + fn bond_extra_updates_recorded_data() { + ExtBuilder::default().add_members(vec![(20, 20)]).build_and_execute(|| { + MaxPoolMembers::::set(None); + MaxPoolMembersPerPool::::set(None); + + // initial state of pool 1. + { + let (member_10, _, reward_pool_10) = Pools::get_member_with_pools(&10).unwrap(); + + assert_eq!(reward_pool_10.last_recorded_total_payouts, 0); + assert_eq!(reward_pool_10.total_rewards_claimed, 0); + assert_eq!(reward_pool_10.last_recorded_reward_counter, 0.into()); + + assert_eq!(member_10.last_recorded_reward_counter, 0.into()); + } + + Balances::make_free_balance_be(&10, 100); + Balances::make_free_balance_be(&20, 100); + + // 10 bonds extra without any rewards. + { + assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::FreeBalance(10))); + let (member, _, reward_pool) = Pools::get_member_with_pools(&10).unwrap(); + assert_eq!(member.last_recorded_reward_counter, 0.into()); + assert_eq!(reward_pool.last_recorded_total_payouts, 0); + assert_eq!(reward_pool.last_recorded_reward_counter, 0.into()); + } + + // 10 bonds extra again with some rewards. This reward should be split equally between + // 10 and 20, as they both have equal points now. + Balances::mutate_account(&default_reward_account(), |f| f.free += 30).unwrap(); + + { + assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::FreeBalance(10))); + let (member, _, reward_pool) = Pools::get_member_with_pools(&10).unwrap(); + // explanation: before bond_extra takes place, there is 40 points and 30 balance in + // the system, RewardCounter is therefore 7.5 + assert_eq!(member.last_recorded_reward_counter, RewardCounter::from_float(0.75)); assert_eq!( - pool_events_since_last_call(), - vec![Event::PaidOut { member: 50, pool_id: 1, payout: 50 }] + reward_pool.last_recorded_reward_counter, + RewardCounter::from_float(0.75) ); + assert_eq!(reward_pool.last_recorded_total_payouts, 30); + } - // We expect a payout of 50: (5,000 del virtual points / 7,5000 pool points) * 75 - // pool balance - assert_eq!(del_50, del(50, 200)); + // 20 bonds extra again, without further rewards. + { + assert_ok!(Pools::bond_extra(Origin::signed(20), BondExtra::FreeBalance(10))); + let (member, _, reward_pool) = Pools::get_member_with_pools(&20).unwrap(); + assert_eq!(member.last_recorded_reward_counter, RewardCounter::from_float(0.75)); assert_eq!( - reward_pool, - rew( - 25, - // old pool points + points from new earnings - del points. - // - // points from new earnings = new earnings(50) * bonded_pool.points(100) - // del points = member.points(50) * new_earnings_since_last_claim (100) - (2_500 + 50 * 100) - 50 * 100, - 200, - ) + reward_pool.last_recorded_reward_counter, + RewardCounter::from_float(0.75) ); - assert_eq!(Balances::free_balance(&50), 50 + 50); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 25); + assert_eq!(reward_pool.last_recorded_total_payouts, 30); + } + + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: false }, + Event::PaidOut { member: 10, pool_id: 1, payout: 15 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: false }, + Event::PaidOut { member: 20, pool_id: 1, payout: 15 }, + Event::Bonded { member: 20, pool_id: 1, bonded: 10, joined: false } + ] + ); + }) + } + + #[test] + fn unbond_updates_recorded_data() { + ExtBuilder::default() + .add_members(vec![(20, 20), (30, 20)]) + .build_and_execute(|| { + MaxPoolMembers::::set(None); + MaxPoolMembersPerPool::::set(None); + + // initial state of pool 1. + { + let (member, _, reward_pool) = Pools::get_member_with_pools(&10).unwrap(); + + assert_eq!(reward_pool.last_recorded_total_payouts, 0); + assert_eq!(reward_pool.total_rewards_claimed, 0); + assert_eq!(reward_pool.last_recorded_reward_counter, 0.into()); + + assert_eq!(member.last_recorded_reward_counter, 0.into()); + } + + // 20 unbonds without any rewards. + { + assert_ok!(Pools::unbond(Origin::signed(20), 20, 10)); + let (member, _, reward_pool) = Pools::get_member_with_pools(&20).unwrap(); + assert_eq!(member.last_recorded_reward_counter, 0.into()); + assert_eq!(reward_pool.last_recorded_total_payouts, 0); + assert_eq!(reward_pool.last_recorded_reward_counter, 0.into()); + } - // When - assert_ok!(Pools::do_reward_payout( - &10, - &mut del_10, - &mut bonded_pool, - &mut reward_pool - )); + // some rewards come in. + Balances::mutate_account(&default_reward_account(), |f| f.free += 30).unwrap(); - // Then - assert_eq!( - pool_events_since_last_call(), - vec![Event::PaidOut { member: 10, pool_id: 1, payout: 5 }] - ); + // and 30 also unbonds half. + { + assert_ok!(Pools::unbond(Origin::signed(30), 30, 10)); + let (member, _, reward_pool) = Pools::get_member_with_pools(&30).unwrap(); + // 30 reward in the system, and 40 points before this unbond to collect it, + // RewardCounter is 3/4. + assert_eq!( + member.last_recorded_reward_counter, + RewardCounter::from_float(0.75) + ); + assert_eq!(reward_pool.last_recorded_total_payouts, 30); + assert_eq!( + reward_pool.last_recorded_reward_counter, + RewardCounter::from_float(0.75) + ); + } - // We expect a payout of 5 - assert_eq!(del_10, del(10, 200)); - assert_eq!(reward_pool, rew(20, 2_500 - 10 * 50, 200)); - assert_eq!(Balances::free_balance(&10), 15 + 5); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 20); + // 30 unbonds again, not change this time. + { + assert_ok!(Pools::unbond(Origin::signed(30), 30, 5)); + let (member, _, reward_pool) = Pools::get_member_with_pools(&30).unwrap(); + assert_eq!( + member.last_recorded_reward_counter, + RewardCounter::from_float(0.75) + ); + assert_eq!(reward_pool.last_recorded_total_payouts, 30); + assert_eq!( + reward_pool.last_recorded_reward_counter, + RewardCounter::from_float(0.75) + ); + } - // Given del 40 hasn't claimed and the reward pool has just earned 400 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 400)); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 420); + // 20 unbonds again, not change this time, just collecting their reward. + { + assert_ok!(Pools::unbond(Origin::signed(20), 20, 5)); + let (member, _, reward_pool) = Pools::get_member_with_pools(&20).unwrap(); + assert_eq!( + member.last_recorded_reward_counter, + RewardCounter::from_float(0.75) + ); + assert_eq!(reward_pool.last_recorded_total_payouts, 30); + assert_eq!( + reward_pool.last_recorded_reward_counter, + RewardCounter::from_float(0.75) + ); + } - // When - assert_ok!(Pools::do_reward_payout( - &10, - &mut del_10, - &mut bonded_pool, - &mut reward_pool - )); + // trigger 10's reward as well to see all of the payouts. + assert_ok!(Pools::claim_payout(Origin::signed(10))); - // Then assert_eq!( pool_events_since_last_call(), - vec![Event::PaidOut { member: 10, pool_id: 1, payout: 40 }] + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, + Event::Bonded { member: 30, pool_id: 1, bonded: 20, joined: true }, + Event::Unbonded { member: 20, pool_id: 1, balance: 10, points: 10 }, + Event::PaidOut { member: 30, pool_id: 1, payout: 15 }, + Event::Unbonded { member: 30, pool_id: 1, balance: 10, points: 10 }, + Event::Unbonded { member: 30, pool_id: 1, balance: 5, points: 5 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 7 }, + Event::Unbonded { member: 20, pool_id: 1, balance: 5, points: 5 }, + Event::PaidOut { member: 10, pool_id: 1, payout: 7 } + ] ); + }) + } - // We expect a payout of 40 - assert_eq!(del_10, del(10, 600)); - assert_eq!( - reward_pool, - rew( - 380, - // old pool points + points from new earnings - del points - // - // points from new earnings = new earnings(400) * bonded_pool.points(100) - // del points = member.points(10) * new_earnings_since_last_claim(400) - (2_000 + 400 * 100) - 10 * 400, - 600 - ) - ); - assert_eq!(Balances::free_balance(&10), 20 + 40); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 380); + #[test] + fn rewards_are_rounded_down_depositor_collects_them() { + ExtBuilder::default().add_members(vec![(20, 20)]).build_and_execute(|| { + // initial balance of 10. + assert_eq!(Balances::free_balance(&10), 5); + assert_eq!( + Balances::free_balance(&default_reward_account()), + Balances::minimum_balance() + ); - // Given del 40 + del 50 haven't claimed and the reward pool has earned 20 - assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += 20)); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 400); + // some rewards come in. + Balances::mutate_account(&default_reward_account(), |f| f.free += 40).unwrap(); - // When - assert_ok!(Pools::do_reward_payout( - &10, - &mut del_10, - &mut bonded_pool, - &mut reward_pool - )); + // everyone claims + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); - // Then - assert_eq!( - pool_events_since_last_call(), - vec![Event::PaidOut { member: 10, pool_id: 1, payout: 2 }] - ); + // some dust (1) remains in the reward account. + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, + Event::PaidOut { member: 10, pool_id: 1, payout: 13 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 26 } + ] + ); - // Expect a payout of 2: (200 del virtual points / 38,000 pool points) * 400 pool - // balance - assert_eq!(del_10, del(10, 620)); - assert_eq!(reward_pool, rew(398, (38_000 + 20 * 100) - 10 * 20, 620)); - assert_eq!(Balances::free_balance(&10), 60 + 2); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 398); + // start dismantling the pool. + assert_ok!(Pools::set_state(Origin::signed(902), 1, PoolState::Destroying)); + assert_ok!(fully_unbond_permissioned(20)); - // When - assert_ok!(Pools::do_reward_payout( - &40, - &mut del_40, - &mut bonded_pool, - &mut reward_pool - )); + CurrentEra::set(3); + assert_ok!(Pools::withdraw_unbonded(Origin::signed(20), 20, 0)); + assert_ok!(fully_unbond_permissioned(10)); - // Then - assert_eq!( - pool_events_since_last_call(), - vec![Event::PaidOut { member: 40, pool_id: 1, payout: 188 }] - ); + CurrentEra::set(6); + assert_ok!(Pools::withdraw_unbonded(Origin::signed(10), 10, 0)); - // Expect a payout of 188: (18,800 del virtual points / 39,800 pool points) * 399 - // pool balance - assert_eq!(del_40, del(40, 620)); - assert_eq!(reward_pool, rew(210, 39_800 - 40 * 470, 620)); - assert_eq!(Balances::free_balance(&40), 60 + 188); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 210); + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::StateChanged { pool_id: 1, new_state: PoolState::Destroying }, + Event::Unbonded { member: 20, pool_id: 1, balance: 20, points: 20 }, + Event::Withdrawn { member: 20, pool_id: 1, balance: 20, points: 20 }, + Event::MemberRemoved { pool_id: 1, member: 20 }, + Event::Unbonded { member: 10, pool_id: 1, balance: 10, points: 10 }, + Event::Withdrawn { member: 10, pool_id: 1, balance: 10, points: 10 }, + Event::MemberRemoved { pool_id: 1, member: 10 }, + Event::Destroyed { pool_id: 1 } + ] + ); - // When - assert_ok!(Pools::do_reward_payout( - &50, - &mut del_50, - &mut bonded_pool, - &mut reward_pool - )); + // original ed + ed put into reward account + reward + bond + dust. + assert_eq!(Balances::free_balance(&10), 5 + 5 + 13 + 10 + 1); + }) + } + + #[test] + fn claim_payout_large_numbers() { + let unit = 10u128.pow(12); // akin to KSM + ExistentialDeposit::set(unit); + StakingMinBond::set(unit * 1000); + + ExtBuilder::default() + .max_members(Some(4)) + .max_members_per_pool(Some(4)) + .add_members(vec![(20, 1500 * unit), (21, 2500 * unit), (22, 5000 * unit)]) + .build_and_execute(|| { + // some rewards come in. + assert_eq!(Balances::free_balance(&default_reward_account()), unit); + Balances::mutate_account(&default_reward_account(), |f| f.free += unit / 1000) + .unwrap(); + + // everyone claims + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + assert_ok!(Pools::claim_payout(Origin::signed(21))); + assert_ok!(Pools::claim_payout(Origin::signed(22))); - // Then assert_eq!( pool_events_since_last_call(), - vec![Event::PaidOut { member: 50, pool_id: 1, payout: 210 }] + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { + member: 10, + pool_id: 1, + bonded: 1000000000000000, + joined: true + }, + Event::Bonded { + member: 20, + pool_id: 1, + bonded: 1500000000000000, + joined: true + }, + Event::Bonded { + member: 21, + pool_id: 1, + bonded: 2500000000000000, + joined: true + }, + Event::Bonded { + member: 22, + pool_id: 1, + bonded: 5000000000000000, + joined: true + }, + Event::PaidOut { member: 10, pool_id: 1, payout: 100000000 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 150000000 }, + Event::PaidOut { member: 21, pool_id: 1, payout: 250000000 }, + Event::PaidOut { member: 22, pool_id: 1, payout: 500000000 } + ] ); - - // Expect payout of 210: (21,000 / 21,000) * 210 - assert_eq!(del_50, del(50, 620)); - assert_eq!(reward_pool, rew(0, 21_000 - 50 * 420, 620)); - assert_eq!(Balances::free_balance(&50), 100 + 210); - assert_eq!(Balances::free_balance(&default_reward_account()), ed + 0); - }); + }) } } @@ -2461,7 +2846,8 @@ mod withdraw_unbonded { assert_eq!( SubPoolsStorage::::get(&1).unwrap().with_era, - unbonding_pools_with_era! { 3 => UnbondPool { points: 550 / 2 + 40 / 2, balance: 550 / 2 + 40 / 2 }} + unbonding_pools_with_era! { 3 => UnbondPool { points: 550 / 2 + 40 / 2, balance: 550 / 2 + 40 / 2 + }} ); assert_eq!( @@ -3089,11 +3475,7 @@ mod create { ); assert_eq!( RewardPools::::get(2).unwrap(), - RewardPool { - balance: Zero::zero(), - points: U256::zero(), - total_earnings: Zero::zero(), - } + RewardPool { ..Default::default() } ); assert_eq!( @@ -3688,3 +4070,310 @@ mod update_roles { }) } } + +mod reward_counter_precision { + use sp_runtime::FixedU128; + + use super::*; + + const DOT: Balance = 10u128.pow(10u32); + const POLKADOT_TOTAL_ISSUANCE_GENESIS: Balance = DOT * 10u128.pow(9u32); + + const fn inflation(years: u128) -> u128 { + let mut i = 0; + let mut start = POLKADOT_TOTAL_ISSUANCE_GENESIS; + while i < years { + start = start + start / 10; + i += 1 + } + start + } + + fn default_pool_reward_counter() -> FixedU128 { + RewardPools::::get(1) + .unwrap() + .current_reward_counter(1, BondedPools::::get(1).unwrap().points) + .unwrap() + } + + fn pending_rewards(of: AccountId) -> Option> { + let member = PoolMembers::::get(of).unwrap(); + assert_eq!(member.pool_id, 1); + let rc = default_pool_reward_counter(); + member.pending_rewards(rc).ok() + } + + #[test] + fn smallest_claimable_reward() { + // create a pool that has all of the polkadot issuance in 50 years. + let pool_bond = inflation(50); + ExtBuilder::default().ed(DOT).min_bond(pool_bond).build_and_execute(|| { + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { + member: 10, + pool_id: 1, + bonded: 1173908528796953165005, + joined: true, + } + ] + ); + + // the smallest reward that this pool can handle is + let expected_smallest_reward = inflation(50) / 10u128.pow(18); + + // tad bit less. cannot be paid out. + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += + expected_smallest_reward - 1)); + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_eq!(pool_events_since_last_call(), vec![]); + // revert it. + + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free -= + expected_smallest_reward - 1)); + + // tad bit more. can be claimed. + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += + expected_smallest_reward + 1)); + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 10, pool_id: 1, payout: 1173 }] + ); + }) + } + + #[test] + fn reward_counter_calc_wont_fail_in_normal_polkadot_future() { + // create a pool that has roughly half of the polkadot issuance in 10 years. + let pool_bond = inflation(10) / 2; + ExtBuilder::default().ed(DOT).min_bond(pool_bond).build_and_execute(|| { + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { + member: 10, + pool_id: 1, + bonded: 12_968_712_300_500_000_000, + joined: true, + } + ] + ); + + // in 10 years, the total claimed rewards are large values as well. assuming that a pool + // is earning all of the inflation per year (which is really unrealistic, but worse + // case), that will be: + let pool_total_earnings_10_years = inflation(10) - POLKADOT_TOTAL_ISSUANCE_GENESIS; + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += + pool_total_earnings_10_years)); + + // some whale now joins with the other half ot the total issuance. This will bloat all + // the calculation regarding current reward counter. + Balances::make_free_balance_be(&20, pool_bond * 2); + assert_ok!(Pools::join(Origin::signed(20), pool_bond, 1)); + + assert_eq!( + pool_events_since_last_call(), + vec![Event::Bonded { + member: 20, + pool_id: 1, + bonded: 12_968_712_300_500_000_000, + joined: true + }] + ); + + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 10, pool_id: 1, payout: 15937424600999999996 }] + ); + + // now let a small member join with 10 DOTs. + Balances::make_free_balance_be(&30, 20 * DOT); + assert_ok!(Pools::join(Origin::signed(30), 10 * DOT, 1)); + + // and give a reasonably small reward to the pool. + assert_ok!(Balances::mutate_account(&default_reward_account(), |a| a.free += DOT)); + + assert_ok!(Pools::claim_payout(Origin::signed(30))); + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Bonded { member: 30, pool_id: 1, bonded: 100000000000, joined: true }, + // quite small, but working fine. + Event::PaidOut { member: 30, pool_id: 1, payout: 38 } + ] + ); + }) + } + + #[test] + fn reward_counter_update_can_fail_if_pool_is_highly_slashed() { + // create a pool that has roughly half of the polkadot issuance in 10 years. + let pool_bond = inflation(10) / 2; + ExtBuilder::default().ed(DOT).min_bond(pool_bond).build_and_execute(|| { + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { + member: 10, + pool_id: 1, + bonded: 12_968_712_300_500_000_000, + joined: true, + } + ] + ); + + // slash this pool by 99% of that. + StakingMock::set_bonded_balance(default_bonded_account(), DOT + pool_bond / 100); + + // some whale now joins with the other half ot the total issuance. This will trigger an + // overflow. This test is actually a bit too lenient because all the reward counters are + // set to zero. In other tests that we want to assert a scenario won't fail, we should + // also set the reward counters to some large value. + Balances::make_free_balance_be(&20, pool_bond * 2); + assert_err!(Pools::join(Origin::signed(20), pool_bond, 1), Error::::OverflowRisk); + }) + } + + #[test] + fn if_small_member_waits_long_enough_they_will_earn_rewards() { + // create a pool that has a quarter of the current polkadot issuance + ExtBuilder::default() + .ed(DOT) + .min_bond(POLKADOT_TOTAL_ISSUANCE_GENESIS / 4) + .build_and_execute(|| { + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { + member: 10, + pool_id: 1, + bonded: 2500000000000000000, + joined: true, + } + ] + ); + + // and have a tiny fish join the pool as well.. + Balances::make_free_balance_be(&20, 20 * DOT); + assert_ok!(Pools::join(Origin::signed(20), 10 * DOT, 1)); + + // earn some small rewards + assert_ok!( + Balances::mutate_account(&default_reward_account(), |a| a.free += DOT / 1000) + ); + + // no point in claiming for 20 (nonetheless, it should be harmless) + assert!(pending_rewards(20).unwrap().is_zero()); + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Bonded { + member: 20, + pool_id: 1, + bonded: 100000000000, + joined: true + }, + Event::PaidOut { member: 10, pool_id: 1, payout: 9999997 } + ] + ); + + // earn some small more, still nothing can be claimed for 20, but 10 claims their + // share. + assert_ok!( + Balances::mutate_account(&default_reward_account(), |a| a.free += DOT / 1000) + ); + assert!(pending_rewards(20).unwrap().is_zero()); + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_eq!( + pool_events_since_last_call(), + vec![Event::PaidOut { member: 10, pool_id: 1, payout: 10000000 }] + ); + + // earn some more rewards, this time 20 can also claim. + assert_ok!( + Balances::mutate_account(&default_reward_account(), |a| a.free += DOT / 1000) + ); + assert_eq!(pending_rewards(20).unwrap(), 1); + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::PaidOut { member: 10, pool_id: 1, payout: 10000000 }, + Event::PaidOut { member: 20, pool_id: 1, payout: 1 } + ] + ); + }); + } + + #[test] + fn zero_reward_claim_does_not_update_reward_counter() { + // create a pool that has a quarter of the current polkadot issuance + ExtBuilder::default() + .ed(DOT) + .min_bond(POLKADOT_TOTAL_ISSUANCE_GENESIS / 4) + .build_and_execute(|| { + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Created { depositor: 10, pool_id: 1 }, + Event::Bonded { + member: 10, + pool_id: 1, + bonded: 2500000000000000000, + joined: true, + } + ] + ); + + // and have a tiny fish join the pool as well.. + Balances::make_free_balance_be(&20, 20 * DOT); + assert_ok!(Pools::join(Origin::signed(20), 10 * DOT, 1)); + + // earn some small rewards + assert_ok!( + Balances::mutate_account(&default_reward_account(), |a| a.free += DOT / 1000) + ); + + // if 20 claims now, their reward counter should stay the same, so that they have a + // chance of claiming this if they let it accumulate. Also see + // `if_small_member_waits_long_enough_they_will_earn_rewards` + assert_ok!(Pools::claim_payout(Origin::signed(10))); + assert_ok!(Pools::claim_payout(Origin::signed(20))); + assert_eq!( + pool_events_since_last_call(), + vec![ + Event::Bonded { + member: 20, + pool_id: 1, + bonded: 100000000000, + joined: true + }, + Event::PaidOut { member: 10, pool_id: 1, payout: 9999997 } + ] + ); + + let current_reward_counter = default_pool_reward_counter(); + // has been updated, because they actually claimed something. + assert_eq!( + PoolMembers::::get(10).unwrap().last_recorded_reward_counter, + current_reward_counter + ); + // has not be updated, even though the claim transaction went through okay. + assert_eq!( + PoolMembers::::get(20).unwrap().last_recorded_reward_counter, + Default::default() + ); + }); + } +} diff --git a/frame/nomination-pools/src/weights.rs b/frame/nomination-pools/src/weights.rs index 8e3facfc5ec26..a9003ffd3fb4c 100644 --- a/frame/nomination-pools/src/weights.rs +++ b/frame/nomination-pools/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_nomination_pools //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2022-06-10, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2022-06-15, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! HOSTNAME: `bm3`, CPU: `Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 @@ -70,7 +70,7 @@ impl WeightInfo for SubstrateWeight { // Storage: NominationPools PoolMembers (r:1 w:1) // Storage: NominationPools BondedPools (r:1 w:1) // Storage: Staking Ledger (r:1 w:1) - // Storage: NominationPools RewardPools (r:1 w:0) + // Storage: NominationPools RewardPools (r:1 w:1) // Storage: System Account (r:2 w:1) // Storage: NominationPools MaxPoolMembersPerPool (r:1 w:0) // Storage: NominationPools MaxPoolMembers (r:1 w:0) @@ -80,22 +80,22 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn join() -> Weight { - (124_508_000 as Weight) + (123_947_000 as Weight) .saturating_add(T::DbWeight::get().reads(17 as Weight)) - .saturating_add(T::DbWeight::get().writes(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(12 as Weight)) } // Storage: NominationPools PoolMembers (r:1 w:1) // Storage: NominationPools BondedPools (r:1 w:1) // Storage: NominationPools RewardPools (r:1 w:1) - // Storage: System Account (r:2 w:2) + // Storage: System Account (r:3 w:2) // Storage: Staking Ledger (r:1 w:1) // Storage: Staking Bonded (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn bond_extra_transfer() -> Weight { - (115_185_000 as Weight) - .saturating_add(T::DbWeight::get().reads(13 as Weight)) + (118_236_000 as Weight) + .saturating_add(T::DbWeight::get().reads(14 as Weight)) .saturating_add(T::DbWeight::get().writes(12 as Weight)) } // Storage: NominationPools PoolMembers (r:1 w:1) @@ -108,7 +108,7 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn bond_extra_reward() -> Weight { - (132_723_000 as Weight) + (132_475_000 as Weight) .saturating_add(T::DbWeight::get().reads(14 as Weight)) .saturating_add(T::DbWeight::get().writes(13 as Weight)) } @@ -117,7 +117,7 @@ impl WeightInfo for SubstrateWeight { // Storage: NominationPools RewardPools (r:1 w:1) // Storage: System Account (r:1 w:1) fn claim_payout() -> Weight { - (52_498_000 as Weight) + (50_299_000 as Weight) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } @@ -136,7 +136,7 @@ impl WeightInfo for SubstrateWeight { // Storage: NominationPools SubPoolsStorage (r:1 w:1) // Storage: NominationPools CounterForSubPoolsStorage (r:1 w:1) fn unbond() -> Weight { - (121_645_000 as Weight) + (121_254_000 as Weight) .saturating_add(T::DbWeight::get().reads(18 as Weight)) .saturating_add(T::DbWeight::get().writes(13 as Weight)) } @@ -146,9 +146,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) /// The range of component `s` is `[0, 100]`. fn pool_withdraw_unbonded(s: u32, ) -> Weight { - (43_320_000 as Weight) + (41_928_000 as Weight) // Standard Error: 0 - .saturating_add((49_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((52_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -162,9 +162,9 @@ impl WeightInfo for SubstrateWeight { // Storage: NominationPools CounterForPoolMembers (r:1 w:1) /// The range of component `s` is `[0, 100]`. fn withdraw_unbonded_update(s: u32, ) -> Weight { - (83_195_000 as Weight) - // Standard Error: 5_000 - .saturating_add((57_000 as Weight).saturating_mul(s as Weight)) + (81_611_000 as Weight) + // Standard Error: 1_000 + .saturating_add((56_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(7 as Weight)) } @@ -189,7 +189,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:0 w:1) /// The range of component `s` is `[0, 100]`. fn withdraw_unbonded_kill(_s: u32, ) -> Weight { - (143_495_000 as Weight) + (139_849_000 as Weight) .saturating_add(T::DbWeight::get().reads(19 as Weight)) .saturating_add(T::DbWeight::get().writes(16 as Weight)) } @@ -216,7 +216,7 @@ impl WeightInfo for SubstrateWeight { // Storage: NominationPools BondedPools (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn create() -> Weight { - (127_998_000 as Weight) + (126_246_000 as Weight) .saturating_add(T::DbWeight::get().reads(22 as Weight)) .saturating_add(T::DbWeight::get().writes(15 as Weight)) } @@ -234,9 +234,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking CounterForNominators (r:1 w:1) /// The range of component `n` is `[1, 16]`. fn nominate(n: u32, ) -> Weight { - (49_929_000 as Weight) - // Standard Error: 16_000 - .saturating_add((2_319_000 as Weight).saturating_mul(n as Weight)) + (48_829_000 as Weight) + // Standard Error: 10_000 + .saturating_add((2_204_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(5 as Weight)) @@ -244,7 +244,7 @@ impl WeightInfo for SubstrateWeight { // Storage: NominationPools BondedPools (r:1 w:1) // Storage: Staking Ledger (r:1 w:0) fn set_state() -> Weight { - (27_399_000 as Weight) + (26_761_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -253,7 +253,7 @@ impl WeightInfo for SubstrateWeight { // Storage: NominationPools CounterForMetadata (r:1 w:1) /// The range of component `n` is `[1, 256]`. fn set_metadata(n: u32, ) -> Weight { - (14_813_000 as Weight) + (14_519_000 as Weight) // Standard Error: 0 .saturating_add((1_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) @@ -265,12 +265,12 @@ impl WeightInfo for SubstrateWeight { // Storage: NominationPools MinCreateBond (r:0 w:1) // Storage: NominationPools MaxPools (r:0 w:1) fn set_configs() -> Weight { - (6_115_000 as Weight) + (6_173_000 as Weight) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } // Storage: NominationPools BondedPools (r:1 w:1) fn update_roles() -> Weight { - (22_546_000 as Weight) + (22_261_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -283,7 +283,7 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill() -> Weight { - (48_243_000 as Weight) + (47_959_000 as Weight) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } @@ -295,7 +295,7 @@ impl WeightInfo for () { // Storage: NominationPools PoolMembers (r:1 w:1) // Storage: NominationPools BondedPools (r:1 w:1) // Storage: Staking Ledger (r:1 w:1) - // Storage: NominationPools RewardPools (r:1 w:0) + // Storage: NominationPools RewardPools (r:1 w:1) // Storage: System Account (r:2 w:1) // Storage: NominationPools MaxPoolMembersPerPool (r:1 w:0) // Storage: NominationPools MaxPoolMembers (r:1 w:0) @@ -305,22 +305,22 @@ impl WeightInfo for () { // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn join() -> Weight { - (124_508_000 as Weight) + (123_947_000 as Weight) .saturating_add(RocksDbWeight::get().reads(17 as Weight)) - .saturating_add(RocksDbWeight::get().writes(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(12 as Weight)) } // Storage: NominationPools PoolMembers (r:1 w:1) // Storage: NominationPools BondedPools (r:1 w:1) // Storage: NominationPools RewardPools (r:1 w:1) - // Storage: System Account (r:2 w:2) + // Storage: System Account (r:3 w:2) // Storage: Staking Ledger (r:1 w:1) // Storage: Staking Bonded (r:1 w:0) // Storage: Balances Locks (r:1 w:1) // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn bond_extra_transfer() -> Weight { - (115_185_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(13 as Weight)) + (118_236_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(14 as Weight)) .saturating_add(RocksDbWeight::get().writes(12 as Weight)) } // Storage: NominationPools PoolMembers (r:1 w:1) @@ -333,7 +333,7 @@ impl WeightInfo for () { // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn bond_extra_reward() -> Weight { - (132_723_000 as Weight) + (132_475_000 as Weight) .saturating_add(RocksDbWeight::get().reads(14 as Weight)) .saturating_add(RocksDbWeight::get().writes(13 as Weight)) } @@ -342,7 +342,7 @@ impl WeightInfo for () { // Storage: NominationPools RewardPools (r:1 w:1) // Storage: System Account (r:1 w:1) fn claim_payout() -> Weight { - (52_498_000 as Weight) + (50_299_000 as Weight) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } @@ -361,7 +361,7 @@ impl WeightInfo for () { // Storage: NominationPools SubPoolsStorage (r:1 w:1) // Storage: NominationPools CounterForSubPoolsStorage (r:1 w:1) fn unbond() -> Weight { - (121_645_000 as Weight) + (121_254_000 as Weight) .saturating_add(RocksDbWeight::get().reads(18 as Weight)) .saturating_add(RocksDbWeight::get().writes(13 as Weight)) } @@ -371,9 +371,9 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) /// The range of component `s` is `[0, 100]`. fn pool_withdraw_unbonded(s: u32, ) -> Weight { - (43_320_000 as Weight) + (41_928_000 as Weight) // Standard Error: 0 - .saturating_add((49_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((52_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -387,9 +387,9 @@ impl WeightInfo for () { // Storage: NominationPools CounterForPoolMembers (r:1 w:1) /// The range of component `s` is `[0, 100]`. fn withdraw_unbonded_update(s: u32, ) -> Weight { - (83_195_000 as Weight) - // Standard Error: 5_000 - .saturating_add((57_000 as Weight).saturating_mul(s as Weight)) + (81_611_000 as Weight) + // Standard Error: 1_000 + .saturating_add((56_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(7 as Weight)) } @@ -414,7 +414,7 @@ impl WeightInfo for () { // Storage: Staking Payee (r:0 w:1) /// The range of component `s` is `[0, 100]`. fn withdraw_unbonded_kill(_s: u32, ) -> Weight { - (143_495_000 as Weight) + (139_849_000 as Weight) .saturating_add(RocksDbWeight::get().reads(19 as Weight)) .saturating_add(RocksDbWeight::get().writes(16 as Weight)) } @@ -441,7 +441,7 @@ impl WeightInfo for () { // Storage: NominationPools BondedPools (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn create() -> Weight { - (127_998_000 as Weight) + (126_246_000 as Weight) .saturating_add(RocksDbWeight::get().reads(22 as Weight)) .saturating_add(RocksDbWeight::get().writes(15 as Weight)) } @@ -459,9 +459,9 @@ impl WeightInfo for () { // Storage: Staking CounterForNominators (r:1 w:1) /// The range of component `n` is `[1, 16]`. fn nominate(n: u32, ) -> Weight { - (49_929_000 as Weight) - // Standard Error: 16_000 - .saturating_add((2_319_000 as Weight).saturating_mul(n as Weight)) + (48_829_000 as Weight) + // Standard Error: 10_000 + .saturating_add((2_204_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) @@ -469,7 +469,7 @@ impl WeightInfo for () { // Storage: NominationPools BondedPools (r:1 w:1) // Storage: Staking Ledger (r:1 w:0) fn set_state() -> Weight { - (27_399_000 as Weight) + (26_761_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -478,7 +478,7 @@ impl WeightInfo for () { // Storage: NominationPools CounterForMetadata (r:1 w:1) /// The range of component `n` is `[1, 256]`. fn set_metadata(n: u32, ) -> Weight { - (14_813_000 as Weight) + (14_519_000 as Weight) // Standard Error: 0 .saturating_add((1_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) @@ -490,12 +490,12 @@ impl WeightInfo for () { // Storage: NominationPools MinCreateBond (r:0 w:1) // Storage: NominationPools MaxPools (r:0 w:1) fn set_configs() -> Weight { - (6_115_000 as Weight) + (6_173_000 as Weight) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } // Storage: NominationPools BondedPools (r:1 w:1) fn update_roles() -> Weight { - (22_546_000 as Weight) + (22_261_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -508,7 +508,7 @@ impl WeightInfo for () { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill() -> Weight { - (48_243_000 as Weight) + (47_959_000 as Weight) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } diff --git a/frame/nomination-pools/test-staking/src/lib.rs b/frame/nomination-pools/test-staking/src/lib.rs index be7752ce113b4..3b9350f25040b 100644 --- a/frame/nomination-pools/test-staking/src/lib.rs +++ b/frame/nomination-pools/test-staking/src/lib.rs @@ -26,6 +26,7 @@ use pallet_nomination_pools::{ PoolState, }; use pallet_staking::{CurrentEra, Event as StakingEvent, Payee, RewardDestination}; +use sp_runtime::traits::Zero; #[test] fn pool_lifecycle_e2e() { @@ -296,7 +297,7 @@ fn pool_slash_e2e() { PoolMember { pool_id: 1, points: 0, - reward_pool_total_earnings: 0, + last_recorded_reward_counter: Zero::zero(), // the 10 points unlocked just now correspond to 5 points in the unbond pool. unbonding_eras: bounded_btree_map!(5 => 10, 6 => 5) } @@ -351,7 +352,7 @@ fn pool_slash_e2e() { PoolMember { pool_id: 1, points: 0, - reward_pool_total_earnings: 0, + last_recorded_reward_counter: Zero::zero(), unbonding_eras: bounded_btree_map!(4 => 10, 5 => 10, 9 => 10) } ); diff --git a/frame/nomination-pools/test-staking/src/mock.rs b/frame/nomination-pools/test-staking/src/mock.rs index d13d8aa341c12..852b2c319f376 100644 --- a/frame/nomination-pools/test-staking/src/mock.rs +++ b/frame/nomination-pools/test-staking/src/mock.rs @@ -16,8 +16,17 @@ // limitations under the License. use frame_election_provider_support::VoteWeight; -use frame_support::{assert_ok, pallet_prelude::*, parameter_types, traits::ConstU64, PalletId}; -use sp_runtime::traits::{Convert, IdentityLookup}; +use frame_support::{ + assert_ok, + pallet_prelude::*, + parameter_types, + traits::{ConstU64, ConstU8}, + PalletId, +}; +use sp_runtime::{ + traits::{Convert, IdentityLookup}, + FixedU128, +}; type AccountId = u128; type AccountIndex = u32; @@ -159,13 +168,15 @@ impl pallet_nomination_pools::Config for Runtime { type Event = Event; type WeightInfo = (); type Currency = Balances; + type CurrencyBalance = Balance; + type RewardCounter = FixedU128; type BalanceToU256 = BalanceToU256; type U256ToBalance = U256ToBalance; type StakingInterface = Staking; type PostUnbondingPoolsWindow = PostUnbondingPoolsWindow; type MaxMetadataLen = ConstU32<256>; type MaxUnbonding = ConstU32<8>; - type MinPointsToBalance = ConstU32<10>; + type MaxPointsToBalance = ConstU8<10>; type PalletId = PoolsPalletId; } diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index c09a33cf3f16a..f77f92c625c9d 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -472,9 +472,10 @@ pub enum State { #[clap(short, long)] snapshot_path: Option, - /// The pallets to scrape. If empty, entire chain state will be scraped. + /// A pallet to scrape. Can be provided multiple times. If empty, entire chain state will + /// be scraped. #[clap(short, long, multiple_values = true)] - pallets: Vec, + pallet: Vec, /// Fetch the child-keys as well. /// @@ -498,7 +499,7 @@ impl State { Builder::::new().mode(Mode::Offline(OfflineConfig { state_snapshot: SnapshotConfig::new(snapshot_path), })), - State::Live { snapshot_path, pallets, uri, at, child_tree } => { + State::Live { snapshot_path, pallet, uri, at, child_tree } => { let at = match at { Some(at_str) => Some(hash_of::(at_str)?), None => None, @@ -507,7 +508,7 @@ impl State { .mode(Mode::Online(OnlineConfig { transport: uri.to_owned().into(), state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), - pallets: pallets.clone(), + pallets: pallet.clone(), scrape_children: true, at, })) From 49b06901eb65f2c61ff0934d66987fd955d5b8f5 Mon Sep 17 00:00:00 2001 From: Emre Surmeli Date: Thu, 14 Jul 2022 04:21:21 -0400 Subject: [PATCH 09/46] Fix typo (#11832) --- frame/election-provider-multi-phase/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 2f1f6463df719..e1d3cb8ed5dee 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -51,7 +51,7 @@ //! In the signed phase, solutions (of type [`RawSolution`]) are submitted and queued on chain. A //! deposit is reserved, based on the size of the solution, for the cost of keeping this solution //! on-chain for a number of blocks, and the potential weight of the solution upon being checked. A -//! maximum of `pallet::Config::MaxSignedSubmissions` solutions are stored. The queue is always +//! maximum of `pallet::Config::SignedMaxSubmissions` solutions are stored. The queue is always //! sorted based on score (worse to best). //! //! Upon arrival of a new solution: From 1f0253c4c5fe70213fc2a53e78b5e9648e71d2c7 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Thu, 14 Jul 2022 10:57:59 +0100 Subject: [PATCH 10/46] Fix nomination pools unbonding logic (#11746) * make pool roles optional * undo lock file changes? * add migration * add the ability for pools to chill themselves * boilerplate of tests * somewhat stable, but I think I found another bug as well * Fix it all * Add more more sophisticated test + capture one more bug. * Update frame/staking/src/lib.rs * reduce the diff a little bit * add some test for the slashing bug * cleanup * fix lock file? * Fix * fmt * Update frame/nomination-pools/src/lib.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/lib.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/lib.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/mock.rs Co-authored-by: Oliver Tale-Yazdi * Fix build * fix some fishy tests.. * add one last integrity check for MinCreateBond * remove bad assertion -- needs to be dealt with later * nits * fix tests and add benchmarks for chill * remove stuff * fix benchmarks * cargo run --quiet --profile=production --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark pallet --chain=dev --steps=50 --repeat=20 --pallet=pallet_nomination_pools --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/nomination-pools/src/weights.rs --template=./.maintain/frame-weight-template.hbs * remove defensive * first working version * bring back all tests * ALL new tests work now * cleanup * make sure benchmarks and all work * cargo run --quiet --profile=production --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark pallet --chain=dev --steps=50 --repeat=20 --pallet=pallet_nomination_pools --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/nomination-pools/src/weights.rs --template=./.maintain/frame-weight-template.hbs * round of self-review, make arithmetic safe * fix warn * add migration code * Fix doc * add precision notes * make arithmetic fallible * fix node runtime * a lot of precision tests and notes and stuff * document MaxPOintsToBalance better * :round of self-review * fmt * fix some comments * new logic, some broken tests * Check if after unbonding remaining balance is more or equal to MinJoinBond and is not zero * incorporate nikos' work * make it work again * merge * Fix all tests * fix all tests * some updates * Add tests * remove erroneoysly placed comment * Try to make lint pass * Try to make lint pass * revamp the tests for unbond * fix docs * Fix proportional slashing logic * Update frame/nomination-pools/src/tests.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/tests.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/lib.rs Co-authored-by: Oliver Tale-Yazdi * track poinst in migration * fix * fmt * fix migration * remove event read * Apply suggestions from code review * Update frame/nomination-pools/src/lib.rs Co-authored-by: Shawn Tabrizi * remove log * Update frame/staking/src/lib.rs Co-authored-by: Shawn Tabrizi * Update frame/nomination-pools/src/lib.rs Co-authored-by: Shawn Tabrizi * Update frame/nomination-pools/src/lib.rs Co-authored-by: Shawn Tabrizi * update * fmt * fmt * add one last test * fmrt * Update frame/nomination-pools/src/lib.rs Co-authored-by: Oliver Tale-Yazdi * Update frame/nomination-pools/src/tests.rs Co-authored-by: Oliver Tale-Yazdi Co-authored-by: Oliver Tale-Yazdi Co-authored-by: Shawn Tabrizi Co-authored-by: Parity Bot Co-authored-by: wirednkod --- frame/nomination-pools/src/lib.rs | 135 +++-- frame/nomination-pools/src/mock.rs | 16 +- frame/nomination-pools/src/tests.rs | 480 ++++++++++++++---- .../nomination-pools/test-staking/src/lib.rs | 6 +- 4 files changed, 464 insertions(+), 173 deletions(-) diff --git a/frame/nomination-pools/src/lib.rs b/frame/nomination-pools/src/lib.rs index 53e1c48a5e39e..514267e8bf4af 100644 --- a/frame/nomination-pools/src/lib.rs +++ b/frame/nomination-pools/src/lib.rs @@ -469,7 +469,7 @@ impl PoolMember { self.points = new_points; Ok(()) } else { - Err(Error::::NotEnoughPointsToUnbond) + Err(Error::::MinimumBondNotMet) } } @@ -781,45 +781,60 @@ impl BondedPool { let is_depositor = *target_account == self.roles.depositor; let is_full_unbond = unbonding_points == target_member.active_points(); + let balance_after_unbond = { + let new_depositor_points = + target_member.active_points().saturating_sub(unbonding_points); + let mut target_member_after_unbond = (*target_member).clone(); + target_member_after_unbond.points = new_depositor_points; + target_member_after_unbond.active_balance() + }; + // any partial unbonding is only ever allowed if this unbond is permissioned. ensure!( is_permissioned || is_full_unbond, Error::::PartialUnbondNotAllowedPermissionlessly ); + // any unbond must comply with the balance condition: + ensure!( + is_full_unbond || + balance_after_unbond >= + if is_depositor { + Pallet::::depositor_min_bond() + } else { + MinJoinBond::::get() + }, + Error::::MinimumBondNotMet + ); + + // additional checks: match (is_permissioned, is_depositor) { - // If the pool is blocked, then an admin with kicking permissions can remove a - // member. If the pool is being destroyed, anyone can remove a member + (true, false) => (), + (true, true) => { + // permission depositor unbond: if destroying and pool is empty, always allowed, + // with no additional limits. + if self.is_destroying_and_only_depositor(target_member.active_points()) { + // everything good, let them unbond anything. + } else { + // depositor cannot fully unbond yet. + ensure!(!is_full_unbond, Error::::MinimumBondNotMet); + } + }, (false, false) => { + // If the pool is blocked, then an admin with kicking permissions can remove a + // member. If the pool is being destroyed, anyone can remove a member + debug_assert!(is_full_unbond); ensure!( self.can_kick(caller) || self.is_destroying(), Error::::NotKickerOrDestroying ) }, - // Any member who is not the depositor can always unbond themselves - (true, false) => (), - (_, true) => { - if self.is_destroying_and_only_depositor(target_member.active_points()) { - // if the pool is about to be destroyed, anyone can unbond the depositor, and - // they can fully unbond. - } else { - // only the depositor can partially unbond, and they can only unbond up to the - // threshold. - ensure!(is_permissioned, Error::::DoesNotHavePermission); - let balance_after_unbond = { - let new_depositor_points = - target_member.active_points().saturating_sub(unbonding_points); - let mut depositor_after_unbond = (*target_member).clone(); - depositor_after_unbond.points = new_depositor_points; - depositor_after_unbond.active_balance() - }; - ensure!( - balance_after_unbond >= MinCreateBond::::get(), - Error::::NotOnlyPoolMember - ); - } + (false, true) => { + // the depositor can simply not be unbonded permissionlessly, period. + return Err(Error::::DoesNotHavePermission.into()) }, }; + Ok(()) } @@ -830,25 +845,14 @@ impl BondedPool { &self, caller: &T::AccountId, target_account: &T::AccountId, - target_member: &PoolMember, - sub_pools: &SubPools, ) -> Result<(), DispatchError> { - if *target_account == self.roles.depositor { - ensure!( - sub_pools.sum_unbonding_points() == target_member.unbonding_points(), - Error::::NotOnlyPoolMember - ); - debug_assert_eq!(self.member_counter, 1, "only member must exist at this point"); - Ok(()) - } else { - // This isn't a depositor - let is_permissioned = caller == target_account; - ensure!( - is_permissioned || self.can_kick(caller) || self.is_destroying(), - Error::::NotKickerOrDestroying - ); - Ok(()) - } + // This isn't a depositor + let is_permissioned = caller == target_account; + ensure!( + is_permissioned || self.can_kick(caller) || self.is_destroying(), + Error::::NotKickerOrDestroying + ); + Ok(()) } /// Bond exactly `amount` from `who`'s funds into this pool. @@ -1100,15 +1104,6 @@ impl SubPools { self } - /// The sum of all unbonding points, regardless of whether they are actually unlocked or not. - fn sum_unbonding_points(&self) -> BalanceOf { - self.no_era.points.saturating_add( - self.with_era - .values() - .fold(BalanceOf::::zero(), |acc, pool| acc.saturating_add(pool.points)), - ) - } - /// The sum of all unbonding balance, regardless of whether they are actually unlocked or not. #[cfg(any(test, debug_assertions))] fn sum_unbonding_balance(&self) -> BalanceOf { @@ -1419,15 +1414,16 @@ pub mod pallet { /// None of the funds can be withdrawn yet because the bonding duration has not passed. CannotWithdrawAny, /// The amount does not meet the minimum bond to either join or create a pool. + /// + /// The depositor can never unbond to a value less than + /// `Pallet::depositor_min_bond`. The caller does not have nominating + /// permissions for the pool. Members can never unbond to a value below `MinJoinBond`. MinimumBondNotMet, /// The transaction could not be executed due to overflow risk for the pool. OverflowRisk, /// A pool must be in [`PoolState::Destroying`] in order for the depositor to unbond or for /// other members to be permissionlessly unbonded. NotDestroying, - /// The depositor must be the only member in the bonded pool in order to unbond. And the - /// depositor must be the only member in the sub pools in order to withdraw unbonded. - NotOnlyPoolMember, /// The caller does not have nominating permissions for the pool. NotNominator, /// Either a) the caller cannot make a valid kick or b) the pool is not destroying. @@ -1447,8 +1443,6 @@ pub mod pallet { /// Some error occurred that should never happen. This should be reported to the /// maintainers. Defensive(DefensiveError), - /// Not enough points. Ty unbonding less. - NotEnoughPointsToUnbond, /// Partial unbonding now allowed permissionlessly. PartialUnbondNotAllowedPermissionlessly, } @@ -1758,12 +1752,7 @@ pub mod pallet { let mut sub_pools = SubPoolsStorage::::get(member.pool_id) .defensive_ok_or::>(DefensiveError::SubPoolsNotFound.into())?; - bonded_pool.ok_to_withdraw_unbonded_with( - &caller, - &member_account, - &member, - &sub_pools, - )?; + bonded_pool.ok_to_withdraw_unbonded_with(&caller, &member_account)?; // NOTE: must do this after we have done the `ok_to_withdraw_unbonded_other_with` check. let withdrawn_points = member.withdraw_unlocked(current_era); @@ -1878,13 +1867,7 @@ pub mod pallet { ) -> DispatchResult { let who = ensure_signed(origin)?; - ensure!( - amount >= - T::StakingInterface::minimum_bond() - .max(MinCreateBond::::get()) - .max(MinJoinBond::::get()), - Error::::MinimumBondNotMet - ); + ensure!(amount >= Pallet::::depositor_min_bond(), Error::::MinimumBondNotMet); ensure!( MaxPools::::get() .map_or(true, |max_pools| BondedPools::::count() < max_pools), @@ -2162,6 +2145,18 @@ pub mod pallet { } impl Pallet { + /// The amount of bond that MUST REMAIN IN BONDED in ALL POOLS. + /// + /// It is the responsibility of the depositor to put these funds into the pool initially. Upon + /// unbond, they can never unbond to a value below this amount. + /// + /// It is essentially `max { MinNominatorBond, MinCreateBond, MinJoinBond }`, where the former + /// is coming from the staking pallet and the latter two are configured in this pallet. + fn depositor_min_bond() -> BalanceOf { + T::StakingInterface::minimum_bond() + .max(MinCreateBond::::get()) + .max(MinJoinBond::::get()) + } /// Remove everything related to the given bonded pool. /// /// All sub-pools are also deleted. All accounts are dusted and the leftover of the reward diff --git a/frame/nomination-pools/src/mock.rs b/frame/nomination-pools/src/mock.rs index 042a0b666efb1..5138c55afccac 100644 --- a/frame/nomination-pools/src/mock.rs +++ b/frame/nomination-pools/src/mock.rs @@ -22,6 +22,7 @@ pub fn default_reward_account() -> AccountId { } parameter_types! { + pub static MinJoinBondConfig: Balance = 2; pub static CurrentEra: EraIndex = 0; pub static BondingDuration: EraIndex = 3; pub storage BondedBalanceMap: BTreeMap = Default::default(); @@ -245,6 +246,11 @@ impl ExtBuilder { self } + pub(crate) fn min_join_bond(self, min: Balance) -> Self { + MinJoinBondConfig::set(min); + self + } + pub(crate) fn with_check(self, level: u8) -> Self { CheckLevel::set(level); self @@ -261,11 +267,12 @@ impl ExtBuilder { } pub(crate) fn build(self) -> sp_io::TestExternalities { + sp_tracing::try_init_simple(); let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); let _ = crate::GenesisConfig:: { - min_join_bond: 2, + min_join_bond: MinJoinBondConfig::get(), min_create_bond: 2, max_pools: Some(2), max_members_per_pool: self.max_members_per_pool, @@ -280,8 +287,8 @@ impl ExtBuilder { frame_system::Pallet::::set_block_number(1); // make a pool - let amount_to_bond = ::StakingInterface::minimum_bond(); - Balances::make_free_balance_be(&10, amount_to_bond * 2); + let amount_to_bond = Pools::depositor_min_bond(); + Balances::make_free_balance_be(&10, amount_to_bond * 5); assert_ok!(Pools::create(RawOrigin::Signed(10).into(), amount_to_bond, 900, 901, 902)); let last_pool = LastPoolId::::get(); @@ -302,12 +309,13 @@ impl ExtBuilder { } } -pub(crate) fn unsafe_set_state(pool_id: PoolId, state: PoolState) -> Result<(), ()> { +pub(crate) fn unsafe_set_state(pool_id: PoolId, state: PoolState) { BondedPools::::try_mutate(pool_id, |maybe_bonded_pool| { maybe_bonded_pool.as_mut().ok_or(()).map(|bonded_pool| { bonded_pool.state = state; }) }) + .unwrap() } parameter_types! { diff --git a/frame/nomination-pools/src/tests.rs b/frame/nomination-pools/src/tests.rs index 9989893d86462..c971239bef507 100644 --- a/frame/nomination-pools/src/tests.rs +++ b/frame/nomination-pools/src/tests.rs @@ -538,13 +538,13 @@ mod join { StakingMock::set_bonded_balance(Pools::create_bonded_account(1), max_points_to_balance); // Cannot join a pool that isn't open - unsafe_set_state(123, PoolState::Blocked).unwrap(); + unsafe_set_state(123, PoolState::Blocked); assert_noop!( Pools::join(Origin::signed(11), max_points_to_balance, 123), Error::::NotOpen ); - unsafe_set_state(123, PoolState::Destroying).unwrap(); + unsafe_set_state(123, PoolState::Destroying); assert_noop!( Pools::join(Origin::signed(11), max_points_to_balance, 123), Error::::NotOpen @@ -1824,7 +1824,8 @@ mod claim_payout { fn rewards_are_rounded_down_depositor_collects_them() { ExtBuilder::default().add_members(vec![(20, 20)]).build_and_execute(|| { // initial balance of 10. - assert_eq!(Balances::free_balance(&10), 5); + + assert_eq!(Balances::free_balance(&10), 35); assert_eq!( Balances::free_balance(&default_reward_account()), Balances::minimum_balance() @@ -1875,7 +1876,7 @@ mod claim_payout { ); // original ed + ed put into reward account + reward + bond + dust. - assert_eq!(Balances::free_balance(&10), 5 + 5 + 13 + 10 + 1); + assert_eq!(Balances::free_balance(&10), 35 + 5 + 13 + 10 + 1); }) } @@ -1942,10 +1943,269 @@ mod claim_payout { mod unbond { use super::*; + #[test] + fn member_unbond_open() { + // depositor in pool, pool state open + // - member unbond above limit + // - member unbonds to 0 + // - member cannot unbond between within limit and 0 + ExtBuilder::default() + .min_join_bond(10) + .add_members(vec![(20, 20)]) + .build_and_execute(|| { + // can unbond to above limit + assert_ok!(Pools::unbond(Origin::signed(20), 20, 5)); + assert_eq!(PoolMembers::::get(20).unwrap().active_points(), 15); + assert_eq!(PoolMembers::::get(20).unwrap().unbonding_points(), 5); + + // cannot go to below 10: + assert_noop!( + Pools::unbond(Origin::signed(20), 20, 10), + Error::::MinimumBondNotMet + ); + + // but can go to 0 + assert_ok!(Pools::unbond(Origin::signed(20), 20, 15)); + assert_eq!(PoolMembers::::get(20).unwrap().active_points(), 0); + assert_eq!(PoolMembers::::get(20).unwrap().unbonding_points(), 20); + }) + } + + #[test] + fn member_kicked() { + // depositor in pool, pool state blocked + // - member cannot be kicked to above limit + // - member cannot be kicked between within limit and 0 + // - member kicked to 0 + ExtBuilder::default() + .min_join_bond(10) + .add_members(vec![(20, 20)]) + .build_and_execute(|| { + unsafe_set_state(1, PoolState::Blocked); + let kicker = DEFAULT_ROLES.state_toggler.unwrap(); + + // cannot be kicked to above the limit. + assert_noop!( + Pools::unbond(Origin::signed(kicker), 20, 5), + Error::::PartialUnbondNotAllowedPermissionlessly + ); + + // cannot go to below 10: + assert_noop!( + Pools::unbond(Origin::signed(kicker), 20, 15), + Error::::PartialUnbondNotAllowedPermissionlessly + ); + + // but they themselves can do an unbond + assert_ok!(Pools::unbond(Origin::signed(20), 20, 2)); + assert_eq!(PoolMembers::::get(20).unwrap().active_points(), 18); + assert_eq!(PoolMembers::::get(20).unwrap().unbonding_points(), 2); + + // can be kicked to 0. + assert_ok!(Pools::unbond(Origin::signed(kicker), 20, 18)); + assert_eq!(PoolMembers::::get(20).unwrap().active_points(), 0); + assert_eq!(PoolMembers::::get(20).unwrap().unbonding_points(), 20); + }) + } + + #[test] + fn member_unbond_destroying() { + // depositor in pool, pool state destroying + // - member cannot be permissionlessly unbonded to above limit + // - member cannot be permissionlessly unbonded between within limit and 0 + // - member permissionlessly unbonded to 0 + ExtBuilder::default() + .min_join_bond(10) + .add_members(vec![(20, 20)]) + .build_and_execute(|| { + unsafe_set_state(1, PoolState::Destroying); + let random = 123; + + // cannot be kicked to above the limit. + assert_noop!( + Pools::unbond(Origin::signed(random), 20, 5), + Error::::PartialUnbondNotAllowedPermissionlessly + ); + + // cannot go to below 10: + assert_noop!( + Pools::unbond(Origin::signed(random), 20, 15), + Error::::PartialUnbondNotAllowedPermissionlessly + ); + + // but they themselves can do an unbond + assert_ok!(Pools::unbond(Origin::signed(20), 20, 2)); + assert_eq!(PoolMembers::::get(20).unwrap().active_points(), 18); + assert_eq!(PoolMembers::::get(20).unwrap().unbonding_points(), 2); + + // but can go to 0 + assert_ok!(Pools::unbond(Origin::signed(random), 20, 18)); + assert_eq!(PoolMembers::::get(20).unwrap().active_points(), 0); + assert_eq!(PoolMembers::::get(20).unwrap().unbonding_points(), 20); + }) + } + + #[test] + fn depositor_unbond_open() { + // depositor in pool, pool state open + // - depositor unbonds to above limit + // - depositor cannot unbond to below limit or 0 + ExtBuilder::default().min_join_bond(10).build_and_execute(|| { + // give the depositor some extra funds. + assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::FreeBalance(10))); + assert_eq!(PoolMembers::::get(10).unwrap().points, 20); + + // can unbond to above the limit. + assert_ok!(Pools::unbond(Origin::signed(10), 10, 5)); + assert_eq!(PoolMembers::::get(10).unwrap().active_points(), 15); + assert_eq!(PoolMembers::::get(10).unwrap().unbonding_points(), 5); + + // cannot go to below 10: + assert_noop!(Pools::unbond(Origin::signed(10), 10, 10), Error::::MinimumBondNotMet); + + // cannot go to 0 either. + assert_noop!(Pools::unbond(Origin::signed(10), 10, 15), Error::::MinimumBondNotMet); + }) + } + + #[test] + fn depositor_kick() { + // depositor in pool, pool state blocked + // - depositor can never be kicked. + ExtBuilder::default().min_join_bond(10).build_and_execute(|| { + // give the depositor some extra funds. + assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::FreeBalance(10))); + assert_eq!(PoolMembers::::get(10).unwrap().points, 20); + + // set the stage + unsafe_set_state(1, PoolState::Blocked); + let kicker = DEFAULT_ROLES.state_toggler.unwrap(); + + // cannot be kicked to above limit. + assert_noop!( + Pools::unbond(Origin::signed(kicker), 10, 5), + Error::::PartialUnbondNotAllowedPermissionlessly + ); + + // or below the limit + assert_noop!( + Pools::unbond(Origin::signed(kicker), 10, 15), + Error::::PartialUnbondNotAllowedPermissionlessly + ); + + // or 0. + assert_noop!( + Pools::unbond(Origin::signed(kicker), 10, 20), + Error::::DoesNotHavePermission + ); + + // they themselves cannot do it either + assert_noop!(Pools::unbond(Origin::signed(10), 10, 20), Error::::MinimumBondNotMet); + }) + } + + #[test] + fn depositor_unbond_destroying_permissionless() { + // depositor can never be permissionlessly unbonded. + ExtBuilder::default().min_join_bond(10).build_and_execute(|| { + // give the depositor some extra funds. + assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::FreeBalance(10))); + assert_eq!(PoolMembers::::get(10).unwrap().points, 20); + + // set the stage + unsafe_set_state(1, PoolState::Destroying); + let random = 123; + + // cannot be kicked to above limit. + assert_noop!( + Pools::unbond(Origin::signed(random), 10, 5), + Error::::PartialUnbondNotAllowedPermissionlessly + ); + + // or below the limit + assert_noop!( + Pools::unbond(Origin::signed(random), 10, 15), + Error::::PartialUnbondNotAllowedPermissionlessly + ); + + // or 0. + assert_noop!( + Pools::unbond(Origin::signed(random), 10, 20), + Error::::DoesNotHavePermission + ); + + // they themselves can do it in this case though. + assert_ok!(Pools::unbond(Origin::signed(10), 10, 20)); + }) + } + + #[test] + fn depositor_unbond_destroying_not_last_member() { + // deposit in pool, pool state destroying + // - depositor can never leave if there is another member in the pool. + ExtBuilder::default() + .min_join_bond(10) + .add_members(vec![(20, 20)]) + .build_and_execute(|| { + // give the depositor some extra funds. + assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::FreeBalance(10))); + assert_eq!(PoolMembers::::get(10).unwrap().points, 20); + + // set the stage + unsafe_set_state(1, PoolState::Destroying); + + // can go above the limit + assert_ok!(Pools::unbond(Origin::signed(10), 10, 5)); + assert_eq!(PoolMembers::::get(10).unwrap().active_points(), 15); + assert_eq!(PoolMembers::::get(10).unwrap().unbonding_points(), 5); + + // but not below the limit + assert_noop!( + Pools::unbond(Origin::signed(10), 10, 10), + Error::::MinimumBondNotMet + ); + + // and certainly not zero + assert_noop!( + Pools::unbond(Origin::signed(10), 10, 15), + Error::::MinimumBondNotMet + ); + }) + } + + #[test] + fn depositor_unbond_destroying_last_member() { + // deposit in pool, pool state destroying + // - depositor can unbond to above limit always. + // - depositor cannot unbond to below limit if last. + // - depositor can unbond to 0 if last and destroying. + ExtBuilder::default().min_join_bond(10).build_and_execute(|| { + // give the depositor some extra funds. + assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::FreeBalance(10))); + assert_eq!(PoolMembers::::get(10).unwrap().points, 20); + + // set the stage + unsafe_set_state(1, PoolState::Destroying); + + // can unbond to above the limit. + assert_ok!(Pools::unbond(Origin::signed(10), 10, 5)); + assert_eq!(PoolMembers::::get(10).unwrap().active_points(), 15); + assert_eq!(PoolMembers::::get(10).unwrap().unbonding_points(), 5); + + // still cannot go to below limit + assert_noop!(Pools::unbond(Origin::signed(10), 10, 10), Error::::MinimumBondNotMet); + + // can go to 0 too. + assert_ok!(Pools::unbond(Origin::signed(10), 10, 15)); + assert_eq!(PoolMembers::::get(10).unwrap().active_points(), 0); + assert_eq!(PoolMembers::::get(10).unwrap().unbonding_points(), 20); + }) + } + #[test] fn unbond_of_1_works() { ExtBuilder::default().build_and_execute(|| { - unsafe_set_state(1, PoolState::Destroying).unwrap(); + unsafe_set_state(1, PoolState::Destroying); assert_ok!(fully_unbond_permissioned(10)); assert_eq!( @@ -2021,7 +2281,7 @@ mod unbond { assert_eq!(Balances::free_balance(&40), 40 + 40); // We claim rewards when unbonding // When - unsafe_set_state(1, PoolState::Destroying).unwrap(); + unsafe_set_state(1, PoolState::Destroying); assert_ok!(fully_unbond_permissioned(550)); // Then @@ -2111,7 +2371,7 @@ mod unbond { }, }, ); - unsafe_set_state(1, PoolState::Destroying).unwrap(); + unsafe_set_state(1, PoolState::Destroying); // When let current_era = 1 + TotalUnbondingPools::::get(); @@ -2148,7 +2408,7 @@ mod unbond { .add_members(vec![(100, 100), (200, 200)]) .build_and_execute(|| { // Given - unsafe_set_state(1, PoolState::Blocked).unwrap(); + unsafe_set_state(1, PoolState::Blocked); let bonded_pool = BondedPool::::get(1).unwrap(); assert_eq!(bonded_pool.roles.root.unwrap(), 900); assert_eq!(bonded_pool.roles.nominator.unwrap(), 901); @@ -2216,7 +2476,7 @@ mod unbond { // Scenarios where non-admin accounts can unbond others ExtBuilder::default().add_members(vec![(100, 100)]).build_and_execute(|| { // Given the pool is blocked - unsafe_set_state(1, PoolState::Blocked).unwrap(); + unsafe_set_state(1, PoolState::Blocked); // A permissionless unbond attempt errors assert_noop!( @@ -2231,16 +2491,17 @@ mod unbond { ); // Given the pool is destroying - unsafe_set_state(1, PoolState::Destroying).unwrap(); + unsafe_set_state(1, PoolState::Destroying); // The depositor cannot be fully unbonded until they are the last member assert_noop!( Pools::fully_unbond(Origin::signed(10), 10), - Error::::NotOnlyPoolMember + Error::::MinimumBondNotMet, ); // Any account can unbond a member that is not the depositor assert_ok!(Pools::fully_unbond(Origin::signed(420), 100)); + assert_eq!( pool_events_since_last_call(), vec![ @@ -2258,7 +2519,7 @@ mod unbond { ); // Given the pool is blocked - unsafe_set_state(1, PoolState::Blocked).unwrap(); + unsafe_set_state(1, PoolState::Blocked); // The depositor cannot be unbonded assert_noop!( @@ -2267,7 +2528,7 @@ mod unbond { ); // Given the pools is destroying - unsafe_set_state(1, PoolState::Destroying).unwrap(); + unsafe_set_state(1, PoolState::Destroying); // The depositor cannot be unbonded yet. assert_noop!( @@ -2285,8 +2546,13 @@ mod unbond { Error::::PartialUnbondNotAllowedPermissionlessly, ); - // but full unbond works. - assert_ok!(Pools::fully_unbond(Origin::signed(420), 10)); + // depositor can never be unbonded permissionlessly . + assert_noop!( + Pools::fully_unbond(Origin::signed(420), 10), + Error::::DoesNotHavePermission + ); + // but depositor itself can do it. + assert_ok!(Pools::fully_unbond(Origin::signed(10), 10)); assert_eq!(BondedPools::::get(1).unwrap().points, 0); assert_eq!( @@ -2346,6 +2612,12 @@ mod unbond { #[test] fn partial_unbond_era_tracking() { ExtBuilder::default().build_and_execute(|| { + // to make the depositor capable of withdrawing. + StakingMinBond::set(1); + MinCreateBond::::set(1); + MinJoinBond::::set(1); + assert_eq!(Pools::depositor_min_bond(), 1); + // given assert_eq!(PoolMembers::::get(10).unwrap().active_points(), 10); assert_eq!(PoolMembers::::get(10).unwrap().unbonding_points(), 0); @@ -2360,7 +2632,7 @@ mod unbond { assert_eq!(BondingDuration::get(), 3); // so the depositor can leave, just keeps the test simpler. - unsafe_set_state(1, PoolState::Destroying).unwrap(); + unsafe_set_state(1, PoolState::Destroying); // when: casual unbond assert_ok!(Pools::unbond(Origin::signed(10), 10, 1)); @@ -2444,13 +2716,13 @@ mod unbond { ); // when: unbonding more than our active: error - assert_err!( + assert_noop!( frame_support::storage::in_storage_layer(|| Pools::unbond( Origin::signed(10), 10, 5 )), - Error::::NotEnoughPointsToUnbond + Error::::MinimumBondNotMet ); // instead: assert_ok!(Pools::unbond(Origin::signed(10), 10, 3)); @@ -2482,26 +2754,24 @@ mod unbond { #[test] fn partial_unbond_max_chunks() { - ExtBuilder::default().ed(1).build_and_execute(|| { - // so the depositor can leave, just keeps the test simpler. - unsafe_set_state(1, PoolState::Destroying).unwrap(); + ExtBuilder::default().add_members(vec![(20, 20)]).ed(1).build_and_execute(|| { MaxUnbonding::set(2); // given - assert_ok!(Pools::unbond(Origin::signed(10), 10, 2)); + assert_ok!(Pools::unbond(Origin::signed(20), 20, 2)); CurrentEra::set(1); - assert_ok!(Pools::unbond(Origin::signed(10), 10, 3)); + assert_ok!(Pools::unbond(Origin::signed(20), 20, 3)); assert_eq!( - PoolMembers::::get(10).unwrap().unbonding_eras, + PoolMembers::::get(20).unwrap().unbonding_eras, member_unbonding_eras!(3 => 2, 4 => 3) ); // when CurrentEra::set(2); - assert_err!( + assert_noop!( frame_support::storage::in_storage_layer(|| Pools::unbond( - Origin::signed(10), - 10, + Origin::signed(20), + 20, 4 )), Error::::MaxUnbondingLimit @@ -2509,30 +2779,35 @@ mod unbond { // when MaxUnbonding::set(3); - assert_ok!(Pools::unbond(Origin::signed(10), 10, 1)); + assert_ok!(Pools::unbond(Origin::signed(20), 20, 1)); + assert_eq!( - PoolMembers::::get(10).unwrap().unbonding_eras, + PoolMembers::::get(20).unwrap().unbonding_eras, member_unbonding_eras!(3 => 2, 4 => 3, 5 => 1) ); + assert_eq!( pool_events_since_last_call(), vec![ Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, - Event::Unbonded { member: 10, pool_id: 1, points: 2, balance: 2 }, - Event::Unbonded { member: 10, pool_id: 1, points: 3, balance: 3 }, - Event::Unbonded { member: 10, pool_id: 1, points: 1, balance: 1 } + Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, + Event::Unbonded { member: 20, pool_id: 1, balance: 2, points: 2 }, + Event::Unbonded { member: 20, pool_id: 1, balance: 3, points: 3 }, + Event::Unbonded { member: 20, pool_id: 1, balance: 1, points: 1 } ] ); }) } - // depositor can unbond inly up to `MinCreateBond`. + // depositor can unbond only up to `MinCreateBond`. #[test] fn depositor_permissioned_partial_unbond() { ExtBuilder::default().ed(1).build_and_execute(|| { // given - assert_eq!(MinCreateBond::::get(), 2); + StakingMinBond::set(5); + assert_eq!(Pools::depositor_min_bond(), 5); + assert_eq!(PoolMembers::::get(10).unwrap().active_points(), 10); assert_eq!(PoolMembers::::get(10).unwrap().unbonding_points(), 0); @@ -2544,7 +2819,7 @@ mod unbond { // but not less than 2 assert_noop!( Pools::unbond(Origin::signed(10), 10, 6), - Error::::NotOnlyPoolMember + Error::::MinimumBondNotMet ); assert_eq!( @@ -2558,7 +2833,6 @@ mod unbond { }); } - // same as above, but the pool is slashed and therefore the depositor cannot partially unbond. #[test] fn depositor_permissioned_partial_unbond_slashed() { ExtBuilder::default().ed(1).build_and_execute(|| { @@ -2573,78 +2847,69 @@ mod unbond { // cannot unbond even 7, because the value of shares is now less. assert_noop!( Pools::unbond(Origin::signed(10), 10, 7), - Error::::NotOnlyPoolMember - ); - assert_eq!( - pool_events_since_last_call(), - vec![ - Event::Created { depositor: 10, pool_id: 1 }, - Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, - ] + Error::::MinimumBondNotMet ); }); } #[test] fn every_unbonding_triggers_payout() { - ExtBuilder::default().build_and_execute(|| { - let initial_reward_account = Balances::free_balance(Pools::create_reward_account(1)); + ExtBuilder::default().add_members(vec![(20, 20)]).build_and_execute(|| { + let initial_reward_account = Balances::free_balance(default_reward_account()); assert_eq!(initial_reward_account, Balances::minimum_balance()); assert_eq!(initial_reward_account, 5); - // set the pool to destroying so that depositor can leave. - unsafe_set_state(1, PoolState::Destroying).unwrap(); - Balances::make_free_balance_be( - &Pools::create_reward_account(1), - 2 * Balances::minimum_balance(), + &default_reward_account(), + 4 * Balances::minimum_balance(), ); - assert_ok!(Pools::unbond(Origin::signed(10), 10, 2)); + assert_ok!(Pools::unbond(Origin::signed(20), 20, 2)); assert_eq!( pool_events_since_last_call(), vec![ + // 2/3 of ed, which is 20's share. Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, - // exactly equal to ed, all that can be claimed. - Event::PaidOut { member: 10, pool_id: 1, payout: 5 }, - Event::Unbonded { member: 10, pool_id: 1, points: 2, balance: 2 } + Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, + Event::PaidOut { member: 20, pool_id: 1, payout: 10 }, + Event::Unbonded { member: 20, pool_id: 1, balance: 2, points: 2 } ] ); CurrentEra::set(1); Balances::make_free_balance_be( - &Pools::create_reward_account(1), - 2 * Balances::minimum_balance(), + &default_reward_account(), + 4 * Balances::minimum_balance(), ); - assert_ok!(Pools::unbond(Origin::signed(10), 10, 3)); + assert_ok!(Pools::unbond(Origin::signed(20), 20, 3)); assert_eq!( pool_events_since_last_call(), vec![ - // exactly equal to ed, all that can be claimed. - Event::PaidOut { member: 10, pool_id: 1, payout: 5 }, - Event::Unbonded { member: 10, pool_id: 1, points: 3, balance: 3 } + // 2/3 of ed, which is 20's share. + Event::PaidOut { member: 20, pool_id: 1, payout: 6 }, + Event::Unbonded { member: 20, pool_id: 1, points: 3, balance: 3 } ] ); CurrentEra::set(2); Balances::make_free_balance_be( - &Pools::create_reward_account(1), - 2 * Balances::minimum_balance(), + &default_reward_account(), + 4 * Balances::minimum_balance(), ); - assert_ok!(Pools::unbond(Origin::signed(10), 10, 5)); + assert_ok!(Pools::unbond(Origin::signed(20), 20, 5)); assert_eq!( pool_events_since_last_call(), vec![ - Event::PaidOut { member: 10, pool_id: 1, payout: 5 }, - Event::Unbonded { member: 10, pool_id: 1, points: 5, balance: 5 } + Event::PaidOut { member: 20, pool_id: 1, payout: 3 }, + Event::Unbonded { member: 20, pool_id: 1, points: 5, balance: 5 } ] ); assert_eq!( - PoolMembers::::get(10).unwrap().unbonding_eras, + PoolMembers::::get(20).unwrap().unbonding_eras, member_unbonding_eras!(3 => 2, 4 => 3, 5 => 5) ); }); @@ -2801,7 +3066,7 @@ mod withdraw_unbonded { ); // now, finally, the depositor can take out its share. - unsafe_set_state(1, PoolState::Destroying).unwrap(); + unsafe_set_state(1, PoolState::Destroying); assert_ok!(fully_unbond_permissioned(10)); current_era += 3; @@ -2911,7 +3176,7 @@ mod withdraw_unbonded { assert!(SubPoolsStorage::::get(&1).unwrap().with_era.is_empty()); // now, finally, the depositor can take out its share. - unsafe_set_state(1, PoolState::Destroying).unwrap(); + unsafe_set_state(1, PoolState::Destroying); assert_ok!(fully_unbond_permissioned(10)); // because everyone else has left, the points @@ -2926,7 +3191,7 @@ mod withdraw_unbonded { assert_ok!(Pools::withdraw_unbonded(Origin::signed(10), 10, 0)); // then - assert_eq!(Balances::free_balance(&10), 10 + 5); + assert_eq!(Balances::free_balance(&10), 10 + 35); assert_eq!(Balances::free_balance(&default_bonded_account()), 0); // in this test 10 also gets a fair share of the slash, because the slash was @@ -2955,9 +3220,9 @@ mod withdraw_unbonded { ExtBuilder::default().build_and_execute(|| { // Given assert_eq!(Balances::minimum_balance(), 5); - assert_eq!(Balances::free_balance(&10), 5); + assert_eq!(Balances::free_balance(&10), 35); assert_eq!(Balances::free_balance(&default_bonded_account()), 10); - unsafe_set_state(1, PoolState::Destroying).unwrap(); + unsafe_set_state(1, PoolState::Destroying); assert_ok!(Pools::fully_unbond(Origin::signed(10), 10)); // Simulate a slash that is not accounted for in the sub pools. @@ -2974,7 +3239,7 @@ mod withdraw_unbonded { assert_ok!(Pools::withdraw_unbonded(Origin::signed(10), 10, 0)); // Then - assert_eq!(Balances::free_balance(10), 10 + 5); + assert_eq!(Balances::free_balance(10), 10 + 35); assert_eq!(Balances::free_balance(&default_bonded_account()), 0); }); } @@ -3054,7 +3319,7 @@ mod withdraw_unbonded { ); // Given - unsafe_set_state(1, PoolState::Blocked).unwrap(); + unsafe_set_state(1, PoolState::Blocked); // Cannot kick as a nominator assert_noop!( @@ -3112,7 +3377,7 @@ mod withdraw_unbonded { ); // Given - unsafe_set_state(1, PoolState::Destroying).unwrap(); + unsafe_set_state(1, PoolState::Destroying); // Can permissionlesly withdraw a member that is not the depositor assert_ok!(Pools::withdraw_unbonded(Origin::signed(420), 100, 0)); @@ -3137,8 +3402,8 @@ mod withdraw_unbonded { #[test] fn partial_withdraw_unbonded_depositor() { ExtBuilder::default().ed(1).build_and_execute(|| { - // so the depositor can leave, just keeps the test simpler. - unsafe_set_state(1, PoolState::Destroying).unwrap(); + assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::FreeBalance(10))); + unsafe_set_state(1, PoolState::Destroying); // given assert_ok!(Pools::unbond(Origin::signed(10), 10, 6)); @@ -3158,13 +3423,14 @@ mod withdraw_unbonded { } } ); - assert_eq!(PoolMembers::::get(10).unwrap().active_points(), 3); + assert_eq!(PoolMembers::::get(10).unwrap().active_points(), 13); assert_eq!(PoolMembers::::get(10).unwrap().unbonding_points(), 7); assert_eq!( pool_events_since_last_call(), vec![ Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: false }, Event::Unbonded { member: 10, pool_id: 1, points: 6, balance: 6 }, Event::Unbonded { member: 10, pool_id: 1, points: 1, balance: 1 } ] @@ -3368,50 +3634,72 @@ mod withdraw_unbonded { #[test] fn full_multi_step_withdrawing_depositor() { ExtBuilder::default().ed(1).build_and_execute(|| { - // given + // depositor now has 20, they can unbond to 10. + assert_eq!(Pools::depositor_min_bond(), 10); + assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::FreeBalance(10))); + + // now they can. assert_ok!(Pools::unbond(Origin::signed(10), 10, 7)); // progress one era and unbond the leftover. CurrentEra::set(1); - unsafe_set_state(1, PoolState::Destroying).unwrap(); assert_ok!(Pools::unbond(Origin::signed(10), 10, 3)); + assert_eq!( PoolMembers::::get(10).unwrap().unbonding_eras, member_unbonding_eras!(3 => 7, 4 => 3) ); + // they can't unbond to a value below 10 other than 0.. assert_noop!( - Pools::withdraw_unbonded(Origin::signed(10), 10, 0), - Error::::CannotWithdrawAny + Pools::unbond(Origin::signed(10), 10, 5), + Error::::MinimumBondNotMet ); + // but not even full, because they pool is not yet destroying. + assert_noop!( + Pools::unbond(Origin::signed(10), 10, 10), + Error::::MinimumBondNotMet + ); + + // but now they can. + unsafe_set_state(1, PoolState::Destroying); + assert_noop!( + Pools::unbond(Origin::signed(10), 10, 5), + Error::::MinimumBondNotMet + ); + assert_ok!(Pools::unbond(Origin::signed(10), 10, 10)); + // now the 7 should be free. CurrentEra::set(3); assert_ok!(Pools::withdraw_unbonded(Origin::signed(10), 10, 0)); + assert_eq!( pool_events_since_last_call(), vec![ Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, - Event::Unbonded { member: 10, pool_id: 1, points: 7, balance: 7 }, - Event::Unbonded { member: 10, pool_id: 1, points: 3, balance: 3 }, - Event::Withdrawn { member: 10, pool_id: 1, points: 7, balance: 7 } + Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: false }, + Event::Unbonded { member: 10, pool_id: 1, balance: 7, points: 7 }, + Event::Unbonded { member: 10, pool_id: 1, balance: 3, points: 3 }, + Event::Unbonded { member: 10, pool_id: 1, balance: 10, points: 10 }, + Event::Withdrawn { member: 10, pool_id: 1, balance: 7, points: 7 } ] ); assert_eq!( PoolMembers::::get(10).unwrap().unbonding_eras, - member_unbonding_eras!(4 => 3) + member_unbonding_eras!(4 => 13) ); - // the 25 should be free now, and the member removed. + // the 13 should be free now, and the member removed. CurrentEra::set(4); assert_ok!(Pools::withdraw_unbonded(Origin::signed(10), 10, 0)); + assert_eq!( pool_events_since_last_call(), vec![ - Event::Withdrawn { member: 10, pool_id: 1, points: 3, balance: 3 }, + Event::Withdrawn { member: 10, pool_id: 1, points: 13, balance: 13 }, Event::MemberRemoved { pool_id: 1, member: 10 }, - // the pool is also destroyed now. Event::Destroyed { pool_id: 1 }, ] ); @@ -3640,7 +3928,7 @@ mod set_state { // If the pool is not ok to be open, then anyone can set it to destroying // Given - unsafe_set_state(1, PoolState::Open).unwrap(); + unsafe_set_state(1, PoolState::Open); let mut bonded_pool = BondedPool::::get(1).unwrap(); bonded_pool.points = 100; bonded_pool.put(); @@ -3651,7 +3939,7 @@ mod set_state { // Given Balances::make_free_balance_be(&default_bonded_account(), Balance::max_value() / 10); - unsafe_set_state(1, PoolState::Open).unwrap(); + unsafe_set_state(1, PoolState::Open); // When assert_ok!(Pools::set_state(Origin::signed(11), 1, PoolState::Destroying)); // Then @@ -3659,7 +3947,7 @@ mod set_state { // If the pool is not ok to be open, it cannot be permissionleslly set to a state that // isn't destroying - unsafe_set_state(1, PoolState::Open).unwrap(); + unsafe_set_state(1, PoolState::Open); assert_noop!( Pools::set_state(Origin::signed(11), 1, PoolState::Blocked), Error::::CanNotChangeState @@ -3820,13 +4108,13 @@ mod bond_extra { // given assert_eq!(PoolMembers::::get(10).unwrap().points, 10); assert_eq!(BondedPools::::get(1).unwrap().points, 10); - assert_eq!(Balances::free_balance(10), 5); + assert_eq!(Balances::free_balance(10), 35); // when assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::Rewards)); // then - assert_eq!(Balances::free_balance(10), 5); + assert_eq!(Balances::free_balance(10), 35); assert_eq!(PoolMembers::::get(10).unwrap().points, 10 + claimable_reward); assert_eq!(BondedPools::::get(1).unwrap().points, 10 + claimable_reward); @@ -3862,14 +4150,14 @@ mod bond_extra { assert_eq!(PoolMembers::::get(10).unwrap().points, 10); assert_eq!(PoolMembers::::get(20).unwrap().points, 20); assert_eq!(BondedPools::::get(1).unwrap().points, 30); - assert_eq!(Balances::free_balance(10), 5); + assert_eq!(Balances::free_balance(10), 35); assert_eq!(Balances::free_balance(20), 20); // when assert_ok!(Pools::bond_extra(Origin::signed(10), BondExtra::Rewards)); // then - assert_eq!(Balances::free_balance(10), 5); + assert_eq!(Balances::free_balance(10), 35); // 10's share of the reward is 1/3, since they gave 10/30 of the total shares. assert_eq!(PoolMembers::::get(10).unwrap().points, 10 + 1); assert_eq!(BondedPools::::get(1).unwrap().points, 30 + 1); diff --git a/frame/nomination-pools/test-staking/src/lib.rs b/frame/nomination-pools/test-staking/src/lib.rs index 3b9350f25040b..eed8e5fd390cd 100644 --- a/frame/nomination-pools/test-staking/src/lib.rs +++ b/frame/nomination-pools/test-staking/src/lib.rs @@ -72,7 +72,7 @@ fn pool_lifecycle_e2e() { // depositor cannot unbond yet. assert_noop!( Pools::unbond(Origin::signed(10), 10, 50), - PoolsError::::NotOnlyPoolMember, + PoolsError::::MinimumBondNotMet, ); // now the members want to unbond. @@ -103,7 +103,7 @@ fn pool_lifecycle_e2e() { // depositor cannot still unbond assert_noop!( Pools::unbond(Origin::signed(10), 10, 50), - PoolsError::::NotOnlyPoolMember, + PoolsError::::MinimumBondNotMet, ); for e in 1..BondingDuration::get() { @@ -120,7 +120,7 @@ fn pool_lifecycle_e2e() { // depositor cannot still unbond assert_noop!( Pools::unbond(Origin::signed(10), 10, 50), - PoolsError::::NotOnlyPoolMember, + PoolsError::::MinimumBondNotMet, ); // but members can now withdraw. From 37cca710eed3dadd4ed5364c7686608f5175cce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Thei=C3=9Fen?= Date: Thu, 14 Jul 2022 20:14:28 +0200 Subject: [PATCH 11/46] contracts: Composable `ChainExtension` (#11816) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add `RegisteredChainExtension` * Add tests * Update frame/contracts/src/chain_extension.rs Co-authored-by: Michael Müller * Add more docs * Remove debugging leftover * Make ChainExtension-registry lowercase * Apply suggestions from code review Co-authored-by: Hernando Castano * Improve clarity of test inputs Co-authored-by: Michael Müller Co-authored-by: Hernando Castano --- Cargo.lock | 1 + frame/contracts/Cargo.toml | 1 + frame/contracts/fixtures/chain_extension.wat | 10 +- frame/contracts/src/chain_extension.rs | 59 ++++++- frame/contracts/src/tests.rs | 175 +++++++++++++++++-- 5 files changed, 216 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b7c315be4390..791d8037f3335 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5505,6 +5505,7 @@ dependencies = [ "frame-support", "frame-system", "hex-literal", + "impl-trait-for-tuples", "log", "pallet-balances", "pallet-contracts-primitives", diff --git a/frame/contracts/Cargo.toml b/frame/contracts/Cargo.toml index d27801df33bda..ac85c469354fe 100644 --- a/frame/contracts/Cargo.toml +++ b/frame/contracts/Cargo.toml @@ -26,6 +26,7 @@ smallvec = { version = "1", default-features = false, features = [ "const_generics", ] } wasmi-validation = { version = "0.4", default-features = false } +impl-trait-for-tuples = "0.2" # Only used in benchmarking to generate random contract code rand = { version = "0.8", optional = true, default-features = false } diff --git a/frame/contracts/fixtures/chain_extension.wat b/frame/contracts/fixtures/chain_extension.wat index db7e83fd96b42..9b2534c540ab8 100644 --- a/frame/contracts/fixtures/chain_extension.wat +++ b/frame/contracts/fixtures/chain_extension.wat @@ -15,12 +15,12 @@ ) ;; [0, 4) len of input output - (data (i32.const 0) "\02") + (data (i32.const 0) "\08") ;; [4, 12) buffer for input - ;; [12, 16) len of output buffer - (data (i32.const 12) "\02") + ;; [12, 48) len of output buffer + (data (i32.const 12) "\20") ;; [16, inf) buffer for output @@ -31,7 +31,7 @@ ;; the chain extension passes through the input and returns it as output (call $seal_call_chain_extension - (i32.load8_u (i32.const 4)) ;; func_id + (i32.load (i32.const 4)) ;; func_id (i32.const 4) ;; input_ptr (i32.load (i32.const 0)) ;; input_len (i32.const 16) ;; output_ptr @@ -39,7 +39,7 @@ ) ;; the chain extension passes through the func_id - (call $assert (i32.eq (i32.load8_u (i32.const 4)))) + (call $assert (i32.eq (i32.load (i32.const 4)))) (call $seal_return (i32.const 0) (i32.const 16) (i32.load (i32.const 12))) ) diff --git a/frame/contracts/src/chain_extension.rs b/frame/contracts/src/chain_extension.rs index ed447719933be..536d58c94f68f 100644 --- a/frame/contracts/src/chain_extension.rs +++ b/frame/contracts/src/chain_extension.rs @@ -29,6 +29,22 @@ //! required for this endeavour are defined or re-exported in this module. There is an //! implementation on `()` which can be used to signal that no chain extension is available. //! +//! # Using multiple chain extensions +//! +//! Often there is a need for having multiple chain extensions. This is often the case when +//! some generally useful off-the-shelf extensions should be included. To have multiple chain +//! extensions they can be put into a tuple which is then passed to `[Config::ChainExtension]` like +//! this `type Extensions = (ExtensionA, ExtensionB)`. +//! +//! However, only extensions implementing [`RegisteredChainExtension`] can be put into a tuple. +//! This is because the [`RegisteredChainExtension::ID`] is used to decide which of those extensions +//! should should be used when the contract calls a chain extensions. Extensions which are generally +//! useful should claim their `ID` with [the registry](https://github.com/paritytech/chainextension-registry) +//! so that no collisions with other vendors will occur. +//! +//! **Chain specific extensions must use the reserved `ID = 0` so that they can't be registered with +//! the registry.** +//! //! # Security //! //! The chain author alone is responsible for the security of the chain extension. @@ -112,20 +128,51 @@ pub trait ChainExtension { } } -/// Implementation that indicates that no chain extension is available. -impl ChainExtension for () { - fn call(_func_id: u32, mut _env: Environment) -> Result +/// A [`ChainExtension`] that can be composed with other extensions using a tuple. +/// +/// An extension that implements this trait can be put in a tuple in order to have multiple +/// extensions available. The tuple implementation routes requests based on the first two +/// most significant bytes of the `func_id` passed to `call`. +/// +/// If this extensions is to be used by multiple runtimes consider +/// [registering it](https://github.com/paritytech/chainextension-registry) to ensure that there +/// are no collisions with other vendors. +/// +/// # Note +/// +/// Currently, we support tuples of up to ten registred chain extensions. If more chain extensions +/// are needed consider opening an issue. +pub trait RegisteredChainExtension: ChainExtension { + /// The extensions globally unique identifier. + const ID: u16; +} + +#[impl_trait_for_tuples::impl_for_tuples(10)] +#[tuple_types_custom_trait_bound(RegisteredChainExtension)] +impl ChainExtension for Tuple { + fn call(func_id: u32, mut env: Environment) -> Result where E: Ext, ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>, { - // Never called since [`Self::enabled()`] is set to `false`. Because we want to - // avoid panics at all costs we supply a sensible error value here instead - // of an `unimplemented!`. + for_tuples!( + #( + if (Tuple::ID == (func_id >> 16) as u16) && Tuple::enabled() { + return Tuple::call(func_id, env); + } + )* + ); Err(Error::::NoChainExtension.into()) } fn enabled() -> bool { + for_tuples!( + #( + if Tuple::enabled() { + return true; + } + )* + ); false } } diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index bbac18142a658..85a0e9977d2d7 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -17,8 +17,8 @@ use crate::{ chain_extension::{ - ChainExtension, Environment, Ext, InitState, Result as ExtensionResult, RetVal, - ReturnFlags, SysConfig, UncheckedFrom, + ChainExtension, Environment, Ext, InitState, RegisteredChainExtension, + Result as ExtensionResult, RetVal, ReturnFlags, SysConfig, UncheckedFrom, }, exec::{FixSizedKey, Frame}, storage::Storage, @@ -118,6 +118,10 @@ pub struct TestExtension { last_seen_inputs: (u32, u32, u32, u32), } +pub struct RevertingExtension; + +pub struct DisabledExtension; + impl TestExtension { fn disable() { TEST_EXTENSION.with(|e| e.borrow_mut().enabled = false) @@ -147,7 +151,7 @@ impl ChainExtension for TestExtension { match func_id { 0 => { let mut env = env.buf_in_buf_out(); - let input = env.read(2)?; + let input = env.read(8)?; env.write(&input, false, None)?; TEST_EXTENSION.with(|e| e.borrow_mut().last_seen_buffer = input); Ok(RetVal::Converging(func_id)) @@ -162,7 +166,7 @@ impl ChainExtension for TestExtension { }, 2 => { let mut env = env.buf_in_buf_out(); - let weight = env.read(2)?[1].into(); + let weight = env.read(5)?[4].into(); env.charge_weight(weight)?; Ok(RetVal::Converging(func_id)) }, @@ -178,6 +182,46 @@ impl ChainExtension for TestExtension { } } +impl RegisteredChainExtension for TestExtension { + const ID: u16 = 0; +} + +impl ChainExtension for RevertingExtension { + fn call(_func_id: u32, _env: Environment) -> ExtensionResult + where + E: Ext, + ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>, + { + Ok(RetVal::Diverging { flags: ReturnFlags::REVERT, data: vec![0x4B, 0x1D] }) + } + + fn enabled() -> bool { + TEST_EXTENSION.with(|e| e.borrow().enabled) + } +} + +impl RegisteredChainExtension for RevertingExtension { + const ID: u16 = 1; +} + +impl ChainExtension for DisabledExtension { + fn call(_func_id: u32, _env: Environment) -> ExtensionResult + where + E: Ext, + ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>, + { + panic!("Disabled chain extensions are never called") + } + + fn enabled() -> bool { + false + } +} + +impl RegisteredChainExtension for DisabledExtension { + const ID: u16 = 2; +} + parameter_types! { pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(2 * WEIGHT_PER_SECOND); @@ -281,7 +325,7 @@ impl Config for Test { type CallStack = [Frame; 31]; type WeightPrice = Self; type WeightInfo = (); - type ChainExtension = TestExtension; + type ChainExtension = (TestExtension, DisabledExtension, RevertingExtension); type DeletionQueueDepth = ConstU32<1024>; type DeletionWeightLimit = ConstU64<500_000_000_000>; type Schedule = MySchedule; @@ -1523,6 +1567,23 @@ fn disabled_chain_extension_errors_on_call() { #[test] fn chain_extension_works() { + struct Input<'a> { + extension_id: u16, + func_id: u16, + extra: &'a [u8], + } + + impl<'a> From> for Vec { + fn from(input: Input) -> Vec { + ((input.extension_id as u32) << 16 | (input.func_id as u32)) + .to_le_bytes() + .iter() + .chain(input.extra) + .cloned() + .collect() + } + } + let (code, hash) = compile_module::("chain_extension").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { let min_balance = ::Currency::minimum_balance(); @@ -1543,31 +1604,107 @@ fn chain_extension_works() { // func_id. // 0 = read input buffer and pass it through as output + let input: Vec = Input { extension_id: 0, func_id: 0, extra: &[99] }.into(); let result = - Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![0, 99], false); - let gas_consumed = result.gas_consumed; - assert_eq!(TestExtension::last_seen_buffer(), vec![0, 99]); - assert_eq!(result.result.unwrap().data, Bytes(vec![0, 99])); + Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, input.clone(), false); + assert_eq!(TestExtension::last_seen_buffer(), input); + assert_eq!(result.result.unwrap().data, Bytes(input)); // 1 = treat inputs as integer primitives and store the supplied integers - Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![1], false) - .result - .unwrap(); + Contracts::bare_call( + ALICE, + addr.clone(), + 0, + GAS_LIMIT, + None, + Input { extension_id: 0, func_id: 1, extra: &[] }.into(), + false, + ) + .result + .unwrap(); // those values passed in the fixture - assert_eq!(TestExtension::last_seen_inputs(), (4, 1, 16, 12)); + assert_eq!(TestExtension::last_seen_inputs(), (4, 4, 16, 12)); - // 2 = charge some extra weight (amount supplied in second byte) - let result = - Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![2, 42], false); + // 2 = charge some extra weight (amount supplied in the fifth byte) + let result = Contracts::bare_call( + ALICE, + addr.clone(), + 0, + GAS_LIMIT, + None, + Input { extension_id: 0, func_id: 2, extra: &[0] }.into(), + false, + ); + assert_ok!(result.result); + let gas_consumed = result.gas_consumed; + let result = Contracts::bare_call( + ALICE, + addr.clone(), + 0, + GAS_LIMIT, + None, + Input { extension_id: 0, func_id: 2, extra: &[42] }.into(), + false, + ); assert_ok!(result.result); assert_eq!(result.gas_consumed, gas_consumed + 42); + let result = Contracts::bare_call( + ALICE, + addr.clone(), + 0, + GAS_LIMIT, + None, + Input { extension_id: 0, func_id: 2, extra: &[95] }.into(), + false, + ); + assert_ok!(result.result); + assert_eq!(result.gas_consumed, gas_consumed + 95); // 3 = diverging chain extension call that sets flags to 0x1 and returns a fixed buffer - let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![3], false) - .result - .unwrap(); + let result = Contracts::bare_call( + ALICE, + addr.clone(), + 0, + GAS_LIMIT, + None, + Input { extension_id: 0, func_id: 3, extra: &[] }.into(), + false, + ) + .result + .unwrap(); assert_eq!(result.flags, ReturnFlags::REVERT); assert_eq!(result.data, Bytes(vec![42, 99])); + + // diverging to second chain extension that sets flags to 0x1 and returns a fixed buffer + // We set the MSB part to 1 (instead of 0) which routes the request into the second + // extension + let result = Contracts::bare_call( + ALICE, + addr.clone(), + 0, + GAS_LIMIT, + None, + Input { extension_id: 1, func_id: 0, extra: &[] }.into(), + false, + ) + .result + .unwrap(); + assert_eq!(result.flags, ReturnFlags::REVERT); + assert_eq!(result.data, Bytes(vec![0x4B, 0x1D])); + + // Diverging to third chain extension that is disabled + // We set the MSB part to 2 (instead of 0) which routes the request into the third extension + assert_err_ignore_postinfo!( + Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + Input { extension_id: 2, func_id: 0, extra: &[] }.into(), + ), + Error::::NoChainExtension, + ); }); } From 4400af779a733417c5bba098e30fd2e809af465f Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Fri, 15 Jul 2022 09:42:40 +0200 Subject: [PATCH 12/46] [ci] fix job `cancel-pipeline` (#11844) * [ci] fix cancel-pipeline job * test fail * remove debug --- .gitlab-ci.yml | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4933333b3418..fba7b7a9d393f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -234,23 +234,42 @@ rusty-cachier-notify: # This job cancels the whole pipeline if any of provided jobs fail. # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests # to fail the pipeline as soon as possible to shorten the feedback loop. -cancel-pipeline: +.cancel-pipeline-template: stage: .post + rules: + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + when: on_failure + variables: + PROJECT_ID: "${CI_PROJECT_ID}" + PIPELINE_ID: "${CI_PIPELINE_ID}" + trigger: "parity/infrastructure/ci_cd/pipeline-stopper" + +cancel-pipeline-test-linux-stable: + extends: .cancel-pipeline-template needs: - job: test-linux-stable artifacts: false + +cancel-pipeline-test-linux-stable-int: + extends: .cancel-pipeline-template + needs: - job: test-linux-stable-int artifacts: false + +cancel-pipeline-cargo-check-subkey: + extends: .cancel-pipeline-template + needs: - job: cargo-check-subkey artifacts: false + +cancel-pipeline-cargo-check-benches: + extends: .cancel-pipeline-template + needs: - job: cargo-check-benches artifacts: false + +cancel-pipeline-check-tracing: + extends: .cancel-pipeline-template + needs: - job: check-tracing artifacts: false - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - when: on_failure - variables: - PROJECT_ID: "${CI_PROJECT_ID}" - PIPELINE_ID: "${CI_PIPELINE_ID}" - trigger: "parity/infrastructure/ci_cd/pipeline-stopper" From 6eca5272deb30ca1abe9782ce8918bfb9047728c Mon Sep 17 00:00:00 2001 From: Vlad Date: Fri, 15 Jul 2022 10:58:53 +0300 Subject: [PATCH 13/46] Speed up `rusty-cachier-notify` job (#11841) * Speed up `rusty-cachier-notify` job * Don't forget to setup `rusty-cachier` first --- .gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fba7b7a9d393f..9e80277404549 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -223,10 +223,15 @@ deploy-prometheus-alerting-rules: # This job notifies rusty-cachier about the latest commit with the cache. # This info is later used for the cache distribution and an overlay creation. +# Note that we don't use any .rusty-cachier references as we assume that a pipeline has reached this stage with working rusty-cachier. rusty-cachier-notify: stage: notify - extends: .docker-env + extends: .kubernetes-env + variables: + CI_IMAGE: paritytech/rusty-cachier-env:latest + GIT_STRATEGY: none script: + - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash - rusty-cachier cache notify #### stage: .post From 66d2023925e28d7bc880dad5d38b5a0b0ed03855 Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Fri, 15 Jul 2022 10:44:46 +0200 Subject: [PATCH 14/46] [ci] improvments to make pipeline faster (#11829) * [DO NOT MERGE] test-linux-stable parallel on 3 ci nodes * add debug message * adjust rusty-cachier * empty commit * move test-linux-stable to test.yml * make cargo-check-benches and test-wasmer-sandbox parallel * fix comment * Update scripts/ci/gitlab/pipeline/test.yml Co-authored-by: Denis Pisarev * Update scripts/ci/gitlab/pipeline/test.yml Co-authored-by: Denis Pisarev * if to case * use case instead if in cargo-check-benches * format * add comments to output * add comment * add quotes * Update scripts/ci/gitlab/pipeline/test.yml Co-authored-by: Denis Pisarev Co-authored-by: parity-processbot <> Co-authored-by: Denis Pisarev --- .gitlab-ci.yml | 1 - scripts/ci/gitlab/pipeline/test.yml | 58 ++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e80277404549..fff11e07326d8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -196,7 +196,6 @@ include: # publish jobs - scripts/ci/gitlab/pipeline/publish.yml - #### stage: deploy deploy-prometheus-alerting-rules: diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index cc167410f94a4..dcec896ddc18b 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -74,7 +74,9 @@ cargo-check-nixos: cargo-check-benches: stage: test variables: - RUSTY_CACHIER_TOOLCHAIN: nightly + # Override to use nightly toolchain + RUSTY_CACHIER_TOOLCHAIN: "nightly" + CI_JOB_NAME: "cargo-check-benches" extends: - .docker-env - .test-refs @@ -91,16 +93,25 @@ cargo-check-benches: git config user.email "ci@gitlab.parity.io"; git merge $CI_COMMIT_REF_NAME --verbose --no-edit; fi + parallel: 2 script: - rusty-cachier snapshot create - mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA - - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all - - 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json - | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json' - - 'cargo run --release -p node-bench -- ::trie::read::small --json - | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json' - - sccache -s - - rusty-cachier cache upload + # this job is executed in parallel on two runners + - echo "___Running benchmarks___"; + - case ${CI_NODE_INDEX} in + 1) + SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all; + cargo run --release -p node-bench -- ::trie::read::small --json + | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json; + echo "___Uploading cache for rusty-cachier___"; + rusty-cachier cache upload + ;; + 2) + cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json + | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json + ;; + esac tags: - linux-docker-benches @@ -116,7 +127,7 @@ node-bench-regression-guard: # this is a DAG - job: cargo-check-benches artifacts: true - # this does not like a DAG, just polls the artifact + # polls artifact from master to compare with current result - project: $CI_PROJECT_PATH job: cargo-check-benches ref: master @@ -213,14 +224,27 @@ test-linux-stable: WASM_BUILD_RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" # Ensure we run the UI tests. RUN_UI_TESTS: 1 + # needed for rusty-cachier to keep cache in test-linux-stable folder and not in test-linux-stable-1/3 + CI_JOB_NAME: "test-linux-stable" + parallel: 3 script: - rusty-cachier snapshot create - # TODO: add to paritytech/ci-linux image + # TODO: remove when current paritytech/ci-linux:staging (with rust stable 1.62) is switched to production - time cargo install cargo-nextest # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests + # tests are partitioned by nextest and executed in parallel on $CI_NODE_TOTAL runners # node-cli is excluded until https://github.com/paritytech/substrate/issues/11321 fixed - - time cargo nextest run --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml --exclude node-cli - - rusty-cachier cache upload + - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}" + - time cargo nextest run --workspace + --locked + --release + --verbose + --features runtime-benchmarks + --manifest-path ./bin/node/cli/Cargo.toml + --exclude node-cli + --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} + # we need to update cache only from one job + - if [ ${CI_NODE_INDEX} == 1 ]; then rusty-cachier cache upload; fi test-frame-support: stage: test @@ -356,10 +380,16 @@ test-wasmer-sandbox: extends: - .docker-env - .test-refs-wasmer-sandbox + variables: + CI_JOB_NAME: "test-wasmer-sandbox" + parallel: 3 script: - rusty-cachier snapshot create - - time cargo test --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests - - rusty-cachier cache upload + # TODO: remove when current paritytech/ci-linux:staging (with rust stable 1.62) is switched to production + - cargo install cargo-nextest + - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}" + - time cargo nextest run --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} + - if [ ${CI_NODE_INDEX} == 1 ]; then rusty-cachier cache upload; fi cargo-check-macos: stage: test From 4a19d408095ebcb47840ad1f2f086f53a0020cb7 Mon Sep 17 00:00:00 2001 From: Arkadiy Paronyan Date: Fri, 15 Jul 2022 13:55:18 +0200 Subject: [PATCH 15/46] Fixed sync target detection (#11817) * Fixed sync target detection * Always report sync_target metric * Clamp median across all peers --- client/network/sync/src/lib.rs | 9 +++++++-- client/service/src/metrics.rs | 7 ++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/client/network/sync/src/lib.rs b/client/network/sync/src/lib.rs index aff773bd12ed6..2f837cd6c4f51 100644 --- a/client/network/sync/src/lib.rs +++ b/client/network/sync/src/lib.rs @@ -1719,7 +1719,7 @@ where Ok(sync) } - /// Returns the best seen block. + /// Returns the best seen block number if we don't have that block yet, `None` otherwise. fn best_seen(&self) -> Option> { let mut best_seens = self.peers.values().map(|p| p.best_number).collect::>(); @@ -1729,7 +1729,12 @@ where let middle = best_seens.len() / 2; // Not the "perfect median" when we have an even number of peers. - Some(*best_seens.select_nth_unstable(middle).1) + let median = *best_seens.select_nth_unstable(middle).1; + if median > self.best_queued_number { + Some(median) + } else { + None + } } } diff --git a/client/service/src/metrics.rs b/client/service/src/metrics.rs index 555023f894488..56b145ccdf7f7 100644 --- a/client/service/src/metrics.rs +++ b/client/service/src/metrics.rs @@ -298,9 +298,10 @@ impl MetricsService { UniqueSaturatedInto::::unique_saturated_into(num) }); - if let Some(best_seen_block) = best_seen_block { - metrics.block_height.with_label_values(&["sync_target"]).set(best_seen_block); - } + metrics + .block_height + .with_label_values(&["sync_target"]) + .set(best_seen_block.unwrap_or(best_number)); } } } From ddfd02ef2e71f3b80564ae8807aac5a46f6c7134 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Sun, 17 Jul 2022 00:24:46 +0300 Subject: [PATCH 16/46] Ignore boot and reserved nodes with peer ID the same as in local node itself (#11851) --- client/network/src/service.rs | 45 +++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/client/network/src/service.rs b/client/network/src/service.rs index ef7ef2f5a2deb..0d28b50df1821 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -148,6 +148,47 @@ where /// for the network processing to advance. From it, you can extract a `NetworkService` using /// `worker.service()`. The `NetworkService` can be shared through the codebase. pub fn new(mut params: Params) -> Result { + // Private and public keys configuration. + let local_identity = params.network_config.node_key.clone().into_keypair()?; + let local_public = local_identity.public(); + let local_peer_id = local_public.to_peer_id(); + + params.network_config.boot_nodes = params + .network_config + .boot_nodes + .into_iter() + .filter(|boot_node| { + if boot_node.peer_id == local_peer_id { + warn!( + target: "sub-libp2p", + "Local peer ID used in bootnode, ignoring: {}", + boot_node, + ); + false + } else { + true + } + }) + .collect(); + params.network_config.default_peers_set.reserved_nodes = params + .network_config + .default_peers_set + .reserved_nodes + .into_iter() + .filter(|reserved_node| { + if reserved_node.peer_id == local_peer_id { + warn!( + target: "sub-libp2p", + "Local peer ID used in reserved node, ignoring: {}", + reserved_node, + ); + false + } else { + true + } + }) + .collect(); + // Ensure the listen addresses are consistent with the transport. ensure_addresses_consistent_with_transport( params.network_config.listen_addresses.iter(), @@ -190,10 +231,6 @@ where .extra_sets .insert(0, transactions_handler_proto.set_config()); - // Private and public keys configuration. - let local_identity = params.network_config.node_key.clone().into_keypair()?; - let local_public = local_identity.public(); - let local_peer_id = local_public.to_peer_id(); info!( target: "sub-libp2p", "🏷 Local node identity is: {}", From 6f652840cc78629ac149bf36d23ab34345c65331 Mon Sep 17 00:00:00 2001 From: Sergej Sakac <73715684+Szegoo@users.noreply.github.com> Date: Sun, 17 Jul 2022 00:38:38 +0300 Subject: [PATCH 17/46] removed evaluation.rs (#11766) * removed evaluation.rs * no need for parent_hash * fix --- .../basic-authorship/src/basic_authorship.rs | 12 +-- primitives/consensus/common/src/evaluation.rs | 74 ------------------- primitives/consensus/common/src/lib.rs | 1 - 3 files changed, 1 insertion(+), 86 deletions(-) delete mode 100644 primitives/consensus/common/src/evaluation.rs diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index b67008bc6f44b..bc328c40edb3c 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -34,9 +34,7 @@ use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_INFO}; use sc_transaction_pool_api::{InPoolTransaction, TransactionPool}; use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_blockchain::{ApplyExtrinsicFailed::Validity, Error::ApplyExtrinsicFailed, HeaderBackend}; -use sp_consensus::{ - evaluation, DisableProofRecording, EnableProofRecording, ProofRecording, Proposal, -}; +use sp_consensus::{DisableProofRecording, EnableProofRecording, ProofRecording, Proposal}; use sp_core::traits::SpawnNamed; use sp_inherents::InherentData; use sp_runtime::{ @@ -205,7 +203,6 @@ where let proposer = Proposer::<_, _, _, _, PR> { spawn_handle: self.spawn_handle.clone(), client: self.client.clone(), - parent_hash, parent_id: id, parent_number: *parent_header.number(), transaction_pool: self.transaction_pool.clone(), @@ -250,7 +247,6 @@ where pub struct Proposer { spawn_handle: Box, client: Arc, - parent_hash: ::Hash, parent_id: BlockId, parent_number: <::Header as HeaderT>::Number, transaction_pool: Arc, @@ -536,12 +532,6 @@ where "hash" => ?::Hash::from(block.header().hash()), ); - if let Err(err) = - evaluation::evaluate_initial(&block, &self.parent_hash, self.parent_number) - { - error!("Failed to evaluate authored block: {:?}", err); - } - let proof = PR::into_proof(proof).map_err(|e| sp_blockchain::Error::Application(Box::new(e)))?; diff --git a/primitives/consensus/common/src/evaluation.rs b/primitives/consensus/common/src/evaluation.rs deleted file mode 100644 index d1ce8e9fc5109..0000000000000 --- a/primitives/consensus/common/src/evaluation.rs +++ /dev/null @@ -1,74 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Block evaluation and evaluation errors. - -use codec::Encode; -use sp_runtime::traits::{Block as BlockT, CheckedConversion, Header as HeaderT, One}; - -// This is just a best effort to encode the number. None indicated that it's too big to encode -// in a u128. -type BlockNumber = Option; - -/// Result type alias. -pub type Result = std::result::Result; - -/// Error type. -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error("Failed to verify block encoding/decoding")] - BadBlockCodec, - /// Proposal provided not a block. - #[error("Proposal provided not a block: decoding error: {0}")] - BadProposalFormat(#[from] codec::Error), - /// Proposal had wrong parent hash. - #[error("Proposal had wrong parent hash. Expected {expected:?}, got {got:?}")] - WrongParentHash { expected: String, got: String }, - /// Proposal had wrong number. - #[error("Proposal had wrong number. Expected {expected:?}, got {got:?}")] - WrongNumber { expected: BlockNumber, got: BlockNumber }, -} - -/// Attempt to evaluate a substrate block as a node block, returning error -/// upon any initial validity checks failing. -pub fn evaluate_initial( - proposal: &Block, - parent_hash: &::Hash, - parent_number: <::Header as HeaderT>::Number, -) -> Result<()> { - let encoded = Encode::encode(proposal); - let block = Block::decode(&mut &encoded[..]).map_err(Error::BadProposalFormat)?; - if &block != proposal { - return Err(Error::BadBlockCodec) - } - - if *parent_hash != *block.header().parent_hash() { - return Err(Error::WrongParentHash { - expected: format!("{:?}", *parent_hash), - got: format!("{:?}", block.header().parent_hash()), - }) - } - - if parent_number + One::one() != *block.header().number() { - return Err(Error::WrongNumber { - expected: parent_number.checked_into::().map(|x| x + 1), - got: (*block.header().number()).checked_into::(), - }) - } - - Ok(()) -} diff --git a/primitives/consensus/common/src/lib.rs b/primitives/consensus/common/src/lib.rs index 2743f434c209b..4539cec2c8e0a 100644 --- a/primitives/consensus/common/src/lib.rs +++ b/primitives/consensus/common/src/lib.rs @@ -33,7 +33,6 @@ use sp_state_machine::StorageProof; pub mod block_validation; pub mod error; -pub mod evaluation; mod select_chain; pub use self::error::Error; From 62a7f2a7db20516673af63884fc83ced4f5d29cf Mon Sep 17 00:00:00 2001 From: lumir-mrkva Date: Sun, 17 Jul 2022 21:28:32 +0200 Subject: [PATCH 18/46] assert noop notifies that storage has been mutated (#11805) --- frame/support/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index f2cd7d1e165ec..8e43df82a284c 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -679,7 +679,7 @@ macro_rules! assert_noop { ) => { let h = $crate::storage_root($crate::StateVersion::V1); $crate::assert_err!($x, $y); - assert_eq!(h, $crate::storage_root($crate::StateVersion::V1)); + assert_eq!(h, $crate::storage_root($crate::StateVersion::V1), "storage has been mutated"); }; } @@ -826,6 +826,7 @@ pub mod tests { pub struct Module for enum Call where origin: T::Origin, system=self {} } } + use self::module::Module; decl_storage! { @@ -851,6 +852,7 @@ pub mod tests { } struct Test; + impl Config for Test { type BlockNumber = u32; type Origin = u32; @@ -867,6 +869,7 @@ pub mod tests { trait Sorted { fn sorted(self) -> Self; } + impl Sorted for Vec { fn sorted(mut self) -> Self { self.sort(); From d2c1c5f36faf0950bd0de467a2d38682eab1dc38 Mon Sep 17 00:00:00 2001 From: yjh Date: Mon, 18 Jul 2022 06:43:04 +0800 Subject: [PATCH 19/46] arrange node-template service (#11795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * make template more clear * fmt check_equivocation Co-authored-by: Bastian Köcher --- bin/node-template/node/src/service.rs | 34 ++++++++++++------------ client/consensus/slots/src/aux_schema.rs | 6 ++--- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index f45f914d94f44..ffb2440caa0ed 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -296,24 +296,24 @@ pub fn new_full(mut config: Configuration) -> Result .spawn_blocking("aura", Some("block-authoring"), aura); } - // if the node isn't actively participating in consensus then it doesn't - // need a keystore, regardless of which protocol we use below. - let keystore = - if role.is_authority() { Some(keystore_container.sync_keystore()) } else { None }; - - let grandpa_config = sc_finality_grandpa::Config { - // FIXME #1578 make this available through chainspec - gossip_duration: Duration::from_millis(333), - justification_period: 512, - name: Some(name), - observer_enabled: false, - keystore, - local_role: role, - telemetry: telemetry.as_ref().map(|x| x.handle()), - protocol_name: grandpa_protocol_name, - }; - if enable_grandpa { + // if the node isn't actively participating in consensus then it doesn't + // need a keystore, regardless of which protocol we use below. + let keystore = + if role.is_authority() { Some(keystore_container.sync_keystore()) } else { None }; + + let grandpa_config = sc_finality_grandpa::Config { + // FIXME #1578 make this available through chainspec + gossip_duration: Duration::from_millis(333), + justification_period: 512, + name: Some(name), + observer_enabled: false, + keystore, + local_role: role, + telemetry: telemetry.as_ref().map(|x| x.handle()), + protocol_name: grandpa_protocol_name, + }; + // start the full GRANDPA voter // NOTE: non-authorities could run the GRANDPA observer protocol, but at // this point the full voter should provide better guarantees of block diff --git a/client/consensus/slots/src/aux_schema.rs b/client/consensus/slots/src/aux_schema.rs index eeaec68d369d2..c1d01500ffe47 100644 --- a/client/consensus/slots/src/aux_schema.rs +++ b/client/consensus/slots/src/aux_schema.rs @@ -89,8 +89,8 @@ where // 1) signed by the same voter, if prev_signer == signer { // 2) with different hash - if header.hash() != prev_header.hash() { - return Ok(Some(EquivocationProof { + return if header.hash() != prev_header.hash() { + Ok(Some(EquivocationProof { slot, offender: signer.clone(), first_header: prev_header.clone(), @@ -100,7 +100,7 @@ where // We don't need to continue in case of duplicated header, // since it's already saved and a possible equivocation // would have been detected before. - return Ok(None) + Ok(None) } } } From 64b0daa64d983738229af08f130f20aafc371a17 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 18 Jul 2022 00:57:13 +0200 Subject: [PATCH 20/46] Expose and link trie migration weights (#11775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * expose and link trie migration weights * actually limit to one trait * Run test on dummy weights * fmt Co-authored-by: Bastian Köcher --- frame/state-trie-migration/src/lib.rs | 69 +++++++++++++-------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/frame/state-trie-migration/src/lib.rs b/frame/state-trie-migration/src/lib.rs index 9ac128a0e3108..94f6c1f223b9c 100644 --- a/frame/state-trie-migration/src/lib.rs +++ b/frame/state-trie-migration/src/lib.rs @@ -56,6 +56,7 @@ #![cfg_attr(not(feature = "std"), no_std)] pub use pallet::*; +pub mod weights; const LOG_TARGET: &str = "runtime::state-trie-migration"; @@ -71,6 +72,9 @@ macro_rules! log { #[frame_support::pallet] pub mod pallet { + + pub use crate::weights::WeightInfo; + use frame_support::{ dispatch::{DispatchErrorWithPostInfo, PostDispatchInfo}, ensure, @@ -90,41 +94,6 @@ pub mod pallet { pub(crate) type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; - /// The weight information of this pallet. - pub trait WeightInfo { - fn process_top_key(x: u32) -> Weight; - fn continue_migrate() -> Weight; - fn continue_migrate_wrong_witness() -> Weight; - fn migrate_custom_top_fail() -> Weight; - fn migrate_custom_top_success() -> Weight; - fn migrate_custom_child_fail() -> Weight; - fn migrate_custom_child_success() -> Weight; - } - - impl WeightInfo for () { - fn process_top_key(_: u32) -> Weight { - 1000000 - } - fn continue_migrate() -> Weight { - 1000000 - } - fn continue_migrate_wrong_witness() -> Weight { - 1000000 - } - fn migrate_custom_top_fail() -> Weight { - 1000000 - } - fn migrate_custom_top_success() -> Weight { - 1000000 - } - fn migrate_custom_child_fail() -> Weight { - 1000000 - } - fn migrate_custom_child_success() -> Weight { - 1000000 - } - } - /// The progress of either the top or child keys. #[derive( CloneNoBound, @@ -1072,6 +1041,7 @@ mod mock { use frame_support::{ parameter_types, traits::{ConstU32, ConstU64, Hooks}, + weights::Weight, }; use frame_system::{EnsureRoot, EnsureSigned}; use sp_core::{ @@ -1148,6 +1118,33 @@ mod mock { type WeightInfo = (); } + /// Test only Weights for state migration. + pub struct StateMigrationTestWeight; + + impl WeightInfo for StateMigrationTestWeight { + fn process_top_key(_: u32) -> Weight { + 1000000 + } + fn continue_migrate() -> Weight { + 1000000 + } + fn continue_migrate_wrong_witness() -> Weight { + 1000000 + } + fn migrate_custom_top_fail() -> Weight { + 1000000 + } + fn migrate_custom_top_success() -> Weight { + 1000000 + } + fn migrate_custom_child_fail() -> Weight { + 1000000 + } + fn migrate_custom_child_success() -> Weight { + 1000000 + } + } + impl pallet_state_trie_migration::Config for Test { type Event = Event; type ControlOrigin = EnsureRoot; @@ -1156,7 +1153,7 @@ mod mock { type SignedDepositPerItem = SignedDepositPerItem; type SignedDepositBase = SignedDepositBase; type SignedFilter = EnsureSigned; - type WeightInfo = (); + type WeightInfo = StateMigrationTestWeight; } pub fn new_test_ext( From 8016da9ddf1c10a5a74cc5a273905cfeb60ed82c Mon Sep 17 00:00:00 2001 From: Koute Date: Mon, 18 Jul 2022 20:51:38 +0900 Subject: [PATCH 21/46] Improve `wasmtime` error reporting (#11856) * Improve `wasmtime` error reporting * cargo fmt --- client/executor/wasmtime/src/imports.rs | 2 +- .../executor/wasmtime/src/instance_wrapper.rs | 7 +++--- client/executor/wasmtime/src/runtime.rs | 24 +++++++++---------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/client/executor/wasmtime/src/imports.rs b/client/executor/wasmtime/src/imports.rs index fae8d75854ef3..c80952a2541ce 100644 --- a/client/executor/wasmtime/src/imports.rs +++ b/client/executor/wasmtime/src/imports.rs @@ -112,7 +112,7 @@ impl<'a, 'b> sp_wasm_interface::HostFunctionRegistry for Registry<'a, 'b> { if self.pending_func_imports.remove(fn_name).is_some() { self.linker.func_wrap("env", fn_name, func).map_err(|error| { WasmError::Other(format!( - "failed to register host function '{}' with the WASM linker: {}", + "failed to register host function '{}' with the WASM linker: {:#}", fn_name, error )) })?; diff --git a/client/executor/wasmtime/src/instance_wrapper.rs b/client/executor/wasmtime/src/instance_wrapper.rs index 6abcbca1bba6f..5d272accd3524 100644 --- a/client/executor/wasmtime/src/instance_wrapper.rs +++ b/client/executor/wasmtime/src/instance_wrapper.rs @@ -186,9 +186,10 @@ impl InstanceWrapper { ) -> Result { let mut store = create_store(engine, max_memory_size); let instance = instance_pre.instantiate(&mut store).map_err(|error| { - WasmError::Other( - format!("failed to instantiate a new WASM module instance: {}", error,), - ) + WasmError::Other(format!( + "failed to instantiate a new WASM module instance: {:#}", + error, + )) })?; let memory = get_linear_memory(&instance, &mut store)?; diff --git a/client/executor/wasmtime/src/runtime.rs b/client/executor/wasmtime/src/runtime.rs index c63a802df07cc..fb5c6cb16993a 100644 --- a/client/executor/wasmtime/src/runtime.rs +++ b/client/executor/wasmtime/src/runtime.rs @@ -257,12 +257,12 @@ fn setup_wasmtime_caching( let wasmtime_cache_root = cache_path.join("wasmtime"); fs::create_dir_all(&wasmtime_cache_root) - .map_err(|err| format!("cannot create the dirs to cache: {:?}", err))?; + .map_err(|err| format!("cannot create the dirs to cache: {}", err))?; // Canonicalize the path after creating the directories. let wasmtime_cache_root = wasmtime_cache_root .canonicalize() - .map_err(|err| format!("failed to canonicalize the path: {:?}", err))?; + .map_err(|err| format!("failed to canonicalize the path: {}", err))?; // Write the cache config file let cache_config_path = wasmtime_cache_root.join("cache-config.toml"); @@ -275,11 +275,11 @@ directory = \"{cache_dir}\" cache_dir = wasmtime_cache_root.display() ); fs::write(&cache_config_path, config_content) - .map_err(|err| format!("cannot write the cache config: {:?}", err))?; + .map_err(|err| format!("cannot write the cache config: {}", err))?; config .cache_config_load(cache_config_path) - .map_err(|err| format!("failed to parse the config: {:?}", err))?; + .map_err(|err| format!("failed to parse the config: {:#}", err))?; Ok(()) } @@ -304,14 +304,14 @@ fn common_config(semantics: &Semantics) -> std::result::Result { @@ -626,7 +626,7 @@ where let serialized_blob = blob.clone().serialize(); let module = wasmtime::Module::new(&engine, &serialized_blob) - .map_err(|e| WasmError::Other(format!("cannot create module: {}", e)))?; + .map_err(|e| WasmError::Other(format!("cannot create module: {:#}", e)))?; match config.semantics.instantiation_strategy { InstantiationStrategy::LegacyInstanceReuse => { @@ -664,7 +664,7 @@ where // // See [`create_runtime_from_artifact`] for more details. let module = wasmtime::Module::deserialize_file(&engine, compiled_artifact_path) - .map_err(|e| WasmError::Other(format!("cannot deserialize module: {}", e)))?; + .map_err(|e| WasmError::Other(format!("cannot deserialize module: {:#}", e)))?; (module, InternalInstantiationStrategy::Builtin) }, @@ -677,7 +677,7 @@ where crate::instance_wrapper::create_store(module.engine(), config.semantics.max_memory_size); let instance_pre = linker .instantiate_pre(&mut store, &module) - .map_err(|e| WasmError::Other(format!("cannot preinstantiate module: {}", e)))?; + .map_err(|e| WasmError::Other(format!("cannot preinstantiate module: {:#}", e)))?; Ok(WasmtimeRuntime { engine, @@ -729,11 +729,11 @@ pub fn prepare_runtime_artifact( let blob = prepare_blob_for_compilation(blob, &semantics)?; let engine = Engine::new(&common_config(&semantics)?) - .map_err(|e| WasmError::Other(format!("cannot create the engine: {}", e)))?; + .map_err(|e| WasmError::Other(format!("cannot create the engine: {:#}", e)))?; engine .precompile_module(&blob.serialize()) - .map_err(|e| WasmError::Other(format!("cannot precompile module: {}", e))) + .map_err(|e| WasmError::Other(format!("cannot precompile module: {:#}", e))) } fn perform_call( From 32bc42ba64f88a8ea65734ba9a404779fafbc0df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20St=C3=BCber?= Date: Mon, 18 Jul 2022 14:56:22 +0200 Subject: [PATCH 22/46] Fix invalid state transitions in pallet-bounties (#11630) * Pallet-bounty: disallow invalid state transitions * Fix tests: funding only at even block numbers * Fix benchmarks: bounties need to be funded * fix on_initialize Co-authored-by: Shawn Tabrizi --- frame/bounties/src/lib.rs | 2 +- frame/bounties/src/tests.rs | 8 ++++---- frame/child-bounties/src/benchmarking.rs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/frame/bounties/src/lib.rs b/frame/bounties/src/lib.rs index 8e28cacb9d929..fca758fd96b8e 100644 --- a/frame/bounties/src/lib.rs +++ b/frame/bounties/src/lib.rs @@ -390,7 +390,7 @@ pub mod pallet { Bounties::::try_mutate_exists(bounty_id, |maybe_bounty| -> DispatchResult { let mut bounty = maybe_bounty.as_mut().ok_or(Error::::InvalidIndex)?; match bounty.status { - BountyStatus::Proposed | BountyStatus::Approved | BountyStatus::Funded => {}, + BountyStatus::Funded => {}, _ => return Err(Error::::UnexpectedStatus.into()), }; diff --git a/frame/bounties/src/tests.rs b/frame/bounties/src/tests.rs index ff220600794d4..b4ce039b35fbc 100644 --- a/frame/bounties/src/tests.rs +++ b/frame/bounties/src/tests.rs @@ -1126,8 +1126,8 @@ fn accept_curator_handles_different_deposit_calculations() { assert_ok!(Bounties::propose_bounty(Origin::signed(0), value, b"12345".to_vec())); assert_ok!(Bounties::approve_bounty(Origin::root(), bounty_index)); - System::set_block_number(3); - >::on_initialize(3); + System::set_block_number(4); + >::on_initialize(4); assert_ok!(Bounties::propose_curator(Origin::root(), bounty_index, user, fee)); assert_ok!(Bounties::accept_curator(Origin::signed(user), bounty_index)); @@ -1150,8 +1150,8 @@ fn accept_curator_handles_different_deposit_calculations() { assert_ok!(Bounties::propose_bounty(Origin::signed(0), value, b"12345".to_vec())); assert_ok!(Bounties::approve_bounty(Origin::root(), bounty_index)); - System::set_block_number(3); - >::on_initialize(3); + System::set_block_number(6); + >::on_initialize(6); assert_ok!(Bounties::propose_curator(Origin::root(), bounty_index, user, fee)); assert_ok!(Bounties::accept_curator(Origin::signed(user), bounty_index)); diff --git a/frame/child-bounties/src/benchmarking.rs b/frame/child-bounties/src/benchmarking.rs index dcb54361fac89..ca5af50276b9d 100644 --- a/frame/child-bounties/src/benchmarking.rs +++ b/frame/child-bounties/src/benchmarking.rs @@ -221,7 +221,7 @@ benchmarks! { unassign_curator { setup_pot_account::(); let bounty_setup = activate_child_bounty::(0, T::MaximumReasonLength::get())?; - Bounties::::on_initialize(T::BlockNumber::zero()); + Treasury::::on_initialize(T::BlockNumber::zero()); frame_system::Pallet::::set_block_number(T::BountyUpdatePeriod::get() + 1u32.into()); let caller = whitelisted_caller(); }: _(RawOrigin::Signed(caller), bounty_setup.bounty_id, @@ -295,7 +295,7 @@ benchmarks! { close_child_bounty_active { setup_pot_account::(); let bounty_setup = activate_child_bounty::(0, T::MaximumReasonLength::get())?; - Bounties::::on_initialize(T::BlockNumber::zero()); + Treasury::::on_initialize(T::BlockNumber::zero()); }: close_child_bounty(RawOrigin::Root, bounty_setup.bounty_id, bounty_setup.child_bounty_id) verify { assert_last_event::(Event::Canceled { From a434d07ab18982eceff528c6e7f1e95a6f71bea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 18 Jul 2022 21:25:18 +0200 Subject: [PATCH 23/46] Disable the interest cache (#11854) * Disable the interest cache The feature is currently broken with the latest `tracing-core`. We will enable it again, when it is fixed upstream. * FMT --- Cargo.lock | 8 +++----- client/tracing/Cargo.toml | 2 +- client/tracing/src/logging/mod.rs | 5 +---- primitives/io/Cargo.toml | 2 +- primitives/runtime-interface/test/Cargo.toml | 2 +- primitives/tracing/Cargo.toml | 2 +- 6 files changed, 8 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 791d8037f3335..279864e056b0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11066,11 +11066,11 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.26" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7" dependencies = [ - "lazy_static", + "once_cell", "valuable", ] @@ -11090,10 +11090,8 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ - "ahash", "lazy_static", "log", - "lru", "tracing-core", ] diff --git a/client/tracing/Cargo.toml b/client/tracing/Cargo.toml index 3f0bbbe922d3f..476e03ee741f3 100644 --- a/client/tracing/Cargo.toml +++ b/client/tracing/Cargo.toml @@ -26,7 +26,7 @@ rustc-hash = "1.1.0" serde = "1.0.136" thiserror = "1.0.30" tracing = "0.1.29" -tracing-log = { version = "0.1.3", features = ["interest-cache"] } +tracing-log = "0.1.3" tracing-subscriber = { version = "0.2.25", features = ["parking_lot"] } sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-rpc-server = { version = "4.0.0-dev", path = "../rpc-servers" } diff --git a/client/tracing/src/logging/mod.rs b/client/tracing/src/logging/mod.rs index 33c83dd87189e..58941617bfb6a 100644 --- a/client/tracing/src/logging/mod.rs +++ b/client/tracing/src/logging/mod.rs @@ -155,10 +155,7 @@ where let max_level_hint = Layer::::max_level_hint(&env_filter); let max_level = to_log_level_filter(max_level_hint); - tracing_log::LogTracer::builder() - .with_max_level(max_level) - .with_interest_cache(tracing_log::InterestCacheConfig::default()) - .init()?; + tracing_log::LogTracer::builder().with_max_level(max_level).init()?; // If we're only logging `INFO` entries then we'll use a simplified logging format. let detailed_output = match max_level_hint { diff --git a/primitives/io/Cargo.toml b/primitives/io/Cargo.toml index 09a087d509416..03cd8535665ac 100644 --- a/primitives/io/Cargo.toml +++ b/primitives/io/Cargo.toml @@ -32,7 +32,7 @@ futures = { version = "0.3.21", features = ["thread-pool"], optional = true } parking_lot = { version = "0.12.0", optional = true } secp256k1 = { version = "0.21.2", features = ["recovery", "global-context"], optional = true } tracing = { version = "0.1.29", default-features = false } -tracing-core = { version = "0.1.26", default-features = false} +tracing-core = { version = "0.1.28", default-features = false} [features] default = ["std"] diff --git a/primitives/runtime-interface/test/Cargo.toml b/primitives/runtime-interface/test/Cargo.toml index e897fc0bab71c..880d03902b421 100644 --- a/primitives/runtime-interface/test/Cargo.toml +++ b/primitives/runtime-interface/test/Cargo.toml @@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] tracing = "0.1.29" -tracing-core = "0.1.26" +tracing-core = "0.1.28" sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" } sc-executor-common = { version = "0.10.0-dev", path = "../../../client/executor/common" } sp-io = { version = "6.0.0", path = "../../io" } diff --git a/primitives/tracing/Cargo.toml b/primitives/tracing/Cargo.toml index d305b756e2d68..c2ca57d2b5a43 100644 --- a/primitives/tracing/Cargo.toml +++ b/primitives/tracing/Cargo.toml @@ -23,7 +23,7 @@ codec = { version = "3.0.0", package = "parity-scale-codec", default-features = "derive", ] } tracing = { version = "0.1.29", default-features = false } -tracing-core = { version = "0.1.26", default-features = false } +tracing-core = { version = "0.1.28", default-features = false } tracing-subscriber = { version = "0.2.25", optional = true, features = [ "tracing-log", ] } From ced41695aafa648ebdb01305e563fa7ca89756bc Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Tue, 19 Jul 2022 08:10:15 +0200 Subject: [PATCH 24/46] Add `benchmark extrinsic` command (#11456) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Benchmark extrinsic Signed-off-by: Oliver Tale-Yazdi * Reduce warmup and repeat Running this 1000 times with a full block takes ~33 minutes 🙈. Reducing it to ~3 minutes per default. Signed-off-by: Oliver Tale-Yazdi * Make ExistentialDeposit public Signed-off-by: Oliver Tale-Yazdi * Add 'bechmark extrinsic' command Signed-off-by: Oliver Tale-Yazdi * fmt Signed-off-by: Oliver Tale-Yazdi * Add --list and cleanup Signed-off-by: Oliver Tale-Yazdi * Unrelated Clippy Signed-off-by: Oliver Tale-Yazdi * Clippy Signed-off-by: Oliver Tale-Yazdi * Fix tests and doc Signed-off-by: Oliver Tale-Yazdi * Move implementations up + fmt Signed-off-by: Oliver Tale-Yazdi * Dont use parameter_types macro Signed-off-by: Oliver Tale-Yazdi * Cache to_lowercase() call The .to_lowercase() on the builder is actually not needes since its already documented to only return lower case. Signed-off-by: Oliver Tale-Yazdi * Spelling Signed-off-by: Oliver Tale-Yazdi * Use correct nightly for fmt... Signed-off-by: Oliver Tale-Yazdi * Rename ED Signed-off-by: Oliver Tale-Yazdi * Fix compile Signed-off-by: Oliver Tale-Yazdi * Subtract block base weight Signed-off-by: Oliver Tale-Yazdi * Fixes Signed-off-by: Oliver Tale-Yazdi * Use tmp folder for test This should already be the case since --dev is passed but somehow not... Signed-off-by: Oliver Tale-Yazdi * Fix test Signed-off-by: Oliver Tale-Yazdi --- .../{command_helper.rs => benchmarking.rs} | 66 ++++++++- bin/node-template/node/src/command.rs | 26 +++- bin/node-template/node/src/main.rs | 2 +- bin/node-template/runtime/src/lib.rs | 5 +- .../{command_helper.rs => benchmarking.rs} | 70 +++++++-- bin/node/cli/src/command.rs | 23 ++- bin/node/cli/src/lib.rs | 4 +- .../cli/tests/benchmark_extrinsic_works.rs | 46 ++++++ .../src/{overhead => extrinsic}/bench.rs | 107 +++++++------- .../benchmarking-cli/src/extrinsic/cmd.rs | 134 ++++++++++++++++++ .../src/extrinsic/extrinsic_factory.rs | 70 +++++++++ .../benchmarking-cli/src/extrinsic/mod.rs | 27 ++++ utils/frame/benchmarking-cli/src/lib.rs | 7 +- .../benchmarking-cli/src/overhead/cmd.rs | 52 +++++-- .../benchmarking-cli/src/overhead/mod.rs | 5 +- .../benchmarking-cli/src/overhead/template.rs | 2 +- 16 files changed, 537 insertions(+), 109 deletions(-) rename bin/node-template/node/src/{command_helper.rs => benchmarking.rs} (71%) rename bin/node/cli/src/{command_helper.rs => benchmarking.rs} (52%) create mode 100644 bin/node/cli/tests/benchmark_extrinsic_works.rs rename utils/frame/benchmarking-cli/src/{overhead => extrinsic}/bench.rs (70%) create mode 100644 utils/frame/benchmarking-cli/src/extrinsic/cmd.rs create mode 100644 utils/frame/benchmarking-cli/src/extrinsic/extrinsic_factory.rs create mode 100644 utils/frame/benchmarking-cli/src/extrinsic/mod.rs diff --git a/bin/node-template/node/src/command_helper.rs b/bin/node-template/node/src/benchmarking.rs similarity index 71% rename from bin/node-template/node/src/command_helper.rs rename to bin/node-template/node/src/benchmarking.rs index 287e81b1e96bd..f0e32104cd3ee 100644 --- a/bin/node-template/node/src/command_helper.rs +++ b/bin/node-template/node/src/benchmarking.rs @@ -16,13 +16,14 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Contains code to setup the command invocations in [`super::command`] which would -//! otherwise bloat that module. +//! Setup code for [`super::command`] which would otherwise bloat that module. +//! +//! Should only be used for benchmarking as it may break in other contexts. use crate::service::FullClient; use node_template_runtime as runtime; -use runtime::SystemCall; +use runtime::{AccountId, Balance, BalancesCall, SystemCall}; use sc_cli::Result; use sc_client_api::BlockBackend; use sp_core::{Encode, Pair}; @@ -35,19 +36,27 @@ use std::{sync::Arc, time::Duration}; /// Generates extrinsics for the `benchmark overhead` command. /// /// Note: Should only be used for benchmarking. -pub struct BenchmarkExtrinsicBuilder { +pub struct RemarkBuilder { client: Arc, } -impl BenchmarkExtrinsicBuilder { +impl RemarkBuilder { /// Creates a new [`Self`] from the given client. pub fn new(client: Arc) -> Self { Self { client } } } -impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { - fn remark(&self, nonce: u32) -> std::result::Result { +impl frame_benchmarking_cli::ExtrinsicBuilder for RemarkBuilder { + fn pallet(&self) -> &str { + "system" + } + + fn extrinsic(&self) -> &str { + "remark" + } + + fn build(&self, nonce: u32) -> std::result::Result { let acc = Sr25519Keyring::Bob.pair(); let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic( self.client.as_ref(), @@ -61,6 +70,49 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { } } +/// Generates `Balances::TransferKeepAlive` extrinsics for the benchmarks. +/// +/// Note: Should only be used for benchmarking. +pub struct TransferKeepAliveBuilder { + client: Arc, + dest: AccountId, + value: Balance, +} + +impl TransferKeepAliveBuilder { + /// Creates a new [`Self`] from the given client. + pub fn new(client: Arc, dest: AccountId, value: Balance) -> Self { + Self { client, dest, value } + } +} + +impl frame_benchmarking_cli::ExtrinsicBuilder for TransferKeepAliveBuilder { + fn pallet(&self) -> &str { + "balances" + } + + fn extrinsic(&self) -> &str { + "transfer_keep_alive" + } + + fn build(&self, nonce: u32) -> std::result::Result { + let acc = Sr25519Keyring::Bob.pair(); + let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic( + self.client.as_ref(), + acc, + BalancesCall::transfer_keep_alive { + dest: self.dest.clone().into(), + value: self.value.into(), + } + .into(), + nonce, + ) + .into(); + + Ok(extrinsic) + } +} + /// Create a transaction using the given `call`. /// /// Note: Should only be used for benchmarking. diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index e3e10007929e6..142f0b40c325e 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -1,14 +1,14 @@ use crate::{ + benchmarking::{inherent_benchmark_data, RemarkBuilder, TransferKeepAliveBuilder}, chain_spec, cli::{Cli, Subcommand}, - command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder}, service, }; -use frame_benchmarking_cli::{BenchmarkCmd, SUBSTRATE_REFERENCE_HARDWARE}; -use node_template_runtime::Block; +use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE}; +use node_template_runtime::{Block, EXISTENTIAL_DEPOSIT}; use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; -use std::sync::Arc; +use sp_keyring::Sr25519Keyring; impl SubstrateCli for Cli { fn impl_name() -> String { @@ -137,9 +137,23 @@ pub fn run() -> sc_cli::Result<()> { }, BenchmarkCmd::Overhead(cmd) => { let PartialComponents { client, .. } = service::new_partial(&config)?; - let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); + let ext_builder = RemarkBuilder::new(client.clone()); - cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) + cmd.run(config, client, inherent_benchmark_data()?, &ext_builder) + }, + BenchmarkCmd::Extrinsic(cmd) => { + let PartialComponents { client, .. } = service::new_partial(&config)?; + // Register the *Remark* and *TKA* builders. + let ext_factory = ExtrinsicFactory(vec![ + Box::new(RemarkBuilder::new(client.clone())), + Box::new(TransferKeepAliveBuilder::new( + client.clone(), + Sr25519Keyring::Alice.to_account_id(), + EXISTENTIAL_DEPOSIT, + )), + ]); + + cmd.run(client, inherent_benchmark_data()?, &ext_factory) }, BenchmarkCmd::Machine(cmd) => cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()), diff --git a/bin/node-template/node/src/main.rs b/bin/node-template/node/src/main.rs index 0f2fbd5a909c6..426cbabb6fbf7 100644 --- a/bin/node-template/node/src/main.rs +++ b/bin/node-template/node/src/main.rs @@ -4,9 +4,9 @@ mod chain_spec; #[macro_use] mod service; +mod benchmarking; mod cli; mod command; -mod command_helper; mod rpc; fn main() -> sc_cli::Result<()> { diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index c514cdf6c25fd..a2f595f571a90 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -234,6 +234,9 @@ impl pallet_timestamp::Config for Runtime { type WeightInfo = (); } +/// Existential deposit. +pub const EXISTENTIAL_DEPOSIT: u128 = 500; + impl pallet_balances::Config for Runtime { type MaxLocks = ConstU32<50>; type MaxReserves = (); @@ -243,7 +246,7 @@ impl pallet_balances::Config for Runtime { /// The ubiquitous event type. type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ConstU128<500>; + type ExistentialDeposit = ConstU128; type AccountStore = System; type WeightInfo = pallet_balances::weights::SubstrateWeight; } diff --git a/bin/node/cli/src/command_helper.rs b/bin/node/cli/src/benchmarking.rs similarity index 52% rename from bin/node/cli/src/command_helper.rs rename to bin/node/cli/src/benchmarking.rs index 84d85ee367cab..52657016c046a 100644 --- a/bin/node/cli/src/command_helper.rs +++ b/bin/node/cli/src/benchmarking.rs @@ -16,12 +16,14 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Contains code to setup the command invocations in [`super::command`] which would -//! otherwise bloat that module. +//! Setup code for [`super::command`] which would otherwise bloat that module. +//! +//! Should only be used for benchmarking as it may break in other contexts. use crate::service::{create_extrinsic, FullClient}; -use node_runtime::SystemCall; +use node_primitives::{AccountId, Balance}; +use node_runtime::{BalancesCall, SystemCall}; use sc_cli::Result; use sp_inherents::{InherentData, InherentDataProvider}; use sp_keyring::Sr25519Keyring; @@ -29,20 +31,30 @@ use sp_runtime::OpaqueExtrinsic; use std::{sync::Arc, time::Duration}; -/// Generates extrinsics for the `benchmark overhead` command. -pub struct BenchmarkExtrinsicBuilder { +/// Generates `System::Remark` extrinsics for the benchmarks. +/// +/// Note: Should only be used for benchmarking. +pub struct RemarkBuilder { client: Arc, } -impl BenchmarkExtrinsicBuilder { +impl RemarkBuilder { /// Creates a new [`Self`] from the given client. pub fn new(client: Arc) -> Self { Self { client } } } -impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { - fn remark(&self, nonce: u32) -> std::result::Result { +impl frame_benchmarking_cli::ExtrinsicBuilder for RemarkBuilder { + fn pallet(&self) -> &str { + "system" + } + + fn extrinsic(&self) -> &str { + "remark" + } + + fn build(&self, nonce: u32) -> std::result::Result { let acc = Sr25519Keyring::Bob.pair(); let extrinsic: OpaqueExtrinsic = create_extrinsic( self.client.as_ref(), @@ -56,6 +68,48 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { } } +/// Generates `Balances::TransferKeepAlive` extrinsics for the benchmarks. +/// +/// Note: Should only be used for benchmarking. +pub struct TransferKeepAliveBuilder { + client: Arc, + dest: AccountId, + value: Balance, +} + +impl TransferKeepAliveBuilder { + /// Creates a new [`Self`] from the given client. + pub fn new(client: Arc, dest: AccountId, value: Balance) -> Self { + Self { client, dest, value } + } +} + +impl frame_benchmarking_cli::ExtrinsicBuilder for TransferKeepAliveBuilder { + fn pallet(&self) -> &str { + "balances" + } + + fn extrinsic(&self) -> &str { + "transfer_keep_alive" + } + + fn build(&self, nonce: u32) -> std::result::Result { + let acc = Sr25519Keyring::Bob.pair(); + let extrinsic: OpaqueExtrinsic = create_extrinsic( + self.client.as_ref(), + acc, + BalancesCall::transfer_keep_alive { + dest: self.dest.clone().into(), + value: self.value.into(), + }, + Some(nonce), + ) + .into(); + + Ok(extrinsic) + } +} + /// Generates inherent data for the `benchmark overhead` command. pub fn inherent_benchmark_data() -> Result { let mut inherent_data = InherentData::new(); diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index b17a26fa02935..df3d5a891e2ab 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use super::command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder}; +use super::benchmarking::{inherent_benchmark_data, RemarkBuilder, TransferKeepAliveBuilder}; use crate::{ chain_spec, service, service::{new_partial, FullClient}, @@ -25,9 +25,10 @@ use crate::{ use frame_benchmarking_cli::*; use node_executor::ExecutorDispatch; use node_primitives::Block; -use node_runtime::RuntimeApi; +use node_runtime::{ExistentialDeposit, RuntimeApi}; use sc_cli::{ChainSpec, Result, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; +use sp_keyring::Sr25519Keyring; use std::sync::Arc; @@ -126,9 +127,23 @@ pub fn run() -> Result<()> { }, BenchmarkCmd::Overhead(cmd) => { let PartialComponents { client, .. } = new_partial(&config)?; - let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); + let ext_builder = RemarkBuilder::new(client.clone()); - cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) + cmd.run(config, client, inherent_benchmark_data()?, &ext_builder) + }, + BenchmarkCmd::Extrinsic(cmd) => { + let PartialComponents { client, .. } = service::new_partial(&config)?; + // Register the *Remark* and *TKA* builders. + let ext_factory = ExtrinsicFactory(vec![ + Box::new(RemarkBuilder::new(client.clone())), + Box::new(TransferKeepAliveBuilder::new( + client.clone(), + Sr25519Keyring::Alice.to_account_id(), + ExistentialDeposit::get(), + )), + ]); + + cmd.run(client, inherent_benchmark_data()?, &ext_factory) }, BenchmarkCmd::Machine(cmd) => cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()), diff --git a/bin/node/cli/src/lib.rs b/bin/node/cli/src/lib.rs index 06c0bcccbc296..13c074268e50f 100644 --- a/bin/node/cli/src/lib.rs +++ b/bin/node/cli/src/lib.rs @@ -35,11 +35,11 @@ pub mod chain_spec; #[macro_use] pub mod service; #[cfg(feature = "cli")] +mod benchmarking; +#[cfg(feature = "cli")] mod cli; #[cfg(feature = "cli")] mod command; -#[cfg(feature = "cli")] -mod command_helper; #[cfg(feature = "cli")] pub use cli::*; diff --git a/bin/node/cli/tests/benchmark_extrinsic_works.rs b/bin/node/cli/tests/benchmark_extrinsic_works.rs new file mode 100644 index 0000000000000..69800ad3c6c13 --- /dev/null +++ b/bin/node/cli/tests/benchmark_extrinsic_works.rs @@ -0,0 +1,46 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; + +/// Tests that the `benchmark extrinsic` command works for +/// remark and transfer_keep_alive within the substrate dev runtime. +#[test] +fn benchmark_extrinsic_works() { + benchmark_extrinsic("system", "remark"); + benchmark_extrinsic("balances", "transfer_keep_alive"); +} + +/// Checks that the `benchmark extrinsic` command works for the given pallet and extrinsic. +fn benchmark_extrinsic(pallet: &str, extrinsic: &str) { + let base_dir = tempdir().expect("could not create a temp dir"); + + let status = Command::new(cargo_bin("substrate")) + .args(&["benchmark", "extrinsic", "--dev"]) + .arg("-d") + .arg(base_dir.path()) + .args(&["--pallet", pallet, "--extrinsic", extrinsic]) + // Run with low repeats for faster execution. + .args(["--warmup=10", "--repeat=10", "--max-ext-per-block=10"]) + .status() + .unwrap(); + + assert!(status.success()); +} diff --git a/utils/frame/benchmarking-cli/src/overhead/bench.rs b/utils/frame/benchmarking-cli/src/extrinsic/bench.rs similarity index 70% rename from utils/frame/benchmarking-cli/src/overhead/bench.rs rename to utils/frame/benchmarking-cli/src/extrinsic/bench.rs index be7dac24021cf..27fc40fb34774 100644 --- a/utils/frame/benchmarking-cli/src/overhead/bench.rs +++ b/utils/frame/benchmarking-cli/src/extrinsic/bench.rs @@ -36,8 +36,8 @@ use log::info; use serde::Serialize; use std::{marker::PhantomData, sync::Arc, time::Instant}; -use super::cmd::ExtrinsicBuilder; -use crate::shared::Stats; +use super::ExtrinsicBuilder; +use crate::shared::{StatSelect, Stats}; /// Parameters to configure an *overhead* benchmark. #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] @@ -60,21 +60,11 @@ pub struct BenchmarkParams { /// The results of multiple runs in nano seconds. pub(crate) type BenchRecord = Vec; -/// Type of a benchmark. -#[derive(Serialize, Clone, PartialEq, Copy)] -pub(crate) enum BenchmarkType { - /// Measure the per-extrinsic execution overhead. - Extrinsic, - /// Measure the per-block execution overhead. - Block, -} - /// Holds all objects needed to run the *overhead* benchmarks. pub(crate) struct Benchmark { client: Arc, params: BenchmarkParams, inherent_data: sp_inherents::InherentData, - ext_builder: Arc, _p: PhantomData<(Block, BA)>, } @@ -90,23 +80,51 @@ where client: Arc, params: BenchmarkParams, inherent_data: sp_inherents::InherentData, - ext_builder: Arc, ) -> Self { - Self { client, params, inherent_data, ext_builder, _p: PhantomData } + Self { client, params, inherent_data, _p: PhantomData } } - /// Run the specified benchmark. - pub fn bench(&self, bench_type: BenchmarkType) -> Result { - let (block, num_ext) = self.build_block(bench_type)?; - let record = self.measure_block(&block, num_ext, bench_type)?; + /// Benchmark a block with only inherents. + pub fn bench_block(&self) -> Result { + let (block, _) = self.build_block(None)?; + let record = self.measure_block(&block)?; Stats::new(&record) } - /// Builds a block for the given benchmark type. + /// Benchmark the time of an extrinsic in a full block. + /// + /// First benchmarks an empty block, analogous to `bench_block` and use it as baseline. + /// Then benchmarks a full block built with the given `ext_builder` and subtracts the baseline + /// from the result. + /// This is necessary to account for the time the inherents use. + pub fn bench_extrinsic(&self, ext_builder: &dyn ExtrinsicBuilder) -> Result { + let (block, _) = self.build_block(None)?; + let base = self.measure_block(&block)?; + let base_time = Stats::new(&base)?.select(StatSelect::Average); + + let (block, num_ext) = self.build_block(Some(ext_builder))?; + let num_ext = num_ext.ok_or_else(|| Error::Input("Block was empty".into()))?; + let mut records = self.measure_block(&block)?; + + for r in &mut records { + // Subtract the base time. + *r = r.saturating_sub(base_time); + // Divide by the number of extrinsics in the block. + *r = ((*r as f64) / (num_ext as f64)).ceil() as u64; + } + + Stats::new(&records) + } + + /// Builds a block with some optional extrinsics. /// /// Returns the block and the number of extrinsics in the block /// that are not inherents. - fn build_block(&self, bench_type: BenchmarkType) -> Result<(Block, u64)> { + /// Returns a block with only inherents if `ext_builder` is `None`. + fn build_block( + &self, + ext_builder: Option<&dyn ExtrinsicBuilder>, + ) -> Result<(Block, Option)> { let mut builder = self.client.new_block(Default::default())?; // Create and insert the inherents. let inherents = builder.create_inherents(self.inherent_data.clone())?; @@ -114,16 +132,18 @@ where builder.push(inherent)?; } - // Return early if we just want a block with inherents and no additional extrinsics. - if bench_type == BenchmarkType::Block { - return Ok((builder.build()?.block, 0)) - } + // Return early if `ext_builder` is `None`. + let ext_builder = if let Some(ext_builder) = ext_builder { + ext_builder + } else { + return Ok((builder.build()?.block, None)) + }; // Put as many extrinsics into the block as possible and count them. info!("Building block, this takes some time..."); let mut num_ext = 0; for nonce in 0..self.max_ext_per_block() { - let ext = self.ext_builder.remark(nonce)?; + let ext = ext_builder.build(nonce)?; match builder.push(ext.clone()) { Ok(()) => {}, Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid( @@ -139,20 +159,12 @@ where info!("Extrinsics per block: {}", num_ext); let block = builder.build()?.block; - Ok((block, num_ext)) + Ok((block, Some(num_ext))) } /// Measures the time that it take to execute a block or an extrinsic. - fn measure_block( - &self, - block: &Block, - num_ext: u64, - bench_type: BenchmarkType, - ) -> Result { + fn measure_block(&self, block: &Block) -> Result { let mut record = BenchRecord::new(); - if bench_type == BenchmarkType::Extrinsic && num_ext == 0 { - return Err("Cannot measure the extrinsic time of an empty block".into()) - } let genesis = BlockId::Number(Zero::zero()); info!("Running {} warmups...", self.params.warmup); @@ -176,12 +188,7 @@ where .map_err(|e| Error::Client(RuntimeApiError(e)))?; let elapsed = start.elapsed().as_nanos(); - if bench_type == BenchmarkType::Extrinsic { - // Checked for non-zero div above. - record.push((elapsed as f64 / num_ext as f64).ceil() as u64); - } else { - record.push(elapsed as u64); - } + record.push(elapsed as u64); } Ok(record) @@ -191,21 +198,3 @@ where self.params.max_ext_per_block.unwrap_or(u32::MAX) } } - -impl BenchmarkType { - /// Short name of the benchmark type. - pub(crate) fn short_name(&self) -> &'static str { - match self { - Self::Extrinsic => "extrinsic", - Self::Block => "block", - } - } - - /// Long name of the benchmark type. - pub(crate) fn long_name(&self) -> &'static str { - match self { - Self::Extrinsic => "ExtrinsicBase", - Self::Block => "BlockExecution", - } - } -} diff --git a/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs b/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs new file mode 100644 index 0000000000000..6b4fd0fad6638 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs @@ -0,0 +1,134 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; +use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams}; +use sc_client_api::Backend as ClientBackend; +use sp_api::{ApiExt, ProvideRuntimeApi}; +use sp_runtime::{traits::Block as BlockT, OpaqueExtrinsic}; + +use clap::{Args, Parser}; +use log::info; +use serde::Serialize; +use std::{fmt::Debug, sync::Arc}; + +use super::{ + bench::{Benchmark, BenchmarkParams}, + extrinsic_factory::ExtrinsicFactory, +}; + +/// Benchmark the execution time of different extrinsics. +/// +/// This is calculated by filling a block with a specific extrinsic and executing the block. +/// The result time is then divided by the number of extrinsics in that block. +/// +/// NOTE: The BlockExecutionWeight is ignored in this case since it +// is very small compared to the total block execution time. +#[derive(Debug, Parser)] +pub struct ExtrinsicCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub import_params: ImportParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub params: ExtrinsicParams, +} + +/// The params for the [`ExtrinsicCmd`]. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct ExtrinsicParams { + #[clap(flatten)] + pub bench: BenchmarkParams, + + /// List all available pallets and extrinsics. + /// + /// The format is CSV with header `pallet, extrinsic`. + #[clap(long)] + pub list: bool, + + /// Pallet name of the extrinsic to benchmark. + #[clap(long, value_name = "PALLET", required_unless_present = "list")] + pub pallet: Option, + + /// Extrinsic to benchmark. + #[clap(long, value_name = "EXTRINSIC", required_unless_present = "list")] + pub extrinsic: Option, +} + +impl ExtrinsicCmd { + /// Benchmark the execution time of a specific type of extrinsic. + /// + /// The output will be printed to console. + pub fn run( + &self, + client: Arc, + inherent_data: sp_inherents::InherentData, + ext_factory: &ExtrinsicFactory, + ) -> Result<()> + where + Block: BlockT, + BA: ClientBackend, + C: BlockBuilderProvider + ProvideRuntimeApi, + C::Api: ApiExt + BlockBuilderApi, + { + // Short circuit if --list was specified. + if self.params.list { + let list: Vec = ext_factory.0.iter().map(|b| b.name()).collect(); + info!( + "Listing available extrinsics ({}):\npallet, extrinsic\n{}", + list.len(), + list.join("\n") + ); + return Ok(()) + } + + let pallet = self.params.pallet.clone().unwrap_or_default(); + let extrinsic = self.params.extrinsic.clone().unwrap_or_default(); + let ext_builder = match ext_factory.try_get(&pallet, &extrinsic) { + Some(ext_builder) => ext_builder, + None => + return Err("Unknown pallet or extrinsic. Use --list for a complete list.".into()), + }; + + let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data); + let stats = bench.bench_extrinsic(ext_builder)?; + info!( + "Executing a {}::{} extrinsic takes[ns]:\n{:?}", + ext_builder.pallet(), + ext_builder.extrinsic(), + stats + ); + + Ok(()) + } +} + +// Boilerplate +impl CliConfiguration for ExtrinsicCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) + } +} diff --git a/utils/frame/benchmarking-cli/src/extrinsic/extrinsic_factory.rs b/utils/frame/benchmarking-cli/src/extrinsic/extrinsic_factory.rs new file mode 100644 index 0000000000000..7e1a22ccd1419 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/extrinsic/extrinsic_factory.rs @@ -0,0 +1,70 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Provides the [`ExtrinsicFactory`] and the [`ExtrinsicBuilder`] types. +//! Is used by the *overhead* and *extrinsic* benchmarks to build extrinsics. + +use sp_runtime::OpaqueExtrinsic; + +/// Helper to manage [`ExtrinsicBuilder`] instances. +#[derive(Default)] +pub struct ExtrinsicFactory(pub Vec>); + +impl ExtrinsicFactory { + /// Returns a builder for a pallet and extrinsic name. + /// + /// Is case in-sensitive. + pub fn try_get(&self, pallet: &str, extrinsic: &str) -> Option<&dyn ExtrinsicBuilder> { + let pallet = pallet.to_lowercase(); + let extrinsic = extrinsic.to_lowercase(); + + self.0 + .iter() + .find(|b| b.pallet() == pallet && b.extrinsic() == extrinsic) + .map(|b| b.as_ref()) + } +} + +/// Used by the benchmark to build signed extrinsics. +/// +/// The built extrinsics only need to be valid in the first block +/// who's parent block is the genesis block. +/// This assumption simplifies the generation of the extrinsics. +/// The signer should be one of the pre-funded dev accounts. +pub trait ExtrinsicBuilder { + /// Name of the pallet this builder is for. + /// + /// Should be all lowercase. + fn pallet(&self) -> &str; + + /// Name of the extrinsic this builder is for. + /// + /// Should be all lowercase. + fn extrinsic(&self) -> &str; + + /// Builds an extrinsic. + /// + /// Will be called multiple times with increasing nonces. + fn build(&self, nonce: u32) -> std::result::Result; +} + +impl dyn ExtrinsicBuilder + '_ { + /// Name of this builder in CSV format: `pallet, extrinsic`. + pub fn name(&self) -> String { + format!("{}, {}", self.pallet(), self.extrinsic()) + } +} diff --git a/utils/frame/benchmarking-cli/src/extrinsic/mod.rs b/utils/frame/benchmarking-cli/src/extrinsic/mod.rs new file mode 100644 index 0000000000000..12a09c3b1af63 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/extrinsic/mod.rs @@ -0,0 +1,27 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Benchmark the time it takes to execute a specific extrinsic. +//! This is a generalization of the *overhead* benchmark which can only measure `System::Remark` +//! extrinsics. + +pub mod bench; +pub mod cmd; +pub mod extrinsic_factory; + +pub use cmd::ExtrinsicCmd; +pub use extrinsic_factory::{ExtrinsicBuilder, ExtrinsicFactory}; diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index d0eee3d2939fc..96c7a997895d4 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -18,6 +18,7 @@ //! Contains the root [`BenchmarkCmd`] command and exports its sub-commands. mod block; +mod extrinsic; mod machine; mod overhead; mod pallet; @@ -25,8 +26,9 @@ mod shared; mod storage; pub use block::BlockCmd; +pub use extrinsic::{ExtrinsicBuilder, ExtrinsicCmd, ExtrinsicFactory}; pub use machine::{MachineCmd, Requirements, SUBSTRATE_REFERENCE_HARDWARE}; -pub use overhead::{ExtrinsicBuilder, OverheadCmd}; +pub use overhead::OverheadCmd; pub use pallet::PalletCmd; pub use storage::StorageCmd; @@ -41,8 +43,8 @@ pub enum BenchmarkCmd { Storage(StorageCmd), Overhead(OverheadCmd), Block(BlockCmd), - #[clap(hide = true)] // Hidden until fully completed. Machine(MachineCmd), + Extrinsic(ExtrinsicCmd), } /// Unwraps a [`BenchmarkCmd`] into its concrete sub-command. @@ -58,6 +60,7 @@ macro_rules! unwrap_cmd { BenchmarkCmd::Overhead($cmd) => $code, BenchmarkCmd::Block($cmd) => $code, BenchmarkCmd::Machine($cmd) => $code, + BenchmarkCmd::Extrinsic($cmd) => $code, } } } diff --git a/utils/frame/benchmarking-cli/src/overhead/cmd.rs b/utils/frame/benchmarking-cli/src/overhead/cmd.rs index 357c89d97a5ac..28ceea63f1572 100644 --- a/utils/frame/benchmarking-cli/src/overhead/cmd.rs +++ b/utils/frame/benchmarking-cli/src/overhead/cmd.rs @@ -31,10 +31,11 @@ use serde::Serialize; use std::{fmt::Debug, sync::Arc}; use crate::{ - overhead::{ - bench::{Benchmark, BenchmarkParams, BenchmarkType}, - template::TemplateData, + extrinsic::{ + bench::{Benchmark, BenchmarkParams as ExtrinsicBenchmarkParams}, + ExtrinsicBuilder, }, + overhead::template::TemplateData, shared::{HostInfoParams, WeightParams}, }; @@ -63,20 +64,20 @@ pub struct OverheadParams { #[allow(missing_docs)] #[clap(flatten)] - pub bench: BenchmarkParams, + pub bench: ExtrinsicBenchmarkParams, #[allow(missing_docs)] #[clap(flatten)] pub hostinfo: HostInfoParams, } -/// Used by the benchmark to build signed extrinsics. -/// -/// The built extrinsics only need to be valid in the first block -/// who's parent block is the genesis block. -pub trait ExtrinsicBuilder { - /// Build a `System::remark` extrinsic. - fn remark(&self, nonce: u32) -> std::result::Result; +/// Type of a benchmark. +#[derive(Serialize, Clone, PartialEq, Copy)] +pub(crate) enum BenchmarkType { + /// Measure the per-extrinsic execution overhead. + Extrinsic, + /// Measure the per-block execution overhead. + Block, } impl OverheadCmd { @@ -89,7 +90,7 @@ impl OverheadCmd { cfg: Configuration, client: Arc, inherent_data: sp_inherents::InherentData, - ext_builder: Arc, + ext_builder: &dyn ExtrinsicBuilder, ) -> Result<()> where Block: BlockT, @@ -97,18 +98,21 @@ impl OverheadCmd { C: BlockBuilderProvider + ProvideRuntimeApi, C::Api: ApiExt + BlockBuilderApi, { - let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data, ext_builder); + if ext_builder.pallet() != "system" || ext_builder.extrinsic() != "remark" { + return Err(format!("The extrinsic builder is required to build `System::Remark` extrinsics but builds `{}` extrinsics instead", ext_builder.name()).into()); + } + let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data); // per-block execution overhead { - let stats = bench.bench(BenchmarkType::Block)?; + let stats = bench.bench_block()?; info!("Per-block execution overhead [ns]:\n{:?}", stats); let template = TemplateData::new(BenchmarkType::Block, &cfg, &self.params, &stats)?; template.write(&self.params.weight.weight_path)?; } // per-extrinsic execution overhead { - let stats = bench.bench(BenchmarkType::Extrinsic)?; + let stats = bench.bench_extrinsic(ext_builder)?; info!("Per-extrinsic execution overhead [ns]:\n{:?}", stats); let template = TemplateData::new(BenchmarkType::Extrinsic, &cfg, &self.params, &stats)?; template.write(&self.params.weight.weight_path)?; @@ -118,6 +122,24 @@ impl OverheadCmd { } } +impl BenchmarkType { + /// Short name of the benchmark type. + pub(crate) fn short_name(&self) -> &'static str { + match self { + Self::Extrinsic => "extrinsic", + Self::Block => "block", + } + } + + /// Long name of the benchmark type. + pub(crate) fn long_name(&self) -> &'static str { + match self { + Self::Extrinsic => "ExtrinsicBase", + Self::Block => "BlockExecution", + } + } +} + // Boilerplate impl CliConfiguration for OverheadCmd { fn shared_params(&self) -> &SharedParams { diff --git a/utils/frame/benchmarking-cli/src/overhead/mod.rs b/utils/frame/benchmarking-cli/src/overhead/mod.rs index abdeac22b7898..fc3db912f7a30 100644 --- a/utils/frame/benchmarking-cli/src/overhead/mod.rs +++ b/utils/frame/benchmarking-cli/src/overhead/mod.rs @@ -15,8 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod bench; pub mod cmd; -mod template; +pub mod template; -pub use cmd::{ExtrinsicBuilder, OverheadCmd}; +pub use cmd::OverheadCmd; diff --git a/utils/frame/benchmarking-cli/src/overhead/template.rs b/utils/frame/benchmarking-cli/src/overhead/template.rs index 33c2c7999039a..aa82e45cf6db9 100644 --- a/utils/frame/benchmarking-cli/src/overhead/template.rs +++ b/utils/frame/benchmarking-cli/src/overhead/template.rs @@ -27,7 +27,7 @@ use serde::Serialize; use std::{env, fs, path::PathBuf}; use crate::{ - overhead::{bench::BenchmarkType, cmd::OverheadParams}, + overhead::cmd::{BenchmarkType, OverheadParams}, shared::{Stats, UnderscoreHelper}, }; From 752980bf4a16488f0b37a4741bdefdd2f90e93bf Mon Sep 17 00:00:00 2001 From: Vlad Date: Tue, 19 Jul 2022 15:47:51 +0300 Subject: [PATCH 25/46] Don't download artifacts into the `rusty-cachier-notify` job (#11850) --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fff11e07326d8..9608d88e9554d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -229,6 +229,7 @@ rusty-cachier-notify: variables: CI_IMAGE: paritytech/rusty-cachier-env:latest GIT_STRATEGY: none + dependencies: [] script: - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash - rusty-cachier cache notify From 7b81072843e584ef2bee449081aed5c963021ee9 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Wed, 20 Jul 2022 10:50:54 +0200 Subject: [PATCH 26/46] Bump wasmtime default-pages (#11864) Signed-off-by: Oliver Tale-Yazdi --- client/executor/wasmtime/src/runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/executor/wasmtime/src/runtime.rs b/client/executor/wasmtime/src/runtime.rs index fb5c6cb16993a..dcf07c454e46d 100644 --- a/client/executor/wasmtime/src/runtime.rs +++ b/client/executor/wasmtime/src/runtime.rs @@ -366,7 +366,7 @@ fn common_config(semantics: &Semantics) -> std::result::Result Date: Wed, 20 Jul 2022 18:04:31 -0400 Subject: [PATCH 27/46] Make reading json genesis file faster (#11868) * Make reading json genesis file faster * Formatting * fmt --- client/chain-spec/src/chain_spec.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index efb40d46f216a..5aafc28524dbf 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -61,7 +61,16 @@ impl GenesisSource { let file = File::open(path).map_err(|e| { format!("Error opening spec file at `{}`: {}", path.display(), e) })?; - let genesis: GenesisContainer = json::from_reader(file) + // SAFETY: `mmap` is fundamentally unsafe since technically the file can change + // underneath us while it is mapped; in practice it's unlikely to be a + // problem + let bytes = unsafe { + memmap2::Mmap::map(&file).map_err(|e| { + format!("Error mmaping spec file `{}`: {}", path.display(), e) + })? + }; + + let genesis: GenesisContainer = json::from_slice(&bytes) .map_err(|e| format!("Error parsing spec file: {}", e))?; Ok(genesis.genesis) }, From 58a80806f01bab8bcf9da9be8afdfd70ad50e7fe Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 21 Jul 2022 11:36:00 +0300 Subject: [PATCH 28/46] Cleanup light client leftovers (#11865) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove --light cli option * Cleanup light client leftovers * Remove commented-out code and clean-up more light client leftovers * Fix formatting with `cargo +nightly fmt` * Remove FIXME regarding db directory structure Co-authored-by: Bastian Köcher Co-authored-by: Bastian Köcher --- client/cli/src/commands/run_cmd.rs | 17 +--- client/cli/src/config.rs | 8 +- .../src/communication/gossip.rs | 14 +-- client/network/src/config.rs | 8 -- client/network/src/protocol/message.rs | 1 - client/network/src/service.rs | 15 +-- client/network/src/transactions.rs | 13 --- client/rpc-api/src/system/helpers.rs | 2 - client/service/src/builder.rs | 93 +++++++------------ client/service/src/lib.rs | 7 -- client/service/src/metrics.rs | 2 +- 11 files changed, 44 insertions(+), 136 deletions(-) diff --git a/client/cli/src/commands/run_cmd.rs b/client/cli/src/commands/run_cmd.rs index 12774eecc4174..99eaa4be1cd0f 100644 --- a/client/cli/src/commands/run_cmd.rs +++ b/client/cli/src/commands/run_cmd.rs @@ -50,10 +50,6 @@ pub struct RunCmd { #[clap(long)] pub no_grandpa: bool, - /// Experimental: Run in light client mode. - #[clap(long)] - pub light: bool, - /// Listen to all RPC interfaces. /// /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use an RPC @@ -337,7 +333,7 @@ impl CliConfiguration for RunCmd { fn dev_key_seed(&self, is_dev: bool) -> Result> { Ok(self.get_keyring().map(|a| format!("//{}", a)).or_else(|| { - if is_dev && !self.light { + if is_dev { Some("//Alice".into()) } else { None @@ -363,16 +359,9 @@ impl CliConfiguration for RunCmd { fn role(&self, is_dev: bool) -> Result { let keyring = self.get_keyring(); - let is_light = self.light; - let is_authority = (self.validator || is_dev || keyring.is_some()) && !is_light; + let is_authority = self.validator || is_dev || keyring.is_some(); - Ok(if is_light { - sc_service::Role::Light - } else if is_authority { - sc_service::Role::Authority - } else { - sc_service::Role::Full - }) + Ok(if is_authority { sc_service::Role::Authority } else { sc_service::Role::Full }) } fn force_authoring(&self) -> Result { diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index 01c541bf75255..4ebbc8c72c19a 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -211,12 +211,8 @@ pub trait CliConfiguration: Sized { base_path: &PathBuf, cache_size: usize, database: Database, - role: &Role, ) -> Result { - let role_dir = match role { - Role::Light => "light", - Role::Full | Role::Authority => "full", - }; + let role_dir = "full"; let rocksdb_path = base_path.join("db").join(role_dir); let paritydb_path = base_path.join("paritydb").join(role_dir); Ok(match database { @@ -536,7 +532,7 @@ pub trait CliConfiguration: Sized { )?, keystore_remote, keystore, - database: self.database_config(&config_dir, database_cache_size, database, &role)?, + database: self.database_config(&config_dir, database_cache_size, database)?, state_cache_size: self.state_cache_size()?, state_cache_child_ratio: self.state_cache_child_ratio()?, state_pruning: self.state_pruning()?, diff --git a/client/finality-grandpa/src/communication/gossip.rs b/client/finality-grandpa/src/communication/gossip.rs index 250e640cbf037..65d7dfb783aa3 100644 --- a/client/finality-grandpa/src/communication/gossip.rs +++ b/client/finality-grandpa/src/communication/gossip.rs @@ -728,11 +728,7 @@ type MaybeMessage = Option<(Vec, NeighborPacket> impl Inner { fn new(config: crate::Config) -> Self { - let catch_up_config = if config.local_role.is_light() { - // if we are a light client we shouldn't be issuing any catch-up requests - // as we don't participate in the full GRANDPA protocol - CatchUpConfig::disabled() - } else if config.observer_enabled { + let catch_up_config = if config.observer_enabled { if config.local_role.is_authority() { // since the observer protocol is enabled, we will only issue // catch-up requests if we are an authority (and only to other @@ -1231,10 +1227,6 @@ impl Inner { None => return false, }; - if self.config.local_role.is_light() { - return false - } - if round_elapsed < round_duration.mul_f32(PROPAGATION_SOME) { self.peers.first_stage_peers.contains(who) } else if round_elapsed < round_duration.mul_f32(PROPAGATION_ALL) { @@ -1266,10 +1258,6 @@ impl Inner { None => return false, }; - if self.config.local_role.is_light() { - return false - } - if round_elapsed < round_duration.mul_f32(PROPAGATION_ALL) { self.peers.first_stage_peers.contains(who) || self.peers.second_stage_peers.contains(who) || diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 430efd697a18c..58349fe973330 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -147,8 +147,6 @@ where pub enum Role { /// Regular full node. Full, - /// Regular light node. - Light, /// Actual authority. Authority, } @@ -158,18 +156,12 @@ impl Role { pub fn is_authority(&self) -> bool { matches!(self, Self::Authority { .. }) } - - /// True for [`Role::Light`]. - pub fn is_light(&self) -> bool { - matches!(self, Self::Light { .. }) - } } impl fmt::Display for Role { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Full => write!(f, "FULL"), - Self::Light => write!(f, "LIGHT"), Self::Authority { .. } => write!(f, "AUTHORITY"), } } diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index c9512f82e23bb..50c4a264a5f95 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -107,7 +107,6 @@ pub mod generic { fn from(roles: &'a crate::config::Role) -> Self { match roles { crate::config::Role::Full => Self::FULL, - crate::config::Role::Light => Self::LIGHT, crate::config::Role::Authority { .. } => Self::AUTHORITY, } } diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 0d28b50df1821..bc6cfc73bec36 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -245,15 +245,11 @@ where }; let chain_sync = (params.create_chain_sync)( - if params.role.is_light() { - SyncMode::Light - } else { - match params.network_config.sync_mode { - config::SyncMode::Full => SyncMode::Full, - config::SyncMode::Fast { skip_proofs, storage_chain_mode } => - SyncMode::LightState { skip_proofs, storage_chain_mode }, - config::SyncMode::Warp => SyncMode::Warp, - } + match params.network_config.sync_mode { + config::SyncMode::Full => SyncMode::Full, + config::SyncMode::Fast { skip_proofs, storage_chain_mode } => + SyncMode::LightState { skip_proofs, storage_chain_mode }, + config::SyncMode::Warp => SyncMode::Warp, }, params.chain.clone(), warp_sync_provider, @@ -489,7 +485,6 @@ where let (tx_handler, tx_handler_controller) = transactions_handler_proto.build( service.clone(), - params.role, params.transaction_pool, params.metrics_registry.as_ref(), )?; diff --git a/client/network/src/transactions.rs b/client/network/src/transactions.rs index 1f54f05d7446f..043bdeff7ebfc 100644 --- a/client/network/src/transactions.rs +++ b/client/network/src/transactions.rs @@ -83,8 +83,6 @@ mod rep { pub const GOOD_TRANSACTION: Rep = Rep::new(1 << 7, "Good transaction"); /// Reputation change when a peer sends us a bad transaction. pub const BAD_TRANSACTION: Rep = Rep::new(-(1 << 12), "Bad transaction"); - /// We received an unexpected transaction packet. - pub const UNEXPECTED_TRANSACTIONS: Rep = Rep::new_fatal("Unexpected transactions packet"); } struct Metrics { @@ -160,7 +158,6 @@ impl TransactionsHandlerPrototype { pub fn build( self, service: Arc>, - local_role: config::Role, transaction_pool: Arc>, metrics_registry: Option<&Registry>, ) -> error::Result<(TransactionsHandler, TransactionsHandlerController)> { @@ -178,7 +175,6 @@ impl TransactionsHandlerPrototype { event_stream, peers: HashMap::new(), transaction_pool, - local_role, from_controller, metrics: if let Some(r) = metrics_registry { Some(Metrics::register(r)?) @@ -247,7 +243,6 @@ pub struct TransactionsHandler { peers: HashMap>, transaction_pool: Arc>, gossip_enabled: Arc, - local_role: config::Role, from_controller: mpsc::UnboundedReceiver>, /// Prometheus metrics. metrics: Option, @@ -360,14 +355,6 @@ impl TransactionsHandler { /// Called when peer sends us new transactions fn on_transactions(&mut self, who: PeerId, transactions: message::Transactions) { - // sending transaction to light node is considered a bad behavior - if matches!(self.local_role, config::Role::Light) { - debug!(target: "sync", "Peer {} is trying to send transactions to the light node", who); - self.service.disconnect_peer(who, self.protocol_name.clone()); - self.service.report_peer(who, rep::UNEXPECTED_TRANSACTIONS); - return - } - // Accept transactions only when enabled if !self.gossip_enabled.load(Ordering::Relaxed) { trace!(target: "sync", "{} Ignoring transactions while disabled", who); diff --git a/client/rpc-api/src/system/helpers.rs b/client/rpc-api/src/system/helpers.rs index e17e0f518c3e6..4561fccc1e81b 100644 --- a/client/rpc-api/src/system/helpers.rs +++ b/client/rpc-api/src/system/helpers.rs @@ -76,8 +76,6 @@ pub struct PeerInfo { pub enum NodeRole { /// The node is a full node Full, - /// The node is a light client - LightClient, /// The node is an authority Authority, } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index fe0ae5db53e70..0a18943f45006 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -37,17 +37,12 @@ use sc_client_db::{Backend, DatabaseSettings}; use sc_consensus::import_queue::ImportQueue; use sc_executor::RuntimeVersionOf; use sc_keystore::LocalKeystore; -use sc_network::{ - config::{Role, SyncMode}, - NetworkService, -}; +use sc_network::{config::SyncMode, NetworkService}; use sc_network_common::sync::warp::WarpSyncProvider; -use sc_network_light::light_client_requests::{self, handler::LightClientRequestHandler}; +use sc_network_light::light_client_requests::handler::LightClientRequestHandler; use sc_network_sync::{ - block_request_handler::{self, BlockRequestHandler}, - state_request_handler::{self, StateRequestHandler}, - warp_request_handler::{self, RequestHandler as WarpSyncRequestHandler}, - ChainSync, + block_request_handler::BlockRequestHandler, state_request_handler::StateRequestHandler, + warp_request_handler::RequestHandler as WarpSyncRequestHandler, ChainSync, }; use sc_rpc::{ author::AuthorApiServer, @@ -731,11 +726,8 @@ where } } - let transaction_pool_adapter = Arc::new(TransactionPoolAdapter { - imports_external_transactions: !matches!(config.role, Role::Light), - pool: transaction_pool, - client: client.clone(), - }); + let transaction_pool_adapter = + Arc::new(TransactionPoolAdapter { pool: transaction_pool, client: client.clone() }); let protocol_id = config.protocol_id(); @@ -746,63 +738,42 @@ where }; let block_request_protocol_config = { - if matches!(config.role, Role::Light) { - // Allow outgoing requests but deny incoming requests. - block_request_handler::generate_protocol_config(&protocol_id) - } else { - // Allow both outgoing and incoming requests. - let (handler, protocol_config) = BlockRequestHandler::new( - &protocol_id, - client.clone(), - config.network.default_peers_set.in_peers as usize + - config.network.default_peers_set.out_peers as usize, - ); - spawn_handle.spawn("block-request-handler", Some("networking"), handler.run()); - protocol_config - } + // Allow both outgoing and incoming requests. + let (handler, protocol_config) = BlockRequestHandler::new( + &protocol_id, + client.clone(), + config.network.default_peers_set.in_peers as usize + + config.network.default_peers_set.out_peers as usize, + ); + spawn_handle.spawn("block-request-handler", Some("networking"), handler.run()); + protocol_config }; let state_request_protocol_config = { - if matches!(config.role, Role::Light) { - // Allow outgoing requests but deny incoming requests. - state_request_handler::generate_protocol_config(&protocol_id) - } else { - // Allow both outgoing and incoming requests. - let (handler, protocol_config) = StateRequestHandler::new( - &protocol_id, - client.clone(), - config.network.default_peers_set_num_full as usize, - ); - spawn_handle.spawn("state-request-handler", Some("networking"), handler.run()); - protocol_config - } + // Allow both outgoing and incoming requests. + let (handler, protocol_config) = StateRequestHandler::new( + &protocol_id, + client.clone(), + config.network.default_peers_set_num_full as usize, + ); + spawn_handle.spawn("state-request-handler", Some("networking"), handler.run()); + protocol_config }; let warp_sync_params = warp_sync.map(|provider| { - let protocol_config = if matches!(config.role, Role::Light) { - // Allow outgoing requests but deny incoming requests. - warp_request_handler::generate_request_response_config(protocol_id.clone()) - } else { - // Allow both outgoing and incoming requests. - let (handler, protocol_config) = - WarpSyncRequestHandler::new(protocol_id.clone(), provider.clone()); - spawn_handle.spawn("warp-sync-request-handler", Some("networking"), handler.run()); - protocol_config - }; + // Allow both outgoing and incoming requests. + let (handler, protocol_config) = + WarpSyncRequestHandler::new(protocol_id.clone(), provider.clone()); + spawn_handle.spawn("warp-sync-request-handler", Some("networking"), handler.run()); (provider, protocol_config) }); let light_client_request_protocol_config = { - if matches!(config.role, Role::Light) { - // Allow outgoing requests but deny incoming requests. - light_client_requests::generate_protocol_config(&protocol_id) - } else { - // Allow both outgoing and incoming requests. - let (handler, protocol_config) = - LightClientRequestHandler::new(&protocol_id, client.clone()); - spawn_handle.spawn("light-client-request-handler", Some("networking"), handler.run()); - protocol_config - } + // Allow both outgoing and incoming requests. + let (handler, protocol_config) = + LightClientRequestHandler::new(&protocol_id, client.clone()); + spawn_handle.spawn("light-client-request-handler", Some("networking"), handler.run()); + protocol_config }; let max_parallel_downloads = config.network.max_parallel_downloads; diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 24ba670cfcd65..1b70227a70488 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -253,7 +253,6 @@ async fn build_network_future< let node_role = match role { Role::Authority { .. } => NodeRole::Authority, - Role::Light => NodeRole::LightClient, Role::Full => NodeRole::Full, }; @@ -377,7 +376,6 @@ where /// Transaction pool adapter. pub struct TransactionPoolAdapter { - imports_external_transactions: bool, pool: Arc

, client: Arc, } @@ -425,11 +423,6 @@ where } fn import(&self, transaction: B::Extrinsic) -> TransactionImportFuture { - if !self.imports_external_transactions { - debug!("Transaction rejected"); - return Box::pin(futures::future::ready(TransactionImport::None)) - } - let encoded = transaction.encode(); let uxt = match Decode::decode(&mut &encoded[..]) { Ok(uxt) => uxt, diff --git a/client/service/src/metrics.rs b/client/service/src/metrics.rs index 56b145ccdf7f7..ef3132f61ab99 100644 --- a/client/service/src/metrics.rs +++ b/client/service/src/metrics.rs @@ -160,7 +160,7 @@ impl MetricsService { ) -> Result { let role_bits = match config.role { Role::Full => 1u64, - Role::Light => 2u64, + // 2u64 used to represent light client role Role::Authority { .. } => 4u64, }; From cac91f59b9e3fb8fd59842c023f87b4206931993 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 21 Jul 2022 12:11:19 +0300 Subject: [PATCH 29/46] Stop RPC servers on drop (#11679) * Stop RPC servers on drop * Switch to existing wrappers that stop RPC servers * Apply formatting --- client/service/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 1b70227a70488..2e7b611ffb187 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -369,7 +369,8 @@ where match tokio::task::block_in_place(|| { config.tokio_handle.block_on(futures::future::try_join(http_fut, ws_fut)) }) { - Ok((http, ws)) => Ok(Box::new((http, ws))), + Ok((http, ws)) => + Ok(Box::new((waiting::HttpServer(Some(http)), waiting::WsServer(Some(ws))))), Err(e) => Err(Error::Application(e)), } } From b28bf15e5160f33e257e2cc6066bdd6c55a3eb42 Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Thu, 21 Jul 2022 15:35:18 +0200 Subject: [PATCH 30/46] [ci] Generate rustdocs without dependencies (#11885) --- scripts/ci/gitlab/pipeline/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index 25ecad5bc5264..eedd2ee0bb409 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -149,7 +149,7 @@ build-rustdoc: - ./crate-docs/ script: - rusty-cachier snapshot create - - time cargo +nightly doc --workspace --all-features --verbose + - time cargo +nightly doc --workspace --all-features --verbose --no-deps - rm -f $CARGO_TARGET_DIR/doc/.lock - mv $CARGO_TARGET_DIR/doc ./crate-docs # FIXME: remove me after CI image gets nonroot From 1399afdcd8ab4d7fad149c96f9cadcaf04b94f86 Mon Sep 17 00:00:00 2001 From: Robert Hambrock Date: Thu, 21 Jul 2022 17:09:21 +0200 Subject: [PATCH 31/46] pallet-mmr: fix batch proof failures (#11840) * pallet-mmr: extend batch proof verification test covers all possible 2-leaf combinations now, including current verification failures that batch proof item count limit is too low sometimes. * raise upper bound on proof item number as described in https://github.com/paritytech/substrate/issues/11753#issuecomment-1179838174 * test for powerset of leaves * refactor batch proof verification test * test all batch proofs for mmr sizes up to n=13 * limit mmr size to reduce batch proof test duration * use saturating integer addition for proof check * extract common chain building in batch proof tests note: right now, since not killing old chain, it keeps growing by 7 blocks for every leaf selection (added after proof generation), hence heavier to compute. * only add blocks after a proof generation once * increase batch proof testing range * register offchain extensions only once * fmt & remove unused util --- Cargo.lock | 1 + frame/merkle-mountain-range/Cargo.toml | 1 + frame/merkle-mountain-range/src/lib.rs | 2 +- .../merkle-mountain-range/src/mmr/storage.rs | 2 +- frame/merkle-mountain-range/src/mmr/utils.rs | 22 +----- frame/merkle-mountain-range/src/tests.rs | 70 ++++++++++++++----- 6 files changed, 58 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 279864e056b0b..3d5bfe5e51201 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5871,6 +5871,7 @@ dependencies = [ "frame-support", "frame-system", "hex-literal", + "itertools", "parity-scale-codec", "scale-info", "sp-core", diff --git a/frame/merkle-mountain-range/Cargo.toml b/frame/merkle-mountain-range/Cargo.toml index f5f8bdee0855d..75301afed0094 100644 --- a/frame/merkle-mountain-range/Cargo.toml +++ b/frame/merkle-mountain-range/Cargo.toml @@ -27,6 +27,7 @@ sp-std = { version = "4.0.0", default-features = false, path = "../../primitives [dev-dependencies] env_logger = "0.9" hex-literal = "0.3" +itertools = "0.10.3" [features] default = ["std"] diff --git a/frame/merkle-mountain-range/src/lib.rs b/frame/merkle-mountain-range/src/lib.rs index 9274e8e72c508..4644ebcb7da1c 100644 --- a/frame/merkle-mountain-range/src/lib.rs +++ b/frame/merkle-mountain-range/src/lib.rs @@ -351,7 +351,7 @@ impl, I: 'static> Pallet { ) -> Result<(), primitives::Error> { if proof.leaf_count > Self::mmr_leaves() || proof.leaf_count == 0 || - proof.items.len() as u32 > mmr::utils::NodesUtils::new(proof.leaf_count).depth() + (proof.items.len().saturating_add(leaves.len())) as u64 > proof.leaf_count { return Err(primitives::Error::Verify .log_debug("The proof has incorrect number of leaves or proof items.")) diff --git a/frame/merkle-mountain-range/src/mmr/storage.rs b/frame/merkle-mountain-range/src/mmr/storage.rs index d31262d4d7f2f..8b623edf56957 100644 --- a/frame/merkle-mountain-range/src/mmr/storage.rs +++ b/frame/merkle-mountain-range/src/mmr/storage.rs @@ -136,7 +136,7 @@ where let window_size = ::BlockHashCount::get().unique_saturated_into(); if leaves >= window_size { - // Move the rolling window towards the end of `block_num->hash` mappings available + // Move the rolling window towards the end of `block_num->hash` mappings available // in the runtime: we "canonicalize" the leaf at the end, let to_canon_leaf = leaves.saturating_sub(window_size); // and all the nodes added by that leaf. diff --git a/frame/merkle-mountain-range/src/mmr/utils.rs b/frame/merkle-mountain-range/src/mmr/utils.rs index 3734ea514782d..0b8e88a9283da 100644 --- a/frame/merkle-mountain-range/src/mmr/utils.rs +++ b/frame/merkle-mountain-range/src/mmr/utils.rs @@ -46,15 +46,6 @@ impl NodesUtils { 2 * self.no_of_leaves - self.number_of_peaks() } - /// Calculate maximal depth of the MMR. - pub fn depth(&self) -> u32 { - if self.no_of_leaves == 0 { - return 0 - } - - 64 - self.no_of_leaves.next_power_of_two().leading_zeros() - } - /// Calculate `LeafIndex` for the leaf that added `node_index` to the MMR. pub fn leaf_index_that_added_node(node_index: NodeIndex) -> LeafIndex { let rightmost_leaf_pos = Self::rightmost_leaf_node_index_from_pos(node_index); @@ -128,18 +119,7 @@ mod tests { } #[test] - fn should_calculate_number_of_leaves_correctly() { - assert_eq!( - vec![0, 1, 2, 3, 4, 9, 15, 21] - .into_iter() - .map(|n| NodesUtils::new(n).depth()) - .collect::>(), - vec![0, 1, 2, 3, 3, 5, 5, 6] - ); - } - - #[test] - fn should_calculate_depth_correclty() { + fn should_calculate_depth_correctly() { assert_eq!( vec![0, 1, 2, 3, 4, 9, 15, 21] .into_iter() diff --git a/frame/merkle-mountain-range/src/tests.rs b/frame/merkle-mountain-range/src/tests.rs index 53226f8419988..566a051823d5e 100644 --- a/frame/merkle-mountain-range/src/tests.rs +++ b/frame/merkle-mountain-range/src/tests.rs @@ -347,28 +347,64 @@ fn should_verify() { } #[test] -fn should_verify_batch_proof() { +fn should_verify_batch_proofs() { + fn generate_and_verify_batch_proof( + ext: &mut sp_io::TestExternalities, + leaves: &Vec, + blocks_to_add: usize, + ) { + let (leaves, proof) = ext + .execute_with(|| crate::Pallet::::generate_batch_proof(leaves.to_vec()).unwrap()); + + ext.execute_with(|| { + add_blocks(blocks_to_add); + // then + assert_eq!(crate::Pallet::::verify_leaves(leaves, proof), Ok(())); + }) + } + let _ = env_logger::try_init(); - // Start off with chain initialisation and storing indexing data off-chain - // (MMR Leafs) - let mut ext = new_test_ext(); - ext.execute_with(|| add_blocks(7)); - ext.persist_offchain_overlay(); + use itertools::Itertools; - // Try to generate proof now. This requires the offchain extensions to be present - // to retrieve full leaf data. + let mut ext = new_test_ext(); + // require the offchain extensions to be present + // to retrieve full leaf data when generating proofs register_offchain_ext(&mut ext); - let (leaves, proof) = ext.execute_with(|| { - // when - crate::Pallet::::generate_batch_proof(vec![0, 4, 5]).unwrap() - }); - ext.execute_with(|| { - add_blocks(7); - // then - assert_eq!(crate::Pallet::::verify_leaves(leaves, proof), Ok(())); - }); + // verify that up to n=10, valid proofs are generated for all possible leaf combinations + for n in 0..10 { + ext.execute_with(|| new_block()); + ext.persist_offchain_overlay(); + + // generate powerset (skipping empty set) of all possible leaf combinations for mmr size n + let leaves_set: Vec> = (0..n).into_iter().powerset().skip(1).collect(); + + leaves_set.iter().for_each(|leaves_subset| { + generate_and_verify_batch_proof(&mut ext, leaves_subset, 0); + ext.persist_offchain_overlay(); + }); + } + + // verify that up to n=15, valid proofs are generated for all possible 2-leaf combinations + for n in 10..15 { + // (MMR Leafs) + ext.execute_with(|| new_block()); + ext.persist_offchain_overlay(); + + // generate all possible 2-leaf combinations for mmr size n + let leaves_set: Vec> = (0..n).into_iter().combinations(2).collect(); + + leaves_set.iter().for_each(|leaves_subset| { + generate_and_verify_batch_proof(&mut ext, leaves_subset, 0); + ext.persist_offchain_overlay(); + }); + } + + generate_and_verify_batch_proof(&mut ext, &vec![7, 11], 20); + ext.execute_with(|| add_blocks(1000)); + ext.persist_offchain_overlay(); + generate_and_verify_batch_proof(&mut ext, &vec![7, 11, 100, 800], 100); } #[test] From dbdbef95c84644328bee60bd170d79814c04e32b Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 22 Jul 2022 15:08:52 +0300 Subject: [PATCH 32/46] Clarify wrong bootnode peer ID error message (#11872) --- client/network/src/service.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/client/network/src/service.rs b/client/network/src/service.rs index bc6cfc73bec36..aeb5e25497bb3 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -2020,12 +2020,16 @@ where if this.boot_node_ids.contains(&peer_id) { if let DialError::WrongPeerId { obtained, endpoint } = &error { - error!( - "💔 The bootnode you want to connect provided a different peer ID than the one you expect: `{}` with `{}`:`{:?}`.", - peer_id, - obtained, - endpoint, - ); + if let ConnectedPoint::Dialer { address, role_override: _ } = + endpoint + { + error!( + "💔 The bootnode you want to connect to at `{}` provided a different peer ID `{}` than the one you expect `{}`.", + address, + obtained, + peer_id, + ); + } } } } From 7bb4936d788c84bfd9ed1bbde49336538459c5aa Mon Sep 17 00:00:00 2001 From: Branislav Kontur Date: Sat, 23 Jul 2022 20:01:57 +0200 Subject: [PATCH 33/46] Fix for 'note_applied_extrinsic' - consider also 'extract_actual_pays_fee' (#11849) --- frame/support/src/weights.rs | 37 +++++++++++ frame/system/src/lib.rs | 5 +- frame/system/src/tests.rs | 119 ++++++++++++++++++++++++++++++++++- 3 files changed, 157 insertions(+), 4 deletions(-) diff --git a/frame/support/src/weights.rs b/frame/support/src/weights.rs index 24240c7cc4e6a..dd49409ccc912 100644 --- a/frame/support/src/weights.rs +++ b/frame/support/src/weights.rs @@ -358,6 +358,15 @@ pub fn extract_actual_weight(result: &DispatchResultWithPostInfo, info: &Dispatc .calc_actual_weight(info) } +/// Extract the actual pays_fee from a dispatch result if any or fall back to the default weight. +pub fn extract_actual_pays_fee(result: &DispatchResultWithPostInfo, info: &DispatchInfo) -> Pays { + match result { + Ok(post_info) => post_info, + Err(err) => &err.post_info, + } + .pays_fee(info) +} + impl From<(Option, Pays)> for PostDispatchInfo { fn from(post_weight_info: (Option, Pays)) -> Self { let (actual_weight, pays_fee) = post_weight_info; @@ -956,6 +965,34 @@ mod tests { ); } + #[test] + fn extract_actual_pays_fee_works() { + let pre = DispatchInfo { weight: 1000, ..Default::default() }; + assert_eq!(extract_actual_pays_fee(&Ok(Some(7).into()), &pre), Pays::Yes); + assert_eq!(extract_actual_pays_fee(&Ok(Some(1000).into()), &pre), Pays::Yes); + assert_eq!(extract_actual_pays_fee(&Ok((Some(1000), Pays::Yes).into()), &pre), Pays::Yes); + assert_eq!(extract_actual_pays_fee(&Ok((Some(1000), Pays::No).into()), &pre), Pays::No); + assert_eq!( + extract_actual_pays_fee(&Err(DispatchError::BadOrigin.with_weight(9)), &pre), + Pays::Yes + ); + assert_eq!( + extract_actual_pays_fee( + &Err(DispatchErrorWithPostInfo { + post_info: PostDispatchInfo { actual_weight: None, pays_fee: Pays::No }, + error: DispatchError::BadOrigin, + }), + &pre + ), + Pays::No + ); + + let pre = DispatchInfo { weight: 1000, pays_fee: Pays::No, ..Default::default() }; + assert_eq!(extract_actual_pays_fee(&Ok(Some(7).into()), &pre), Pays::No); + assert_eq!(extract_actual_pays_fee(&Ok(Some(1000).into()), &pre), Pays::No); + assert_eq!(extract_actual_pays_fee(&Ok((Some(1000), Pays::Yes).into()), &pre), Pays::No); + } + type Balance = u64; // 0.5x^3 + 2.333x^2 + 7x - 10_000 diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 3c6f514808b5d..94605c2da59bd 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -91,8 +91,8 @@ use frame_support::{ OriginTrait, PalletInfo, SortedMembers, StoredMap, TypedGet, }, weights::{ - extract_actual_weight, DispatchClass, DispatchInfo, PerDispatchClass, RuntimeDbWeight, - Weight, + extract_actual_pays_fee, extract_actual_weight, DispatchClass, DispatchInfo, + PerDispatchClass, RuntimeDbWeight, Weight, }, Parameter, }; @@ -1500,6 +1500,7 @@ impl Pallet { /// To be called immediately after an extrinsic has been applied. pub fn note_applied_extrinsic(r: &DispatchResultWithPostInfo, mut info: DispatchInfo) { info.weight = extract_actual_weight(r, &info); + info.pays_fee = extract_actual_pays_fee(r, &info); Self::deposit_event(match r { Ok(_) => Event::ExtrinsicSuccess { dispatch_info: info }, Err(err) => { diff --git a/frame/system/src/tests.rs b/frame/system/src/tests.rs index 0facd796b2a0c..417dca12045ee 100644 --- a/frame/system/src/tests.rs +++ b/frame/system/src/tests.rs @@ -17,7 +17,9 @@ use crate::*; use frame_support::{ - assert_noop, assert_ok, dispatch::PostDispatchInfo, weights::WithPostDispatchInfo, + assert_noop, assert_ok, + dispatch::PostDispatchInfo, + weights::{Pays, WithPostDispatchInfo}, }; use mock::{Origin, *}; use sp_core::H256; @@ -216,7 +218,7 @@ fn deposit_event_should_work() { } #[test] -fn deposit_event_uses_actual_weight() { +fn deposit_event_uses_actual_weight_and_pays_fee() { new_test_ext().execute_with(|| { System::reset_events(); System::initialize(&1, &[0u8; 32].into(), &Default::default()); @@ -230,8 +232,34 @@ fn deposit_event_uses_actual_weight() { &Ok(Some(1200).into()), pre_info, ); + System::note_applied_extrinsic(&Ok((Some(2_500_000), Pays::Yes).into()), pre_info); + System::note_applied_extrinsic(&Ok(Pays::No.into()), pre_info); + System::note_applied_extrinsic(&Ok((Some(2_500_000), Pays::No).into()), pre_info); + System::note_applied_extrinsic(&Ok((Some(500), Pays::No).into()), pre_info); System::note_applied_extrinsic(&Err(DispatchError::BadOrigin.with_weight(999)), pre_info); + System::note_applied_extrinsic( + &Err(DispatchErrorWithPostInfo { + post_info: PostDispatchInfo { actual_weight: None, pays_fee: Pays::Yes }, + error: DispatchError::BadOrigin, + }), + pre_info, + ); + System::note_applied_extrinsic( + &Err(DispatchErrorWithPostInfo { + post_info: PostDispatchInfo { actual_weight: Some(800), pays_fee: Pays::Yes }, + error: DispatchError::BadOrigin, + }), + pre_info, + ); + System::note_applied_extrinsic( + &Err(DispatchErrorWithPostInfo { + post_info: PostDispatchInfo { actual_weight: Some(800), pays_fee: Pays::No }, + error: DispatchError::BadOrigin, + }), + pre_info, + ); + assert_eq!( System::events(), vec![ @@ -261,6 +289,54 @@ fn deposit_event_uses_actual_weight() { }, EventRecord { phase: Phase::ApplyExtrinsic(3), + event: SysEvent::ExtrinsicSuccess { + dispatch_info: DispatchInfo { + weight: 1000, + pays_fee: Pays::Yes, + ..Default::default() + }, + } + .into(), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(4), + event: SysEvent::ExtrinsicSuccess { + dispatch_info: DispatchInfo { + weight: 1000, + pays_fee: Pays::No, + ..Default::default() + }, + } + .into(), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(5), + event: SysEvent::ExtrinsicSuccess { + dispatch_info: DispatchInfo { + weight: 1000, + pays_fee: Pays::No, + ..Default::default() + }, + } + .into(), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(6), + event: SysEvent::ExtrinsicSuccess { + dispatch_info: DispatchInfo { + weight: 500, + pays_fee: Pays::No, + ..Default::default() + }, + } + .into(), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(7), event: SysEvent::ExtrinsicFailed { dispatch_error: DispatchError::BadOrigin.into(), dispatch_info: DispatchInfo { weight: 999, ..Default::default() }, @@ -268,6 +344,45 @@ fn deposit_event_uses_actual_weight() { .into(), topics: vec![] }, + EventRecord { + phase: Phase::ApplyExtrinsic(8), + event: SysEvent::ExtrinsicFailed { + dispatch_error: DispatchError::BadOrigin.into(), + dispatch_info: DispatchInfo { + weight: 1000, + pays_fee: Pays::Yes, + ..Default::default() + }, + } + .into(), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(9), + event: SysEvent::ExtrinsicFailed { + dispatch_error: DispatchError::BadOrigin.into(), + dispatch_info: DispatchInfo { + weight: 800, + pays_fee: Pays::Yes, + ..Default::default() + }, + } + .into(), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(10), + event: SysEvent::ExtrinsicFailed { + dispatch_error: DispatchError::BadOrigin.into(), + dispatch_info: DispatchInfo { + weight: 800, + pays_fee: Pays::No, + ..Default::default() + }, + } + .into(), + topics: vec![] + }, ] ); }); From ca570cffee47d9fa4806715b0e8b284ec6db7e65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 23 Jul 2022 21:28:25 +0200 Subject: [PATCH 34/46] pallet-assets: Update docs (#11877) --- frame/assets/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/assets/src/lib.rs b/frame/assets/src/lib.rs index 3f9146c5d229b..e0b00c5642c81 100644 --- a/frame/assets/src/lib.rs +++ b/frame/assets/src/lib.rs @@ -79,8 +79,6 @@ //! * `create`: Creates a new asset class, taking the required deposit. //! * `transfer`: Transfer sender's assets to another account. //! * `transfer_keep_alive`: Transfer sender's assets to another account, keeping the sender alive. -//! * `set_metadata`: Set the metadata of an asset class. -//! * `clear_metadata`: Remove the metadata of an asset class. //! * `approve_transfer`: Create or increase an delegated transfer. //! * `cancel_approval`: Rescind a previous approval. //! * `transfer_approved`: Transfer third-party's assets to another account. @@ -103,6 +101,8 @@ //! * `transfer_ownership`: Changes an asset class's Owner; called by the asset class's Owner. //! * `set_team`: Changes an asset class's Admin, Freezer and Issuer; called by the asset class's //! Owner. +//! * `set_metadata`: Set the metadata of an asset class; called by the asset class's Owner. +//! * `clear_metadata`: Remove the metadata of an asset class; called by the asset class's Owner. //! //! Please refer to the [`Call`] enum and its associated variants for documentation on each //! function. From fe3437e7ffd3207199f140ca14558f303567c955 Mon Sep 17 00:00:00 2001 From: Ikko Ashimine Date: Mon, 25 Jul 2022 05:42:49 +0900 Subject: [PATCH 35/46] Fix typo in limits.rs (#11894) alway -> always --- frame/system/src/limits.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/system/src/limits.rs b/frame/system/src/limits.rs index d3c108afb6f32..6076414ba6bcb 100644 --- a/frame/system/src/limits.rs +++ b/frame/system/src/limits.rs @@ -369,7 +369,7 @@ impl BlockWeightsBuilder { /// /// This is to make sure that extrinsics don't stay forever in the pool, /// because they could seamingly fit the block (since they are below `max_block`), - /// but the cost of calling `on_initialize` alway prevents them from being included. + /// but the cost of calling `on_initialize` always prevents them from being included. pub fn avg_block_initialization(mut self, init_cost: Perbill) -> Self { self.init_cost = Some(init_cost); self From 64d77551cb9ae0a9572a955d8a678d0de41af4bf Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Mon, 25 Jul 2022 11:15:47 +0200 Subject: [PATCH 36/46] [ci] fix node-bench-regression-guard job (#11901) * [Do not merge] [ci] debug node-bench-regression-guard * debug * fix artifacts path * add debug job * debug * debug * fix job, return pipeline --- scripts/ci/gitlab/pipeline/test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index dcec896ddc18b..d900d425ad494 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -129,7 +129,11 @@ node-bench-regression-guard: artifacts: true # polls artifact from master to compare with current result - project: $CI_PROJECT_PATH - job: cargo-check-benches + job: "cargo-check-benches 1/2" + ref: master + artifacts: true + - project: $CI_PROJECT_PATH + job: "cargo-check-benches 2/2" ref: master artifacts: true variables: From 618f26bc9fe1a5ae8ee0a485e4e24da3e5979af6 Mon Sep 17 00:00:00 2001 From: Davide Galassi Date: Mon, 25 Jul 2022 12:19:43 +0200 Subject: [PATCH 37/46] Remove unused leaves-set fields (#11895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove unused leaves-set fields * Fix undo_import method Old leaf sould be inserted using the displaced number * Fix leaves count * Apply code review suggestions Co-authored-by: Bastian Köcher Co-authored-by: Bastian Köcher --- client/api/src/leaves.rs | 81 +++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/client/api/src/leaves.rs b/client/api/src/leaves.rs index 5859290777433..2e5d4be3a5462 100644 --- a/client/api/src/leaves.rs +++ b/client/api/src/leaves.rs @@ -67,8 +67,6 @@ impl FinalizationDisplaced { #[derive(Debug, Clone, PartialEq, Eq)] pub struct LeafSet { storage: BTreeMap, Vec>, - pending_added: Vec<(H, N)>, - pending_removed: Vec, } impl LeafSet @@ -78,7 +76,7 @@ where { /// Construct a new, blank leaf set. pub fn new() -> Self { - Self { storage: BTreeMap::new(), pending_added: Vec::new(), pending_removed: Vec::new() } + Self { storage: BTreeMap::new() } } /// Read the leaf list from the DB, using given prefix for keys. @@ -97,21 +95,21 @@ where }, None => {}, } - Ok(Self { storage, pending_added: Vec::new(), pending_removed: Vec::new() }) + Ok(Self { storage }) } - /// update the leaf list on import. returns a displaced leaf if there was one. + /// Update the leaf list on import. + /// Returns a displaced leaf if there was one. pub fn import(&mut self, hash: H, number: N, parent_hash: H) -> Option> { // avoid underflow for genesis. let displaced = if number != N::zero() { - let new_number = Reverse(number.clone() - N::one()); - let was_displaced = self.remove_leaf(&new_number, &parent_hash); + let parent_number = Reverse(number.clone() - N::one()); + let was_displaced = self.remove_leaf(&parent_number, &parent_hash); if was_displaced { - self.pending_removed.push(parent_hash.clone()); Some(ImportDisplaced { new_hash: hash.clone(), - displaced: LeafSetItem { hash: parent_hash, number: new_number }, + displaced: LeafSetItem { hash: parent_hash, number: parent_number }, }) } else { None @@ -121,7 +119,6 @@ where }; self.insert_leaf(Reverse(number.clone()), hash.clone()); - self.pending_added.push((hash, number)); displaced } @@ -140,8 +137,6 @@ where }; let below_boundary = self.storage.split_off(&Reverse(boundary)); - self.pending_removed - .extend(below_boundary.values().flat_map(|h| h.iter()).cloned()); FinalizationDisplaced { leaves: below_boundary } } @@ -188,8 +183,6 @@ where self.remove_leaf(number, hash), "item comes from an iterator over storage; qed", ); - - self.pending_removed.push(hash.clone()); } } @@ -203,7 +196,6 @@ where // this is an invariant of regular block import. if !leaves_contains_best { self.insert_leaf(best_number.clone(), best_hash.clone()); - self.pending_added.push((best_hash, best_number.0)); } } @@ -213,9 +205,9 @@ where self.storage.iter().flat_map(|(_, hashes)| hashes.iter()).cloned().collect() } - /// Number of known leaves + /// Number of known leaves. pub fn count(&self) -> usize { - self.storage.len() + self.storage.values().map(|level| level.len()).sum() } /// Write the leaf list to the database transaction. @@ -227,8 +219,6 @@ where ) { let leaves: Vec<_> = self.storage.iter().map(|(n, h)| (n.0.clone(), h.clone())).collect(); tx.set_from_vec(column, prefix, leaves.encode()); - self.pending_added.clear(); - self.pending_removed.clear(); } /// Check if given block is a leaf. @@ -242,7 +232,7 @@ where self.storage.entry(number).or_insert_with(Vec::new).push(hash); } - // returns true if this leaf was contained, false otherwise. + // Returns true if this leaf was contained, false otherwise. fn remove_leaf(&mut self, number: &Reverse, hash: &H) -> bool { let mut empty = false; let removed = self.storage.get_mut(number).map_or(false, |leaves| { @@ -285,7 +275,7 @@ where pub fn undo_import(&mut self, displaced: ImportDisplaced) { let new_number = Reverse(displaced.displaced.number.0.clone() + N::one()); self.inner.remove_leaf(&new_number, &displaced.new_hash); - self.inner.insert_leaf(new_number, displaced.displaced.hash); + self.inner.insert_leaf(displaced.displaced.number, displaced.displaced.hash); } /// Undo a finalization operation by providing the displaced leaves. @@ -294,13 +284,6 @@ where } } -impl<'a, H: 'a, N: 'a> Drop for Undo<'a, H, N> { - fn drop(&mut self) { - self.inner.pending_added.clear(); - self.inner.pending_removed.clear(); - } -} - #[cfg(test)] mod tests { use super::*; @@ -315,15 +298,20 @@ mod tests { set.import(2_1, 2, 1_1); set.import(3_1, 3, 2_1); + assert_eq!(set.count(), 1); assert!(set.contains(3, 3_1)); assert!(!set.contains(2, 2_1)); assert!(!set.contains(1, 1_1)); assert!(!set.contains(0, 0)); set.import(2_2, 2, 1_1); + set.import(1_2, 1, 0); + set.import(2_3, 2, 1_2); + assert_eq!(set.count(), 3); assert!(set.contains(3, 3_1)); assert!(set.contains(2, 2_2)); + assert!(set.contains(2, 2_3)); } #[test] @@ -392,17 +380,42 @@ mod tests { } #[test] - fn undo_finalization() { + fn undo_import() { let mut set = LeafSet::new(); set.import(10_1u32, 10u32, 0u32); - set.import(11_1, 11, 10_2); - set.import(11_2, 11, 10_2); - set.import(12_1, 12, 11_123); + set.import(11_1, 11, 10_1); + set.import(11_2, 11, 10_1); + + let displaced = set.import(12_1, 12, 11_1).unwrap(); + assert_eq!(set.count(), 2); + assert!(set.contains(11, 11_2)); + assert!(set.contains(12, 12_1)); + + set.undo().undo_import(displaced); + assert_eq!(set.count(), 2); + assert!(set.contains(11, 11_1)); + assert!(set.contains(11, 11_2)); + } + + #[test] + fn undo_finalization() { + let mut set = LeafSet::new(); + set.import(9_1u32, 9u32, 0u32); + set.import(10_1, 10, 9_1); + set.import(10_2, 10, 9_1); + set.import(11_1, 11, 10_1); + set.import(11_2, 11, 10_1); + set.import(12_1, 12, 11_2); let displaced = set.finalize_height(11); - assert!(!set.contains(10, 10_1)); + assert_eq!(set.count(), 2); + assert!(set.contains(11, 11_1)); + assert!(set.contains(12, 12_1)); set.undo().undo_finalization(displaced); - assert!(set.contains(10, 10_1)); + assert_eq!(set.count(), 3); + assert!(set.contains(11, 11_1)); + assert!(set.contains(12, 12_1)); + assert!(set.contains(10, 10_2)); } } From c38d02bfa2e5cc11e905bc1bc1ded8529f445f61 Mon Sep 17 00:00:00 2001 From: Falco Hirschenberger Date: Mon, 25 Jul 2022 16:53:12 +0200 Subject: [PATCH 38/46] Add era to `Unbonded` event (#11770) * Add era to `Unbonded` event fixes #11749 * Fix missing tests * Add comment for `era` field in `Unbonded` struct. Co-authored-by: parity-processbot <> --- frame/nomination-pools/src/lib.rs | 4 +- frame/nomination-pools/src/tests.rs | 131 ++++++++++++------ .../nomination-pools/test-staking/src/lib.rs | 60 ++++++-- 3 files changed, 134 insertions(+), 61 deletions(-) diff --git a/frame/nomination-pools/src/lib.rs b/frame/nomination-pools/src/lib.rs index 514267e8bf4af..203c13f2f3496 100644 --- a/frame/nomination-pools/src/lib.rs +++ b/frame/nomination-pools/src/lib.rs @@ -1349,7 +1349,7 @@ pub mod pallet { /// pool. /// - `points` is the number of points that are issued as a result of `balance` being /// dissolved into the corresponding unbonding pool. - /// + /// - `era` is the era in which the balance will be unbonded. /// In the absence of slashing, these values will match. In the presence of slashing, the /// number of points that are issued in the unbonding pool will be less than the amount /// requested to be unbonded. @@ -1358,6 +1358,7 @@ pub mod pallet { pool_id: PoolId, balance: BalanceOf, points: BalanceOf, + era: EraIndex, }, /// A member has withdrawn from their pool. /// @@ -1683,6 +1684,7 @@ pub mod pallet { pool_id: member.pool_id, points: points_unbonded, balance: unbonding_balance, + era: unbond_era, }); // Now that we know everything has worked write the items to storage. diff --git a/frame/nomination-pools/src/tests.rs b/frame/nomination-pools/src/tests.rs index c971239bef507..b59f18cca72a2 100644 --- a/frame/nomination-pools/src/tests.rs +++ b/frame/nomination-pools/src/tests.rs @@ -869,7 +869,7 @@ mod claim_payout { Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 11, pool_id: 1, bonded: 11, joined: true }, - Event::Unbonded { member: 11, pool_id: 1, points: 11, balance: 11 } + Event::Unbonded { member: 11, pool_id: 1, points: 11, balance: 11, era: 3 } ] ); }); @@ -1374,7 +1374,7 @@ mod claim_payout { assert_eq!( pool_events_since_last_call(), vec![ - Event::Unbonded { member: 20, pool_id: 1, balance: 10, points: 10 }, + Event::Unbonded { member: 20, pool_id: 1, balance: 10, points: 10, era: 3 }, Event::PaidOut { member: 10, pool_id: 1, payout: 50 }, Event::PaidOut { member: 20, pool_id: 1, payout: 50 }, ] @@ -1808,12 +1808,12 @@ mod claim_payout { Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, Event::Bonded { member: 30, pool_id: 1, bonded: 20, joined: true }, - Event::Unbonded { member: 20, pool_id: 1, balance: 10, points: 10 }, + Event::Unbonded { member: 20, pool_id: 1, balance: 10, points: 10, era: 3 }, Event::PaidOut { member: 30, pool_id: 1, payout: 15 }, - Event::Unbonded { member: 30, pool_id: 1, balance: 10, points: 10 }, - Event::Unbonded { member: 30, pool_id: 1, balance: 5, points: 5 }, + Event::Unbonded { member: 30, pool_id: 1, balance: 10, points: 10, era: 3 }, + Event::Unbonded { member: 30, pool_id: 1, balance: 5, points: 5, era: 3 }, Event::PaidOut { member: 20, pool_id: 1, payout: 7 }, - Event::Unbonded { member: 20, pool_id: 1, balance: 5, points: 5 }, + Event::Unbonded { member: 20, pool_id: 1, balance: 5, points: 5, era: 3 }, Event::PaidOut { member: 10, pool_id: 1, payout: 7 } ] ); @@ -1865,10 +1865,10 @@ mod claim_payout { pool_events_since_last_call(), vec![ Event::StateChanged { pool_id: 1, new_state: PoolState::Destroying }, - Event::Unbonded { member: 20, pool_id: 1, balance: 20, points: 20 }, + Event::Unbonded { member: 20, pool_id: 1, balance: 20, points: 20, era: 3 }, Event::Withdrawn { member: 20, pool_id: 1, balance: 20, points: 20 }, Event::MemberRemoved { pool_id: 1, member: 20 }, - Event::Unbonded { member: 10, pool_id: 1, balance: 10, points: 10 }, + Event::Unbonded { member: 10, pool_id: 1, balance: 10, points: 10, era: 6 }, Event::Withdrawn { member: 10, pool_id: 1, balance: 10, points: 10 }, Event::MemberRemoved { pool_id: 1, member: 10 }, Event::Destroyed { pool_id: 1 } @@ -2269,7 +2269,7 @@ mod unbond { Event::Bonded { member: 40, pool_id: 1, bonded: 40, joined: true }, Event::Bonded { member: 550, pool_id: 1, bonded: 550, joined: true }, Event::PaidOut { member: 40, pool_id: 1, payout: 40 }, - Event::Unbonded { member: 40, pool_id: 1, points: 6, balance: 6 } + Event::Unbonded { member: 40, pool_id: 1, points: 6, balance: 6, era: 3 } ] ); @@ -2311,7 +2311,13 @@ mod unbond { pool_events_since_last_call(), vec![ Event::PaidOut { member: 550, pool_id: 1, payout: 550 }, - Event::Unbonded { member: 550, pool_id: 1, points: 92, balance: 92 } + Event::Unbonded { + member: 550, + pool_id: 1, + points: 92, + balance: 92, + era: 3 + } ] ); @@ -2349,7 +2355,7 @@ mod unbond { Event::Withdrawn { member: 550, pool_id: 1, points: 92, balance: 92 }, Event::MemberRemoved { pool_id: 1, member: 550 }, Event::PaidOut { member: 10, pool_id: 1, payout: 10 }, - Event::Unbonded { member: 10, pool_id: 1, points: 2, balance: 2 } + Event::Unbonded { member: 10, pool_id: 1, points: 2, balance: 2, era: 6 } ] ); }); @@ -2395,7 +2401,7 @@ mod unbond { vec![ Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, - Event::Unbonded { member: 10, pool_id: 1, points: 10, balance: 10 } + Event::Unbonded { member: 10, pool_id: 1, points: 10, balance: 10, era: 9 } ] ); }); @@ -2430,7 +2436,13 @@ mod unbond { Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 100, pool_id: 1, bonded: 100, joined: true }, Event::Bonded { member: 200, pool_id: 1, bonded: 200, joined: true }, - Event::Unbonded { member: 100, pool_id: 1, points: 100, balance: 100 }, + Event::Unbonded { + member: 100, + pool_id: 1, + points: 100, + balance: 100, + era: 3 + }, ] ); @@ -2439,7 +2451,13 @@ mod unbond { assert_eq!( pool_events_since_last_call(), - vec![Event::Unbonded { member: 200, pool_id: 1, points: 200, balance: 200 }] + vec![Event::Unbonded { + member: 200, + pool_id: 1, + points: 200, + balance: 200, + era: 3 + }] ); assert_eq!( @@ -2508,7 +2526,7 @@ mod unbond { Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 100, pool_id: 1, bonded: 100, joined: true }, - Event::Unbonded { member: 100, pool_id: 1, points: 100, balance: 100 } + Event::Unbonded { member: 100, pool_id: 1, points: 100, balance: 100, era: 3 } ] ); @@ -2659,7 +2677,7 @@ mod unbond { vec![ Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, - Event::Unbonded { member: 10, pool_id: 1, points: 1, balance: 1 } + Event::Unbonded { member: 10, pool_id: 1, points: 1, balance: 1, era: 3 } ] ); @@ -2685,7 +2703,7 @@ mod unbond { ); assert_eq!( pool_events_since_last_call(), - vec![Event::Unbonded { member: 10, pool_id: 1, points: 5, balance: 5 }] + vec![Event::Unbonded { member: 10, pool_id: 1, points: 5, balance: 5, era: 3 }] ); // when: casual further unbond, next era. @@ -2712,7 +2730,7 @@ mod unbond { ); assert_eq!( pool_events_since_last_call(), - vec![Event::Unbonded { member: 10, pool_id: 1, points: 1, balance: 1 }] + vec![Event::Unbonded { member: 10, pool_id: 1, points: 1, balance: 1, era: 4 }] ); // when: unbonding more than our active: error @@ -2747,7 +2765,7 @@ mod unbond { ); assert_eq!( pool_events_since_last_call(), - vec![Event::Unbonded { member: 10, pool_id: 1, points: 3, balance: 3 }] + vec![Event::Unbonded { member: 10, pool_id: 1, points: 3, balance: 3, era: 4 }] ); }); } @@ -2792,9 +2810,9 @@ mod unbond { Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, - Event::Unbonded { member: 20, pool_id: 1, balance: 2, points: 2 }, - Event::Unbonded { member: 20, pool_id: 1, balance: 3, points: 3 }, - Event::Unbonded { member: 20, pool_id: 1, balance: 1, points: 1 } + Event::Unbonded { member: 20, pool_id: 1, points: 2, balance: 2, era: 3 }, + Event::Unbonded { member: 20, pool_id: 1, points: 3, balance: 3, era: 4 }, + Event::Unbonded { member: 20, pool_id: 1, points: 1, balance: 1, era: 5 } ] ); }) @@ -2827,7 +2845,7 @@ mod unbond { vec![ Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, - Event::Unbonded { member: 10, pool_id: 1, points: 3, balance: 3 } + Event::Unbonded { member: 10, pool_id: 1, points: 3, balance: 3, era: 3 } ] ); }); @@ -2873,7 +2891,7 @@ mod unbond { Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: true }, Event::PaidOut { member: 20, pool_id: 1, payout: 10 }, - Event::Unbonded { member: 20, pool_id: 1, balance: 2, points: 2 } + Event::Unbonded { member: 20, pool_id: 1, balance: 2, points: 2, era: 3 } ] ); @@ -2889,7 +2907,7 @@ mod unbond { vec![ // 2/3 of ed, which is 20's share. Event::PaidOut { member: 20, pool_id: 1, payout: 6 }, - Event::Unbonded { member: 20, pool_id: 1, points: 3, balance: 3 } + Event::Unbonded { member: 20, pool_id: 1, points: 3, balance: 3, era: 4 } ] ); @@ -2904,7 +2922,7 @@ mod unbond { pool_events_since_last_call(), vec![ Event::PaidOut { member: 20, pool_id: 1, payout: 3 }, - Event::Unbonded { member: 20, pool_id: 1, points: 5, balance: 5 } + Event::Unbonded { member: 20, pool_id: 1, points: 5, balance: 5, era: 5 } ] ); @@ -3003,7 +3021,6 @@ mod withdraw_unbonded { with_era: Default::default() } ); - assert_eq!( pool_events_since_last_call(), vec![ @@ -3011,8 +3028,14 @@ mod withdraw_unbonded { Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 40, pool_id: 1, bonded: 40, joined: true }, Event::Bonded { member: 550, pool_id: 1, bonded: 550, joined: true }, - Event::Unbonded { member: 550, pool_id: 1, points: 550, balance: 550 }, - Event::Unbonded { member: 40, pool_id: 1, points: 40, balance: 40 }, + Event::Unbonded { + member: 550, + pool_id: 1, + points: 550, + balance: 550, + era: 3 + }, + Event::Unbonded { member: 40, pool_id: 1, points: 40, balance: 40, era: 3 }, ] ); assert_eq!( @@ -3077,7 +3100,7 @@ mod withdraw_unbonded { assert_eq!( pool_events_since_last_call(), vec![ - Event::Unbonded { member: 10, pool_id: 1, balance: 5, points: 5 }, + Event::Unbonded { member: 10, pool_id: 1, balance: 5, points: 5, era: 9 }, Event::Withdrawn { member: 10, pool_id: 1, balance: 5, points: 5 }, Event::MemberRemoved { pool_id: 1, member: 10 }, Event::Destroyed { pool_id: 1 } @@ -3122,8 +3145,14 @@ mod withdraw_unbonded { Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 40, pool_id: 1, bonded: 40, joined: true }, Event::Bonded { member: 550, pool_id: 1, bonded: 550, joined: true }, - Event::Unbonded { member: 40, pool_id: 1, balance: 20, points: 20 }, - Event::Unbonded { member: 550, pool_id: 1, balance: 275, points: 275 }, + Event::Unbonded { member: 40, pool_id: 1, balance: 20, points: 20, era: 3 }, + Event::Unbonded { + member: 550, + pool_id: 1, + balance: 275, + points: 275, + era: 3, + } ] ); assert_eq!( @@ -3199,7 +3228,7 @@ mod withdraw_unbonded { assert_eq!( pool_events_since_last_call(), vec![ - Event::Unbonded { member: 10, pool_id: 1, points: 5, balance: 5 }, + Event::Unbonded { member: 10, pool_id: 1, points: 5, balance: 5, era: 6 }, Event::Withdrawn { member: 10, pool_id: 1, points: 5, balance: 5 }, Event::MemberRemoved { pool_id: 1, member: 10 }, Event::Destroyed { pool_id: 1 } @@ -3313,8 +3342,20 @@ mod withdraw_unbonded { Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 100, pool_id: 1, bonded: 100, joined: true }, Event::Bonded { member: 200, pool_id: 1, bonded: 200, joined: true }, - Event::Unbonded { member: 100, pool_id: 1, points: 100, balance: 100 }, - Event::Unbonded { member: 200, pool_id: 1, points: 200, balance: 200 } + Event::Unbonded { + member: 100, + pool_id: 1, + points: 100, + balance: 100, + era: 3 + }, + Event::Unbonded { + member: 200, + pool_id: 1, + points: 200, + balance: 200, + era: 3 + } ] ); @@ -3391,7 +3432,7 @@ mod withdraw_unbonded { Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 100, pool_id: 1, bonded: 100, joined: true }, - Event::Unbonded { member: 100, pool_id: 1, points: 100, balance: 100 }, + Event::Unbonded { member: 100, pool_id: 1, points: 100, balance: 100, era: 3 }, Event::Withdrawn { member: 100, pool_id: 1, points: 100, balance: 100 }, Event::MemberRemoved { pool_id: 1, member: 100 } ] @@ -3431,8 +3472,8 @@ mod withdraw_unbonded { Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: false }, - Event::Unbonded { member: 10, pool_id: 1, points: 6, balance: 6 }, - Event::Unbonded { member: 10, pool_id: 1, points: 1, balance: 1 } + Event::Unbonded { member: 10, pool_id: 1, points: 6, balance: 6, era: 3 }, + Event::Unbonded { member: 10, pool_id: 1, points: 1, balance: 1, era: 4 } ] ); @@ -3518,8 +3559,8 @@ mod withdraw_unbonded { Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 11, pool_id: 1, bonded: 10, joined: true }, - Event::Unbonded { member: 11, pool_id: 1, points: 6, balance: 6 }, - Event::Unbonded { member: 11, pool_id: 1, points: 1, balance: 1 } + Event::Unbonded { member: 11, pool_id: 1, points: 6, balance: 6, era: 3 }, + Event::Unbonded { member: 11, pool_id: 1, points: 1, balance: 1, era: 4 } ] ); @@ -3608,8 +3649,8 @@ mod withdraw_unbonded { Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 100, pool_id: 1, bonded: 100, joined: true }, - Event::Unbonded { member: 100, pool_id: 1, points: 75, balance: 75 }, - Event::Unbonded { member: 100, pool_id: 1, points: 25, balance: 25 }, + Event::Unbonded { member: 100, pool_id: 1, points: 75, balance: 75, era: 3 }, + Event::Unbonded { member: 100, pool_id: 1, points: 25, balance: 25, era: 4 }, Event::Withdrawn { member: 100, pool_id: 1, points: 75, balance: 75 }, ] ); @@ -3680,9 +3721,9 @@ mod withdraw_unbonded { Event::Created { depositor: 10, pool_id: 1 }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: true }, Event::Bonded { member: 10, pool_id: 1, bonded: 10, joined: false }, - Event::Unbonded { member: 10, pool_id: 1, balance: 7, points: 7 }, - Event::Unbonded { member: 10, pool_id: 1, balance: 3, points: 3 }, - Event::Unbonded { member: 10, pool_id: 1, balance: 10, points: 10 }, + Event::Unbonded { member: 10, pool_id: 1, balance: 7, points: 7, era: 3 }, + Event::Unbonded { member: 10, pool_id: 1, balance: 3, points: 3, era: 4 }, + Event::Unbonded { member: 10, pool_id: 1, balance: 10, points: 10, era: 4 }, Event::Withdrawn { member: 10, pool_id: 1, balance: 7, points: 7 } ] ); diff --git a/frame/nomination-pools/test-staking/src/lib.rs b/frame/nomination-pools/test-staking/src/lib.rs index eed8e5fd390cd..5a7cd494362ca 100644 --- a/frame/nomination-pools/test-staking/src/lib.rs +++ b/frame/nomination-pools/test-staking/src/lib.rs @@ -95,8 +95,8 @@ fn pool_lifecycle_e2e() { pool_events_since_last_call(), vec![ PoolsEvent::StateChanged { pool_id: 1, new_state: PoolState::Destroying }, - PoolsEvent::Unbonded { member: 20, pool_id: 1, points: 10, balance: 10 }, - PoolsEvent::Unbonded { member: 21, pool_id: 1, points: 10, balance: 10 }, + PoolsEvent::Unbonded { member: 20, pool_id: 1, points: 10, balance: 10, era: 3 }, + PoolsEvent::Unbonded { member: 21, pool_id: 1, points: 10, balance: 10, era: 3 }, ] ); @@ -159,7 +159,7 @@ fn pool_lifecycle_e2e() { ); assert_eq!( pool_events_since_last_call(), - vec![PoolsEvent::Unbonded { member: 10, pool_id: 1, points: 50, balance: 50 }] + vec![PoolsEvent::Unbonded { member: 10, pool_id: 1, points: 50, balance: 50, era: 6 }] ); // waiting another bonding duration: @@ -237,8 +237,8 @@ fn pool_slash_e2e() { assert_eq!( pool_events_since_last_call(), vec![ - PoolsEvent::Unbonded { member: 10, pool_id: 1, balance: 10, points: 10 }, - PoolsEvent::Unbonded { member: 20, pool_id: 1, balance: 10, points: 10 } + PoolsEvent::Unbonded { member: 10, pool_id: 1, balance: 10, points: 10, era: 4 }, + PoolsEvent::Unbonded { member: 20, pool_id: 1, balance: 10, points: 10, era: 4 } ] ); @@ -262,9 +262,9 @@ fn pool_slash_e2e() { assert_eq!( pool_events_since_last_call(), vec![ - PoolsEvent::Unbonded { member: 10, pool_id: 1, balance: 10, points: 10 }, - PoolsEvent::Unbonded { member: 20, pool_id: 1, balance: 10, points: 10 }, - PoolsEvent::Unbonded { member: 21, pool_id: 1, balance: 10, points: 10 }, + PoolsEvent::Unbonded { member: 10, pool_id: 1, balance: 10, points: 10, era: 5 }, + PoolsEvent::Unbonded { member: 20, pool_id: 1, balance: 10, points: 10, era: 5 }, + PoolsEvent::Unbonded { member: 21, pool_id: 1, balance: 10, points: 10, era: 5 }, ] ); @@ -305,7 +305,7 @@ fn pool_slash_e2e() { assert_eq!(staking_events_since_last_call(), vec![StakingEvent::Unbonded(POOL1_BONDED, 5)]); assert_eq!( pool_events_since_last_call(), - vec![PoolsEvent::Unbonded { member: 21, pool_id: 1, balance: 5, points: 5 }] + vec![PoolsEvent::Unbonded { member: 21, pool_id: 1, balance: 5, points: 5, era: 6 }] ); // now we start withdrawing. we do it all at once, at era 6 where 20 and 21 are fully free. @@ -342,7 +342,7 @@ fn pool_slash_e2e() { pool_events_since_last_call(), vec![ PoolsEvent::StateChanged { pool_id: 1, new_state: PoolState::Destroying }, - PoolsEvent::Unbonded { member: 10, pool_id: 1, points: 10, balance: 10 } + PoolsEvent::Unbonded { member: 10, pool_id: 1, points: 10, balance: 10, era: 9 } ] ); @@ -432,7 +432,13 @@ fn pool_slash_proportional() { ); assert_eq!( pool_events_since_last_call(), - vec![PoolsEvent::Unbonded { member: 20, pool_id: 1, balance: bond, points: bond }] + vec![PoolsEvent::Unbonded { + member: 20, + pool_id: 1, + balance: bond, + points: bond, + era: 127 + }] ); CurrentEra::::set(Some(100)); @@ -443,7 +449,13 @@ fn pool_slash_proportional() { ); assert_eq!( pool_events_since_last_call(), - vec![PoolsEvent::Unbonded { member: 21, pool_id: 1, balance: bond, points: bond }] + vec![PoolsEvent::Unbonded { + member: 21, + pool_id: 1, + balance: bond, + points: bond, + era: 128 + }] ); CurrentEra::::set(Some(101)); @@ -454,7 +466,13 @@ fn pool_slash_proportional() { ); assert_eq!( pool_events_since_last_call(), - vec![PoolsEvent::Unbonded { member: 22, pool_id: 1, balance: bond, points: bond }] + vec![PoolsEvent::Unbonded { + member: 22, + pool_id: 1, + balance: bond, + points: bond, + era: 129 + }] ); // Apply a slash that happened in era 100. This is typically applied with a delay. @@ -531,7 +549,13 @@ fn pool_slash_non_proportional_only_bonded_pool() { ); assert_eq!( pool_events_since_last_call(), - vec![PoolsEvent::Unbonded { member: 20, pool_id: 1, balance: bond, points: bond }] + vec![PoolsEvent::Unbonded { + member: 20, + pool_id: 1, + balance: bond, + points: bond, + era: 127 + }] ); // slash for 30. This will be deducted only from the bonded pool. @@ -598,7 +622,13 @@ fn pool_slash_non_proportional_bonded_pool_and_chunks() { ); assert_eq!( pool_events_since_last_call(), - vec![PoolsEvent::Unbonded { member: 20, pool_id: 1, balance: bond, points: bond }] + vec![PoolsEvent::Unbonded { + member: 20, + pool_id: 1, + balance: bond, + points: bond, + era: 127 + }] ); // slash 50. This will be deducted only from the bonded pool and one of the unbonding pools. From eefb4fad378181d8916c28f320a1ba383157c990 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Mon, 25 Jul 2022 16:20:19 +0100 Subject: [PATCH 39/46] remove FunctionOf (#11897) * remove FunctionOf * fix docs Co-authored-by: parity-processbot <> --- frame/support/src/weights.rs | 89 +++--------------------------------- 1 file changed, 6 insertions(+), 83 deletions(-) diff --git a/frame/support/src/weights.rs b/frame/support/src/weights.rs index dd49409ccc912..c37a72536bddf 100644 --- a/frame/support/src/weights.rs +++ b/frame/support/src/weights.rs @@ -91,27 +91,19 @@ //! # fn main() {} //! ``` //! -//! ### 2. Define weights as a function of input arguments using `FunctionOf` tuple struct. +//! ### 2. Define weights as a function of input arguments. //! -//! This struct works in a similar manner as above. 3 items must be provided and each can be either -//! a fixed value or a function/closure with the same parameters list as the dispatchable function -//! itself, wrapper in a tuple. -//! -//! Using this only makes sense if you want to use a function for at least one of the elements. If -//! all 3 are static values, providing a raw tuple is easier. +//! The arguments of the dispatch are available in the weight expressions as a borrowed value. //! //! ``` //! # use frame_system::Config; -//! # use frame_support::weights::{DispatchClass, FunctionOf, Pays}; +//! # use frame_support::weights::{DispatchClass, Pays}; //! frame_support::decl_module! { //! pub struct Module for enum Call where origin: T::Origin { -//! #[weight = FunctionOf( -//! // weight, function. -//! |args: (&u32, &u64)| *args.0 as u64 + args.1, -//! // class, fixed. +//! #[weight = ( +//! *a as u64 + *b, //! DispatchClass::Operational, -//! // pays fee, function. -//! |args: (&u32, &u64)| if *args.0 > 1000 { Pays::Yes } else { Pays::No }, +//! if *a > 1000 { Pays::Yes } else { Pays::No } //! )] //! fn dispatching(origin, a: u32, b: u64) { unimplemented!() } //! } @@ -508,75 +500,6 @@ impl PaysFee for (Weight, Pays) { } } -/// A struct to represent a weight which is a function of the input arguments. The given items have -/// the following types: -/// -/// - `WD`: a raw `Weight` value or a closure that returns a `Weight` with the same argument list as -/// the dispatched, wrapped in a tuple. -/// - `CD`: a raw `DispatchClass` value or a closure that returns a `DispatchClass` with the same -/// argument list as the dispatched, wrapped in a tuple. -/// - `PF`: a `Pays` variant for whether this dispatch pays fee or not or a closure that returns a -/// `Pays` variant with the same argument list as the dispatched, wrapped in a tuple. -#[deprecated = "Function arguments are available directly inside the annotation now."] -pub struct FunctionOf(pub WD, pub CD, pub PF); - -// `WeighData` as a raw value -#[allow(deprecated)] -impl WeighData for FunctionOf { - fn weigh_data(&self, _: Args) -> Weight { - self.0 - } -} - -// `WeighData` as a closure -#[allow(deprecated)] -impl WeighData for FunctionOf -where - WD: Fn(Args) -> Weight, -{ - fn weigh_data(&self, args: Args) -> Weight { - (self.0)(args) - } -} - -// `ClassifyDispatch` as a raw value -#[allow(deprecated)] -impl ClassifyDispatch for FunctionOf { - fn classify_dispatch(&self, _: Args) -> DispatchClass { - self.1 - } -} - -// `ClassifyDispatch` as a raw value -#[allow(deprecated)] -impl ClassifyDispatch for FunctionOf -where - CD: Fn(Args) -> DispatchClass, -{ - fn classify_dispatch(&self, args: Args) -> DispatchClass { - (self.1)(args) - } -} - -// `PaysFee` as a raw value -#[allow(deprecated)] -impl PaysFee for FunctionOf { - fn pays_fee(&self, _: Args) -> Pays { - self.2 - } -} - -// `PaysFee` as a closure -#[allow(deprecated)] -impl PaysFee for FunctionOf -where - PF: Fn(Args) -> Pays, -{ - fn pays_fee(&self, args: Args) -> Pays { - (self.2)(args) - } -} - /// Implementation for unchecked extrinsic. impl GetDispatchInfo for UncheckedExtrinsic From 720093428b90363cb909162ef7c05e5bd51ae626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Thei=C3=9Fen?= Date: Mon, 25 Jul 2022 17:48:01 +0200 Subject: [PATCH 40/46] contracts: Allow `ChainExtension::call()` to access `&mut self` (#11874) * Give chain extensions the ability to store some temporary values * Update frame/contracts/src/wasm/runtime.rs Co-authored-by: Hernando Castano * Rename func_id -> id * Replace `id` param by two functions on `env` Co-authored-by: Hernando Castano --- frame/contracts/fixtures/chain_extension.wat | 4 +- .../fixtures/chain_extension_temp_storage.wat | 85 ++++++++++ frame/contracts/src/chain_extension.rs | 41 +++-- frame/contracts/src/lib.rs | 2 +- frame/contracts/src/tests.rs | 150 +++++++++++++----- frame/contracts/src/wasm/runtime.rs | 28 +++- 6 files changed, 254 insertions(+), 56 deletions(-) create mode 100644 frame/contracts/fixtures/chain_extension_temp_storage.wat diff --git a/frame/contracts/fixtures/chain_extension.wat b/frame/contracts/fixtures/chain_extension.wat index 9b2534c540ab8..7cc7335052e90 100644 --- a/frame/contracts/fixtures/chain_extension.wat +++ b/frame/contracts/fixtures/chain_extension.wat @@ -31,14 +31,14 @@ ;; the chain extension passes through the input and returns it as output (call $seal_call_chain_extension - (i32.load (i32.const 4)) ;; func_id + (i32.load (i32.const 4)) ;; id (i32.const 4) ;; input_ptr (i32.load (i32.const 0)) ;; input_len (i32.const 16) ;; output_ptr (i32.const 12) ;; output_len_ptr ) - ;; the chain extension passes through the func_id + ;; the chain extension passes through the id (call $assert (i32.eq (i32.load (i32.const 4)))) (call $seal_return (i32.const 0) (i32.const 16) (i32.load (i32.const 12))) diff --git a/frame/contracts/fixtures/chain_extension_temp_storage.wat b/frame/contracts/fixtures/chain_extension_temp_storage.wat new file mode 100644 index 0000000000000..b481abb5bc7c9 --- /dev/null +++ b/frame/contracts/fixtures/chain_extension_temp_storage.wat @@ -0,0 +1,85 @@ +;; Call chain extension two times with the specified func_ids +;; It then calls itself once +(module + (import "seal0" "seal_call_chain_extension" + (func $seal_call_chain_extension (param i32 i32 i32 i32 i32) (result i32)) + ) + (import "seal0" "seal_input" (func $seal_input (param i32 i32))) + (import "seal0" "seal_address" (func $seal_address (param i32 i32))) + (import "seal1" "seal_call" (func $seal_call (param i32 i32 i64 i32 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 16 16)) + + (func $assert (param i32) + (block $ok + (br_if $ok (get_local 0)) + (unreachable) + ) + ) + + ;; [0, 4) len of input buffer: 8 byte (func_ids) + 1byte (stop_recurse) + (data (i32.const 0) "\09") + + ;; [4, 16) buffer for input + + ;; [16, 48] buffer for self address + + ;; [48, 52] len of self address buffer + (data (i32.const 48) "\20") + + (func (export "deploy")) + + (func (export "call") + ;; input: (func_id1: i32, func_id2: i32, stop_recurse: i8) + (call $seal_input (i32.const 4) (i32.const 0)) + + (call $seal_call_chain_extension + (i32.load (i32.const 4)) ;; id + (i32.const 0) ;; input_ptr + (i32.const 0) ;; input_len + (i32.const 0xffffffff) ;; u32 max sentinel value: do not copy output + (i32.const 0) ;; output_len_ptr + ) + drop + + (call $seal_call_chain_extension + (i32.load (i32.const 8)) ;; _id + (i32.const 0) ;; input_ptr + (i32.const 0) ;; input_len + (i32.const 0xffffffff) ;; u32 max sentinel value: do not copy output + (i32.const 0) ;; output_len_ptr + ) + drop + + (if (i32.eqz (i32.load8_u (i32.const 12))) + (then + ;; stop recursion + (i32.store8 (i32.const 12) (i32.const 1)) + + ;; load own address into buffer + (call $seal_address (i32.const 16) (i32.const 48)) + + ;; call function 2 + 3 of chainext 3 next time + ;; (3 << 16) | 2 + ;; (3 << 16) | 3 + (i32.store (i32.const 4) (i32.const 196610)) + (i32.store (i32.const 8) (i32.const 196611)) + + ;; call self + (call $seal_call + (i32.const 8) ;; Set ALLOW_REENTRY + (i32.const 16) ;; Pointer to "callee" address. + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 512) ;; Pointer to the buffer with value to transfer + (i32.const 4) ;; Pointer to input data buffer address + (i32.load (i32.const 0)) ;; Length of input data buffer + (i32.const 4294967295) ;; u32 max value is the sentinel value: do not copy output + (i32.const 0) ;; Length is ignored in this case + ) + + ;; check that call succeeded of call + (call $assert (i32.eqz)) + ) + (else) + ) + ) +) diff --git a/frame/contracts/src/chain_extension.rs b/frame/contracts/src/chain_extension.rs index 536d58c94f68f..23242a2a542c1 100644 --- a/frame/contracts/src/chain_extension.rs +++ b/frame/contracts/src/chain_extension.rs @@ -33,7 +33,7 @@ //! //! Often there is a need for having multiple chain extensions. This is often the case when //! some generally useful off-the-shelf extensions should be included. To have multiple chain -//! extensions they can be put into a tuple which is then passed to `[Config::ChainExtension]` like +//! extensions they can be put into a tuple which is then passed to [`Config::ChainExtension`] like //! this `type Extensions = (ExtensionA, ExtensionB)`. //! //! However, only extensions implementing [`RegisteredChainExtension`] can be put into a tuple. @@ -94,6 +94,12 @@ pub type Result = sp_std::result::Result; /// In order to create a custom chain extension this trait must be implemented and supplied /// to the pallet contracts configuration trait as the associated type of the same name. /// Consult the [module documentation](self) for a general explanation of chain extensions. +/// +/// # Lifetime +/// +/// The extension will be [`Default`] initialized at the beginning of each call +/// (**not** per call stack) and dropped afterwards. Hence any value held inside the extension +/// can be used as a per-call scratch buffer. pub trait ChainExtension { /// Call the chain extension logic. /// @@ -102,8 +108,6 @@ pub trait ChainExtension { /// imported wasm function. /// /// # Parameters - /// - `func_id`: The first argument to `seal_call_chain_extension`. Usually used to determine - /// which function to realize. /// - `env`: Access to the remaining arguments and the execution environment. /// /// # Return @@ -111,7 +115,7 @@ pub trait ChainExtension { /// In case of `Err` the contract execution is immediately suspended and the passed error /// is returned to the caller. Otherwise the value of [`RetVal`] determines the exit /// behaviour. - fn call(func_id: u32, env: Environment) -> Result + fn call(&mut self, env: Environment) -> Result where E: Ext, ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>; @@ -132,7 +136,7 @@ pub trait ChainExtension { /// /// An extension that implements this trait can be put in a tuple in order to have multiple /// extensions available. The tuple implementation routes requests based on the first two -/// most significant bytes of the `func_id` passed to `call`. +/// most significant bytes of the `id` passed to `call`. /// /// If this extensions is to be used by multiple runtimes consider /// [registering it](https://github.com/paritytech/chainextension-registry) to ensure that there @@ -150,15 +154,15 @@ pub trait RegisteredChainExtension: ChainExtension { #[impl_trait_for_tuples::impl_for_tuples(10)] #[tuple_types_custom_trait_bound(RegisteredChainExtension)] impl ChainExtension for Tuple { - fn call(func_id: u32, mut env: Environment) -> Result + fn call(&mut self, mut env: Environment) -> Result where E: Ext, ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>, { for_tuples!( #( - if (Tuple::ID == (func_id >> 16) as u16) && Tuple::enabled() { - return Tuple::call(func_id, env); + if (Tuple::ID == env.ext_id()) && Tuple::enabled() { + return Tuple.call(env); } )* ); @@ -206,6 +210,22 @@ impl<'a, 'b, E: Ext, S: state::State> Environment<'a, 'b, E, S> where ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>, { + /// The function id within the `id` passed by a contract. + /// + /// It returns the two least significant bytes of the `id` passed by a contract as the other + /// two bytes represent the chain extension itself (the code which is calling this function). + pub fn func_id(&self) -> u16 { + (self.inner.id & 0x00FF) as u16 + } + + /// The chain extension id within the `id` passed by a contract. + /// + /// It returns the two most significant bytes of the `id` passed by a contract which represent + /// the chain extension itself (the code which is calling this function). + pub fn ext_id(&self) -> u16 { + (self.inner.id >> 16) as u16 + } + /// Charge the passed `amount` of weight from the overall limit. /// /// It returns `Ok` when there the remaining weight budget is larger than the passed @@ -251,13 +271,14 @@ impl<'a, 'b, E: Ext> Environment<'a, 'b, E, state::Init> { /// ever create this type. Chain extensions merely consume it. pub(crate) fn new( runtime: &'a mut Runtime<'b, E>, + id: u32, input_ptr: u32, input_len: u32, output_ptr: u32, output_len_ptr: u32, ) -> Self { Environment { - inner: Inner { runtime, input_ptr, input_len, output_ptr, output_len_ptr }, + inner: Inner { runtime, id, input_ptr, input_len, output_ptr, output_len_ptr }, phantom: PhantomData, } } @@ -406,6 +427,8 @@ struct Inner<'a, 'b, E: Ext> { /// The runtime contains all necessary functions to interact with the running contract. runtime: &'a mut Runtime<'b, E>, /// Verbatim argument passed to `seal_call_chain_extension`. + id: u32, + /// Verbatim argument passed to `seal_call_chain_extension`. input_ptr: u32, /// Verbatim argument passed to `seal_call_chain_extension`. input_len: u32, diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs index 60b30ffa25005..319bacaab7789 100644 --- a/frame/contracts/src/lib.rs +++ b/frame/contracts/src/lib.rs @@ -280,7 +280,7 @@ pub mod pallet { type WeightInfo: WeightInfo; /// Type that allows the runtime authors to add new host functions for a contract to call. - type ChainExtension: chain_extension::ChainExtension; + type ChainExtension: chain_extension::ChainExtension + Default; /// Cost schedule and limits. #[pallet::constant] diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index 85a0e9977d2d7..0febfec929b6e 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -118,10 +118,17 @@ pub struct TestExtension { last_seen_inputs: (u32, u32, u32, u32), } +#[derive(Default)] pub struct RevertingExtension; +#[derive(Default)] pub struct DisabledExtension; +#[derive(Default)] +pub struct TempStorageExtension { + storage: u32, +} + impl TestExtension { fn disable() { TEST_EXTENSION.with(|e| e.borrow_mut().enabled = false) @@ -143,18 +150,20 @@ impl Default for TestExtension { } impl ChainExtension for TestExtension { - fn call(func_id: u32, env: Environment) -> ExtensionResult + fn call(&mut self, env: Environment) -> ExtensionResult where E: Ext, ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>, { + let func_id = env.func_id(); + let id = env.ext_id() as u32 | func_id as u32; match func_id { 0 => { let mut env = env.buf_in_buf_out(); let input = env.read(8)?; env.write(&input, false, None)?; TEST_EXTENSION.with(|e| e.borrow_mut().last_seen_buffer = input); - Ok(RetVal::Converging(func_id)) + Ok(RetVal::Converging(id)) }, 1 => { let env = env.only_in(); @@ -162,17 +171,17 @@ impl ChainExtension for TestExtension { e.borrow_mut().last_seen_inputs = (env.val0(), env.val1(), env.val2(), env.val3()) }); - Ok(RetVal::Converging(func_id)) + Ok(RetVal::Converging(id)) }, 2 => { let mut env = env.buf_in_buf_out(); let weight = env.read(5)?[4].into(); env.charge_weight(weight)?; - Ok(RetVal::Converging(func_id)) + Ok(RetVal::Converging(id)) }, 3 => Ok(RetVal::Diverging { flags: ReturnFlags::REVERT, data: vec![42, 99] }), _ => { - panic!("Passed unknown func_id to test chain extension: {}", func_id); + panic!("Passed unknown id to test chain extension: {}", func_id); }, } } @@ -187,7 +196,7 @@ impl RegisteredChainExtension for TestExtension { } impl ChainExtension for RevertingExtension { - fn call(_func_id: u32, _env: Environment) -> ExtensionResult + fn call(&mut self, _env: Environment) -> ExtensionResult where E: Ext, ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>, @@ -205,7 +214,7 @@ impl RegisteredChainExtension for RevertingExtension { } impl ChainExtension for DisabledExtension { - fn call(_func_id: u32, _env: Environment) -> ExtensionResult + fn call(&mut self, _env: Environment) -> ExtensionResult where E: Ext, ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>, @@ -222,6 +231,37 @@ impl RegisteredChainExtension for DisabledExtension { const ID: u16 = 2; } +impl ChainExtension for TempStorageExtension { + fn call(&mut self, env: Environment) -> ExtensionResult + where + E: Ext, + ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>, + { + let func_id = env.func_id(); + match func_id { + 0 => self.storage = 42, + 1 => assert_eq!(self.storage, 42, "Storage is preserved inside the same call."), + 2 => { + assert_eq!(self.storage, 0, "Storage is different for different calls."); + self.storage = 99; + }, + 3 => assert_eq!(self.storage, 99, "Storage is preserved inside the same call."), + _ => { + panic!("Passed unknown id to test chain extension: {}", func_id); + }, + } + Ok(RetVal::Converging(0)) + } + + fn enabled() -> bool { + TEST_EXTENSION.with(|e| e.borrow().enabled) + } +} + +impl RegisteredChainExtension for TempStorageExtension { + const ID: u16 = 3; +} + parameter_types! { pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(2 * WEIGHT_PER_SECOND); @@ -325,7 +365,8 @@ impl Config for Test { type CallStack = [Frame; 31]; type WeightPrice = Self; type WeightInfo = (); - type ChainExtension = (TestExtension, DisabledExtension, RevertingExtension); + type ChainExtension = + (TestExtension, DisabledExtension, RevertingExtension, TempStorageExtension); type DeletionQueueDepth = ConstU32<1024>; type DeletionWeightLimit = ConstU64<500_000_000_000>; type Schedule = MySchedule; @@ -396,6 +437,29 @@ fn initialize_block(number: u64) { System::initialize(&number, &[0u8; 32].into(), &Default::default()); } +struct ExtensionInput<'a> { + extension_id: u16, + func_id: u16, + extra: &'a [u8], +} + +impl<'a> ExtensionInput<'a> { + fn to_vec(&self) -> Vec { + ((self.extension_id as u32) << 16 | (self.func_id as u32)) + .to_le_bytes() + .iter() + .chain(self.extra) + .cloned() + .collect() + } +} + +impl<'a> From> for Vec { + fn from(input: ExtensionInput) -> Vec { + input.to_vec() + } +} + // Perform a call to a plain account. // The actual transfer fails because we can only call contracts. // Then we check that at least the base costs where charged (no runtime gas costs.) @@ -1567,23 +1631,6 @@ fn disabled_chain_extension_errors_on_call() { #[test] fn chain_extension_works() { - struct Input<'a> { - extension_id: u16, - func_id: u16, - extra: &'a [u8], - } - - impl<'a> From> for Vec { - fn from(input: Input) -> Vec { - ((input.extension_id as u32) << 16 | (input.func_id as u32)) - .to_le_bytes() - .iter() - .chain(input.extra) - .cloned() - .collect() - } - } - let (code, hash) = compile_module::("chain_extension").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { let min_balance = ::Currency::minimum_balance(); @@ -1599,12 +1646,8 @@ fn chain_extension_works() { ),); let addr = Contracts::contract_address(&ALICE, &hash, &[]); - // The contract takes a up to 2 byte buffer where the first byte passed is used as - // as func_id to the chain extension which behaves differently based on the - // func_id. - // 0 = read input buffer and pass it through as output - let input: Vec = Input { extension_id: 0, func_id: 0, extra: &[99] }.into(); + let input: Vec = ExtensionInput { extension_id: 0, func_id: 0, extra: &[99] }.into(); let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, input.clone(), false); assert_eq!(TestExtension::last_seen_buffer(), input); @@ -1617,7 +1660,7 @@ fn chain_extension_works() { 0, GAS_LIMIT, None, - Input { extension_id: 0, func_id: 1, extra: &[] }.into(), + ExtensionInput { extension_id: 0, func_id: 1, extra: &[] }.into(), false, ) .result @@ -1632,7 +1675,7 @@ fn chain_extension_works() { 0, GAS_LIMIT, None, - Input { extension_id: 0, func_id: 2, extra: &[0] }.into(), + ExtensionInput { extension_id: 0, func_id: 2, extra: &[0] }.into(), false, ); assert_ok!(result.result); @@ -1643,7 +1686,7 @@ fn chain_extension_works() { 0, GAS_LIMIT, None, - Input { extension_id: 0, func_id: 2, extra: &[42] }.into(), + ExtensionInput { extension_id: 0, func_id: 2, extra: &[42] }.into(), false, ); assert_ok!(result.result); @@ -1654,7 +1697,7 @@ fn chain_extension_works() { 0, GAS_LIMIT, None, - Input { extension_id: 0, func_id: 2, extra: &[95] }.into(), + ExtensionInput { extension_id: 0, func_id: 2, extra: &[95] }.into(), false, ); assert_ok!(result.result); @@ -1667,7 +1710,7 @@ fn chain_extension_works() { 0, GAS_LIMIT, None, - Input { extension_id: 0, func_id: 3, extra: &[] }.into(), + ExtensionInput { extension_id: 0, func_id: 3, extra: &[] }.into(), false, ) .result @@ -1684,7 +1727,7 @@ fn chain_extension_works() { 0, GAS_LIMIT, None, - Input { extension_id: 1, func_id: 0, extra: &[] }.into(), + ExtensionInput { extension_id: 1, func_id: 0, extra: &[] }.into(), false, ) .result @@ -1701,13 +1744,46 @@ fn chain_extension_works() { 0, GAS_LIMIT, None, - Input { extension_id: 2, func_id: 0, extra: &[] }.into(), + ExtensionInput { extension_id: 2, func_id: 0, extra: &[] }.into(), ), Error::::NoChainExtension, ); }); } +#[test] +fn chain_extension_temp_storage_works() { + let (code, hash) = compile_module::("chain_extension_temp_storage").unwrap(); + ExtBuilder::default().existential_deposit(50).build().execute_with(|| { + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + min_balance * 100, + GAS_LIMIT, + None, + code, + vec![], + vec![], + ),); + let addr = Contracts::contract_address(&ALICE, &hash, &[]); + + // Call func 0 and func 1 back to back. + let stop_recursion = 0u8; + let mut input: Vec = ExtensionInput { extension_id: 3, func_id: 0, extra: &[] }.into(); + input.extend_from_slice( + ExtensionInput { extension_id: 3, func_id: 1, extra: &[stop_recursion] } + .to_vec() + .as_ref(), + ); + + assert_ok!( + Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, input.clone(), false) + .result + ); + }) +} + #[test] fn lazy_removal_works() { let (code, hash) = compile_module::("self_destruct").unwrap(); diff --git a/frame/contracts/src/wasm/runtime.rs b/frame/contracts/src/wasm/runtime.rs index c1757ba06412a..3d5e62f2c1333 100644 --- a/frame/contracts/src/wasm/runtime.rs +++ b/frame/contracts/src/wasm/runtime.rs @@ -65,7 +65,7 @@ impl KeyType { /// This enum can be extended in the future: New codes can be added but existing codes /// will not be changed or removed. This means that any contract **must not** exhaustively /// match return codes. Instead, contracts should prepare for unknown variants and deal with -/// those errors gracefuly in order to be forward compatible. +/// those errors gracefully in order to be forward compatible. #[repr(u32)] pub enum ReturnCode { /// API call successful. @@ -101,8 +101,9 @@ pub enum ReturnCode { } impl ConvertibleToWasm for ReturnCode { - type NativeType = Self; const VALUE_TYPE: ValueType = ValueType::I32; + type NativeType = Self; + fn to_typed_value(self) -> sp_sandbox::Value { sp_sandbox::Value::I32(self as i32) } @@ -439,6 +440,7 @@ pub struct Runtime<'a, E: Ext + 'a> { input_data: Option>, memory: sp_sandbox::default_executor::Memory, trap_reason: Option, + chain_extension: Option::ChainExtension>>, } impl<'a, E> Runtime<'a, E> @@ -452,7 +454,13 @@ where input_data: Vec, memory: sp_sandbox::default_executor::Memory, ) -> Self { - Runtime { ext, input_data: Some(input_data), memory, trap_reason: None } + Runtime { + ext, + input_data: Some(input_data), + memory, + trap_reason: None, + chain_extension: Some(Box::new(Default::default())), + } } /// Converts the sandbox result and the runtime state into the execution outcome. @@ -2006,7 +2014,7 @@ define_env!(Env, , // module error. [seal0] seal_call_chain_extension( ctx, - func_id: u32, + id: u32, input_ptr: u32, input_len: u32, output_ptr: u32, @@ -2016,14 +2024,20 @@ define_env!(Env, , if !::ChainExtension::enabled() { return Err(Error::::NoChainExtension.into()); } - let env = Environment::new(ctx, input_ptr, input_len, output_ptr, output_len_ptr); - match ::ChainExtension::call(func_id, env)? { + let mut chain_extension = ctx.chain_extension.take().expect( + "Constructor initializes with `Some`. This is the only place where it is set to `None`.\ + It is always reset to `Some` afterwards. qed" + ); + let env = Environment::new(ctx, id, input_ptr, input_len, output_ptr, output_len_ptr); + let ret = match chain_extension.call(env)? { RetVal::Converging(val) => Ok(val), RetVal::Diverging{flags, data} => Err(TrapReason::Return(ReturnData { flags: flags.bits(), data, })), - } + }; + ctx.chain_extension = Some(chain_extension); + ret }, // Emit a custom debug message. From 9b139a5117081dc1d70890c02e5a09a46bc95fb6 Mon Sep 17 00:00:00 2001 From: Qinxuan Chen Date: Tue, 26 Jul 2022 00:52:11 +0800 Subject: [PATCH 41/46] Pruned duplicated dependencies (#11900) * Update comfy-table v5.0.1 => v6.0.0 Signed-off-by: koushiro * Update strum v0.23.0 => v0.24.1 Signed-off-by: koushiro * Update h2 v0.3.9 => v0.3.13 Signed-off-by: koushiro * Update file-per-thread-logger v0.1.4 => v0.1.5 Signed-off-by: koushiro * Update mio v0.8.0 => v0.8.4 Signed-off-by: koushiro * revert twox-hash Signed-off-by: koushiro * Update secp256k1 v0.21.2 => v0.24.0 Signed-off-by: koushiro --- Cargo.lock | 324 +++--------------- client/beefy/Cargo.toml | 2 +- .../election-provider-multi-phase/Cargo.toml | 9 +- primitives/core/Cargo.toml | 2 +- primitives/core/src/ecdsa.rs | 2 +- primitives/io/Cargo.toml | 2 +- primitives/keyring/Cargo.toml | 2 +- utils/frame/benchmarking-cli/Cargo.toml | 2 +- utils/wasm-builder/Cargo.toml | 2 +- 9 files changed, 65 insertions(+), 282 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d5bfe5e51201..8c19c80bde1a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -230,7 +230,7 @@ dependencies = [ "parking", "polling", "slab", - "socket2 0.4.4", + "socket2", "waker-fn", "winapi", ] @@ -310,7 +310,7 @@ dependencies = [ "futures-io", "futures-util", "pin-utils", - "socket2 0.4.4", + "socket2", "trust-dns-resolver", ] @@ -382,12 +382,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "autocfg" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" - [[package]] name = "autocfg" version = "1.0.1" @@ -512,7 +506,7 @@ name = "beefy-merkle-tree" version = "4.0.0-dev" dependencies = [ "beefy-primitives", - "env_logger 0.9.0", + "env_logger", "hex", "hex-literal", "log", @@ -1005,7 +999,7 @@ version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro-error", "proc-macro2", "quote", @@ -1021,15 +1015,6 @@ dependencies = [ "os_str_bytes", ] -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", -] - [[package]] name = "cmake" version = "0.1.46" @@ -1041,9 +1026,9 @@ dependencies = [ [[package]] name = "comfy-table" -version = "5.0.1" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b103d85ca6e209388771bfb7aa6b68a7aeec4afbf6f0a0264bfbf50360e5212e" +checksum = "121d8a5b0346092c18a4b2fd6f620d7a06f0eb7ac0a45860939a0884bc579c56" dependencies = [ "strum", "strum_macros", @@ -1831,7 +1816,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro2", "quote", "syn", @@ -1898,19 +1883,6 @@ dependencies = [ "syn", ] -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "atty", - "humantime 1.3.0", - "log", - "regex", - "termcolor", -] - [[package]] name = "env_logger" version = "0.9.0" @@ -1918,7 +1890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" dependencies = [ "atty", - "humantime 2.1.0", + "humantime", "log", "regex", "termcolor", @@ -2008,11 +1980,11 @@ dependencies = [ [[package]] name = "file-per-thread-logger" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fdbe0d94371f9ce939b555dd342d0686cc4c0cadbcd4b61d70af5ff97eb4126" +checksum = "21e16290574b39ee41c71aeb90ae960c504ebaf1e2a1c87bd52aa56ed6e1a02f" dependencies = [ - "env_logger 0.7.1", + "env_logger", "log", ] @@ -2445,12 +2417,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "funty" version = "2.0.0" @@ -2751,9 +2717,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.9" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f072413d126e57991455e0a922b31e4c8ba7c2ffbebf6b78b4f8521397d65cd" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ "bytes", "fnv", @@ -2764,7 +2730,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.6.7", + "tokio-util", "tracing", ] @@ -2821,15 +2787,6 @@ dependencies = [ "ahash", ] -[[package]] -name = "heck" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.4.0" @@ -2950,15 +2907,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error 1.2.3", -] - [[package]] name = "humantime" version = "2.1.0" @@ -2982,7 +2930,7 @@ dependencies = [ "httpdate", "itoa 0.4.8", "pin-project-lite 0.2.6", - "socket2 0.4.4", + "socket2", "tokio", "tower-service", "tracing", @@ -3084,7 +3032,7 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" dependencies = [ - "autocfg 1.0.1", + "autocfg", "hashbrown 0.11.2", "serde", ] @@ -3131,7 +3079,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98" dependencies = [ - "socket2 0.4.4", + "socket2", "widestring", "winapi", "winreg", @@ -3213,7 +3161,7 @@ dependencies = [ "thiserror", "tokio", "tokio-rustls", - "tokio-util 0.7.1", + "tokio-util", "tracing", "webpki-roots", ] @@ -3316,7 +3264,7 @@ dependencies = [ "soketto", "tokio", "tokio-stream", - "tokio-util 0.7.1", + "tokio-util", "tracing", ] @@ -3695,7 +3643,7 @@ dependencies = [ "log", "rand 0.8.4", "smallvec", - "socket2 0.4.4", + "socket2", "void", ] @@ -3913,7 +3861,7 @@ dependencies = [ "libc", "libp2p-core", "log", - "socket2 0.4.4", + "socket2", ] [[package]] @@ -4276,7 +4224,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" dependencies = [ - "autocfg 1.0.1", + "autocfg", ] [[package]] @@ -4321,30 +4269,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", - "autocfg 1.0.1", + "autocfg", ] [[package]] name = "mio" -version = "0.8.0" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" dependencies = [ "libc", "log", - "miow", - "ntapi", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" -dependencies = [ - "socket2 0.3.19", - "winapi", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.36.1", ] [[package]] @@ -5015,22 +4952,13 @@ dependencies = [ "version_check", ] -[[package]] -name = "ntapi" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" -dependencies = [ - "winapi", -] - [[package]] name = "num-bigint" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-integer", "num-traits", ] @@ -5060,7 +4988,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-traits", ] @@ -5070,7 +4998,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-bigint", "num-integer", "num-traits", @@ -5082,7 +5010,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-integer", "num-traits", ] @@ -5093,7 +5021,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg 1.0.1", + "autocfg", "libm", ] @@ -5500,7 +5428,7 @@ version = "4.0.0-dev" dependencies = [ "assert_matches", "bitflags", - "env_logger 0.9.0", + "env_logger", "frame-benchmarking", "frame-support", "frame-system", @@ -5866,7 +5794,7 @@ name = "pallet-mmr" version = "4.0.0-dev" dependencies = [ "ckb-merkle-mountain-range", - "env_logger 0.9.0", + "env_logger", "frame-benchmarking", "frame-support", "frame-system", @@ -7082,7 +7010,7 @@ dependencies = [ "bytes", "cfg-if 1.0.0", "cmake", - "heck 0.4.0", + "heck", "itertools", "lazy_static", "log", @@ -7207,25 +7135,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.7", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.2", - "rand_hc 0.1.0", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg 0.1.2", - "rand_xorshift", - "winapi", -] - [[package]] name = "rand" version = "0.7.3" @@ -7252,16 +7161,6 @@ dependencies = [ "rand_hc 0.3.0", ] -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.7", - "rand_core 0.3.1", -] - [[package]] name = "rand_chacha" version = "0.2.2" @@ -7282,21 +7181,6 @@ dependencies = [ "rand_core 0.6.2", ] -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.5.1" @@ -7325,15 +7209,6 @@ dependencies = [ "rand 0.8.4", ] -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "rand_hc" version = "0.2.0" @@ -7352,50 +7227,6 @@ dependencies = [ "rand_core 0.6.2", ] -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi", - "fuchsia-cprng", - "libc", - "rand_core 0.4.2", - "rdrand", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.7", - "rand_core 0.4.2", -] - [[package]] name = "rand_pcg" version = "0.2.1" @@ -7414,15 +7245,6 @@ dependencies = [ "rand_core 0.6.2", ] -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "rawpointer" version = "0.2.1" @@ -7435,7 +7257,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" dependencies = [ - "autocfg 1.0.1", + "autocfg", "crossbeam-deque", "either", "rayon-core", @@ -7454,15 +7276,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "redox_syscall" version = "0.2.10" @@ -7580,7 +7393,7 @@ dependencies = [ name = "remote-externalities" version = "0.10.0-dev" dependencies = [ - "env_logger 0.9.0", + "env_logger", "frame-support", "jsonrpsee", "log", @@ -8317,7 +8130,7 @@ name = "sc-executor" version = "0.10.0-dev" dependencies = [ "criterion", - "env_logger 0.9.0", + "env_logger", "hex-literal", "lazy_static", "lru", @@ -8747,7 +8560,7 @@ name = "sc-rpc" version = "4.0.0-dev" dependencies = [ "assert_matches", - "env_logger 0.9.0", + "env_logger", "futures", "hash-db", "jsonrpsee", @@ -9189,19 +9002,18 @@ dependencies = [ [[package]] name = "secp256k1" -version = "0.21.2" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7883017d5b21f011ef8040ea9c6c7ac90834c0df26a69e4c0b06276151f125" +checksum = "b7649a0b3ffb32636e60c7ce0d70511eda9c52c658cd0634e194d5a19943aeff" dependencies = [ - "rand 0.6.5", "secp256k1-sys", ] [[package]] name = "secp256k1-sys" -version = "0.4.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957da2573cde917463ece3570eab4a0b3f19de6f1646cde62e6fd3868f566036" +checksum = "7058dc8eaf3f2810d7828680320acda0b25a288f6d288e19278e249bbf74226b" dependencies = [ "cc", ] @@ -9510,17 +9322,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "socket2" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "winapi", -] - [[package]] name = "socket2" version = "0.4.4" @@ -10427,20 +10228,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strum" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.23.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" +checksum = "4faebde00e8ff94316c01800f9054fd2ba77d30d9e922541913051d1d978918b" dependencies = [ - "heck 0.3.2", + "heck", "proc-macro2", "quote", "rustversion", @@ -10947,7 +10748,7 @@ dependencies = [ "parking_lot 0.12.0", "pin-project-lite 0.2.6", "signal-hook-registry", - "socket2 0.4.4", + "socket2", "tokio-macros", "winapi", ] @@ -10998,20 +10799,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "tokio-util" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.6", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.1" @@ -11024,6 +10811,7 @@ dependencies = [ "futures-sink", "pin-project-lite 0.2.6", "tokio", + "tracing", ] [[package]] @@ -11347,12 +11135,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" - [[package]] name = "unicode-width" version = "0.1.8" @@ -11493,6 +11275,12 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.77" diff --git a/client/beefy/Cargo.toml b/client/beefy/Cargo.toml index bd25496f2dfea..b910f7ce307a5 100644 --- a/client/beefy/Cargo.toml +++ b/client/beefy/Cargo.toml @@ -39,7 +39,7 @@ sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } [dev-dependencies] serde = "1.0.136" -strum = { version = "0.23", features = ["derive"] } +strum = { version = "0.24.1", features = ["derive"] } tempfile = "3.1.0" tokio = "1.17.0" sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" } diff --git a/frame/election-provider-multi-phase/Cargo.toml b/frame/election-provider-multi-phase/Cargo.toml index 6c8e537a72c64..bbb0adf02e366 100644 --- a/frame/election-provider-multi-phase/Cargo.toml +++ b/frame/election-provider-multi-phase/Cargo.toml @@ -36,13 +36,8 @@ frame-election-provider-support = { version = "4.0.0-dev", default-features = fa # Optional imports for benchmarking frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } pallet-election-provider-support-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support/benchmarking", optional = true } -rand = { version = "0.7.3", default-features = false, optional = true, features = [ - "alloc", - "small_rng", -] } -strum = { optional = true, default-features = false, version = "0.23.0", features = [ - "derive", -] } +rand = { version = "0.7.3", default-features = false, features = ["alloc", "small_rng"], optional = true } +strum = { version = "0.24.1", default-features = false, features = ["derive"], optional = true } [dev-dependencies] parking_lot = "0.12.0" diff --git a/primitives/core/Cargo.toml b/primitives/core/Cargo.toml index cdc12c677e4f3..fced678293140 100644 --- a/primitives/core/Cargo.toml +++ b/primitives/core/Cargo.toml @@ -56,7 +56,7 @@ schnorrkel = { version = "0.9.1", features = [ hex = { version = "0.4", default-features = false, optional = true } libsecp256k1 = { version = "0.7", default-features = false, features = ["static-context"], optional = true } merlin = { version = "2.0", default-features = false, optional = true } -secp256k1 = { version = "0.21.2", default-features = false, features = ["recovery", "alloc"], optional = true } +secp256k1 = { version = "0.24.0", default-features = false, features = ["recovery", "alloc"], optional = true } ss58-registry = { version = "1.18.0", default-features = false } sp-core-hashing = { version = "4.0.0", path = "./hashing", default-features = false, optional = true } sp-runtime-interface = { version = "6.0.0", default-features = false, path = "../runtime-interface" } diff --git a/primitives/core/src/ecdsa.rs b/primitives/core/src/ecdsa.rs index 485e39d3a71db..d56f65fd289e7 100644 --- a/primitives/core/src/ecdsa.rs +++ b/primitives/core/src/ecdsa.rs @@ -499,7 +499,7 @@ impl TraitPair for Pair { impl Pair { /// Get the seed for this key. pub fn seed(&self) -> Seed { - self.secret.serialize_secret() + self.secret.secret_bytes() } /// Exactly as `from_string` except that if no matches are found then, the the first 32 diff --git a/primitives/io/Cargo.toml b/primitives/io/Cargo.toml index 03cd8535665ac..77ba4be677f31 100644 --- a/primitives/io/Cargo.toml +++ b/primitives/io/Cargo.toml @@ -30,7 +30,7 @@ sp-tracing = { version = "5.0.0", default-features = false, path = "../tracing" log = { version = "0.4.17", optional = true } futures = { version = "0.3.21", features = ["thread-pool"], optional = true } parking_lot = { version = "0.12.0", optional = true } -secp256k1 = { version = "0.21.2", features = ["recovery", "global-context"], optional = true } +secp256k1 = { version = "0.24.0", features = ["recovery", "global-context"], optional = true } tracing = { version = "0.1.29", default-features = false } tracing-core = { version = "0.1.28", default-features = false} diff --git a/primitives/keyring/Cargo.toml b/primitives/keyring/Cargo.toml index 8f1adb6cf81f3..982abfd09a553 100644 --- a/primitives/keyring/Cargo.toml +++ b/primitives/keyring/Cargo.toml @@ -15,6 +15,6 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] lazy_static = "1.4.0" -strum = { version = "0.23.0", features = ["derive"] } +strum = { version = "0.24.1", features = ["derive"] } sp-core = { version = "6.0.0", path = "../core" } sp-runtime = { version = "6.0.0", path = "../runtime" } diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index ea26bf0c9261c..31b8f1332a653 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] chrono = "0.4" clap = { version = "3.1.18", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0" } -comfy-table = { version = "5.0.1", default-features = false } +comfy-table = { version = "6.0.0", default-features = false } handlebars = "4.2.2" hash-db = "0.15.2" hex = "0.4.3" diff --git a/utils/wasm-builder/Cargo.toml b/utils/wasm-builder/Cargo.toml index 8f887e45ec176..ac0ba5dbdb85a 100644 --- a/utils/wasm-builder/Cargo.toml +++ b/utils/wasm-builder/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] ansi_term = "0.12.1" build-helper = "0.1.1" cargo_metadata = "0.14.2" -strum = { version = "0.23.0", features = ["derive"] } +strum = { version = "0.24.1", features = ["derive"] } tempfile = "3.1.0" toml = "0.5.4" walkdir = "2.3.2" From 134985fdf880e16de6ba5e8bf2bf9a7d8b768fb3 Mon Sep 17 00:00:00 2001 From: Sacha Lansky Date: Mon, 25 Jul 2022 23:09:50 +0100 Subject: [PATCH 42/46] Fix typos (#11914) --- frame/transaction-payment/src/lib.rs | 2 +- test-utils/runtime/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index 0f5c0321130be..fe37acb214452 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -609,7 +609,7 @@ where /// and user-included tip. /// /// The priority is based on the amount of `tip` the user is willing to pay per unit of either - /// `weight` or `length`, depending which one is more limitting. For `Operational` extrinsics + /// `weight` or `length`, depending which one is more limiting. For `Operational` extrinsics /// we add a "virtual tip" to the calculations. /// /// The formula should simply be `tip / bounded_{weight|length}`, but since we are using diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index ea62f2ac84f3d..e5cfae49da56d 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -237,7 +237,7 @@ impl sp_runtime::traits::Dispatchable for Extrinsic { type Info = (); type PostInfo = (); fn dispatch(self, _origin: Self::Origin) -> sp_runtime::DispatchResultWithInfo { - panic!("This implemention should not be used for actual dispatch."); + panic!("This implementation should not be used for actual dispatch."); } } From 5709dfa7095659d398323fb2c9515f66b2617b17 Mon Sep 17 00:00:00 2001 From: Sergej Sakac <73715684+Szegoo@users.noreply.github.com> Date: Tue, 26 Jul 2022 11:06:48 +0200 Subject: [PATCH 43/46] Rpc for pending rewards (#11831) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * rpc pending rewards * commit * remove unused imports * fix * fix * fmt * fix * fmt * fix * docs * docs & formatting * better formatting * temporary fix * error handling * fix? * fmt * use to_string * fmt * fixed error handling * fix * rpc added to client * Update Cargo.toml * Update Cargo.toml * fix wrong reward counter * expose function * move implementation * docs * docs * docs * Update lib.rs * Update lib.rs * unexpose functions * unused dependency * update Cargo.lock * Update frame/nomination-pools/src/lib.rs * Update lib.rs * Update lib.rs * Update frame/nomination-pools/rpc/runtime-api/src/lib.rs Co-authored-by: Bastian Köcher * remove rpc * remove rpc directory * final fix Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: Bastian Köcher --- Cargo.lock | 10 ++++++ Cargo.toml | 1 + bin/node/runtime/Cargo.toml | 2 ++ bin/node/runtime/src/lib.rs | 6 ++++ frame/nomination-pools/runtime-api/Cargo.toml | 26 +++++++++++++++ frame/nomination-pools/runtime-api/README.md | 3 ++ frame/nomination-pools/runtime-api/src/lib.rs | 33 +++++++++++++++++++ frame/nomination-pools/src/lib.rs | 16 ++++++++- 8 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 frame/nomination-pools/runtime-api/Cargo.toml create mode 100644 frame/nomination-pools/runtime-api/README.md create mode 100644 frame/nomination-pools/runtime-api/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 8c19c80bde1a9..50603c6235a23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4759,6 +4759,7 @@ dependencies = [ "pallet-multisig", "pallet-nomination-pools", "pallet-nomination-pools-benchmarking", + "pallet-nomination-pools-runtime-api", "pallet-offences", "pallet-offences-benchmarking", "pallet-preimage", @@ -5911,6 +5912,15 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-nomination-pools-runtime-api" +version = "1.0.0-dev" +dependencies = [ + "parity-scale-codec", + "sp-api", + "sp-std", +] + [[package]] name = "pallet-nomination-pools-test-staking" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 9909e6f893877..1f22343c002a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,6 +116,7 @@ members = [ "frame/nomination-pools", "frame/nomination-pools/benchmarking", "frame/nomination-pools/test-staking", + "frame/nomination-pools/runtime-api", "frame/randomness-collective-flip", "frame/ranked-collective", "frame/recovery", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index ca971f29e93c9..d3138ce51de9c 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -79,6 +79,7 @@ pallet-mmr = { version = "4.0.0-dev", default-features = false, path = "../../.. pallet-multisig = { version = "4.0.0-dev", default-features = false, path = "../../../frame/multisig" } pallet-nomination-pools = { version = "1.0.0", default-features = false, path = "../../../frame/nomination-pools"} pallet-nomination-pools-benchmarking = { version = "1.0.0", default-features = false, optional = true, path = "../../../frame/nomination-pools/benchmarking" } +pallet-nomination-pools-runtime-api = { version = "1.0.0-dev", default-features = false, path = "../../../frame/nomination-pools/runtime-api" } pallet-offences = { version = "4.0.0-dev", default-features = false, path = "../../../frame/offences" } pallet-offences-benchmarking = { version = "4.0.0-dev", path = "../../../frame/offences/benchmarking", default-features = false, optional = true } pallet-preimage = { version = "4.0.0-dev", default-features = false, path = "../../../frame/preimage" } @@ -145,6 +146,7 @@ std = [ "pallet-mmr/std", "pallet-multisig/std", "pallet-nomination-pools/std", + "pallet-nomination-pools-runtime-api/std", "pallet-identity/std", "pallet-scheduler/std", "node-primitives/std", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index b2efcb196787d..037211155cf0e 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1835,6 +1835,12 @@ impl_runtime_apis! { } } + impl pallet_nomination_pools_runtime_api::NominationPoolsApi for Runtime { + fn pending_rewards(member_account: AccountId) -> Balance { + NominationPools::pending_rewards(member_account) + } + } + impl sp_consensus_babe::BabeApi for Runtime { fn configuration() -> sp_consensus_babe::BabeGenesisConfiguration { // The choice of `c` parameter (where `1 - c` represents the diff --git a/frame/nomination-pools/runtime-api/Cargo.toml b/frame/nomination-pools/runtime-api/Cargo.toml new file mode 100644 index 0000000000000..dde925c62fe0d --- /dev/null +++ b/frame/nomination-pools/runtime-api/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "pallet-nomination-pools-runtime-api" +version = "1.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "Runtime API for nomination-pools FRAME pallet" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" } +sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } + +[features] +default = ["std"] +std = [ + "codec/std", + "sp-api/std", + "sp-std/std", +] diff --git a/frame/nomination-pools/runtime-api/README.md b/frame/nomination-pools/runtime-api/README.md new file mode 100644 index 0000000000000..af90b31733b0b --- /dev/null +++ b/frame/nomination-pools/runtime-api/README.md @@ -0,0 +1,3 @@ +Runtime API definition for nomination-pools pallet. + +License: Apache-2.0 \ No newline at end of file diff --git a/frame/nomination-pools/runtime-api/src/lib.rs b/frame/nomination-pools/runtime-api/src/lib.rs new file mode 100644 index 0000000000000..aa3ca57ca5b8b --- /dev/null +++ b/frame/nomination-pools/runtime-api/src/lib.rs @@ -0,0 +1,33 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Runtime API definition for nomination-pools pallet. +//! Currently supports only one rpc endpoint. + +#![cfg_attr(not(feature = "std"), no_std)] + +use codec::Codec; + +sp_api::decl_runtime_apis! { + /// Runtime api for accessing information about nomination pools. + pub trait NominationPoolsApi + where AccountId: Codec, Balance: Codec + { + /// Returns the pending rewards for the member that the AccountId was given for. + fn pending_rewards(member: AccountId) -> Balance; + } +} diff --git a/frame/nomination-pools/src/lib.rs b/frame/nomination-pools/src/lib.rs index 203c13f2f3496..09f1746b8e5ba 100644 --- a/frame/nomination-pools/src/lib.rs +++ b/frame/nomination-pools/src/lib.rs @@ -939,7 +939,7 @@ pub struct RewardPool { impl RewardPool { /// Getter for [`RewardPool::last_recorded_reward_counter`]. - fn last_recorded_reward_counter(&self) -> T::RewardCounter { + pub(crate) fn last_recorded_reward_counter(&self) -> T::RewardCounter { self.last_recorded_reward_counter } @@ -2147,6 +2147,20 @@ pub mod pallet { } impl Pallet { + /// Returns the pending rewards for the specified `member_account`. + /// + /// In the case of error the function returns balance of zero. + pub fn pending_rewards(member_account: T::AccountId) -> BalanceOf { + if let Some(pool_member) = PoolMembers::::get(member_account) { + if let Some(reward_pool) = RewardPools::::get(pool_member.pool_id) { + return pool_member + .pending_rewards(reward_pool.last_recorded_reward_counter()) + .unwrap_or_default() + } + } + BalanceOf::::default() + } + /// The amount of bond that MUST REMAIN IN BONDED in ALL POOLS. /// /// It is the responsibility of the depositor to put these funds into the pool initially. Upon From 531d7c3a5b26f5849aa6e23518c45a56d67faccb Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Tue, 26 Jul 2022 11:37:54 +0200 Subject: [PATCH 44/46] [ci] remove cargo-check-nixos job (#11873) * [ci] remove cargo-check-nixos job * remove shell.nix --- scripts/ci/gitlab/pipeline/test.yml | 24 ++++-------------------- shell.nix | 27 --------------------------- 2 files changed, 4 insertions(+), 47 deletions(-) delete mode 100644 shell.nix diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index d900d425ad494..b2daf5bada24f 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -51,26 +51,6 @@ cargo-clippy: - SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo +nightly clippy --all-targets - rusty-cachier cache upload -cargo-check-nixos: - stage: test - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: cargo-clippy - artifacts: false - extends: - - .docker-env - - .test-refs - before_script: [] - # Don't use CI_IMAGE here because it breaks nightly checks of paritytech/ci-linux image - image: nixos/nix - variables: - SNAP: "DUMMY" - WS_API: "DUMMY" - script: - - nix-channel --update - - nix-shell shell.nix - - nix-shell --run "cargo check --workspace --all-targets --all-features" - cargo-check-benches: stage: test variables: @@ -337,6 +317,8 @@ test-linux-stable-int: time cargo test -p node-cli --release --verbose --locked -- --ignored - rusty-cachier cache upload +# more information about this job can be found here: +# https://github.com/paritytech/substrate/pull/6916 check-tracing: stage: test # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs @@ -355,6 +337,8 @@ check-tracing: - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features --features=with-tracing - rusty-cachier cache upload +# more information about this job can be found here: +# https://github.com/paritytech/substrate/pull/3778 test-full-crypto-feature: stage: test # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs diff --git a/shell.nix b/shell.nix deleted file mode 100644 index c318995605a38..0000000000000 --- a/shell.nix +++ /dev/null @@ -1,27 +0,0 @@ -let - mozillaOverlay = - import (builtins.fetchGit { - url = "https://github.com/mozilla/nixpkgs-mozilla.git"; - rev = "15b7a05f20aab51c4ffbefddb1b448e862dccb7d"; - }); - nixpkgs = import { overlays = [ mozillaOverlay ]; }; - rust-nightly = with nixpkgs; ((rustChannelOf { date = "2022-04-20"; channel = "nightly"; }).rust.override { - extensions = [ "rust-src" ]; - targets = [ "wasm32-unknown-unknown" ]; - }); -in -with nixpkgs; pkgs.mkShell { - buildInputs = [ - clang - openssl.dev - pkg-config - rust-nightly - ] ++ lib.optionals stdenv.isDarwin [ - darwin.apple_sdk.frameworks.Security - ]; - - RUST_SRC_PATH = "${rust-nightly}/lib/rustlib/src/rust/src"; - LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib"; - PROTOC = "${protobuf}/bin/protoc"; - ROCKSDB_LIB_DIR = "${rocksdb}/lib"; -} From 9428b0ea492d405f320753fcd93bc59ebb3bf33b Mon Sep 17 00:00:00 2001 From: Sebastian Kunert Date: Tue, 26 Jul 2022 14:37:05 +0200 Subject: [PATCH 45/46] Prepare for rust 1.62.1 (#11903) * Update UI test output for rust 1.62.1 * switch ci to staging image to check that everything works * fix artifacts node-bench-regression-guard * Imeplement `scale_info::TypeInfo` manually to silence aggressive rust warning * Fix more clippy lints * Make clippy happy by relying on auto-deref were possible * Add tracking issue to the comments * pin ci image Co-authored-by: alvicsam --- .gitlab-ci.yml | 3 +- bin/node/bench/src/simple_trie.rs | 2 +- bin/node/cli/benches/transaction_pool.rs | 8 +-- client/beefy/src/tests.rs | 14 ++--- client/beefy/src/worker.rs | 4 +- client/cli/src/commands/run_cmd.rs | 2 +- client/consensus/common/src/block_import.rs | 4 +- client/db/src/lib.rs | 2 +- .../finality-grandpa/src/communication/mod.rs | 6 +- client/finality-grandpa/src/import.rs | 2 +- client/finality-grandpa/src/observer.rs | 2 +- client/rpc/src/offchain/mod.rs | 4 +- client/rpc/src/state/state_full.rs | 2 +- client/service/src/client/client.rs | 4 +- client/tracing/src/logging/stderr_writer.rs | 2 +- client/transaction-pool/tests/pool.rs | 14 ++--- frame/contracts/src/exec.rs | 2 +- frame/executive/src/lib.rs | 4 +- frame/offences/src/mock.rs | 2 +- .../procedural/src/pallet/parse/call.rs | 2 +- .../no_std_genesis_config.stderr | 13 ++++ .../undefined_event_part.stderr | 26 ++++++++ .../undefined_genesis_config_part.stderr | 13 ++++ .../undefined_origin_part.stderr | 26 ++++++++ ...ed_keyword_two_times_integrity_test.stderr | 14 ++--- frame/support/test/tests/decl_storage.rs | 2 +- .../derive_no_bound_ui/partial_eq.stderr | 4 +- .../call_argument_invalid_bound.stderr | 5 -- .../call_argument_invalid_bound_2.stderr | 5 -- .../call_argument_invalid_bound_3.stderr | 4 ++ .../error_does_not_derive_pallet_error.stderr | 10 ++++ .../pallet_ui/event_field_not_member.stderr | 5 -- .../genesis_default_not_satisfied.stderr | 4 ++ ...age_ensure_span_are_ok_on_wrong_gen.stderr | 60 +++++++++++++++++++ ...re_span_are_ok_on_wrong_gen_unnamed.stderr | 60 +++++++++++++++++++ .../pallet_ui/storage_info_unsatisfied.stderr | 10 ++++ .../storage_info_unsatisfied_nmap.stderr | 10 ++++ .../api/proc-macro/src/impl_runtime_apis.rs | 2 +- primitives/runtime/src/bounded/bounded_vec.rs | 40 ++++++++++++- primitives/runtime/src/curve.rs | 32 +++++++++- primitives/runtime/src/traits.rs | 2 +- primitives/trie/src/lib.rs | 12 ++-- .../proc-macro/src/decl_runtime_version.rs | 2 +- rustfmt.toml | 1 + utils/frame/remote-externalities/src/lib.rs | 2 +- 45 files changed, 365 insertions(+), 84 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9608d88e9554d..d1de666009bf1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,7 +47,8 @@ variables: &default-vars CARGO_INCREMENTAL: 0 DOCKER_OS: "debian:stretch" ARCH: "x86_64" - CI_IMAGE: "paritytech/ci-linux:production" + # change to production when this image is published + CI_IMAGE: "paritytech/ci-linux:staging@sha256:2c90b67f1452ed2d7236c2cd13f6224053a833d521b3630650b679f00874e0a9" RUSTY_CACHIER_SINGLE_BRANCH: master RUSTY_CACHIER_DONT_OPERATE_ON_MAIN_BRANCH: "true" diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index c59389570e534..aa9c96a1cbd3f 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -33,7 +33,7 @@ pub struct SimpleTrie<'a> { impl<'a> AsHashDB for SimpleTrie<'a> { fn as_hash_db(&self) -> &dyn hash_db::HashDB { - &*self + self } fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { diff --git a/bin/node/cli/benches/transaction_pool.rs b/bin/node/cli/benches/transaction_pool.rs index 5b96355b9ca70..d88c3a65b6138 100644 --- a/bin/node/cli/benches/transaction_pool.rs +++ b/bin/node/cli/benches/transaction_pool.rs @@ -243,25 +243,25 @@ fn transaction_pool_benchmarks(c: &mut Criterion) { move |b| { b.iter_batched( || { - let prepare_extrinsics = create_account_extrinsics(&*node.client, &accounts); + let prepare_extrinsics = create_account_extrinsics(&node.client, &accounts); runtime.block_on(future::join_all(prepare_extrinsics.into_iter().map(|tx| { submit_tx_and_wait_for_inclusion( &node.transaction_pool, tx, - &*node.client, + &node.client, true, ) }))); - create_benchmark_extrinsics(&*node.client, &accounts, extrinsics_per_account) + create_benchmark_extrinsics(&node.client, &accounts, extrinsics_per_account) }, |extrinsics| { runtime.block_on(future::join_all(extrinsics.into_iter().map(|tx| { submit_tx_and_wait_for_inclusion( &node.transaction_pool, tx, - &*node.client, + &node.client, false, ) }))); diff --git a/client/beefy/src/tests.rs b/client/beefy/src/tests.rs index 8090c425e71db..78e697a6ada81 100644 --- a/client/beefy/src/tests.rs +++ b/client/beefy/src/tests.rs @@ -467,7 +467,7 @@ fn finalize_block_and_wait_for_beefy( finalize_targets: &[u64], expected_beefy: &[u64], ) { - let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers); + let (best_blocks, signed_commitments) = get_beefy_streams(&mut net.lock(), peers); for block in finalize_targets { let finalize = BlockId::number(*block); @@ -555,7 +555,7 @@ fn lagging_validators() { // Alice finalizes #25, Bob lags behind let finalize = BlockId::number(25); - let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers); + let (best_blocks, signed_commitments) = get_beefy_streams(&mut net.lock(), peers); net.lock().peer(0).client().as_client().finalize_block(finalize, None).unwrap(); // verify nothing gets finalized by BEEFY let timeout = Some(Duration::from_millis(250)); @@ -563,7 +563,7 @@ fn lagging_validators() { streams_empty_after_timeout(signed_commitments, &net, &mut runtime, None); // Bob catches up and also finalizes #25 - let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers); + let (best_blocks, signed_commitments) = get_beefy_streams(&mut net.lock(), peers); net.lock().peer(1).client().as_client().finalize_block(finalize, None).unwrap(); // expected beefy finalizes block #17 from diff-power-of-two wait_for_best_beefy_blocks(best_blocks, &net, &mut runtime, &[23, 24, 25]); @@ -577,7 +577,7 @@ fn lagging_validators() { // validator set). // Alice finalizes session-boundary mandatory block #60, Bob lags behind - let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers); + let (best_blocks, signed_commitments) = get_beefy_streams(&mut net.lock(), peers); let finalize = BlockId::number(60); net.lock().peer(0).client().as_client().finalize_block(finalize, None).unwrap(); // verify nothing gets finalized by BEEFY @@ -586,7 +586,7 @@ fn lagging_validators() { streams_empty_after_timeout(signed_commitments, &net, &mut runtime, None); // Bob catches up and also finalizes #60 (and should have buffered Alice's vote on #60) - let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers); + let (best_blocks, signed_commitments) = get_beefy_streams(&mut net.lock(), peers); net.lock().peer(1).client().as_client().finalize_block(finalize, None).unwrap(); // verify beefy skips intermediary votes, and successfully finalizes mandatory block #40 wait_for_best_beefy_blocks(best_blocks, &net, &mut runtime, &[60]); @@ -629,7 +629,7 @@ fn correct_beefy_payload() { finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[10], &[1, 9]); let (best_blocks, signed_commitments) = - get_beefy_streams(&mut *net.lock(), &[BeefyKeyring::Alice]); + get_beefy_streams(&mut net.lock(), &[BeefyKeyring::Alice]); // now 2 good validators and 1 bad one are voting net.lock() @@ -658,7 +658,7 @@ fn correct_beefy_payload() { // 3rd good validator catches up and votes as well let (best_blocks, signed_commitments) = - get_beefy_streams(&mut *net.lock(), &[BeefyKeyring::Alice]); + get_beefy_streams(&mut net.lock(), &[BeefyKeyring::Alice]); net.lock() .peer(2) .client() diff --git a/client/beefy/src/worker.rs b/client/beefy/src/worker.rs index 735dea0170a62..dccf7ba7504dd 100644 --- a/client/beefy/src/worker.rs +++ b/client/beefy/src/worker.rs @@ -439,7 +439,7 @@ where let commitment = Commitment { payload, block_number: target_number, validator_set_id }; let encoded_commitment = commitment.encode(); - let signature = match self.key_store.sign(&authority_id, &*encoded_commitment) { + let signature = match self.key_store.sign(&authority_id, &encoded_commitment) { Ok(sig) => sig, Err(err) => { warn!(target: "beefy", "🥩 Error signing commitment: {:?}", err); @@ -451,7 +451,7 @@ where target: "beefy", "🥩 Produced signature using {:?}, is_valid: {:?}", authority_id, - BeefyKeystore::verify(&authority_id, &signature, &*encoded_commitment) + BeefyKeystore::verify(&authority_id, &signature, &encoded_commitment) ); let message = VoteMessage { commitment, id: authority_id, signature }; diff --git a/client/cli/src/commands/run_cmd.rs b/client/cli/src/commands/run_cmd.rs index 99eaa4be1cd0f..3a74fdd9700f2 100644 --- a/client/cli/src/commands/run_cmd.rs +++ b/client/cli/src/commands/run_cmd.rs @@ -556,7 +556,7 @@ impl std::error::Error for TelemetryParsingError {} impl std::fmt::Display for TelemetryParsingError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &*self { + match self { TelemetryParsingError::MissingVerbosity => write!(f, "Verbosity level missing"), TelemetryParsingError::VerbosityParsingError(e) => write!(f, "{}", e), } diff --git a/client/consensus/common/src/block_import.rs b/client/consensus/common/src/block_import.rs index f81c8eb7e8dee..10739f63ef779 100644 --- a/client/consensus/common/src/block_import.rs +++ b/client/consensus/common/src/block_import.rs @@ -433,10 +433,10 @@ impl JustificationSyncLink for () { impl> JustificationSyncLink for Arc { fn request_justification(&self, hash: &B::Hash, number: NumberFor) { - L::request_justification(&*self, hash, number); + L::request_justification(self, hash, number); } fn clear_justification_requests(&self) { - L::clear_justification_requests(&*self); + L::clear_justification_requests(self); } } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index f1adbd3df1a0f..7dd49f9831f1c 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2292,7 +2292,7 @@ impl sc_client_api::backend::Backend for Backend { } fn get_import_lock(&self) -> &RwLock<()> { - &*self.import_lock + &self.import_lock } fn requires_full_sync(&self) -> bool { diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs index 0555a03db149f..378501cffdd62 100644 --- a/client/finality-grandpa/src/communication/mod.rs +++ b/client/finality-grandpa/src/communication/mod.rs @@ -320,7 +320,7 @@ impl> NetworkBridge { voters: Arc>, has_voted: HasVoted, ) -> (impl Stream> + Unpin, OutgoingMessages) { - self.note_round(round, set_id, &*voters); + self.note_round(round, set_id, &voters); let keystore = keystore.and_then(|ks| { let id = ks.local_id(); @@ -637,9 +637,9 @@ fn incoming_global( .filter_map(move |(notification, msg)| { future::ready(match msg { GossipMessage::Commit(msg) => - process_commit(msg, notification, &gossip_engine, &gossip_validator, &*voters), + process_commit(msg, notification, &gossip_engine, &gossip_validator, &voters), GossipMessage::CatchUp(msg) => - process_catch_up(msg, notification, &gossip_engine, &gossip_validator, &*voters), + process_catch_up(msg, notification, &gossip_engine, &gossip_validator, &voters), _ => { debug!(target: "afg", "Skipping unknown message type"); None diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index eefb3d3f0aee4..b5a0d7be70f19 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -284,7 +284,7 @@ where impl<'a, H, N> InnerGuard<'a, H, N> { fn as_mut(&mut self) -> &mut AuthoritySet { - &mut **self.guard.as_mut().expect("only taken on deconstruction; qed") + self.guard.as_mut().expect("only taken on deconstruction; qed") } fn set_old(&mut self, old: AuthoritySet) { diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs index 85d80dcfe7cde..9bcb03c0555c2 100644 --- a/client/finality-grandpa/src/observer.rs +++ b/client/finality-grandpa/src/observer.rs @@ -289,7 +289,7 @@ where network.note_round( crate::communication::Round(round), crate::communication::SetId(set_id), - &*voters, + &voters, ) } }; diff --git a/client/rpc/src/offchain/mod.rs b/client/rpc/src/offchain/mod.rs index b66b78274a64e..6896b82619166 100644 --- a/client/rpc/src/offchain/mod.rs +++ b/client/rpc/src/offchain/mod.rs @@ -57,7 +57,7 @@ impl OffchainApiServer for Offchain { StorageKind::PERSISTENT => sp_offchain::STORAGE_PREFIX, StorageKind::LOCAL => return Err(JsonRpseeError::from(Error::UnavailableStorageKind)), }; - self.storage.write().set(prefix, &*key, &*value); + self.storage.write().set(prefix, &key, &value); Ok(()) } @@ -69,6 +69,6 @@ impl OffchainApiServer for Offchain { StorageKind::LOCAL => return Err(JsonRpseeError::from(Error::UnavailableStorageKind)), }; - Ok(self.storage.read().get(prefix, &*key).map(Into::into)) + Ok(self.storage.read().get(prefix, &key).map(Into::into)) } } diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index c58638c870ab3..5a8a83cdd4851 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -199,7 +199,7 @@ where .call( &BlockId::Hash(block), &method, - &*call_data, + &call_data, self.client.execution_extensions().strategies().other, None, ) diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index fb73b4c34c040..d61d7f7fa3781 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1614,11 +1614,11 @@ where RA: Send + Sync, { fn header(&self, id: BlockId) -> sp_blockchain::Result> { - (**self).backend.blockchain().header(id) + self.backend.blockchain().header(id) } fn info(&self) -> blockchain::Info { - (**self).backend.blockchain().info() + self.backend.blockchain().info() } fn status(&self, id: BlockId) -> sp_blockchain::Result { diff --git a/client/tracing/src/logging/stderr_writer.rs b/client/tracing/src/logging/stderr_writer.rs index e62c5e82c1ac7..de78a61af41a2 100644 --- a/client/tracing/src/logging/stderr_writer.rs +++ b/client/tracing/src/logging/stderr_writer.rs @@ -89,7 +89,7 @@ fn flush_logs(mut buffer: parking_lot::lock_api::MutexGuard>(); assert_eq!(expected_ready, ready); - let event = block_event_with_retracted(f1_header, f0, &*pool.api()); + let event = block_event_with_retracted(f1_header, f0, pool.api()); block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 3); diff --git a/frame/contracts/src/exec.rs b/frame/contracts/src/exec.rs index d93c646872c6f..5ca74e681e5dd 100644 --- a/frame/contracts/src/exec.rs +++ b/frame/contracts/src/exec.rs @@ -1315,7 +1315,7 @@ where fn deposit_event(topics: Vec, event: Event) { >::deposit_event_indexed( - &*topics, + &topics, ::Event::from(event).into(), ) } diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index c40fdf94806aa..cd3e1c500db26 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -573,7 +573,7 @@ mod tests { use pallet_balances::Call as BalancesCall; use pallet_transaction_payment::CurrencyAdapter; - const TEST_KEY: &[u8] = &*b":test:key:"; + const TEST_KEY: &[u8] = b":test:key:"; #[frame_support::pallet] mod custom { @@ -806,7 +806,7 @@ mod tests { type TestUncheckedExtrinsic = TestXt; // Will contain `true` when the custom runtime logic was called. - const CUSTOM_ON_RUNTIME_KEY: &[u8] = &*b":custom:on_runtime"; + const CUSTOM_ON_RUNTIME_KEY: &[u8] = b":custom:on_runtime"; struct CustomOnRuntimeUpgrade; impl OnRuntimeUpgrade for CustomOnRuntimeUpgrade { diff --git a/frame/offences/src/mock.rs b/frame/offences/src/mock.rs index b3dfbdd90b19d..6a69b54b3cca0 100644 --- a/frame/offences/src/mock.rs +++ b/frame/offences/src/mock.rs @@ -67,7 +67,7 @@ impl offence::OnOffenceHandler } pub fn with_on_offence_fractions) -> R>(f: F) -> R { - ON_OFFENCE_PERBILL.with(|fractions| f(&mut *fractions.borrow_mut())) + ON_OFFENCE_PERBILL.with(|fractions| f(&mut fractions.borrow_mut())) } type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; diff --git a/frame/support/procedural/src/pallet/parse/call.rs b/frame/support/procedural/src/pallet/parse/call.rs index 75c85474dcfe7..d8a81d699b8c2 100644 --- a/frame/support/procedural/src/pallet/parse/call.rs +++ b/frame/support/procedural/src/pallet/parse/call.rs @@ -188,7 +188,7 @@ impl CallDef { return Err(syn::Error::new(method.sig.span(), msg)) }, Some(syn::FnArg::Typed(arg)) => { - check_dispatchable_first_arg_type(&*arg.ty)?; + check_dispatchable_first_arg_type(&arg.ty)?; }, } diff --git a/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.stderr b/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.stderr index 404c0c3627b7b..1f08ab87c1f79 100644 --- a/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.stderr +++ b/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.stderr @@ -29,6 +29,19 @@ help: consider importing this struct | 1 | use frame_system::GenesisConfig; | +help: if you import `GenesisConfig`, refer to it directly + | +40 - construct_runtime! { +41 - pub enum Runtime where +42 - Block = Block, +43 - NodeBlock = Block, +44 - UncheckedExtrinsic = UncheckedExtrinsic +45 - { +46 - System: frame_system::{Pallet, Call, Storage, Config, Event}, +47 - Pallet: test_pallet::{Pallet, Config}, +48 - } +49 - } + | error[E0283]: type annotations needed --> tests/construct_runtime_ui/no_std_genesis_config.rs:40:1 diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_event_part.stderr b/frame/support/test/tests/construct_runtime_ui/undefined_event_part.stderr index 31229f8c93cb6..2af4d3fb15000 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_event_part.stderr +++ b/frame/support/test/tests/construct_runtime_ui/undefined_event_part.stderr @@ -32,6 +32,19 @@ help: consider importing this enum | 1 | use frame_system::Event; | +help: if you import `Event`, refer to it directly + | +49 - construct_runtime! { +50 - pub enum Runtime where +51 - Block = Block, +52 - NodeBlock = Block, +53 - UncheckedExtrinsic = UncheckedExtrinsic +54 - { +55 - System: frame_system::{Pallet, Call, Storage, Config, Event}, +56 - Pallet: pallet::{Pallet, Event}, +57 - } +58 - } + | error[E0412]: cannot find type `Event` in module `pallet` --> tests/construct_runtime_ui/undefined_event_part.rs:49:1 @@ -52,3 +65,16 @@ help: consider importing one of these items | 1 | use frame_system::Event; | +help: if you import `Event`, refer to it directly + | +49 - construct_runtime! { +50 - pub enum Runtime where +51 - Block = Block, +52 - NodeBlock = Block, +53 - UncheckedExtrinsic = UncheckedExtrinsic +54 - { +55 - System: frame_system::{Pallet, Call, Storage, Config, Event}, +56 - Pallet: pallet::{Pallet, Event}, +57 - } +58 - } + | diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.stderr b/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.stderr index fa1cee1ac7e2f..1bc109a45ac57 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.stderr +++ b/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.stderr @@ -32,6 +32,19 @@ help: consider importing this struct | 1 | use frame_system::GenesisConfig; | +help: if you import `GenesisConfig`, refer to it directly + | +49 - construct_runtime! { +50 - pub enum Runtime where +51 - Block = Block, +52 - NodeBlock = Block, +53 - UncheckedExtrinsic = UncheckedExtrinsic +54 - { +55 - System: frame_system::{Pallet, Call, Storage, Config, Event}, +56 - Pallet: pallet::{Pallet, Config}, +57 - } +58 - } + | error[E0283]: type annotations needed --> tests/construct_runtime_ui/undefined_genesis_config_part.rs:49:1 diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.stderr b/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.stderr index 06e845618d44f..c692cd61bae8b 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.stderr +++ b/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.stderr @@ -32,6 +32,19 @@ help: consider importing this type alias | 1 | use frame_system::Origin; | +help: if you import `Origin`, refer to it directly + | +49 - construct_runtime! { +50 - pub enum Runtime where +51 - Block = Block, +52 - NodeBlock = Block, +53 - UncheckedExtrinsic = UncheckedExtrinsic +54 - { +55 - System: frame_system::{Pallet, Call, Storage, Config, Event}, +56 - Pallet: pallet::{Pallet, Origin}, +57 - } +58 - } + | error[E0412]: cannot find type `Origin` in module `pallet` --> tests/construct_runtime_ui/undefined_origin_part.rs:49:1 @@ -52,6 +65,19 @@ help: consider importing one of these items | 1 | use frame_system::Origin; | +help: if you import `Origin`, refer to it directly + | +49 - construct_runtime! { +50 - pub enum Runtime where +51 - Block = Block, +52 - NodeBlock = Block, +53 - UncheckedExtrinsic = UncheckedExtrinsic +54 - { +55 - System: frame_system::{Pallet, Call, Storage, Config, Event}, +56 - Pallet: pallet::{Pallet, Origin}, +57 - } +58 - } + | error[E0282]: type annotations needed --> tests/construct_runtime_ui/undefined_origin_part.rs:49:1 diff --git a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr index 86c427d8080be..880695d9b77e2 100644 --- a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr +++ b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr @@ -1,5 +1,5 @@ error: `integrity_test` can only be passed once as input. - --> $DIR/reserved_keyword_two_times_integrity_test.rs:1:1 + --> tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs:1:1 | 1 | / frame_support::decl_module! { 2 | | pub struct Module for enum Call where origin: T::Origin, system=self { @@ -13,13 +13,7 @@ error: `integrity_test` can only be passed once as input. = note: this error originates in the macro `$crate::decl_module` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0601]: `main` function not found in crate `$CRATE` - --> $DIR/reserved_keyword_two_times_integrity_test.rs:1:1 + --> tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs:7:2 | -1 | / frame_support::decl_module! { -2 | | pub struct Module for enum Call where origin: T::Origin, system=self { -3 | | fn integrity_test() {} -4 | | -5 | | fn integrity_test() {} -6 | | } -7 | | } - | |_^ consider adding a `main` function to `$DIR/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs` +7 | } + | ^ consider adding a `main` function to `$DIR/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs` diff --git a/frame/support/test/tests/decl_storage.rs b/frame/support/test/tests/decl_storage.rs index be5d70be17f69..7ce43cd5d44d1 100644 --- a/frame/support/test/tests/decl_storage.rs +++ b/frame/support/test/tests/decl_storage.rs @@ -797,7 +797,7 @@ mod test_append_and_len { TestExternalities::default().execute_with(|| { let key = JustVec::hashed_key(); // Set it to some invalid value. - frame_support::storage::unhashed::put_raw(&key, &*b"1"); + frame_support::storage::unhashed::put_raw(&key, b"1"); assert!(JustVec::get().is_empty()); assert_eq!(frame_support::storage::unhashed::get_raw(&key), Some(b"1".to_vec())); diff --git a/frame/support/test/tests/derive_no_bound_ui/partial_eq.stderr b/frame/support/test/tests/derive_no_bound_ui/partial_eq.stderr index 64f844e547be0..1c230db376a49 100644 --- a/frame/support/test/tests/derive_no_bound_ui/partial_eq.stderr +++ b/frame/support/test/tests/derive_no_bound_ui/partial_eq.stderr @@ -1,7 +1,5 @@ error[E0369]: binary operation `==` cannot be applied to type `::C` - --> $DIR/partial_eq.rs:7:2 + --> tests/derive_no_bound_ui/partial_eq.rs:7:2 | 7 | c: T::C, | ^ - | - = note: the trait `std::cmp::PartialEq` is not implemented for `::C` diff --git a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound.stderr b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound.stderr index 3a636d9f659c7..1d581ea7ed572 100644 --- a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound.stderr +++ b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound.stderr @@ -19,8 +19,3 @@ error[E0369]: binary operation `==` cannot be applied to type `&, bar: T::Bar) -> DispatchResultWithPostInfo { | ^^^ - | -help: consider further restricting this bound - | -17 | #[pallet::call + std::cmp::PartialEq] - | +++++++++++++++++++++ diff --git a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr index f182382d18f11..b1487776eac50 100644 --- a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr +++ b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr @@ -19,11 +19,6 @@ error[E0369]: binary operation `==` cannot be applied to type `&, bar: T::Bar) -> DispatchResultWithPostInfo { | ^^^ - | -help: consider further restricting this bound - | -17 | #[pallet::call + std::cmp::PartialEq] - | +++++++++++++++++++++ error[E0277]: the trait bound `::Bar: WrapperTypeEncode` is not satisfied --> tests/pallet_ui/call_argument_invalid_bound_2.rs:20:36 diff --git a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.stderr b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.stderr index c196b28a31ce6..a0418760ba7e2 100644 --- a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.stderr +++ b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.stderr @@ -8,3 +8,7 @@ error[E0277]: `Bar` doesn't implement `std::fmt::Debug` = note: add `#[derive(Debug)]` to `Bar` or manually `impl std::fmt::Debug for Bar` = note: required because of the requirements on the impl of `std::fmt::Debug` for `&Bar` = note: required for the cast to the object type `dyn std::fmt::Debug` +help: consider annotating `Bar` with `#[derive(Debug)]` + | +17 | #[derive(Debug)] + | diff --git a/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr b/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr index 0f2ea7e161c4e..7edb55a62d401 100644 --- a/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr +++ b/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr @@ -4,4 +4,14 @@ error[E0277]: the trait bound `MyError: PalletError` is not satisfied 1 | #[frame_support::pallet] | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `PalletError` is not implemented for `MyError` | + = help: the following other types implement trait `PalletError`: + () + (TupleElement0, TupleElement1) + (TupleElement0, TupleElement1, TupleElement2) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4, TupleElement5) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4, TupleElement5, TupleElement6) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4, TupleElement5, TupleElement6, TupleElement7) + and 36 others = note: this error originates in the derive macro `frame_support::PalletError` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/frame/support/test/tests/pallet_ui/event_field_not_member.stderr b/frame/support/test/tests/pallet_ui/event_field_not_member.stderr index 3db258a819fcb..92623e0329fe3 100644 --- a/frame/support/test/tests/pallet_ui/event_field_not_member.stderr +++ b/frame/support/test/tests/pallet_ui/event_field_not_member.stderr @@ -9,11 +9,6 @@ error[E0369]: binary operation `==` cannot be applied to type `& { - | +++++++++++++++++++++ error[E0277]: `::Bar` doesn't implement `std::fmt::Debug` --> tests/pallet_ui/event_field_not_member.rs:23:7 diff --git a/frame/support/test/tests/pallet_ui/genesis_default_not_satisfied.stderr b/frame/support/test/tests/pallet_ui/genesis_default_not_satisfied.stderr index e01f66fa3d19d..80903928585b4 100644 --- a/frame/support/test/tests/pallet_ui/genesis_default_not_satisfied.stderr +++ b/frame/support/test/tests/pallet_ui/genesis_default_not_satisfied.stderr @@ -9,3 +9,7 @@ note: required by a bound in `GenesisBuild` | | pub trait GenesisBuild: Default + sp_runtime::traits::MaybeSerializeDeserialize { | ^^^^^^^ required by this bound in `GenesisBuild` +help: consider annotating `pallet::GenesisConfig` with `#[derive(Default)]` + | +19 | #[derive(Default)] + | diff --git a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr index 87528751a0a7a..e1c1b32d65c39 100644 --- a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr +++ b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr @@ -4,6 +4,11 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied 10 | #[pallet::without_storage_info] | ^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` | + = help: the following other types implement trait `WrapperTypeDecode`: + Arc + Box + Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes = note: required because of the requirements on the impl of `Decode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` @@ -14,6 +19,16 @@ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied 10 | #[pallet::without_storage_info] | ^^^^^^^^^^^^^^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` | + = help: the following other types implement trait `EncodeLike`: + <&&T as EncodeLike> + <&T as EncodeLike> + <&T as EncodeLike> + <&[(K, V)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[T] as EncodeLike>> + and 273 others = note: required because of the requirements on the impl of `FullEncode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` @@ -24,6 +39,16 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied 10 | #[pallet::without_storage_info] | ^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` | + = help: the following other types implement trait `WrapperTypeEncode`: + &T + &mut T + Arc + Box + Cow<'a, T> + Rc + Vec + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes + and 2 others = note: required because of the requirements on the impl of `Encode` for `Bar` = note: required because of the requirements on the impl of `FullEncode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` @@ -35,6 +60,16 @@ error[E0277]: the trait bound `Bar: TypeInfo` is not satisfied 21 | #[pallet::storage] | ^^^^^^^ the trait `TypeInfo` is not implemented for `Bar` | + = help: the following other types implement trait `TypeInfo`: + &T + &mut T + () + (A, B) + (A, B, C) + (A, B, C, D) + (A, B, C, D, E) + (A, B, C, D, E, F) + and 158 others = note: required because of the requirements on the impl of `StaticTypeInfo` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` @@ -44,6 +79,11 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied 21 | #[pallet::storage] | ^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` | + = help: the following other types implement trait `WrapperTypeDecode`: + Arc + Box + Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes = note: required because of the requirements on the impl of `Decode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` @@ -54,6 +94,16 @@ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied 21 | #[pallet::storage] | ^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` | + = help: the following other types implement trait `EncodeLike`: + <&&T as EncodeLike> + <&T as EncodeLike> + <&T as EncodeLike> + <&[(K, V)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[T] as EncodeLike>> + and 273 others = note: required because of the requirements on the impl of `FullEncode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` @@ -64,6 +114,16 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied 21 | #[pallet::storage] | ^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` | + = help: the following other types implement trait `WrapperTypeEncode`: + &T + &mut T + Arc + Box + Cow<'a, T> + Rc + Vec + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes + and 2 others = note: required because of the requirements on the impl of `Encode` for `Bar` = note: required because of the requirements on the impl of `FullEncode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` diff --git a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr index 6c3f6dc662fbc..7d8c448f00193 100644 --- a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr +++ b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr @@ -4,6 +4,11 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied 10 | #[pallet::without_storage_info] | ^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` | + = help: the following other types implement trait `WrapperTypeDecode`: + Arc + Box + Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes = note: required because of the requirements on the impl of `Decode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` @@ -14,6 +19,16 @@ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied 10 | #[pallet::without_storage_info] | ^^^^^^^^^^^^^^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` | + = help: the following other types implement trait `EncodeLike`: + <&&T as EncodeLike> + <&T as EncodeLike> + <&T as EncodeLike> + <&[(K, V)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[T] as EncodeLike>> + and 273 others = note: required because of the requirements on the impl of `FullEncode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` @@ -24,6 +39,16 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied 10 | #[pallet::without_storage_info] | ^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` | + = help: the following other types implement trait `WrapperTypeEncode`: + &T + &mut T + Arc + Box + Cow<'a, T> + Rc + Vec + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes + and 2 others = note: required because of the requirements on the impl of `Encode` for `Bar` = note: required because of the requirements on the impl of `FullEncode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` @@ -35,6 +60,16 @@ error[E0277]: the trait bound `Bar: TypeInfo` is not satisfied 21 | #[pallet::storage] | ^^^^^^^ the trait `TypeInfo` is not implemented for `Bar` | + = help: the following other types implement trait `TypeInfo`: + &T + &mut T + () + (A, B) + (A, B, C) + (A, B, C, D) + (A, B, C, D, E) + (A, B, C, D, E, F) + and 158 others = note: required because of the requirements on the impl of `StaticTypeInfo` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` @@ -44,6 +79,11 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied 21 | #[pallet::storage] | ^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` | + = help: the following other types implement trait `WrapperTypeDecode`: + Arc + Box + Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes = note: required because of the requirements on the impl of `Decode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` @@ -54,6 +94,16 @@ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied 21 | #[pallet::storage] | ^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` | + = help: the following other types implement trait `EncodeLike`: + <&&T as EncodeLike> + <&T as EncodeLike> + <&T as EncodeLike> + <&[(K, V)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[T] as EncodeLike>> + and 273 others = note: required because of the requirements on the impl of `FullEncode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` @@ -64,6 +114,16 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied 21 | #[pallet::storage] | ^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` | + = help: the following other types implement trait `WrapperTypeEncode`: + &T + &mut T + Arc + Box + Cow<'a, T> + Rc + Vec + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes + and 2 others = note: required because of the requirements on the impl of `Encode` for `Bar` = note: required because of the requirements on the impl of `FullEncode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` diff --git a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.stderr b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.stderr index 68856f122c7ac..d87e6900197f5 100644 --- a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.stderr +++ b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.stderr @@ -4,4 +4,14 @@ error[E0277]: the trait bound `Bar: MaxEncodedLen` is not satisfied 9 | #[pallet::pallet] | ^^^^^^ the trait `MaxEncodedLen` is not implemented for `Bar` | + = help: the following other types implement trait `MaxEncodedLen`: + () + (TupleElement0, TupleElement1) + (TupleElement0, TupleElement1, TupleElement2) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4, TupleElement5) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4, TupleElement5, TupleElement6) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4, TupleElement5, TupleElement6, TupleElement7) + and 70 others = note: required because of the requirements on the impl of `StorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` diff --git a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr index 226cb40f1d48b..bc8e99cd65006 100644 --- a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr +++ b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr @@ -4,5 +4,15 @@ error[E0277]: the trait bound `Bar: MaxEncodedLen` is not satisfied 12 | #[pallet::pallet] | ^^^^^^ the trait `MaxEncodedLen` is not implemented for `Bar` | + = help: the following other types implement trait `MaxEncodedLen`: + () + (TupleElement0, TupleElement1) + (TupleElement0, TupleElement1, TupleElement2) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4, TupleElement5) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4, TupleElement5, TupleElement6) + (TupleElement0, TupleElement1, TupleElement2, TupleElement3, TupleElement4, TupleElement5, TupleElement6, TupleElement7) + and 70 others = note: required because of the requirements on the impl of `KeyGeneratorMaxEncodedLen` for `Key` = note: required because of the requirements on the impl of `StorageInfoTrait` for `frame_support::pallet_prelude::StorageNMap<_GeneratedPrefixForStorageFoo, Key, u32>` diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 0ac3cfbe1244e..02ef37370ffeb 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -613,7 +613,7 @@ fn generate_api_impl_for_runtime_api(impls: &[ItemImpl]) -> Result let mut visitor = ApiRuntimeImplToApiRuntimeApiImpl { runtime_block, runtime_mod_path: &runtime_mod_path, - runtime_type: &*runtime_type, + runtime_type, trait_generic_arguments: &trait_generic_arguments, impl_trait: &impl_trait.ident, }; diff --git a/primitives/runtime/src/bounded/bounded_vec.rs b/primitives/runtime/src/bounded/bounded_vec.rs index c9c9f851d3249..10d9fc608c273 100644 --- a/primitives/runtime/src/bounded/bounded_vec.rs +++ b/primitives/runtime/src/bounded/bounded_vec.rs @@ -104,10 +104,46 @@ where /// A bounded slice. /// /// Similar to a `BoundedVec`, but not owned and cannot be decoded. -#[derive(Encode, scale_info::TypeInfo)] -#[scale_info(skip_type_params(S))] +#[derive(Encode)] pub struct BoundedSlice<'a, T, S>(pub(super) &'a [T], PhantomData); +// This can be replaced with +// #[derive(scale_info::TypeInfo)] +// #[scale_info(skip_type_params(S))] +// again once this issue is fixed in the rust compiler: https://github.com/rust-lang/rust/issues/96956 +// Tracking issues: https://github.com/paritytech/substrate/issues/11915 +impl<'a, T, S> scale_info::TypeInfo for BoundedSlice<'a, T, S> +where + &'a [T]: scale_info::TypeInfo + 'static, + PhantomData: scale_info::TypeInfo + 'static, + T: scale_info::TypeInfo + 'static, + S: 'static, +{ + type Identity = Self; + + fn type_info() -> ::scale_info::Type { + scale_info::Type::builder() + .path(scale_info::Path::new("BoundedSlice", "sp_runtime::bounded::bounded_vec")) + .type_params(<[_]>::into_vec(Box::new([ + scale_info::TypeParameter::new( + "T", + core::option::Option::Some(::scale_info::meta_type::()), + ), + scale_info::TypeParameter::new("S", ::core::option::Option::None), + ]))) + .docs(&[ + "A bounded slice.", + "", + "Similar to a `BoundedVec`, but not owned and cannot be decoded.", + ]) + .composite( + scale_info::build::Fields::unnamed() + .field(|f| f.ty::<&'static [T]>().type_name("&'static[T]").docs(&[])) + .field(|f| f.ty::>().type_name("PhantomData").docs(&[])), + ) + } +} + // `BoundedSlice`s encode to something which will always decode into a `BoundedVec`, // `WeakBoundedVec`, or a `Vec`. impl<'a, T: Encode + Decode, S: Get> EncodeLike> for BoundedSlice<'a, T, S> {} diff --git a/primitives/runtime/src/curve.rs b/primitives/runtime/src/curve.rs index c6bfa66017870..99733dbbe9a55 100644 --- a/primitives/runtime/src/curve.rs +++ b/primitives/runtime/src/curve.rs @@ -24,7 +24,7 @@ use crate::{ use core::ops::Sub; /// Piecewise Linear function in [0, 1] -> [0, 1]. -#[derive(PartialEq, Eq, sp_core::RuntimeDebug, scale_info::TypeInfo)] +#[derive(PartialEq, Eq, sp_core::RuntimeDebug)] pub struct PiecewiseLinear<'a> { /// Array of points. Must be in order from the lowest abscissas to the highest. pub points: &'a [(Perbill, Perbill)], @@ -32,6 +32,36 @@ pub struct PiecewiseLinear<'a> { pub maximum: Perbill, } +// This can be replaced with +// #[derive(scale_info::TypeInfo)] +// #[scale_info(skip_type_params(S))] +// again once this issue is fixed in the rust compiler: https://github.com/rust-lang/rust/issues/96956 +// Tracking issues: https://github.com/paritytech/substrate/issues/11915 +impl scale_info::TypeInfo for PiecewiseLinear<'static> { + type Identity = Self; + fn type_info() -> ::scale_info::Type { + scale_info::Type::builder() + .path(scale_info::Path::new("PiecewiseLinear", "sp_runtime::curve")) + .type_params(crate::Vec::new()) + .docs(&["Piecewise Linear function in [0, 1] -> [0, 1]."]) + .composite( + scale_info::build::Fields::named() + .field(|f| { + f.ty::<&'static[(Perbill, Perbill)]>() + .name("points") + .type_name("&'static[(Perbill, Perbill)]") + .docs(&["Array of points. Must be in order from the lowest abscissas to the highest."]) + }) + .field(|f| { + f.ty::() + .name("maximum") + .type_name("Perbill") + .docs(&["The maximum value that can be returned."]) + }), + ) + } +} + fn abs_sub + Clone>(a: N, b: N) -> N where { a.clone().max(b.clone()) - a.min(b) } diff --git a/primitives/runtime/src/traits.rs b/primitives/runtime/src/traits.rs index fba3117c41617..a82ae1d62f56a 100644 --- a/primitives/runtime/src/traits.rs +++ b/primitives/runtime/src/traits.rs @@ -55,7 +55,7 @@ pub trait Lazy { impl<'a> Lazy<[u8]> for &'a [u8] { fn get(&mut self) -> &[u8] { - &**self + self } } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 1dca617acf912..7a17d44aa5b69 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -280,7 +280,7 @@ where L: TrieConfiguration, DB: hash_db::HashDBRef, { - TrieDB::::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec())) + TrieDB::::new(db, root)?.get(key).map(|x| x.map(|val| val.to_vec())) } /// Read a value from the trie with given Query. @@ -295,7 +295,7 @@ where Q: Query, DB: hash_db::HashDBRef, { - TrieDB::::new(&*db, root)? + TrieDB::::new(db, root)? .get_with(key, query) .map(|x| x.map(|val| val.to_vec())) } @@ -354,7 +354,7 @@ pub fn record_all_keys( where DB: hash_db::HashDBRef, { - let trie = TrieDB::::new(&*db, root)?; + let trie = TrieDB::::new(db, root)?; let iter = trie.iter()?; for x in iter { @@ -379,7 +379,7 @@ pub fn read_child_trie_value( where DB: hash_db::HashDBRef, { - let db = KeySpacedDB::new(&*db, keyspace); + let db = KeySpacedDB::new(db, keyspace); TrieDB::::new(&db, root)?.get(key).map(|x| x.map(|val| val.to_vec())) } @@ -400,7 +400,7 @@ where // root is fetched from DB, not writable by runtime, so it's always valid. root.as_mut().copy_from_slice(root_slice); - let db = KeySpacedDB::new(&*db, keyspace); + let db = KeySpacedDB::new(db, keyspace); TrieDB::::new(&db, &root)? .get_with(key, query) .map(|x| x.map(|val| val.to_vec())) @@ -501,7 +501,7 @@ where T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, { fn as_hash_db(&self) -> &dyn hash_db::HashDB { - &*self + self } fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { diff --git a/primitives/version/proc-macro/src/decl_runtime_version.rs b/primitives/version/proc-macro/src/decl_runtime_version.rs index ee81940b53993..9a25adfa5fca2 100644 --- a/primitives/version/proc-macro/src/decl_runtime_version.rs +++ b/primitives/version/proc-macro/src/decl_runtime_version.rs @@ -37,7 +37,7 @@ pub fn decl_runtime_version_impl(input: proc_macro::TokenStream) -> proc_macro:: } fn decl_runtime_version_impl_inner(item: ItemConst) -> Result { - let runtime_version = ParseRuntimeVersion::parse_expr(&*item.expr)?.build(item.expr.span())?; + let runtime_version = ParseRuntimeVersion::parse_expr(&item.expr)?.build(item.expr.span())?; let link_section = generate_emit_link_section_decl(&runtime_version.encode(), "runtime_version"); diff --git a/rustfmt.toml b/rustfmt.toml index 441913f619cdc..f6fbe80064fce 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -21,3 +21,4 @@ match_block_trailing_comma = true trailing_comma = "Vertical" trailing_semicolon = false use_field_init_shorthand = true +edition = "2021" diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 1d9dc5dcd11da..202560f18cf84 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -130,7 +130,7 @@ pub enum Transport { impl Transport { fn as_client(&self) -> Option<&WsClient> { match self { - Self::RemoteClient(client) => Some(&*client), + Self::RemoteClient(client) => Some(client), _ => None, } } From fdfdc73f9e64dc47934b72eb9af3e1989e4ba699 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Tue, 26 Jul 2022 13:44:48 +0100 Subject: [PATCH 46/46] Properly defer slashes (#11823) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * initial draft of fixing slashing * fix test * Update frame/staking/src/tests.rs Co-authored-by: Piotr Mikołajczyk * last touches * add more detail about unbonding * add migration * fmt Co-authored-by: Piotr Mikołajczyk Co-authored-by: parity-processbot <> --- frame/staking/src/lib.rs | 11 +-- frame/staking/src/migrations.rs | 23 ++++++ frame/staking/src/mock.rs | 15 ++++ frame/staking/src/pallet/impls.rs | 47 ++++++------ frame/staking/src/pallet/mod.rs | 4 - frame/staking/src/tests.rs | 118 +++++++++++++++++++++++++++--- 6 files changed, 177 insertions(+), 41 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 50c033bdc7e28..3fff6312f333f 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -872,11 +872,12 @@ enum Releases { V2_0_0, V3_0_0, V4_0_0, - V5_0_0, // blockable validators. - V6_0_0, // removal of all storage associated with offchain phragmen. - V7_0_0, // keep track of number of nominators / validators in map - V8_0_0, // populate `VoterList`. - V9_0_0, // inject validators into `VoterList` as well. + V5_0_0, // blockable validators. + V6_0_0, // removal of all storage associated with offchain phragmen. + V7_0_0, // keep track of number of nominators / validators in map + V8_0_0, // populate `VoterList`. + V9_0_0, // inject validators into `VoterList` as well. + V10_0_0, // remove `EarliestUnappliedSlash`. } impl Default for Releases { diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 14846da8a5d54..101cac0a31348 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -20,6 +20,29 @@ use super::*; use frame_election_provider_support::SortedListProvider; use frame_support::traits::OnRuntimeUpgrade; +pub mod v10 { + use super::*; + use frame_support::storage_alias; + + #[storage_alias] + type EarliestUnappliedSlash = StorageValue, EraIndex>; + + pub struct MigrateToV10(sp_std::marker::PhantomData); + impl OnRuntimeUpgrade for MigrateToV10 { + fn on_runtime_upgrade() -> frame_support::weights::Weight { + if StorageVersion::::get() == Releases::V9_0_0 { + EarliestUnappliedSlash::::kill(); + StorageVersion::::put(Releases::V10_0_0); + + T::DbWeight::get().reads_writes(1, 1) + } else { + log!(warn, "MigrateToV10 should be removed."); + T::DbWeight::get().reads(1) + } + } + } +} + pub mod v9 { use super::*; diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index bd2d8cdc32ce9..70d00c2b648d8 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -549,6 +549,7 @@ impl ExtBuilder { ext } pub fn build_and_execute(self, test: impl FnOnce() -> ()) { + sp_tracing::try_init_simple(); let mut ext = self.build(); ext.execute_with(test); ext.execute_with(post_conditions); @@ -884,6 +885,20 @@ pub(crate) fn staking_events() -> Vec> { .collect() } +parameter_types! { + static StakingEventsIndex: usize = 0; +} + +pub(crate) fn staking_events_since_last_call() -> Vec> { + let all: Vec<_> = System::events() + .into_iter() + .filter_map(|r| if let Event::Staking(inner) = r.event { Some(inner) } else { None }) + .collect(); + let seen = StakingEventsIndex::get(); + StakingEventsIndex::set(all.len()); + all.into_iter().skip(seen).collect() +} + pub(crate) fn balances(who: &AccountId) -> (Balance, Balance) { (Balances::free_balance(who), Balances::reserved_balance(who)) } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 7656eec80a5ff..68aa97db8a324 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -32,7 +32,7 @@ use frame_support::{ use frame_system::{pallet_prelude::BlockNumberFor, RawOrigin}; use pallet_session::historical; use sp_runtime::{ - traits::{Bounded, Convert, SaturatedConversion, Saturating, StaticLookup, Zero}, + traits::{Bounded, Convert, One, SaturatedConversion, Saturating, StaticLookup, Zero}, Perbill, }; use sp_staking::{ @@ -599,20 +599,17 @@ impl Pallet { /// Apply previously-unapplied slashes on the beginning of a new era, after a delay. fn apply_unapplied_slashes(active_era: EraIndex) { - let slash_defer_duration = T::SlashDeferDuration::get(); - ::EarliestUnappliedSlash::mutate(|earliest| { - if let Some(ref mut earliest) = earliest { - let keep_from = active_era.saturating_sub(slash_defer_duration); - for era in (*earliest)..keep_from { - let era_slashes = ::UnappliedSlashes::take(&era); - for slash in era_slashes { - slashing::apply_slash::(slash, era); - } - } - - *earliest = (*earliest).max(keep_from) - } - }) + let era_slashes = ::UnappliedSlashes::take(&active_era); + log!( + debug, + "found {} slashes scheduled to be executed in era {:?}", + era_slashes.len(), + active_era, + ); + for slash in era_slashes { + let slash_era = active_era.saturating_sub(T::SlashDeferDuration::get()); + slashing::apply_slash::(slash, slash_era); + } } /// Add reward points to validators using their stash account ID. @@ -1209,11 +1206,6 @@ where } }; - ::EarliestUnappliedSlash::mutate(|earliest| { - if earliest.is_none() { - *earliest = Some(active_era) - } - }); add_db_reads_writes(1, 1); let slash_defer_duration = T::SlashDeferDuration::get(); @@ -1263,9 +1255,18 @@ where } } else { // Defer to end of some `slash_defer_duration` from now. - ::UnappliedSlashes::mutate(active_era, move |for_later| { - for_later.push(unapplied) - }); + log!( + debug, + "deferring slash of {:?}% happened in {:?} (reported in {:?}) to {:?}", + slash_fraction, + slash_era, + active_era, + slash_era + slash_defer_duration + 1, + ); + ::UnappliedSlashes::mutate( + slash_era.saturating_add(slash_defer_duration).saturating_add(One::one()), + move |for_later| for_later.push(unapplied), + ); add_db_reads_writes(1, 1); } } else { diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index e53464195de23..c999020c28167 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -477,10 +477,6 @@ pub mod pallet { ValueQuery, >; - /// The earliest era for which we have a pending, unapplied slash. - #[pallet::storage] - pub(crate) type EarliestUnappliedSlash = StorageValue<_, EraIndex>; - /// The last planned session scheduled by the session pallet. /// /// This is basically in sync with the call to [`pallet_session::SessionManager::new_session`]. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index b76126f0c5d04..d14d8c4a75f2e 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -2777,12 +2777,103 @@ fn deferred_slashes_are_deferred() { assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(101), 2000); + System::reset_events(); + // at the start of era 4, slashes from era 1 are processed, // after being deferred for at least 2 full eras. mock::start_active_era(4); assert_eq!(Balances::free_balance(11), 900); assert_eq!(Balances::free_balance(101), 2000 - (nominated_value / 10)); + + assert_eq!( + staking_events_since_last_call(), + vec![ + Event::StakersElected, + Event::EraPaid(3, 11075, 33225), + Event::Slashed(11, 100), + Event::Slashed(101, 12) + ] + ); + }) +} + +#[test] +fn retroactive_deferred_slashes_two_eras_before() { + ExtBuilder::default().slash_defer_duration(2).build_and_execute(|| { + assert_eq!(BondingDuration::get(), 3); + + mock::start_active_era(1); + let exposure_11_at_era1 = Staking::eras_stakers(active_era(), 11); + + mock::start_active_era(3); + on_offence_in_era( + &[OffenceDetails { offender: (11, exposure_11_at_era1), reporters: vec![] }], + &[Perbill::from_percent(10)], + 1, // should be deferred for two full eras, and applied at the beginning of era 4. + DisableStrategy::Never, + ); + System::reset_events(); + + mock::start_active_era(4); + + assert_eq!( + staking_events_since_last_call(), + vec![ + Event::StakersElected, + Event::EraPaid(3, 7100, 21300), + Event::Slashed(11, 100), + Event::Slashed(101, 12) + ] + ); + }) +} + +#[test] +fn retroactive_deferred_slashes_one_before() { + ExtBuilder::default().slash_defer_duration(2).build_and_execute(|| { + assert_eq!(BondingDuration::get(), 3); + + mock::start_active_era(1); + let exposure_11_at_era1 = Staking::eras_stakers(active_era(), 11); + + // unbond at slash era. + mock::start_active_era(2); + assert_ok!(Staking::chill(Origin::signed(10))); + assert_ok!(Staking::unbond(Origin::signed(10), 100)); + + mock::start_active_era(3); + on_offence_in_era( + &[OffenceDetails { offender: (11, exposure_11_at_era1), reporters: vec![] }], + &[Perbill::from_percent(10)], + 2, // should be deferred for two full eras, and applied at the beginning of era 5. + DisableStrategy::Never, + ); + System::reset_events(); + + mock::start_active_era(4); + assert_eq!( + staking_events_since_last_call(), + vec![Event::StakersElected, Event::EraPaid(3, 11075, 33225)] + ); + + assert_eq!(Staking::ledger(10).unwrap().total, 1000); + // slash happens after the next line. + mock::start_active_era(5); + assert_eq!( + staking_events_since_last_call(), + vec![ + Event::StakersElected, + Event::EraPaid(4, 11075, 33225), + Event::Slashed(11, 100), + Event::Slashed(101, 12) + ] + ); + + // their ledger has already been slashed. + assert_eq!(Staking::ledger(10).unwrap().total, 900); + assert_ok!(Staking::unbond(Origin::signed(10), 1000)); + assert_eq!(Staking::ledger(10).unwrap().total, 900); }) } @@ -2871,6 +2962,7 @@ fn remove_deferred() { assert_eq!(Balances::free_balance(101), 2000); let nominated_value = exposure.others.iter().find(|o| o.who == 101).unwrap().value; + // deferred to start of era 4. on_offence_now( &[OffenceDetails { offender: (11, exposure.clone()), reporters: vec![] }], &[Perbill::from_percent(10)], @@ -2881,6 +2973,7 @@ fn remove_deferred() { mock::start_active_era(2); + // reported later, but deferred to start of era 4 as well. on_offence_in_era( &[OffenceDetails { offender: (11, exposure.clone()), reporters: vec![] }], &[Perbill::from_percent(15)], @@ -2894,7 +2987,8 @@ fn remove_deferred() { Error::::EmptyTargets ); - assert_ok!(Staking::cancel_deferred_slash(Origin::root(), 1, vec![0])); + // cancel one of them. + assert_ok!(Staking::cancel_deferred_slash(Origin::root(), 4, vec![0])); assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(101), 2000); @@ -2906,13 +3000,19 @@ fn remove_deferred() { // at the start of era 4, slashes from era 1 are processed, // after being deferred for at least 2 full eras. + System::reset_events(); mock::start_active_era(4); - // the first slash for 10% was cancelled, so no effect. - assert_eq!(Balances::free_balance(11), 1000); - assert_eq!(Balances::free_balance(101), 2000); - - mock::start_active_era(5); + // the first slash for 10% was cancelled, but the 15% one + assert_eq!( + staking_events_since_last_call(), + vec![ + Event::StakersElected, + Event::EraPaid(3, 11075, 33225), + Event::Slashed(11, 50), + Event::Slashed(101, 7) + ] + ); let slash_10 = Perbill::from_percent(10); let slash_15 = Perbill::from_percent(15); @@ -2965,7 +3065,7 @@ fn remove_multi_deferred() { &[Perbill::from_percent(25)], ); - assert_eq!(::UnappliedSlashes::get(&1).len(), 5); + assert_eq!(::UnappliedSlashes::get(&4).len(), 5); // fails if list is not sorted assert_noop!( @@ -2983,9 +3083,9 @@ fn remove_multi_deferred() { Error::::InvalidSlashIndex ); - assert_ok!(Staking::cancel_deferred_slash(Origin::root(), 1, vec![0, 2, 4])); + assert_ok!(Staking::cancel_deferred_slash(Origin::root(), 4, vec![0, 2, 4])); - let slashes = ::UnappliedSlashes::get(&1); + let slashes = ::UnappliedSlashes::get(&4); assert_eq!(slashes.len(), 2); assert_eq!(slashes[0].validator, 21); assert_eq!(slashes[1].validator, 42);