From d6b5eb9614df0d244c95d66f479413934d9b0098 Mon Sep 17 00:00:00 2001 From: quake Date: Mon, 20 May 2024 13:21:25 +0900 Subject: [PATCH 1/3] feat: add estimate_fee_rate rpc --- rpc/README.md | 48 ++++++++++++++++++ rpc/src/module/pool.rs | 65 ++++++++++++++++++++++++- tx-pool/src/component/pool_map.rs | 29 ++++++++++- tx-pool/src/component/tests/estimate.rs | 56 +++++++++++++++++++++ tx-pool/src/component/tests/mod.rs | 1 + tx-pool/src/pool.rs | 12 ++++- tx-pool/src/process.rs | 6 +++ tx-pool/src/service.rs | 19 ++++++++ 8 files changed, 233 insertions(+), 3 deletions(-) create mode 100644 tx-pool/src/component/tests/estimate.rs diff --git a/rpc/README.md b/rpc/README.md index 016b32b8b8..6dddc8d57a 100644 --- a/rpc/README.md +++ b/rpc/README.md @@ -104,6 +104,7 @@ The crate `ckb-rpc`'s minimum supported rustc version is 1.71.1. * [Module Pool](#module-pool) [👉 OpenRPC spec](http://playground.open-rpc.org/?uiSchema[appBar][ui:title]=CKB-Pool&uiSchema[appBar][ui:splitView]=false&uiSchema[appBar][ui:examplesDropdown]=false&uiSchema[appBar][ui:logoUrl]=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/ckb-logo.jpg&schemaUrl=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/json/pool_rpc_doc.json) * [Method `send_transaction`](#pool-send_transaction) + * [Method `estimate_fee_rate`](#pool-estimate_fee_rate) * [Method `test_tx_pool_accept`](#pool-test_tx_pool_accept) * [Method `remove_transaction`](#pool-remove_transaction) * [Method `tx_pool_info`](#pool-tx_pool_info) @@ -4422,6 +4423,53 @@ Response } ``` + +#### Method `estimate_fee_rate` +* `estimate_fee_rate(target_to_be_committed)` + * `target_to_be_committed`: [`Uint64`](#type-uint64) +* result: [`Uint64`](#type-uint64) + +Estimate fee rate for a transaction to be committed within target block number by using a simple strategy. + +Since CKB transaction confirmation involves a two-step process—1) propose and 2) commit, it is complex to +predict the transaction fee accurately with the expectation that it will be included within a certain block height. + +This method relies on two assumptions and uses a simple strategy to estimate the transaction fee: 1) all transactions +in the pool are waiting to be proposed, and 2) no new transactions will be added to the pool. + +In practice, this simple method should achieve good accuracy fee rate and running performance. + +###### Params + +* `target_to_be_committed` - The target block number to be committed, minimum value is 3 and maximum value is 131. + +###### Returns + +The estimated fee rate in shannons per kilobyte. + +###### Examples + +```json +{ + "id": 42, + "jsonrpc": "2.0", + "method": "estimate_fee_rate", + "params": [ + "0x4" + ] +} +``` + +Response + +```json +{ + "id": 42, + "jsonrpc": "2.0", + "result": "0x3e8" +} +``` + #### Method `test_tx_pool_accept` * `test_tx_pool_accept(tx, outputs_validator)` diff --git a/rpc/src/module/pool.rs b/rpc/src/module/pool.rs index aab1071e0d..0551f9de78 100644 --- a/rpc/src/module/pool.rs +++ b/rpc/src/module/pool.rs @@ -3,7 +3,8 @@ use async_trait::async_trait; use ckb_chain_spec::consensus::Consensus; use ckb_constant::hardfork::{mainnet, testnet}; use ckb_jsonrpc_types::{ - EntryCompleted, OutputsValidator, PoolTxDetailInfo, RawTxPool, Script, Transaction, TxPoolInfo, + BlockNumber, EntryCompleted, OutputsValidator, PoolTxDetailInfo, RawTxPool, Script, + Transaction, TxPoolInfo, Uint64, }; use ckb_logger::error; use ckb_shared::shared::Shared; @@ -110,6 +111,50 @@ pub trait PoolRpc { outputs_validator: Option, ) -> Result; + /// Estimate fee rate for a transaction to be committed within target block number by using a simple strategy. + /// + /// Since CKB transaction confirmation involves a two-step process—1) propose and 2) commit, it is complex to + /// predict the transaction fee accurately with the expectation that it will be included within a certain block height. + /// + /// This method relies on two assumptions and uses a simple strategy to estimate the transaction fee: 1) all transactions + /// in the pool are waiting to be proposed, and 2) no new transactions will be added to the pool. + /// + /// In practice, this simple method should achieve good accuracy fee rate and running performance. + /// + /// ## Params + /// + /// * `target_to_be_committed` - The target block number to be committed, minimum value is 3 and maximum value is 131. + /// + /// ## Returns + /// + /// The estimated fee rate in shannons per kilobyte. + /// + /// ## Examples + /// + /// ```json + /// { + /// "id": 42, + /// "jsonrpc": "2.0", + /// "method": "estimate_fee_rate", + /// "params": [ + /// "0x4" + /// ] + /// } + /// ``` + /// + /// Response + /// + /// ```json + /// { + /// "id": 42, + /// "jsonrpc": "2.0", + /// "result": "0x3e8" + /// } + /// ``` + /// + #[rpc(name = "estimate_fee_rate")] + fn estimate_fee_rate(&self, target_to_be_committed: BlockNumber) -> Result; + /// Test if a transaction can be accepted by the transaction pool without inserting it into the pool or rebroadcasting it to peers. /// The parameters and errors of this method are the same as `send_transaction`. /// @@ -604,6 +649,24 @@ impl PoolRpc for PoolRpcImpl { } } + fn estimate_fee_rate(&self, target_to_be_committed: BlockNumber) -> Result { + let target_to_be_committed = target_to_be_committed.value(); + if !(3..=131).contains(&target_to_be_committed) { + return Err(RPCError::invalid_params( + "target to be committed block number must be in range [3, 131]", + )); + } + let fee_rate = self + .shared + .tx_pool_controller() + .estimate_fee_rate(target_to_be_committed) + .map_err(|e| { + error!("Send estimate_fee_rate request error {}", e); + RPCError::ckb_internal_error(e) + })?; + Ok(fee_rate.as_u64().into()) + } + fn test_tx_pool_accept( &self, tx: Transaction, diff --git a/tx-pool/src/component/pool_map.rs b/tx-pool/src/component/pool_map.rs index a421f10e04..ba31c1fb5a 100644 --- a/tx-pool/src/component/pool_map.rs +++ b/tx-pool/src/component/pool_map.rs @@ -9,7 +9,7 @@ use crate::error::Reject; use crate::TxEntry; use ckb_logger::{debug, error, trace}; use ckb_types::core::error::OutPointError; -use ckb_types::core::Cycle; +use ckb_types::core::{Cycle, FeeRate}; use ckb_types::packed::OutPoint; use ckb_types::prelude::*; use ckb_types::{ @@ -329,6 +329,33 @@ impl PoolMap { conflicts } + pub(crate) fn estimate_fee_rate( + &self, + mut target_blocks: usize, + max_block_bytes: usize, + max_block_cycles: Cycle, + min_fee_rate: FeeRate, + ) -> FeeRate { + debug_assert!(target_blocks > 0); + let iter = self.entries.iter_by_score().rev(); + let mut current_block_bytes = 0; + let mut current_block_cycles = 0; + for entry in iter { + current_block_bytes += entry.inner.size; + current_block_cycles += entry.inner.cycles; + if current_block_bytes >= max_block_bytes || current_block_cycles >= max_block_cycles { + target_blocks -= 1; + if target_blocks == 0 { + return entry.inner.fee_rate(); + } + current_block_bytes = entry.inner.size; + current_block_cycles = entry.inner.cycles; + } + } + + min_fee_rate + } + // find the pending txs sorted by score, and return their proposal short ids pub(crate) fn get_proposals( &self, diff --git a/tx-pool/src/component/tests/estimate.rs b/tx-pool/src/component/tests/estimate.rs new file mode 100644 index 0000000000..fa222fd91a --- /dev/null +++ b/tx-pool/src/component/tests/estimate.rs @@ -0,0 +1,56 @@ +use crate::component::tests::util::build_tx; +use crate::component::{ + entry::TxEntry, + pool_map::{PoolMap, Status}, +}; +use ckb_types::core::{Capacity, Cycle, FeeRate}; + +#[test] +fn test_estimate_fee_rate() { + let mut pool = PoolMap::new(1000); + for i in 0..1024 { + let tx = build_tx(vec![(&Default::default(), i as u32)], 1); + let entry = TxEntry::dummy_resolve(tx, i + 1, Capacity::shannons(i + 1), 1000); + pool.add_entry(entry, Status::Pending).unwrap(); + } + + assert_eq!( + FeeRate::from_u64(42), + pool.estimate_fee_rate(1, usize::MAX, Cycle::MAX, FeeRate::from_u64(42)) + ); + + assert_eq!( + FeeRate::from_u64(1024), + pool.estimate_fee_rate(1, 1000, Cycle::MAX, FeeRate::from_u64(1)) + ); + assert_eq!( + FeeRate::from_u64(1023), + pool.estimate_fee_rate(1, 2000, Cycle::MAX, FeeRate::from_u64(1)) + ); + assert_eq!( + FeeRate::from_u64(1016), + pool.estimate_fee_rate(2, 5000, Cycle::MAX, FeeRate::from_u64(1)) + ); + + assert_eq!( + FeeRate::from_u64(1024), + pool.estimate_fee_rate(1, usize::MAX, 1, FeeRate::from_u64(1)) + ); + assert_eq!( + FeeRate::from_u64(1023), + pool.estimate_fee_rate(1, usize::MAX, 2047, FeeRate::from_u64(1)) + ); + assert_eq!( + FeeRate::from_u64(1015), + pool.estimate_fee_rate(2, usize::MAX, 5110, FeeRate::from_u64(1)) + ); + + assert_eq!( + FeeRate::from_u64(624), + pool.estimate_fee_rate(100, 5000, 5110, FeeRate::from_u64(1)) + ); + assert_eq!( + FeeRate::from_u64(1), + pool.estimate_fee_rate(1000, 5000, 5110, FeeRate::from_u64(1)) + ); +} diff --git a/tx-pool/src/component/tests/mod.rs b/tx-pool/src/component/tests/mod.rs index ac625eede3..4f344ce443 100644 --- a/tx-pool/src/component/tests/mod.rs +++ b/tx-pool/src/component/tests/mod.rs @@ -1,5 +1,6 @@ mod chunk; mod entry; +mod estimate; mod links; mod orphan; mod pending; diff --git a/tx-pool/src/pool.rs b/tx-pool/src/pool.rs index 54b2395e4c..1780f20a5f 100644 --- a/tx-pool/src/pool.rs +++ b/tx-pool/src/pool.rs @@ -12,7 +12,7 @@ use ckb_logger::{debug, error, warn}; use ckb_snapshot::Snapshot; use ckb_store::ChainStore; use ckb_types::core::tx_pool::PoolTxDetailInfo; -use ckb_types::core::CapacityError; +use ckb_types::core::{BlockNumber, CapacityError, FeeRate}; use ckb_types::packed::OutPoint; use ckb_types::{ core::{ @@ -528,6 +528,16 @@ impl TxPool { (entries, size, cycles) } + pub(crate) fn estimate_fee_rate(&self, target_to_be_committed: BlockNumber) -> FeeRate { + self.pool_map.estimate_fee_rate( + (target_to_be_committed - self.snapshot.consensus().tx_proposal_window().closest()) + as usize, + self.snapshot.consensus().max_block_bytes() as usize, + self.snapshot.consensus().max_block_cycles(), + self.config.min_fee_rate, + ) + } + pub(crate) fn check_rbf( &self, snapshot: &Snapshot, diff --git a/tx-pool/src/process.rs b/tx-pool/src/process.rs index 39627ccf9a..f898309255 100644 --- a/tx-pool/src/process.rs +++ b/tx-pool/src/process.rs @@ -20,6 +20,7 @@ use ckb_snapshot::Snapshot; use ckb_store::data_loader_wrapper::AsDataLoader; use ckb_store::ChainStore; use ckb_types::core::error::OutPointError; +use ckb_types::core::{BlockNumber, FeeRate}; use ckb_types::{ core::{cell::ResolvedTransaction, BlockView, Capacity, Cycle, HeaderView, TransactionView}, packed::{Byte32, ProposalShortId}, @@ -339,6 +340,11 @@ impl TxPoolService { } } + pub(crate) async fn estimate_fee_rate(&self, target_to_be_committed: BlockNumber) -> FeeRate { + let pool = self.tx_pool.read().await; + pool.estimate_fee_rate(target_to_be_committed) + } + pub(crate) async fn test_accept_tx(&self, tx: TransactionView) -> Result { // non contextual verify first self.non_contextual_verify(&tx, None)?; diff --git a/tx-pool/src/service.rs b/tx-pool/src/service.rs index 64f17f4aa9..db48fc48d6 100644 --- a/tx-pool/src/service.rs +++ b/tx-pool/src/service.rs @@ -19,6 +19,7 @@ use ckb_network::{NetworkController, PeerIndex}; use ckb_snapshot::Snapshot; use ckb_stop_handler::new_tokio_exit_rx; use ckb_types::core::tx_pool::{EntryCompleted, PoolTxDetailInfo, TransactionWithStatus, TxStatus}; +use ckb_types::core::{BlockNumber, FeeRate}; use ckb_types::{ core::{ tx_pool::{Reject, TxPoolEntryInfo, TxPoolIds, TxPoolInfo, TRANSACTION_SIZE_LIMIT}, @@ -91,6 +92,7 @@ pub(crate) enum Message { BlockTemplate(Request), SubmitLocalTx(Request), RemoveLocalTx(Request), + EstimateFeeRate(Request), TestAcceptTx(Request), SubmitRemoteTx(Request<(TransactionView, Cycle, PeerIndex), ()>), NotifyTxs(Notify>), @@ -228,6 +230,14 @@ impl TxPoolController { send_message!(self, SubmitLocalTx, tx) } + /// Estimate fee rate for a transaction to be committed within target block number by using a simple strategy + pub fn estimate_fee_rate( + &self, + target_to_be_committed: BlockNumber, + ) -> Result { + send_message!(self, EstimateFeeRate, target_to_be_committed) + } + /// test if a tx can be accepted by tx-pool /// Won't be broadcasted to network /// won't be insert to tx-pool @@ -706,6 +716,15 @@ async fn process(mut service: TxPoolService, message: Message) { error!("Responder sending remove_tx result failed {:?}", e); }; } + Message::EstimateFeeRate(Request { + responder, + arguments: target_to_be_committed, + }) => { + let fee_rate = service.estimate_fee_rate(target_to_be_committed).await; + if let Err(e) = responder.send(fee_rate) { + error!("Responder sending estimate_fee_rate failed {:?}", e); + }; + } Message::TestAcceptTx(Request { responder, arguments: tx, From d3fa3196d57de0aab741be4948157b414f5077ec Mon Sep 17 00:00:00 2001 From: Boyu Yang Date: Mon, 20 May 2024 18:51:32 +0800 Subject: [PATCH 2/3] feat: (experimental) optional fee estimator with different algorithms --- Cargo.lock | 13 + Cargo.toml | 1 + chain/src/chain.rs | 7 + resource/ckb.toml | 4 + rpc/README.md | 61 ++ rpc/src/module/experiment.rs | 58 +- rpc/src/tests/examples.rs | 2 + shared/Cargo.toml | 1 + shared/src/shared_builder.rs | 50 +- spec/src/consensus.rs | 9 +- tx-pool/Cargo.toml | 1 + tx-pool/src/process.rs | 15 +- tx-pool/src/service.rs | 38 +- util/app-config/src/app_config.rs | 3 + util/app-config/src/configs/fee_estimator.rs | 18 + util/app-config/src/configs/mod.rs | 2 + util/app-config/src/legacy/mod.rs | 4 + util/fee-estimator/Cargo.toml | 16 + util/fee-estimator/src/constants.rs | 24 + util/fee-estimator/src/error.rs | 14 + .../src/estimator/confirmation_fraction.rs | 600 ++++++++++++++++++ util/fee-estimator/src/estimator/mod.rs | 92 +++ .../src/estimator/weight_units_flow.rs | 463 ++++++++++++++ util/fee-estimator/src/lib.rs | 8 + util/jsonrpc-types/src/fee_estimator.rs | 43 ++ util/jsonrpc-types/src/lib.rs | 2 + util/launcher/Cargo.toml | 1 - util/launcher/src/lib.rs | 1 + util/types/src/core/fee_estimator.rs | 14 + util/types/src/core/mod.rs | 2 + 30 files changed, 1555 insertions(+), 12 deletions(-) create mode 100644 util/app-config/src/configs/fee_estimator.rs create mode 100644 util/fee-estimator/Cargo.toml create mode 100644 util/fee-estimator/src/constants.rs create mode 100644 util/fee-estimator/src/error.rs create mode 100644 util/fee-estimator/src/estimator/confirmation_fraction.rs create mode 100644 util/fee-estimator/src/estimator/mod.rs create mode 100644 util/fee-estimator/src/estimator/weight_units_flow.rs create mode 100644 util/fee-estimator/src/lib.rs create mode 100644 util/jsonrpc-types/src/fee_estimator.rs create mode 100644 util/types/src/core/fee_estimator.rs diff --git a/Cargo.lock b/Cargo.lock index 96aa488a20..775ac7bba0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -864,6 +864,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ckb-fee-estimator" +version = "0.117.0-pre" +dependencies = [ + "ckb-chain-spec", + "ckb-logger", + "ckb-types", + "ckb-util", + "thiserror", +] + [[package]] name = "ckb-fixed-hash" version = "0.117.0-pre" @@ -1505,6 +1516,7 @@ dependencies = [ "ckb-db", "ckb-db-schema", "ckb-error", + "ckb-fee-estimator", "ckb-logger", "ckb-migrate", "ckb-notify", @@ -1673,6 +1685,7 @@ dependencies = [ "ckb-dao", "ckb-db", "ckb-error", + "ckb-fee-estimator", "ckb-hash", "ckb-jsonrpc-types", "ckb-logger", diff --git a/Cargo.toml b/Cargo.toml index 3361e53e1d..5c35229b42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,7 @@ members = [ "util/instrument", "rpc", "util/light-client-protocol-server", + "util/fee-estimator", "util/launcher", "devtools/doc/rpc-gen", "ckb-bin" diff --git a/chain/src/chain.rs b/chain/src/chain.rs index c1915ed48e..1851ae50b0 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -460,6 +460,7 @@ impl ChainService { // is_better_than let new_best_block = cannon_total_difficulty > current_total_difficulty; + let in_ibd = self.shared.is_initial_block_download(); if new_best_block { debug!( "Newly found best block : {} => {:#x}, difficulty diff = {:#x}", @@ -517,6 +518,9 @@ impl ChainService { ) { error!("Notify update_tx_pool_for_reorg error {}", e); } + if let Err(e) = tx_pool_controller.update_ibd_state(in_ibd) { + error!("Notify update_ibd_state error {}", e); + } } let block_ref: &BlockView = █ @@ -546,6 +550,9 @@ impl ChainService { if let Err(e) = tx_pool_controller.notify_new_uncle(block_ref.as_uncle()) { error!("Notify new_uncle error {}", e); } + if let Err(e) = tx_pool_controller.update_ibd_state(in_ibd) { + error!("Notify update_ibd_state error {}", e); + } } } diff --git a/resource/ckb.toml b/resource/ckb.toml index 1ccd93b623..22438aefc9 100644 --- a/resource/ckb.toml +++ b/resource/ckb.toml @@ -218,3 +218,7 @@ block_uncles_cache_size = 30 # db_port = 5432 # db_user = "postgres" # db_password = "123456" +# +# # [fee_estimator] +# # Specifies the fee estimates algorithm. Current algorithms: ConfirmationFraction, WeightUnitsFlow. +# # algorithm = "WeightUnitsFlow" diff --git a/rpc/README.md b/rpc/README.md index 016b32b8b8..d6318f070c 100644 --- a/rpc/README.md +++ b/rpc/README.md @@ -70,6 +70,7 @@ The crate `ckb-rpc`'s minimum supported rustc version is 1.71.1. * [Method `dry_run_transaction`](#experiment-dry_run_transaction) * [Method `calculate_dao_maximum_withdraw`](#experiment-calculate_dao_maximum_withdraw) + * [Method `get_fee_estimates`](#experiment-get_fee_estimates) * [Module Indexer](#module-indexer) [👉 OpenRPC spec](http://playground.open-rpc.org/?uiSchema[appBar][ui:title]=CKB-Indexer&uiSchema[appBar][ui:splitView]=false&uiSchema[appBar][ui:examplesDropdown]=false&uiSchema[appBar][ui:logoUrl]=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/ckb-logo.jpg&schemaUrl=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/json/indexer_rpc_doc.json) * [Method `get_indexer_tip`](#indexer-get_indexer_tip) @@ -208,6 +209,7 @@ The crate `ckb-rpc`'s minimum supported rustc version is 1.71.1. * [Type `Ratio`](#type-ratio) * [Type `RationalU256`](#type-rationalu256) * [Type `RawTxPool`](#type-rawtxpool) + * [Type `RecommendedFeeRates`](#type-recommendedfeerates) * [Type `RemoteNode`](#type-remotenode) * [Type `RemoteNodeProtocol`](#type-remotenodeprotocol) * [Type `ResponseFormat`](#type-responseformat_for_blockview) @@ -2162,6 +2164,50 @@ Response } ``` + +#### Method `get_fee_estimates` +* `get_fee_estimates()` + +* result: [`RecommendedFeeRates`](#type-recommendedfeerates) + +Get fee estimates. + +###### Returns + +Recommended fee rates in 4 levels of priorities: +- No priority (about 2 hours). +- Low priority (about 1 hour). +- Medium priority (about 10 minutes). +- High priority (as soon as possible). + +###### Examples + +Request + +```json +{ + "id": 42, + "jsonrpc": "2.0", + "method": "get_fee_estimates", + "params": [] +} +``` + +Response + +```json +{ + "id": 42, + "jsonrpc": "2.0", + "result": { + "no_priority": 1000, + "low_priority": 1000, + "medium_priority": 1000, + "high_priority": 1000 + } +} +``` + ### Module `Indexer` - [👉 OpenRPC spec](http://playground.open-rpc.org/?uiSchema[appBar][ui:title]=CKB-Indexer&uiSchema[appBar][ui:splitView]=false&uiSchema[appBar][ui:examplesDropdown]=false&uiSchema[appBar][ui:logoUrl]=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/ckb-logo.jpg&schemaUrl=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/json/indexer_rpc_doc.json) @@ -6606,6 +6652,21 @@ All transactions in tx-pool. [`TxPoolIds`]: struct.TxPoolIds.html [`TxPoolEntries`]: struct.TxPoolEntries.html +### Type `RecommendedFeeRates` +Recommended fee rates. + +#### Fields + +`RecommendedFeeRates` is a JSON object with the following fields. + +* `high_priority`: `integer` - High-priority fee rate. + +* `low_priority`: `integer` - Low-priority fee rate. + +* `medium_priority`: `integer` - Medium-priority fee rate. + +* `no_priority`: `integer` - Default fee rate. + ### Type `RemoteNode` Information of a remote node. diff --git a/rpc/src/module/experiment.rs b/rpc/src/module/experiment.rs index 90e910dc1a..6a0f8fec06 100644 --- a/rpc/src/module/experiment.rs +++ b/rpc/src/module/experiment.rs @@ -3,7 +3,8 @@ use crate::module::chain::CyclesEstimator; use async_trait::async_trait; use ckb_dao::DaoCalculator; use ckb_jsonrpc_types::{ - Capacity, DaoWithdrawingCalculationKind, EstimateCycles, OutPoint, Transaction, + Capacity, DaoWithdrawingCalculationKind, EstimateCycles, OutPoint, RecommendedFeeRates, + Transaction, }; use ckb_shared::{shared::Shared, Snapshot}; use ckb_store::ChainStore; @@ -162,6 +163,46 @@ pub trait ExperimentRpc { out_point: OutPoint, kind: DaoWithdrawingCalculationKind, ) -> Result; + + /// Get fee estimates. + /// + /// ## Returns + /// + /// Recommended fee rates in 4 levels of priorities: + /// - No priority (about 2 hours). + /// - Low priority (about 1 hour). + /// - Medium priority (about 10 minutes). + /// - High priority (as soon as possible). + /// + /// ## Examples + /// + /// Request + /// + /// ```json + /// { + /// "id": 42, + /// "jsonrpc": "2.0", + /// "method": "get_fee_estimates", + /// "params": [] + /// } + /// ``` + /// + /// Response + /// + /// ```json + /// { + /// "id": 42, + /// "jsonrpc": "2.0", + /// "result": { + /// "no_priority": 1000, + /// "low_priority": 1000, + /// "medium_priority": 1000, + /// "high_priority": 1000 + /// } + /// } + /// ``` + #[rpc(name = "get_fee_estimates")] + fn get_fee_estimates(&self) -> Result; } #[derive(Clone)] @@ -241,4 +282,19 @@ impl ExperimentRpc for ExperimentRpcImpl { } } } + + fn get_fee_estimates(&self) -> Result { + let tx_pool = self.shared.tx_pool_controller(); + let fee_rates_res = tx_pool + .get_fee_estimates() + .map_err(|err| RPCError::custom(RPCError::CKBInternalError, err.to_string()))?; + if let Ok(Some(fee_rates)) = fee_rates_res { + Ok(fee_rates) + } else { + // TODO merge code from PR#4465 + let msg = "fallback fee estimates algorithm is unfinished"; + let err = RPCError::custom(RPCError::CKBInternalError, msg.to_owned()); + Err(err) + } + } } diff --git a/rpc/src/tests/examples.rs b/rpc/src/tests/examples.rs index 03afbb9d3c..d9dfa030fc 100644 --- a/rpc/src/tests/examples.rs +++ b/rpc/src/tests/examples.rs @@ -15,6 +15,7 @@ use std::hash; use std::io::{self, BufRead}; use std::path::PathBuf; +use ckb_jsonrpc_types::RecommendedFeeRates; use ckb_types::{ core::{capacity_bytes, Capacity, TransactionBuilder, TransactionView}, h256, @@ -389,6 +390,7 @@ fn mock_rpc_response(example: &RpcTestExample, response: &mut RpcTestResponse) { "get_pool_tx_detail_info" => { response.result["timestamp"] = example.response.result["timestamp"].clone() } + "get_fee_estimates" => replace_rpc_response::(example, response), _ => {} } } diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 794f800946..96ae061663 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -29,6 +29,7 @@ ckb-systemtime = { path = "../util/systemtime", version = "= 0.117.0-pre" } ckb-channel = { path = "../util/channel", version = "= 0.117.0-pre" } ckb-app-config = {path = "../util/app-config", version = "= 0.117.0-pre"} ckb-migrate = { path = "../util/migrate", version = "= 0.117.0-pre" } +ckb-fee-estimator = {path = "../util/fee-estimator", version = "= 0.117.0-pre"} once_cell = "1.8.0" tempfile.workspace = true diff --git a/shared/src/shared_builder.rs b/shared/src/shared_builder.rs index 985add3ba0..4a47f858da 100644 --- a/shared/src/shared_builder.rs +++ b/shared/src/shared_builder.rs @@ -13,12 +13,14 @@ use ckb_proposal_table::ProposalView; use ckb_snapshot::{Snapshot, SnapshotMgr}; use ckb_app_config::{ - BlockAssemblerConfig, DBConfig, ExitCode, NotifyConfig, StoreConfig, TxPoolConfig, + BlockAssemblerConfig, DBConfig, ExitCode, FeeEstimatorAlgo, FeeEstimatorConfig, NotifyConfig, + StoreConfig, TxPoolConfig, }; use ckb_async_runtime::{new_background_runtime, Handle}; use ckb_db::RocksDB; use ckb_db_schema::COLUMNS; use ckb_error::{Error, InternalErrorKind}; +use ckb_fee_estimator::FeeEstimator; use ckb_logger::{error, info}; use ckb_migrate::migrate::Migrate; use ckb_notify::{NotifyController, NotifyService}; @@ -45,6 +47,7 @@ pub struct SharedBuilder { block_assembler_config: Option, notify_config: Option, async_handle: Handle, + fee_estimator_config: Option, } /// Open or create a rocksdb @@ -148,6 +151,7 @@ impl SharedBuilder { store_config: None, block_assembler_config: None, async_handle, + fee_estimator_config: None, }) } @@ -193,6 +197,7 @@ impl SharedBuilder { store_config: None, block_assembler_config: None, async_handle: runtime.get_or_init(new_background_runtime).clone(), + fee_estimator_config: None, }) } } @@ -228,6 +233,12 @@ impl SharedBuilder { self } + /// Sets the configuration for the fee estimator. + pub fn fee_estimator_config(mut self, config: FeeEstimatorConfig) -> Self { + self.fee_estimator_config = Some(config); + self + } + /// specifies the async_handle for the shared pub fn async_handle(mut self, async_handle: Handle) -> Self { self.async_handle = async_handle; @@ -328,6 +339,7 @@ impl SharedBuilder { block_assembler_config, notify_config, async_handle, + fee_estimator_config, } = self; let tx_pool_config = tx_pool_config.unwrap_or_default(); @@ -354,6 +366,17 @@ impl SharedBuilder { let (sender, receiver) = ckb_channel::unbounded(); + let fee_estimator_algo = fee_estimator_config + .map(|config| config.algorithm) + .unwrap_or(None); + let fee_estimator = match fee_estimator_algo { + Some(FeeEstimatorAlgo::WeightUnitsFlow) => FeeEstimator::new_weight_units_flow(), + Some(FeeEstimatorAlgo::ConfirmationFraction) => { + FeeEstimator::new_confirmation_fraction() + } + None => FeeEstimator::new_dummy(), + }; + let (mut tx_pool_builder, tx_pool_controller) = TxPoolServiceBuilder::new( tx_pool_config, Arc::clone(&snapshot), @@ -361,9 +384,14 @@ impl SharedBuilder { Arc::clone(&txs_verify_cache), &async_handle, sender, + fee_estimator.clone(), ); - register_tx_pool_callback(&mut tx_pool_builder, notify_controller.clone()); + register_tx_pool_callback( + &mut tx_pool_builder, + notify_controller.clone(), + fee_estimator, + ); let ibd_finished = Arc::new(AtomicBool::new(false)); let shared = Shared::new( @@ -387,7 +415,11 @@ impl SharedBuilder { } } -fn register_tx_pool_callback(tx_pool_builder: &mut TxPoolServiceBuilder, notify: NotifyController) { +fn register_tx_pool_callback( + tx_pool_builder: &mut TxPoolServiceBuilder, + notify: NotifyController, + fee_estimator: FeeEstimator, +) { let notify_pending = notify.clone(); let tx_relay_sender = tx_pool_builder.tx_relay_sender(); @@ -398,10 +430,15 @@ fn register_tx_pool_callback(tx_pool_builder: &mut TxPoolServiceBuilder, notify: fee: entry.fee, timestamp: entry.timestamp, }; + + let fee_estimator_clone = fee_estimator.clone(); tx_pool_builder.register_pending(Box::new(move |entry: &TxEntry| { // notify let notify_tx_entry = create_notify_entry(entry); notify_pending.notify_new_transaction(notify_tx_entry); + let tx_hash = entry.transaction().hash(); + let entry_info = entry.to_info(); + fee_estimator_clone.accept_tx(tx_hash, entry_info); })); let notify_proposed = notify.clone(); @@ -428,7 +465,9 @@ fn register_tx_pool_callback(tx_pool_builder: &mut TxPoolServiceBuilder, notify: } if reject.is_allowed_relay() { - if let Err(e) = tx_relay_sender.send(TxVerificationResult::Reject { tx_hash }) { + if let Err(e) = tx_relay_sender.send(TxVerificationResult::Reject { + tx_hash: tx_hash.clone(), + }) { error!("tx-pool tx_relay_sender internal error {}", e); } } @@ -436,6 +475,9 @@ fn register_tx_pool_callback(tx_pool_builder: &mut TxPoolServiceBuilder, notify: // notify let notify_tx_entry = create_notify_entry(entry); notify_reject.notify_reject_transaction(notify_tx_entry, reject); + + // fee estimator + fee_estimator.reject_tx(&tx_hash); }, )); } diff --git a/spec/src/consensus.rs b/spec/src/consensus.rs index 2d8a16c481..3d312a9a8b 100644 --- a/spec/src/consensus.rs +++ b/spec/src/consensus.rs @@ -45,7 +45,8 @@ pub(crate) const DEFAULT_SECONDARY_EPOCH_REWARD: Capacity = Capacity::shannons(6 // 4.2 billion per year pub(crate) const INITIAL_PRIMARY_EPOCH_REWARD: Capacity = Capacity::shannons(1_917_808_21917808); const MAX_UNCLE_NUM: usize = 2; -pub(crate) const TX_PROPOSAL_WINDOW: ProposalWindow = ProposalWindow(2, 10); +/// Default transaction proposal window. +pub const TX_PROPOSAL_WINDOW: ProposalWindow = ProposalWindow(2, 10); // Cellbase outputs are "locked" and require 4 epoch confirmations (approximately 16 hours) before // they mature sufficiently to be spendable, // This is to reduce the risk of later txs being reversed if a chain reorganization occurs. @@ -138,17 +139,17 @@ pub const TYPE_ID_CODE_HASH: H256 = h256!("0x545950455f4944"); /// impl ProposalWindow { /// The w_close parameter - pub fn closest(&self) -> BlockNumber { + pub const fn closest(&self) -> BlockNumber { self.0 } /// The w_far parameter - pub fn farthest(&self) -> BlockNumber { + pub const fn farthest(&self) -> BlockNumber { self.1 } /// The proposal window length - pub fn length(&self) -> BlockNumber { + pub const fn length(&self) -> BlockNumber { self.1 - self.0 + 1 } } diff --git a/tx-pool/Cargo.toml b/tx-pool/Cargo.toml index 7ced816523..308fe4a770 100644 --- a/tx-pool/Cargo.toml +++ b/tx-pool/Cargo.toml @@ -42,6 +42,7 @@ multi_index_map = "0.6.0" slab = "0.4" rustc-hash = "1.1" tokio-util = "0.7.8" +ckb-fee-estimator = { path = "../util/fee-estimator", version = "= 0.117.0-pre" } [dev-dependencies] tempfile.workspace = true diff --git a/tx-pool/src/process.rs b/tx-pool/src/process.rs index 39627ccf9a..3436409fa7 100644 --- a/tx-pool/src/process.rs +++ b/tx-pool/src/process.rs @@ -12,7 +12,7 @@ use crate::util::{ }; use ckb_chain_spec::consensus::MAX_BLOCK_PROPOSALS_LIMIT; use ckb_error::{AnyError, InternalErrorKind}; -use ckb_jsonrpc_types::BlockTemplate; +use ckb_jsonrpc_types::{BlockTemplate, RecommendedFeeRates}; use ckb_logger::Level::Trace; use ckb_logger::{debug, error, info, log_enabled_target, trace_target}; use ckb_network::PeerIndex; @@ -924,6 +924,7 @@ impl TxPoolService { } for blk in attached_blocks { + self.fee_estimator.commit_block(&blk); attached.extend(blk.transactions().into_iter().skip(1)); } let retain: Vec = detached.difference(&attached).cloned().collect(); @@ -1089,6 +1090,18 @@ impl TxPoolService { } } + pub(crate) async fn update_ibd_state(&self, in_ibd: bool) { + self.fee_estimator.update_ibd_state(in_ibd); + } + + pub(crate) async fn get_fee_estimates(&self) -> Result, AnyError> { + let all_entry_info = self.tx_pool.read().await.get_all_entry_info(); + self.fee_estimator + .get_fee_estimates(all_entry_info) + .map(|inner| inner.map(Into::into)) + .map_err(Into::into) + } + // # Notice // // This method assumes that the inputs transactions are sorted. diff --git a/tx-pool/src/service.rs b/tx-pool/src/service.rs index 64f17f4aa9..28510f55c7 100644 --- a/tx-pool/src/service.rs +++ b/tx-pool/src/service.rs @@ -13,7 +13,8 @@ use ckb_async_runtime::Handle; use ckb_chain_spec::consensus::Consensus; use ckb_channel::oneshot; use ckb_error::AnyError; -use ckb_jsonrpc_types::BlockTemplate; +use ckb_fee_estimator::FeeEstimator; +use ckb_jsonrpc_types::{BlockTemplate, RecommendedFeeRates}; use ckb_logger::{error, info}; use ckb_network::{NetworkController, PeerIndex}; use ckb_snapshot::Snapshot; @@ -87,6 +88,8 @@ pub(crate) type ChainReorgArgs = ( Arc, ); +pub(crate) type FeeEstimatesResult = Result, AnyError>; + pub(crate) enum Message { BlockTemplate(Request), SubmitLocalTx(Request), @@ -107,6 +110,9 @@ pub(crate) enum Message { SavePool(Request<(), ()>), GetPoolTxDetails(Request), + UpdateIBDState(Request), + GetFeeEstimates(Request<(), FeeEstimatesResult>), + // test #[cfg(feature = "internal")] PlugEntry(Request<(Vec, PlugTarget), ()>), @@ -323,6 +329,16 @@ impl TxPoolController { send_message!(self, SavePool, ()) } + /// Updates IBD state. + pub fn update_ibd_state(&self, in_ibd: bool) -> Result<(), AnyError> { + send_message!(self, UpdateIBDState, in_ibd) + } + + /// Gets fee estimates. + pub fn get_fee_estimates(&self) -> Result { + send_message!(self, GetFeeEstimates, ()) + } + /// Sends suspend chunk process cmd pub fn suspend_chunk_process(&self) -> Result<(), AnyError> { self.chunk_tx @@ -394,6 +410,7 @@ pub struct TxPoolServiceBuilder { mpsc::Sender, mpsc::Receiver, ), + pub(crate) fee_estimator: FeeEstimator, } impl TxPoolServiceBuilder { @@ -405,6 +422,7 @@ impl TxPoolServiceBuilder { txs_verify_cache: Arc>, handle: &Handle, tx_relay_sender: ckb_channel::Sender, + fee_estimator: FeeEstimator, ) -> (TxPoolServiceBuilder, TxPoolController) { let (sender, receiver) = mpsc::channel(DEFAULT_CHANNEL_SIZE); let block_assembler_channel = mpsc::channel(BLOCK_ASSEMBLER_CHANNEL_SIZE); @@ -440,6 +458,7 @@ impl TxPoolServiceBuilder { chunk, started, block_assembler_channel, + fee_estimator, }; (builder, controller) @@ -495,6 +514,7 @@ impl TxPoolServiceBuilder { consensus, delay: Arc::new(RwLock::new(LinkedHashMap::new())), after_delay: Arc::new(AtomicBool::new(after_delay_window)), + fee_estimator: self.fee_estimator, }; let signal_receiver = self.signal_receiver.clone(); @@ -650,6 +670,7 @@ pub(crate) struct TxPoolService { pub(crate) block_assembler_sender: mpsc::Sender, pub(crate) delay: Arc>>, pub(crate) after_delay: Arc, + pub(crate) fee_estimator: FeeEstimator, } /// tx verification result @@ -903,6 +924,21 @@ async fn process(mut service: TxPoolService, message: Message) { error!("Responder sending save_pool failed {:?}", e) }; } + Message::UpdateIBDState(Request { + responder, + arguments: in_ibd, + }) => { + service.update_ibd_state(in_ibd).await; + if let Err(e) = responder.send(()) { + error!("Responder sending update_ibd_state failed {:?}", e) + }; + } + Message::GetFeeEstimates(Request { responder, .. }) => { + let fee_estimates_result = service.get_fee_estimates().await; + if let Err(e) = responder.send(fee_estimates_result) { + error!("Responder sending fee_estimates_result failed {:?}", e) + }; + } #[cfg(feature = "internal")] Message::PlugEntry(Request { responder, diff --git a/util/app-config/src/app_config.rs b/util/app-config/src/app_config.rs index 963bd7ef18..d10819c876 100644 --- a/util/app-config/src/app_config.rs +++ b/util/app-config/src/app_config.rs @@ -92,6 +92,9 @@ pub struct CKBAppConfig { /// Indexer config options. #[serde(default)] pub indexer: IndexerConfig, + /// Fee estimator config options. + #[serde(default)] + pub fee_estimator: FeeEstimatorConfig, } /// The miner config file for `ckb miner`. Usually it is the `ckb-miner.toml` in the CKB root diff --git a/util/app-config/src/configs/fee_estimator.rs b/util/app-config/src/configs/fee_estimator.rs new file mode 100644 index 0000000000..5ca05d46d8 --- /dev/null +++ b/util/app-config/src/configs/fee_estimator.rs @@ -0,0 +1,18 @@ +use serde::{Deserialize, Serialize}; + +/// Fee estimator config options. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Config { + /// The algorithm for fee estimator. + pub algorithm: Option, +} + +/// Specifies the fee estimates algorithm. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, Eq)] +pub enum Algorithm { + /// Confirmation Fraction Fee Estimator + ConfirmationFraction, + /// Weight-Units Flow Fee Estimator + WeightUnitsFlow, +} diff --git a/util/app-config/src/configs/mod.rs b/util/app-config/src/configs/mod.rs index 95b0c79508..5a20d1ea16 100644 --- a/util/app-config/src/configs/mod.rs +++ b/util/app-config/src/configs/mod.rs @@ -1,4 +1,5 @@ mod db; +mod fee_estimator; mod indexer; mod memory_tracker; mod miner; @@ -11,6 +12,7 @@ mod store; mod tx_pool; pub use db::Config as DBConfig; +pub use fee_estimator::{Algorithm as FeeEstimatorAlgo, Config as FeeEstimatorConfig}; pub use indexer::{IndexerConfig, IndexerSyncConfig}; pub use memory_tracker::Config as MemoryTrackerConfig; pub use miner::{ diff --git a/util/app-config/src/legacy/mod.rs b/util/app-config/src/legacy/mod.rs index 79d8c29b27..77084810d3 100644 --- a/util/app-config/src/legacy/mod.rs +++ b/util/app-config/src/legacy/mod.rs @@ -59,6 +59,8 @@ pub(crate) struct CKBAppConfig { notify: crate::NotifyConfig, #[serde(default)] indexer_v2: crate::IndexerConfig, + #[serde(default)] + fee_estimator: crate::FeeEstimatorConfig, } #[derive(Clone, Debug, Deserialize)] @@ -106,6 +108,7 @@ impl From for crate::CKBAppConfig { alert_signature, notify, indexer_v2, + fee_estimator, } = input; #[cfg(not(feature = "with_sentry"))] let _ = sentry; @@ -131,6 +134,7 @@ impl From for crate::CKBAppConfig { alert_signature, notify, indexer: indexer_v2, + fee_estimator, } } } diff --git a/util/fee-estimator/Cargo.toml b/util/fee-estimator/Cargo.toml new file mode 100644 index 0000000000..4cdc57c484 --- /dev/null +++ b/util/fee-estimator/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "ckb-fee-estimator" +version = "0.117.0-pre" +license = "MIT" +authors = ["Nervos Core Dev "] +edition = "2021" +description = "The ckb fee estimator" +homepage = "https://github.com/nervosnetwork/ckb" +repository = "https://github.com/nervosnetwork/ckb" + +[dependencies] +ckb-logger = { path = "../logger", version = "= 0.117.0-pre" } +ckb-types = { path = "../types", version = "= 0.117.0-pre" } +ckb-util = { path = "../../util", version = "= 0.117.0-pre" } +ckb-chain-spec = { path = "../../spec", version = "= 0.117.0-pre" } +thiserror = "1.0" diff --git a/util/fee-estimator/src/constants.rs b/util/fee-estimator/src/constants.rs new file mode 100644 index 0000000000..fcfc934f81 --- /dev/null +++ b/util/fee-estimator/src/constants.rs @@ -0,0 +1,24 @@ +//! The constants for the fee estimator. + +use ckb_chain_spec::consensus::{MAX_BLOCK_INTERVAL, MIN_BLOCK_INTERVAL, TX_PROPOSAL_WINDOW}; +use ckb_types::core::{BlockNumber, FeeRate}; + +/// Average block interval. +pub(crate) const AVG_BLOCK_INTERVAL: u64 = (MAX_BLOCK_INTERVAL + MIN_BLOCK_INTERVAL) / 2; + +/// Max target blocks, about 2 hours. +pub(crate) const MAX_TARGET: BlockNumber = (60 * 2 * 60) / AVG_BLOCK_INTERVAL; +/// Min target blocks, in next block. +pub(crate) const MIN_TARGET: BlockNumber = TX_PROPOSAL_WINDOW.closest() + 1; + +/// Lowest fee rate. +pub(crate) const LOWEST_FEE_RATE: FeeRate = FeeRate::from_u64(1000); + +/// Target blocks for default priority (lowest priority, about 2 hours). +pub(crate) const DEFAULT_TARGET: BlockNumber = MAX_TARGET; +/// Target blocks for low priority (about 1 hour). +pub(crate) const LOW_TARGET: BlockNumber = DEFAULT_TARGET / 2; +/// Target blocks for medium priority (about 10 minutes). +pub(crate) const MEDIUM_TARGET: BlockNumber = LOW_TARGET / 6; +/// Target blocks for high priority. +pub(crate) const HIGH_TARGET: BlockNumber = MIN_TARGET; diff --git a/util/fee-estimator/src/error.rs b/util/fee-estimator/src/error.rs new file mode 100644 index 0000000000..66702a59ae --- /dev/null +++ b/util/fee-estimator/src/error.rs @@ -0,0 +1,14 @@ +//! The error type for the fee estimator. + +use thiserror::Error; + +/// A list specifying general categories of fee estimator errors. +#[derive(Error, Debug)] +pub enum Error { + /// Not ready for do estimate. + #[error("not ready")] + NotReady, + /// Lack of empirical data. + #[error("lack of empirical data")] + LackData, +} diff --git a/util/fee-estimator/src/estimator/confirmation_fraction.rs b/util/fee-estimator/src/estimator/confirmation_fraction.rs new file mode 100644 index 0000000000..10edbb131f --- /dev/null +++ b/util/fee-estimator/src/estimator/confirmation_fraction.rs @@ -0,0 +1,600 @@ +//! Confirmation Fraction Fee Estimator +//! +//! Copy from https://github.com/nervosnetwork/ckb/tree/v0.39.1/util/fee-estimator +//! Ref: https://github.com/nervosnetwork/ckb/pull/1659 + +use std::{ + cmp, + collections::{BTreeMap, HashMap}, +}; + +use ckb_types::{ + core::{ + tx_pool::{get_transaction_weight, TxEntryInfo}, + BlockNumber, BlockView, FeeRate, RecommendedFeeRates, + }, + packed::Byte32, +}; + +use crate::{constants, Error}; + +const DEFAULT_MIN_CONFIRM_RATE: f64 = 0.9; + +#[derive(Default, Debug, Clone)] +struct BucketStat { + total_fee_rate: FeeRate, + txs_count: f64, + old_unconfirmed_txs: usize, +} + +/// TxConfirmStat is a struct to help to estimate txs fee rate, +/// This struct record txs fee_rate and blocks that txs to be committed. +/// +/// We start from track unconfirmed txs, +/// When tx added to txpool, we increase the count of unconfirmed tx, we do opposite tx removed. +/// When a tx get committed, put it into bucket by tx fee_rate and confirmed blocks, +/// then decrease the count of unconfirmed txs. +/// +/// So we get a group of samples which includes txs count, average fee rate and confirmed blocks, etc. +/// For estimate, we loop through each bucket, calculate the confirmed txs rate, until meet the required_confirm_rate. +#[derive(Clone)] +struct TxConfirmStat { + min_fee_rate: FeeRate, + /// per bucket stat + bucket_stats: Vec, + /// bucket upper bound fee_rate => bucket index + fee_rate_to_bucket: BTreeMap, + /// confirm_blocks => bucket index => confirmed txs count + confirm_blocks_to_confirmed_txs: Vec>, + /// confirm_blocks => bucket index => failed txs count + confirm_blocks_to_failed_txs: Vec>, + /// Track recent N blocks unconfirmed txs + /// tracked block index => bucket index => TxTracker + block_unconfirmed_txs: Vec>, + decay_factor: f64, +} + +#[derive(Clone)] +struct TxRecord { + height: u64, + bucket_index: usize, + fee_rate: FeeRate, +} + +/// Estimator track new block and tx_pool to collect data +/// we track every new tx enter txpool and record the tip height and fee_rate, +/// when tx is packed into a new block or dropped by txpool, +/// we get a sample about how long a tx with X fee_rate can get confirmed or get dropped. +/// +/// In inner, we group samples by predefined fee_rate buckets. +/// To estimator fee_rate for a confirm target(how many blocks that a tx can get committed), +/// we travel through fee_rate buckets, try to find a fee_rate X to let a tx get committed +/// with high probilities within confirm target blocks. +/// +#[derive(Clone)] +pub struct Algorithm { + best_height: u64, + start_height: u64, + /// a data struct to track tx confirm status + tx_confirm_stat: TxConfirmStat, + tracked_txs: HashMap, + + current_tip: BlockNumber, + is_ready: bool, +} + +impl BucketStat { + // add a new fee rate to this bucket + fn new_fee_rate_sample(&mut self, fee_rate: FeeRate) { + self.txs_count += 1f64; + let total_fee_rate = self + .total_fee_rate + .as_u64() + .saturating_add(fee_rate.as_u64()); + self.total_fee_rate = FeeRate::from_u64(total_fee_rate); + } + + // get average fee rate from a bucket + fn avg_fee_rate(&self) -> Option { + if self.txs_count > 0f64 { + Some(FeeRate::from_u64( + ((self.total_fee_rate.as_u64() as f64) / self.txs_count) as u64, + )) + } else { + None + } + } +} + +impl Default for TxConfirmStat { + fn default() -> Self { + let max_confirm_blocks = constants::MAX_TARGET as usize; + let min_bucket_feerate = f64::from(constants::LOWEST_FEE_RATE.as_u64() as u32); + let max_bucket_feerate = min_bucket_feerate * 1000.0; + let fee_spacing = min_bucket_feerate; + // half life each 100 blocks, math.exp(math.log(0.5) / 100) + let decay_factor: f64 = (0.5f64.ln() / 100.0).exp(); + + let mut buckets = Vec::new(); + let mut bucket_fee_boundary = min_bucket_feerate; + // initialize fee_rate buckets + while bucket_fee_boundary <= max_bucket_feerate { + buckets.push(FeeRate::from_u64(bucket_fee_boundary as u64)); + bucket_fee_boundary *= fee_spacing; + } + Self::new(buckets, max_confirm_blocks, decay_factor) + } +} + +impl TxConfirmStat { + fn new(buckets: Vec, max_confirm_blocks: usize, decay_factor: f64) -> Self { + // max_confirm_blocsk: The number of blocks that the esitmator will trace the statistics. + let min_fee_rate = buckets[0]; + let bucket_stats = vec![BucketStat::default(); buckets.len()]; + let confirm_blocks_to_confirmed_txs = vec![vec![0f64; buckets.len()]; max_confirm_blocks]; + let confirm_blocks_to_failed_txs = vec![vec![0f64; buckets.len()]; max_confirm_blocks]; + let block_unconfirmed_txs = vec![vec![0; buckets.len()]; max_confirm_blocks]; + let fee_rate_to_bucket = buckets + .into_iter() + .enumerate() + .map(|(i, fee_rate)| (fee_rate, i)) + .collect(); + TxConfirmStat { + min_fee_rate, + bucket_stats, + fee_rate_to_bucket, + block_unconfirmed_txs, + confirm_blocks_to_confirmed_txs, + confirm_blocks_to_failed_txs, + decay_factor, + } + } + + /// Return upper bound fee_rate bucket + /// assume we have three buckets with fee_rate [1.0, 2.0, 3.0], we return index 1 for fee_rate 1.5 + fn bucket_index_by_fee_rate(&self, fee_rate: FeeRate) -> Option { + self.fee_rate_to_bucket + .range(fee_rate..) + .next() + .map(|(_fee_rate, index)| *index) + } + + fn max_confirms(&self) -> usize { + self.confirm_blocks_to_confirmed_txs.len() + } + + // add confirmed sample + fn add_confirmed_tx(&mut self, blocks_to_confirm: usize, fee_rate: FeeRate) { + if blocks_to_confirm < 1 { + return; + } + let bucket_index = match self.bucket_index_by_fee_rate(fee_rate) { + Some(index) => index, + None => return, + }; + // increase txs_count in buckets + for i in (blocks_to_confirm - 1)..self.max_confirms() { + self.confirm_blocks_to_confirmed_txs[i][bucket_index] += 1f64; + } + let stat = &mut self.bucket_stats[bucket_index]; + stat.new_fee_rate_sample(fee_rate); + } + + // track an unconfirmed tx + // entry_height - tip number when tx enter txpool + fn add_unconfirmed_tx(&mut self, entry_height: u64, fee_rate: FeeRate) -> Option { + let bucket_index = match self.bucket_index_by_fee_rate(fee_rate) { + Some(index) => index, + None => return None, + }; + let block_index = (entry_height % (self.block_unconfirmed_txs.len() as u64)) as usize; + self.block_unconfirmed_txs[block_index][bucket_index] += 1; + Some(bucket_index) + } + + fn remove_unconfirmed_tx( + &mut self, + entry_height: u64, + tip_height: u64, + bucket_index: usize, + count_failure: bool, + ) { + let tx_age = tip_height.saturating_sub(entry_height) as usize; + if tx_age < 1 { + return; + } + if tx_age >= self.block_unconfirmed_txs.len() { + self.bucket_stats[bucket_index].old_unconfirmed_txs -= 1; + } else { + let block_index = (entry_height % self.block_unconfirmed_txs.len() as u64) as usize; + self.block_unconfirmed_txs[block_index][bucket_index] -= 1; + } + if count_failure { + self.confirm_blocks_to_failed_txs[tx_age - 1][bucket_index] += 1f64; + } + } + + fn move_track_window(&mut self, height: u64) { + let block_index = (height % (self.block_unconfirmed_txs.len() as u64)) as usize; + for bucket_index in 0..self.bucket_stats.len() { + // mark unconfirmed txs as old_unconfirmed_txs + self.bucket_stats[bucket_index].old_unconfirmed_txs += + self.block_unconfirmed_txs[block_index][bucket_index]; + self.block_unconfirmed_txs[block_index][bucket_index] = 0; + } + } + + /// apply decay factor on stats, smoothly reduce the effects of old samples. + fn decay(&mut self) { + let decay_factor = self.decay_factor; + for (bucket_index, bucket) in self.bucket_stats.iter_mut().enumerate() { + self.confirm_blocks_to_confirmed_txs + .iter_mut() + .for_each(|buckets| { + buckets[bucket_index] *= decay_factor; + }); + + self.confirm_blocks_to_failed_txs + .iter_mut() + .for_each(|buckets| { + buckets[bucket_index] *= decay_factor; + }); + bucket.total_fee_rate = + FeeRate::from_u64((bucket.total_fee_rate.as_u64() as f64 * decay_factor) as u64); + bucket.txs_count *= decay_factor; + // TODO do we need decay the old unconfirmed? + } + } + + /// The naive estimate implementation + /// 1. find best range of buckets satisfy the given condition + /// 2. get median fee_rate from best range bucekts + fn estimate_median( + &self, + confirm_blocks: usize, + required_samples: usize, + required_confirm_rate: f64, + ) -> Option { + // A tx need 1 block to propose, then 2 block to get confirmed + // so at least confirm blocks is 3 blocks. + if confirm_blocks < 3 || required_samples == 0 { + ckb_logger::debug!( + "confirm_blocks(={}) < 3 || required_samples(={}) == 0", + confirm_blocks, + required_samples + ); + return None; + } + let mut confirmed_txs = 0f64; + let mut txs_count = 0f64; + let mut failure_count = 0f64; + let mut extra_count = 0; + let mut best_bucket_start = 0; + let mut best_bucket_end = 0; + let mut start_bucket_index = 0; + let mut find_best = false; + // try find enough sample data from buckets + for (bucket_index, stat) in self.bucket_stats.iter().enumerate() { + confirmed_txs += self.confirm_blocks_to_confirmed_txs[confirm_blocks - 1][bucket_index]; + failure_count += self.confirm_blocks_to_failed_txs[confirm_blocks - 1][bucket_index]; + extra_count += &self.block_unconfirmed_txs[confirm_blocks - 1][bucket_index]; + txs_count += stat.txs_count; + // we have enough data + while txs_count as usize >= required_samples { + let confirm_rate = confirmed_txs / (txs_count + failure_count + extra_count as f64); + // satisfied required_confirm_rate, find the best buckets range + if confirm_rate >= required_confirm_rate { + best_bucket_start = start_bucket_index; + best_bucket_end = bucket_index; + find_best = true; + break; + } else { + // remove sample data of the first bucket in the range, then retry + let stat = &self.bucket_stats[start_bucket_index]; + confirmed_txs -= self.confirm_blocks_to_confirmed_txs[confirm_blocks - 1] + [start_bucket_index]; + failure_count -= + self.confirm_blocks_to_failed_txs[confirm_blocks - 1][start_bucket_index]; + extra_count -= + &self.block_unconfirmed_txs[confirm_blocks - 1][start_bucket_index]; + txs_count -= stat.txs_count; + start_bucket_index += 1; + continue; + } + } + + // end loop if we found the best buckets + if find_best { + break; + } + } + + if find_best { + let best_range_txs_count: f64 = self.bucket_stats[best_bucket_start..=best_bucket_end] + .iter() + .map(|b| b.txs_count) + .sum(); + + // find median bucket + if best_range_txs_count != 0f64 { + let mut half_count = best_range_txs_count / 2f64; + for bucket in &self.bucket_stats[best_bucket_start..=best_bucket_end] { + // find the median bucket + if bucket.txs_count >= half_count { + return bucket + .avg_fee_rate() + .map(|fee_rate| cmp::max(fee_rate, self.min_fee_rate)); + } else { + half_count -= bucket.txs_count; + } + } + } + ckb_logger::trace!("no best fee rate"); + } else { + ckb_logger::trace!("no best bucket"); + } + + None + } +} + +impl Default for Algorithm { + fn default() -> Self { + Self::new() + } +} + +impl Algorithm { + /// Creates a new estimator. + pub fn new() -> Self { + Self { + best_height: 0, + start_height: 0, + tx_confirm_stat: Default::default(), + tracked_txs: Default::default(), + current_tip: 0, + is_ready: false, + } + } + + fn process_block_tx(&mut self, height: u64, tx_hash: &Byte32) -> bool { + if let Some(tx) = self.drop_tx_inner(tx_hash, false) { + let blocks_to_confirm = height.saturating_sub(tx.height) as usize; + self.tx_confirm_stat + .add_confirmed_tx(blocks_to_confirm, tx.fee_rate); + true + } else { + // tx is not tracked + false + } + } + + /// process new block + /// record confirm blocks for txs which we tracked before. + fn process_block(&mut self, height: u64, txs: impl Iterator) { + // For simpfy, we assume chain reorg will not effect tx fee. + if height <= self.best_height { + return; + } + self.best_height = height; + // update tx confirm stat + self.tx_confirm_stat.move_track_window(height); + self.tx_confirm_stat.decay(); + let processed_txs = txs.filter(|tx| self.process_block_tx(height, tx)).count(); + if self.start_height == 0 && processed_txs > 0 { + // start record + self.start_height = self.best_height; + ckb_logger::debug!("start recording at {}", self.start_height); + } + } + + /// track a tx that entered txpool + fn track_tx(&mut self, tx_hash: Byte32, fee_rate: FeeRate, height: u64) { + if self.tracked_txs.contains_key(&tx_hash) { + // already in track + return; + } + if height != self.best_height { + // ignore wrong height txs + return; + } + if let Some(bucket_index) = self.tx_confirm_stat.add_unconfirmed_tx(height, fee_rate) { + self.tracked_txs.insert( + tx_hash, + TxRecord { + height, + bucket_index, + fee_rate, + }, + ); + } + } + + fn drop_tx_inner(&mut self, tx_hash: &Byte32, count_failure: bool) -> Option { + self.tracked_txs.remove(tx_hash).map(|tx_record| { + self.tx_confirm_stat.remove_unconfirmed_tx( + tx_record.height, + self.best_height, + tx_record.bucket_index, + count_failure, + ); + tx_record + }) + } + + /// tx removed from txpool + fn drop_tx(&mut self, tx_hash: &Byte32) -> bool { + self.drop_tx_inner(tx_hash, true).is_some() + } + + /// estimate a fee rate for confirm target + fn estimate(&self, expect_confirm_blocks: BlockNumber) -> Option { + let min_estimate_samples = constants::MIN_TARGET as usize * 2; + self.tx_confirm_stat.estimate_median( + expect_confirm_blocks as usize, + min_estimate_samples, + DEFAULT_MIN_CONFIRM_RATE, + ) + } +} + +impl Algorithm { + pub fn update_ibd_state(&mut self, in_ibd: bool) { + if self.is_ready { + if in_ibd { + self.clear(); + self.is_ready = false; + } + } else if !in_ibd { + self.clear(); + self.is_ready = true; + } + } + + fn clear(&mut self) { + self.best_height = 0; + self.start_height = 0; + self.tx_confirm_stat = Default::default(); + self.tracked_txs.clear(); + self.current_tip = 0; + } + + pub fn commit_block(&mut self, block: &BlockView) { + let tip_number = block.number(); + self.current_tip = tip_number; + self.process_block(tip_number, block.tx_hashes().iter().map(ToOwned::to_owned)); + } + + pub fn accept_tx(&mut self, tx_hash: Byte32, info: TxEntryInfo) { + let weight = get_transaction_weight(info.size as usize, info.cycles); + let fee_rate = FeeRate::calculate(info.fee, weight); + self.track_tx(tx_hash, fee_rate, self.current_tip) + } + + pub fn reject_tx(&mut self, tx_hash: &Byte32) { + let _ = self.drop_tx(tx_hash); + } + + pub fn get_fee_estimates(&self) -> Result, Error> { + if !self.is_ready { + return Err(Error::NotReady); + } + + let high = if let Some(fee_rate) = self.estimate(constants::HIGH_TARGET) { + fee_rate + } else { + return Ok(None); + }; + + let medium = if let Some(fee_rate) = self.estimate(constants::MEDIUM_TARGET) { + fee_rate + } else { + let rates = RecommendedFeeRates { + default: high, + low: high, + medium: high, + high, + }; + return Ok(Some(rates)); + }; + + let low = if let Some(fee_rate) = self.estimate(constants::LOW_TARGET) { + fee_rate + } else { + let rates = RecommendedFeeRates { + default: medium, + low: medium, + medium, + high, + }; + return Ok(Some(rates)); + }; + + let default = if let Some(fee_rate) = self.estimate(constants::DEFAULT_TARGET) { + fee_rate + } else { + let rates = RecommendedFeeRates { + default: low, + low, + medium, + high, + }; + return Ok(Some(rates)); + }; + + let rates = RecommendedFeeRates { + default, + low, + medium, + high, + }; + Ok(Some(rates)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_estimate_median() { + let mut bucket_fee_rate = 1000; + let bucket_end_fee_rate = 5000; + let rate = 1.1f64; + // decay = exp(ln(0.5) / 100), so decay.pow(100) =~ 0.5 + let decay = 0.993f64; + let max_confirm_blocks = 1000; + // prepare fee rate buckets + let mut buckets = vec![]; + while bucket_fee_rate < bucket_end_fee_rate { + buckets.push(FeeRate::from_u64(bucket_fee_rate)); + bucket_fee_rate = (rate * bucket_fee_rate as f64) as u64; + } + let mut stat = TxConfirmStat::new(buckets, max_confirm_blocks, decay); + // txs data + let fee_rate_and_confirms = vec![ + (2500, 5), + (3000, 5), + (3500, 5), + (1500, 10), + (2000, 10), + (2100, 10), + (2200, 10), + (1200, 15), + (1000, 15), + ]; + for (fee_rate, blocks_to_confirm) in fee_rate_and_confirms { + stat.add_confirmed_tx(blocks_to_confirm, FeeRate::from_u64(fee_rate)); + } + // test basic median fee rate + assert_eq!( + stat.estimate_median(5, 3, 1f64), + Some(FeeRate::from_u64(3000)) + ); + // test different required samples + assert_eq!( + stat.estimate_median(10, 1, 1f64), + Some(FeeRate::from_u64(1500)) + ); + assert_eq!( + stat.estimate_median(10, 3, 1f64), + Some(FeeRate::from_u64(2050)) + ); + assert_eq!( + stat.estimate_median(10, 4, 1f64), + Some(FeeRate::from_u64(2050)) + ); + assert_eq!( + stat.estimate_median(15, 2, 1f64), + Some(FeeRate::from_u64(1000)) + ); + assert_eq!( + stat.estimate_median(15, 3, 1f64), + Some(FeeRate::from_u64(1200)) + ); + // test return zero if confirm_blocks or required_samples is zero + assert_eq!(stat.estimate_median(0, 4, 1f64), None); + assert_eq!(stat.estimate_median(15, 0, 1f64), None); + assert_eq!(stat.estimate_median(0, 3, 1f64), None); + } +} diff --git a/util/fee-estimator/src/estimator/mod.rs b/util/fee-estimator/src/estimator/mod.rs new file mode 100644 index 0000000000..1d71bfeecf --- /dev/null +++ b/util/fee-estimator/src/estimator/mod.rs @@ -0,0 +1,92 @@ +use std::sync::Arc; + +use ckb_types::{ + core::{ + tx_pool::{TxEntryInfo, TxPoolEntryInfo}, + BlockView, RecommendedFeeRates, + }, + packed::Byte32, +}; +use ckb_util::RwLock; + +use crate::Error; + +mod confirmation_fraction; +mod weight_units_flow; + +/// The fee estimator with a chosen algorithm. +#[derive(Clone)] +pub enum FeeEstimator { + /// Dummy fee estimate algorithm; just do nothing. + Dummy, + /// Confirmation fraction fee estimator algorithm. + ConfirmationFraction(Arc>), + /// Weight-Units flow fee estimator algorithm. + WeightUnitsFlow(Arc>), +} + +impl FeeEstimator { + /// Creates a new dummy fee estimator. + pub fn new_dummy() -> Self { + FeeEstimator::Dummy + } + + /// Creates a new confirmation fraction fee estimator. + pub fn new_confirmation_fraction() -> Self { + let algo = confirmation_fraction::Algorithm::new(); + FeeEstimator::ConfirmationFraction(Arc::new(RwLock::new(algo))) + } + + /// Creates a new weight-units flow fee estimator. + pub fn new_weight_units_flow() -> Self { + let algo = weight_units_flow::Algorithm::new(); + FeeEstimator::WeightUnitsFlow(Arc::new(RwLock::new(algo))) + } + + /// Updates the IBD state. + pub fn update_ibd_state(&self, in_ibd: bool) { + match self { + Self::Dummy => {} + Self::ConfirmationFraction(algo) => algo.write().update_ibd_state(in_ibd), + Self::WeightUnitsFlow(algo) => algo.write().update_ibd_state(in_ibd), + } + } + + /// Commits a block. + pub fn commit_block(&self, block: &BlockView) { + match self { + Self::Dummy => {} + Self::ConfirmationFraction(algo) => algo.write().commit_block(block), + Self::WeightUnitsFlow(algo) => algo.write().commit_block(block), + } + } + + /// Accepts a tx. + pub fn accept_tx(&self, tx_hash: Byte32, info: TxEntryInfo) { + match self { + Self::Dummy => {} + Self::ConfirmationFraction(algo) => algo.write().accept_tx(tx_hash, info), + Self::WeightUnitsFlow(algo) => algo.write().accept_tx(info), + } + } + + /// Rejects a tx. + pub fn reject_tx(&self, tx_hash: &Byte32) { + match self { + Self::Dummy | Self::WeightUnitsFlow(_) => {} + Self::ConfirmationFraction(algo) => algo.write().reject_tx(tx_hash), + } + } + + /// Gets fee estimates. + pub fn get_fee_estimates( + &self, + all_entry_info: TxPoolEntryInfo, + ) -> Result, Error> { + match self { + Self::Dummy => Ok(None), + Self::ConfirmationFraction(algo) => algo.read().get_fee_estimates(), + Self::WeightUnitsFlow(algo) => algo.read().get_fee_estimates(all_entry_info), + } + } +} diff --git a/util/fee-estimator/src/estimator/weight_units_flow.rs b/util/fee-estimator/src/estimator/weight_units_flow.rs new file mode 100644 index 0000000000..16d209bd4b --- /dev/null +++ b/util/fee-estimator/src/estimator/weight_units_flow.rs @@ -0,0 +1,463 @@ +//! Weight-Units Flow Fee Estimator +//! +//! ### Summary +//! +//! This algorithm is migrated from a Bitcoin fee estimates algorithm. +//! +//! The original algorithm could be found in . +//! +//! ### Details +//! +//! #### Inputs +//! +//! The mempool is categorized into "fee buckets". +//! A bucket represents data about all transactions with a fee greater than or +//! equal to some amount (in `weight`). +//! +//! Each bucket contains 2 numeric values: +//! +//! - `current_weight`, represents the transactions currently sitting in the +//! mempool. +//! +//! - `flow`, represents the speed at which new transactions are entering the +//! mempool. +//! +//! It's sampled by observing the flow of transactions during twice the blocks +//! count of each target interval (ex: last 60 blocks for the 30 blocks target +//! interval). +//! +//! For simplicity, transactions are not looked at individually. +//! Focus is on the weight, like a fluid flowing from bucket to bucket. +//! +//! #### Computations +//! +//! Let's simulate what's going to happen during each timespan lasting blocks: +//! +//! - New transactions entering the mempool. +//! +//! While it's impossible to predict sudden changes to the speed at which new +//! weight is added to the mempool, for simplicty's sake we're going to assume +//! the flow we measured remains constant: `added_weight = flow * blocks`. +//! +//! - Transactions leaving the mempool due to mined blocks. Each block removes +//! up to `MAX_BLOCK_BYTES` weight from a bucket. +//! +//! Once we know the minimum expected number of blocks we can compute how that +//! would affect the bucket's weight: +//! `removed_weight = MAX_BLOCK_BYTES * blocks`. +//! +//! - Finally we can compute the expected final weight of the bucket: +//! `final_weight = current_weight + added_weight - removed_weight`. +//! +//! The cheapest bucket whose `final_weight` is less than or equal to 0 is going +//! to be the one selected as the estimate. + +use std::collections::HashMap; + +use ckb_chain_spec::consensus::MAX_BLOCK_BYTES; +use ckb_types::core::{ + tx_pool::{get_transaction_weight, TxEntryInfo, TxPoolEntryInfo}, + BlockNumber, BlockView, FeeRate, RecommendedFeeRates, +}; + +use crate::{constants, Error}; + +const FEE_RATE_UNIT: u64 = 1000; + +#[derive(Clone)] +pub struct Algorithm { + boot_tip: BlockNumber, + current_tip: BlockNumber, + txs: HashMap>, + + is_ready: bool, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +struct TxStatus { + weight: u64, + fee_rate: FeeRate, +} + +impl PartialOrd for TxStatus { + fn partial_cmp(&self, other: &TxStatus) -> Option<::std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for TxStatus { + fn cmp(&self, other: &Self) -> ::std::cmp::Ordering { + self.fee_rate + .cmp(&other.fee_rate) + .then_with(|| other.weight.cmp(&self.weight)) + } +} + +impl TxStatus { + fn new_from_entry_info(info: TxEntryInfo) -> Self { + let weight = get_transaction_weight(info.size as usize, info.cycles); + let fee_rate = FeeRate::calculate(info.fee, weight); + Self { weight, fee_rate } + } +} + +impl Default for Algorithm { + fn default() -> Self { + Self::new() + } +} + +impl Algorithm { + pub fn new() -> Self { + Self { + boot_tip: 0, + current_tip: 0, + txs: Default::default(), + is_ready: false, + } + } + + pub fn update_ibd_state(&mut self, in_ibd: bool) { + if self.is_ready { + if in_ibd { + self.clear(); + self.is_ready = false; + } + } else if !in_ibd { + self.clear(); + self.is_ready = true; + } + } + + fn clear(&mut self) { + self.boot_tip = 0; + self.current_tip = 0; + self.txs.clear(); + } + + pub fn commit_block(&mut self, block: &BlockView) { + let tip_number = block.number(); + if self.boot_tip == 0 { + self.boot_tip = tip_number; + } + self.current_tip = tip_number; + self.expire(); + } + + fn expire(&mut self) { + let historical_blocks = Self::historical_blocks(constants::MAX_TARGET); + let expired_tip = self.current_tip.saturating_sub(historical_blocks); + self.txs.retain(|&num, _| num >= expired_tip); + } + + pub fn accept_tx(&mut self, info: TxEntryInfo) { + if self.current_tip == 0 { + return; + } + let item = TxStatus::new_from_entry_info(info); + self.txs + .entry(self.current_tip) + .and_modify(|items| items.push(item)) + .or_insert_with(|| vec![item]); + } + + pub fn get_fee_estimates( + &self, + all_entry_info: TxPoolEntryInfo, + ) -> Result, Error> { + if !self.is_ready { + return Err(Error::NotReady); + } + + let sorted_current_txs = { + let mut current_txs: Vec<_> = all_entry_info + .pending + .into_values() + .chain(all_entry_info.proposed.into_values()) + .map(TxStatus::new_from_entry_info) + .collect(); + current_txs.sort_unstable_by(|a, b| b.cmp(a)); + current_txs + }; + + let high = if let Some(fee_rate) = + self.do_estimate(constants::HIGH_TARGET, &sorted_current_txs)? + { + fee_rate + } else { + return Ok(None); + }; + + let medium = if let Ok(Some(fee_rate)) = + self.do_estimate(constants::MEDIUM_TARGET, &sorted_current_txs) + { + fee_rate + } else { + let rates = RecommendedFeeRates { + default: high, + low: high, + medium: high, + high, + }; + return Ok(Some(rates)); + }; + + let low = if let Ok(Some(fee_rate)) = + self.do_estimate(constants::LOW_TARGET, &sorted_current_txs) + { + fee_rate + } else { + let rates = RecommendedFeeRates { + default: medium, + low: medium, + medium, + high, + }; + return Ok(Some(rates)); + }; + + let default = if let Ok(Some(fee_rate)) = + self.do_estimate(constants::DEFAULT_TARGET, &sorted_current_txs) + { + fee_rate + } else { + let rates = RecommendedFeeRates { + default: low, + low, + medium, + high, + }; + return Ok(Some(rates)); + }; + + let rates = RecommendedFeeRates { + default, + low, + medium, + high, + }; + Ok(Some(rates)) + } +} + +impl Algorithm { + fn do_estimate( + &self, + target_blocks: BlockNumber, + sorted_current_txs: &[TxStatus], + ) -> Result, Error> { + ckb_logger::debug!( + "boot: {}, current: {}, target: {target_blocks} blocks", + self.boot_tip, + self.current_tip, + ); + let historical_blocks = Self::historical_blocks(target_blocks); + ckb_logger::debug!("required: {historical_blocks} blocks"); + if historical_blocks > self.current_tip.saturating_sub(self.boot_tip) { + return Err(Error::LackData); + } + + let max_fee_rate = if let Some(fee_rate) = sorted_current_txs.first().map(|tx| tx.fee_rate) + { + fee_rate + } else { + return Ok(Some(constants::LOWEST_FEE_RATE)); + }; + + ckb_logger::debug!("max fee rate of current transactions: {max_fee_rate}"); + + let max_bucket_index = Self::max_bucket_index_by_fee_rate(max_fee_rate); + ckb_logger::debug!("current weight buckets size: {}", max_bucket_index + 1); + + // Create weight buckets. + let current_weight_buckets = { + let mut buckets = vec![0u64; max_bucket_index + 1]; + let mut index_curr = max_bucket_index; + for tx in sorted_current_txs { + let index = Self::max_bucket_index_by_fee_rate(tx.fee_rate); + if index < index_curr { + let weight_curr = buckets[index_curr]; + for i in buckets.iter_mut().take(index_curr).skip(index) { + *i = weight_curr; + } + } + buckets[index] += tx.weight; + index_curr = index; + } + buckets + }; + for (index, weight) in current_weight_buckets.iter().enumerate() { + if *weight != 0 { + ckb_logger::trace!(">>> current_weight[{index}]: {weight}"); + } + } + + // Calculate flow speeds for buckets. + let flow_speed_buckets = { + let historical_tip = self.current_tip - historical_blocks; + let sorted_flowed = self.sorted_flowed(historical_tip); + let mut buckets = vec![0u64; max_bucket_index + 1]; + let mut index_curr = max_bucket_index; + for tx in &sorted_flowed { + let index = Self::max_bucket_index_by_fee_rate(tx.fee_rate); + if index > max_bucket_index { + continue; + } + if index < index_curr { + let flowed_curr = buckets[index_curr]; + for i in buckets.iter_mut().take(index_curr).skip(index) { + *i = flowed_curr; + } + } + buckets[index] += tx.weight; + index_curr = index; + } + buckets + .into_iter() + .map(|value| value / historical_blocks) + .collect::>() + }; + for (index, speed) in flow_speed_buckets.iter().enumerate() { + if *speed != 0 { + ckb_logger::trace!(">>> flow_speed[{index}]: {speed}"); + } + } + + for bucket_index in 1..=max_bucket_index { + let current_weight = current_weight_buckets[bucket_index]; + let added_weight = flow_speed_buckets[bucket_index] * target_blocks; + let removed_weight = MAX_BLOCK_BYTES * target_blocks; + let passed = current_weight + added_weight <= removed_weight; + ckb_logger::trace!( + ">>> bucket[{}]: {}; {} + {} - {}", + bucket_index, + passed, + current_weight, + added_weight, + removed_weight + ); + if passed { + let fee_rate = Self::lowest_fee_rate_by_bucket_index(bucket_index); + return Ok(Some(fee_rate)); + } + } + + Ok(None) + } + + fn sorted_flowed(&self, historical_tip: BlockNumber) -> Vec { + let mut statuses: Vec<_> = self + .txs + .iter() + .filter(|(&num, _)| num >= historical_tip) + .flat_map(|(_, statuses)| statuses.to_owned()) + .collect(); + statuses.sort_unstable_by(|a, b| b.cmp(a)); + statuses + } +} + +impl Algorithm { + fn historical_blocks(target_blocks: BlockNumber) -> BlockNumber { + if target_blocks < constants::MIN_TARGET { + constants::MIN_TARGET * 2 + } else { + target_blocks * 2 + } + } + + fn lowest_fee_rate_by_bucket_index(index: usize) -> FeeRate { + let t = FEE_RATE_UNIT; + let value = match index as u64 { + // 0->0 + 0 => 0, + // 1->1000, 2->2000, .., 10->10000 + x if x <= 10 => t * x, + // 11->12000, 12->14000, .., 30->50000 + x if x <= 30 => t * (10 + (x - 10) * 2), + // 31->55000, 32->60000, ..., 60->200000 + x if x <= 60 => t * (10 + 20 * 2 + (x - 30) * 5), + // 61->210000, 62->220000, ..., 90->500000 + x if x <= 90 => t * (10 + 20 * 2 + 30 * 5 + (x - 60) * 10), + // 91->520000, 92->540000, ..., 115 -> 1000000 + x if x <= 115 => t * (10 + 20 * 2 + 30 * 5 + 30 * 10 + (x - 90) * 20), + // 116->1050000, 117->1100000, ..., 135->2000000 + x if x <= 135 => t * (10 + 20 * 2 + 30 * 5 + 30 * 10 + 25 * 20 + (x - 115) * 50), + // 136->2100000, 137->2200000, ... + x => t * (10 + 20 * 2 + 30 * 5 + 30 * 10 + 25 * 20 + 20 * 50 + (x - 135) * 100), + }; + FeeRate::from_u64(value) + } + + fn max_bucket_index_by_fee_rate(fee_rate: FeeRate) -> usize { + let t = FEE_RATE_UNIT; + let index = match fee_rate.as_u64() { + x if x <= 10_000 => x / t, + x if x <= 50_000 => (x + t * 10) / (2 * t), + x if x <= 200_000 => (x + t * 100) / (5 * t), + x if x <= 500_000 => (x + t * 400) / (10 * t), + x if x <= 1_000_000 => (x + t * 1_300) / (20 * t), + x if x <= 2_000_000 => (x + t * 4_750) / (50 * t), + x => (x + t * 11_500) / (100 * t), + }; + index as usize + } +} + +#[cfg(test)] +mod tests { + use super::Algorithm; + use ckb_types::core::FeeRate; + + #[test] + fn test_bucket_index_and_fee_rate_expected() { + let testdata = [ + (0, 0), + (1, 1_000), + (2, 2_000), + (10, 10_000), + (11, 12_000), + (12, 14_000), + (30, 50_000), + (31, 55_000), + (32, 60_000), + (60, 200_000), + (61, 210_000), + (62, 220_000), + (90, 500_000), + (91, 520_000), + (92, 540_000), + (115, 1_000_000), + (116, 1_050_000), + (117, 1_100_000), + (135, 2_000_000), + (136, 2_100_000), + (137, 2_200_000), + ]; + for (bucket_index, fee_rate) in &testdata[..] { + let expected_fee_rate = + Algorithm::lowest_fee_rate_by_bucket_index(*bucket_index).as_u64(); + assert_eq!(expected_fee_rate, *fee_rate); + let actual_bucket_index = + Algorithm::max_bucket_index_by_fee_rate(FeeRate::from_u64(*fee_rate)); + assert_eq!(actual_bucket_index, *bucket_index); + } + } + + #[test] + fn test_bucket_index_and_fee_rate_continuous() { + for fee_rate in 0..3_000_000 { + let bucket_index = Algorithm::max_bucket_index_by_fee_rate(FeeRate::from_u64(fee_rate)); + let fee_rate_le = Algorithm::lowest_fee_rate_by_bucket_index(bucket_index).as_u64(); + let fee_rate_gt = Algorithm::lowest_fee_rate_by_bucket_index(bucket_index + 1).as_u64(); + assert!( + fee_rate_le <= fee_rate && fee_rate < fee_rate_gt, + "Error for bucket[{}]: {} <= {} < {}", + bucket_index, + fee_rate_le, + fee_rate, + fee_rate_gt, + ); + } + } +} diff --git a/util/fee-estimator/src/lib.rs b/util/fee-estimator/src/lib.rs new file mode 100644 index 0000000000..e2c444e35b --- /dev/null +++ b/util/fee-estimator/src/lib.rs @@ -0,0 +1,8 @@ +//! CKB's built-in fee estimator, which shares data with the ckb node through the tx-pool service. + +pub mod constants; +pub(crate) mod error; +pub(crate) mod estimator; + +pub use error::Error; +pub use estimator::FeeEstimator; diff --git a/util/jsonrpc-types/src/fee_estimator.rs b/util/jsonrpc-types/src/fee_estimator.rs new file mode 100644 index 0000000000..d46db1882d --- /dev/null +++ b/util/jsonrpc-types/src/fee_estimator.rs @@ -0,0 +1,43 @@ +use ckb_types::core; +use serde::{Deserialize, Serialize}; + +use schemars::JsonSchema; + +/// Recommended fee rates. +#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize, JsonSchema)] +pub struct RecommendedFeeRates { + /// Default fee rate. + #[serde(rename = "no_priority")] + pub default: u64, + /// Low-priority fee rate. + #[serde(rename = "low_priority")] + pub low: u64, + /// Medium-priority fee rate. + #[serde(rename = "medium_priority")] + pub medium: u64, + /// High-priority fee rate. + #[serde(rename = "high_priority")] + pub high: u64, +} + +impl From for core::RecommendedFeeRates { + fn from(json: RecommendedFeeRates) -> Self { + core::RecommendedFeeRates { + default: core::FeeRate::from_u64(json.default), + low: core::FeeRate::from_u64(json.low), + medium: core::FeeRate::from_u64(json.medium), + high: core::FeeRate::from_u64(json.high), + } + } +} + +impl From for RecommendedFeeRates { + fn from(data: core::RecommendedFeeRates) -> Self { + RecommendedFeeRates { + default: data.default.as_u64(), + low: data.low.as_u64(), + medium: data.medium.as_u64(), + high: data.high.as_u64(), + } + } +} diff --git a/util/jsonrpc-types/src/lib.rs b/util/jsonrpc-types/src/lib.rs index ac70de3ec3..be965af45e 100644 --- a/util/jsonrpc-types/src/lib.rs +++ b/util/jsonrpc-types/src/lib.rs @@ -6,6 +6,7 @@ mod bytes; mod cell; mod debug; mod experiment; +mod fee_estimator; mod fee_rate; mod fixed_bytes; mod indexer; @@ -37,6 +38,7 @@ pub use self::bytes::JsonBytes; pub use self::cell::{CellData, CellInfo, CellWithStatus}; pub use self::debug::{ExtraLoggerConfig, MainLoggerConfig}; pub use self::experiment::{DaoWithdrawingCalculationKind, EstimateCycles}; +pub use self::fee_estimator::RecommendedFeeRates; pub use self::fee_rate::FeeRateDef; pub use self::fixed_bytes::Byte32; pub use self::info::{ChainInfo, DeploymentInfo, DeploymentPos, DeploymentState, DeploymentsInfo}; diff --git a/util/launcher/Cargo.toml b/util/launcher/Cargo.toml index e9d04dc543..fe089d6f69 100644 --- a/util/launcher/Cargo.toml +++ b/util/launcher/Cargo.toml @@ -33,7 +33,6 @@ ckb-tx-pool = { path = "../../tx-pool", version = "= 0.117.0-pre" } ckb-light-client-protocol-server = { path = "../light-client-protocol-server", version = "= 0.117.0-pre" } ckb-block-filter = { path = "../../block-filter", version = "= 0.117.0-pre" } - [features] with_sentry = [ "ckb-sync/with_sentry", "ckb-network/with_sentry", "ckb-app-config/with_sentry" ] portable = ["ckb-shared/portable"] diff --git a/util/launcher/src/lib.rs b/util/launcher/src/lib.rs index 0370339a54..776b0ef4af 100644 --- a/util/launcher/src/lib.rs +++ b/util/launcher/src/lib.rs @@ -203,6 +203,7 @@ impl Launcher { .notify_config(self.args.config.notify.clone()) .store_config(self.args.config.store) .block_assembler_config(block_assembler_config) + .fee_estimator_config(self.args.config.fee_estimator.clone()) .build()?; // internal check migrate_version diff --git a/util/types/src/core/fee_estimator.rs b/util/types/src/core/fee_estimator.rs new file mode 100644 index 0000000000..ef31bbbdcb --- /dev/null +++ b/util/types/src/core/fee_estimator.rs @@ -0,0 +1,14 @@ +use crate::core::FeeRate; + +/// Recommended fee rates. +#[derive(Clone, Copy, Debug)] +pub struct RecommendedFeeRates { + /// Default fee rate. + pub default: FeeRate, + /// Low-priority fee rate. + pub low: FeeRate, + /// Medium-priority fee rate. + pub medium: FeeRate, + /// High-priority fee rate. + pub high: FeeRate, +} diff --git a/util/types/src/core/mod.rs b/util/types/src/core/mod.rs index 5b0cbdb5b3..ace2a15ec6 100644 --- a/util/types/src/core/mod.rs +++ b/util/types/src/core/mod.rs @@ -23,6 +23,7 @@ mod tests; mod advanced_builders; mod blockchain; mod extras; +mod fee_estimator; mod fee_rate; mod reward; mod transaction_meta; @@ -31,6 +32,7 @@ mod views; pub use advanced_builders::{BlockBuilder, HeaderBuilder, TransactionBuilder}; pub use blockchain::DepType; pub use extras::{BlockExt, EpochExt, EpochNumberWithFraction, TransactionInfo}; +pub use fee_estimator::RecommendedFeeRates; pub use fee_rate::FeeRate; pub use reward::{BlockEconomicState, BlockIssuance, BlockReward, MinerReward}; pub use transaction_meta::{TransactionMeta, TransactionMetaBuilder}; From a6c1a8656d51042fb43a71c11c26e0e91b004cf6 Mon Sep 17 00:00:00 2001 From: Boyu Yang Date: Fri, 18 Oct 2024 12:16:06 +0800 Subject: [PATCH 3/3] fix(fee-estimator): adjusting parameters based on tests --- util/fee-estimator/src/constants.rs | 17 +++++++++-------- .../src/estimator/confirmation_fraction.rs | 18 +++++++++++------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/util/fee-estimator/src/constants.rs b/util/fee-estimator/src/constants.rs index 5cf781de80..ca08d9fa0e 100644 --- a/util/fee-estimator/src/constants.rs +++ b/util/fee-estimator/src/constants.rs @@ -3,22 +3,23 @@ use ckb_chain_spec::consensus::{MAX_BLOCK_INTERVAL, MIN_BLOCK_INTERVAL, TX_PROPOSAL_WINDOW}; use ckb_types::core::{BlockNumber, FeeRate}; -/// Average block interval. +/// Average block interval (28). pub(crate) const AVG_BLOCK_INTERVAL: u64 = (MAX_BLOCK_INTERVAL + MIN_BLOCK_INTERVAL) / 2; -/// Max target blocks, about 1 hour. +/// Max target blocks, about 1 hour (128). pub(crate) const MAX_TARGET: BlockNumber = (60 * 60) / AVG_BLOCK_INTERVAL; -/// Min target blocks, in next block. -pub(crate) const MIN_TARGET: BlockNumber = TX_PROPOSAL_WINDOW.closest() + 1; +/// Min target blocks, in next block (5). +/// NOTE After tests, 3 blocks are too strict; so to adjust larger: 5. +pub(crate) const MIN_TARGET: BlockNumber = (TX_PROPOSAL_WINDOW.closest() + 1) + 2; /// Lowest fee rate. pub(crate) const LOWEST_FEE_RATE: FeeRate = FeeRate::from_u64(1000); -/// Target blocks for no priority (lowest priority, about 1 hour). +/// Target blocks for no priority (lowest priority, about 1 hour, 128). pub const DEFAULT_TARGET: BlockNumber = MAX_TARGET; -/// Target blocks for low priority (about 30 minutes). +/// Target blocks for low priority (about 30 minutes, 64). pub const LOW_TARGET: BlockNumber = DEFAULT_TARGET / 2; -/// Target blocks for medium priority (about 10 minutes). +/// Target blocks for medium priority (about 10 minutes, 42). pub const MEDIUM_TARGET: BlockNumber = LOW_TARGET / 3; -/// Target blocks for high priority. +/// Target blocks for high priority (3). pub const HIGH_TARGET: BlockNumber = MIN_TARGET; diff --git a/util/fee-estimator/src/estimator/confirmation_fraction.rs b/util/fee-estimator/src/estimator/confirmation_fraction.rs index 5514078c8a..cc21d07c7d 100644 --- a/util/fee-estimator/src/estimator/confirmation_fraction.rs +++ b/util/fee-estimator/src/estimator/confirmation_fraction.rs @@ -18,7 +18,10 @@ use ckb_types::{ use crate::{constants, Error}; -const DEFAULT_MIN_CONFIRM_RATE: f64 = 0.9; +/// The number of blocks that the esitmator will trace the statistics. +const MAX_CONFIRM_BLOCKS: usize = 1000; +const DEFAULT_MIN_SAMPLES: usize = 20; +const DEFAULT_MIN_CONFIRM_RATE: f64 = 0.85; #[derive(Default, Debug, Clone)] struct BucketStat { @@ -108,10 +111,12 @@ impl BucketStat { impl Default for TxConfirmStat { fn default() -> Self { - let max_confirm_blocks = constants::MAX_TARGET as usize; let min_bucket_feerate = f64::from(constants::LOWEST_FEE_RATE.as_u64() as u32); - let max_bucket_feerate = min_bucket_feerate * 1000.0; - let fee_spacing = min_bucket_feerate; + // MULTIPLE = max_bucket_feerate / min_bucket_feerate + const MULTIPLE: f64 = 10000.0; + let max_bucket_feerate = min_bucket_feerate * MULTIPLE; + // expect 200 buckets + let fee_spacing = (MULTIPLE.ln() / 200.0f64).exp(); // half life each 100 blocks, math.exp(math.log(0.5) / 100) let decay_factor: f64 = (0.5f64.ln() / 100.0).exp(); @@ -122,7 +127,7 @@ impl Default for TxConfirmStat { buckets.push(FeeRate::from_u64(bucket_fee_boundary as u64)); bucket_fee_boundary *= fee_spacing; } - Self::new(buckets, max_confirm_blocks, decay_factor) + Self::new(buckets, MAX_CONFIRM_BLOCKS, decay_factor) } } @@ -430,10 +435,9 @@ impl Algorithm { /// estimate a fee rate for confirm target fn estimate(&self, expect_confirm_blocks: BlockNumber) -> Result { - let min_estimate_samples = constants::MIN_TARGET as usize * 2; self.tx_confirm_stat.estimate_median( expect_confirm_blocks as usize, - min_estimate_samples, + DEFAULT_MIN_SAMPLES, DEFAULT_MIN_CONFIRM_RATE, ) }