Merge #1659

1659: feat: Fee estimate RPC r=quake,zhangsoledad a=jjyr

This PR adds a new RPC [estimate_fee_rate](https://github.com/nervosnetwork/ckb/pull/1659/files#diff-622e6d119ac5d43f7eb41cb596159f9fR907). It takes the basic idea from bitcoin's [estimatesmartfee](https://bitcoincore.org/en/doc/0.16.0/rpc/util/estimatesmartfee/), however, we ignore the magic numbers and tricks from the original code.

We estimate the tx fee rate by track txs that entered tx pool. See details https://github.com/nervosnetwork/ckb/pull/1659/files#diff-ff03764a87b23e747dadc645fcf8df8bR21

Co-authored-by: jjy <[email protected]>

Cargo.toml

@@ -31,6 +31,7 @@ members = [
     "util/types",
     "util/future-executor",
     "util/jsonrpc-types",
+    "util/fee-estimator",
     "script/data-loader",
     "db",
     "resource",

benches/benches/benchmarks/overall.rs

@@ -8,7 +8,7 @@ use ckb_shared::{
     Snapshot,
 };
 use ckb_store::ChainStore;
-use ckb_tx_pool::{fee_rate::FeeRate, BlockAssemblerConfig, TxPoolConfig};
+use ckb_tx_pool::{BlockAssemblerConfig, FeeRate, TxPoolConfig};
 use ckb_types::{
     bytes::Bytes,
     core::{

rpc/Cargo.toml

@@ -15,6 +15,7 @@ ckb-sync = { path = "../sync" }
 ckb-chain = { path = "../chain" }
 ckb-logger = { path = "../util/logger"}
 ckb-network-alert = { path = "../util/network-alert" }
+ckb-fee-estimator = { path = "../util/fee-estimator" }
 jsonrpc-core = "10.1"
 jsonrpc-derive = "10.1"
 jsonrpc-http-server = { git = "https://github.com/nervosnetwork/jsonrpc", rev = "7c101f83a8fe34369c1b7a0e9b6721fcb0f91ee0" }
@@ -42,3 +43,4 @@ ckb-test-chain-utils = { path = "../util/test-chain-utils" }
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
 ckb-dao-utils = { path = "../util/dao/utils" }
+rand = "0.7"

rpc/README.md

@@ -21,6 +21,7 @@ NOTE: This file is auto-generated. Please don't update this file directly; inste
     *   [`dry_run_transaction`](#dry_run_transaction)
     *   [`_compute_transaction_hash`](#_compute_transaction_hash)
     *   [`calculate_dao_maximum_withdraw`](#calculate_dao_maximum_withdraw)
+    *   [`estimate_fee_rate`](#estimate_fee_rate)
     *   [`_compute_script_hash`](#_compute_script_hash)
 *   [`Indexer`](#indexer)
     *   [`index_lock_hash`](#index_lock_hash)
@@ -898,6 +899,41 @@ http://localhost:8114
 }
 ```
 
+### `estimate_fee_rate`
+
+Estimate a fee rate (capacity/KB) for a transaction that to be committed in expect blocks.
+
+This method estimate fee rate by sample transactions that collected from p2p network
+expected_confirm_blocks must be between 3 and 1000
+an error will return if samples is not enough
+
+
+#### Examples
+
+```bash
+echo '{
+    "id": 2,
+    "jsonrpc": "2.0",
+    "method": "estimate_fee_rate",
+    "params": [
+        "0xa"
+    ]
+}' \
+| tr -d '\n' \
+| curl -H 'content-type: application/json' -d @- \
+http://localhost:8114
+```
+
+```json
+{
+    "id": 2,
+    "jsonrpc": "2.0",
+    "result": {
+        "fee_rate": "0x7d0"
+    }
+}
+```
+
 ### `_compute_script_hash`
 
 Returns script hash of given transaction script

rpc/json/rpc.json

@@ -559,6 +559,17 @@
             }
         ]
     },
+    {
+        "description": "Estimate a fee rate (capacity/KB) for a transaction that to be committed in expect blocks.\n\nThis method estimate fee rate by sample transactions that collected from p2p network\nexpected_confirm_blocks must be between 3 and 1000\nan error will return if samples is not enough",
+        "method": "estimate_fee_rate",
+        "module": "experiment",
+        "params": [
+            "0xa"
+        ],
+        "result": {
+            "fee_rate": "0x7d0"
+        }
+    },
     {
         "description": "Send new transaction into transaction pool\n\nIf <block_hash> of <previsous_output> is not specified, loads the corresponding input cell. If <block_hash> is specified, load the corresponding input cell only if the corresponding block exist and contain this cell as output.",
         "method": "send_transaction",

rpc/src/module/experiment.rs

@@ -1,6 +1,9 @@
 use crate::error::RPCError;
 use ckb_dao::DaoCalculator;
-use ckb_jsonrpc_types::{Capacity, DryRunResult, OutPoint, Script, Transaction};
+use ckb_fee_estimator::MAX_CONFIRM_BLOCKS;
+use ckb_jsonrpc_types::{
+    Capacity, DryRunResult, EstimateResult, OutPoint, Script, Transaction, Uint64,
+};
 use ckb_logger::error;
 use ckb_shared::{shared::Shared, Snapshot};
 use ckb_store::ChainStore;
@@ -31,6 +34,10 @@ pub trait ExperimentRpc {
     #[rpc(name = "calculate_dao_maximum_withdraw")]
     fn calculate_dao_maximum_withdraw(&self, _out_point: OutPoint, _hash: H256)
         -> Result<Capacity>;
+
+    // Estimate fee
+    #[rpc(name = "estimate_fee_rate")]
+    fn estimate_fee_rate(&self, expect_confirm_blocks: Uint64) -> Result<EstimateResult>;
 }
 
 pub(crate) struct ExperimentRpcImpl {
@@ -65,6 +72,40 @@ impl ExperimentRpc for ExperimentRpcImpl {
             }
         }
     }
+
+    fn estimate_fee_rate(&self, expect_confirm_blocks: Uint64) -> Result<EstimateResult> {
+        let expect_confirm_blocks = expect_confirm_blocks.value() as usize;
+        // A tx need 1 block to propose, then 2 block to get confirmed
+        // so at least confirm blocks is 3 blocks.
+        if expect_confirm_blocks < 3 || expect_confirm_blocks > MAX_CONFIRM_BLOCKS {
+            return Err(RPCError::custom(
+                RPCError::Invalid,
+                format!(
+                    "expect_confirm_blocks should between 3 and {}, got {}",
+                    MAX_CONFIRM_BLOCKS, expect_confirm_blocks
+                ),
+            ));
+        }
+
+        let tx_pool = self.shared.tx_pool_controller();
+        let fee_rate = tx_pool.estimate_fee_rate(expect_confirm_blocks);
+        if let Err(e) = fee_rate {
+            error!("send estimate_fee_rate request error {}", e);
+            return Err(Error::internal_error());
+        };
+        let fee_rate = fee_rate.unwrap();
+
+        if fee_rate.as_u64() == 0 {
+            return Err(RPCError::custom(
+                RPCError::Invalid,
+                "collected samples is not enough, please make sure node has peers and try later"
+                    .into(),
+            ));
+        }
+        Ok(EstimateResult {
+            fee_rate: fee_rate.as_u64().into(),
+        })
+    }
 }
 
 // DryRunner dry run given transaction, and return the result, including execution cycles.

rpc/src/module/pool.rs

@@ -4,7 +4,7 @@ use ckb_logger::error;
 use ckb_network::PeerIndex;
 use ckb_shared::shared::Shared;
 use ckb_sync::SyncSharedState;
-use ckb_tx_pool::{error::SubmitTxError, fee_rate::FeeRate};
+use ckb_tx_pool::{error::SubmitTxError, FeeRate};
 use ckb_types::{
     core::{self},
     packed,

rpc/src/service_builder.rs

@@ -11,7 +11,7 @@ use ckb_network_alert::{notifier::Notifier as AlertNotifier, verifier::Verifier
 use ckb_shared::shared::Shared;
 use ckb_sync::SyncSharedState;
 use ckb_sync::Synchronizer;
-use ckb_tx_pool::fee_rate::FeeRate;
+use ckb_tx_pool::FeeRate;
 use ckb_util::Mutex;
 use jsonrpc_core::IoHandler;
 use std::sync::Arc;

rpc/src/test.rs

@@ -20,14 +20,16 @@ use ckb_shared::{
 use ckb_store::ChainStore;
 use ckb_sync::{SyncSharedState, Synchronizer};
 use ckb_test_chain_utils::{always_success_cell, always_success_cellbase};
-use ckb_tx_pool::fee_rate::FeeRate;
+use ckb_tx_pool::FeeRate;
 use ckb_types::{
     core::{
         capacity_bytes, cell::resolve_transaction, BlockBuilder, BlockView, Capacity,
         EpochNumberWithFraction, HeaderView, TransactionBuilder, TransactionView,
     },
     h256,
-    packed::{AlertBuilder, CellDep, CellInput, CellOutputBuilder, OutPoint, RawAlertBuilder},
+    packed::{
+        AlertBuilder, Byte32, CellDep, CellInput, CellOutputBuilder, OutPoint, RawAlertBuilder,
+    },
     prelude::*,
     H256,
 };
@@ -36,6 +38,7 @@ use jsonrpc_http_server::ServerBuilder;
 use jsonrpc_server_utils::cors::AccessControlAllowOrigin;
 use jsonrpc_server_utils::hosts::DomainsValidation;
 use pretty_assertions::assert_eq as pretty_assert_eq;
+use rand::{thread_rng, Rng};
 use reqwest;
 use serde::ser::Serialize;
 use serde_derive::{Deserialize, Serialize};
@@ -146,17 +149,42 @@ fn setup_node(height: u64) -> (Shared, ChainController, RpcServer) {
         .build()
         .unwrap();
     let chain_controller = ChainService::new(shared.clone(), table).start::<&str>(None);
+    let tx_pool_controller = shared.tx_pool_controller();
 
     // Build chain, insert [1, height) blocks
     let mut parent = always_success_consensus().genesis_block;
 
+    // prepare fee estimator samples
+    let sample_txs: Vec<Byte32> = (0..30)
+        .map(|_| {
+            let mut buf = [0u8; 32];
+            let mut rng = thread_rng();
+            rng.fill(&mut buf);
+            buf.pack()
+        })
+        .collect();
+    let fee_rate = FeeRate::from_u64(2_000);
+    let send_height = height.saturating_sub(9);
+
     for _ in 0..height {
         let block = next_block(&shared, &parent.header());
         chain_controller
             .process_block(Arc::new(block.clone()))
             .expect("processing new block should be ok");
+        // Fake fee estimator samples
+        if block.header().number() == send_height {
+            for tx_hash in sample_txs.clone() {
+                tx_pool_controller
+                    .estimator_track_tx(tx_hash, fee_rate, send_height)
+                    .expect("prepare estimator samples");
+            }
+        }
         parent = block;
     }
+    // mark txs as confirmed
+    tx_pool_controller
+        .estimator_process_block(height + 1, sample_txs.into_iter())
+        .expect("process estimator samples");
 
     // Start network services
     let dir = tempfile::tempdir()
@@ -402,6 +430,7 @@ fn params_of(shared: &Shared, method: &str) -> Value {
             let json_script: ckb_jsonrpc_types::Script = script.into();
             vec![json!(json_script)]
         }
+        "estimate_fee_rate" => vec![json!("0xa")],
         "calculate_dao_maximum_withdraw" => vec![json!(always_success_out_point), json!(tip_hash)],
         "get_block_template" => vec![json!(null), json!(null), json!(null)],
         "submit_block" => {

sync/src/relayer/mod.rs

@@ -28,7 +28,7 @@ use crate::BAD_MESSAGE_BAN_TIME;
 use ckb_chain::chain::ChainController;
 use ckb_logger::{debug_target, info_target, trace_target};
 use ckb_network::{CKBProtocolContext, CKBProtocolHandler, PeerIndex, TargetSession};
-use ckb_tx_pool::fee_rate::FeeRate;
+use ckb_tx_pool::FeeRate;
 use ckb_types::{
     core::{self},
     packed::{self, Byte32, ProposalShortId},

sync/src/relayer/tests/helper.rs

@@ -7,7 +7,7 @@ use ckb_network::{
 use ckb_shared::shared::{Shared, SharedBuilder};
 use ckb_store::ChainStore;
 use ckb_test_chain_utils::always_success_cell;
-use ckb_tx_pool::fee_rate::FeeRate;
+use ckb_tx_pool::FeeRate;
 use ckb_types::prelude::*;
 use ckb_types::{
     bytes::Bytes,

test/src/main.rs

@@ -250,6 +250,7 @@ fn all_specs() -> SpecMap {
         Box::new(ReferenceHeaderMaturity),
         Box::new(ValidSince),
         Box::new(SendLowFeeRateTx),
+        Box::new(FeeEstimate),
         Box::new(DifferentTxsWithSameInput),
         Box::new(CompactBlockEmpty),
         Box::new(CompactBlockEmptyParentUnknown),

test/src/rpc.rs

@@ -1,8 +1,9 @@
 use ckb_jsonrpc_types::{
     Alert, BannedAddr, Block, BlockNumber, BlockReward, BlockTemplate, BlockView, Capacity,
     CellOutputWithOutPoint, CellTransaction, CellWithStatus, ChainInfo, Cycle, DryRunResult,
-    EpochNumber, EpochView, HeaderView, LiveCell, LockHashIndexState, Node, OutPoint, PeerState,
-    Timestamp, Transaction, TransactionWithStatus, TxPoolInfo, Uint64, Version,
+    EpochNumber, EpochView, EstimateResult, HeaderView, LiveCell, LockHashIndexState, Node,
+    OutPoint, PeerState, Timestamp, Transaction, TransactionWithStatus, TxPoolInfo, Uint64,
+    Version,
 };
 use ckb_types::core::{
     BlockNumber as CoreBlockNumber, Capacity as CoreCapacity, EpochNumber as CoreEpochNumber,
@@ -359,6 +360,14 @@ impl RpcClient {
             .expect("rpc call get_cellbase_output_capacity_details")
             .expect("get_cellbase_output_capacity_details return none")
     }
+
+    pub fn estimate_fee_rate(&self, expect_confirm_blocks: Uint64) -> EstimateResult {
+        self.inner()
+            .lock()
+            .estimate_fee_rate(expect_confirm_blocks)
+            .call()
+            .expect("rpc call estimate_fee_rate")
+    }
 }
 
 jsonrpc_client!(pub struct Inner {
@@ -420,4 +429,5 @@ jsonrpc_client!(pub struct Inner {
     pub fn calculate_dao_maximum_withdraw(&mut self, _out_point: OutPoint, _hash: H256) -> RpcRequest<Capacity>;
     pub fn get_cellbase_output_capacity_details(&mut self, _hash: H256) -> RpcRequest<Option<BlockReward>>;
     pub fn broadcast_transaction(&mut self, tx: Transaction, cycles: Cycle) -> RpcRequest<H256>;
+    pub fn estimate_fee_rate(&mut self, expect_confirm_blocks: Uint64) -> RpcRequest<EstimateResult>;
 });

test/src/specs/mod.rs

@@ -23,7 +23,7 @@ use ckb_app_config::CKBAppConfig;
 use ckb_chain_spec::ChainSpec;
 use ckb_network::{ProtocolId, ProtocolVersion};
 use ckb_sync::NetworkProtocol;
-use ckb_tx_pool::fee_rate::FeeRate;
+use ckb_tx_pool::FeeRate;
 
 #[macro_export]
 macro_rules! name {

test/src/specs/relay/transaction_relay_low_fee_rate.rs

@@ -2,7 +2,7 @@ use crate::utils::wait_until;
 use crate::{Net, Spec, DEFAULT_TX_PROPOSAL_WINDOW};
 use ckb_app_config::CKBAppConfig;
 use ckb_jsonrpc_types::Status;
-use ckb_tx_pool::fee_rate::FeeRate;
+use ckb_tx_pool::FeeRate;
 use ckb_types::{core::TransactionView, packed, prelude::*};
 use log::info;