From 47af66c8f2bd75af045fa6f2bc9da9128fbb33a0 Mon Sep 17 00:00:00 2001 From: Victor Gao <10379359+vgao1996@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:20:39 -0700 Subject: [PATCH] [CLI] introducing local simulation, benchmarking and replay (#12832) --- Cargo.lock | 5 +- .../aptos-debugger/src/aptos_debugger.rs | 2 +- aptos-move/aptos-vm/src/aptos_vm.rs | 2 +- crates/aptos/Cargo.toml | 3 + crates/aptos/src/common/local_simulation.rs | 136 ++++++++++++++ crates/aptos/src/common/mod.rs | 1 + crates/aptos/src/common/types.rs | 122 +++++++----- crates/aptos/src/common/utils.rs | 10 + crates/aptos/src/move_tool/mod.rs | 174 +++++++++++++++++- .../Cargo.toml | 1 + 10 files changed, 405 insertions(+), 51 deletions(-) create mode 100644 crates/aptos/src/common/local_simulation.rs diff --git a/Cargo.lock b/Cargo.lock index e1325c55c5f6e..ac8a89576c43d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,6 +264,8 @@ dependencies = [ "aptos-types", "aptos-vm", "aptos-vm-genesis", + "aptos-vm-logging", + "aptos-vm-types", "async-trait", "base64 0.13.1", "bcs 0.1.4", @@ -313,6 +315,7 @@ dependencies = [ "tonic 0.11.0", "tracing", "tracing-subscriber 0.3.18", + "url", "version-compare", ] @@ -2055,7 +2058,7 @@ dependencies = [ [[package]] name = "aptos-indexer-grpc-in-memory-cache-benchmark" -version = "0.0.0" +version = "0.1.0" dependencies = [ "anyhow", "aptos-indexer-grpc-utils", diff --git a/aptos-move/aptos-debugger/src/aptos_debugger.rs b/aptos-move/aptos-debugger/src/aptos_debugger.rs index b11f1c7f886b0..c7a923ae2fb6d 100644 --- a/aptos-move/aptos-debugger/src/aptos_debugger.rs +++ b/aptos-move/aptos-debugger/src/aptos_debugger.rs @@ -132,7 +132,7 @@ impl AptosDebugger { // Deprecated. TransactionPayload::ModuleBundle(..) => { - unreachable!("Module bundle payload has already been checked") + unreachable!("Module bundle payload has already been checked because before this function is called") }, }; Ok(gas_profiler) diff --git a/aptos-move/aptos-vm/src/aptos_vm.rs b/aptos-move/aptos-vm/src/aptos_vm.rs index 924f46041073c..5fb18ab86eb94 100644 --- a/aptos-move/aptos-vm/src/aptos_vm.rs +++ b/aptos-move/aptos-vm/src/aptos_vm.rs @@ -1716,7 +1716,7 @@ impl AptosVM { }) } - fn execute_user_transaction( + pub fn execute_user_transaction( &self, resolver: &impl AptosMoveResolver, txn: &SignedTransaction, diff --git a/crates/aptos/Cargo.toml b/crates/aptos/Cargo.toml index f1e3820d602ec..a4d7c1629cc1b 100644 --- a/crates/aptos/Cargo.toml +++ b/crates/aptos/Cargo.toml @@ -46,6 +46,8 @@ aptos-temppath = { workspace = true } aptos-types = { workspace = true } aptos-vm = { workspace = true, features = ["testing"] } aptos-vm-genesis = { workspace = true } +aptos-vm-logging = { workspace = true } +aptos-vm-types = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } bcs = { workspace = true } @@ -97,6 +99,7 @@ toml = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +url = { workspace = true } version-compare = { workspace = true } [target.'cfg(unix)'.dependencies] diff --git a/crates/aptos/src/common/local_simulation.rs b/crates/aptos/src/common/local_simulation.rs new file mode 100644 index 0000000000000..f25d6a943a31d --- /dev/null +++ b/crates/aptos/src/common/local_simulation.rs @@ -0,0 +1,136 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use crate::common::types::{CliError, CliTypedResult}; +use aptos_crypto::HashValue; +use aptos_gas_profiling::FrameName; +use aptos_move_debugger::aptos_debugger::AptosDebugger; +use aptos_types::transaction::SignedTransaction; +use aptos_vm::{data_cache::AsMoveResolver, AptosVM}; +use aptos_vm_logging::log_schema::AdapterLogSchema; +use aptos_vm_types::{output::VMOutput, resolver::StateStorageView}; +use move_core_types::vm_status::VMStatus; +use std::{path::Path, time::Instant}; + +pub fn run_transaction_using_debugger( + debugger: &AptosDebugger, + version: u64, + transaction: SignedTransaction, + _hash: HashValue, +) -> CliTypedResult<(VMStatus, VMOutput)> { + let state_view = debugger.state_view_at_version(version); + let resolver = state_view.as_move_resolver(); + + let vm = AptosVM::new(&resolver, None); + let log_context = AdapterLogSchema::new(resolver.id(), 0); + + let (vm_status, vm_output) = vm.execute_user_transaction(&resolver, &transaction, &log_context); + + Ok((vm_status, vm_output)) +} + +pub fn benchmark_transaction_using_debugger( + debugger: &AptosDebugger, + version: u64, + transaction: SignedTransaction, + _hash: HashValue, +) -> CliTypedResult<(VMStatus, VMOutput)> { + let state_view = debugger.state_view_at_version(version); + let resolver = state_view.as_move_resolver(); + + let vm = AptosVM::new(&resolver, None); + let log_context = AdapterLogSchema::new(resolver.id(), 0); + + let (vm_status, vm_output) = vm.execute_user_transaction(&resolver, &transaction, &log_context); + + let time_cold = { + let n = 15; + + let mut times = vec![]; + for _i in 0..n { + // Create a new VM each time so to include code loading as part of the + // total running time. + let vm = AptosVM::new(&resolver, None); + let log_context = AdapterLogSchema::new(resolver.id(), 0); + + let t1 = Instant::now(); + std::hint::black_box(vm.execute_user_transaction( + &resolver, + &transaction, + &log_context, + )); + let t2 = Instant::now(); + + times.push(t2 - t1); + } + times.sort(); + + times[n / 2] + }; + + let time_warm = { + let mut times = vec![]; + let n = 15; + + for _i in 0..n { + // Reuse the existing VM with warm code cache so to measure only the + // execution time. + let t1 = Instant::now(); + std::hint::black_box(vm.execute_user_transaction( + &resolver, + &transaction, + &log_context, + )); + let t2 = Instant::now(); + + times.push(t2 - t1); + } + + times[n / 2] + }; + + println!("Running time (cold code cache): {:?}", time_cold); + println!("Running time (warm code cache): {:?}", time_warm); + + Ok((vm_status, vm_output)) +} + +pub fn profile_transaction_using_debugger( + debugger: &AptosDebugger, + version: u64, + transaction: SignedTransaction, + hash: HashValue, +) -> CliTypedResult<(VMStatus, VMOutput)> { + let (vm_status, vm_output, gas_log) = debugger + .execute_transaction_at_version_with_gas_profiler(version, transaction) + .map_err(|err| { + CliError::UnexpectedError(format!("failed to simulate txn with gas profiler: {}", err)) + })?; + + // Generate a humen-readable name for the report + let entry_point = gas_log.entry_point(); + + let human_readable_name = match entry_point { + FrameName::Script => "script".to_string(), + FrameName::Function { + module_id, name, .. + } => { + let addr_short = module_id.address().short_str_lossless(); + let addr_truncated = if addr_short.len() > 4 { + &addr_short[..4] + } else { + addr_short.as_str() + }; + format!("0x{}-{}-{}", addr_truncated, module_id.name(), name) + }, + }; + let raw_file_name = format!("txn-{}-{}", hash, human_readable_name); + + // Generate the report + let path = Path::new("gas-profiling").join(raw_file_name); + gas_log.generate_html_report(&path, format!("Gas Report - {}", human_readable_name))?; + + println!("Gas report saved to {}.", path.display()); + + Ok((vm_status, vm_output)) +} diff --git a/crates/aptos/src/common/mod.rs b/crates/aptos/src/common/mod.rs index 1be94da1d100b..016e3e18ca6f4 100644 --- a/crates/aptos/src/common/mod.rs +++ b/crates/aptos/src/common/mod.rs @@ -2,5 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 pub mod init; +pub mod local_simulation; pub mod types; pub mod utils; diff --git a/crates/aptos/src/common/types.rs b/crates/aptos/src/common/types.rs index 713d700994e7c..aead564855588 100644 --- a/crates/aptos/src/common/types.rs +++ b/crates/aptos/src/common/types.rs @@ -5,6 +5,7 @@ use super::utils::fund_account; use crate::{ common::{ init::Network, + local_simulation, utils::{ check_if_file_exists, create_dir_if_not_exist, dir_default_to_current, get_account_with_state, get_auth_key, get_sequence_number, parse_json_file, @@ -23,7 +24,6 @@ use aptos_crypto::{ encoding_type::{EncodingError, EncodingType}, x25519, PrivateKey, ValidCryptoMaterialStringExt, }; -use aptos_gas_profiling::FrameName; use aptos_global_constants::adjust_gas_headroom; use aptos_keygen::KeyGen; use aptos_logger::Level; @@ -44,10 +44,13 @@ use aptos_types::{ SignedTransaction, TransactionArgument, TransactionPayload, TransactionStatus, }, }; +use aptos_vm_types::output::VMOutput; use async_trait::async_trait; use clap::{Parser, ValueEnum}; use hex::FromHexError; -use move_core_types::{account_address::AccountAddress, language_storage::TypeTag}; +use move_core_types::{ + account_address::AccountAddress, language_storage::TypeTag, vm_status::VMStatus, +}; use move_model::metadata::{CompilerVersion, LanguageVersion}; use serde::{Deserialize, Serialize}; #[cfg(unix)] @@ -57,7 +60,7 @@ use std::{ convert::TryFrom, fmt::{Debug, Display, Formatter}, fs::OpenOptions, - path::{Path, PathBuf}, + path::PathBuf, str::FromStr, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; @@ -1508,6 +1511,14 @@ pub struct TransactionOptions { #[clap(flatten)] pub(crate) prompt_options: PromptOptions, + /// If this option is set, simulate the transaction locally. + #[clap(long)] + pub(crate) local: bool, + + /// If this option is set, benchmark the transaction locally. + #[clap(long)] + pub(crate) benchmark: bool, + /// If this option is set, simulate the transaction locally using the debugger and generate /// flamegraphs that reflect the gas usage. #[clap(long)] @@ -1729,14 +1740,20 @@ impl TransactionOptions { } } - /// Simulate the transaction locally using the debugger, with the gas profiler enabled. - pub async fn profile_gas( + /// Simulates a transaction locally, using the debugger to fetch required data from remote. + async fn simulate_using_debugger( &self, payload: TransactionPayload, - ) -> CliTypedResult { - println!(); - println!("Simulating transaction locally with the gas profiler..."); - + execute: F, + ) -> CliTypedResult + where + F: FnOnce( + &AptosDebugger, + u64, + SignedTransaction, + aptos_crypto::HashValue, + ) -> CliTypedResult<(VMStatus, VMOutput)>, + { let client = self.rest_client()?; // Fetch the chain states required for the simulation @@ -1768,7 +1785,6 @@ impl TransactionOptions { } }); - // Create and sign the transaction let transaction_factory = TransactionFactory::new(chain_id) .with_gas_unit_price(gas_unit_price) .with_max_gas_amount(max_gas) @@ -1778,47 +1794,17 @@ impl TransactionOptions { sender_account.sign_with_transaction_builder(transaction_factory.payload(payload)); let hash = transaction.clone().committed_hash(); - // Execute the transaction using the debugger let debugger = AptosDebugger::rest_client(client).unwrap(); - let res = debugger.execute_transaction_at_version_with_gas_profiler(version, transaction); - let (vm_status, output, gas_log) = res.map_err(|err| { - CliError::UnexpectedError(format!("failed to simulate txn with gas profiler: {}", err)) - })?; + let (vm_status, vm_output) = execute(&debugger, version, transaction, hash)?; - // Generate a humen-readable name for the report - let entry_point = gas_log.entry_point(); - - let human_readable_name = match entry_point { - FrameName::Script => "script".to_string(), - FrameName::Function { - module_id, name, .. - } => { - let addr_short = module_id.address().short_str_lossless(); - let addr_truncated = if addr_short.len() > 4 { - &addr_short[..4] - } else { - addr_short.as_str() - }; - format!("0x{}-{}-{}", addr_truncated, module_id.name(), name) - }, - }; - let raw_file_name = format!("txn-{}-{}", hash, human_readable_name); - - // Generate the report - let path = Path::new("gas-profiling").join(raw_file_name); - gas_log.generate_html_report(path, format!("Gas Report - {}", human_readable_name))?; - - // Generate the transaction summary - - // TODO(Gas): double check if this is correct. - let success = match output.status() { + let success = match vm_output.status() { TransactionStatus::Keep(exec_status) => Some(exec_status.is_success()), TransactionStatus::Discard(_) | TransactionStatus::Retry => None, }; - Ok(TransactionSummary { + let summary = TransactionSummary { transaction_hash: hash.into(), - gas_used: Some(output.gas_used()), + gas_used: Some(vm_output.gas_used()), gas_unit_price: Some(gas_unit_price), pending: None, sender: Some(sender_address), @@ -1827,7 +1813,53 @@ impl TransactionOptions { timestamp_us: None, version: Some(version), // The transaction is not comitted so there is no new version. vm_status: Some(vm_status.to_string()), - }) + }; + + Ok(summary) + } + + /// Simulates a transaction locally. + pub async fn simulate_locally( + &self, + payload: TransactionPayload, + ) -> CliTypedResult { + println!(); + println!("Simulating transaction locally..."); + + self.simulate_using_debugger(payload, local_simulation::run_transaction_using_debugger) + .await + } + + /// Benchmarks the transaction payload locally. + /// The transaction is executed multiple times, and the median value is calculated to improve + /// the accuracy of the measurement results. + pub async fn benchmark_locally( + &self, + payload: TransactionPayload, + ) -> CliTypedResult { + println!(); + println!("Benchmarking transaction locally..."); + + self.simulate_using_debugger( + payload, + local_simulation::benchmark_transaction_using_debugger, + ) + .await + } + + /// Simulates the transaction locally with the gas profiler enabled. + pub async fn profile_gas( + &self, + payload: TransactionPayload, + ) -> CliTypedResult { + println!(); + println!("Simulating transaction locally using the gas profiler..."); + + self.simulate_using_debugger( + payload, + local_simulation::profile_transaction_using_debugger, + ) + .await } pub async fn estimate_gas_price(&self) -> CliTypedResult { diff --git a/crates/aptos/src/common/utils.rs b/crates/aptos/src/common/utils.rs index 0eee2c5ea3ad9..9fcd7bc507c08 100644 --- a/crates/aptos/src/common/utils.rs +++ b/crates/aptos/src/common/utils.rs @@ -497,9 +497,19 @@ pub async fn profile_or_submit( payload: TransactionPayload, txn_options_ref: &TransactionOptions, ) -> CliTypedResult { + if txn_options_ref.profile_gas && txn_options_ref.benchmark { + return Err(CliError::UnexpectedError( + "Cannot perform benchmarking and gas profiling at the same time.".to_string(), + )); + } + // Profile gas if needed. if txn_options_ref.profile_gas { txn_options_ref.profile_gas(payload).await + } else if txn_options_ref.benchmark { + txn_options_ref.benchmark_locally(payload).await + } else if txn_options_ref.local { + txn_options_ref.simulate_locally(payload).await } else { // Otherwise submit the transaction. txn_options_ref diff --git a/crates/aptos/src/move_tool/mod.rs b/crates/aptos/src/move_tool/mod.rs index b1db6deefd5e6..c1369d83e8c43 100644 --- a/crates/aptos/src/move_tool/mod.rs +++ b/crates/aptos/src/move_tool/mod.rs @@ -4,6 +4,7 @@ use crate::{ account::derive_resource_account::ResourceAccountSeed, common::{ + local_simulation, types::{ load_account_arg, ArgWithTypeJSON, CliConfig, CliError, CliTypedResult, ConfigSearchMode, EntryFunctionArguments, EntryFunctionArgumentsJSON, @@ -30,15 +31,18 @@ use aptos_framework::{ BuildOptions, BuiltPackage, }; use aptos_gas_schedule::{MiscGasParameters, NativeGasParameters}; -use aptos_rest_client::aptos_api_types::{ - EntryFunctionId, HexEncodedBytes, IdentifierWrapper, MoveModuleId, +use aptos_move_debugger::aptos_debugger::AptosDebugger; +use aptos_rest_client::{ + aptos_api_types::{EntryFunctionId, HexEncodedBytes, IdentifierWrapper, MoveModuleId}, + Client, }; use aptos_types::{ account_address::{create_resource_address, AccountAddress}, object_address::create_object_code_deployment_address, on_chain_config::aptos_test_feature_flags_genesis, - transaction::{TransactionArgument, TransactionPayload}, + transaction::{Transaction, TransactionArgument, TransactionPayload, TransactionStatus}, }; +use aptos_vm::data_cache::AsMoveResolver; use async_trait::async_trait; use clap::{Parser, Subcommand, ValueEnum}; use itertools::Itertools; @@ -59,6 +63,7 @@ use std::{ }; pub use stored_package::*; use tokio::task; +use url::Url; mod aptos_debug_natives; mod bytecode; @@ -99,6 +104,7 @@ pub enum MoveTool { Test(TestPackage), VerifyPackage(VerifyPackage), View(ViewFunction), + Replay(Replay), } impl MoveTool { @@ -130,6 +136,7 @@ impl MoveTool { MoveTool::Test(tool) => tool.execute_serialized().await, MoveTool::VerifyPackage(tool) => tool.execute_serialized().await, MoveTool::View(tool) => tool.execute_serialized().await, + MoveTool::Replay(tool) => tool.execute_serialized().await, } } } @@ -1491,6 +1498,167 @@ impl CliCommand for RunScript { } } +#[derive(Clone, Debug)] +pub enum ReplayNetworkSelection { + Mainnet, + Testnet, + Devnet, + RestEndpoint(String), +} + +/// Replay a comitted transaction using a local VM. +#[derive(Parser, Debug)] +pub struct Replay { + /// The network to replay on. + /// + /// Possible values: + /// mainnet, testnet, + #[clap(long)] + pub(crate) network: ReplayNetworkSelection, + + /// The id of the transaction to replay. Also being referred to as "version" in some contexts. + #[clap(long)] + pub(crate) txn_id: u64, + + /// If this option is set, benchmark the transaction and report the running time(s). + #[clap(long)] + pub(crate) benchmark: bool, + + /// If this option is set, profile the transaction and generate a detailed report of its gas usage. + #[clap(long)] + pub(crate) profile_gas: bool, + + /// If present, skip the comparison against the expected transaction output. + #[clap(long)] + pub(crate) skip_comparison: bool, +} + +impl FromStr for ReplayNetworkSelection { + type Err = CliError; + + fn from_str(s: &str) -> Result { + Ok(match s { + "mainnet" => Self::Mainnet, + "testnet" => Self::Testnet, + "devnet" => Self::Devnet, + _ => Self::RestEndpoint(s.to_owned()), + }) + } +} + +#[async_trait] +impl CliCommand for Replay { + fn command_name(&self) -> &'static str { + "Replay" + } + + async fn execute(self) -> CliTypedResult { + use ReplayNetworkSelection::*; + + if self.profile_gas && self.benchmark { + return Err(CliError::UnexpectedError( + "Cannot perform benchmarking and gas profiling at the same time.".to_string(), + )); + } + + let rest_endpoint = match &self.network { + Mainnet => "https://fullnode.mainnet.aptoslabs.com", + Testnet => "https://fullnode.testnet.aptoslabs.com", + Devnet => "https://fullnode.devnet.aptoslabs.com", + RestEndpoint(url) => url, + }; + + let debugger = AptosDebugger::rest_client(Client::new( + Url::parse(rest_endpoint) + .map_err(|_err| CliError::UnableToParse("url", rest_endpoint.to_string()))?, + ))?; + + // Fetch the transaction to replay. + let (txn, txn_info) = debugger + .get_committed_transaction_at_version(self.txn_id) + .await?; + + let txn = match txn { + Transaction::UserTransaction(txn) => txn, + _ => { + return Err(CliError::UnexpectedError( + "Unsupported transaction type. Only user transactions are supported." + .to_string(), + )) + }, + }; + + let hash = txn.clone().committed_hash(); + + // Execute the transaction. + let (vm_status, vm_output) = if self.profile_gas { + println!("Profiling transaction..."); + local_simulation::profile_transaction_using_debugger( + &debugger, + self.txn_id, + txn.clone(), + hash, + )? + } else if self.benchmark { + println!("Benchmarking transaction..."); + local_simulation::benchmark_transaction_using_debugger( + &debugger, + self.txn_id, + txn.clone(), + hash, + )? + } else { + println!("Replaying transaction..."); + local_simulation::run_transaction_using_debugger( + &debugger, + self.txn_id, + txn.clone(), + hash, + )? + }; + + // Materialize into transaction output and check if the outputs match. + let state_view = debugger.state_view_at_version(self.txn_id); + let resolver = state_view.as_move_resolver(); + + let txn_output = vm_output + .try_materialize_into_transaction_output(&resolver) + .map_err(|err| { + CliError::UnexpectedError(format!( + "Failed to materialize into transaction output: {}", + err + )) + })?; + + if !self.skip_comparison { + txn_output + .ensure_match_transaction_info(self.txn_id, &txn_info, None, None) + .map_err(|msg| CliError::UnexpectedError(msg.to_string()))?; + } + + // Generate the transaction summary. + let success = match txn_output.status() { + TransactionStatus::Keep(exec_status) => Some(exec_status.is_success()), + TransactionStatus::Discard(_) | TransactionStatus::Retry => None, + }; + + let summary = TransactionSummary { + transaction_hash: txn.clone().committed_hash().into(), + gas_used: Some(txn_output.gas_used()), + gas_unit_price: Some(txn.gas_unit_price()), + pending: None, + sender: Some(txn.sender()), + sequence_number: Some(txn.sequence_number()), + success, + timestamp_us: None, + version: Some(self.txn_id), + vm_status: Some(vm_status.to_string()), + }; + + Ok(summary) + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum FunctionArgType { Address, diff --git a/ecosystem/indexer-grpc/indexer-grpc-in-memory-cache-benchmark/Cargo.toml b/ecosystem/indexer-grpc/indexer-grpc-in-memory-cache-benchmark/Cargo.toml index 8cbfaee7e2e2e..c40527464dcf5 100644 --- a/ecosystem/indexer-grpc/indexer-grpc-in-memory-cache-benchmark/Cargo.toml +++ b/ecosystem/indexer-grpc/indexer-grpc-in-memory-cache-benchmark/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "aptos-indexer-grpc-in-memory-cache-benchmark" +version = "0.1.0" # Workspace inherited keys authors = { workspace = true }