diff --git a/crates/cli/src/commands/report/chart/chart_id.rs b/crates/cli/src/commands/report/chart/chart_id.rs index 1b84676a..eb9ee7fb 100644 --- a/crates/cli/src/commands/report/chart/chart_id.rs +++ b/crates/cli/src/commands/report/chart/chart_id.rs @@ -1,4 +1,4 @@ -use crate::commands::report::report_dir; +use crate::util::report_dir; pub enum ReportChartId { Heatmap, diff --git a/crates/cli/src/commands/report/command.rs b/crates/cli/src/commands/report/command.rs new file mode 100644 index 00000000..84d531c0 --- /dev/null +++ b/crates/cli/src/commands/report/command.rs @@ -0,0 +1,131 @@ +use super::block_trace::get_block_trace_data; +use super::cache::CacheFile; +use super::chart::{ + DrawableChart, GasPerBlockChart, HeatMapChart, PendingTxsChart, ReportChartId, + TimeToInclusionChart, TxGasUsedChart, +}; +use super::gen_html::{build_html_report, ReportMetadata}; +use crate::util::{report_dir, write_run_txs}; +use alloy::network::AnyNetwork; +use alloy::providers::DynProvider; +use alloy::{providers::ProviderBuilder, transports::http::reqwest::Url}; +use contender_core::db::{DbOps, RunTx}; +use csv::WriterBuilder; +use std::str::FromStr; + +pub async fn report( + last_run_id: Option, + preceding_runs: u64, + db: &(impl DbOps + Clone + Send + Sync + 'static), + rpc_url: &str, +) -> Result<(), Box> { + let num_runs = db.num_runs()?; + + if num_runs == 0 { + println!("No runs found in the database. Exiting."); + return Ok(()); + } + + // if id is provided, check if it's valid + let end_run_id = if let Some(id) = last_run_id { + if id == 0 || id > num_runs { + // panic!("Invalid run ID: {}", id); + return Err(format!("Invalid run ID: {}", id).into()); + } + id + } else { + // get latest run + println!("No run ID provided. Using latest run ID: {}", num_runs); + num_runs + }; + + // collect CSV report for each run_id + let start_run_id = end_run_id - preceding_runs; + let mut all_txs = vec![]; + for id in start_run_id..=end_run_id { + let txs = db.get_run_txs(id)?; + all_txs.extend_from_slice(&txs); + save_csv_report(id, &txs)?; + } + + // get run data + let mut run_data = vec![]; + for id in start_run_id..=end_run_id { + let run = db.get_run(id)?; + if let Some(run) = run { + run_data.push(run); + } + } + // collect all unique scenario_name values from run_data + let scenario_names: Vec = run_data + .iter() + .map(|run| run.scenario_name.clone()) + .collect::>() + .iter() + .map(|v| { + // return only the filename without the path and extension + let re = regex::Regex::new(r".*/(.*)\.toml$").unwrap(); + re.replace(v, "$1").to_string() + }) + .collect(); + let scenario_title = scenario_names + .into_iter() + .reduce(|acc, v| format!("{}, {}", acc, v)) + .unwrap_or_default(); + + // get trace data for reports + let url = Url::from_str(rpc_url).expect("Invalid URL"); + let rpc_client = DynProvider::new(ProviderBuilder::new().network::().on_http(url)); + let (trace_data, blocks) = get_block_trace_data(&all_txs, &rpc_client).await?; + + // cache data to file + let cache_data = CacheFile::new(trace_data, blocks); + cache_data.save()?; + + // make heatmap + let heatmap = HeatMapChart::new(&cache_data.traces)?; + heatmap.draw(&ReportChartId::Heatmap.filename(start_run_id, end_run_id)?)?; + + // make gasPerBlock chart + let gas_per_block = GasPerBlockChart::new(&cache_data.blocks); + gas_per_block.draw(&ReportChartId::GasPerBlock.filename(start_run_id, end_run_id)?)?; + + // make timeToInclusion chart + let time_to_inclusion = TimeToInclusionChart::new(&all_txs); + time_to_inclusion.draw(&ReportChartId::TimeToInclusion.filename(start_run_id, end_run_id)?)?; + + // make txGasUsed chart + let tx_gas_used = TxGasUsedChart::new(&cache_data.traces); + tx_gas_used.draw(&ReportChartId::TxGasUsed.filename(start_run_id, end_run_id)?)?; + + // make pendingTxs chart + let pending_txs = PendingTxsChart::new(&all_txs); + pending_txs.draw(&ReportChartId::PendingTxs.filename(start_run_id, end_run_id)?)?; + + // compile report + let report_path = build_html_report(ReportMetadata { + scenario_name: scenario_title, + start_run_id, + end_run_id, + start_block: cache_data.blocks.first().unwrap().header.number, + end_block: cache_data.blocks.last().unwrap().header.number, + rpc_url: rpc_url.to_string(), + })?; + + // Open the report in the default web browser + webbrowser::open(&report_path)?; + + Ok(()) +} + +/// Saves RunTxs to `{data_dir}/reports/{id}.csv`. +fn save_csv_report(id: u64, txs: &[RunTx]) -> Result<(), Box> { + let report_dir = report_dir()?; + let out_path = format!("{report_dir}/{id}.csv"); + + println!("Exporting report for run #{:?} to {:?}", id, out_path); + let mut writer = WriterBuilder::new().has_headers(true).from_path(out_path)?; + write_run_txs(&mut writer, txs)?; + + Ok(()) +} diff --git a/crates/cli/src/commands/report/gen_html.rs b/crates/cli/src/commands/report/gen_html.rs index 4c4f2d0e..3d858436 100644 --- a/crates/cli/src/commands/report/gen_html.rs +++ b/crates/cli/src/commands/report/gen_html.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use super::{report_dir, ReportChartId}; +use crate::{commands::report::chart::ReportChartId, util::report_dir}; pub struct ReportMetadata { pub scenario_name: String, diff --git a/crates/cli/src/commands/report/mod.rs b/crates/cli/src/commands/report/mod.rs index 08c5e356..78bd29bb 100644 --- a/crates/cli/src/commands/report/mod.rs +++ b/crates/cli/src/commands/report/mod.rs @@ -1,144 +1,8 @@ mod block_trace; mod cache; mod chart; +mod command; mod gen_html; mod util; -use crate::util::{data_dir, write_run_txs}; -use alloy::network::AnyNetwork; -use alloy::providers::DynProvider; -use alloy::{providers::ProviderBuilder, transports::http::reqwest::Url}; -use block_trace::get_block_trace_data; -use cache::CacheFile; -use chart::{ - DrawableChart, GasPerBlockChart, HeatMapChart, PendingTxsChart, ReportChartId, - TimeToInclusionChart, TxGasUsedChart, -}; -use contender_core::db::{DbOps, RunTx}; -use csv::WriterBuilder; -use gen_html::{build_html_report, ReportMetadata}; -use std::str::FromStr; - -/// Returns the fully-qualified path to the report directory. -fn report_dir() -> Result> { - let path = format!("{}/reports", data_dir()?); - std::fs::create_dir_all(&path)?; - Ok(path) -} - -pub async fn report( - last_run_id: Option, - preceding_runs: u64, - db: &(impl DbOps + Clone + Send + Sync + 'static), - rpc_url: &str, -) -> Result<(), Box> { - let num_runs = db.num_runs()?; - - if num_runs == 0 { - println!("No runs found in the database. Exiting."); - return Ok(()); - } - - // if id is provided, check if it's valid - let end_run_id = if let Some(id) = last_run_id { - if id == 0 || id > num_runs { - // panic!("Invalid run ID: {}", id); - return Err(format!("Invalid run ID: {}", id).into()); - } - id - } else { - // get latest run - println!("No run ID provided. Using latest run ID: {}", num_runs); - num_runs - }; - - // collect CSV report for each run_id - let start_run_id = end_run_id - preceding_runs; - let mut all_txs = vec![]; - for id in start_run_id..=end_run_id { - let txs = db.get_run_txs(id)?; - all_txs.extend_from_slice(&txs); - save_csv_report(id, &txs)?; - } - - // get run data - let mut run_data = vec![]; - for id in start_run_id..=end_run_id { - let run = db.get_run(id)?; - if let Some(run) = run { - run_data.push(run); - } - } - // collect all unique scenario_name values from run_data - let scenario_names: Vec = run_data - .iter() - .map(|run| run.scenario_name.clone()) - .collect::>() - .iter() - .map(|v| { - // return only the filename without the path and extension - let re = regex::Regex::new(r".*/(.*)\.toml$").unwrap(); - re.replace(v, "$1").to_string() - }) - .collect(); - let scenario_title = scenario_names - .into_iter() - .reduce(|acc, v| format!("{}, {}", acc, v)) - .unwrap_or_default(); - - // get trace data for reports - let url = Url::from_str(rpc_url).expect("Invalid URL"); - let rpc_client = DynProvider::new(ProviderBuilder::new().network::().on_http(url)); - let (trace_data, blocks) = get_block_trace_data(&all_txs, &rpc_client).await?; - - // cache data to file - let cache_data = CacheFile::new(trace_data, blocks); - cache_data.save()?; - - // make heatmap - let heatmap = HeatMapChart::new(&cache_data.traces)?; - heatmap.draw(&ReportChartId::Heatmap.filename(start_run_id, end_run_id)?)?; - - // make gasPerBlock chart - let gas_per_block = GasPerBlockChart::new(&cache_data.blocks); - gas_per_block.draw(&ReportChartId::GasPerBlock.filename(start_run_id, end_run_id)?)?; - - // make timeToInclusion chart - let time_to_inclusion = TimeToInclusionChart::new(&all_txs); - time_to_inclusion.draw(&ReportChartId::TimeToInclusion.filename(start_run_id, end_run_id)?)?; - - // make txGasUsed chart - let tx_gas_used = TxGasUsedChart::new(&cache_data.traces); - tx_gas_used.draw(&ReportChartId::TxGasUsed.filename(start_run_id, end_run_id)?)?; - - // make pendingTxs chart - let pending_txs = PendingTxsChart::new(&all_txs); - pending_txs.draw(&ReportChartId::PendingTxs.filename(start_run_id, end_run_id)?)?; - - // compile report - let report_path = build_html_report(ReportMetadata { - scenario_name: scenario_title, - start_run_id, - end_run_id, - start_block: cache_data.blocks.first().unwrap().header.number, - end_block: cache_data.blocks.last().unwrap().header.number, - rpc_url: rpc_url.to_string(), - })?; - - // Open the report in the default web browser - webbrowser::open(&report_path)?; - - Ok(()) -} - -/// Saves RunTxs to `{data_dir}/reports/{id}.csv`. -fn save_csv_report(id: u64, txs: &[RunTx]) -> Result<(), Box> { - let report_dir = report_dir()?; - let out_path = format!("{report_dir}/{id}.csv"); - - println!("Exporting report for run #{:?} to {:?}", id, out_path); - let mut writer = WriterBuilder::new().has_headers(true).from_path(out_path)?; - write_run_txs(&mut writer, txs)?; - - Ok(()) -} +pub use command::report; diff --git a/crates/cli/src/commands/spam.rs b/crates/cli/src/commands/spam.rs index b01ad394..ae2ff4e4 100644 --- a/crates/cli/src/commands/spam.rs +++ b/crates/cli/src/commands/spam.rs @@ -1,5 +1,8 @@ -use std::sync::Arc; - +use super::common::{ScenarioSendTxsCliArgs, SendSpamCliArgs}; +use crate::util::{ + check_private_keys, fund_accounts, get_signers_with_defaults, spam_callback_default, + SpamCallbackType, +}; use alloy::{ consensus::TxType, network::AnyNetwork, @@ -7,29 +10,20 @@ use alloy::{ utils::{format_ether, parse_ether}, U256, }, - providers::{DynProvider, Provider, ProviderBuilder}, + providers::{DynProvider, ProviderBuilder}, transports::http::reqwest::Url, }; use contender_core::{ agent_controller::AgentStore, db::DbOps, error::ContenderError, - generator::{ - seeder::Seeder, templater::Templater, types::AnyProvider, Generator, PlanConfig, PlanType, - RandSeed, - }, - spammer::{BlockwiseSpammer, ExecutionPayload, Spammer, TimedSpammer}, + generator::{seeder::Seeder, templater::Templater, types::AnyProvider, PlanConfig, RandSeed}, + spammer::{BlockwiseSpammer, Spammer, TimedSpammer}, test_scenario::{TestScenario, TestScenarioParams}, }; use contender_sqlite::SqliteDb; use contender_testfile::TestConfig; - -use crate::util::{ - check_private_keys, fund_accounts, get_signers_with_defaults, spam_callback_default, - SpamCallbackType, -}; - -use super::common::{ScenarioSendTxsCliArgs, SendSpamCliArgs}; +use std::sync::Arc; #[derive(Debug)] pub struct SpamCommandArgs { @@ -200,7 +194,7 @@ async fn init_scenario( builder_rpc_url: builder_url .to_owned() .map(|url| Url::parse(&url).expect("Invalid builder URL")), - signers: user_signers, + signers: user_signers.to_owned(), agent_store: agents.to_owned(), tx_type: *tx_type, gas_price_percent_add: *gas_price_percent_add, @@ -209,8 +203,7 @@ async fn init_scenario( .await?; // don't multiply by TPS or TPB, because that number scales the number of accounts; this cost is per account - let total_cost = - U256::from(duration) * get_max_spam_cost(scenario.to_owned(), &rpc_client).await?; + let total_cost = U256::from(duration) * scenario.get_max_spam_cost(&user_signers).await?; if min_balance < U256::from(total_cost) { return Err(ContenderError::SpamError( "min_balance is not enough to cover the cost of the spam transactions", @@ -312,80 +305,3 @@ pub async fn spam< Ok(run_id) } - -/// Returns the maximum cost of a single spam transaction. -/// -/// We take `scenario` by value rather than by reference, because we call `prepare_tx_request` -/// and `prepare_spam` which will mutate the scenario (namely the internal nonce counter). -/// We're not going to run the transactions we generate here; we just want to see the cost of -/// our spam txs, so we can estimate how much the user should provide for `min_balance`. -async fn get_max_spam_cost< - D: DbOps + Send + Sync + 'static, - S: Seeder + Send + Sync + Clone, - P: PlanConfig + Templater + Send + Sync + Clone, ->( - scenario: TestScenario, - rpc_client: &AnyProvider, -) -> Result> { - let mut scenario = scenario; - - // load a sample of each spam tx from the scenario - let sample_txs = scenario - .prepare_spam( - &scenario - .load_txs(PlanType::Spam( - scenario - .config - .get_spam_steps() - .map(|s| s.len()) // take the number of spam txs from the testfile - .unwrap_or(0), - |_named_req| { - // we can look at the named request here if needed - Ok(None) - }, - )) - .await?, - ) - .await? - .iter() - .map(|ex_payload| match ex_payload { - ExecutionPayload::SignedTx(_envelope, tx_req) => vec![tx_req.to_owned()], - ExecutionPayload::SignedTxBundle(_envelopes, tx_reqs) => tx_reqs - .iter() - .map(|tx| Box::new(tx.to_owned())) - .collect::>(), - }) - .collect::>() - .concat(); - - let gas_price = rpc_client.get_gas_price().await?; - - // get gas limit for each tx - let mut prepared_sample_txs = vec![]; - for tx in sample_txs { - let tx_req = tx.tx; - let (prepared_req, _signer) = scenario.prepare_tx_request(&tx_req, gas_price).await?; - - prepared_sample_txs.push(prepared_req); - } - - // get the highest gas cost of all spam txs - let highest_gas_cost = prepared_sample_txs - .iter() - .map(|tx| { - let mut gas_price = tx.max_fee_per_gas.unwrap_or(tx.gas_price.unwrap_or(0)); - if let Some(priority_fee) = tx.max_priority_fee_per_gas { - gas_price += priority_fee; - } - println!("gas_price={:?}", gas_price); - U256::from(gas_price * tx.gas.unwrap_or(0) as u128) + tx.value.unwrap_or(U256::ZERO) - }) - .max() - .ok_or(ContenderError::SpamError( - "failed to get max gas cost for spam txs", - None, - ))?; - - // we assume the highest possible cost to minimize the chances of running out of ETH mid-test - Ok(highest_gas_cost) -} diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index 74a4b627..6d9449f1 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -331,6 +331,13 @@ pub fn data_dir() -> Result> { Ok(dir) } +/// Returns the fully-qualified path to the report directory. +pub fn report_dir() -> Result> { + let path = format!("{}/reports", data_dir()?); + std::fs::create_dir_all(&path)?; + Ok(path) +} + /// Returns path to default contender DB file. pub fn db_file() -> Result> { let data_path = data_dir()?; diff --git a/crates/core/src/test_scenario.rs b/crates/core/src/test_scenario.rs index d3d1fa76..fa0fa95a 100644 --- a/crates/core/src/test_scenario.rs +++ b/crates/core/src/test_scenario.rs @@ -757,6 +757,95 @@ where }, ) } + + /// Returns the maximum cost of a single spam transaction by creating a new scenario + /// and running estimateGas calls to estimate the cost of the spam transactions. + pub async fn get_max_spam_cost(&self, user_signers: &[PrivateKeySigner]) -> Result { + let mut scenario = TestScenario::new( + self.config.to_owned(), + self.db.clone(), + self.rand_seed.clone(), + TestScenarioParams { + rpc_url: self.rpc_url.clone(), + builder_rpc_url: self.builder_rpc_url.clone(), + signers: user_signers.to_owned(), + agent_store: self.agent_store.clone(), + tx_type: self.tx_type, + gas_price_percent_add: Some(self.gas_price_percent_add as u16), + }, + ) + .await?; + + // load a sample of each spam tx from the scenario + let sample_txs = scenario + .prepare_spam( + &scenario + .load_txs(PlanType::Spam( + scenario + .config + .get_spam_steps() + .map(|s| s.len()) // take the number of spam txs from the testfile + .unwrap_or(0), + |_named_req| { + // we can look at the named request here if needed + Ok(None) + }, + )) + .await?, + ) + .await? + .iter() + .map(|ex_payload| match ex_payload { + ExecutionPayload::SignedTx(_envelope, tx_req) => vec![tx_req.to_owned()], + ExecutionPayload::SignedTxBundle(_envelopes, tx_reqs) => tx_reqs + .iter() + .map(|tx| Box::new(tx.to_owned())) + .collect::>(), + }) + .collect::>() + .concat(); + + let gas_price = scenario + .rpc_client + .get_gas_price() + .await + .map_err(|e| ContenderError::with_err(e, "failed to get gas price"))?; + + // get gas limit for each tx + let mut prepared_sample_txs = vec![]; + for tx in sample_txs { + let tx_req = tx.tx; + let (prepared_req, _signer) = scenario.prepare_tx_request(&tx_req, gas_price).await?; + println!( + "tx_request gas={:?} gas_price={:?} ({:?}, {:?})", + prepared_req.gas, + prepared_req.gas_price, + prepared_req.max_fee_per_gas, + prepared_req.max_priority_fee_per_gas + ); + prepared_sample_txs.push(prepared_req); + } + + // get the highest gas cost of all spam txs + let highest_gas_cost = prepared_sample_txs + .iter() + .map(|tx| { + let mut gas_price = tx.max_fee_per_gas.unwrap_or(tx.gas_price.unwrap_or(0)); + if let Some(priority_fee) = tx.max_priority_fee_per_gas { + gas_price += priority_fee; + } + println!("gas_price={:?}", gas_price); + U256::from(gas_price * tx.gas.unwrap_or(0) as u128) + tx.value.unwrap_or(U256::ZERO) + }) + .max() + .ok_or(ContenderError::SpamError( + "failed to get max gas cost for spam txs", + None, + ))?; + + // we assume the highest possible cost to minimize the chances of running out of ETH mid-test + Ok(highest_gas_cost) + } } async fn sync_nonces( diff --git a/scenarios/simple.toml b/scenarios/simple.toml index a1b35039..a8e908c6 100644 --- a/scenarios/simple.toml +++ b/scenarios/simple.toml @@ -7,5 +7,5 @@ bytecode = "0x608060405234801561000f575f5ffd5b5060405180602001604052805f8152505f [spam.tx] to = "{SpamMe5}" from_pool = "spammers" -signature = "consumeGas(string memory method, uint256 iterations)" -args = ["hash_sha256", "5"] +signature = "callPrecompile(string memory method, uint256 iterations)" +args = ["ecMul", "5"]