Skip to content

Commit

Permalink
feat: make --gas-report JSON output compatible (foundry-rs#9063)
Browse files Browse the repository at this point in the history 
* add gas report generation in JSON

* skip junit for now

* add json formatted tests, trailing space and invalid formatting

* avoid redundant modifications for calls count

* replace existing tests with snapbox

* clean up snapbox tests

* merge in master

* calls -> frames

* use .is_jsonlines()
  • Loading branch information
@zerosnacks @rplusq
zerosnacks authored and rplusq committed Nov 29, 2024
1 parent b0525f7 commit f572a51
Show file tree
Hide file tree
Showing 3 changed files with 367 additions and 270 deletions.
71 changes: 42 additions & 29 deletions crates/forge/bin/cmd/test/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use clap::{Parser, ValueHint};
use eyre::{Context, OptionExt, Result};
use forge::{
decode::decode_console_logs,
gas_report::GasReport,
gas_report::{GasReport, GasReportKind},
multi_runner::matches_contract,
result::{SuiteResult, TestOutcome, TestStatus},
traces::{
Expand Down Expand Up @@ -112,7 +112,7 @@ pub struct TestArgs {
json: bool,

/// Output test results as JUnit XML report.
#[arg(long, conflicts_with = "json", help_heading = "Display options")]
#[arg(long, conflicts_with_all(["json", "gas_report"]), help_heading = "Display options")]
junit: bool,

/// Stop running tests after the first failure.
Expand Down Expand Up @@ -474,6 +474,9 @@ impl TestArgs {

trace!(target: "forge::test", "running all tests");

// If we need to render to a serialized format, we should not print anything else to stdout.
let silent = self.gas_report && self.json;

let num_filtered = runner.matching_test_functions(filter).count();
if num_filtered != 1 && (self.debug.is_some() || self.flamegraph || self.flamechart) {
let action = if self.flamegraph {
Expand All @@ -500,7 +503,7 @@ impl TestArgs {
}

// Run tests in a non-streaming fashion and collect results for serialization.
if self.json {
if !self.gas_report && self.json {
let mut results = runner.test_collect(filter);
results.values_mut().for_each(|suite_result| {
for test_result in suite_result.test_results.values_mut() {
Expand Down Expand Up @@ -565,9 +568,13 @@ impl TestArgs {
}
let mut decoder = builder.build();

let mut gas_report = self
.gas_report
.then(|| GasReport::new(config.gas_reports.clone(), config.gas_reports_ignore.clone()));
let mut gas_report = self.gas_report.then(|| {
GasReport::new(
config.gas_reports.clone(),
config.gas_reports_ignore.clone(),
if self.json { GasReportKind::JSON } else { GasReportKind::Markdown },
)
});

let mut gas_snapshots = BTreeMap::<String, BTreeMap<String, String>>::new();

Expand All @@ -588,30 +595,34 @@ impl TestArgs {
self.flamechart;

// Print suite header.
println!();
for warning in suite_result.warnings.iter() {
eprintln!("{} {warning}", "Warning:".yellow().bold());
}
if !tests.is_empty() {
let len = tests.len();
let tests = if len > 1 { "tests" } else { "test" };
println!("Ran {len} {tests} for {contract_name}");
if !silent {
println!();
for warning in suite_result.warnings.iter() {
eprintln!("{} {warning}", "Warning:".yellow().bold());
}
if !tests.is_empty() {
let len = tests.len();
let tests = if len > 1 { "tests" } else { "test" };
println!("Ran {len} {tests} for {contract_name}");
}
}

// Process individual test results, printing logs and traces when necessary.
for (name, result) in tests {
shell::println(result.short_result(name))?;

// We only display logs at level 2 and above
if verbosity >= 2 {
// We only decode logs from Hardhat and DS-style console events
let console_logs = decode_console_logs(&result.logs);
if !console_logs.is_empty() {
println!("Logs:");
for log in console_logs {
println!(" {log}");
if !silent {
shell::println(result.short_result(name))?;

// We only display logs at level 2 and above
if verbosity >= 2 {
// We only decode logs from Hardhat and DS-style console events
let console_logs = decode_console_logs(&result.logs);
if !console_logs.is_empty() {
println!("Logs:");
for log in console_logs {
println!(" {log}");
}
println!();
}
println!();
}
}

Expand Down Expand Up @@ -653,7 +664,7 @@ impl TestArgs {
}
}

if !decoded_traces.is_empty() {
if !silent && !decoded_traces.is_empty() {
shell::println("Traces:")?;
for trace in &decoded_traces {
shell::println(trace)?;
Expand Down Expand Up @@ -760,7 +771,9 @@ impl TestArgs {
}

// Print suite summary.
shell::println(suite_result.summary())?;
if !silent {
shell::println(suite_result.summary())?;
}

// Add the suite result to the outcome.
outcome.results.insert(contract_name, suite_result);
Expand All @@ -781,7 +794,7 @@ impl TestArgs {
outcome.gas_report = Some(finalized);
}

if !outcome.results.is_empty() {
if !silent && !outcome.results.is_empty() {
shell::println(outcome.summary(duration))?;

if self.summary {
Expand Down Expand Up @@ -1063,7 +1076,7 @@ contract FooBarTest is DSTest {
let call_cnts = gas_report
.contracts
.values()
.flat_map(|c| c.functions.values().flat_map(|f| f.values().map(|v| v.calls.len())))
.flat_map(|c| c.functions.values().flat_map(|f| f.values().map(|v| v.frames.len())))
.collect::<Vec<_>>();
// assert that all functions were called at least 100 times
assert!(call_cnts.iter().all(|c| *c > 100));
Expand Down
43 changes: 34 additions & 9 deletions crates/forge/src/gas_report.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,25 @@ use serde::{Deserialize, Serialize};
use std::{collections::BTreeMap, fmt::Display};
use yansi::Paint;

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub enum GasReportKind {
Markdown,
JSON,
}

impl Default for GasReportKind {
fn default() -> Self {
Self::Markdown
}
}

/// Represents the gas report for a set of contracts.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct GasReport {
/// Whether to report any contracts.
report_any: bool,
/// What kind of report to generate.
report_type: GasReportKind,
/// Contracts to generate the report for.
report_for: HashSet<String>,
/// Contracts to ignore when generating the report.
Expand All @@ -30,11 +44,13 @@ impl GasReport {
pub fn new(
report_for: impl IntoIterator<Item = String>,
ignore: impl IntoIterator<Item = String>,
report_kind: GasReportKind,
) -> Self {
let report_for = report_for.into_iter().collect::<HashSet<_>>();
let ignore = ignore.into_iter().collect::<HashSet<_>>();
let report_any = report_for.is_empty() || report_for.contains("*");
Self { report_any, report_for, ignore, ..Default::default() }
let report_type = report_kind;
Self { report_any, report_type, report_for, ignore, ..Default::default() }
}

/// Whether the given contract should be reported.
Expand Down Expand Up @@ -113,7 +129,7 @@ impl GasReport {
.or_default()
.entry(signature.clone())
.or_default();
gas_info.calls.push(trace.gas_used);
gas_info.frames.push(trace.gas_used);
}
}
}
Expand All @@ -125,11 +141,12 @@ impl GasReport {
for contract in self.contracts.values_mut() {
for sigs in contract.functions.values_mut() {
for func in sigs.values_mut() {
func.calls.sort_unstable();
func.min = func.calls.first().copied().unwrap_or_default();
func.max = func.calls.last().copied().unwrap_or_default();
func.mean = calc::mean(&func.calls);
func.median = calc::median_sorted(&func.calls);
func.frames.sort_unstable();
func.min = func.frames.first().copied().unwrap_or_default();
func.max = func.frames.last().copied().unwrap_or_default();
func.mean = calc::mean(&func.frames);
func.median = calc::median_sorted(&func.frames);
func.calls = func.frames.len() as u64;
}
}
}
Expand All @@ -145,6 +162,11 @@ impl Display for GasReport {
continue;
}

if self.report_type == GasReportKind::JSON {
writeln!(f, "{}", serde_json::to_string(&contract).unwrap())?;
continue;
}

let mut table = Table::new();
table.load_preset(ASCII_MARKDOWN);
table.set_header([Cell::new(format!("{name} contract"))
Expand Down Expand Up @@ -176,7 +198,7 @@ impl Display for GasReport {
Cell::new(gas_info.mean.to_string()).fg(Color::Yellow),
Cell::new(gas_info.median.to_string()).fg(Color::Yellow),
Cell::new(gas_info.max.to_string()).fg(Color::Red),
Cell::new(gas_info.calls.len().to_string()),
Cell::new(gas_info.calls.to_string()),
]);
})
});
Expand All @@ -197,9 +219,12 @@ pub struct ContractInfo {

#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct GasInfo {
pub calls: Vec<u64>,
pub calls: u64,
pub min: u64,
pub mean: u64,
pub median: u64,
pub max: u64,

#[serde(skip)]
pub frames: Vec<u64>,
}
Loading

0 comments on commit f572a51

Please sign in to comment.