use std::collections::HashMap;
use std::fs;
use std::path::PathBuf;
use serde::Serialize;
use inflector::Inflector;
use crate::BenchmarkCmd;
use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis, RegressionModel};
use sp_runtime::traits::Zero;
const VERSION: &'static str = env!("CARGO_PKG_VERSION");
const TEMPLATE: &str = include_str!("./template.hbs");
#[derive(Serialize, Default, Debug, Clone)]
struct TemplateData {
args: Vec<String>,
date: String,
version: String,
pallet: String,
instance: String,
header: String,
cmd: CmdData,
benchmarks: Vec<BenchmarkData>,
}
#[derive(Serialize, Default, Debug, Clone)]
struct BenchmarkData {
name: String,
components: Vec<Component>,
#[serde(serialize_with = "string_serialize")]
base_weight: u128,
#[serde(serialize_with = "string_serialize")]
base_reads: u128,
#[serde(serialize_with = "string_serialize")]
base_writes: u128,
component_weight: Vec<ComponentSlope>,
component_reads: Vec<ComponentSlope>,
component_writes: Vec<ComponentSlope>,
}
#[derive(Serialize, Default, Debug, Clone)]
struct CmdData {
steps: Vec<u32>,
repeat: u32,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
execution: String,
wasm_execution: String,
chain: String,
db_cache: u32,
}
#[derive(Serialize, Debug, Clone, Eq, PartialEq)]
struct Component {
name: String,
is_used: bool,
}
#[derive(Serialize, Debug, Clone, Eq, PartialEq)]
struct ComponentSlope {
name: String,
#[serde(serialize_with = "string_serialize")]
slope: u128,
#[serde(serialize_with = "string_serialize")]
error: u128,
}
fn io_error(s: &str) -> std::io::Error {
use std::io::{Error, ErrorKind};
Error::new(ErrorKind::Other, s)
}
fn map_results(batches: &[BenchmarkBatch]) -> Result<HashMap<(String, String), Vec<BenchmarkData>>, std::io::Error> {
if batches.is_empty() { return Err(io_error("empty batches")) }
let mut all_benchmarks = HashMap::new();
let mut pallet_benchmarks = Vec::new();
let mut batches_iter = batches.iter().peekable();
while let Some(batch) = batches_iter.next() {
if batch.results.is_empty() { continue }
let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
let instance_string = String::from_utf8(batch.instance.clone()).unwrap();
let benchmark_data = get_benchmark_data(batch);
pallet_benchmarks.push(benchmark_data);
if let Some(next) = batches_iter.peek() {
let next_pallet = String::from_utf8(next.pallet.clone()).unwrap();
let next_instance = String::from_utf8(next.instance.clone()).unwrap();
if next_pallet != pallet_string || next_instance != instance_string {
all_benchmarks.insert((pallet_string, instance_string), pallet_benchmarks.clone());
pallet_benchmarks = Vec::new();
}
} else {
all_benchmarks.insert((pallet_string, instance_string), pallet_benchmarks.clone());
}
}
Ok(all_benchmarks)
}
fn extract_errors(model: &Option<RegressionModel>) -> impl Iterator<Item=u128> + '_ {
let mut errors = model.as_ref().map(|m| m.se.regressor_values.iter());
std::iter::from_fn(move || {
match &mut errors {
Some(model) => model.next().map(|val| *val as u128),
_ => Some(0),
}
})
}
fn get_benchmark_data(batch: &BenchmarkBatch) -> BenchmarkData {
let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap();
let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap();
let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap();
let mut used_components = Vec::new();
let mut used_extrinsic_time = Vec::new();
let mut used_reads = Vec::new();
let mut used_writes = Vec::new();
extrinsic_time.slopes.into_iter()
.zip(extrinsic_time.names.iter())
.zip(extract_errors(&extrinsic_time.model))
.for_each(|((slope, name), error)| {
if !slope.is_zero() {
if !used_components.contains(&name) { used_components.push(name); }
used_extrinsic_time.push(ComponentSlope {
name: name.clone(),
slope: slope.saturating_mul(1000),
error: error.saturating_mul(1000),
});
}
});
reads.slopes.into_iter()
.zip(reads.names.iter())
.zip(extract_errors(&reads.model))
.for_each(|((slope, name), error)| {
if !slope.is_zero() {
if !used_components.contains(&name) { used_components.push(name); }
used_reads.push(ComponentSlope {
name: name.clone(),
slope,
error,
});
}
});
writes.slopes.into_iter()
.zip(writes.names.iter())
.zip(extract_errors(&writes.model))
.for_each(|((slope, name), error)| {
if !slope.is_zero() {
if !used_components.contains(&name) { used_components.push(name); }
used_writes.push(ComponentSlope {
name: name.clone(),
slope,
error,
});
}
});
let components = batch.results[0].components
.iter()
.map(|(name, _)| -> Component {
let name_string = name.to_string();
let is_used = used_components.contains(&&name_string);
Component { name: name_string, is_used }
})
.collect::<Vec<_>>();
BenchmarkData {
name: String::from_utf8(batch.benchmark.clone()).unwrap(),
components,
base_weight: extrinsic_time.base.saturating_mul(1000),
base_reads: reads.base,
base_writes: writes.base,
component_weight: used_extrinsic_time,
component_reads: used_reads,
component_writes: used_writes,
}
}
pub fn write_results(
batches: &[BenchmarkBatch],
path: &PathBuf,
cmd: &BenchmarkCmd,
) -> Result<(), std::io::Error> {
let template: String = match &cmd.template {
Some(template_file) => {
fs::read_to_string(template_file)?
},
None => {
TEMPLATE.to_string()
},
};
let header_text = match &cmd.header {
Some(header_file) => {
let text = fs::read_to_string(header_file)?;
text
},
None => String::new(),
};
let date = chrono::Utc::now().format("%Y-%m-%d").to_string();
let args = std::env::args().collect::<Vec<String>>();
let cmd_data = CmdData {
steps: cmd.steps.clone(),
repeat: cmd.repeat.clone(),
lowest_range_values: cmd.lowest_range_values.clone(),
highest_range_values: cmd.highest_range_values.clone(),
execution: format!("{:?}", cmd.execution),
wasm_execution: cmd.wasm_method.to_string(),
chain: format!("{:?}", cmd.shared_params.chain),
db_cache: cmd.database_cache_size,
};
let mut handlebars = handlebars::Handlebars::new();
handlebars.register_helper("underscore", Box::new(UnderscoreHelper));
handlebars.register_helper("join", Box::new(JoinHelper));
handlebars.register_escape_fn(|s| -> String { s.to_string() });
let all_results = map_results(batches)?;
for ((pallet, instance), results) in all_results.iter() {
let mut file_path = path.clone();
if file_path.is_dir() {
if all_results.keys().any(|(p, i)| p == pallet && i != instance) {
file_path.push(pallet.clone() + "_" + &instance.to_snake_case());
} else {
file_path.push(pallet.clone());
}
file_path.set_extension("rs");
}
let hbs_data = TemplateData {
args: args.clone(),
date: date.clone(),
version: VERSION.to_string(),
pallet: pallet.to_string(),
instance: instance.to_string(),
header: header_text.clone(),
cmd: cmd_data.clone(),
benchmarks: results.clone(),
};
let mut output_file = fs::File::create(file_path)?;
handlebars.render_template_to_write(&template, &hbs_data, &mut output_file)
.map_err(|e| io_error(&e.to_string()))?;
}
Ok(())
}
fn underscore<Number>(i: Number) -> String
where Number: std::string::ToString
{
let mut s = String::new();
let i_str = i.to_string();
let a = i_str.chars().rev().enumerate();
for (idx, val) in a {
if idx != 0 && idx % 3 == 0 {
s.insert(0, '_');
}
s.insert(0, val);
}
s
}
#[derive(Clone, Copy)]
struct UnderscoreHelper;
impl handlebars::HelperDef for UnderscoreHelper {
fn call<'reg: 'rc, 'rc>(
&self, h: &handlebars::Helper,
_: &handlebars::Handlebars,
_: &handlebars::Context,
_rc: &mut handlebars::RenderContext,
out: &mut dyn handlebars::Output
) -> handlebars::HelperResult {
use handlebars::JsonRender;
let param = h.param(0).unwrap();
let underscore_param = underscore(param.value().render());
out.write(&underscore_param)?;
Ok(())
}
}
#[derive(Clone, Copy)]
struct JoinHelper;
impl handlebars::HelperDef for JoinHelper {
fn call<'reg: 'rc, 'rc>(
&self, h: &handlebars::Helper,
_: &handlebars::Handlebars,
_: &handlebars::Context,
_rc: &mut handlebars::RenderContext,
out: &mut dyn handlebars::Output
) -> handlebars::HelperResult {
use handlebars::JsonRender;
let param = h.param(0).unwrap();
let value = param.value();
let joined = if value.is_array() {
value.as_array().unwrap()
.iter()
.map(|v| v.render())
.collect::<Vec<String>>()
.join(" ")
} else {
value.render()
};
out.write(&joined)?;
Ok(())
}
}
fn string_serialize<S>(x: &u128, s: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
s.serialize_str(&x.to_string())
}
#[cfg(test)]
mod test {
use super::*;
use frame_benchmarking::{BenchmarkBatch, BenchmarkParameter, BenchmarkResults};
fn test_data(pallet: &[u8], benchmark: &[u8], param: BenchmarkParameter, base: u32, slope: u32) -> BenchmarkBatch {
let mut results = Vec::new();
for i in 0 .. 5 {
results.push(
BenchmarkResults {
components: vec![(param, i), (BenchmarkParameter::z, 0)],
extrinsic_time: (base + slope * i).into(),
storage_root_time: (base + slope * i).into(),
reads: (base + slope * i).into(),
repeat_reads: 0,
writes: (base + slope * i).into(),
repeat_writes: 0,
}
)
}
return BenchmarkBatch {
pallet: [pallet.to_vec(), b"_pallet".to_vec()].concat(),
instance: b"instance".to_vec(),
benchmark: [benchmark.to_vec(), b"_benchmark".to_vec()].concat(),
results,
}
}
fn check_data(benchmark: &BenchmarkData, component: &str, base: u128, slope: u128) {
assert_eq!(
benchmark.components,
vec![
Component { name: component.to_string(), is_used: true },
Component { name: "z".to_string(), is_used: false},
],
);
assert_eq!(benchmark.base_weight, base * 1_000);
assert_eq!(
benchmark.component_weight,
vec![ComponentSlope {
name: component.to_string(),
slope: slope * 1_000,
error: 0,
}]
);
assert_eq!(benchmark.base_reads, base);
assert_eq!(
benchmark.component_reads,
vec![ComponentSlope {
name: component.to_string(),
slope,
error: 0,
}]
);
assert_eq!(benchmark.base_writes, base);
assert_eq!(
benchmark.component_writes,
vec![ComponentSlope {
name: component.to_string(),
slope,
error: 0,
}]
);
}
#[test]
fn map_results_works() {
let mapped_results = map_results(&[
test_data(b"first", b"first", BenchmarkParameter::a, 10, 3),
test_data(b"first", b"second", BenchmarkParameter::b, 9, 2),
test_data(b"second", b"first", BenchmarkParameter::c, 3, 4),
]).unwrap();
let first_benchmark = &mapped_results.get(
&("first_pallet".to_string(), "instance".to_string())
).unwrap()[0];
assert_eq!(first_benchmark.name, "first_benchmark");
check_data(first_benchmark, "a", 10, 3);
let second_benchmark = &mapped_results.get(
&("first_pallet".to_string(), "instance".to_string())
).unwrap()[1];
assert_eq!(second_benchmark.name, "second_benchmark");
check_data(second_benchmark, "b", 9, 2);
let second_pallet_benchmark = &mapped_results.get(
&("second_pallet".to_string(), "instance".to_string())
).unwrap()[0];
assert_eq!(second_pallet_benchmark.name, "first_benchmark");
check_data(second_pallet_benchmark, "c", 3, 4);
}
}