From f26867e56561470df417dac38eccda5a85b55913 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 31 Jul 2024 17:16:52 +0200 Subject: [PATCH 1/6] feat: implement baseline comparison --- src/config.rs | 20 ++ src/goose.rs | 2 +- src/lib.rs | 12 +- src/metrics.rs | 85 +++--- src/metrics/common.rs | 611 ++++++++++++++++++++++++++++------------- src/metrics/delta.rs | 171 ++++++++++++ src/report.rs | 212 +++++++++----- src/report/markdown.rs | 16 +- 8 files changed, 808 insertions(+), 321 deletions(-) create mode 100644 src/metrics/delta.rs diff --git a/src/config.rs b/src/config.rs index 59a5920e..5424dcf3 100644 --- a/src/config.rs +++ b/src/config.rs @@ -103,6 +103,9 @@ pub struct GooseConfiguration { /// Create reports, can be used multiple times (supports .html, .htm, .md, .json) #[options(no_short, meta = "NAME")] pub report_file: Vec, + /// An optional baseline, for rendering the report + #[options(no_short, meta = "NAME")] + pub baseline_file: Option, /// Disable granular graphs in report file #[options(no_short)] pub no_granular_report: bool, @@ -283,6 +286,8 @@ pub(crate) struct GooseDefaults { pub no_error_summary: Option, /// An optional default for the html-formatted report file name. pub report_file: Option>, + /// An optional baseline file for the reports. + pub baseline_file: Option, /// An optional default for the flag that disables granular data in HTML report graphs. pub no_granular_report: Option, /// An optional default for the requests log file name. @@ -1598,6 +1603,21 @@ impl GooseConfiguration { ]) .unwrap_or_default(); + self.baseline_file = self.get_value(vec![ + // Use --baseline-file if set. + GooseValue { + value: self.baseline_file.clone(), + filter: self.baseline_file.is_none(), + message: "baseline_file", + }, + // Otherwise, use GooseDefault if set. + GooseValue { + value: defaults.baseline_file.clone(), + filter: defaults.baseline_file.is_none(), + message: "baseline_file", + }, + ]); + // Configure `no_granular_report`. self.no_debug_body = self .get_value(vec![ diff --git a/src/goose.rs b/src/goose.rs index e785b1a4..a346cfdc 100644 --- a/src/goose.rs +++ b/src/goose.rs @@ -670,7 +670,7 @@ pub enum GooseUserCommand { } /// Supported HTTP methods. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Ord, PartialOrd)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Ord, PartialOrd, Hash)] pub enum GooseMethod { Delete, Get, diff --git a/src/lib.rs b/src/lib.rs index bb4363de..bcd061ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,7 +67,7 @@ use crate::controller::{ControllerProtocol, ControllerRequest}; use crate::goose::{GooseUser, GooseUserCommand, Scenario, Transaction}; use crate::graph::GraphData; use crate::logger::{GooseLoggerJoinHandle, GooseLoggerTx}; -use crate::metrics::{GooseMetric, GooseMetrics}; +use crate::metrics::{load_baseline_file, GooseMetric, GooseMetrics}; use crate::test_plan::{TestPlan, TestPlanHistory, TestPlanStepAction}; /// Constant defining Goose's default telnet Controller port. @@ -1722,6 +1722,16 @@ impl GooseAttack { goose_attack_run_state.throttle_threads_tx = throttle_threads_tx; goose_attack_run_state.parent_to_throttle_tx = parent_to_throttle_tx; + // If enabled, try loading the baseline + if let Some(baseline_file) = &self.configuration.baseline_file { + let _data = + load_baseline_file(baseline_file).map_err(|err| GooseError::InvalidOption { + option: "--baseline-file".to_string(), + value: baseline_file.to_string(), + detail: err.to_string(), + }); + } + // Try to create the requested report files, to confirm access. self.create_reports().await?; diff --git a/src/metrics.rs b/src/metrics.rs index fe8313a5..77966b01 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -9,8 +9,10 @@ //! [`GooseErrorMetrics`] are displayed in tables. mod common; +mod delta; -pub(crate) use common::ReportData; +pub(crate) use common::{load_baseline_file, ReportData}; +pub(crate) use delta::*; use crate::config::GooseDefaults; use crate::goose::{get_base_url, GooseMethod, Scenario}; @@ -25,8 +27,7 @@ use itertools::Itertools; use num_format::{Locale, ToFormattedString}; use regex::RegexSet; use reqwest::StatusCode; -use serde::ser::SerializeStruct; -use serde::{Deserialize, Serialize, Serializer}; +use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::collections::{BTreeMap, HashMap, HashSet}; use std::ffi::OsStr; @@ -1026,12 +1027,13 @@ impl ScenarioMetricAggregate { /// Ok(()) /// } /// ``` -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] pub struct GooseMetrics { /// A hash of the load test, primarily used to validate all Workers in a Gaggle /// are running the same load test. pub hash: u64, /// A vector recording the history of each load test step. + #[serde(skip)] pub history: Vec, /// Total number of seconds the load test ran. pub duration: usize, @@ -2564,27 +2566,6 @@ impl GooseMetrics { } } -impl Serialize for GooseMetrics { - // GooseMetrics serialization can't be derived because of the started field. - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut s = serializer.serialize_struct("GooseMetrics", 10)?; - s.serialize_field("hash", &self.hash)?; - s.serialize_field("duration", &self.duration)?; - s.serialize_field("maximum_users", &self.maximum_users)?; - s.serialize_field("total_users", &self.total_users)?; - s.serialize_field("requests", &self.requests)?; - s.serialize_field("transactions", &self.transactions)?; - s.serialize_field("errors", &self.errors)?; - s.serialize_field("final_metrics", &self.final_metrics)?; - s.serialize_field("display_status_codes", &self.display_status_codes)?; - s.serialize_field("display_metrics", &self.display_metrics)?; - s.end() - } -} - /// Implement format trait to allow displaying metrics. impl fmt::Display for GooseMetrics { // Implement display of metrics with `{}` marker. @@ -2671,6 +2652,7 @@ pub struct GooseErrorMetricAggregate { /// A counter reflecting how many times this error occurred. pub occurrences: usize, } + impl GooseErrorMetricAggregate { pub(crate) fn new(method: GooseMethod, name: String, error: String) -> Self { GooseErrorMetricAggregate { @@ -2790,11 +2772,9 @@ impl GooseAttack { let key = format!("{} {}", request_metric.raw.method, request_metric.name); let mut merge_request = match self.metrics.requests.get(&key) { Some(m) => m.clone(), - None => GooseRequestMetricAggregate::new( - &request_metric.name, - request_metric.raw.method.clone(), - 0, - ), + None => { + GooseRequestMetricAggregate::new(&request_metric.name, request_metric.raw.method, 0) + } }; // Handle a metrics update. @@ -2984,7 +2964,7 @@ impl GooseAttack { Some(m) => m.clone(), // First time we've seen this error. None => GooseErrorMetricAggregate::new( - raw_request.raw.method.clone(), + raw_request.raw.method, raw_request.name.to_string(), raw_request.error.to_string(), ), @@ -2996,11 +2976,10 @@ impl GooseAttack { // Update metrics showing how long the load test has been running. // 1.2 seconds will round down to 1 second. 1.6 seconds will round up to 2 seconds. pub(crate) fn update_duration(&mut self) { - self.metrics.duration = if self.started.is_some() { - self.started.unwrap().elapsed().as_secs_f32().round() as usize - } else { - 0 - }; + self.metrics.duration = self + .started + .map(|started| started.elapsed().as_secs_f32().round() as usize) + .unwrap_or_default(); } /// Process all requested reports. @@ -3018,25 +2997,32 @@ impl GooseAttack { }) }; + let baseline = self + .configuration + .baseline_file + .as_ref() + .map(load_baseline_file) + .transpose()?; + for report in &self.configuration.report_file { let path = PathBuf::from(report); match path.extension().map(OsStr::to_string_lossy).as_deref() { Some("html" | "htm") => { let file = create(path).await?; if write { - self.write_html_report(file, report).await?; + self.write_html_report(file, &baseline, report).await?; } } Some("json") => { let file = create(path).await?; if write { - self.write_json_report(file).await?; + self.write_json_report(file, &baseline).await?; } } Some("md") => { let file = create(path).await?; if write { - self.write_markdown_report(file).await?; + self.write_markdown_report(file, &baseline).await?; } } None => { @@ -3070,7 +3056,11 @@ impl GooseAttack { } /// Write a JSON report. - pub(crate) async fn write_json_report(&self, report_file: File) -> Result<(), GooseError> { + pub(crate) async fn write_json_report( + &self, + report_file: File, + baseline: &Option>, + ) -> Result<(), GooseError> { let data = common::prepare_data( ReportOptions { no_transaction_metrics: self.configuration.no_transaction_metrics, @@ -3078,6 +3068,7 @@ impl GooseAttack { no_status_codes: self.configuration.no_status_codes, }, &self.metrics, + baseline, ); serde_json::to_writer_pretty(BufWriter::new(report_file.into_std().await), &data)?; @@ -3086,7 +3077,11 @@ impl GooseAttack { } /// Write a Markdown report. - pub(crate) async fn write_markdown_report(&self, report_file: File) -> Result<(), GooseError> { + pub(crate) async fn write_markdown_report( + &self, + report_file: File, + baseline: &Option>, + ) -> Result<(), GooseError> { let data = common::prepare_data( ReportOptions { no_transaction_metrics: self.configuration.no_transaction_metrics, @@ -3094,6 +3089,7 @@ impl GooseAttack { no_status_codes: self.configuration.no_status_codes, }, &self.metrics, + baseline, ); report::write_markdown_report(&mut BufWriter::new(report_file.into_std().await), data) @@ -3103,6 +3099,7 @@ impl GooseAttack { pub(crate) async fn write_html_report( &self, mut report_file: File, + baseline: &Option>, path: &str, ) -> Result<(), GooseError> { // Only write the report if enabled. @@ -3195,6 +3192,7 @@ impl GooseAttack { no_status_codes: self.configuration.no_status_codes, }, &self.metrics, + baseline, ); // Compile the request metrics template. @@ -3265,7 +3263,10 @@ impl GooseAttack { let errors_template = errors .map(|errors| { - let error_rows = errors.into_iter().map(report::error_row).join("\n"); + let error_rows = errors + .into_iter() + .map(|error| report::error_row(&error)) + .join("\n"); report::errors_template( &error_rows, diff --git a/src/metrics/common.rs b/src/metrics/common.rs index b337c111..766d5257 100644 --- a/src/metrics/common.rs +++ b/src/metrics/common.rs @@ -1,20 +1,27 @@ use super::{ - merge_times, per_second_calculations, prepare_status_codes, report, update_max_time, - update_min_time, GooseErrorMetricAggregate, GooseMetrics, + delta::*, merge_times, per_second_calculations, prepare_status_codes, update_max_time, + update_min_time, GooseMetrics, }; +use crate::report::ErrorMetric; use crate::{ report::{ - CORequestMetric, RequestMetric, ResponseMetric, ScenarioMetric, StatusCodeMetric, - TransactionMetric, + get_response_metric, CORequestMetric, RequestMetric, ResponseMetric, ScenarioMetric, + StatusCodeMetric, TransactionMetric, }, - util, + util, GooseError, }; use itertools::Itertools; +use std::borrow::Cow; use std::collections::{BTreeMap, HashMap}; +use std::fmt::Debug; +use std::fs::File; +use std::hash::Hash; +use std::io::BufReader; +use std::path::Path; -#[derive(Debug, serde::Serialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] pub(crate) struct ReportData<'m> { - pub raw_metrics: &'m GooseMetrics, + pub raw_metrics: Cow<'m, GooseMetrics>, pub raw_request_metrics: Vec, pub raw_response_metrics: Vec, @@ -27,7 +34,7 @@ pub(crate) struct ReportData<'m> { pub status_code_metrics: Option>, - pub errors: Option>, + pub errors: Option>, } pub struct ReportOptions { @@ -36,111 +43,199 @@ pub struct ReportOptions { pub no_status_codes: bool, } -pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportData { - // Prepare requests and responses variables. - let mut raw_request_metrics = Vec::new(); - let mut co_request_metrics = Vec::new(); - let mut raw_response_metrics = Vec::new(); - let mut co_response_metrics = Vec::new(); - let mut raw_aggregate_total_count = 0; - let mut co_aggregate_total_count = 0; - let mut raw_aggregate_fail_count = 0; - let mut raw_aggregate_response_time_counter: usize = 0; - let mut raw_aggregate_response_time_minimum: usize = 0; - let mut raw_aggregate_response_time_maximum: usize = 0; - let mut raw_aggregate_response_times: BTreeMap = BTreeMap::new(); - let mut co_aggregate_response_time_counter: usize = 0; - let mut co_aggregate_response_time_maximum: usize = 0; - let mut co_aggregate_response_times: BTreeMap = BTreeMap::new(); - let mut co_data = false; - - for (request_key, request) in metrics.requests.iter().sorted() { +struct RawIntermediate { + raw_aggregate_response_time_counter: usize, + raw_aggregate_response_time_minimum: usize, + raw_aggregate_total_count: usize, +} + +struct Prepare<'m, 'b> { + options: ReportOptions, + metrics: &'m GooseMetrics, + baseline: &'b Option>, + + co_data: bool, +} + +impl<'m, 'b> Prepare<'m, 'b> { + fn new( + options: ReportOptions, + metrics: &'m GooseMetrics, + baseline: &'b Option>, + ) -> Self { + Self { + options, + metrics, + baseline, + co_data: false, + } + } + + fn build(mut self) -> ReportData<'m> { // Determine whether or not to include Coordinated Omission data. - if !co_data && request.coordinated_omission_data.is_some() { - co_data = true; + self.co_data = self + .metrics + .requests + .values() + .any(|request| request.coordinated_omission_data.is_some()); + + let (raw_request_metrics, raw_response_metrics, intermediate) = self.build_raw(); + let (co_request_metrics, co_response_metrics) = self.build_co(&intermediate); + + ReportData { + raw_metrics: Cow::Borrowed(self.metrics), + raw_request_metrics, + raw_response_metrics, + co_request_metrics, + co_response_metrics, + scenario_metrics: self.build_scenario(), + transaction_metrics: self.build_transaction(&intermediate), + status_code_metrics: self.build_status_code(), + errors: self.build_errors(), } - let method = format!("{}", request.method); - // The request_key is "{method} {name}", so by stripping the "{method} " - // prefix we get the name. - let name = request_key - .strip_prefix(&format!("{} ", request.method)) - .unwrap() - .to_string(); - let total_request_count = request.success_count + request.fail_count; - let (requests_per_second, failures_per_second) = - per_second_calculations(metrics.duration, total_request_count, request.fail_count); - // Prepare per-request metrics. - raw_request_metrics.push(report::RequestMetric { - method: method.to_string(), - name: name.to_string(), - number_of_requests: total_request_count, - number_of_failures: request.fail_count, - response_time_average: request.raw_data.total_time as f32 - / request.raw_data.counter as f32, - response_time_minimum: request.raw_data.minimum_time, - response_time_maximum: request.raw_data.maximum_time, - requests_per_second, - failures_per_second, - }); + } - // Prepare per-response metrics. - raw_response_metrics.push(report::get_response_metric( - &method, - &name, - &request.raw_data.times, - request.raw_data.counter, - request.raw_data.minimum_time, - request.raw_data.maximum_time, - )); + fn build_raw(&self) -> (Vec, Vec, RawIntermediate) { + // Prepare requests and responses variables. + let mut raw_request_metrics = vec![]; + let mut raw_response_metrics = vec![]; + let mut raw_aggregate_total_count = 0; + let mut raw_aggregate_fail_count = 0; + let mut raw_aggregate_response_time_counter: usize = 0; + let mut raw_aggregate_response_time_minimum: usize = 0; + let mut raw_aggregate_response_time_maximum: usize = 0; + let mut raw_aggregate_response_times: BTreeMap = BTreeMap::new(); + + for (request_key, request) in self.metrics.requests.iter().sorted() { + let method = format!("{}", request.method); + // The request_key is "{method} {name}", so by stripping the "{method} " + // prefix we get the name. + let name = request_key + .strip_prefix(&format!("{} ", request.method)) + .unwrap() + .to_string(); + let total_request_count = request.success_count + request.fail_count; + let (requests_per_second, failures_per_second) = per_second_calculations( + self.metrics.duration, + total_request_count, + request.fail_count, + ); + // Prepare per-request metrics. + raw_request_metrics.push(RequestMetric { + method: method.to_string(), + name: name.to_string(), + number_of_requests: total_request_count.into(), + number_of_failures: request.fail_count.into(), + response_time_average: (request.raw_data.total_time as f32 + / request.raw_data.counter as f32) + .into(), + response_time_minimum: request.raw_data.minimum_time.into(), + response_time_maximum: request.raw_data.maximum_time.into(), + requests_per_second: requests_per_second.into(), + failures_per_second: failures_per_second.into(), + }); + + // Prepare per-response metrics. + raw_response_metrics.push(get_response_metric( + &method, + &name, + &request.raw_data.times, + request.raw_data.counter, + request.raw_data.minimum_time, + request.raw_data.maximum_time, + )); + + // Collect aggregated request and response metrics. + raw_aggregate_total_count += total_request_count; + raw_aggregate_fail_count += request.fail_count; + raw_aggregate_response_time_counter += request.raw_data.total_time; + raw_aggregate_response_time_minimum = update_min_time( + raw_aggregate_response_time_minimum, + request.raw_data.minimum_time, + ); + raw_aggregate_response_time_maximum = update_max_time( + raw_aggregate_response_time_maximum, + request.raw_data.maximum_time, + ); + raw_aggregate_response_times = + merge_times(raw_aggregate_response_times, request.raw_data.times.clone()); + } + + // Prepare aggregate per-request metrics. + let (raw_aggregate_requests_per_second, raw_aggregate_failures_per_second) = + per_second_calculations( + self.metrics.duration, + raw_aggregate_total_count, + raw_aggregate_fail_count, + ); + raw_request_metrics.push(RequestMetric { + method: "".to_string(), + name: "Aggregated".to_string(), + number_of_requests: raw_aggregate_total_count.into(), + number_of_failures: raw_aggregate_fail_count.into(), + response_time_average: (raw_aggregate_response_time_counter as f32 + / raw_aggregate_total_count as f32) + .into(), + response_time_minimum: raw_aggregate_response_time_minimum.into(), + response_time_maximum: raw_aggregate_response_time_maximum.into(), + requests_per_second: raw_aggregate_requests_per_second.into(), + failures_per_second: raw_aggregate_failures_per_second.into(), + }); - // Collect aggregated request and response metrics. - raw_aggregate_total_count += total_request_count; - raw_aggregate_fail_count += request.fail_count; - raw_aggregate_response_time_counter += request.raw_data.total_time; - raw_aggregate_response_time_minimum = update_min_time( + // Prepare aggregate per-response metrics. + raw_response_metrics.push(get_response_metric( + "", + "Aggregated", + &raw_aggregate_response_times, + raw_aggregate_total_count, raw_aggregate_response_time_minimum, - request.raw_data.minimum_time, - ); - raw_aggregate_response_time_maximum = update_max_time( raw_aggregate_response_time_maximum, - request.raw_data.maximum_time, - ); - raw_aggregate_response_times = - merge_times(raw_aggregate_response_times, request.raw_data.times.clone()); + )); + + // correlate with baseline + + if let Some(baseline) = self.baseline { + correlate_deltas( + &mut raw_request_metrics, + &baseline.raw_request_metrics, + |entry| (entry.method.clone(), entry.name.clone()), + ); + correlate_deltas( + &mut raw_response_metrics, + &baseline.raw_response_metrics, + |entry| (entry.method.clone(), entry.name.clone()), + ); + } + + // return result + + ( + raw_request_metrics, + raw_response_metrics, + RawIntermediate { + raw_aggregate_response_time_counter, + raw_aggregate_response_time_minimum, + raw_aggregate_total_count, + }, + ) } - // Prepare aggregate per-request metrics. - let (raw_aggregate_requests_per_second, raw_aggregate_failures_per_second) = - per_second_calculations( - metrics.duration, - raw_aggregate_total_count, - raw_aggregate_fail_count, - ); - raw_request_metrics.push(report::RequestMetric { - method: "".to_string(), - name: "Aggregated".to_string(), - number_of_requests: raw_aggregate_total_count, - number_of_failures: raw_aggregate_fail_count, - response_time_average: raw_aggregate_response_time_counter as f32 - / raw_aggregate_total_count as f32, - response_time_minimum: raw_aggregate_response_time_minimum, - response_time_maximum: raw_aggregate_response_time_maximum, - requests_per_second: raw_aggregate_requests_per_second, - failures_per_second: raw_aggregate_failures_per_second, - }); - - // Prepare aggregate per-response metrics. - raw_response_metrics.push(report::get_response_metric( - "", - "Aggregated", - &raw_aggregate_response_times, - raw_aggregate_total_count, - raw_aggregate_response_time_minimum, - raw_aggregate_response_time_maximum, - )); - - let (co_request_metrics, co_response_metrics) = if co_data { - for (request_key, request) in metrics.requests.iter().sorted() { + fn build_co( + &self, + intermediate: &RawIntermediate, + ) -> (Option>, Option>) { + if !self.co_data { + return (None, None); + } + + let mut co_request_metrics = Vec::new(); + let mut co_response_metrics = Vec::new(); + let mut co_aggregate_total_count = 0; + let mut co_aggregate_response_time_counter: usize = 0; + let mut co_aggregate_response_time_maximum: usize = 0; + let mut co_aggregate_response_times: BTreeMap = BTreeMap::new(); + + for (request_key, request) in self.metrics.requests.iter().sorted() { if let Some(coordinated_omission_data) = request.coordinated_omission_data.as_ref() { let method = format!("{}", request.method); // The request_key is "{method} {name}", so by stripping the "{method} " @@ -154,19 +249,20 @@ pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportDat let co_average = coordinated_omission_data.total_time as f32 / coordinated_omission_data.counter as f32; // Prepare per-request metrics. - co_request_metrics.push(report::CORequestMetric { + co_request_metrics.push(CORequestMetric { method: method.to_string(), name: name.to_string(), - response_time_average: co_average, + response_time_average: co_average.into(), response_time_standard_deviation: util::standard_deviation( raw_average, co_average, - ), - response_time_maximum: coordinated_omission_data.maximum_time, + ) + .into(), + response_time_maximum: coordinated_omission_data.maximum_time.into(), }); // Prepare per-response metrics. - co_response_metrics.push(report::get_response_metric( + co_response_metrics.push(get_response_metric( &method, &name, &coordinated_omission_data.times, @@ -191,34 +287,56 @@ pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportDat } let co_average = co_aggregate_response_time_counter as f32 / co_aggregate_total_count as f32; - let raw_average = - raw_aggregate_response_time_counter as f32 / raw_aggregate_total_count as f32; - co_request_metrics.push(report::CORequestMetric { + let raw_average = intermediate.raw_aggregate_response_time_counter as f32 + / intermediate.raw_aggregate_total_count as f32; + co_request_metrics.push(CORequestMetric { method: "".to_string(), name: "Aggregated".to_string(), - response_time_average: co_aggregate_response_time_counter as f32 - / co_aggregate_total_count as f32, - response_time_standard_deviation: util::standard_deviation(raw_average, co_average), - response_time_maximum: co_aggregate_response_time_maximum, + response_time_average: (co_aggregate_response_time_counter as f32 + / co_aggregate_total_count as f32) + .into(), + response_time_standard_deviation: util::standard_deviation(raw_average, co_average) + .into(), + response_time_maximum: co_aggregate_response_time_maximum.into(), }); // Prepare aggregate per-response metrics. - co_response_metrics.push(report::get_response_metric( + co_response_metrics.push(get_response_metric( "", "Aggregated", &co_aggregate_response_times, co_aggregate_total_count, - raw_aggregate_response_time_minimum, + intermediate.raw_aggregate_response_time_minimum, co_aggregate_response_time_maximum, )); + if let Some(baseline) = self.baseline { + if let Some(baseline_co_request_metrics) = &baseline.co_request_metrics { + correlate_deltas( + &mut co_request_metrics, + baseline_co_request_metrics, + |entry| (entry.method.clone(), entry.name.clone()), + ); + } + if let Some(baseline_co_response_metrics) = &baseline.co_response_metrics { + correlate_deltas( + &mut co_response_metrics, + baseline_co_response_metrics, + |entry| (entry.method.clone(), entry.name.clone()), + ); + } + } + (Some(co_request_metrics), Some(co_response_metrics)) - } else { - (None, None) - }; + } + + fn build_transaction(&self, intermediate: &RawIntermediate) -> Option> { + if self.options.no_transaction_metrics { + return None; + } + + // Only build the transactions template if --no-transaction-metrics isn't enabled. - // Only build the transactions template if --no-transaction-metrics isn't enabled. - let transaction_metrics = if !options.no_transaction_metrics { let mut transaction_metrics = Vec::new(); let mut aggregate_total_count = 0; let mut aggregate_fail_count = 0; @@ -226,26 +344,26 @@ pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportDat let mut aggregate_transaction_time_minimum: usize = 0; let mut aggregate_transaction_time_maximum: usize = 0; let mut aggregate_transaction_times: BTreeMap = BTreeMap::new(); - for (scenario_counter, scenario) in metrics.transactions.iter().enumerate() { + for (scenario_counter, scenario) in self.metrics.transactions.iter().enumerate() { for (transaction_counter, transaction) in scenario.iter().enumerate() { if transaction_counter == 0 { // Only the scenario_name is used for scenarios. - transaction_metrics.push(report::TransactionMetric { + transaction_metrics.push(TransactionMetric { is_scenario: true, transaction: "".to_string(), name: transaction.scenario_name.to_string(), - number_of_requests: 0, - number_of_failures: 0, + number_of_requests: 0.into(), + number_of_failures: 0.into(), response_time_average: None, - response_time_minimum: 0, - response_time_maximum: 0, + response_time_minimum: 0.into(), + response_time_maximum: 0.into(), requests_per_second: None, failures_per_second: None, }); } let total_run_count = transaction.success_count + transaction.fail_count; let (requests_per_second, failures_per_second) = per_second_calculations( - metrics.duration, + self.metrics.duration, total_run_count, transaction.fail_count, ); @@ -253,17 +371,17 @@ pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportDat 0 => 0.00, _ => transaction.total_time as f32 / transaction.counter as f32, }; - transaction_metrics.push(report::TransactionMetric { + transaction_metrics.push(TransactionMetric { is_scenario: false, transaction: format!("{}.{}", scenario_counter, transaction_counter), name: transaction.transaction_name.to_string(), - number_of_requests: total_run_count, - number_of_failures: transaction.fail_count, - response_time_average: Some(average), - response_time_minimum: transaction.min_time, - response_time_maximum: transaction.max_time, - requests_per_second: Some(requests_per_second), - failures_per_second: Some(failures_per_second), + number_of_requests: total_run_count.into(), + number_of_failures: transaction.fail_count.into(), + response_time_average: Some(average.into()), + response_time_minimum: transaction.min_time.into(), + response_time_maximum: transaction.max_time.into(), + requests_per_second: Some(requests_per_second.into()), + failures_per_second: Some(failures_per_second.into()), }); aggregate_total_count += total_run_count; @@ -280,31 +398,49 @@ pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportDat let (aggregate_requests_per_second, aggregate_failures_per_second) = per_second_calculations( - metrics.duration, + self.metrics.duration, aggregate_total_count, aggregate_fail_count, ); - transaction_metrics.push(report::TransactionMetric { + transaction_metrics.push(TransactionMetric { is_scenario: false, transaction: "".to_string(), name: "Aggregated".to_string(), - number_of_requests: aggregate_total_count, - number_of_failures: aggregate_fail_count, + number_of_requests: aggregate_total_count.into(), + number_of_failures: aggregate_fail_count.into(), response_time_average: Some( - raw_aggregate_response_time_counter as f32 / aggregate_total_count as f32, + (intermediate.raw_aggregate_response_time_counter as f32 + / aggregate_total_count as f32) + .into(), ), - response_time_minimum: aggregate_transaction_time_minimum, - response_time_maximum: aggregate_transaction_time_maximum, - requests_per_second: Some(aggregate_requests_per_second), - failures_per_second: Some(aggregate_failures_per_second), + response_time_minimum: aggregate_transaction_time_minimum.into(), + response_time_maximum: aggregate_transaction_time_maximum.into(), + requests_per_second: Some(aggregate_requests_per_second.into()), + failures_per_second: Some(aggregate_failures_per_second.into()), }); + + if let Some(baseline_transaction_metrics) = self + .baseline + .as_ref() + .and_then(|baseline| baseline.transaction_metrics.as_ref()) + { + correlate_deltas( + &mut transaction_metrics, + baseline_transaction_metrics, + |entry| (entry.transaction.clone(), entry.name.clone()), + ); + } + Some(transaction_metrics) - } else { - None - }; + } + + fn build_scenario(&self) -> Option> { + // Only build the scenarios template if --no-senario-metrics isn't enabled. + + if self.options.no_scenario_metrics { + return None; + } - // Only build the scenarios template if --no-senario-metrics isn't enabled. - let scenario_metrics = if !options.no_scenario_metrics { let mut scenario_metrics = Vec::new(); let mut aggregate_users = 0; let mut aggregate_count = 0; @@ -314,23 +450,26 @@ pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportDat let mut aggregate_scenario_times: BTreeMap = BTreeMap::new(); let mut aggregate_iterations = 0.0; let mut aggregate_response_time_counter = 0.0; - for scenario in &metrics.scenarios { + for scenario in &self.metrics.scenarios { let (count_per_second, _failures_per_second) = - per_second_calculations(metrics.duration, scenario.counter, 0); + per_second_calculations(self.metrics.duration, scenario.counter, 0); let average = match scenario.counter { 0 => 0.00, _ => scenario.total_time as f32 / scenario.counter as f32, }; - let iterations = scenario.counter as f32 / scenario.users.len() as f32; - scenario_metrics.push(report::ScenarioMetric { + let iterations = match scenario.users.len() { + 0 => 0f32, + n => scenario.counter as f32 / n as f32, + }; + scenario_metrics.push(ScenarioMetric { name: scenario.name.to_string(), - users: scenario.users.len(), - count: scenario.counter, - response_time_average: average, - response_time_minimum: scenario.min_time, - response_time_maximum: scenario.max_time, - count_per_second, - iterations, + users: scenario.users.len().into(), + count: scenario.counter.into(), + response_time_average: average.into(), + response_time_minimum: scenario.min_time.into(), + response_time_maximum: scenario.max_time.into(), + count_per_second: count_per_second.into(), + iterations: iterations.into(), }); aggregate_users += scenario.users.len(); @@ -347,27 +486,40 @@ pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportDat } let (aggregate_count_per_second, _aggregate_failures_per_second) = - per_second_calculations(metrics.duration, aggregate_count, 0); - scenario_metrics.push(report::ScenarioMetric { + per_second_calculations(self.metrics.duration, aggregate_count, 0); + scenario_metrics.push(ScenarioMetric { name: "Aggregated".to_string(), - users: aggregate_users, - count: aggregate_count, - response_time_average: aggregate_response_time_counter / aggregate_count as f32, - response_time_minimum: aggregate_scenario_time_minimum, - response_time_maximum: aggregate_scenario_time_maximum, - count_per_second: aggregate_count_per_second, - iterations: aggregate_iterations, + users: aggregate_users.into(), + count: aggregate_count.into(), + response_time_average: (aggregate_response_time_counter / aggregate_count as f32) + .into(), + response_time_minimum: aggregate_scenario_time_minimum.into(), + response_time_maximum: aggregate_scenario_time_maximum.into(), + count_per_second: aggregate_count_per_second.into(), + iterations: aggregate_iterations.into(), }); + if let Some(baseline_scenario_metrics) = self + .baseline + .as_ref() + .and_then(|baseline| baseline.scenario_metrics.as_ref()) + { + correlate_deltas(&mut scenario_metrics, baseline_scenario_metrics, |entry| { + entry.name.clone() + }); + } + Some(scenario_metrics) - } else { - None - }; + } + + fn build_status_code(&self) -> Option> { + if self.options.no_status_codes { + return None; + } - let status_code_metrics = if !options.no_status_codes { let mut status_code_metrics = Vec::new(); let mut aggregated_status_code_counts: HashMap = HashMap::new(); - for (request_key, request) in metrics.requests.iter().sorted() { + for (request_key, request) in self.metrics.requests.iter().sorted() { let method = format!("{}", request.method); // The request_key is "{method} {name}", so by stripping the "{method} " // prefix we get the name. @@ -383,7 +535,7 @@ pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportDat ); // Add a row of data for the status code table. - status_code_metrics.push(report::StatusCodeMetric { + status_code_metrics.push(StatusCodeMetric { method, name, status_codes: codes, @@ -394,29 +546,96 @@ pub fn prepare_data(options: ReportOptions, metrics: &GooseMetrics) -> ReportDat let aggregated_codes = prepare_status_codes(&aggregated_status_code_counts, &mut None); // Add a final row of aggregate data for the status code table. - status_code_metrics.push(report::StatusCodeMetric { + status_code_metrics.push(StatusCodeMetric { method: "".to_string(), name: "Aggregated".to_string(), status_codes: aggregated_codes, }); Some(status_code_metrics) - } else { - None - }; - - ReportData { - raw_metrics: metrics, - raw_request_metrics, - raw_response_metrics, - co_request_metrics, - co_response_metrics, - scenario_metrics, - transaction_metrics, - status_code_metrics, - errors: metrics + } + + fn build_errors(&self) -> Option> { + if self.metrics.errors.is_empty() { + return None; + } + + let mut errors = self + .metrics .errors - .is_empty() - .then(|| metrics.errors.values().collect::>()), + .values() + .map(|error| ErrorMetric { + method: error.method, + name: error.name.clone(), + error: error.error.clone(), + occurrences: error.occurrences.into(), + }) + .collect::>(); + + if let Some(baseline_errors) = self + .baseline + .as_ref() + .and_then(|baseline| baseline.errors.as_ref()) + { + correlate_deltas(&mut errors, baseline_errors, |error| { + (error.method, error.name.clone(), error.error.clone()) + }); + } + + Some(errors) + } +} + +pub fn prepare_data<'a, 'b>( + options: ReportOptions, + metrics: &'a GooseMetrics, + baseline: &'b Option>, +) -> ReportData<'a> { + Prepare::new(options, metrics, baseline).build() +} + +/// Load a baseline file +pub(crate) fn load_baseline_file( + path: impl AsRef, +) -> Result, GooseError> { + Ok(serde_json::from_reader(BufReader::new(File::open(path)?))?) +} + +/// take a current slice of metrics, and apply correlated baseline metrics. +/// +/// This will iterate over all the current metrics, fetch the correlated baseline metrics and call +/// [`DeltaEval::eval`] on it. Entries are correlated by the key returned from the function `f`. +fn correlate_deltas(current: &mut [T], baseline: &[T], f: F) +where + T: DeltaTo, + F: Fn(&T) -> K, + K: Eq + Hash, +{ + let mut current = current + .iter_mut() + .map(|request| (f(request), request)) + .collect::>(); + let previous = baseline + .iter() + .map(|request| (f(request), request)) + .collect::>(); + + for (k, v) in &mut current { + if let Some(previous) = previous.get(k) { + v.delta_to(previous); + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn delta_value_usize() { + assert_eq!(100usize.delta(50usize), 50isize); + assert_eq!(usize::MAX.delta(usize::MAX), 0isize); + assert_eq!(usize::MAX.delta(0usize), isize::MAX); + assert_eq!(0usize.delta(usize::MAX), isize::MIN); } } diff --git a/src/metrics/delta.rs b/src/metrics/delta.rs new file mode 100644 index 00000000..28220cd7 --- /dev/null +++ b/src/metrics/delta.rs @@ -0,0 +1,171 @@ +use std::fmt::{Debug, Display, Formatter, Write}; + +pub trait DeltaValue: Copy + Debug + Display { + type Delta: Copy + Display; + + fn delta(self, value: Self) -> Self::Delta; + + /// It's positive if it's not negative or zero + fn is_delta_positive(value: Self::Delta) -> bool; +} + +impl DeltaValue for usize { + type Delta = isize; + + fn delta(self, value: Self) -> Self::Delta { + if self >= value { + // the result will be positive, so just limit to isize::MAX + (self - value).min(isize::MAX as usize) as isize + } else { + // the result will be negative, we will calculate the absolute value of that... + let delta = value - self; + if delta > 9223372036854775808 + /* the absolute value of isize::MIN as usize */ + { + // ... which is too big to fix into the negative space of isize, so we limit to isize::MIN + isize::MIN + } else { + // ... which fits, so we return the negative value + -(delta as isize) + } + } + } + + fn is_delta_positive(value: Self::Delta) -> bool { + value.is_positive() + } +} + +impl DeltaValue for f32 { + type Delta = f32; + + fn delta(self, value: Self) -> Self::Delta { + self - value + } + + fn is_delta_positive(value: Self::Delta) -> bool { + !value.is_sign_negative() + } +} + +#[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub(crate) enum Value { + Plain(T), + Delta { value: T, delta: T::Delta }, +} + +impl From for Value { + fn from(value: T) -> Self { + Self::Plain(value) + } +} + +impl Value { + pub fn diff(&mut self, other: T) { + match self { + Self::Plain(value) => { + *self = Self::Delta { + value: *value, + delta: value.delta(other), + }; + } + Self::Delta { value, delta: _ } => { + *self = Self::Delta { + value: *value, + delta: value.delta(other), + } + } + } + } +} + +impl DeltaEval for Value { + fn eval(&mut self, other: Self) { + self.diff(other.value()) + } +} + +impl DeltaEval for Option> { + fn eval(&mut self, other: Self) { + if let (Some(value), Some(other)) = (self, other) { + value.eval(other); + } + } +} + +pub trait DeltaEval { + fn eval(&mut self, other: Self); +} + +impl Value { + pub fn value(&self) -> T { + match self { + Self::Plain(value) => *value, + Self::Delta { value, delta: _ } => *value, + } + } +} + +impl Display for Value { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Plain(value) => Display::fmt(value, f), + Self::Delta { value, delta } => { + // we can pass on the actual value + Display::fmt(value, f)?; + + // format delta as `({delta:+})`, keeping the actual format options + f.write_str(" (")?; + + // for the delta, we want a plus sign, in the case of a positive value, zero excluded + if T::is_delta_positive(*delta) { + f.write_char('+')?; + Display::fmt(delta, f)?; + } else { + Display::fmt(delta, f)?; + } + + f.write_char(')')?; + + // done + Ok(()) + } + } + } +} + +/// Build a delta to a baseline +pub trait DeltaTo { + fn delta_to(&mut self, other: &Self); +} + +#[cfg(test)] +mod test { + use super::*; + use crate::metrics::Value; + + #[test] + fn eval_optional() { + assert_eq!( + { + let mut value = Some(Value::Plain(10)); + value.eval(Some(Value::Plain(5))); + value + }, + Some(Value::Delta { + value: 10, + delta: 5 + }) + ); + + assert_eq!( + { + let mut value = None; + value.eval(Some(Value::Plain(5))); + value + }, + None + ); + } +} diff --git a/src/report.rs b/src/report.rs index 9a5ceae8..b3e07175 100644 --- a/src/report.rs +++ b/src/report.rs @@ -4,11 +4,12 @@ mod markdown; pub(crate) use markdown::write_markdown_report; +use crate::goose::GooseMethod; use crate::{ - metrics::{self, format_number}, + metrics::{self, DeltaEval, DeltaTo, Value}, report::common::OrEmpty, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; /// The following templates are necessary to build an html-formatted summary report. @@ -28,80 +29,153 @@ pub(crate) struct GooseReportTemplates<'a> { } /// Defines the metrics reported about requests. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct RequestMetric { pub method: String, pub name: String, - pub number_of_requests: usize, - pub number_of_failures: usize, - pub response_time_average: f32, - pub response_time_minimum: usize, - pub response_time_maximum: usize, - pub requests_per_second: f32, - pub failures_per_second: f32, + pub number_of_requests: Value, + pub number_of_failures: Value, + pub response_time_average: Value, + pub response_time_minimum: Value, + pub response_time_maximum: Value, + pub requests_per_second: Value, + pub failures_per_second: Value, +} + +impl DeltaTo for RequestMetric { + fn delta_to(&mut self, other: &Self) { + self.number_of_requests.eval(other.number_of_requests); + self.number_of_requests.eval(other.number_of_requests); + self.response_time_average.eval(other.response_time_average); + self.response_time_minimum.eval(other.response_time_minimum); + self.response_time_maximum.eval(other.response_time_maximum); + self.requests_per_second.eval(other.requests_per_second); + self.failures_per_second.eval(other.failures_per_second); + } } /// Defines the metrics reported about Coordinated Omission requests. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct CORequestMetric { pub method: String, pub name: String, - pub response_time_average: f32, - pub response_time_standard_deviation: f32, - pub response_time_maximum: usize, + pub response_time_average: Value, + pub response_time_standard_deviation: Value, + pub response_time_maximum: Value, +} + +impl DeltaTo for CORequestMetric { + fn delta_to(&mut self, other: &Self) { + self.response_time_average.eval(other.response_time_average); + self.response_time_standard_deviation + .eval(other.response_time_standard_deviation); + self.response_time_maximum.eval(other.response_time_maximum); + } } /// Defines the metrics reported about responses. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct ResponseMetric { pub method: String, pub name: String, - pub percentile_50: usize, - pub percentile_60: usize, - pub percentile_70: usize, - pub percentile_80: usize, - pub percentile_90: usize, - pub percentile_95: usize, - pub percentile_99: usize, - pub percentile_100: usize, + pub percentile_50: Value, + pub percentile_60: Value, + pub percentile_70: Value, + pub percentile_80: Value, + pub percentile_90: Value, + pub percentile_95: Value, + pub percentile_99: Value, + pub percentile_100: Value, +} + +impl DeltaTo for ResponseMetric { + fn delta_to(&mut self, other: &Self) { + self.percentile_50.eval(other.percentile_50); + self.percentile_60.eval(other.percentile_60); + self.percentile_70.eval(other.percentile_70); + self.percentile_80.eval(other.percentile_80); + self.percentile_90.eval(other.percentile_90); + self.percentile_95.eval(other.percentile_95); + self.percentile_99.eval(other.percentile_99); + self.percentile_100.eval(other.percentile_100); + } } /// Defines the metrics reported about transactions. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct TransactionMetric { pub is_scenario: bool, pub transaction: String, pub name: String, - pub number_of_requests: usize, - pub number_of_failures: usize, - pub response_time_average: Option, - pub response_time_minimum: usize, - pub response_time_maximum: usize, - pub requests_per_second: Option, - pub failures_per_second: Option, + pub number_of_requests: Value, + pub number_of_failures: Value, + pub response_time_average: Option>, + pub response_time_minimum: Value, + pub response_time_maximum: Value, + pub requests_per_second: Option>, + pub failures_per_second: Option>, +} + +impl DeltaTo for TransactionMetric { + fn delta_to(&mut self, other: &Self) { + self.number_of_requests.eval(other.number_of_requests); + self.number_of_failures.eval(other.number_of_failures); + self.response_time_average.eval(other.response_time_average); + self.response_time_minimum.eval(other.response_time_minimum); + self.response_time_maximum.eval(other.response_time_maximum); + self.requests_per_second.eval(other.requests_per_second); + self.failures_per_second.eval(other.failures_per_second); + } } /// Defines the metrics reported about scenarios. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct ScenarioMetric { pub name: String, - pub users: usize, - pub count: usize, - pub response_time_average: f32, - pub response_time_minimum: usize, - pub response_time_maximum: usize, - pub count_per_second: f32, - pub iterations: f32, + pub users: Value, + pub count: Value, + pub response_time_average: Value, + pub response_time_minimum: Value, + pub response_time_maximum: Value, + pub count_per_second: Value, + pub iterations: Value, +} + +impl DeltaTo for ScenarioMetric { + fn delta_to(&mut self, other: &Self) { + self.users.eval(other.users); + self.count.eval(other.count); + self.response_time_average.eval(other.response_time_average); + self.response_time_minimum.eval(other.response_time_minimum); + self.response_time_maximum.eval(other.response_time_maximum); + self.count_per_second.eval(other.count_per_second); + self.iterations.eval(other.iterations); + } } /// Defines the metrics reported about status codes. -#[derive(Debug, serde::Serialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] pub(crate) struct StatusCodeMetric { pub method: String, pub name: String, pub status_codes: String, } +/// Defines the metrics report about errors +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] +pub struct ErrorMetric { + pub method: GooseMethod, + pub name: String, + pub error: String, + pub occurrences: Value, +} + +impl DeltaTo for ErrorMetric { + fn delta_to(&mut self, other: &Self) { + self.occurrences.eval(other.occurrences); + } +} + /// Helper to generate a single response metric. pub(crate) fn get_response_metric( method: &str, @@ -127,14 +201,14 @@ pub(crate) fn get_response_metric( ResponseMetric { method: method.to_string(), name: name.to_string(), - percentile_50: percentiles[0], - percentile_60: percentiles[1], - percentile_70: percentiles[2], - percentile_80: percentiles[3], - percentile_90: percentiles[4], - percentile_95: percentiles[5], - percentile_99: percentiles[6], - percentile_100: percentiles[7], + percentile_50: percentiles[0].into(), + percentile_60: percentiles[1].into(), + percentile_70: percentiles[2].into(), + percentile_80: percentiles[3].into(), + percentile_90: percentiles[4].into(), + percentile_95: percentiles[5].into(), + percentile_99: percentiles[6].into(), + percentile_100: percentiles[7].into(), } } @@ -181,14 +255,14 @@ pub(crate) fn response_metrics_row(metric: ResponseMetric) -> String { "#, method = metric.method, name = metric.name, - percentile_50 = format_number(metric.percentile_50), - percentile_60 = format_number(metric.percentile_60), - percentile_70 = format_number(metric.percentile_70), - percentile_80 = format_number(metric.percentile_80), - percentile_90 = format_number(metric.percentile_90), - percentile_95 = format_number(metric.percentile_95), - percentile_99 = format_number(metric.percentile_99), - percentile_100 = format_number(metric.percentile_100), + percentile_50 = metric.percentile_50, + percentile_60 = metric.percentile_60, + percentile_70 = metric.percentile_70, + percentile_80 = metric.percentile_80, + percentile_90 = metric.percentile_90, + percentile_95 = metric.percentile_95, + percentile_99 = metric.percentile_99, + percentile_100 = metric.percentile_100, ) } @@ -284,14 +358,14 @@ pub(crate) fn coordinated_omission_response_metrics_row(metric: ResponseMetric) "#, method = metric.method, name = metric.name, - percentile_50 = format_number(metric.percentile_50), - percentile_60 = format_number(metric.percentile_60), - percentile_70 = format_number(metric.percentile_70), - percentile_80 = format_number(metric.percentile_80), - percentile_90 = format_number(metric.percentile_90), - percentile_95 = format_number(metric.percentile_95), - percentile_99 = format_number(metric.percentile_99), - percentile_100 = format_number(metric.percentile_100), + percentile_50 = metric.percentile_50, + percentile_60 = metric.percentile_60, + percentile_70 = metric.percentile_70, + percentile_80 = metric.percentile_80, + percentile_90 = metric.percentile_90, + percentile_95 = metric.percentile_95, + percentile_99 = metric.percentile_99, + percentile_100 = metric.percentile_100, ) } @@ -386,8 +460,8 @@ pub(crate) fn transaction_metrics_row(metric: TransactionMetric) -> String { "#, transaction = metric.transaction, name = metric.name, - number_of_requests = format_number(metric.number_of_requests), - number_of_failures = format_number(metric.number_of_failures), + number_of_requests = metric.number_of_requests, + number_of_failures = metric.number_of_failures, response_time_average = OrEmpty(metric.response_time_average), response_time_minimum = metric.response_time_minimum, response_time_maximum = metric.response_time_maximum, @@ -442,8 +516,8 @@ pub(crate) fn scenario_metrics_row(metric: ScenarioMetric) -> String { {iterations:.2} "#, name = metric.name, - users = format_number(metric.users), - count = format_number(metric.count), + users = metric.users, + count = metric.count, response_time_average = metric.response_time_average, response_time_minimum = metric.response_time_minimum, response_time_maximum = metric.response_time_maximum, @@ -478,7 +552,7 @@ pub(crate) fn errors_template(error_rows: &str, graph: String) -> String { } /// Build an individual error row in the html report. -pub fn error_row(error: &metrics::GooseErrorMetricAggregate) -> String { +pub fn error_row(error: &ErrorMetric) -> String { format!( r#" {occurrences} diff --git a/src/report/markdown.rs b/src/report/markdown.rs index 9f921dee..7d1b6478 100644 --- a/src/report/markdown.rs +++ b/src/report/markdown.rs @@ -1,8 +1,8 @@ use crate::{ - metrics::{format_number, GooseErrorMetricAggregate, ReportData}, + metrics::ReportData, report::{ - common::OrEmpty, RequestMetric, ResponseMetric, ScenarioMetric, StatusCodeMetric, - TransactionMetric, + common::OrEmpty, ErrorMetric, RequestMetric, ResponseMetric, ScenarioMetric, + StatusCodeMetric, TransactionMetric, }, test_plan::TestPlanStepAction, GooseError, @@ -161,14 +161,6 @@ impl<'m, 'w, W: Write> Markdown<'m, 'w, W> { writeln!( self.w, r#"| {method} | {name} | {percentile_50} | {percentile_60 } | {percentile_70 } | {percentile_80} | {percentile_90} | {percentile_95} | {percentile_99} | {percentile_100} |"#, - percentile_50 = format_number(*percentile_50), - percentile_60 = format_number(*percentile_60), - percentile_70 = format_number(*percentile_70), - percentile_80 = format_number(*percentile_80), - percentile_90 = format_number(*percentile_90), - percentile_95 = format_number(*percentile_95), - percentile_99 = format_number(*percentile_99), - percentile_100 = format_number(*percentile_100), )?; } @@ -295,7 +287,7 @@ impl<'m, 'w, W: Write> Markdown<'m, 'w, W> { "# )?; - for GooseErrorMetricAggregate { + for ErrorMetric { method, name, error, From 362de7d645dcc50ddef12834ef8a8d328c1ea4ff Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Mon, 26 Aug 2024 09:39:25 +0200 Subject: [PATCH 2/6] chore: work towards better number formatting --- src/metrics/delta.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/metrics/delta.rs b/src/metrics/delta.rs index 28220cd7..24a52ce9 100644 --- a/src/metrics/delta.rs +++ b/src/metrics/delta.rs @@ -1,3 +1,4 @@ +use num_format::ToFormattedStr; use std::fmt::{Debug, Display, Formatter, Write}; pub trait DeltaValue: Copy + Debug + Display { @@ -140,6 +141,18 @@ pub trait DeltaTo { fn delta_to(&mut self, other: &Self); } +pub struct Formatted(pub T); + +impl Display for Formatted { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + use num_format::{Locale, ToFormattedString}; + + f.write_str(&self.0.to_formatted_string(&Locale::en))?; + + Ok(()) + } +} + #[cfg(test)] mod test { use super::*; From e08926531654e81421684481ec21ad0ca43a1379 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 28 Aug 2024 15:02:02 +0200 Subject: [PATCH 3/6] chore: implement number format for delta values too --- src/metrics.rs | 10 ++++- src/metrics/delta.rs | 96 ++++++++++++++++++++++++++++++++++++------ src/report.rs | 44 +++++++++---------- src/report/common.rs | 25 ++++++++++- src/report/markdown.rs | 9 ++++ 5 files changed, 145 insertions(+), 39 deletions(-) diff --git a/src/metrics.rs b/src/metrics.rs index 77966b01..cd0186c6 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -3360,7 +3360,15 @@ fn determine_precision(value: f32) -> usize { /// Format large number in locale appropriate style. pub(crate) fn format_number(number: usize) -> String { - (number).to_formatted_string(&Locale::en) + number.to_formatted_string(&Locale::en) +} + +/// Format large value in locale appropriate style. +pub(crate) fn format_value(value: &Value) -> String +where + T: DeltaValue + ToFormattedString, +{ + value.formatted_number(&Locale::en) } /// A helper function that merges together times. diff --git a/src/metrics/delta.rs b/src/metrics/delta.rs index 24a52ce9..a4a36a47 100644 --- a/src/metrics/delta.rs +++ b/src/metrics/delta.rs @@ -1,9 +1,14 @@ -use num_format::ToFormattedStr; +use num_format::{Format, ToFormattedString}; use std::fmt::{Debug, Display, Formatter, Write}; +/// A value that can be used to provide a delta +/// +/// As the actual value can be an unsigned type, we require an associated type which defines the +/// type of the delta. pub trait DeltaValue: Copy + Debug + Display { type Delta: Copy + Display; + /// Generate the delta between this and the provided value fn delta(self, value: Self) -> Self::Delta; /// It's positive if it's not negative or zero @@ -23,7 +28,7 @@ impl DeltaValue for usize { if delta > 9223372036854775808 /* the absolute value of isize::MIN as usize */ { - // ... which is too big to fix into the negative space of isize, so we limit to isize::MIN + // ... which is too big to fit into the negative space of isize, so we limit to isize::MIN isize::MIN } else { // ... which fits, so we return the negative value @@ -49,6 +54,7 @@ impl DeltaValue for f32 { } } +/// A value, being either a plain value of a value with delta to a baseline #[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] #[serde(untagged)] pub(crate) enum Value { @@ -81,6 +87,29 @@ impl Value { } } +impl Value +where + T: DeltaValue + ToFormattedString, +{ + pub fn formatted_number(&self, format: &impl Format) -> String { + match self { + Self::Plain(value) => value.to_formatted_string(format), + Self::Delta { value, delta } => { + let s = if T::is_delta_positive(*delta) { + "+" + } else { + "" + }; + format!( + "{} ({s}{})", + value.to_formatted_string(format), + delta.to_formatted_string(format) + ) + } + } + } +} + impl DeltaEval for Value { fn eval(&mut self, other: Self) { self.diff(other.value()) @@ -141,22 +170,11 @@ pub trait DeltaTo { fn delta_to(&mut self, other: &Self); } -pub struct Formatted(pub T); - -impl Display for Formatted { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - use num_format::{Locale, ToFormattedString}; - - f.write_str(&self.0.to_formatted_string(&Locale::en))?; - - Ok(()) - } -} - #[cfg(test)] mod test { use super::*; use crate::metrics::Value; + use num_format::Locale; #[test] fn eval_optional() { @@ -181,4 +199,54 @@ mod test { None ); } + + #[test] + fn delta_to_string() { + assert_eq!(format!("{}", 0.delta(10)), "-10"); + assert_eq!(format!("{}", 10.delta(10)), "0"); + assert_eq!(format!("{}", 10.delta(0)), "10"); + } + + #[test] + fn value_to_string() { + fn value(value: T, baseline: T) -> Value { + let mut result = Value::from(value); + result.diff(baseline); + result + } + + assert_eq!(format!("{}", value(0, 1000)), "0 (-1000)"); + assert_eq!(format!("{}", value(1000, 1000)), "1000 (0)"); + assert_eq!(format!("{}", value(1000, 0)), "1000 (+1000)"); + } + + #[test] + fn value_with_delta_to_string_num() { + fn value(value: T, baseline: T) -> Value { + let mut result = Value::from(value); + result.diff(baseline); + result + } + + assert_eq!( + format!("{}", value(0, 1000).formatted_number(&Locale::en)), + "0 (-1,000)" + ); + assert_eq!( + format!("{}", value(1000, 1000).formatted_number(&Locale::en)), + "1,000 (0)" + ); + assert_eq!( + format!("{}", value(1000, 0).formatted_number(&Locale::en)), + "1,000 (+1,000)" + ); + } + + #[test] + fn value_to_string_num() { + assert_eq!( + format!("{}", Value::from(1000).formatted_number(&Locale::en)), + "1,000" + ); + } } diff --git a/src/report.rs b/src/report.rs index b3e07175..bd21a91d 100644 --- a/src/report.rs +++ b/src/report.rs @@ -4,9 +4,9 @@ mod markdown; pub(crate) use markdown::write_markdown_report; -use crate::goose::GooseMethod; use crate::{ - metrics::{self, DeltaEval, DeltaTo, Value}, + goose::GooseMethod, + metrics::{self, format_value, DeltaEval, DeltaTo, Value}, report::common::OrEmpty, }; use serde::{Deserialize, Serialize}; @@ -255,14 +255,14 @@ pub(crate) fn response_metrics_row(metric: ResponseMetric) -> String { "#, method = metric.method, name = metric.name, - percentile_50 = metric.percentile_50, - percentile_60 = metric.percentile_60, - percentile_70 = metric.percentile_70, - percentile_80 = metric.percentile_80, - percentile_90 = metric.percentile_90, - percentile_95 = metric.percentile_95, - percentile_99 = metric.percentile_99, - percentile_100 = metric.percentile_100, + percentile_50 = format_value(&metric.percentile_50), + percentile_60 = format_value(&metric.percentile_60), + percentile_70 = format_value(&metric.percentile_70), + percentile_80 = format_value(&metric.percentile_80), + percentile_90 = format_value(&metric.percentile_90), + percentile_95 = format_value(&metric.percentile_95), + percentile_99 = format_value(&metric.percentile_99), + percentile_100 = format_value(&metric.percentile_100), ) } @@ -358,14 +358,14 @@ pub(crate) fn coordinated_omission_response_metrics_row(metric: ResponseMetric) "#, method = metric.method, name = metric.name, - percentile_50 = metric.percentile_50, - percentile_60 = metric.percentile_60, - percentile_70 = metric.percentile_70, - percentile_80 = metric.percentile_80, - percentile_90 = metric.percentile_90, - percentile_95 = metric.percentile_95, - percentile_99 = metric.percentile_99, - percentile_100 = metric.percentile_100, + percentile_50 = format_value(&metric.percentile_50), + percentile_60 = format_value(&metric.percentile_60), + percentile_70 = format_value(&metric.percentile_70), + percentile_80 = format_value(&metric.percentile_80), + percentile_90 = format_value(&metric.percentile_90), + percentile_95 = format_value(&metric.percentile_95), + percentile_99 = format_value(&metric.percentile_99), + percentile_100 = format_value(&metric.percentile_100), ) } @@ -460,8 +460,8 @@ pub(crate) fn transaction_metrics_row(metric: TransactionMetric) -> String { "#, transaction = metric.transaction, name = metric.name, - number_of_requests = metric.number_of_requests, - number_of_failures = metric.number_of_failures, + number_of_requests = format_value(&metric.number_of_requests), + number_of_failures = format_value(&metric.number_of_failures), response_time_average = OrEmpty(metric.response_time_average), response_time_minimum = metric.response_time_minimum, response_time_maximum = metric.response_time_maximum, @@ -516,8 +516,8 @@ pub(crate) fn scenario_metrics_row(metric: ScenarioMetric) -> String { {iterations:.2} "#, name = metric.name, - users = metric.users, - count = metric.count, + users = format_value(&metric.users), + count = format_value(&metric.count), response_time_average = metric.response_time_average, response_time_minimum = metric.response_time_minimum, response_time_maximum = metric.response_time_maximum, diff --git a/src/report/common.rs b/src/report/common.rs index ede83130..2ae30502 100644 --- a/src/report/common.rs +++ b/src/report/common.rs @@ -1,3 +1,4 @@ +use num_format::{Locale, ToFormattedString}; use std::fmt::{Display, Formatter}; #[derive(Clone, Debug, PartialEq, Eq)] @@ -12,14 +13,34 @@ impl Display for OrEmpty { } } +pub struct FormattedNumber(pub T) +where + T: ToFormattedString; + +impl Display for FormattedNumber +where + T: ToFormattedString, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0.to_formatted_string(&Locale::en)) + } +} + #[cfg(test)] mod test { - use crate::report::common::OrEmpty; + use crate::report::common::{FormattedNumber, OrEmpty}; #[test] - pub fn format() { + pub fn format_or_empty() { assert_eq!("1.23", format!("{:.2}", OrEmpty(Some(1.23456)))); assert_eq!("1", format!("{:.0}", OrEmpty(Some(1.23456)))); assert_eq!("", format!("{:.2}", OrEmpty::(None))); } + + #[test] + pub fn format_number_format() { + assert_eq!("1", format!("{:.2}", FormattedNumber(1))); + assert_eq!("1,000", format!("{:.2}", FormattedNumber(1000))); + assert_eq!("1,000,000", format!("{:.2}", FormattedNumber(1000000))); + } } diff --git a/src/report/markdown.rs b/src/report/markdown.rs index 7d1b6478..bb42e21d 100644 --- a/src/report/markdown.rs +++ b/src/report/markdown.rs @@ -1,3 +1,4 @@ +use crate::metrics::format_value; use crate::{ metrics::ReportData, report::{ @@ -161,6 +162,14 @@ impl<'m, 'w, W: Write> Markdown<'m, 'w, W> { writeln!( self.w, r#"| {method} | {name} | {percentile_50} | {percentile_60 } | {percentile_70 } | {percentile_80} | {percentile_90} | {percentile_95} | {percentile_99} | {percentile_100} |"#, + percentile_50 = format_value(percentile_50), + percentile_60 = format_value(percentile_60), + percentile_70 = format_value(percentile_70), + percentile_80 = format_value(percentile_80), + percentile_90 = format_value(percentile_90), + percentile_95 = format_value(percentile_95), + percentile_99 = format_value(percentile_99), + percentile_100 = format_value(percentile_100), )?; } From c90e1c3b268c101c22a3b292c5eb6b3c0d32134e Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 28 Aug 2024 16:23:03 +0200 Subject: [PATCH 4/6] chore: work around serde issue with f32 NaNs Serde serializes f32s of value NaN to `null`, but doesn't serialize them back from `null` to NaN. For that we need a custom deserialize. Also see: --- src/metrics.rs | 2 ++ src/metrics/common.rs | 5 ++-- src/metrics/delta.rs | 7 +++++ src/metrics/nullable.rs | 58 +++++++++++++++++++++++++++++++++++++++++ src/report.rs | 24 ++++++++--------- 5 files changed, 81 insertions(+), 15 deletions(-) create mode 100644 src/metrics/nullable.rs diff --git a/src/metrics.rs b/src/metrics.rs index cd0186c6..6ac21445 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -10,9 +10,11 @@ mod common; mod delta; +mod nullable; pub(crate) use common::{load_baseline_file, ReportData}; pub(crate) use delta::*; +pub(crate) use nullable::NullableFloat; use crate::config::GooseDefaults; use crate::goose::{get_base_url, GooseMethod, Scenario}; diff --git a/src/metrics/common.rs b/src/metrics/common.rs index 766d5257..6b564e61 100644 --- a/src/metrics/common.rs +++ b/src/metrics/common.rs @@ -2,11 +2,10 @@ use super::{ delta::*, merge_times, per_second_calculations, prepare_status_codes, update_max_time, update_min_time, GooseMetrics, }; -use crate::report::ErrorMetric; use crate::{ report::{ - get_response_metric, CORequestMetric, RequestMetric, ResponseMetric, ScenarioMetric, - StatusCodeMetric, TransactionMetric, + get_response_metric, CORequestMetric, ErrorMetric, RequestMetric, ResponseMetric, + ScenarioMetric, StatusCodeMetric, TransactionMetric, }, util, GooseError, }; diff --git a/src/metrics/delta.rs b/src/metrics/delta.rs index a4a36a47..8ce40001 100644 --- a/src/metrics/delta.rs +++ b/src/metrics/delta.rs @@ -1,3 +1,4 @@ +use crate::metrics::NullableFloat; use num_format::{Format, ToFormattedString}; use std::fmt::{Debug, Display, Formatter, Write}; @@ -68,6 +69,12 @@ impl From for Value { } } +impl From for Value { + fn from(value: f32) -> Self { + Self::Plain(NullableFloat(value)) + } +} + impl Value { pub fn diff(&mut self, other: T) { match self { diff --git a/src/metrics/nullable.rs b/src/metrics/nullable.rs new file mode 100644 index 00000000..7f3377e2 --- /dev/null +++ b/src/metrics/nullable.rs @@ -0,0 +1,58 @@ +use crate::metrics::DeltaValue; +use serde::Deserializer; +use std::fmt::{Display, Formatter}; +use std::ops::{Deref, DerefMut}; + +/// An `f32` which can deserialize from `null` as `NaN`. +/// +/// Also see: +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, serde::Serialize)] +pub struct NullableFloat(pub f32); + +impl Deref for NullableFloat { + type Target = f32; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for NullableFloat { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl From for NullableFloat { + fn from(value: f32) -> Self { + Self(value) + } +} + +impl<'de> serde::Deserialize<'de> for NullableFloat { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Option::::deserialize(deserializer)?; + Ok(Self(value.unwrap_or(f32::NAN))) + } +} + +impl Display for NullableFloat { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl DeltaValue for NullableFloat { + type Delta = NullableFloat; + + fn delta(self, value: Self) -> Self::Delta { + NullableFloat(self.0.delta(value.0)) + } + + fn is_delta_positive(value: Self::Delta) -> bool { + f32::is_delta_positive(value.0) + } +} diff --git a/src/report.rs b/src/report.rs index bd21a91d..1e4fab72 100644 --- a/src/report.rs +++ b/src/report.rs @@ -6,7 +6,7 @@ pub(crate) use markdown::write_markdown_report; use crate::{ goose::GooseMethod, - metrics::{self, format_value, DeltaEval, DeltaTo, Value}, + metrics::{self, format_value, DeltaEval, DeltaTo, NullableFloat, Value}, report::common::OrEmpty, }; use serde::{Deserialize, Serialize}; @@ -35,11 +35,11 @@ pub(crate) struct RequestMetric { pub name: String, pub number_of_requests: Value, pub number_of_failures: Value, - pub response_time_average: Value, + pub response_time_average: Value, pub response_time_minimum: Value, pub response_time_maximum: Value, - pub requests_per_second: Value, - pub failures_per_second: Value, + pub requests_per_second: Value, + pub failures_per_second: Value, } impl DeltaTo for RequestMetric { @@ -59,8 +59,8 @@ impl DeltaTo for RequestMetric { pub(crate) struct CORequestMetric { pub method: String, pub name: String, - pub response_time_average: Value, - pub response_time_standard_deviation: Value, + pub response_time_average: Value, + pub response_time_standard_deviation: Value, pub response_time_maximum: Value, } @@ -109,11 +109,11 @@ pub(crate) struct TransactionMetric { pub name: String, pub number_of_requests: Value, pub number_of_failures: Value, - pub response_time_average: Option>, + pub response_time_average: Option>, pub response_time_minimum: Value, pub response_time_maximum: Value, - pub requests_per_second: Option>, - pub failures_per_second: Option>, + pub requests_per_second: Option>, + pub failures_per_second: Option>, } impl DeltaTo for TransactionMetric { @@ -134,11 +134,11 @@ pub(crate) struct ScenarioMetric { pub name: String, pub users: Value, pub count: Value, - pub response_time_average: Value, + pub response_time_average: Value, pub response_time_minimum: Value, pub response_time_maximum: Value, - pub count_per_second: Value, - pub iterations: Value, + pub count_per_second: Value, + pub iterations: Value, } impl DeltaTo for ScenarioMetric { From d5ae6ae38d04835e21efe248a07cda0ce0fc1622 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 28 Aug 2024 16:35:11 +0200 Subject: [PATCH 5/6] docs: add documentation for the baseline feature --- src/config.rs | 4 ++-- src/docs/goose-book/src/getting-started/metrics.md | 12 ++++++++++++ .../src/getting-started/runtime-options.md | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index 5424dcf3..7a37920a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -103,7 +103,7 @@ pub struct GooseConfiguration { /// Create reports, can be used multiple times (supports .html, .htm, .md, .json) #[options(no_short, meta = "NAME")] pub report_file: Vec, - /// An optional baseline, for rendering the report + /// An optional baseline JSON report, for rendering the reports with deltas #[options(no_short, meta = "NAME")] pub baseline_file: Option, /// Disable granular graphs in report file @@ -286,7 +286,7 @@ pub(crate) struct GooseDefaults { pub no_error_summary: Option, /// An optional default for the html-formatted report file name. pub report_file: Option>, - /// An optional baseline file for the reports. + /// /// An optional baseline JSON report, for rendering the reports with deltas pub baseline_file: Option, /// An optional default for the flag that disables granular data in HTML report graphs. pub no_granular_report: Option, diff --git a/src/docs/goose-book/src/getting-started/metrics.md b/src/docs/goose-book/src/getting-started/metrics.md index 8b77e244..cd5c6792 100644 --- a/src/docs/goose-book/src/getting-started/metrics.md +++ b/src/docs/goose-book/src/getting-started/metrics.md @@ -352,3 +352,15 @@ The JSON report is a dump of the internal metrics collection. It is a JSON seria ### Developer documentation Additional details about how metrics are collected, stored, and displayed can be found [in the developer documentation](https://docs.rs/goose/*/goose/metrics/index.html). + +## Baseline deltas + +It is possible to use a previously generated JSON report as a baseline for reports. In addition to the actual values, this will then also show the difference from the last. + +To use this, you will need to: + +* Create a baseline report by running it at least once using `--report-file report.json` +* Then start adding `--baseline-file report.json` to later runs + +It is possible to still keep generating a JSON report at the same time as using a baseline file. So that you can +create the next basefile in the same step. diff --git a/src/docs/goose-book/src/getting-started/runtime-options.md b/src/docs/goose-book/src/getting-started/runtime-options.md index d36c5b5a..439ff85a 100644 --- a/src/docs/goose-book/src/getting-started/runtime-options.md +++ b/src/docs/goose-book/src/getting-started/runtime-options.md @@ -34,6 +34,7 @@ Metrics: --no-print-metrics Doesn't display metrics at end of load test --no-error-summary Doesn't display an error summary --report-file NAME Create reports, can be used multiple times (supports .html, .htm, .md, .json) + --baseline-file NAME An optional baseline JSON report, for rendering the reports with deltas --no-granular-report Disable granular graphs in report file -R, --request-log NAME Sets request log file name --request-format FORMAT Sets request log format (csv, json, raw, pretty) From 418b16414c71fc7eea3693c6885c57980a4c0ae2 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Wed, 28 Aug 2024 16:56:33 +0200 Subject: [PATCH 6/6] chore: make it work with Rust versions prior to 1.79.0 --- src/metrics.rs | 3 ++- src/metrics/delta.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/metrics.rs b/src/metrics.rs index 6ac21445..09ef9479 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -3368,7 +3368,8 @@ pub(crate) fn format_number(number: usize) -> String { /// Format large value in locale appropriate style. pub(crate) fn format_value(value: &Value) -> String where - T: DeltaValue + ToFormattedString, + T: DeltaValue + ToFormattedString, + ::Delta: ToFormattedString, { value.formatted_number(&Locale::en) } diff --git a/src/metrics/delta.rs b/src/metrics/delta.rs index 8ce40001..89b2149f 100644 --- a/src/metrics/delta.rs +++ b/src/metrics/delta.rs @@ -96,7 +96,8 @@ impl Value { impl Value where - T: DeltaValue + ToFormattedString, + T: DeltaValue + ToFormattedString, + ::Delta: ToFormattedString, { pub fn formatted_number(&self, format: &impl Format) -> String { match self {