Adding baseline to reports

src/config.rs

@@ -103,6 +103,9 @@ pub struct GooseConfiguration {
     /// Create reports, can be used multiple times (supports .html, .htm, .md, .json)
     #[options(no_short, meta = "NAME")]
     pub report_file: Vec<String>,
+    /// An optional baseline JSON report, for rendering the reports with deltas
+    #[options(no_short, meta = "NAME")]
+    pub baseline_file: Option<String>,
     /// Disable granular graphs in report file
     #[options(no_short)]
     pub no_granular_report: bool,
@@ -283,6 +286,8 @@ pub(crate) struct GooseDefaults {
     pub no_error_summary: Option<bool>,
     /// An optional default for the html-formatted report file name.
     pub report_file: Option<Vec<String>>,
+    /// /// An optional baseline JSON report, for rendering the reports with deltas
+    pub baseline_file: Option<String>,
     /// An optional default for the flag that disables granular data in HTML report graphs.
     pub no_granular_report: Option<bool>,
     /// An optional default for the requests log file name.
@@ -1598,6 +1603,21 @@ impl GooseConfiguration {
             ])
             .unwrap_or_default();
 
+        self.baseline_file = self.get_value(vec![
+            // Use --baseline-file if set.
+            GooseValue {
+                value: self.baseline_file.clone(),
+                filter: self.baseline_file.is_none(),
+                message: "baseline_file",
+            },
+            // Otherwise, use GooseDefault if set.
+            GooseValue {
+                value: defaults.baseline_file.clone(),
+                filter: defaults.baseline_file.is_none(),
+                message: "baseline_file",
+            },
+        ]);
+
         // Configure `no_granular_report`.
         self.no_debug_body = self
             .get_value(vec![

src/docs/goose-book/src/getting-started/metrics.md

@@ -352,3 +352,15 @@ The JSON report is a dump of the internal metrics collection. It is a JSON seria
 
 ### Developer documentation
 Additional details about how metrics are collected, stored, and displayed can be found [in the developer documentation](https://docs.rs/goose/*/goose/metrics/index.html).
+
+## Baseline deltas
+
+It is possible to use a previously generated JSON report as a baseline for reports. In addition to the actual values, this will then also show the difference from the last.
+
+To use this, you will need to:
+
+* Create a baseline report by running it at least once using `--report-file report.json`
+* Then start adding `--baseline-file report.json` to later runs
+
+It is possible to still keep generating a JSON report at the same time as using a baseline file. So that you can
+create the next basefile in the same step.

src/docs/goose-book/src/getting-started/runtime-options.md

@@ -34,6 +34,7 @@ Metrics:
   --no-print-metrics          Doesn't display metrics at end of load test
   --no-error-summary          Doesn't display an error summary
   --report-file NAME          Create reports, can be used multiple times (supports .html, .htm, .md, .json)
+  --baseline-file NAME        An optional baseline JSON report, for rendering the reports with deltas
   --no-granular-report        Disable granular graphs in report file
   -R, --request-log NAME      Sets request log file name
   --request-format FORMAT     Sets request log format (csv, json, raw, pretty)

src/goose.rs

@@ -670,7 +670,7 @@ pub enum GooseUserCommand {
 }
 
 /// Supported HTTP methods.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Ord, PartialOrd)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Ord, PartialOrd, Hash)]
 pub enum GooseMethod {
     Delete,
     Get,

src/lib.rs

@@ -67,7 +67,7 @@ use crate::controller::{ControllerProtocol, ControllerRequest};
 use crate::goose::{GooseUser, GooseUserCommand, Scenario, Transaction};
 use crate::graph::GraphData;
 use crate::logger::{GooseLoggerJoinHandle, GooseLoggerTx};
-use crate::metrics::{GooseMetric, GooseMetrics};
+use crate::metrics::{load_baseline_file, GooseMetric, GooseMetrics};
 use crate::test_plan::{TestPlan, TestPlanHistory, TestPlanStepAction};
 
 /// Constant defining Goose's default telnet Controller port.
@@ -1722,6 +1722,16 @@ impl GooseAttack {
         goose_attack_run_state.throttle_threads_tx = throttle_threads_tx;
         goose_attack_run_state.parent_to_throttle_tx = parent_to_throttle_tx;
 
+        // If enabled, try loading the baseline
+        if let Some(baseline_file) = &self.configuration.baseline_file {
+            let _data =
+                load_baseline_file(baseline_file).map_err(|err| GooseError::InvalidOption {
+                    option: "--baseline-file".to_string(),
+                    value: baseline_file.to_string(),
+                    detail: err.to_string(),
+                });
+        }
+
         // Try to create the requested report files, to confirm access.
         self.create_reports().await?;
 

src/metrics.rs

@@ -9,8 +9,12 @@
 //! [`GooseErrorMetrics`] are displayed in tables.
 
 mod common;
+mod delta;
+mod nullable;
 
-pub(crate) use common::ReportData;
+pub(crate) use common::{load_baseline_file, ReportData};
+pub(crate) use delta::*;
+pub(crate) use nullable::NullableFloat;
 
 use crate::config::GooseDefaults;
 use crate::goose::{get_base_url, GooseMethod, Scenario};
@@ -25,8 +29,7 @@ use itertools::Itertools;
 use num_format::{Locale, ToFormattedString};
 use regex::RegexSet;
 use reqwest::StatusCode;
-use serde::ser::SerializeStruct;
-use serde::{Deserialize, Serialize, Serializer};
+use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::ffi::OsStr;
@@ -1026,12 +1029,13 @@ impl ScenarioMetricAggregate {
 ///     Ok(())
 /// }
 /// ```
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
 pub struct GooseMetrics {
     /// A hash of the load test, primarily used to validate all Workers in a Gaggle
     /// are running the same load test.
     pub hash: u64,
     /// A vector recording the history of each load test step.
+    #[serde(skip)]
     pub history: Vec<TestPlanHistory>,
     /// Total number of seconds the load test ran.
     pub duration: usize,
@@ -2564,27 +2568,6 @@ impl GooseMetrics {
     }
 }
 
-impl Serialize for GooseMetrics {
-    // GooseMetrics serialization can't be derived because of the started field.
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut s = serializer.serialize_struct("GooseMetrics", 10)?;
-        s.serialize_field("hash", &self.hash)?;
-        s.serialize_field("duration", &self.duration)?;
-        s.serialize_field("maximum_users", &self.maximum_users)?;
-        s.serialize_field("total_users", &self.total_users)?;
-        s.serialize_field("requests", &self.requests)?;
-        s.serialize_field("transactions", &self.transactions)?;
-        s.serialize_field("errors", &self.errors)?;
-        s.serialize_field("final_metrics", &self.final_metrics)?;
-        s.serialize_field("display_status_codes", &self.display_status_codes)?;
-        s.serialize_field("display_metrics", &self.display_metrics)?;
-        s.end()
-    }
-}
-
 /// Implement format trait to allow displaying metrics.
 impl fmt::Display for GooseMetrics {
     // Implement display of metrics with `{}` marker.
@@ -2671,6 +2654,7 @@ pub struct GooseErrorMetricAggregate {
     /// A counter reflecting how many times this error occurred.
     pub occurrences: usize,
 }
+
 impl GooseErrorMetricAggregate {
     pub(crate) fn new(method: GooseMethod, name: String, error: String) -> Self {
         GooseErrorMetricAggregate {
@@ -2790,11 +2774,9 @@ impl GooseAttack {
         let key = format!("{} {}", request_metric.raw.method, request_metric.name);
         let mut merge_request = match self.metrics.requests.get(&key) {
             Some(m) => m.clone(),
-            None => GooseRequestMetricAggregate::new(
-                &request_metric.name,
-                request_metric.raw.method.clone(),
-                0,
-            ),
+            None => {
+                GooseRequestMetricAggregate::new(&request_metric.name, request_metric.raw.method, 0)
+            }
         };
 
         // Handle a metrics update.
@@ -2984,7 +2966,7 @@ impl GooseAttack {
             Some(m) => m.clone(),
             // First time we've seen this error.
             None => GooseErrorMetricAggregate::new(
-                raw_request.raw.method.clone(),
+                raw_request.raw.method,
                 raw_request.name.to_string(),
                 raw_request.error.to_string(),
             ),
@@ -2996,11 +2978,10 @@ impl GooseAttack {
     // Update metrics showing how long the load test has been running.
     // 1.2 seconds will round down to 1 second. 1.6 seconds will round up to 2 seconds.
     pub(crate) fn update_duration(&mut self) {
-        self.metrics.duration = if self.started.is_some() {
-            self.started.unwrap().elapsed().as_secs_f32().round() as usize
-        } else {
-            0
-        };
+        self.metrics.duration = self
+            .started
+            .map(|started| started.elapsed().as_secs_f32().round() as usize)
+            .unwrap_or_default();
     }
 
     /// Process all requested reports.
@@ -3018,25 +2999,32 @@ impl GooseAttack {
                 })
         };
 
+        let baseline = self
+            .configuration
+            .baseline_file
+            .as_ref()
+            .map(load_baseline_file)
+            .transpose()?;
+
         for report in &self.configuration.report_file {
             let path = PathBuf::from(report);
             match path.extension().map(OsStr::to_string_lossy).as_deref() {
                 Some("html" | "htm") => {
                     let file = create(path).await?;
                     if write {
-                        self.write_html_report(file, report).await?;
+                        self.write_html_report(file, &baseline, report).await?;
                     }
                 }
                 Some("json") => {
                     let file = create(path).await?;
                     if write {
-                        self.write_json_report(file).await?;
+                        self.write_json_report(file, &baseline).await?;
                     }
                 }
                 Some("md") => {
                     let file = create(path).await?;
                     if write {
-                        self.write_markdown_report(file).await?;
+                        self.write_markdown_report(file, &baseline).await?;
                     }
                 }
                 None => {
@@ -3070,14 +3058,19 @@ impl GooseAttack {
     }
 
     /// Write a JSON report.
-    pub(crate) async fn write_json_report(&self, report_file: File) -> Result<(), GooseError> {
+    pub(crate) async fn write_json_report(
+        &self,
+        report_file: File,
+        baseline: &Option<ReportData<'_>>,
+    ) -> Result<(), GooseError> {
         let data = common::prepare_data(
             ReportOptions {
                 no_transaction_metrics: self.configuration.no_transaction_metrics,
                 no_scenario_metrics: self.configuration.no_scenario_metrics,
                 no_status_codes: self.configuration.no_status_codes,
             },
             &self.metrics,
+            baseline,
         );
 
         serde_json::to_writer_pretty(BufWriter::new(report_file.into_std().await), &data)?;
@@ -3086,14 +3079,19 @@ impl GooseAttack {
     }
 
     /// Write a Markdown report.
-    pub(crate) async fn write_markdown_report(&self, report_file: File) -> Result<(), GooseError> {
+    pub(crate) async fn write_markdown_report(
+        &self,
+        report_file: File,
+        baseline: &Option<ReportData<'_>>,
+    ) -> Result<(), GooseError> {
         let data = common::prepare_data(
             ReportOptions {
                 no_transaction_metrics: self.configuration.no_transaction_metrics,
                 no_scenario_metrics: self.configuration.no_scenario_metrics,
                 no_status_codes: self.configuration.no_status_codes,
             },
             &self.metrics,
+            baseline,
         );
 
         report::write_markdown_report(&mut BufWriter::new(report_file.into_std().await), data)
@@ -3103,6 +3101,7 @@ impl GooseAttack {
     pub(crate) async fn write_html_report(
         &self,
         mut report_file: File,
+        baseline: &Option<ReportData<'_>>,
         path: &str,
     ) -> Result<(), GooseError> {
         // Only write the report if enabled.
@@ -3195,6 +3194,7 @@ impl GooseAttack {
                 no_status_codes: self.configuration.no_status_codes,
             },
             &self.metrics,
+            baseline,
         );
 
         // Compile the request metrics template.
@@ -3265,7 +3265,10 @@ impl GooseAttack {
 
         let errors_template = errors
             .map(|errors| {
-                let error_rows = errors.into_iter().map(report::error_row).join("\n");
+                let error_rows = errors
+                    .into_iter()
+                    .map(|error| report::error_row(&error))
+                    .join("\n");
 
                 report::errors_template(
                     &error_rows,
@@ -3359,7 +3362,16 @@ fn determine_precision(value: f32) -> usize {
 
 /// Format large number in locale appropriate style.
 pub(crate) fn format_number(number: usize) -> String {
-    (number).to_formatted_string(&Locale::en)
+    number.to_formatted_string(&Locale::en)
+}
+
+/// Format large value in locale appropriate style.
+pub(crate) fn format_value<T>(value: &Value<T>) -> String
+where
+    T: DeltaValue + ToFormattedString,
+    <T as DeltaValue>::Delta: ToFormattedString,
+{
+    value.formatted_number(&Locale::en)
 }
 
 /// A helper function that merges together times.