Merge pull request #405 from Boavizta/379-expose-individual-metrics-l…

…abel-metrics-with-resource-ids-and-other-metadata 379 expose individual metrics label metrics with resource ids and other metadata
Boavizta · Jan 10, 2024 · 667c429 · 667c429
2 parents 755ca3b + 4623ff5
commit 667c429
Show file tree

Hide file tree

Showing 12 changed files with 476 additions and 115 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,22 +5,32 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## Unreleased (main branch but not tagged)
+## Unreleased 
 
-_This paragraph may describe WIP/unreleased features_
+_This paragraph may describe WIP/unreleased features. They are merged to main branch but not tagged._
 
 ### Added
 
-- Return instance state (either *Running* or *Stopped*) with the inventory. https://github.com/Boavizta/cloud-scanner/issues/396.
+### Changed
+
+## [2.0.0]-2024-01-10
+
+### Added
+
+- Return instance state (either *Running* or *Stopped*) with the inventory: [Add instance state to the inventory](https://github.com/Boavizta/cloud-scanner/issues/396).
+- Return metrics of individual resources: [Expose individual metrics (label metrics with resource id's and other metadata)](https://github.com/Boavizta/cloud-scanner/issues/379)
 
 ### Changed
 
-- Use API v1.1.0 in docker-compose (support aditional instances): https://github.com/Boavizta/cloud-scanner/issues/386
-- Update logo in documentation https://github.com/Boavizta/cloud-scanner/pull/381
-- Add link checker when pubishing doc https://github.com/Boavizta/cloud-scanner/pull/382
-- Add logo in the doc website https://github.com/Boavizta/cloud-scanner/pull/383
+- **Breaking change**: Renamed the count summary metrics (_instances_ become _resources_ because we now take into account additional resources like storage):
+  - `boavizta_number_of_instances_total` becomes `boavizta_number_of_resources_total`
+  - `boavizta_number_of_instances_assessed` becomes `boavizta_number_of_resources_assessed`
+- Use Boavizta API v1.1.0 in docker-compose (this adds support for additional instances): https://github.com/Boavizta/cloud-scanner/issues/386
+- Update logo in documentation: https://github.com/Boavizta/cloud-scanner/pull/381
+- Add link checker when publishing documentation: https://github.com/Boavizta/cloud-scanner/pull/382
+- Add logo in the doc website: https://github.com/Boavizta/cloud-scanner/pull/383
 
-## [1.0.1]-2023-10-27
+## [1.0.1]-2023-10-28
 
 ### Added
 

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/cloud-scanner-cli/Cargo.toml b/cloud-scanner-cli/Cargo.toml
@@ -2,7 +2,7 @@
 authors = ["boavizta.org", "Olivier de Meringo <demeringo@gmail.com>"]
 edition = "2021"
 name = "cloud-scanner-cli"
-version = "1.0.1"
+version = "2.0.0"
 
 [dependencies]
 aws-types = "0.56.1"

diff --git a/cloud-scanner-cli/src/boavizta_api_v1.rs b/cloud-scanner-cli/src/boavizta_api_v1.rs
@@ -113,9 +113,35 @@ impl BoaviztaApiV1 {
                             }
                         }
                     }
+                    "gp2" | "gp3" => {
+                        // Use impacts of an SSD
+                        let res = component_api::disk_impact_bottom_up_v1_component_ssd_post(
+                            &self.configuration,
+                            Some(verbose),
+                            Some(usage_duration_hours.to_owned()),
+                            Some("DEFAULT"),
+                            Some(criteria),
+                            Some(disk),
+                        )
+                        .await;
+                        match res {
+                            Ok(res) => Some(res),
+                            Err(e) => {
+                                warn!(
+                                    "Warning: Cannot get SSD impact from API for type {}: {}",
+                                    storage_type, e
+                                );
+                                None
+                            }
+                        }
+                    }
                     _ => {
-                        error!("Query ssd {:?}", disk);
-                        // All other types (like gp2, gp3...) are considered SSD
+                        warn!(
+                            "Unknown disk type ({:?}), we use impacts of an ssd {:?}",
+                            storage_type.as_str(),
+                            disk
+                        );
+                        // All other types are considered SSD
                         let res = component_api::disk_impact_bottom_up_v1_component_ssd_post(
                             &self.configuration,
                             Some(verbose),
@@ -294,7 +320,7 @@ mod tests {
                 usage: Some(InstanceUsage {
                     average_cpu_load: 100.0,
                     usage_duration_seconds: 3600,
-                    state: InstanceState::Running
+                    state: InstanceState::Running,
                 }),
             },
             tags: Vec::new(),

diff --git a/cloud-scanner-cli/src/impact_provider.rs b/cloud-scanner-cli/src/impact_provider.rs
@@ -52,9 +52,9 @@ pub struct ResourceImpacts {
 /// The aggregated impacts and meta data about the scan results
 #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
 pub struct ImpactsSummary {
-    pub number_of_instances_total: u32,
-    pub number_of_instances_assessed: u32,
-    pub number_of_instances_not_assessed: u32,
+    pub number_of_resources_total: usize,
+    pub number_of_resources_assessed: usize,
+    pub number_of_resources_not_assessed: usize,
     pub duration_of_use_hours: f64,
     pub adp_manufacture_kgsbeq: f64,
     pub adp_use_kgsbeq: f64,
@@ -75,9 +75,9 @@ impl ImpactsSummary {
         duration_of_use_hours: f64,
     ) -> Self {
         let mut summary = ImpactsSummary {
-            number_of_instances_total: u32::try_from(resources.len()).unwrap(),
-            number_of_instances_assessed: 0,
-            number_of_instances_not_assessed: 0,
+            number_of_resources_total: resources.len(),
+            number_of_resources_assessed: 0,
+            number_of_resources_not_assessed: 0,
             aws_region,
             country,
             duration_of_use_hours,
@@ -92,7 +92,7 @@ impl ImpactsSummary {
         for resource in resources {
             // Only consider the instances for which we have impact data
             if let Some(impacts) = resource.resource_impacts {
-                summary.number_of_instances_assessed += 1;
+                summary.number_of_resources_assessed += 1;
                 summary.adp_manufacture_kgsbeq += impacts.adp_manufacture_kgsbeq;
                 summary.adp_use_kgsbeq += impacts.adp_use_kgsbeq;
                 summary.pe_manufacture_megajoules += impacts.pe_manufacture_megajoules;
@@ -102,7 +102,7 @@ impl ImpactsSummary {
             } else {
                 // Resource was not counted due to no impact
                 debug!("Skipped counting resource: {:#?} while building summary because it has no impact data", resource);
-                summary.number_of_instances_not_assessed += 1;
+                summary.number_of_resources_not_assessed += 1;
             }
         }
         summary

diff --git a/cloud-scanner-cli/src/lib.rs b/cloud-scanner-cli/src/lib.rs
@@ -123,22 +123,24 @@ pub async fn get_default_impacts_as_metrics(
     let summary: ImpactsSummary = ImpactsSummary::new(
         String::from(aws_region),
         usage_location.iso_country_code,
-        instances_with_impacts.impacts,
+        instances_with_impacts.impacts.clone(),
         (*hours_use_time).into(),
     );
     debug!("Summary: {:#?}", summary);
 
-    let metrics = get_metrics(&summary).with_context(|| {
-        format!(
-            "Unable to get default impacts as metrics for {}",
-            aws_region
-        )
-    })?;
 
-    Ok(metrics)
+    let all_metrics =
+        get_all_metrics(&summary, instances_with_impacts.impacts).with_context(|| {
+            format!(
+                "Unable to get resource impacts as metrics for region {}",
+                aws_region
+            )
+        })?;
+
+    Ok(all_metrics)
 }
 
-/// Prints  impacts  to standard output in json format
+/// Prints  impacts to standard output in json format
 pub async fn print_default_impacts_as_json(
     hours_use_time: &f32,
     tags: &[String],
@@ -160,7 +162,7 @@ pub async fn print_default_impacts_as_json(
     Ok(())
 }
 
-/// Prints impacts  to standard output as metrics in prometheus format
+/// Prints impacts to standard output as metrics in prometheus format
 pub async fn print_default_impacts_as_metrics(
     hours_use_time: &f32,
     tags: &[String],