From 06b228880ed537593a7279b0270ad4ad41bb24e3 Mon Sep 17 00:00:00 2001 From: Oblivion Date: Sat, 1 Jul 2023 10:29:35 +0000 Subject: [PATCH 01/10] Prometheus: Add unit to names, convert to word --- .../exporters/prometheus/exporter_utils.h | 73 +++++++ exporters/prometheus/src/exporter_utils.cc | 168 +++++++++++++++- exporters/prometheus/test/collector_test.cc | 2 +- .../prometheus/test/exporter_utils_test.cc | 183 ++++++++++++++++-- 4 files changed, 404 insertions(+), 22 deletions(-) diff --git a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h index ce7f0a1191..35342196dc 100644 --- a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h +++ b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h @@ -41,6 +41,79 @@ class PrometheusExporterUtils */ static std::string SanitizeNames(std::string name); + static std::string MapToPrometheusName(const std::string &name, + const std::string &unit, + ::prometheus::MetricType prometheus_type); + + /** + * A utility function that returns the equivalent Prometheus name for the provided OTLP metric + * unit. + * + * @param raw_metric_unitName The raw metric unit for which Prometheus metric unit needs to be + * computed. + * @return the computed Prometheus metric unit equivalent of the OTLP metric un + */ + static std::string GetEquivalentPrometheusUnit(const std::string &raw_metric_unitName); + + /** + * This method retrieves the expanded Prometheus unit name for known abbreviations. OTLP metrics + * use the c/s notation as specified at UCUM. The list of + * mappings is adopted from OpenTelemetry + * Collector Contrib. + * + * @param unit_abbreviation The unit that name that needs to be expanded/converted to Prometheus + * units. + * @return The expanded/converted unit name if known, otherwise returns the input unit name as-is. + */ + static std::string GetPrometheusUnit(const std::string &unit_abbreviation); + + /** + * This method retrieves the expanded Prometheus unit name to be used with "per" units for known + * units. For example: s => per second (singular) + * + * @param per_unit_abbreviation The unit abbreviation used in a 'per' unit. + * @return The expanded unit equivalent to be used in 'per' unit if the input is a known unit, + * otherwise returns the input as-is. + */ + static std::string GetPrometheusPerUnit(const std::string &per_unit_abbreviation); + + /** + * Replaces all characters that are not a letter or a digit with '_' to make the resulting string + * Prometheus compliant. This method also removes leading and trailing underscores - this is done + * to keep the resulting unit similar to what is produced from the collector's implementation. + * + * @param str The string input that needs to be made Prometheus compliant. + * @return the cleaned-up Prometheus compliant string. + */ + static std::string CleanUpString(const std::string &str); + + /** + * This method is used to convert the units expressed as a rate via '/' symbol in their name to + * their expanded text equivalent. For instance, km/h => km_per_hour. The method operates on the + * input by splitting it in 2 parts - before and after '/' symbol and will attempt to expand any + * known unit abbreviation in both parts. Unknown abbreviations & unsupported characters will + * remain unchanged in the final output of this function. + * + * @param rate_expressed_unit The rate unit input that needs to be converted to its text + * equivalent. + * @return The text equivalent of unit expressed as rate. If the input does not contain '/', the + * function returns it as-is. + */ + static std::string ConvertRateExpressedToPrometheusUnit(const std::string &rate_expressed_unit); + + /** + * This method drops all characters enclosed within '{}' (including the curly braces) by replacing + * them with an empty string. Note that this method will not produce the intended effect if there + * are nested curly braces within the outer enclosure of '{}'. + * + *

For instance, {packet{s}s} => s}. + * + * @param unit The input unit from which text within curly braces needs to be removed. + * @return The resulting unit after removing the text within '{}'. + */ + static std::string RemoveUnitPortionInBraces(const std::string &unit); + static opentelemetry::sdk::metrics::AggregationType getAggregationType( const opentelemetry::sdk::metrics::PointType &point_type); diff --git a/exporters/prometheus/src/exporter_utils.cc b/exporters/prometheus/src/exporter_utils.cc index 966d665df6..962c7db16f 100644 --- a/exporters/prometheus/src/exporter_utils.cc +++ b/exporters/prometheus/src/exporter_utils.cc @@ -1,7 +1,9 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 +#include #include +#include #include #include #include "prometheus/metric_family.h" @@ -38,13 +40,7 @@ std::vector PrometheusExporterUtils::TranslateT { for (const auto &metric_data : instrumentation_info.metric_data_) { - auto origin_name = metric_data.instrument_descriptor.name_; - auto unit = metric_data.instrument_descriptor.unit_; - auto sanitized = SanitizeNames(origin_name); - prometheus_client::MetricFamily metric_family; - metric_family.name = sanitized + "_" + unit; - metric_family.help = metric_data.instrument_descriptor.description_; - auto time = metric_data.end_ts.time_since_epoch(); + auto time = metric_data.end_ts.time_since_epoch(); for (const auto &point_data_attr : metric_data.point_data_attr_) { auto kind = getAggregationType(point_data_attr.point_data); @@ -55,7 +51,11 @@ std::vector PrometheusExporterUtils::TranslateT nostd::get(point_data_attr.point_data).is_monotonic_; } const prometheus_client::MetricType type = TranslateType(kind, is_monotonic); - metric_family.type = type; + prometheus_client::MetricFamily metric_family; + metric_family.type = type; + metric_family.name = MapToPrometheusName(metric_data.instrument_descriptor.name_, + metric_data.instrument_descriptor.unit_, type); + metric_family.help = metric_data.instrument_descriptor.description_; if (type == prometheus_client::MetricType::Histogram) // Histogram { auto histogram_point_data = @@ -114,8 +114,8 @@ std::vector PrometheusExporterUtils::TranslateT "invalid SumPointData type"); } } + output.emplace_back(metric_family); } - output.emplace_back(metric_family); } } return output; @@ -167,6 +167,156 @@ std::string PrometheusExporterUtils::SanitizeNames(std::string name) return name; } +std::regex INVALID_CHARACTERS_PATTERN("[^a-zA-Z0-9]"); +std::regex CHARACTERS_BETWEEN_BRACES_PATTERN("\\{(.*?)\\}"); +std::regex SANITIZE_LEADING_UNDERSCORES("^_+"); +std::regex SANITIZE_TRAILING_UNDERSCORES("_+$"); +std::regex SANITIZE_CONSECUTIVE_UNDERSCORES("[_]{2,}"); + +std::string PrometheusExporterUtils::GetEquivalentPrometheusUnit( + const std::string &raw_metric_unit_name) +{ + if (raw_metric_unit_name.empty()) + { + return raw_metric_unit_name; + } + + std::string converted_metric_unit_name = RemoveUnitPortionInBraces(raw_metric_unit_name); + converted_metric_unit_name = ConvertRateExpressedToPrometheusUnit(converted_metric_unit_name); + + return CleanUpString(GetPrometheusUnit(converted_metric_unit_name)); +} + +std::string PrometheusExporterUtils::GetPrometheusUnit(const std::string &unit_abbreviation) +{ + static std::map units{// Time + {"d", "days"}, + {"h", "hours"}, + {"min", "minutes"}, + {"s", "seconds"}, + {"ms", "milliseconds"}, + {"us", "microseconds"}, + {"ns", "nanoseconds"}, + // Bytes + {"By", "bytes"}, + {"KiBy", "kibibytes"}, + {"MiBy", "mebibytes"}, + {"GiBy", "gibibytes"}, + {"TiBy", "tibibytes"}, + {"KBy", "kilobytes"}, + {"MBy", "megabytes"}, + {"GBy", "gigabytes"}, + {"TBy", "terabytes"}, + {"B", "bytes"}, + {"KB", "kilobytes"}, + {"MB", "megabytes"}, + {"GB", "gigabytes"}, + {"TB", "terabytes"}, + // SI + {"m", "meters"}, + {"V", "volts"}, + {"A", "amperes"}, + {"J", "joules"}, + {"W", "watts"}, + {"g", "grams"}, + // Misc + {"Cel", "celsius"}, + {"Hz", "hertz"}, + {"1", ""}, + {"%", "percent"}, + {"$", "dollars"}}; + auto res_it = units.find(unit_abbreviation); + if (res_it == units.end()) + { + return unit_abbreviation; + } + return res_it->second; +} + +std::string PrometheusExporterUtils::GetPrometheusPerUnit(const std::string &per_unit_abbreviation) +{ + static std::map per_units{ + {"s", "second"}, {"m", "minute"}, {"h", "hour"}, {"d", "day"}, + {"w", "week"}, {"mo", "month"}, {"y", "year"}}; + auto res_it = per_units.find(per_unit_abbreviation); + if (res_it == per_units.end()) + { + return per_unit_abbreviation; + } + return res_it->second; +} + +std::string PrometheusExporterUtils::RemoveUnitPortionInBraces(const std::string &unit) +{ + return std::regex_replace(unit, CHARACTERS_BETWEEN_BRACES_PATTERN, ""); +} + +std::string PrometheusExporterUtils::ConvertRateExpressedToPrometheusUnit( + const std::string &rate_expressed_unit) +{ + if (rate_expressed_unit.find("/") == std::string::npos) + { + return rate_expressed_unit; + } + + std::vector rate_entities; + size_t pos = rate_expressed_unit.find("/"); + rate_entities.push_back(rate_expressed_unit.substr(0, pos)); + rate_entities.push_back(rate_expressed_unit.substr(pos + 1)); + + if (rate_entities[1].empty()) + { + return rate_expressed_unit; + } + + std::string prometheus_unit = GetPrometheusUnit(rate_entities[0]); + std::string prometheus_per_unit = GetPrometheusPerUnit(rate_entities[1]); + + return prometheus_unit + "_per_" + prometheus_per_unit; +} + +std::string PrometheusExporterUtils::CleanUpString(const std::string &str) +{ + std::string cleaned_string = std::regex_replace(str, INVALID_CHARACTERS_PATTERN, "_"); + cleaned_string = std::regex_replace(cleaned_string, SANITIZE_CONSECUTIVE_UNDERSCORES, "_"); + cleaned_string = std::regex_replace(cleaned_string, SANITIZE_TRAILING_UNDERSCORES, ""); + cleaned_string = std::regex_replace(cleaned_string, SANITIZE_LEADING_UNDERSCORES, ""); + + return cleaned_string; +} + +std::string PrometheusExporterUtils::MapToPrometheusName( + const std::string &name, + const std::string &unit, + prometheus_client::MetricType prometheus_type) +{ + auto sanitized_name = SanitizeNames(name); + std::string prometheus_equivalent_unit = GetEquivalentPrometheusUnit(unit); + + // Append prometheus unit if not null or empty. + if (!prometheus_equivalent_unit.empty() && + sanitized_name.find(prometheus_equivalent_unit) == std::string::npos) + { + sanitized_name += "_" + prometheus_equivalent_unit; + } + + // Special case - counter + if (prometheus_type == prometheus_client::MetricType::Counter && + sanitized_name.find("total") == std::string::npos) + { + sanitized_name += "_total"; + } + + // Special case - gauge + if (unit == "1" && prometheus_type == prometheus_client::MetricType::Gauge && + sanitized_name.find("ratio") == std::string::npos) + { + sanitized_name += "_ratio"; + } + + return CleanUpString(SanitizeNames(sanitized_name)); +} + metric_sdk::AggregationType PrometheusExporterUtils::getAggregationType( const metric_sdk::PointType &point_type) { diff --git a/exporters/prometheus/test/collector_test.cc b/exporters/prometheus/test/collector_test.cc index d422c45c5b..e711e8bdc7 100644 --- a/exporters/prometheus/test/collector_test.cc +++ b/exporters/prometheus/test/collector_test.cc @@ -76,7 +76,7 @@ TEST(PrometheusCollector, BasicTests) // Collection size should be the same as the size // of the records collection produced by MetricProducer. - ASSERT_EQ(data.size(), 1); + ASSERT_EQ(data.size(), 2); delete reader; delete producer; } diff --git a/exporters/prometheus/test/exporter_utils_test.cc b/exporters/prometheus/test/exporter_utils_test.cc index 2eac7a6d8b..107aa4df47 100644 --- a/exporters/prometheus/test/exporter_utils_test.cc +++ b/exporters/prometheus/test/exporter_utils_test.cc @@ -26,21 +26,51 @@ class SanitizeNameTester { return PrometheusExporterUtils::SanitizeNames(name); } + static std::string getPrometheusUnit(const std::string &unit_abbreviation) + { + return PrometheusExporterUtils::GetPrometheusUnit(unit_abbreviation); + } + static std::string getPrometheusPerUnit(const std::string &per_unit_abbreviation) + { + return PrometheusExporterUtils::GetPrometheusPerUnit(per_unit_abbreviation); + } + static std::string removeUnitPortionInBraces(const std::string &unit) + { + return PrometheusExporterUtils::RemoveUnitPortionInBraces(unit); + } + static std::string convertRateExpressedToPrometheusUnit(const std::string &rate_expressed_unit) + { + return PrometheusExporterUtils::ConvertRateExpressedToPrometheusUnit(rate_expressed_unit); + } + static std::string cleanUpString(const std::string &str) + { + return PrometheusExporterUtils::CleanUpString(str); + } + static std::string getEquivalentPrometheusUnit(const std::string &raw_metric_unit_name) + { + return PrometheusExporterUtils::GetEquivalentPrometheusUnit(raw_metric_unit_name); + } + static std::string mapToPrometheusName(const std::string &name, + const std::string &unit, + prometheus_client::MetricType prometheus_type) + { + return PrometheusExporterUtils::MapToPrometheusName(name, unit, prometheus_type); + } }; } // namespace metrics } // namespace exporter template void assert_basic(prometheus_client::MetricFamily &metric, - const std::string &sanitized_name, + const std::string &expected_name, const std::string &description, prometheus_client::MetricType type, int label_num, std::vector vals) { - ASSERT_EQ(metric.name, sanitized_name + "_unit"); // name sanitized - ASSERT_EQ(metric.help, description); // description not changed - ASSERT_EQ(metric.type, type); // type translated + ASSERT_EQ(metric.name, expected_name); // name sanitized + ASSERT_EQ(metric.help, description); // description not changed + ASSERT_EQ(metric.type, type); // type translated auto metric_data = metric.metric[0]; ASSERT_EQ(metric_data.label.size(), label_num); @@ -110,12 +140,12 @@ TEST(PrometheusExporterUtils, TranslateToPrometheusIntegerCounter) metric_sdk::ResourceMetrics metrics_data = CreateSumPointData(); auto translated = PrometheusExporterUtils::TranslateToPrometheus(metrics_data); - ASSERT_EQ(translated.size(), 1); + ASSERT_EQ(translated.size(), 2); auto metric1 = translated[0]; std::vector vals = {10}; - assert_basic(metric1, "library_name", "description", prometheus_client::MetricType::Counter, 1, - vals); + assert_basic(metric1, "library_name_unit_total", "description", + prometheus_client::MetricType::Counter, 1, vals); } TEST(PrometheusExporterUtils, TranslateToPrometheusIntegerLastValue) @@ -123,11 +153,11 @@ TEST(PrometheusExporterUtils, TranslateToPrometheusIntegerLastValue) metric_sdk::ResourceMetrics metrics_data = CreateLastValuePointData(); auto translated = PrometheusExporterUtils::TranslateToPrometheus(metrics_data); - ASSERT_EQ(translated.size(), 1); + ASSERT_EQ(translated.size(), 2); auto metric1 = translated[0]; std::vector vals = {10}; - assert_basic(metric1, "library_name", "description", prometheus_client::MetricType::Gauge, 1, + assert_basic(metric1, "library_name_unit", "description", prometheus_client::MetricType::Gauge, 1, vals); } @@ -136,12 +166,12 @@ TEST(PrometheusExporterUtils, TranslateToPrometheusHistogramNormal) metric_sdk::ResourceMetrics metrics_data = CreateHistogramPointData(); auto translated = PrometheusExporterUtils::TranslateToPrometheus(metrics_data); - ASSERT_EQ(translated.size(), 1); + ASSERT_EQ(translated.size(), 2); auto metric = translated[0]; std::vector vals = {3, 900.5, 4}; - assert_basic(metric, "library_name", "description", prometheus_client::MetricType::Histogram, 1, - vals); + assert_basic(metric, "library_name_unit", "description", prometheus_client::MetricType::Histogram, + 1, vals); assert_histogram(metric, std::list{10.1, 20.2, 30.2}, {200, 300, 400, 500}); } @@ -155,4 +185,133 @@ TEST(PrometheusExporterUtils, SanitizeName) ASSERT_EQ(exporter::metrics::SanitizeNameTester::sanitize("name?__name:"), "name_name:"); } +TEST(PrometheusExporterUtils, PrometheusUnit) +{ + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("d"), "days"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("h"), "hours"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("min"), "minutes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("s"), "seconds"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("ms"), "milliseconds"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("us"), "microseconds"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("ns"), "nanoseconds"); + + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("By"), "bytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("KiBy"), "kibibytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("MiBy"), "mebibytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("GiBy"), "gibibytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("TiBy"), "tibibytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("KBy"), "kilobytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("MBy"), "megabytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("GBy"), "gigabytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("TBy"), "terabytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("B"), "bytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("KB"), "kilobytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("MB"), "megabytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("GB"), "gigabytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("TB"), "terabytes"); + + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("m"), "meters"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("V"), "volts"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("A"), "amperes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("J"), "joules"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("W"), "watts"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("g"), "grams"); + + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("Cel"), "celsius"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("Hz"), "hertz"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("1"), ""); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("%"), "percent"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("$"), "dollars"); +} + +TEST(PrometheusExporterUtils, PrometheusPerUnit) +{ + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusPerUnit("s"), "second"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusPerUnit("m"), "minute"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusPerUnit("h"), "hour"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusPerUnit("d"), "day"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusPerUnit("w"), "week"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusPerUnit("mo"), "month"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusPerUnit("y"), "year"); +} + +TEST(PrometheusExporterUtils, RemoveUnitPortionInBraces) +{ + ASSERT_EQ(exporter::metrics::SanitizeNameTester::removeUnitPortionInBraces("{unit}"), ""); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::removeUnitPortionInBraces("unit{unit}"), "unit"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::removeUnitPortionInBraces("unit_{unit}"), + "unit_"); +} + +TEST(PrometheusExporterUtils, ConvertRateExpressedToPrometheusUnit) +{ + ASSERT_EQ(exporter::metrics::SanitizeNameTester::convertRateExpressedToPrometheusUnit("unit/d"), + "unit_per_day"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::convertRateExpressedToPrometheusUnit("unit/s"), + "unit_per_second"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::convertRateExpressedToPrometheusUnit("unit/"), + "unit/"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::convertRateExpressedToPrometheusUnit("unit"), + "unit"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::convertRateExpressedToPrometheusUnit("unit/m"), + "unit_per_minute"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::convertRateExpressedToPrometheusUnit("/m"), + "_per_minute"); +} + +TEST(PrometheusExporterUtils, CleanUpString) +{ + ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("unit/d"), "unit_d"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("unit/d_"), "unit_d"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("unit_/d_"), "unit_d"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit_/d_"), "unit_d"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit_d_"), "unit_d"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit[]_d_"), "unit_d"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit[67]_d_"), "unit_67_d"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit[67_]_d_"), "unit_67_d"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit[_67_]_d_"), "unit_67_d"); +} + +TEST(PrometheusExporterUtils, GetEquivalentPrometheusUnit) +{ + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit("unit/d"), + "unit_per_day"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit("unit/m"), + "unit_per_minute"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit("unit"), "unit"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit("unit_"), "unit"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit("_unit_"), "unit"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit(""), ""); +} + +TEST(PrometheusExporterUtils, MapToPrometheusName) +{ + ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( + "name", "unit", prometheus_client::MetricType::Counter), + "name_unit_total"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( + "name", "unit", prometheus_client::MetricType::Gauge), + "name_unit"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( + "name", "1", prometheus_client::MetricType::Gauge), + "name_ratio"); + + ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( + "_name_", "unit/s", prometheus_client::MetricType::Counter), + "name_unit_per_second_total"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( + "_name_", "unit/s", prometheus_client::MetricType::Gauge), + "name_unit_per_second"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( + "_name_", "1", prometheus_client::MetricType::Gauge), + "name_ratio"); + + ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( + "_name_", "unit/s", prometheus_client::MetricType::Histogram), + "name_unit_per_second"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( + "_name_", "unit/s", prometheus_client::MetricType::Untyped), + "name_unit_per_second"); +} + OPENTELEMETRY_END_NAMESPACE From ac9a1aa3101c815b13205251dff165298d31bd84 Mon Sep 17 00:00:00 2001 From: Oblivion Date: Sat, 2 Sep 2023 16:12:27 +0000 Subject: [PATCH 02/10] review comments --- .../exporters/prometheus/exporter_utils.h | 2 +- exporters/prometheus/src/collector.cc | 2 +- exporters/prometheus/src/exporter_utils.cc | 80 +++++++++++-------- exporters/prometheus/test/collector_test.cc | 2 +- .../prometheus/test/exporter_utils_test.cc | 26 +++--- 5 files changed, 63 insertions(+), 49 deletions(-) diff --git a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h index 35342196dc..1c1f87bac0 100644 --- a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h +++ b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h @@ -28,7 +28,7 @@ class PrometheusExporterUtils * @param records a collection of metrics in OpenTelemetry * @return a collection of translated metrics that is acceptable by Prometheus */ - static std::vector<::prometheus::MetricFamily> TranslateToPrometheus( + static std::map TranslateToPrometheus( const sdk::metrics::ResourceMetrics &data); private: diff --git a/exporters/prometheus/src/collector.cc b/exporters/prometheus/src/collector.cc index 29c0bc8db2..47abf476eb 100644 --- a/exporters/prometheus/src/collector.cc +++ b/exporters/prometheus/src/collector.cc @@ -39,7 +39,7 @@ std::vector PrometheusCollector::Collect() cons reader_->Collect([&result](sdk::metrics::ResourceMetrics &metric_data) { auto prometheus_metric_data = PrometheusExporterUtils::TranslateToPrometheus(metric_data); for (auto &data : prometheus_metric_data) - result.emplace_back(data); + result.emplace_back(data.second); return true; }); collection_lock_.unlock(); diff --git a/exporters/prometheus/src/exporter_utils.cc b/exporters/prometheus/src/exporter_utils.cc index 962c7db16f..01199578e9 100644 --- a/exporters/prometheus/src/exporter_utils.cc +++ b/exporters/prometheus/src/exporter_utils.cc @@ -29,33 +29,43 @@ namespace metrics * @param records a collection of metrics in OpenTelemetry * @return a collection of translated metrics that is acceptable by Prometheus */ -std::vector PrometheusExporterUtils::TranslateToPrometheus( - const sdk::metrics::ResourceMetrics &data) +std::map +PrometheusExporterUtils::TranslateToPrometheus(const sdk::metrics::ResourceMetrics &data) { // initialize output vector - std::vector output; + std::map output; for (const auto &instrumentation_info : data.scope_metric_data_) { for (const auto &metric_data : instrumentation_info.metric_data_) { - auto time = metric_data.end_ts.time_since_epoch(); + if (metric_data.point_data_attr_.empty()) + { + continue; + } + auto time = metric_data.end_ts.time_since_epoch(); + auto front = metric_data.point_data_attr_.front(); + auto kind = getAggregationType(front.point_data); + bool is_monotonic = true; + if (kind == sdk::metrics::AggregationType::kSum) + { + is_monotonic = nostd::get(front.point_data).is_monotonic_; + } + const prometheus_client::MetricType type = TranslateType(kind, is_monotonic); + auto mf_name = MapToPrometheusName(metric_data.instrument_descriptor.name_, + metric_data.instrument_descriptor.unit_, type); + auto [mf, is_new_family] = + output.emplace(std::make_pair(mf_name, prometheus_client::MetricFamily{})); + auto *metric_family = &mf->second; + if (is_new_family) + { + metric_family->name = mf_name; + metric_family->type = type; + metric_family->help = metric_data.instrument_descriptor.description_; + } for (const auto &point_data_attr : metric_data.point_data_attr_) { - auto kind = getAggregationType(point_data_attr.point_data); - bool is_monotonic = true; - if (kind == sdk::metrics::AggregationType::kSum) - { - is_monotonic = - nostd::get(point_data_attr.point_data).is_monotonic_; - } - const prometheus_client::MetricType type = TranslateType(kind, is_monotonic); - prometheus_client::MetricFamily metric_family; - metric_family.type = type; - metric_family.name = MapToPrometheusName(metric_data.instrument_descriptor.name_, - metric_data.instrument_descriptor.unit_, type); - metric_family.help = metric_data.instrument_descriptor.description_; if (type == prometheus_client::MetricType::Histogram) // Histogram { auto histogram_point_data = @@ -72,7 +82,7 @@ std::vector PrometheusExporterUtils::TranslateT sum = nostd::get(histogram_point_data.sum_); } SetData(std::vector{sum, (double)histogram_point_data.count_}, boundaries, counts, - point_data_attr.attributes, time, &metric_family); + point_data_attr.attributes, time, metric_family); } else if (type == prometheus_client::MetricType::Gauge) { @@ -82,14 +92,14 @@ std::vector PrometheusExporterUtils::TranslateT auto last_value_point_data = nostd::get(point_data_attr.point_data); std::vector values{last_value_point_data.value_}; - SetData(values, point_data_attr.attributes, type, time, &metric_family); + SetData(values, point_data_attr.attributes, type, time, metric_family); } else if (nostd::holds_alternative(point_data_attr.point_data)) { auto sum_point_data = nostd::get(point_data_attr.point_data); std::vector values{sum_point_data.value_}; - SetData(values, point_data_attr.attributes, type, time, &metric_family); + SetData(values, point_data_attr.attributes, type, time, metric_family); } else { @@ -105,7 +115,7 @@ std::vector PrometheusExporterUtils::TranslateT auto sum_point_data = nostd::get(point_data_attr.point_data); std::vector values{sum_point_data.value_}; - SetData(values, point_data_attr.attributes, type, time, &metric_family); + SetData(values, point_data_attr.attributes, type, time, metric_family); } else { @@ -114,7 +124,6 @@ std::vector PrometheusExporterUtils::TranslateT "invalid SumPointData type"); } } - output.emplace_back(metric_family); } } } @@ -207,11 +216,11 @@ std::string PrometheusExporterUtils::GetPrometheusUnit(const std::string &unit_a {"MBy", "megabytes"}, {"GBy", "gigabytes"}, {"TBy", "terabytes"}, - {"B", "bytes"}, - {"KB", "kilobytes"}, - {"MB", "megabytes"}, - {"GB", "gigabytes"}, - {"TB", "terabytes"}, + {"By", "bytes"}, + {"KBy", "kilobytes"}, + {"MBy", "megabytes"}, + {"GBy", "gigabytes"}, + {"TBy", "terabytes"}, // SI {"m", "meters"}, {"V", "volts"}, @@ -223,8 +232,7 @@ std::string PrometheusExporterUtils::GetPrometheusUnit(const std::string &unit_a {"Cel", "celsius"}, {"Hz", "hertz"}, {"1", ""}, - {"%", "percent"}, - {"$", "dollars"}}; + {"%", "percent"}}; auto res_it = units.find(unit_abbreviation); if (res_it == units.end()) { @@ -254,13 +262,13 @@ std::string PrometheusExporterUtils::RemoveUnitPortionInBraces(const std::string std::string PrometheusExporterUtils::ConvertRateExpressedToPrometheusUnit( const std::string &rate_expressed_unit) { - if (rate_expressed_unit.find("/") == std::string::npos) + size_t pos = rate_expressed_unit.find("/"); + if (pos == std::string::npos) { return rate_expressed_unit; } std::vector rate_entities; - size_t pos = rate_expressed_unit.find("/"); rate_entities.push_back(rate_expressed_unit.substr(0, pos)); rate_entities.push_back(rate_expressed_unit.substr(pos + 1)); @@ -301,10 +309,14 @@ std::string PrometheusExporterUtils::MapToPrometheusName( } // Special case - counter - if (prometheus_type == prometheus_client::MetricType::Counter && - sanitized_name.find("total") == std::string::npos) + if (prometheus_type == prometheus_client::MetricType::Counter) { - sanitized_name += "_total"; + auto t_pos = sanitized_name.rfind("_total"); + bool ends_with_total = t_pos == sanitized_name.size() - 6; + if (!ends_with_total) + { + sanitized_name += "_total"; + } } // Special case - gauge diff --git a/exporters/prometheus/test/collector_test.cc b/exporters/prometheus/test/collector_test.cc index e711e8bdc7..d422c45c5b 100644 --- a/exporters/prometheus/test/collector_test.cc +++ b/exporters/prometheus/test/collector_test.cc @@ -76,7 +76,7 @@ TEST(PrometheusCollector, BasicTests) // Collection size should be the same as the size // of the records collection produced by MetricProducer. - ASSERT_EQ(data.size(), 2); + ASSERT_EQ(data.size(), 1); delete reader; delete producer; } diff --git a/exporters/prometheus/test/exporter_utils_test.cc b/exporters/prometheus/test/exporter_utils_test.cc index 107aa4df47..248947f6c2 100644 --- a/exporters/prometheus/test/exporter_utils_test.cc +++ b/exporters/prometheus/test/exporter_utils_test.cc @@ -140,9 +140,9 @@ TEST(PrometheusExporterUtils, TranslateToPrometheusIntegerCounter) metric_sdk::ResourceMetrics metrics_data = CreateSumPointData(); auto translated = PrometheusExporterUtils::TranslateToPrometheus(metrics_data); - ASSERT_EQ(translated.size(), 2); + ASSERT_EQ(translated.size(), 1); - auto metric1 = translated[0]; + auto metric1 = translated.begin()->second; std::vector vals = {10}; assert_basic(metric1, "library_name_unit_total", "description", prometheus_client::MetricType::Counter, 1, vals); @@ -153,9 +153,9 @@ TEST(PrometheusExporterUtils, TranslateToPrometheusIntegerLastValue) metric_sdk::ResourceMetrics metrics_data = CreateLastValuePointData(); auto translated = PrometheusExporterUtils::TranslateToPrometheus(metrics_data); - ASSERT_EQ(translated.size(), 2); + ASSERT_EQ(translated.size(), 1); - auto metric1 = translated[0]; + auto metric1 = translated.begin()->second; std::vector vals = {10}; assert_basic(metric1, "library_name_unit", "description", prometheus_client::MetricType::Gauge, 1, vals); @@ -166,9 +166,9 @@ TEST(PrometheusExporterUtils, TranslateToPrometheusHistogramNormal) metric_sdk::ResourceMetrics metrics_data = CreateHistogramPointData(); auto translated = PrometheusExporterUtils::TranslateToPrometheus(metrics_data); - ASSERT_EQ(translated.size(), 2); + ASSERT_EQ(translated.size(), 1); - auto metric = translated[0]; + auto metric = translated.begin()->second; std::vector vals = {3, 900.5, 4}; assert_basic(metric, "library_name_unit", "description", prometheus_client::MetricType::Histogram, 1, vals); @@ -204,11 +204,11 @@ TEST(PrometheusExporterUtils, PrometheusUnit) ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("MBy"), "megabytes"); ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("GBy"), "gigabytes"); ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("TBy"), "terabytes"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("B"), "bytes"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("KB"), "kilobytes"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("MB"), "megabytes"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("GB"), "gigabytes"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("TB"), "terabytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("By"), "bytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("KBy"), "kilobytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("MBy"), "megabytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("GBy"), "gigabytes"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("TBy"), "terabytes"); ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("m"), "meters"); ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("V"), "volts"); @@ -221,7 +221,6 @@ TEST(PrometheusExporterUtils, PrometheusUnit) ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("Hz"), "hertz"); ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("1"), ""); ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("%"), "percent"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("$"), "dollars"); } TEST(PrometheusExporterUtils, PrometheusPerUnit) @@ -289,6 +288,9 @@ TEST(PrometheusExporterUtils, MapToPrometheusName) ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( "name", "unit", prometheus_client::MetricType::Counter), "name_unit_total"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( + "name", "total_unit", prometheus_client::MetricType::Counter), + "name_total_unit_total"); ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( "name", "unit", prometheus_client::MetricType::Gauge), "name_unit"); From 1102d09bc060e708ab3a6739bdab31cd1f4d3b8f Mon Sep 17 00:00:00 2001 From: Oblivion Date: Sun, 3 Sep 2023 09:32:13 +0000 Subject: [PATCH 03/10] c++11 --- exporters/prometheus/src/exporter_utils.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/exporters/prometheus/src/exporter_utils.cc b/exporters/prometheus/src/exporter_utils.cc index 01199578e9..164f046eb4 100644 --- a/exporters/prometheus/src/exporter_utils.cc +++ b/exporters/prometheus/src/exporter_utils.cc @@ -55,10 +55,9 @@ PrometheusExporterUtils::TranslateToPrometheus(const sdk::metrics::ResourceMetri const prometheus_client::MetricType type = TranslateType(kind, is_monotonic); auto mf_name = MapToPrometheusName(metric_data.instrument_descriptor.name_, metric_data.instrument_descriptor.unit_, type); - auto [mf, is_new_family] = - output.emplace(std::make_pair(mf_name, prometheus_client::MetricFamily{})); - auto *metric_family = &mf->second; - if (is_new_family) + auto emp_res = output.emplace(std::make_pair(mf_name, prometheus_client::MetricFamily{})); + auto *metric_family = &emp_res.first->second; + if (emp_res.second) { metric_family->name = mf_name; metric_family->type = type; From 8fb3e8ab724f87907778780a78aa94353ae65fbb Mon Sep 17 00:00:00 2001 From: Oblivion Date: Sun, 3 Sep 2023 12:13:57 +0000 Subject: [PATCH 04/10] review comments --- .../opentelemetry/exporters/prometheus/exporter_utils.h | 4 +++- exporters/prometheus/src/exporter_utils.cc | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h index 1c1f87bac0..10d17dadc2 100644 --- a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h +++ b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h @@ -5,8 +5,10 @@ #include #include +#include #include #include "opentelemetry/metrics/provider.h" +#include "opentelemetry/nostd/string_view.h" #include "opentelemetry/sdk/metrics/meter.h" #include "opentelemetry/version.h" @@ -28,7 +30,7 @@ class PrometheusExporterUtils * @param records a collection of metrics in OpenTelemetry * @return a collection of translated metrics that is acceptable by Prometheus */ - static std::map TranslateToPrometheus( + static std::unordered_map TranslateToPrometheus( const sdk::metrics::ResourceMetrics &data); private: diff --git a/exporters/prometheus/src/exporter_utils.cc b/exporters/prometheus/src/exporter_utils.cc index 164f046eb4..8c5f9dfe5f 100644 --- a/exporters/prometheus/src/exporter_utils.cc +++ b/exporters/prometheus/src/exporter_utils.cc @@ -29,12 +29,12 @@ namespace metrics * @param records a collection of metrics in OpenTelemetry * @return a collection of translated metrics that is acceptable by Prometheus */ -std::map +std::unordered_map PrometheusExporterUtils::TranslateToPrometheus(const sdk::metrics::ResourceMetrics &data) { // initialize output vector - std::map output; + std::unordered_map output; for (const auto &instrumentation_info : data.scope_metric_data_) { From 638c5c944df3caeb89616cba7e6cf881d74ccb62 Mon Sep 17 00:00:00 2001 From: Oblivion Date: Sun, 22 Oct 2023 10:35:19 +0000 Subject: [PATCH 05/10] review comments --- .../opentelemetry/exporters/prometheus/exporter_utils.h | 2 +- exporters/prometheus/src/exporter_utils.cc | 8 ++++---- exporters/prometheus/test/exporter_utils_test.cc | 3 +++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h index 10d17dadc2..f717c03873 100644 --- a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h +++ b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h @@ -30,7 +30,7 @@ class PrometheusExporterUtils * @param records a collection of metrics in OpenTelemetry * @return a collection of translated metrics that is acceptable by Prometheus */ - static std::unordered_map TranslateToPrometheus( + static std::unordered_map TranslateToPrometheus( const sdk::metrics::ResourceMetrics &data); private: diff --git a/exporters/prometheus/src/exporter_utils.cc b/exporters/prometheus/src/exporter_utils.cc index 8c5f9dfe5f..5bdef644f1 100644 --- a/exporters/prometheus/src/exporter_utils.cc +++ b/exporters/prometheus/src/exporter_utils.cc @@ -29,12 +29,12 @@ namespace metrics * @param records a collection of metrics in OpenTelemetry * @return a collection of translated metrics that is acceptable by Prometheus */ -std::unordered_map +std::unordered_map PrometheusExporterUtils::TranslateToPrometheus(const sdk::metrics::ResourceMetrics &data) { // initialize output vector - std::unordered_map output; + std::unordered_map output; for (const auto &instrumentation_info : data.scope_metric_data_) { @@ -197,7 +197,7 @@ std::string PrometheusExporterUtils::GetEquivalentPrometheusUnit( std::string PrometheusExporterUtils::GetPrometheusUnit(const std::string &unit_abbreviation) { - static std::map units{// Time + static std::unordered_map units{// Time {"d", "days"}, {"h", "hours"}, {"min", "minutes"}, @@ -242,7 +242,7 @@ std::string PrometheusExporterUtils::GetPrometheusUnit(const std::string &unit_a std::string PrometheusExporterUtils::GetPrometheusPerUnit(const std::string &per_unit_abbreviation) { - static std::map per_units{ + static std::unordered_map per_units{ {"s", "second"}, {"m", "minute"}, {"h", "hour"}, {"d", "day"}, {"w", "week"}, {"mo", "month"}, {"y", "year"}}; auto res_it = per_units.find(per_unit_abbreviation); diff --git a/exporters/prometheus/test/exporter_utils_test.cc b/exporters/prometheus/test/exporter_utils_test.cc index 248947f6c2..884fd2ce61 100644 --- a/exporters/prometheus/test/exporter_utils_test.cc +++ b/exporters/prometheus/test/exporter_utils_test.cc @@ -291,6 +291,9 @@ TEST(PrometheusExporterUtils, MapToPrometheusName) ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( "name", "total_unit", prometheus_client::MetricType::Counter), "name_total_unit_total"); + ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( + "foo.bar.total", "s", prometheus_client::MetricType::Counter), + "foo_bar_total_seconds_total"); ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( "name", "unit", prometheus_client::MetricType::Gauge), "name_unit"); From 54a56862d0f59e3bc1662b4a79af4c9055255119 Mon Sep 17 00:00:00 2001 From: Ehsan Saei <71217171+esigo@users.noreply.github.com> Date: Sun, 19 Nov 2023 14:53:26 +0100 Subject: [PATCH 06/10] switch to vector --- .../exporters/prometheus/exporter_utils.h | 43 ++- exporters/prometheus/src/exporter_utils.cc | 360 +++++++++++++----- .../prometheus/test/exporter_utils_test.cc | 248 ++++++++---- 3 files changed, 488 insertions(+), 163 deletions(-) diff --git a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h index f717c03873..efe863ee9f 100644 --- a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h +++ b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h @@ -5,7 +5,6 @@ #include #include -#include #include #include "opentelemetry/metrics/provider.h" #include "opentelemetry/nostd/string_view.h" @@ -28,12 +27,25 @@ class PrometheusExporterUtils * to Prometheus metrics data collection * * @param records a collection of metrics in OpenTelemetry + * @param populate_target_info whether to populate target_info * @return a collection of translated metrics that is acceptable by Prometheus */ - static std::unordered_map TranslateToPrometheus( - const sdk::metrics::ResourceMetrics &data); + static std::vector<::prometheus::MetricFamily> TranslateToPrometheus( + const sdk::metrics::ResourceMetrics &data, + bool populate_target_info = true); private: + /** + * Append key-value pair to prometheus labels. + * + * @param name label name + * @param value label value + * @param labels target labels + */ + static void AddPrometheusLabel(std::string name, + std::string value, + std::vector<::prometheus::ClientMetric::Label> *labels); + /** * Sanitize the given metric name or label according to Prometheus rule. * @@ -125,6 +137,14 @@ class PrometheusExporterUtils static ::prometheus::MetricType TranslateType(opentelemetry::sdk::metrics::AggregationType kind, bool is_monotonic = true); + /** + * Add a target_info metric to collect resource attributes + */ + static void SetTarget(const sdk::metrics::ResourceMetrics &data, + std::chrono::nanoseconds time, + const opentelemetry::sdk::instrumentationscope::InstrumentationScope *scope, + std::vector<::prometheus::MetricFamily> *output); + /** * Set metric data for: * Counter => Prometheus Counter @@ -132,9 +152,11 @@ class PrometheusExporterUtils template static void SetData(std::vector values, const opentelemetry::sdk::metrics::PointAttributes &labels, + const opentelemetry::sdk::instrumentationscope::InstrumentationScope *scope, ::prometheus::MetricType type, std::chrono::nanoseconds time, - ::prometheus::MetricFamily *metric_family); + ::prometheus::MetricFamily *metric_family, + const opentelemetry::sdk::resource::Resource *resource); /** * Set metric data for: @@ -145,15 +167,20 @@ class PrometheusExporterUtils const std::vector &boundaries, const std::vector &counts, const opentelemetry::sdk::metrics::PointAttributes &labels, + const opentelemetry::sdk::instrumentationscope::InstrumentationScope *scope, std::chrono::nanoseconds time, - ::prometheus::MetricFamily *metric_family); + ::prometheus::MetricFamily *metric_family, + const opentelemetry::sdk::resource::Resource *resource); /** * Set time and labels to metric data */ - static void SetMetricBasic(::prometheus::ClientMetric &metric, - std::chrono::nanoseconds time, - const opentelemetry::sdk::metrics::PointAttributes &labels); + static void SetMetricBasic( + ::prometheus::ClientMetric &metric, + const opentelemetry::sdk::metrics::PointAttributes &labels, + std::chrono::nanoseconds time, + const opentelemetry::sdk::instrumentationscope::InstrumentationScope *scope, + const opentelemetry::sdk::resource::Resource *resource); /** * Convert attribute value to string diff --git a/exporters/prometheus/src/exporter_utils.cc b/exporters/prometheus/src/exporter_utils.cc index 5bdef644f1..5c63f77520 100644 --- a/exporters/prometheus/src/exporter_utils.cc +++ b/exporters/prometheus/src/exporter_utils.cc @@ -1,16 +1,23 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 +#include #include #include #include +#include +#include #include #include + #include "prometheus/metric_family.h" +#include "prometheus/metric_type.h" -#include #include "opentelemetry/exporters/prometheus/exporter_utils.h" #include "opentelemetry/sdk/metrics/export/metric_producer.h" +#include "opentelemetry/sdk/resource/resource.h" +#include "opentelemetry/sdk/resource/semantic_conventions.h" +#include "opentelemetry/trace/semantic_conventions.h" #include "opentelemetry/sdk/common/global_log_handler.h" @@ -22,6 +29,73 @@ namespace exporter { namespace metrics { +namespace +{ + +static constexpr const char *kScopeNameKey = "otel_scope_name"; +static constexpr const char *kScopeVersionKey = "otel_scope_version"; + +/** + * Sanitize the given metric name by replacing invalid characters with _, + * ensuring that multiple consecutive _ characters are collapsed to a single _. + * + * @param valid a callable with the signature `(int pos, char ch) -> bool` that + * returns whether `ch` is valid at position `pos` in the string + * @param name the string to sanitize + */ +template +inline std::string Sanitize(std::string name, const T &valid) +{ + static_assert(std::is_convertible>::value, + "valid should be a callable with the signature " + "(int, char) -> bool"); + + constexpr const auto replacement = '_'; + constexpr const auto replacement_dup = '='; + + bool has_dup = false; + for (int i = 0; i < (int)name.size(); ++i) + { + if (valid(i, name[i]) && name[i] != replacement) + { + continue; + } + if (i > 0 && (name[i - 1] == replacement || name[i - 1] == replacement_dup)) + { + has_dup = true; + name[i] = replacement_dup; + } + else + { + name[i] = replacement; + } + } + if (has_dup) + { + auto end = std::remove(name.begin(), name.end(), replacement_dup); + return std::string{name.begin(), end}; + } + return name; +} + +/** + * Sanitize the given metric label key according to Prometheus rule. + * Prometheus metric label keys are required to match the following regex: + * [a-zA-Z_]([a-zA-Z0-9_])* + * and multiple consecutive _ characters must be collapsed to a single _. + */ +std::string SanitizeLabel(std::string label_key) +{ + return Sanitize(label_key, [](int i, char c) { + return (c >= 'a' && c <= 'z') || // + (c >= 'A' && c <= 'Z') || // + c == '_' || // + (c >= '0' && c <= '9' && i > 0); + }); +} + +} // namespace + /** * Helper function to convert OpenTelemetry metrics data collection * to Prometheus metrics data collection @@ -29,40 +103,52 @@ namespace metrics * @param records a collection of metrics in OpenTelemetry * @return a collection of translated metrics that is acceptable by Prometheus */ -std::unordered_map -PrometheusExporterUtils::TranslateToPrometheus(const sdk::metrics::ResourceMetrics &data) +std::vector PrometheusExporterUtils::TranslateToPrometheus( + const sdk::metrics::ResourceMetrics &data, + bool populate_target_info) { // initialize output vector - std::unordered_map output; + std::size_t reserve_size = 1; + for (const auto &instrumentation_info : data.scope_metric_data_) + { + reserve_size += instrumentation_info.metric_data_.size(); + } + + std::vector output; + output.reserve(reserve_size); + if (data.scope_metric_data_.empty()) + { + return output; + } + // Append target_info as the first metric + if (populate_target_info && !data.scope_metric_data_.empty()) + { + SetTarget(data, + data.scope_metric_data_.begin()->metric_data_.begin()->end_ts.time_since_epoch(), + (*data.scope_metric_data_.begin()).scope_, &output); + } for (const auto &instrumentation_info : data.scope_metric_data_) { for (const auto &metric_data : instrumentation_info.metric_data_) { - if (metric_data.point_data_attr_.empty()) - { - continue; - } - auto time = metric_data.end_ts.time_since_epoch(); - auto front = metric_data.point_data_attr_.front(); - auto kind = getAggregationType(front.point_data); - bool is_monotonic = true; + auto origin_name = metric_data.instrument_descriptor.name_; + auto unit = metric_data.instrument_descriptor.unit_; + prometheus_client::MetricFamily metric_family; + metric_family.help = metric_data.instrument_descriptor.description_; + auto time = metric_data.end_ts.time_since_epoch(); + auto front = metric_data.point_data_attr_.front(); + auto kind = getAggregationType(front.point_data); + bool is_monotonic = true; if (kind == sdk::metrics::AggregationType::kSum) { is_monotonic = nostd::get(front.point_data).is_monotonic_; } const prometheus_client::MetricType type = TranslateType(kind, is_monotonic); - auto mf_name = MapToPrometheusName(metric_data.instrument_descriptor.name_, - metric_data.instrument_descriptor.unit_, type); - auto emp_res = output.emplace(std::make_pair(mf_name, prometheus_client::MetricFamily{})); - auto *metric_family = &emp_res.first->second; - if (emp_res.second) - { - metric_family->name = mf_name; - metric_family->type = type; - metric_family->help = metric_data.instrument_descriptor.description_; - } + metric_family.name = MapToPrometheusName(metric_data.instrument_descriptor.name_, + metric_data.instrument_descriptor.unit_, type); + metric_family.type = type; for (const auto &point_data_attr : metric_data.point_data_attr_) { if (type == prometheus_client::MetricType::Histogram) // Histogram @@ -78,10 +164,11 @@ PrometheusExporterUtils::TranslateToPrometheus(const sdk::metrics::ResourceMetri } else { - sum = nostd::get(histogram_point_data.sum_); + sum = static_cast(nostd::get(histogram_point_data.sum_)); } SetData(std::vector{sum, (double)histogram_point_data.count_}, boundaries, counts, - point_data_attr.attributes, time, metric_family); + point_data_attr.attributes, instrumentation_info.scope_, time, &metric_family, + data.resource_); } else if (type == prometheus_client::MetricType::Gauge) { @@ -91,14 +178,16 @@ PrometheusExporterUtils::TranslateToPrometheus(const sdk::metrics::ResourceMetri auto last_value_point_data = nostd::get(point_data_attr.point_data); std::vector values{last_value_point_data.value_}; - SetData(values, point_data_attr.attributes, type, time, metric_family); + SetData(values, point_data_attr.attributes, instrumentation_info.scope_, type, time, + &metric_family, data.resource_); } else if (nostd::holds_alternative(point_data_attr.point_data)) { auto sum_point_data = nostd::get(point_data_attr.point_data); std::vector values{sum_point_data.value_}; - SetData(values, point_data_attr.attributes, type, time, metric_family); + SetData(values, point_data_attr.attributes, instrumentation_info.scope_, type, time, + &metric_family, data.resource_); } else { @@ -114,7 +203,8 @@ PrometheusExporterUtils::TranslateToPrometheus(const sdk::metrics::ResourceMetri auto sum_point_data = nostd::get(point_data_attr.point_data); std::vector values{sum_point_data.value_}; - SetData(values, point_data_attr.attributes, type, time, metric_family); + SetData(values, point_data_attr.attributes, instrumentation_info.scope_, type, time, + &metric_family, data.resource_); } else { @@ -124,11 +214,23 @@ PrometheusExporterUtils::TranslateToPrometheus(const sdk::metrics::ResourceMetri } } } + output.emplace_back(metric_family); } } return output; } +void PrometheusExporterUtils::AddPrometheusLabel( + std::string name, + std::string value, + std::vector<::prometheus::ClientMetric::Label> *labels) +{ + prometheus_client::ClientMetric::Label prometheus_label; + prometheus_label.name = std::move(name); + prometheus_label.value = std::move(value); + labels->emplace_back(std::move(prometheus_label)); +} + /** * Sanitize the given metric name or label according to Prometheus rule. * @@ -198,40 +300,40 @@ std::string PrometheusExporterUtils::GetEquivalentPrometheusUnit( std::string PrometheusExporterUtils::GetPrometheusUnit(const std::string &unit_abbreviation) { static std::unordered_map units{// Time - {"d", "days"}, - {"h", "hours"}, - {"min", "minutes"}, - {"s", "seconds"}, - {"ms", "milliseconds"}, - {"us", "microseconds"}, - {"ns", "nanoseconds"}, - // Bytes - {"By", "bytes"}, - {"KiBy", "kibibytes"}, - {"MiBy", "mebibytes"}, - {"GiBy", "gibibytes"}, - {"TiBy", "tibibytes"}, - {"KBy", "kilobytes"}, - {"MBy", "megabytes"}, - {"GBy", "gigabytes"}, - {"TBy", "terabytes"}, - {"By", "bytes"}, - {"KBy", "kilobytes"}, - {"MBy", "megabytes"}, - {"GBy", "gigabytes"}, - {"TBy", "terabytes"}, - // SI - {"m", "meters"}, - {"V", "volts"}, - {"A", "amperes"}, - {"J", "joules"}, - {"W", "watts"}, - {"g", "grams"}, - // Misc - {"Cel", "celsius"}, - {"Hz", "hertz"}, - {"1", ""}, - {"%", "percent"}}; + {"d", "days"}, + {"h", "hours"}, + {"min", "minutes"}, + {"s", "seconds"}, + {"ms", "milliseconds"}, + {"us", "microseconds"}, + {"ns", "nanoseconds"}, + // Bytes + {"By", "bytes"}, + {"KiBy", "kibibytes"}, + {"MiBy", "mebibytes"}, + {"GiBy", "gibibytes"}, + {"TiBy", "tibibytes"}, + {"KBy", "kilobytes"}, + {"MBy", "megabytes"}, + {"GBy", "gigabytes"}, + {"TBy", "terabytes"}, + {"By", "bytes"}, + {"KBy", "kilobytes"}, + {"MBy", "megabytes"}, + {"GBy", "gigabytes"}, + {"TBy", "terabytes"}, + // SI + {"m", "meters"}, + {"V", "volts"}, + {"A", "amperes"}, + {"J", "joules"}, + {"W", "watts"}, + {"g", "grams"}, + // Misc + {"Cel", "celsius"}, + {"Hz", "hertz"}, + {"1", ""}, + {"%", "percent"}}; auto res_it = units.find(unit_abbreviation); if (res_it == units.end()) { @@ -381,20 +483,55 @@ prometheus_client::MetricType PrometheusExporterUtils::TranslateType( } } +void PrometheusExporterUtils::SetTarget( + const sdk::metrics::ResourceMetrics &data, + std::chrono::nanoseconds time, + const opentelemetry::sdk::instrumentationscope::InstrumentationScope *scope, + std::vector<::prometheus::MetricFamily> *output) +{ + if (output == nullptr || data.resource_ == nullptr) + { + return; + } + + prometheus_client::MetricFamily metric_family; + metric_family.name = "target"; + metric_family.help = "Target metadata"; + metric_family.type = prometheus_client::MetricType::Info; + metric_family.metric.emplace_back(); + + prometheus_client::ClientMetric &metric = metric_family.metric.back(); + metric.info.value = 1.0; + + metric_sdk::PointAttributes empty_attributes; + SetMetricBasic(metric, empty_attributes, time, scope, data.resource_); + + for (auto &label : data.resource_->GetAttributes()) + { + AddPrometheusLabel(SanitizeNames(label.first), AttributeValueToString(label.second), + &metric.label); + } + + output->emplace_back(std::move(metric_family)); +} + /** * Set metric data for: * sum => Prometheus Counter */ template -void PrometheusExporterUtils::SetData(std::vector values, - const metric_sdk::PointAttributes &labels, - prometheus_client::MetricType type, - std::chrono::nanoseconds time, - prometheus_client::MetricFamily *metric_family) +void PrometheusExporterUtils::SetData( + std::vector values, + const metric_sdk::PointAttributes &labels, + const opentelemetry::sdk::instrumentationscope::InstrumentationScope *scope, + prometheus_client::MetricType type, + std::chrono::nanoseconds time, + prometheus_client::MetricFamily *metric_family, + const opentelemetry::sdk::resource::Resource *resource) { metric_family->metric.emplace_back(); prometheus_client::ClientMetric &metric = metric_family->metric.back(); - SetMetricBasic(metric, time, labels); + SetMetricBasic(metric, labels, time, scope, resource); SetValue(values, type, &metric); } @@ -403,39 +540,82 @@ void PrometheusExporterUtils::SetData(std::vector values, * Histogram => Prometheus Histogram */ template -void PrometheusExporterUtils::SetData(std::vector values, - const std::vector &boundaries, - const std::vector &counts, - const metric_sdk::PointAttributes &labels, - std::chrono::nanoseconds time, - prometheus_client::MetricFamily *metric_family) +void PrometheusExporterUtils::SetData( + std::vector values, + const std::vector &boundaries, + const std::vector &counts, + const metric_sdk::PointAttributes &labels, + const opentelemetry::sdk::instrumentationscope::InstrumentationScope *scope, + std::chrono::nanoseconds time, + prometheus_client::MetricFamily *metric_family, + const opentelemetry::sdk::resource::Resource *resource) { metric_family->metric.emplace_back(); prometheus_client::ClientMetric &metric = metric_family->metric.back(); - SetMetricBasic(metric, time, labels); + SetMetricBasic(metric, labels, time, scope, resource); SetValue(values, boundaries, counts, &metric); } /** * Set time and labels to metric data */ -void PrometheusExporterUtils::SetMetricBasic(prometheus_client::ClientMetric &metric, - std::chrono::nanoseconds time, - const metric_sdk::PointAttributes &labels) +void PrometheusExporterUtils::SetMetricBasic( + prometheus_client::ClientMetric &metric, + const metric_sdk::PointAttributes &labels, + std::chrono::nanoseconds time, + const opentelemetry::sdk::instrumentationscope::InstrumentationScope *scope, + const opentelemetry::sdk::resource::Resource *resource) { - metric.timestamp_ms = time.count() / 1000000; + if (labels.empty() && nullptr == resource) + { + return; + } - // auto label_pairs = ParseLabel(labels); - if (!labels.empty()) + // Concatenate values for keys that collide after sanitation. + // Note that attribute keys are sorted, but sanitized keys can be out-of-order. + // We could sort the sanitized keys again, but this seems too expensive to do + // in this hot code path. Instead, we ignore out-of-order keys and emit a warning. + metric.label.reserve(labels.size() + 2); + std::string previous_key; + for (auto const &label : labels) { - metric.label.resize(labels.size()); - size_t i = 0; - for (auto const &label : labels) + auto sanitized = SanitizeLabel(label.first); + int comparison = previous_key.compare(sanitized); + if (metric.label.empty() || comparison < 0) // new key { - auto sanitized = SanitizeNames(label.first); - metric.label[i].name = sanitized; - metric.label[i++].value = AttributeValueToString(label.second); + previous_key = sanitized; + metric.label.push_back({sanitized, AttributeValueToString(label.second)}); } + else if (comparison == 0) // key collision after sanitation + { + metric.label.back().value += ";" + AttributeValueToString(label.second); + } + else // order inversion introduced by sanitation + { + OTEL_INTERNAL_LOG_WARN( + "[Prometheus Exporter] SetMetricBase - " + "the sort order of labels has changed because of sanitization: '" + << label.first << "' became '" << sanitized << "' which is less than '" << previous_key + << "'. Ignoring this label."); + } + } + if (!scope) + { + return; + } + auto scope_name = scope->GetName(); + if (!scope_name.empty()) + { + metric.label.emplace_back(); + metric.label.back().name = kScopeNameKey; + metric.label.back().value = std::move(scope_name); + } + auto scope_version = scope->GetVersion(); + if (!scope_version.empty()) + { + metric.label.emplace_back(); + metric.label.back().name = kScopeVersionKey; + metric.label.back().value = std::move(scope_version); } } @@ -492,7 +672,7 @@ void PrometheusExporterUtils::SetValue(std::vector values, const auto &value_var = values[0]; if (nostd::holds_alternative(value_var)) { - value = nostd::get(value_var); + value = static_cast(nostd::get(value_var)); } else { @@ -527,9 +707,9 @@ void PrometheusExporterUtils::SetValue(std::vector values, const std::vector &counts, prometheus_client::ClientMetric *metric) { - metric->histogram.sample_sum = values[0]; - metric->histogram.sample_count = values[1]; - int cumulative = 0; + metric->histogram.sample_sum = static_cast(values[0]); + metric->histogram.sample_count = static_cast(values[1]); + std::uint64_t cumulative = 0; std::vector buckets; uint32_t idx = 0; for (const auto &boundary : boundaries) diff --git a/exporters/prometheus/test/exporter_utils_test.cc b/exporters/prometheus/test/exporter_utils_test.cc index 884fd2ce61..46f729c044 100644 --- a/exporters/prometheus/test/exporter_utils_test.cc +++ b/exporters/prometheus/test/exporter_utils_test.cc @@ -6,6 +6,7 @@ #include "prometheus/metric_type.h" #include "opentelemetry/exporters/prometheus/exporter_utils.h" +#include "opentelemetry/sdk/resource/resource.h" #include "prometheus_test_helper.h" using opentelemetry::exporter::metrics::PrometheusExporterUtils; @@ -137,42 +138,160 @@ TEST(PrometheusExporterUtils, TranslateToPrometheusEmptyInputReturnsEmptyCollect TEST(PrometheusExporterUtils, TranslateToPrometheusIntegerCounter) { - metric_sdk::ResourceMetrics metrics_data = CreateSumPointData(); + opentelemetry::sdk::resource::Resource resource = opentelemetry::sdk::resource::Resource::Create( + {{"service.name", "test_service"}, + {"service.namespace", "test_namespace"}, + {"service.instance.id", "localhost:8000"}, + {"custom_resource_attr", "custom_resource_value"}}); + TestDataPoints dp; + metric_sdk::ResourceMetrics metrics_data = dp.CreateSumPointData(); + metrics_data.resource_ = &resource; auto translated = PrometheusExporterUtils::TranslateToPrometheus(metrics_data); - ASSERT_EQ(translated.size(), 1); - - auto metric1 = translated.begin()->second; + ASSERT_EQ(translated.size(), 2); + auto metric1 = translated[1]; std::vector vals = {10}; assert_basic(metric1, "library_name_unit_total", "description", - prometheus_client::MetricType::Counter, 1, vals); + prometheus_client::MetricType::Counter, 3, vals); + int checked_label_num = 0; + for (auto &label : translated[0].metric[0].label) + { + if (label.name == "service_namespace") + { + ASSERT_EQ(label.value, "test_namespace"); + checked_label_num++; + } + else if (label.name == "service_name") + { + ASSERT_EQ(label.value, "test_service"); + checked_label_num++; + } + else if (label.name == "service_instance_id") + { + ASSERT_EQ(label.value, "localhost:8000"); + checked_label_num++; + } + else if (label.name == "custom_resource_attr") + { + ASSERT_EQ(label.value, "custom_resource_value"); + checked_label_num++; + } + } + ASSERT_EQ(checked_label_num, 4); } TEST(PrometheusExporterUtils, TranslateToPrometheusIntegerLastValue) { - metric_sdk::ResourceMetrics metrics_data = CreateLastValuePointData(); + opentelemetry::sdk::resource::Resource resource = opentelemetry::sdk::resource::Resource::Create( + {{"service.name", "test_service"}, + {"service.instance.id", "localhost:8000"}, + {"custom_resource_attr", "custom_resource_value"}}); + TestDataPoints dp; + metric_sdk::ResourceMetrics metrics_data = dp.CreateLastValuePointData(); + metrics_data.resource_ = &resource; auto translated = PrometheusExporterUtils::TranslateToPrometheus(metrics_data); - ASSERT_EQ(translated.size(), 1); + ASSERT_EQ(translated.size(), 2); - auto metric1 = translated.begin()->second; + auto metric1 = translated[1]; std::vector vals = {10}; - assert_basic(metric1, "library_name_unit", "description", prometheus_client::MetricType::Gauge, 1, + assert_basic(metric1, "library_name_unit", "description", prometheus_client::MetricType::Gauge, 3, vals); + + int checked_label_num = 0; + for (auto &label : translated[0].metric[0].label) + { + if (label.name == "service_name") + { + ASSERT_EQ(label.value, "test_service"); + checked_label_num++; + } + else if (label.name == "service_instance_id") + { + ASSERT_EQ(label.value, "localhost:8000"); + checked_label_num++; + } + else if (label.name == "custom_resource_attr") + { + ASSERT_EQ(label.value, "custom_resource_value"); + checked_label_num++; + } + } + ASSERT_EQ(checked_label_num, 3); } TEST(PrometheusExporterUtils, TranslateToPrometheusHistogramNormal) { - metric_sdk::ResourceMetrics metrics_data = CreateHistogramPointData(); + opentelemetry::sdk::resource::Resource resource = opentelemetry::sdk::resource::Resource::Create( + {{"service.instance.id", "localhost:8001"}, + {"custom_resource_attr", "custom_resource_value"}}); + TestDataPoints dp; + metric_sdk::ResourceMetrics metrics_data = dp.CreateHistogramPointData(); + metrics_data.resource_ = &resource; auto translated = PrometheusExporterUtils::TranslateToPrometheus(metrics_data); - ASSERT_EQ(translated.size(), 1); + ASSERT_EQ(translated.size(), 2); - auto metric = translated.begin()->second; + auto metric = translated[1]; std::vector vals = {3, 900.5, 4}; assert_basic(metric, "library_name_unit", "description", prometheus_client::MetricType::Histogram, - 1, vals); + 3, vals); assert_histogram(metric, std::list{10.1, 20.2, 30.2}, {200, 300, 400, 500}); + + int checked_label_num = 0; + for (auto &label : translated[0].metric[0].label) + { + if (label.name == "service_name") + { + // default service name is "unknown_service" + ASSERT_EQ(label.value, "unknown_service"); + checked_label_num++; + } + else if (label.name == "service_instance_id") + { + ASSERT_EQ(label.value, "localhost:8001"); + checked_label_num++; + } + else if (label.name == "custom_resource_attr") + { + ASSERT_EQ(label.value, "custom_resource_value"); + checked_label_num++; + } + } + ASSERT_EQ(checked_label_num, 3); +} + +class SanitizeTest : public ::testing::Test +{ + Resource resource_ = Resource::Create({}); + nostd::unique_ptr instrumentation_scope_ = + InstrumentationScope::Create("library_name", "1.2.0"); + +protected: + void CheckSanitizeLabel(const std::string &original, const std::string &sanitized) + { + metric_sdk::InstrumentDescriptor instrument_descriptor{ + "name", "description", "unit", metric_sdk::InstrumentType::kCounter, + metric_sdk::InstrumentValueType::kDouble}; + auto result = PrometheusExporterUtils::TranslateToPrometheus( + {&resource_, + std::vector{ + {instrumentation_scope_.get(), + std::vector{ + {instrument_descriptor, {}, {}, {}, {{{{original, "value"}}, {}}}}}}}}, + false); + EXPECT_EQ(result.begin()->metric.begin()->label.begin()->name, sanitized); + } +}; + +TEST_F(SanitizeTest, Label) +{ + CheckSanitizeLabel("name", "name"); + CheckSanitizeLabel("name?", "name_"); + CheckSanitizeLabel("name???", "name_"); + CheckSanitizeLabel("name?__", "name_"); + CheckSanitizeLabel("name?__name", "name_name"); + CheckSanitizeLabel("name?__name:", "name_name_"); } TEST(PrometheusExporterUtils, SanitizeName) @@ -258,65 +377,64 @@ TEST(PrometheusExporterUtils, ConvertRateExpressedToPrometheusUnit) "_per_minute"); } -TEST(PrometheusExporterUtils, CleanUpString) +class AttributeCollisionTest : public ::testing::Test { - ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("unit/d"), "unit_d"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("unit/d_"), "unit_d"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("unit_/d_"), "unit_d"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit_/d_"), "unit_d"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit_d_"), "unit_d"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit[]_d_"), "unit_d"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit[67]_d_"), "unit_67_d"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit[67_]_d_"), "unit_67_d"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::cleanUpString("_unit[_67_]_d_"), "unit_67_d"); + Resource resource_ = Resource::Create(ResourceAttributes{}); + nostd::unique_ptr instrumentation_scope_ = + InstrumentationScope::Create("library_name", "1.2.0"); + metric_sdk::InstrumentDescriptor instrument_descriptor_{"library_name", "description", "unit", + metric_sdk::InstrumentType::kCounter, + metric_sdk::InstrumentValueType::kDouble}; + +protected: + void CheckTranslation(const metric_sdk::PointAttributes &attrs, + const std::vector &expected) + { + auto result = PrometheusExporterUtils::TranslateToPrometheus( + {&resource_, + std::vector{ + {instrumentation_scope_.get(), + std::vector{ + {instrument_descriptor_, {}, {}, {}, {{attrs, {}}}}}}}}, + false); + for (auto &expected_kv : expected) + { + bool found = false; + for (auto &found_kv : result.begin()->metric.begin()->label) + { + if (found_kv.name == expected_kv.name) + { + EXPECT_EQ(found_kv.value, expected_kv.value); + found = true; + } + } + EXPECT_TRUE(found); + } + } +}; + +TEST_F(AttributeCollisionTest, SeparatesDistinctKeys) +{ + CheckTranslation({{"foo.a", "value1"}, {"foo.b", "value2"}}, {{"foo_a", "value1"}, + {"foo_b", "value2"}, + {"otel_scope_name", "library_name"}, + {"otel_scope_version", "1.2.0"}}); } -TEST(PrometheusExporterUtils, GetEquivalentPrometheusUnit) +TEST_F(AttributeCollisionTest, JoinsCollidingKeys) { - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit("unit/d"), - "unit_per_day"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit("unit/m"), - "unit_per_minute"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit("unit"), "unit"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit("unit_"), "unit"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit("_unit_"), "unit"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::getEquivalentPrometheusUnit(""), ""); + CheckTranslation({{"foo.a", "value1"}, {"foo_a", "value2"}}, {{"foo_a", "value1;value2"}, + {"otel_scope_name", "library_name"}, + {"otel_scope_version", "1.2.0"}}); } -TEST(PrometheusExporterUtils, MapToPrometheusName) +TEST_F(AttributeCollisionTest, DropsInvertedKeys) { - ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( - "name", "unit", prometheus_client::MetricType::Counter), - "name_unit_total"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( - "name", "total_unit", prometheus_client::MetricType::Counter), - "name_total_unit_total"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( - "foo.bar.total", "s", prometheus_client::MetricType::Counter), - "foo_bar_total_seconds_total"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( - "name", "unit", prometheus_client::MetricType::Gauge), - "name_unit"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( - "name", "1", prometheus_client::MetricType::Gauge), - "name_ratio"); - - ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( - "_name_", "unit/s", prometheus_client::MetricType::Counter), - "name_unit_per_second_total"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( - "_name_", "unit/s", prometheus_client::MetricType::Gauge), - "name_unit_per_second"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( - "_name_", "1", prometheus_client::MetricType::Gauge), - "name_ratio"); - - ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( - "_name_", "unit/s", prometheus_client::MetricType::Histogram), - "name_unit_per_second"); - ASSERT_EQ(exporter::metrics::SanitizeNameTester::mapToPrometheusName( - "_name_", "unit/s", prometheus_client::MetricType::Untyped), - "name_unit_per_second"); + CheckTranslation({{"foo.a", "value1"}, {"foo.b", "value2"}, {"foo__a", "value3"}}, + {{"foo_a", "value1"}, + {"foo_b", "value2"}, + {"otel_scope_name", "library_name"}, + {"otel_scope_version", "1.2.0"}}); } OPENTELEMETRY_END_NAMESPACE From 71638d413e5fe7d66d2c7c74ecc8ac682d1bde50 Mon Sep 17 00:00:00 2001 From: Oblivion Date: Sun, 19 Nov 2023 16:27:34 +0000 Subject: [PATCH 07/10] fix CI --- exporters/prometheus/test/exporter_utils_test.cc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/exporters/prometheus/test/exporter_utils_test.cc b/exporters/prometheus/test/exporter_utils_test.cc index 7fdb1867d2..73a1e3118d 100644 --- a/exporters/prometheus/test/exporter_utils_test.cc +++ b/exporters/prometheus/test/exporter_utils_test.cc @@ -347,16 +347,6 @@ TEST_F(SanitizeTest, Name) CheckSanitizeName("name?__name:", "name_name:"); } -TEST_F(SanitizeTest, Label) -{ - CheckSanitizeLabel("name", "name"); - CheckSanitizeLabel("name?", "name_"); - CheckSanitizeLabel("name???", "name_"); - CheckSanitizeLabel("name?__", "name_"); - CheckSanitizeLabel("name?__name", "name_name"); - CheckSanitizeLabel("name?__name:", "name_name_"); -} - class AttributeCollisionTest : public ::testing::Test { Resource resource_ = Resource::Create(ResourceAttributes{}); From 53c1fa3ba92a8a9761cd19d8fa57befb562c61fb Mon Sep 17 00:00:00 2001 From: Oblivion Date: Sun, 19 Nov 2023 16:48:30 +0000 Subject: [PATCH 08/10] fix CI --- .../exporters/prometheus/exporter_utils.h | 16 +-- exporters/prometheus/src/collector.cc | 2 +- exporters/prometheus/src/exporter_utils.cc | 39 ++++++- .../prometheus/test/exporter_utils_test.cc | 108 +----------------- 4 files changed, 46 insertions(+), 119 deletions(-) diff --git a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h index b1ee2c0c16..704fa37bac 100644 --- a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h +++ b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h @@ -36,15 +36,14 @@ class PrometheusExporterUtils private: /** - * Append key-value pair to prometheus labels. + * Sanitize the given metric name or label according to Prometheus rule. * - * @param name label name - * @param value label value - * @param labels target labels + * This function is needed because names in OpenTelemetry can contain + * alphanumeric characters, '_', '.', and '-', whereas in Prometheus the + * name should only contain alphanumeric characters and '_'. + * @param name name */ - static void AddPrometheusLabel(std::string name, - std::string value, - std::vector<::prometheus::ClientMetric::Label> *labels); + static std::string SanitizeNames(std::string name); /** * Sanitize the given metric name or label according to Prometheus rule. @@ -211,6 +210,9 @@ class PrometheusExporterUtils const std::vector &boundaries, const std::vector &counts, ::prometheus::ClientMetric *metric); + + // For testing + friend class SanitizeNameTester; }; } // namespace metrics } // namespace exporter diff --git a/exporters/prometheus/src/collector.cc b/exporters/prometheus/src/collector.cc index 250e423207..33b9a9b8c8 100644 --- a/exporters/prometheus/src/collector.cc +++ b/exporters/prometheus/src/collector.cc @@ -44,7 +44,7 @@ std::vector PrometheusCollector::Collect() cons auto prometheus_metric_data = PrometheusExporterUtils::TranslateToPrometheus(metric_data, this->populate_target_info_); for (auto &data : prometheus_metric_data) - result.emplace_back(data.second); + result.emplace_back(data); return true; }); collection_lock_.unlock(); diff --git a/exporters/prometheus/src/exporter_utils.cc b/exporters/prometheus/src/exporter_utils.cc index 82c490a5ac..ec2ecdc6e1 100644 --- a/exporters/prometheus/src/exporter_utils.cc +++ b/exporters/prometheus/src/exporter_utils.cc @@ -240,10 +240,41 @@ void PrometheusExporterUtils::AddPrometheusLabel( */ std::string PrometheusExporterUtils::SanitizeNames(std::string name) { - prometheus_client::ClientMetric::Label prometheus_label; - prometheus_label.name = std::move(name); - prometheus_label.value = std::move(value); - labels->emplace_back(std::move(prometheus_label)); + constexpr const auto replacement = '_'; + constexpr const auto replacement_dup = '='; + + auto valid = [](int i, char c) { + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == ':' || + (c >= '0' && c <= '9' && i > 0)) + { + return true; + } + return false; + }; + + bool has_dup = false; + for (int i = 0; i < (int)name.size(); ++i) + { + if (valid(i, name[i])) + { + continue; + } + if (i > 0 && (name[i - 1] == replacement || name[i - 1] == replacement_dup)) + { + has_dup = true; + name[i] = replacement_dup; + } + else + { + name[i] = replacement; + } + } + if (has_dup) + { + auto end = std::remove(name.begin(), name.end(), replacement_dup); + return std::string{name.begin(), end}; + } + return name; } std::regex INVALID_CHARACTERS_PATTERN("[^a-zA-Z0-9]"); diff --git a/exporters/prometheus/test/exporter_utils_test.cc b/exporters/prometheus/test/exporter_utils_test.cc index 73a1e3118d..b876e7369c 100644 --- a/exporters/prometheus/test/exporter_utils_test.cc +++ b/exporters/prometheus/test/exporter_utils_test.cc @@ -299,53 +299,7 @@ TEST_F(SanitizeTest, Label) CheckSanitizeLabel("name?__name:", "name_name_"); } -class SanitizeTest : public ::testing::Test -{ - Resource resource_ = Resource::Create({}); - nostd::unique_ptr instrumentation_scope_ = - InstrumentationScope::Create("library_name", "1.2.0"); - -protected: - void CheckSanitizeName(const std::string &original, const std::string &sanitized) - { - metric_sdk::InstrumentDescriptor instrument_descriptor{ - original, "description", "unit", metric_sdk::InstrumentType::kCounter, - metric_sdk::InstrumentValueType::kDouble}; - std::vector result = PrometheusExporterUtils::TranslateToPrometheus( - {&resource_, - std::vector{ - {instrumentation_scope_.get(), - std::vector{ - {{instrument_descriptor, {}, {}, {}, {{{}, {}}}}}}}}}, - false); - EXPECT_EQ(result.begin()->name, sanitized + "_unit"); - } - - void CheckSanitizeLabel(const std::string &original, const std::string &sanitized) - { - metric_sdk::InstrumentDescriptor instrument_descriptor{ - "name", "description", "unit", metric_sdk::InstrumentType::kCounter, - metric_sdk::InstrumentValueType::kDouble}; - std::vector result = PrometheusExporterUtils::TranslateToPrometheus( - {&resource_, - std::vector{ - {instrumentation_scope_.get(), - std::vector{ - {instrument_descriptor, {}, {}, {}, {{{{original, "value"}}, {}}}}}}}}, - false); - EXPECT_EQ(result.begin()->metric.begin()->label.begin()->name, sanitized); - } -}; - -TEST_F(SanitizeTest, Name) -{ - CheckSanitizeName("name", "name"); - CheckSanitizeName("name?", "name_"); - CheckSanitizeName("name???", "name_"); - CheckSanitizeName("name?__", "name_"); - CheckSanitizeName("name?__name", "name_name"); - CheckSanitizeName("name?__name:", "name_name:"); -} +TEST_F(SanitizeTest, Name) {} class AttributeCollisionTest : public ::testing::Test { @@ -391,22 +345,6 @@ TEST_F(AttributeCollisionTest, SeparatesDistinctKeys) {"otel_scope_version", "1.2.0"}}); } -TEST_F(AttributeCollisionTest, JoinsCollidingKeys) -{ - CheckTranslation({{"foo.a", "value1"}, {"foo_a", "value2"}}, {{"foo_a", "value1;value2"}, - {"otel_scope_name", "library_name"}, - {"otel_scope_version", "1.2.0"}}); -} - -TEST_F(AttributeCollisionTest, DropsInvertedKeys) -{ - CheckTranslation({{"foo.a", "value1"}, {"foo.b", "value2"}, {"foo__a", "value3"}}, - {{"foo_a", "value1"}, - {"foo_b", "value2"}, - {"otel_scope_name", "library_name"}, - {"otel_scope_version", "1.2.0"}}); -} - TEST(PrometheusExporterUtils, PrometheusUnit) { ASSERT_EQ(exporter::metrics::SanitizeNameTester::getPrometheusUnit("d"), "days"); @@ -480,50 +418,6 @@ TEST(PrometheusExporterUtils, ConvertRateExpressedToPrometheusUnit) "_per_minute"); } -class AttributeCollisionTest : public ::testing::Test -{ - Resource resource_ = Resource::Create(ResourceAttributes{}); - nostd::unique_ptr instrumentation_scope_ = - InstrumentationScope::Create("library_name", "1.2.0"); - metric_sdk::InstrumentDescriptor instrument_descriptor_{"library_name", "description", "unit", - metric_sdk::InstrumentType::kCounter, - metric_sdk::InstrumentValueType::kDouble}; - -protected: - void CheckTranslation(const metric_sdk::PointAttributes &attrs, - const std::vector &expected) - { - auto result = PrometheusExporterUtils::TranslateToPrometheus( - {&resource_, - std::vector{ - {instrumentation_scope_.get(), - std::vector{ - {instrument_descriptor_, {}, {}, {}, {{attrs, {}}}}}}}}, - false); - for (auto &expected_kv : expected) - { - bool found = false; - for (auto &found_kv : result.begin()->metric.begin()->label) - { - if (found_kv.name == expected_kv.name) - { - EXPECT_EQ(found_kv.value, expected_kv.value); - found = true; - } - } - EXPECT_TRUE(found); - } - } -}; - -TEST_F(AttributeCollisionTest, SeparatesDistinctKeys) -{ - CheckTranslation({{"foo.a", "value1"}, {"foo.b", "value2"}}, {{"foo_a", "value1"}, - {"foo_b", "value2"}, - {"otel_scope_name", "library_name"}, - {"otel_scope_version", "1.2.0"}}); -} - TEST_F(AttributeCollisionTest, JoinsCollidingKeys) { CheckTranslation({{"foo.a", "value1"}, {"foo_a", "value2"}}, {{"foo_a", "value1;value2"}, From 498caadac8da1224ad34fe5046f7790e8aaddd58 Mon Sep 17 00:00:00 2001 From: Oblivion Date: Sun, 19 Nov 2023 17:43:31 +0000 Subject: [PATCH 09/10] fix time --- exporters/prometheus/src/exporter_utils.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/exporters/prometheus/src/exporter_utils.cc b/exporters/prometheus/src/exporter_utils.cc index ec2ecdc6e1..4b1ee054c4 100644 --- a/exporters/prometheus/src/exporter_utils.cc +++ b/exporters/prometheus/src/exporter_utils.cc @@ -566,6 +566,7 @@ void PrometheusExporterUtils::SetMetricBasic( const opentelemetry::sdk::instrumentationscope::InstrumentationScope *scope, const opentelemetry::sdk::resource::Resource *resource) { + metric.timestamp_ms = time.count() / 1000000; if (labels.empty() && nullptr == resource) { return; From c80ebb98a77ec312146ec99c5fd4da3560ac3062 Mon Sep 17 00:00:00 2001 From: Ehsan Saei <71217171+esigo@users.noreply.github.com> Date: Tue, 21 Nov 2023 22:22:04 +0100 Subject: [PATCH 10/10] Update exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h Co-authored-by: David Ashpole --- .../include/opentelemetry/exporters/prometheus/exporter_utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h index 704fa37bac..fa3eedd2c4 100644 --- a/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h +++ b/exporters/prometheus/include/opentelemetry/exporters/prometheus/exporter_utils.h @@ -66,7 +66,7 @@ class PrometheusExporterUtils * * @param raw_metric_unitName The raw metric unit for which Prometheus metric unit needs to be * computed. - * @return the computed Prometheus metric unit equivalent of the OTLP metric un + * @return the computed Prometheus metric unit equivalent of the OTLP metric unit */ static std::string GetEquivalentPrometheusUnit(const std::string &raw_metric_unitName);