From 3c450efb31968e4782634c34657c31d9dcff386f Mon Sep 17 00:00:00 2001 From: Maxime Leblanc Date: Fri, 18 Oct 2024 09:49:13 +0200 Subject: [PATCH 1/6] [EXPORTERS]: elastic search log message within `message` key instead of `body` According to ECS logging reference https://www.elastic.co/guide/en/ecs/8.11/ecs-base.html#field-message Refs #3091 --- exporters/elasticsearch/src/es_log_recordable.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporters/elasticsearch/src/es_log_recordable.cc b/exporters/elasticsearch/src/es_log_recordable.cc index 663691273b..a1e3b9542d 100644 --- a/exporters/elasticsearch/src/es_log_recordable.cc +++ b/exporters/elasticsearch/src/es_log_recordable.cc @@ -232,7 +232,7 @@ void ElasticSearchRecordable::SetSeverity(opentelemetry::logs::Severity severity void ElasticSearchRecordable::SetBody(const opentelemetry::common::AttributeValue &message) noexcept { - WriteValue(message, "body"); + WriteValue(message, "message"); } void ElasticSearchRecordable::SetTraceId(const opentelemetry::trace::TraceId &trace_id) noexcept From 8be43d28a2d9296d727e452e768a575bb538aeeb Mon Sep 17 00:00:00 2001 From: Maxime Leblanc Date: Fri, 18 Oct 2024 09:54:47 +0200 Subject: [PATCH 2/6] [EXPORTERS]: elastic search set severity within `log.level` key instead of `severity` According to ECS logging reference https://www.elastic.co/guide/en/ecs/8.11/ecs-log.html#field-log-level Refs #3091 --- exporters/elasticsearch/src/es_log_recordable.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/exporters/elasticsearch/src/es_log_recordable.cc b/exporters/elasticsearch/src/es_log_recordable.cc index a1e3b9542d..2581e7841b 100644 --- a/exporters/elasticsearch/src/es_log_recordable.cc +++ b/exporters/elasticsearch/src/es_log_recordable.cc @@ -216,17 +216,19 @@ void ElasticSearchRecordable::SetObservedTimestamp( void ElasticSearchRecordable::SetSeverity(opentelemetry::logs::Severity severity) noexcept { + auto &severityField = json_["log"]["level"]; + // Convert the severity enum to a string std::uint32_t severity_index = static_cast(severity); if (severity_index >= std::extent::value) { std::stringstream sout; sout << "Invalid severity(" << severity_index << ")"; - json_["severity"] = sout.str(); + severityField = sout.str(); } else { - json_["severity"] = opentelemetry::logs::SeverityNumToText[severity_index]; + severityField = opentelemetry::logs::SeverityNumToText[severity_index]; } } From 53d72bff2364df7795035babf4598b4f09f971f4 Mon Sep 17 00:00:00 2001 From: Maxime Leblanc Date: Fri, 18 Oct 2024 10:23:12 +0200 Subject: [PATCH 3/6] [EXPORTERS]: elastic search set timestamp within `@timestamp` instead of `timestamp` Also changes the format to be a Date string. According to ECS logging reference https://www.elastic.co/guide/en/ecs/8.11/ecs-base.html#field-timestamp Refs #3091 --- .../elasticsearch/src/es_log_recordable.cc | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/exporters/elasticsearch/src/es_log_recordable.cc b/exporters/elasticsearch/src/es_log_recordable.cc index 2581e7841b..c7d2ad3b94 100644 --- a/exporters/elasticsearch/src/es_log_recordable.cc +++ b/exporters/elasticsearch/src/es_log_recordable.cc @@ -205,7 +205,38 @@ nlohmann::json ElasticSearchRecordable::GetJSON() noexcept void ElasticSearchRecordable::SetTimestamp( opentelemetry::common::SystemTimestamp timestamp) noexcept { - json_["timestamp"] = timestamp.time_since_epoch().count(); + const std::chrono::system_clock::time_point timePoint{timestamp}; + + // If built with with at least cpp 20 then use std::format + // Otherwise use the old style to format the timestamp in UTC +#if __cplusplus >= 202002L + const std::string dateStr = std::format("{:%FT%T%Ez}", timePoint); +#else + const static int dateToSecondsSize = 19; + const static int millisecondsSize = 8; + const static int timeZoneSize = 1; + const static int dateSize = dateToSecondsSize + millisecondsSize + timeZoneSize; + + std::time_t time = std::chrono::system_clock::to_time_t(timePoint); + std::tm tm = *std::gmtime(&time); + + char bufferDate[dateSize]; // example: 2024-10-18T07:26:00.123456Z + std::strftime(bufferDate, sizeof(bufferDate), "%Y-%m-%dT%H:%M:%S", &tm); + auto microseconds = + std::chrono::duration_cast(timePoint.time_since_epoch()) % + std::chrono::seconds(1); + + char bufferMilliseconds[millisecondsSize]; + std::snprintf(bufferMilliseconds, sizeof(bufferMilliseconds), ".%06ld", + static_cast(microseconds.count())); + + std::strcat(bufferDate, bufferMilliseconds); + std::strcat(bufferDate, "Z"); + + const std::string dateStr(bufferDate); +#endif + + json_["@timestamp"] = dateStr; } void ElasticSearchRecordable::SetObservedTimestamp( From ff35e7aaf3389fdc66bcb7499d6f9ace4439f56c Mon Sep 17 00:00:00 2001 From: Maxime Leblanc Date: Fri, 18 Oct 2024 10:27:40 +0200 Subject: [PATCH 4/6] [EXPORTERS]: elastic search set instrumentation scope within `log.logger` instead of `name` According to ECS logging reference https://www.elastic.co/guide/en/ecs/8.11/ecs-log.html#field-log-logger Refs #3091 --- exporters/elasticsearch/src/es_log_recordable.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporters/elasticsearch/src/es_log_recordable.cc b/exporters/elasticsearch/src/es_log_recordable.cc index c7d2ad3b94..80ca2e5ac6 100644 --- a/exporters/elasticsearch/src/es_log_recordable.cc +++ b/exporters/elasticsearch/src/es_log_recordable.cc @@ -324,7 +324,7 @@ void ElasticSearchRecordable::SetInstrumentationScope( const opentelemetry::sdk::instrumentationscope::InstrumentationScope &instrumentation_scope) noexcept { - json_["name"] = instrumentation_scope.GetName(); + json_["log"]["logger"] = instrumentation_scope.GetName(); } } // namespace logs From 401ad81a9c37638c342fd11577d86e200b60a638 Mon Sep 17 00:00:00 2001 From: Maxime Leblanc Date: Fri, 18 Oct 2024 10:55:00 +0200 Subject: [PATCH 5/6] EXPORTERS]: elastic search recorable constructor sets `ecs.version` field to 8.11.0 According to ECS guidelines this field is mandatory https://www.elastic.co/guide/en/ecs/8.11/ecs-guidelines.html Refs #3091 --- .../exporters/elasticsearch/es_log_recordable.h | 2 ++ exporters/elasticsearch/src/es_log_recordable.cc | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h b/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h index af4ccb16ef..52d24cada6 100644 --- a/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h +++ b/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h @@ -42,6 +42,8 @@ class ElasticSearchRecordable final : public sdk::logs::Recordable void WriteValue(const opentelemetry::common::AttributeValue &value, const std::string &name); public: + ElasticSearchRecordable() noexcept; + /** * Returns a JSON object contain the log information */ diff --git a/exporters/elasticsearch/src/es_log_recordable.cc b/exporters/elasticsearch/src/es_log_recordable.cc index 80ca2e5ac6..6bdad97235 100644 --- a/exporters/elasticsearch/src/es_log_recordable.cc +++ b/exporters/elasticsearch/src/es_log_recordable.cc @@ -197,6 +197,11 @@ void ElasticSearchRecordable::WriteValue(const opentelemetry::common::AttributeV } } +ElasticSearchRecordable::ElasticSearchRecordable() noexcept : sdk::logs::Recordable() +{ + json_["ecs"]["version"] = "8.11.0"; +} + nlohmann::json ElasticSearchRecordable::GetJSON() noexcept { return json_; From 6d72624edbe9fb6a727df6ceef9bd7157c51edec Mon Sep 17 00:00:00 2001 From: Maxime Leblanc Date: Fri, 18 Oct 2024 10:48:44 +0200 Subject: [PATCH 6/6] [EXPORTERS]: elastic search put attributes in json root instead of under `attributes` This allows user to set other fields that are part of the [ECS log documentation](https://www.elastic.co/guide/en/ecs/8.11/ecs-log.html). For instance, it allows to have an attribute with key `log.file`, that will, thanks to `nlohmann::json`, appear as : ``` { "log": { "file": "xxx" } } ``` Closes #3091 --- exporters/elasticsearch/src/es_log_recordable.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporters/elasticsearch/src/es_log_recordable.cc b/exporters/elasticsearch/src/es_log_recordable.cc index 6bdad97235..56d7cea71b 100644 --- a/exporters/elasticsearch/src/es_log_recordable.cc +++ b/exporters/elasticsearch/src/es_log_recordable.cc @@ -313,7 +313,7 @@ void ElasticSearchRecordable::SetAttribute( nostd::string_view key, const opentelemetry::common::AttributeValue &value) noexcept { - WriteKeyValue(key, value, "attributes"); + WriteValue(value, key.data()); } void ElasticSearchRecordable::SetResource(