From 80f301805d8908cf063b5d45d92661f64f50632d Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 5 May 2023 21:22:31 -0700 Subject: [PATCH] SDK support for the new OTel log API (#2123) --- CHANGELOG.md | 2 + api/include/opentelemetry/logs/event_id.h | 5 +- api/include/opentelemetry/logs/log_record.h | 8 ++ api/include/opentelemetry/logs/logger.h | 14 ++-- .../opentelemetry/logs/logger_type_traits.h | 5 +- .../elasticsearch/es_log_recordable.h | 10 +++ exporters/ostream/src/log_record_exporter.cc | 4 + exporters/ostream/test/ostream_log_test.cc | 83 +++++++++++++++++++ .../exporters/otlp/otlp_log_recordable.h | 10 +++ .../opentelemetry/sdk/logs/multi_recordable.h | 7 ++ .../sdk/logs/read_write_log_record.h | 22 +++++ .../sdk/logs/readable_log_record.h | 12 +++ sdk/src/logs/multi_recordable.cc | 11 +++ sdk/src/logs/read_write_log_record.cc | 20 ++++- .../logs/batch_log_record_processor_test.cc | 2 + sdk/test/logs/logger_provider_sdk_test.cc | 2 + sdk/test/logs/logger_sdk_test.cc | 8 ++ .../logs/simple_log_record_processor_test.cc | 2 + 18 files changed, 215 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3d83a082..7566b78304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ Increment the: ## [Unreleased] +* [SDK] SDK support for the new OTel log + [#2123](https://github.com/open-telemetry/opentelemetry-cpp/pull/2123) * [BUILD] Build break with old curl, macro CURL_VERSION_BITS unknown [#2102](https://github.com/open-telemetry/opentelemetry-cpp/pull/2102) * [API] Add user facing Logging API and Benchmarks diff --git a/api/include/opentelemetry/logs/event_id.h b/api/include/opentelemetry/logs/event_id.h index 1a1ad6f078..131e9e7111 100644 --- a/api/include/opentelemetry/logs/event_id.h +++ b/api/include/opentelemetry/logs/event_id.h @@ -18,14 +18,15 @@ namespace logs class EventId { public: - EventId(int64_t id, nostd::string_view name) noexcept + EventId(int64_t id, nostd::string_view name) noexcept : id_{id} { - id_ = id; name_ = nostd::unique_ptr{new char[name.length() + 1]}; std::copy(name.begin(), name.end(), name_.get()); name_.get()[name.length()] = 0; } + EventId(int64_t id) noexcept : id_{id}, name_{nullptr} {} + public: int64_t id_; nostd::unique_ptr name_; diff --git a/api/include/opentelemetry/logs/log_record.h b/api/include/opentelemetry/logs/log_record.h index 7fb51c2d99..403238900a 100644 --- a/api/include/opentelemetry/logs/log_record.h +++ b/api/include/opentelemetry/logs/log_record.h @@ -58,6 +58,14 @@ class LogRecord virtual void SetAttribute(nostd::string_view key, const opentelemetry::common::AttributeValue &value) noexcept = 0; + /** + * Set the Event Id. + * @param id The event id to set + * @param name Optional event name to set + */ + // TODO: mark this as pure virtual once all exporters have been updated + virtual void SetEventId(int64_t id, nostd::string_view name = {}) noexcept = 0; + /** * Set the trace id for this log. * @param trace_id the trace id to set diff --git a/api/include/opentelemetry/logs/logger.h b/api/include/opentelemetry/logs/logger.h index 5806c6a0d1..f88784a581 100644 --- a/api/include/opentelemetry/logs/logger.h +++ b/api/include/opentelemetry/logs/logger.h @@ -289,7 +289,7 @@ class Logger nostd::string_view format, const common::KeyValueIterable &attributes) noexcept { - this->EmitLogRecord(severity, event_id, format, attributes); + this->EmitLogRecord(severity, EventId{event_id}, format, attributes); } virtual void Log(Severity severity, @@ -318,7 +318,7 @@ class Logger nostd::string_view format, const common::KeyValueIterable &attributes) noexcept { - this->Log(Severity::kTrace, event_id, format, attributes); + this->Log(Severity::kTrace, EventId{event_id}, format, attributes); } inline void Trace(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept @@ -339,7 +339,7 @@ class Logger nostd::string_view format, const common::KeyValueIterable &attributes) noexcept { - this->Log(Severity::kDebug, event_id, format, attributes); + this->Log(Severity::kDebug, EventId{event_id}, format, attributes); } inline void Debug(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept @@ -360,7 +360,7 @@ class Logger nostd::string_view format, const common::KeyValueIterable &attributes) noexcept { - this->Log(Severity::kInfo, event_id, format, attributes); + this->Log(Severity::kInfo, EventId{event_id}, format, attributes); } inline void Info(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept @@ -381,7 +381,7 @@ class Logger nostd::string_view format, const common::KeyValueIterable &attributes) noexcept { - this->Log(Severity::kWarn, event_id, format, attributes); + this->Log(Severity::kWarn, EventId{event_id}, format, attributes); } inline void Warn(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept @@ -402,7 +402,7 @@ class Logger nostd::string_view format, const common::KeyValueIterable &attributes) noexcept { - this->Log(Severity::kError, event_id, format, attributes); + this->Log(Severity::kError, EventId{event_id}, format, attributes); } inline void Error(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept @@ -423,7 +423,7 @@ class Logger nostd::string_view format, const common::KeyValueIterable &attributes) noexcept { - this->Log(Severity::kFatal, event_id, format, attributes); + this->Log(Severity::kFatal, EventId{event_id}, format, attributes); } inline void Fatal(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept diff --git a/api/include/opentelemetry/logs/logger_type_traits.h b/api/include/opentelemetry/logs/logger_type_traits.h index 82f4ee5278..98457058c2 100644 --- a/api/include/opentelemetry/logs/logger_type_traits.h +++ b/api/include/opentelemetry/logs/logger_type_traits.h @@ -48,9 +48,10 @@ template <> struct LogRecordSetterTrait { template - inline static LogRecord *Set(LogRecord *log_record, ArgumentType && /*arg*/) noexcept + inline static LogRecord *Set(LogRecord *log_record, ArgumentType &&arg) noexcept { - // TODO: set log_record + log_record->SetEventId(arg.id_, nostd::string_view{arg.name_.get()}); + return log_record; } }; diff --git a/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h b/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h index d6ed78d117..c5be2a2cd8 100644 --- a/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h +++ b/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h @@ -72,6 +72,16 @@ class ElasticSearchRecordable final : public sdk::logs::Recordable */ void SetBody(const opentelemetry::common::AttributeValue &message) noexcept override; + /** + * Set the Event Id + * @param id the event id to set + * @param name the event name to set + */ + void SetEventId(int64_t /* id */, nostd::string_view /* name */) noexcept override + { + // TODO: implement event id + } + /** * Set the trace id for this log. * @param trace_id the trace id to set diff --git a/exporters/ostream/src/log_record_exporter.cc b/exporters/ostream/src/log_record_exporter.cc index 751de71e2c..18552d7a65 100644 --- a/exporters/ostream/src/log_record_exporter.cc +++ b/exporters/ostream/src/log_record_exporter.cc @@ -52,6 +52,8 @@ sdk::common::ExportResult OStreamLogRecordExporter::Export( continue; } + int64_t event_id = log_record->GetEventId(); + // Convert trace, spanid, traceflags into exportable representation constexpr int trace_id_len = 32; constexpr int span_id__len = 16; @@ -96,6 +98,8 @@ sdk::common::ExportResult OStreamLogRecordExporter::Export( printAttributes(log_record->GetAttributes(), "\n "); sout_ << "\n" + << " event_id : " << event_id << "\n" + << " event_name : " << log_record->GetEventName() << "\n" << " trace_id : " << std::string(trace_id, trace_id_len) << "\n" << " span_id : " << std::string(span_id, span_id__len) << "\n" << " trace_flags : " << std::string(trace_flags, trace_flags_len) << "\n" diff --git a/exporters/ostream/test/ostream_log_test.cc b/exporters/ostream/test/ostream_log_test.cc index 95cdb69380..30cf64c5eb 100644 --- a/exporters/ostream/test/ostream_log_test.cc +++ b/exporters/ostream/test/ostream_log_test.cc @@ -20,6 +20,8 @@ namespace nostd = opentelemetry::nostd; namespace exporterlogs = opentelemetry::exporter::logs; namespace common = opentelemetry::common; +using Attributes = std::initializer_list>; + OPENTELEMETRY_BEGIN_NAMESPACE namespace exporter { @@ -103,6 +105,8 @@ TEST(OstreamLogExporter, DefaultLogRecordToCout) " telemetry.sdk.name: opentelemetry\n", " telemetry.sdk.language: cpp\n", " attributes : \n", + " event_id : 0\n" + " event_name : \n", " trace_id : 00000000000000000000000000000000\n", " span_id : 0000000000000000\n", " trace_flags : 00\n", @@ -176,6 +180,8 @@ TEST(OStreamLogRecordExporter, SimpleLogToCout) " telemetry.sdk.name: opentelemetry\n", " telemetry.sdk.language: cpp\n", " attributes : \n", + " event_id : 0\n" + " event_name : \n", " trace_id : 00000000000000000000000000000000\n", " span_id : 0000000000000000\n", " trace_flags : 00\n", @@ -248,6 +254,8 @@ TEST(OStreamLogRecordExporter, LogWithStringAttributesToCerr) " key1: val1\n", " attributes : \n", " a: 1\n", + " event_id : 0\n" + " event_name : \n", " trace_id : 00000000000000000000000000000000\n", " span_id : 0000000000000000\n", " trace_flags : 00\n", @@ -327,6 +335,8 @@ TEST(OStreamLogRecordExporter, LogWithVariantTypesToClog) " res1: [1,2,3]\n", " attributes : \n", " attr1: [0,1,0]\n", + " event_id : 0\n" + " event_name : \n", " trace_id : 00000000000000000000000000000000\n", " span_id : 0000000000000000\n", " trace_flags : 00\n", @@ -398,6 +408,79 @@ TEST(OStreamLogRecordExporter, IntegrationTest) " telemetry.sdk.name: opentelemetry\n", " telemetry.sdk.language: cpp\n", " attributes : \n", + " event_id : 0\n" + " event_name : \n", + " trace_id : 00000000000000000000000000000000\n", + " span_id : 0000000000000000\n", + " trace_flags : 00\n", + " scope : \n", + " name : opentelelemtry_library\n", + " version : " OPENTELEMETRY_SDK_VERSION "\n", + " schema_url : https://opentelemetry.io/schemas/1.11.0\n", + " attributes : \n", + " scope.attr.key: 123\n", + "}\n"}; + + std::string ostream_output = stdcoutOutput.str(); + for (auto &expected : expected_output) + { + std::string::size_type result = ostream_output.find(expected); + if (result == std::string::npos) + { + std::cout << "Can not find: \"" << expected << "\" in\n" << ostream_output << std::endl; + } + ASSERT_NE(result, std::string::npos); + } +} + +// Test using the simple log processor and ostream exporter to cout +// and use the rest of the logging pipeline (Logger, LoggerProvider, Provider) and user-facing API +// as well +TEST(OStreamLogRecordExporter, IntegrationTestWithEventId) +{ + // Initialize a logger + auto exporter = + std::unique_ptr(new exporterlogs::OStreamLogRecordExporter); + auto sdkProvider = std::shared_ptr(new sdklogs::LoggerProvider()); + sdkProvider->AddProcessor(std::unique_ptr( + new sdklogs::SimpleLogRecordProcessor(std::move(exporter)))); + auto apiProvider = nostd::shared_ptr(sdkProvider); + auto provider = nostd::shared_ptr(apiProvider); + logs_api::Provider::SetLoggerProvider(provider); + const std::string schema_url{"https://opentelemetry.io/schemas/1.11.0"}; + auto logger = logs_api::Provider::GetLoggerProvider()->GetLogger( + "Logger", "opentelelemtry_library", OPENTELEMETRY_SDK_VERSION, schema_url, true, + {{"scope.attr.key", 123}}); + + // Back up cout's streambuf + std::streambuf *original = std::cout.rdbuf(); + + // Redirect cout to our string stream + std::stringstream stdcoutOutput; + std::cout.rdbuf(stdcoutOutput.rdbuf()); + + logs_api::EventId event_id{12345678, "test_event_id"}; + + // Write a log to ostream exporter + logger->Debug(event_id, "Hello {key1} {key2}", + common::MakeKeyValueIterableView({{"key1", 123}, {"key2", "value2"}})); + + // Restore cout's original streambuf + std::cout.rdbuf(original); + + // Compare actual vs expected outputs + std::vector expected_output{ + " severity_num : 5\n" + " severity_text : DEBUG\n" + " body : Hello {key1} {key2}\n", + " resource : \n", + " telemetry.sdk.version: " OPENTELEMETRY_VERSION "\n", + " service.name: unknown_service\n", + " telemetry.sdk.name: opentelemetry\n", + " telemetry.sdk.language: cpp\n", + " attributes : \n", + " event_id : 12345678\n" + " event_name : test_event_id\n", " trace_id : 00000000000000000000000000000000\n", " span_id : 0000000000000000\n", " trace_flags : 00\n", diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_log_recordable.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_log_recordable.h index cef1b3ef8f..0648fe3308 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_log_recordable.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_log_recordable.h @@ -67,6 +67,16 @@ class OtlpLogRecordable final : public opentelemetry::sdk::logs::Recordable */ void SetBody(const opentelemetry::common::AttributeValue &message) noexcept override; + /** + * @brief Set the Event Id for this log. + * @param id the event Id to set + * @param name the event name to set + */ + void SetEventId(int64_t /* id */, nostd::string_view /* name */) noexcept override + { + // TODO: export Event Id to OTLP + } + /** * Set the trace id for this log. * @param trace_id the trace id to set diff --git a/sdk/include/opentelemetry/sdk/logs/multi_recordable.h b/sdk/include/opentelemetry/sdk/logs/multi_recordable.h index 8bbc2ae200..5944d49b39 100644 --- a/sdk/include/opentelemetry/sdk/logs/multi_recordable.h +++ b/sdk/include/opentelemetry/sdk/logs/multi_recordable.h @@ -55,6 +55,13 @@ class MultiRecordable final : public Recordable */ void SetBody(const opentelemetry::common::AttributeValue &message) noexcept override; + /** + * Set the event id + * @param id the event id to set + * @param name the event name to set + */ + void SetEventId(int64_t id, nostd::string_view name) noexcept override; + /** * Set the trace id for this log. * @param trace_id the trace id to set diff --git a/sdk/include/opentelemetry/sdk/logs/read_write_log_record.h b/sdk/include/opentelemetry/sdk/logs/read_write_log_record.h index c81a2da126..a850072134 100644 --- a/sdk/include/opentelemetry/sdk/logs/read_write_log_record.h +++ b/sdk/include/opentelemetry/sdk/logs/read_write_log_record.h @@ -87,6 +87,25 @@ class ReadWriteLogRecord final : public ReadableLogRecord */ const opentelemetry::common::AttributeValue &GetBody() const noexcept override; + /** + * Set the Event Id object + * @param id the event Id to set + * @param name the event name to set + */ + void SetEventId(int64_t id, nostd::string_view name) noexcept override; + + /** + * Get event Id of this log. + * @return the event Id of this log. + */ + int64_t GetEventId() const noexcept override; + + /** + * Get event name of this log. + * @return the event name of this log. + */ + nostd::string_view GetEventName() const noexcept override; + /** * Set the trace id for this log. * @param trace_id the trace id to set @@ -176,6 +195,9 @@ class ReadWriteLogRecord final : public ReadableLogRecord opentelemetry::common::SystemTimestamp timestamp_; opentelemetry::common::SystemTimestamp observed_timestamp_; + int64_t event_id_; + std::string event_name_; + // We do not pay for trace state when not necessary struct TraceState { diff --git a/sdk/include/opentelemetry/sdk/logs/readable_log_record.h b/sdk/include/opentelemetry/sdk/logs/readable_log_record.h index 7b3d8ad502..90a1c4b7b7 100644 --- a/sdk/include/opentelemetry/sdk/logs/readable_log_record.h +++ b/sdk/include/opentelemetry/sdk/logs/readable_log_record.h @@ -64,6 +64,18 @@ class ReadableLogRecord : public Recordable */ virtual const opentelemetry::common::AttributeValue &GetBody() const noexcept = 0; + /** + * Get the Event id. + * @return the event id + */ + virtual int64_t GetEventId() const noexcept = 0; + + /** + * Get the Event Name. + * @return the event name + */ + virtual nostd::string_view GetEventName() const noexcept = 0; + /** * Get the trace id of this log. * @return the trace id of this log diff --git a/sdk/src/logs/multi_recordable.cc b/sdk/src/logs/multi_recordable.cc index 17e2c29777..c1e335ead2 100644 --- a/sdk/src/logs/multi_recordable.cc +++ b/sdk/src/logs/multi_recordable.cc @@ -101,6 +101,17 @@ void MultiRecordable::SetBody(const opentelemetry::common::AttributeValue &messa } } +void MultiRecordable::SetEventId(int64_t id, nostd::string_view name) noexcept +{ + for (auto &recordable : recordables_) + { + if (recordable.second) + { + recordable.second->SetEventId(id, name); + } + } +} + void MultiRecordable::SetTraceId(const opentelemetry::trace::TraceId &trace_id) noexcept { for (auto &recordable : recordables_) diff --git a/sdk/src/logs/read_write_log_record.cc b/sdk/src/logs/read_write_log_record.cc index e452fa20e9..f45c20adff 100644 --- a/sdk/src/logs/read_write_log_record.cc +++ b/sdk/src/logs/read_write_log_record.cc @@ -19,7 +19,9 @@ ReadWriteLogRecord::ReadWriteLogRecord() resource_(nullptr), instrumentation_scope_(nullptr), body_(nostd::string_view()), - observed_timestamp_(std::chrono::system_clock::now()) + observed_timestamp_(std::chrono::system_clock::now()), + event_id_(0), + event_name_("") {} ReadWriteLogRecord::~ReadWriteLogRecord() {} @@ -65,6 +67,22 @@ const opentelemetry::common::AttributeValue &ReadWriteLogRecord::GetBody() const return body_; } +void ReadWriteLogRecord::SetEventId(int64_t id, nostd::string_view name) noexcept +{ + event_id_ = id; + event_name_ = std::string{name}; +} + +int64_t ReadWriteLogRecord::GetEventId() const noexcept +{ + return event_id_; +} + +nostd::string_view ReadWriteLogRecord::GetEventName() const noexcept +{ + return nostd::string_view{event_name_}; +} + void ReadWriteLogRecord::SetTraceId(const opentelemetry::trace::TraceId &trace_id) noexcept { if (!trace_state_) diff --git a/sdk/test/logs/batch_log_record_processor_test.cc b/sdk/test/logs/batch_log_record_processor_test.cc index 462e9b8397..41ae2c5123 100644 --- a/sdk/test/logs/batch_log_record_processor_test.cc +++ b/sdk/test/logs/batch_log_record_processor_test.cc @@ -42,6 +42,8 @@ class MockLogRecordable final : public opentelemetry::sdk::logs::Recordable void SetBody(const std::string &message) noexcept { body_ = message; } + void SetEventId(int64_t, nostd::string_view) noexcept override {} + void SetTraceId(const opentelemetry::trace::TraceId &) noexcept override {} void SetSpanId(const opentelemetry::trace::SpanId &) noexcept override {} diff --git a/sdk/test/logs/logger_provider_sdk_test.cc b/sdk/test/logs/logger_provider_sdk_test.cc index cf993dc44c..ae606876b3 100644 --- a/sdk/test/logs/logger_provider_sdk_test.cc +++ b/sdk/test/logs/logger_provider_sdk_test.cc @@ -123,6 +123,8 @@ class DummyLogRecordable final : public opentelemetry::sdk::logs::Recordable void SetBody(const opentelemetry::common::AttributeValue &) noexcept override {} + void SetEventId(int64_t, nostd::string_view) noexcept override {} + void SetTraceId(const opentelemetry::trace::TraceId &) noexcept override {} void SetSpanId(const opentelemetry::trace::SpanId &) noexcept override {} diff --git a/sdk/test/logs/logger_sdk_test.cc b/sdk/test/logs/logger_sdk_test.cc index 5f6467cd91..f584fe4b7d 100644 --- a/sdk/test/logs/logger_sdk_test.cc +++ b/sdk/test/logs/logger_sdk_test.cc @@ -78,6 +78,12 @@ class MockLogRecordable final : public opentelemetry::sdk::logs::Recordable void SetBody(const std::string &message) noexcept { body_ = message; } + void SetEventId(int64_t id, nostd::string_view name) noexcept override + { + event_id_ = id; + log_record_event_name_ = static_cast(name); + } + void SetTraceId(const opentelemetry::trace::TraceId &trace_id) noexcept override { trace_id_ = trace_id; @@ -144,6 +150,8 @@ class MockLogRecordable final : public opentelemetry::sdk::logs::Recordable private: opentelemetry::logs::Severity severity_ = opentelemetry::logs::Severity::kInvalid; std::string body_; + int64_t event_id_; + std::string log_record_event_name_; opentelemetry::trace::TraceId trace_id_; opentelemetry::trace::SpanId span_id_; opentelemetry::trace::TraceFlags trace_flags_; diff --git a/sdk/test/logs/simple_log_record_processor_test.cc b/sdk/test/logs/simple_log_record_processor_test.cc index 2d377cd924..7d45cc85c2 100644 --- a/sdk/test/logs/simple_log_record_processor_test.cc +++ b/sdk/test/logs/simple_log_record_processor_test.cc @@ -42,6 +42,8 @@ class TestLogRecordable final : public opentelemetry::sdk::logs::Recordable void SetBody(const char *message) noexcept { body_ = message; } + void SetEventId(int64_t, nostd::string_view) noexcept override {} + void SetTraceId(const opentelemetry::trace::TraceId &) noexcept override {} void SetSpanId(const opentelemetry::trace::SpanId &) noexcept override {}