SDK support for the new OTel log API (#2123)

CHANGELOG.md

@@ -15,6 +15,8 @@ Increment the:
 
 ## [Unreleased]
 
+* [SDK] SDK support for the new OTel log
+  [#2123](https://github.com/open-telemetry/opentelemetry-cpp/pull/2123)
 * [BUILD] Build break with old curl, macro CURL_VERSION_BITS unknown
   [#2102](https://github.com/open-telemetry/opentelemetry-cpp/pull/2102)
 * [API] Add user facing Logging API and Benchmarks

api/include/opentelemetry/logs/event_id.h

@@ -18,14 +18,15 @@ namespace logs
 class EventId
 {
 public:
-  EventId(int64_t id, nostd::string_view name) noexcept
+  EventId(int64_t id, nostd::string_view name) noexcept : id_{id}
   {
-    id_   = id;
     name_ = nostd::unique_ptr<char[]>{new char[name.length() + 1]};
     std::copy(name.begin(), name.end(), name_.get());
     name_.get()[name.length()] = 0;
   }
 
+  EventId(int64_t id) noexcept : id_{id}, name_{nullptr} {}
+
 public:
   int64_t id_;
   nostd::unique_ptr<char[]> name_;

api/include/opentelemetry/logs/log_record.h

@@ -58,6 +58,14 @@ class LogRecord
   virtual void SetAttribute(nostd::string_view key,
                             const opentelemetry::common::AttributeValue &value) noexcept = 0;
 
+  /**
+   * Set the Event Id.
+   * @param id The event id to set
+   * @param name Optional event name to set
+   */
+  // TODO: mark this as pure virtual once all exporters have been updated
+  virtual void SetEventId(int64_t id, nostd::string_view name = {}) noexcept = 0;
+
   /**
    * Set the trace id for this log.
    * @param trace_id the trace id to set

api/include/opentelemetry/logs/logger.h

@@ -289,7 +289,7 @@ class Logger
                    nostd::string_view format,
                    const common::KeyValueIterable &attributes) noexcept
   {
-    this->EmitLogRecord(severity, event_id, format, attributes);
+    this->EmitLogRecord(severity, EventId{event_id}, format, attributes);
   }
 
   virtual void Log(Severity severity,
@@ -318,7 +318,7 @@ class Logger
                     nostd::string_view format,
                     const common::KeyValueIterable &attributes) noexcept
   {
-    this->Log(Severity::kTrace, event_id, format, attributes);
+    this->Log(Severity::kTrace, EventId{event_id}, format, attributes);
   }
 
   inline void Trace(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept
@@ -339,7 +339,7 @@ class Logger
                     nostd::string_view format,
                     const common::KeyValueIterable &attributes) noexcept
   {
-    this->Log(Severity::kDebug, event_id, format, attributes);
+    this->Log(Severity::kDebug, EventId{event_id}, format, attributes);
   }
 
   inline void Debug(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept
@@ -360,7 +360,7 @@ class Logger
                    nostd::string_view format,
                    const common::KeyValueIterable &attributes) noexcept
   {
-    this->Log(Severity::kInfo, event_id, format, attributes);
+    this->Log(Severity::kInfo, EventId{event_id}, format, attributes);
   }
 
   inline void Info(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept
@@ -381,7 +381,7 @@ class Logger
                    nostd::string_view format,
                    const common::KeyValueIterable &attributes) noexcept
   {
-    this->Log(Severity::kWarn, event_id, format, attributes);
+    this->Log(Severity::kWarn, EventId{event_id}, format, attributes);
   }
 
   inline void Warn(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept
@@ -402,7 +402,7 @@ class Logger
                     nostd::string_view format,
                     const common::KeyValueIterable &attributes) noexcept
   {
-    this->Log(Severity::kError, event_id, format, attributes);
+    this->Log(Severity::kError, EventId{event_id}, format, attributes);
   }
 
   inline void Error(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept
@@ -423,7 +423,7 @@ class Logger
                     nostd::string_view format,
                     const common::KeyValueIterable &attributes) noexcept
   {
-    this->Log(Severity::kFatal, event_id, format, attributes);
+    this->Log(Severity::kFatal, EventId{event_id}, format, attributes);
   }
 
   inline void Fatal(nostd::string_view format, const common::KeyValueIterable &attributes) noexcept

api/include/opentelemetry/logs/logger_type_traits.h

@@ -48,9 +48,10 @@ template <>
 struct LogRecordSetterTrait<EventId>
 {
   template <class ArgumentType>
-  inline static LogRecord *Set(LogRecord *log_record, ArgumentType && /*arg*/) noexcept
+  inline static LogRecord *Set(LogRecord *log_record, ArgumentType &&arg) noexcept
   {
-    // TODO: set log_record
+    log_record->SetEventId(arg.id_, nostd::string_view{arg.name_.get()});
+
     return log_record;
   }
 };

exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h

@@ -72,6 +72,16 @@ class ElasticSearchRecordable final : public sdk::logs::Recordable
    */
   void SetBody(const opentelemetry::common::AttributeValue &message) noexcept override;
 
+  /**
+   * Set the Event Id
+   * @param id the event id to set
+   * @param name the event name to set
+   */
+  void SetEventId(int64_t /* id */, nostd::string_view /* name */) noexcept override
+  {
+    // TODO: implement event id
+  }
+
   /**
    * Set the trace id for this log.
    * @param trace_id the trace id to set

exporters/ostream/src/log_record_exporter.cc

@@ -52,6 +52,8 @@ sdk::common::ExportResult OStreamLogRecordExporter::Export(
       continue;
     }
 
+    int64_t event_id = log_record->GetEventId();
+
     // Convert trace, spanid, traceflags into exportable representation
     constexpr int trace_id_len    = 32;
     constexpr int span_id__len    = 16;
@@ -96,6 +98,8 @@ sdk::common::ExportResult OStreamLogRecordExporter::Export(
     printAttributes(log_record->GetAttributes(), "\n    ");
 
     sout_ << "\n"
+          << "  event_id           : " << event_id << "\n"
+          << "  event_name         : " << log_record->GetEventName() << "\n"
           << "  trace_id           : " << std::string(trace_id, trace_id_len) << "\n"
           << "  span_id            : " << std::string(span_id, span_id__len) << "\n"
           << "  trace_flags        : " << std::string(trace_flags, trace_flags_len) << "\n"

exporters/ostream/test/ostream_log_test.cc

@@ -20,6 +20,8 @@ namespace nostd        = opentelemetry::nostd;
 namespace exporterlogs = opentelemetry::exporter::logs;
 namespace common       = opentelemetry::common;
 
+using Attributes = std::initializer_list<std::pair<nostd::string_view, common::AttributeValue>>;
+
 OPENTELEMETRY_BEGIN_NAMESPACE
 namespace exporter
 {
@@ -103,6 +105,8 @@ TEST(OstreamLogExporter, DefaultLogRecordToCout)
       "    telemetry.sdk.name: opentelemetry\n",
       "    telemetry.sdk.language: cpp\n",
       "  attributes         : \n",
+      "  event_id           : 0\n"
+      "  event_name         : \n",
       "  trace_id           : 00000000000000000000000000000000\n",
       "  span_id            : 0000000000000000\n",
       "  trace_flags        : 00\n",
@@ -176,6 +180,8 @@ TEST(OStreamLogRecordExporter, SimpleLogToCout)
       "    telemetry.sdk.name: opentelemetry\n",
       "    telemetry.sdk.language: cpp\n",
       "  attributes         : \n",
+      "  event_id           : 0\n"
+      "  event_name         : \n",
       "  trace_id           : 00000000000000000000000000000000\n",
       "  span_id            : 0000000000000000\n",
       "  trace_flags        : 00\n",
@@ -248,6 +254,8 @@ TEST(OStreamLogRecordExporter, LogWithStringAttributesToCerr)
       "    key1: val1\n",
       "  attributes         : \n",
       "    a: 1\n",
+      "  event_id           : 0\n"
+      "  event_name         : \n",
       "  trace_id           : 00000000000000000000000000000000\n",
       "  span_id            : 0000000000000000\n",
       "  trace_flags        : 00\n",
@@ -327,6 +335,8 @@ TEST(OStreamLogRecordExporter, LogWithVariantTypesToClog)
       "    res1: [1,2,3]\n",
       "  attributes         : \n",
       "    attr1: [0,1,0]\n",
+      "  event_id           : 0\n"
+      "  event_name         : \n",
       "  trace_id           : 00000000000000000000000000000000\n",
       "  span_id            : 0000000000000000\n",
       "  trace_flags        : 00\n",
@@ -398,6 +408,79 @@ TEST(OStreamLogRecordExporter, IntegrationTest)
       "    telemetry.sdk.name: opentelemetry\n",
       "    telemetry.sdk.language: cpp\n",
       "  attributes         : \n",
+      "  event_id           : 0\n"
+      "  event_name         : \n",
+      "  trace_id           : 00000000000000000000000000000000\n",
+      "  span_id            : 0000000000000000\n",
+      "  trace_flags        : 00\n",
+      "  scope              : \n",
+      "    name             : opentelelemtry_library\n",
+      "    version          : " OPENTELEMETRY_SDK_VERSION "\n",
+      "    schema_url       : https://opentelemetry.io/schemas/1.11.0\n",
+      "    attributes       : \n",
+      "      scope.attr.key: 123\n",
+      "}\n"};
+
+  std::string ostream_output = stdcoutOutput.str();
+  for (auto &expected : expected_output)
+  {
+    std::string::size_type result = ostream_output.find(expected);
+    if (result == std::string::npos)
+    {
+      std::cout << "Can not find: \"" << expected << "\" in\n" << ostream_output << std::endl;
+    }
+    ASSERT_NE(result, std::string::npos);
+  }
+}
+
+// Test using the simple log processor and ostream exporter to cout
+// and use the rest of the logging pipeline (Logger, LoggerProvider, Provider) and user-facing API
+// as well
+TEST(OStreamLogRecordExporter, IntegrationTestWithEventId)
+{
+  // Initialize a logger
+  auto exporter =
+      std::unique_ptr<sdklogs::LogRecordExporter>(new exporterlogs::OStreamLogRecordExporter);
+  auto sdkProvider = std::shared_ptr<sdklogs::LoggerProvider>(new sdklogs::LoggerProvider());
+  sdkProvider->AddProcessor(std::unique_ptr<sdklogs::LogRecordProcessor>(
+      new sdklogs::SimpleLogRecordProcessor(std::move(exporter))));
+  auto apiProvider = nostd::shared_ptr<logs_api::LoggerProvider>(sdkProvider);
+  auto provider    = nostd::shared_ptr<logs_api::LoggerProvider>(apiProvider);
+  logs_api::Provider::SetLoggerProvider(provider);
+  const std::string schema_url{"https://opentelemetry.io/schemas/1.11.0"};
+  auto logger = logs_api::Provider::GetLoggerProvider()->GetLogger(
+      "Logger", "opentelelemtry_library", OPENTELEMETRY_SDK_VERSION, schema_url, true,
+      {{"scope.attr.key", 123}});
+
+  // Back up cout's streambuf
+  std::streambuf *original = std::cout.rdbuf();
+
+  // Redirect cout to our string stream
+  std::stringstream stdcoutOutput;
+  std::cout.rdbuf(stdcoutOutput.rdbuf());
+
+  logs_api::EventId event_id{12345678, "test_event_id"};
+
+  // Write a log to ostream exporter
+  logger->Debug(event_id, "Hello {key1} {key2}",
+                common::MakeKeyValueIterableView<Attributes>({{"key1", 123}, {"key2", "value2"}}));
+
+  // Restore cout's original streambuf
+  std::cout.rdbuf(original);
+
+  // Compare actual vs expected outputs
+  std::vector<std::string> expected_output{
+      "  severity_num       : 5\n"
+      "  severity_text      : DEBUG\n"
+      "  body               : Hello {key1} {key2}\n",
+      "  resource           : \n",
+      "    telemetry.sdk.version: " OPENTELEMETRY_VERSION "\n",
+      "    service.name: unknown_service\n",
+      "    telemetry.sdk.name: opentelemetry\n",
+      "    telemetry.sdk.language: cpp\n",
+      "  attributes         : \n",
+      "  event_id           : 12345678\n"
+      "  event_name         : test_event_id\n",
       "  trace_id           : 00000000000000000000000000000000\n",
       "  span_id            : 0000000000000000\n",
       "  trace_flags        : 00\n",

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_log_recordable.h

@@ -67,6 +67,16 @@ class OtlpLogRecordable final : public opentelemetry::sdk::logs::Recordable
    */
   void SetBody(const opentelemetry::common::AttributeValue &message) noexcept override;
 
+  /**
+   * @brief Set the Event Id for this log.
+   * @param id the event Id to set
+   * @param name  the event name to set
+   */
+  void SetEventId(int64_t /* id */, nostd::string_view /* name */) noexcept override
+  {
+    // TODO: export Event Id to OTLP
+  }
+
   /**
    * Set the trace id for this log.
    * @param trace_id the trace id to set

sdk/include/opentelemetry/sdk/logs/multi_recordable.h

@@ -55,6 +55,13 @@ class MultiRecordable final : public Recordable
    */
   void SetBody(const opentelemetry::common::AttributeValue &message) noexcept override;
 
+  /**
+   * Set the event id
+   * @param id the event id to set
+   * @param name  the event name to set
+   */
+  void SetEventId(int64_t id, nostd::string_view name) noexcept override;
+
   /**
    * Set the trace id for this log.
    * @param trace_id the trace id to set

sdk/include/opentelemetry/sdk/logs/read_write_log_record.h

@@ -87,6 +87,25 @@ class ReadWriteLogRecord final : public ReadableLogRecord
    */
   const opentelemetry::common::AttributeValue &GetBody() const noexcept override;
 
+  /**
+   * Set the Event Id object
+   * @param id  the event Id to set
+   * @param name the event name to set
+   */
+  void SetEventId(int64_t id, nostd::string_view name) noexcept override;
+
+  /**
+   * Get event Id of this log.
+   * @return the event Id of this log.
+   */
+  int64_t GetEventId() const noexcept override;
+
+  /**
+   * Get event name of this log.
+   * @return the event name of this log.
+   */
+  nostd::string_view GetEventName() const noexcept override;
+
   /**
    * Set the trace id for this log.
    * @param trace_id the trace id to set
@@ -176,6 +195,9 @@ class ReadWriteLogRecord final : public ReadableLogRecord
   opentelemetry::common::SystemTimestamp timestamp_;
   opentelemetry::common::SystemTimestamp observed_timestamp_;
 
+  int64_t event_id_;
+  std::string event_name_;
+
   // We do not pay for trace state when not necessary
   struct TraceState
   {

sdk/include/opentelemetry/sdk/logs/readable_log_record.h

@@ -64,6 +64,18 @@ class ReadableLogRecord : public Recordable
    */
   virtual const opentelemetry::common::AttributeValue &GetBody() const noexcept = 0;
 
+  /**
+   * Get the Event id.
+   * @return the event id
+   */
+  virtual int64_t GetEventId() const noexcept = 0;
+
+  /**
+   * Get the Event Name.
+   * @return the event name
+   */
+  virtual nostd::string_view GetEventName() const noexcept = 0;
+
   /**
    * Get the trace id of this log.
    * @return the trace id of this log

sdk/src/logs/multi_recordable.cc

@@ -101,6 +101,17 @@ void MultiRecordable::SetBody(const opentelemetry::common::AttributeValue &messa
   }
 }
 
+void MultiRecordable::SetEventId(int64_t id, nostd::string_view name) noexcept
+{
+  for (auto &recordable : recordables_)
+  {
+    if (recordable.second)
+    {
+      recordable.second->SetEventId(id, name);
+    }
+  }
+}
+
 void MultiRecordable::SetTraceId(const opentelemetry::trace::TraceId &trace_id) noexcept
 {
   for (auto &recordable : recordables_)