From 15e7725bcf654d96865214566c4070ae0a79e2f6 Mon Sep 17 00:00:00 2001 From: WenTao Ou Date: Tue, 29 Mar 2022 13:09:11 +0800 Subject: [PATCH] Cocurrency otlp http session (#1281) --- .github/workflows/benchmark.yml | 4 +- .github/workflows/ci.yml | 16 +- CHANGELOG.md | 1 + README.md | 1 + .../opentelemetry/common/spin_lock_mutex.h | 37 +- api/include/opentelemetry/metrics/noop.h | 17 +- .../opentelemetry/metrics/sync_instruments.h | 90 +++- api/test/metrics/BUILD | 31 ++ api/test/metrics/noop_sync_instrument_test.cc | 13 +- docker/Dockerfile.centos | 14 +- docs/{ => public}/performance/benchmarks.rst | 2 +- exporters/ostream/test/ostream_metric_test.cc | 4 +- .../exporters/otlp/otlp_environment.h | 2 +- .../exporters/otlp/otlp_http_client.h | 87 ++-- .../exporters/otlp/otlp_http_exporter.h | 5 +- .../exporters/otlp/otlp_http_log_exporter.h | 5 +- exporters/otlp/src/otlp_http_client.cc | 326 ++++++++------ exporters/otlp/src/otlp_http_exporter.cc | 13 +- exporters/otlp/src/otlp_http_log_exporter.cc | 12 +- .../otlp/test/otlp_http_exporter_test.cc | 363 +++++++++------- .../otlp/test/otlp_http_log_exporter_test.cc | 404 ++++++++++-------- sdk/CMakeLists.txt | 6 + .../sdk/logs/batch_log_processor.h | 89 ++-- .../sdk/metrics/aggregation/sum_aggregation.h | 2 +- .../sdk/metrics/data/point_data.h | 2 +- .../metrics/exemplar/always_sample_filter.h | 43 ++ .../opentelemetry/sdk/metrics/exemplar/data.h | 45 ++ .../sdk/metrics/exemplar/filter.h | 39 ++ .../metrics/exemplar/never_sample_filter.h | 43 ++ .../metrics/exemplar/no_exemplar_reservoir.h | 55 +++ .../sdk/metrics/exemplar/reservoir.h | 55 +++ .../opentelemetry/sdk/metrics/instruments.h | 2 +- .../sdk/metrics/measurement_processor.h | 116 ----- .../sdk/metrics/metric_exporter.h | 2 +- .../opentelemetry/sdk/metrics/metric_reader.h | 6 +- .../sdk/metrics/state/metric_collector.h | 4 +- .../sdk/metrics/state/metric_storage.h | 24 +- .../sdk/metrics/state/multi_metric_storage.h | 26 +- .../sdk/metrics/state/sync_metric_storage.h | 26 +- .../sdk/metrics/sync_instruments.h | 26 +- .../sdk/metrics/view/attributes_processor.h | 2 - .../sdk/trace/batch_span_processor.h | 50 +-- sdk/src/logs/batch_log_processor.cc | 214 ++++++---- sdk/src/metrics/meter.cc | 4 +- sdk/src/metrics/metric_reader.cc | 8 +- sdk/src/metrics/state/metric_collector.cc | 4 +- sdk/src/metrics/sync_instruments.cc | 90 +++- sdk/src/trace/batch_span_processor.cc | 208 +++++---- sdk/src/trace/span.cc | 4 + sdk/test/logs/batch_log_processor_test.cc | 27 +- sdk/test/metrics/BUILD | 16 + sdk/test/metrics/CMakeLists.txt | 2 + sdk/test/metrics/async_metric_storage_test.cc | 4 +- sdk/test/metrics/exemplar/BUILD | 47 ++ sdk/test/metrics/exemplar/CMakeLists.txt | 10 + .../exemplar/always_sample_filter_test.cc | 19 + .../exemplar/never_sample_filter_test.cc | 20 + .../exemplar/no_exemplar_reservoir_test.cc | 22 + sdk/test/metrics/metric_reader_test.cc | 10 +- sdk/test/metrics/multi_metric_storage_test.cc | 27 +- sdk/test/metrics/sync_instruments_test.cc | 50 ++- sdk/test/metrics/sync_metric_storage_test.cc | 16 +- sdk/test/trace/batch_span_processor_test.cc | 33 +- sdk/test/trace/tracer_test.cc | 28 ++ 64 files changed, 1990 insertions(+), 983 deletions(-) create mode 100644 api/test/metrics/BUILD rename docs/{ => public}/performance/benchmarks.rst (69%) create mode 100644 sdk/include/opentelemetry/sdk/metrics/exemplar/always_sample_filter.h create mode 100644 sdk/include/opentelemetry/sdk/metrics/exemplar/data.h create mode 100644 sdk/include/opentelemetry/sdk/metrics/exemplar/filter.h create mode 100644 sdk/include/opentelemetry/sdk/metrics/exemplar/never_sample_filter.h create mode 100644 sdk/include/opentelemetry/sdk/metrics/exemplar/no_exemplar_reservoir.h create mode 100644 sdk/include/opentelemetry/sdk/metrics/exemplar/reservoir.h delete mode 100644 sdk/include/opentelemetry/sdk/metrics/measurement_processor.h create mode 100644 sdk/test/metrics/exemplar/BUILD create mode 100644 sdk/test/metrics/exemplar/CMakeLists.txt create mode 100644 sdk/test/metrics/exemplar/always_sample_filter_test.cc create mode 100644 sdk/test/metrics/exemplar/never_sample_filter_test.cc create mode 100644 sdk/test/metrics/exemplar/no_exemplar_reservoir_test.cc diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c11e0c564a..d6b12b82d0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -17,7 +17,7 @@ jobs: with: submodules: 'recursive' - name: Mount Bazel Cache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: bazel_cache with: @@ -67,6 +67,6 @@ jobs: # Show alert with commit comment on detecting possible performance regression alert-threshold: '200%' comment-on-alert: true - fail-on-alert: true + fail-on-alert: false gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fd8f80838..a851728a2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,7 +136,7 @@ jobs: with: submodules: 'recursive' - name: Mount Bazel Cache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: bazel_cache with: @@ -158,7 +158,7 @@ jobs: with: submodules: 'recursive' - name: Mount Bazel Cache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: bazel_cache with: @@ -180,7 +180,7 @@ jobs: with: submodules: 'recursive' - name: Mount Bazel Cache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: bazel_cache with: @@ -202,7 +202,7 @@ jobs: with: submodules: 'recursive' - name: Mount Bazel Cache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: bazel_cache with: @@ -224,7 +224,7 @@ jobs: with: submodules: 'recursive' - name: Mount Bazel Cache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: bazel_cache with: @@ -246,7 +246,7 @@ jobs: with: submodules: 'recursive' - name: Mount Bazel Cache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: bazel_cache with: @@ -268,7 +268,7 @@ jobs: with: submodules: 'recursive' - name: Mount Bazel Cache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: bazel_cache with: @@ -285,7 +285,7 @@ jobs: with: submodules: 'recursive' - name: Mount Bazel Cache - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: bazel_cache with: diff --git a/CHANGELOG.md b/CHANGELOG.md index e33097f0d8..0eac7fa35a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Increment the: ## [Unreleased] +* [SDK] Bugfix: span SetAttribute crash ([#1283](https://github.com/open-telemetry/opentelemetry-cpp/pull/1283)) * [EXPORTER] Jaeger Exporter - Populate Span Links ([#1251](https://github.com/open-telemetry/opentelemetry-cpp/pull/1251)) * [EXPORTER] OTLP http exporter allow concurrency session ([#1209](https://github.com/open-telemetry/opentelemetry-cpp/pull/1209)) diff --git a/README.md b/README.md index 18b9aafd14..7dc77fc940 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ For edit access, get in touch on * [Josh Suereth](https://github.com/jsuereth), Google * [Reiley Yang](https://github.com/reyang), Microsoft +* [WenTao Ou](https://github.com/owent), Tencent [Emeritus Maintainer/Approver/Triager](https://github.com/open-telemetry/community/blob/main/community-membership.md#emeritus-maintainerapprovertriager): diff --git a/api/include/opentelemetry/common/spin_lock_mutex.h b/api/include/opentelemetry/common/spin_lock_mutex.h index b35f54081a..9c51857567 100644 --- a/api/include/opentelemetry/common/spin_lock_mutex.h +++ b/api/include/opentelemetry/common/spin_lock_mutex.h @@ -59,6 +59,26 @@ class SpinLockMutex SpinLockMutex &operator=(const SpinLockMutex &) = delete; SpinLockMutex &operator=(const SpinLockMutex &) volatile = delete; + static inline void fast_yield() noexcept + { +// Issue a Pause/Yield instruction while spinning. +#if defined(_MSC_VER) + YieldProcessor(); +#elif defined(__i386__) || defined(__x86_64__) +# if defined(__clang__) + _mm_pause(); +# else + __builtin_ia32_pause(); +# endif +#elif defined(__arm__) + // This intrinsic should fail to be found if YIELD is not supported on the current + // processor. + __yield(); +#else + // TODO: Issue PAGE/YIELD on other architectures. +#endif + } + /** * Attempts to lock the mutex. Return immediately with `true` (success) or `false` (failure). */ @@ -91,22 +111,7 @@ class SpinLockMutex { return; } -// Issue a Pause/Yield instruction while spinning. -#if defined(_MSC_VER) - YieldProcessor(); -#elif defined(__i386__) || defined(__x86_64__) -# if defined(__clang__) - _mm_pause(); -# else - __builtin_ia32_pause(); -# endif -#elif defined(__arm__) - // This intrinsic should fail to be found if YIELD is not supported on the current - // processor. - __yield(); -#else - // TODO: Issue PAGE/YIELD on other architectures. -#endif + fast_yield(); } // Yield then try again (goal ~100ns) std::this_thread::yield(); diff --git a/api/include/opentelemetry/metrics/noop.h b/api/include/opentelemetry/metrics/noop.h index 19b9443a7d..3332384ee2 100644 --- a/api/include/opentelemetry/metrics/noop.h +++ b/api/include/opentelemetry/metrics/noop.h @@ -24,7 +24,12 @@ class NoopCounter : public Counter nostd::string_view unit) noexcept {} void Add(T value) noexcept override {} + void Add(T value, const opentelemetry::context::Context &context) noexcept override {} void Add(T value, const common::KeyValueIterable &attributes) noexcept override {} + void Add(T value, + const common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override + {} }; template @@ -35,8 +40,11 @@ class NoopHistogram : public Histogram nostd::string_view description, nostd::string_view unit) noexcept {} - void Record(T value) noexcept override {} - void Record(T value, const common::KeyValueIterable &attributes) noexcept override {} + void Record(T value, const opentelemetry::context::Context &context) noexcept override {} + void Record(T value, + const common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override + {} }; template @@ -48,7 +56,12 @@ class NoopUpDownCounter : public UpDownCounter nostd::string_view unit) noexcept {} void Add(T value) noexcept override {} + void Add(T value, const opentelemetry::context::Context &context) noexcept override {} void Add(T value, const common::KeyValueIterable &attributes) noexcept override {} + void Add(T value, + const common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override + {} }; template diff --git a/api/include/opentelemetry/metrics/sync_instruments.h b/api/include/opentelemetry/metrics/sync_instruments.h index 35cb8621ab..e8239743a1 100644 --- a/api/include/opentelemetry/metrics/sync_instruments.h +++ b/api/include/opentelemetry/metrics/sync_instruments.h @@ -6,6 +6,7 @@ # include "opentelemetry/common/attribute_value.h" # include "opentelemetry/common/key_value_iterable_view.h" +# include "opentelemetry/context/context.h" # include "opentelemetry/nostd/span.h" # include "opentelemetry/nostd/string_view.h" # include "opentelemetry/nostd/type_traits.h" @@ -29,6 +30,8 @@ class Counter : public SynchronousInstrument */ virtual void Add(T value) noexcept = 0; + virtual void Add(T value, const opentelemetry::context::Context &context) noexcept = 0; + /** * Add adds the value to the counter's sum. The attributes should contain * the keys and values to be associated with this value. Counters only @@ -40,19 +43,44 @@ class Counter : public SynchronousInstrument virtual void Add(T value, const common::KeyValueIterable &attributes) noexcept = 0; + virtual void Add(T value, + const common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept = 0; + template ::value> * = nullptr> void Add(T value, const U &attributes) noexcept { - this->Add(value, common::KeyValueIterableView{attributes}); + auto context = opentelemetry::context::Context{}; + this->Add(value, common::KeyValueIterableView{attributes}, context); + } + + template ::value> * = nullptr> + void Add(T value, const U &attributes, const opentelemetry::context::Context &context) noexcept + { + this->Add(value, common::KeyValueIterableView{attributes}, context); } void Add(T value, std::initializer_list> attributes) noexcept { - this->Add(value, nostd::span>{ - attributes.begin(), attributes.end()}); + auto context = opentelemetry::context::Context{}; + this->Add(value, + nostd::span>{ + attributes.begin(), attributes.end()}, + context); + } + + void Add(T value, + std::initializer_list> attributes, + const opentelemetry::context::Context &context) noexcept + { + this->Add(value, + nostd::span>{ + attributes.begin(), attributes.end()}, + context); } }; @@ -67,7 +95,7 @@ class Histogram : public SynchronousInstrument * * @param value The increment amount. May be positive, negative or zero. */ - virtual void Record(T value) noexcept = 0; + virtual void Record(T value, const opentelemetry::context::Context &context) noexcept = 0; /** * Records a value with a set of attributes. @@ -75,21 +103,26 @@ class Histogram : public SynchronousInstrument * @param value The increment amount. May be positive, negative or zero. * @param attributes A set of attributes to associate with the count. */ - virtual void Record(T value, const common::KeyValueIterable &attributes) noexcept = 0; + virtual void Record(T value, + const common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept = 0; template ::value> * = nullptr> - void Record(T value, const U &attributes) noexcept + void Record(T value, const U &attributes, const opentelemetry::context::Context &context) noexcept { - this->Record(value, common::KeyValueIterableView{attributes}); + this->Record(value, common::KeyValueIterableView{attributes}, context); } - void Record(T value, - std::initializer_list> - attributes) noexcept + void Record( + T value, + std::initializer_list> attributes, + const opentelemetry::context::Context &context) noexcept { - this->Record(value, nostd::span>{ - attributes.begin(), attributes.end()}); + this->Record(value, + nostd::span>{ + attributes.begin(), attributes.end()}, + context); } }; @@ -106,6 +139,8 @@ class UpDownCounter : public SynchronousInstrument */ virtual void Add(T value) noexcept = 0; + virtual void Add(T value, const opentelemetry::context::Context &context) noexcept = 0; + /** * Add a value with a set of attributes. * @@ -114,19 +149,44 @@ class UpDownCounter : public SynchronousInstrument */ virtual void Add(T value, const common::KeyValueIterable &attributes) noexcept = 0; + virtual void Add(T value, + const common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept = 0; + template ::value> * = nullptr> void Add(T value, const U &attributes) noexcept { - this->Add(value, common::KeyValueIterableView{attributes}); + auto context = opentelemetry::context::Context{}; + this->Add(value, common::KeyValueIterableView{attributes}, context); + } + + template ::value> * = nullptr> + void Add(T value, const U &attributes, const opentelemetry::context::Context &context) noexcept + { + this->Add(value, common::KeyValueIterableView{attributes}, context); } void Add(T value, std::initializer_list> attributes) noexcept { - this->Add(value, nostd::span>{ - attributes.begin(), attributes.end()}); + auto context = opentelemetry::context::Context{}; + this->Add(value, + nostd::span>{ + attributes.begin(), attributes.end()}, + context); + } + + void Add(T value, + std::initializer_list> attributes, + const opentelemetry::context::Context &context) noexcept + { + this->Add(value, + nostd::span>{ + attributes.begin(), attributes.end()}, + context); } }; diff --git a/api/test/metrics/BUILD b/api/test/metrics/BUILD new file mode 100644 index 0000000000..d15d7511f4 --- /dev/null +++ b/api/test/metrics/BUILD @@ -0,0 +1,31 @@ +load("//bazel:otel_cc_benchmark.bzl", "otel_cc_benchmark") + +cc_test( + name = "noop_sync_instrument_test", + srcs = [ + "noop_sync_instrument_test.cc", + ], + tags = [ + "metrics", + "test", + ], + deps = [ + "//api", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "meter_provider_test", + srcs = [ + "meter_provider_test.cc", + ], + tags = [ + "metrics", + "test", + ], + deps = [ + "//api", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/api/test/metrics/noop_sync_instrument_test.cc b/api/test/metrics/noop_sync_instrument_test.cc index 4597e79e8b..62be4f34ab 100644 --- a/api/test/metrics/noop_sync_instrument_test.cc +++ b/api/test/metrics/noop_sync_instrument_test.cc @@ -14,8 +14,11 @@ TEST(Counter, Add) std::map labels = {{"k1", "v1"}}; EXPECT_NO_THROW(counter->Add(10l, labels)); + EXPECT_NO_THROW(counter->Add(10l, labels, opentelemetry::context::Context{})); EXPECT_NO_THROW(counter->Add(2l)); + EXPECT_NO_THROW(counter->Add(2l, opentelemetry::context::Context{})); EXPECT_NO_THROW(counter->Add(10l, {{"k1", "1"}, {"k2", 2}})); + EXPECT_NO_THROW(counter->Add(10l, {{"k1", "1"}, {"k2", 2}}, opentelemetry::context::Context{})); } TEST(histogram, Record) @@ -24,9 +27,10 @@ TEST(histogram, Record) new opentelemetry::metrics::NoopHistogram("test", "none", "unitless")}; std::map labels = {{"k1", "v1"}}; - EXPECT_NO_THROW(counter->Record(10l, labels)); - EXPECT_NO_THROW(counter->Record(2l)); - EXPECT_NO_THROW(counter->Record(10l, {{"k1", "1"}, {"k2", 2}})); + EXPECT_NO_THROW(counter->Record(10l, labels, opentelemetry::context::Context{})); + EXPECT_NO_THROW(counter->Record(2l, opentelemetry::context::Context{})); + EXPECT_NO_THROW( + counter->Record(10l, {{"k1", "1"}, {"k2", 2}}, opentelemetry::context::Context{})); } TEST(UpDownCountr, Record) @@ -36,8 +40,11 @@ TEST(UpDownCountr, Record) std::map labels = {{"k1", "v1"}}; EXPECT_NO_THROW(counter->Add(10l, labels)); + EXPECT_NO_THROW(counter->Add(10l, labels, opentelemetry::context::Context{})); EXPECT_NO_THROW(counter->Add(2l)); + EXPECT_NO_THROW(counter->Add(2l, opentelemetry::context::Context{})); EXPECT_NO_THROW(counter->Add(10l, {{"k1", "1"}, {"k2", 2}})); + EXPECT_NO_THROW(counter->Add(10l, {{"k1", "1"}, {"k2", 2}}, opentelemetry::context::Context{})); } #endif \ No newline at end of file diff --git a/docker/Dockerfile.centos b/docker/Dockerfile.centos index 66ca8f24b3..66138ba60a 100644 --- a/docker/Dockerfile.centos +++ b/docker/Dockerfile.centos @@ -1,13 +1,15 @@ FROM centos:7 +ARG TOOLSET_VER=11 + RUN yum update -y && yum install -y centos-release-scl epel-release -RUN yum install -y devtoolset-11 \ +RUN yum install -y devtoolset-${TOOLSET_VER} \ cmake3 git \ openssl-devel \ libcurl-devel \ - && source /opt/rh/devtoolset-11/enable + && source /opt/rh/devtoolset-${TOOLSET_VER}/enable -RUN echo "source /opt/rh/devtoolset-11/enable" >> /etc/bashrc +RUN echo "source /opt/rh/devtoolset-${TOOLSET_VER}/enable" >> /etc/bashrc RUN echo "BOOST_LIBRARYDIR=/usr/lib64/boost169" >> /etc/bashrc RUN echo "BOOST_INCLUDEDIR=/usr/include/boost169" >> /etc/bashrc @@ -17,7 +19,7 @@ ARG GRPC_VERSION=v1.43.2 RUN git clone --depth=1 -b $GRPC_VERSION https://github.com/grpc/grpc.git \ && cd grpc && git submodule update --init \ && mkdir -p "third_party/abseil-cpp/build" && cd "third_party/abseil-cpp/build" \ - && source /opt/rh/devtoolset-11/enable \ + && source /opt/rh/devtoolset-${TOOLSET_VER}/enable \ && cmake3 -DCMAKE_CXX_STANDARD=17 -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE .. \ && make -j${nproc} install && cd ../../.. \ && mkdir build && cd build \ @@ -40,7 +42,7 @@ RUN yum install -y \ && wget https://github.com/apache/thrift/archive/refs/tags/v$THRIFT_VERSION.tar.gz \ && tar -xvf v$THRIFT_VERSION.tar.gz \ && mkdir -p thrift-$THRIFT_VERSION/build && cd thrift-$THRIFT_VERSION/build \ - && source /opt/rh/devtoolset-11/enable \ + && source /opt/rh/devtoolset-${TOOLSET_VER}/enable \ && export BOOST_INCLUDEDIR=/usr/include/boost169 \ && export BOOST_LIBRARYDIR=/usr/lib64/boost169 \ && cmake3 \ @@ -66,7 +68,7 @@ RUN yum install -y \ RUN git clone --depth=1 https://github.com/open-telemetry/opentelemetry-cpp.git \ && cd opentelemetry-cpp && git submodule update --init \ && mkdir -p build && cd build \ - && source /opt/rh/devtoolset-11/enable \ + && source /opt/rh/devtoolset-${TOOLSET_VER}/enable \ && export BOOST_INCLUDEDIR=/usr/include/boost169 \ && export BOOST_LIBRARYDIR=/usr/lib64/boost169 \ && cmake3 \ diff --git a/docs/performance/benchmarks.rst b/docs/public/performance/benchmarks.rst similarity index 69% rename from docs/performance/benchmarks.rst rename to docs/public/performance/benchmarks.rst index 0f1044be42..39b13571dc 100644 --- a/docs/performance/benchmarks.rst +++ b/docs/public/performance/benchmarks.rst @@ -3,4 +3,4 @@ Performance Tests - Benchmarks Click `here `_ to view the latest performance benchmarks for packages in this repo. -Please note that the flutation in the results are mainly because [machines with different CPUs](https://github.com/benchmark-action/github-action-benchmark/issues/79) are used for tests. +Please note that the flutation in the results are mainly because `machines with different CPUs `_ are used for tests. diff --git a/exporters/ostream/test/ostream_metric_test.cc b/exporters/ostream/test/ostream_metric_test.cc index 51b9882c47..9e98dd4da4 100644 --- a/exporters/ostream/test/ostream_metric_test.cc +++ b/exporters/ostream/test/ostream_metric_test.cc @@ -35,11 +35,11 @@ TEST(OStreamMetricsExporter, ExportSumPointData) record->instrumentation_library_ = instrumentation_library.get(); record->point_data_ = metric_sdk::SumPointData{ opentelemetry::common::SystemTimestamp{}, opentelemetry::common::SystemTimestamp{}, 10.0, - metric_sdk::AggregationTemporarily::kUnspecified, false}; + metric_sdk::AggregationTemporality::kUnspecified, false}; auto record2 = std::unique_ptr(new metric_sdk::MetricData(*record)); record2->point_data_ = metric_sdk::SumPointData{ opentelemetry::common::SystemTimestamp{}, opentelemetry::common::SystemTimestamp{}, 20l, - metric_sdk::AggregationTemporarily::kUnspecified, false}; + metric_sdk::AggregationTemporality::kUnspecified, false}; std::vector> records; records.push_back(std::move(record)); records.push_back(std::move(record2)); diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h index c60de87eb9..bf7ea6a61c 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h @@ -53,7 +53,7 @@ inline const std::string GetOtlpDefaultHttpEndpoint() return endpoint.size() ? endpoint : kOtlpEndpointDefault; } -inline const bool GetOtlpDefaultIsSslEnable() +inline bool GetOtlpDefaultIsSslEnable() { constexpr char kOtlpTracesIsSslEnableEnv[] = "OTEL_EXPORTER_OTLP_TRACES_SSL_ENABLE"; constexpr char kOtlpIsSslEnableEnv[] = "OTEL_EXPORTER_OTLP_SSL_ENABLE"; diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h index 0c1c1fe684..1b21ed9f90 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h @@ -11,6 +11,7 @@ #include "opentelemetry/common/spin_lock_mutex.h" #include "opentelemetry/ext/http/client/http_client.h" +#include "opentelemetry/nostd/variant.h" #include "opentelemetry/sdk/common/exporter_utils.h" #include "opentelemetry/exporters/otlp/otlp_environment.h" @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -76,8 +78,8 @@ struct OtlpHttpClientOptions // Additional HTTP headers OtlpHeaders http_headers = GetOtlpDefaultHeaders(); - // Concurrent sessions - std::size_t concurrent_sessions = 8; + // Concurrent requests + std::size_t max_concurrent_requests = 8; inline OtlpHttpClientOptions(nostd::string_view input_url, HttpRequestContentType input_content_type, @@ -94,7 +96,7 @@ struct OtlpHttpClientOptions console_debug(input_console_debug), timeout(input_timeout), http_headers(input_http_headers), - concurrent_sessions(input_concurrent_sessions) + max_concurrent_requests(input_concurrent_sessions) {} }; @@ -112,12 +114,22 @@ class OtlpHttpClient ~OtlpHttpClient(); /** - * Export + * Sync export * @param message message to export, it should be ExportTraceServiceRequest, * ExportMetricsServiceRequest or ExportLogsServiceRequest */ sdk::common::ExportResult Export(const google::protobuf::Message &message) noexcept; + /** + * Async export + * @param message message to export, it should be ExportTraceServiceRequest, + * ExportMetricsServiceRequest or ExportLogsServiceRequest + * @param result_callback callback to call when the exporting is done + */ + void Export( + const google::protobuf::Message &message, + std::function &&result_callback) noexcept; + /** * Shut down the HTTP client. * @param timeout an optional timeout, the default timeout of 0 means that no @@ -134,14 +146,49 @@ class OtlpHttpClient void ReleaseSession(const opentelemetry::ext::http::client::Session &session) noexcept; private: + struct HttpSessionData + { + std::shared_ptr session; + std::shared_ptr event_handle; + + inline HttpSessionData() = default; + + inline explicit HttpSessionData( + std::shared_ptr &&input_session, + std::shared_ptr &&input_handle) + { + session.swap(input_session); + event_handle.swap(input_handle); + } + + inline explicit HttpSessionData(HttpSessionData &&other) + { + session.swap(other.session); + event_handle.swap(other.event_handle); + } + + inline HttpSessionData &operator=(HttpSessionData &&other) noexcept + { + session.swap(other.session); + event_handle.swap(other.event_handle); + return *this; + } + }; + + /** + * @brief Create a Session object or return a error result + * + * @param message The message to send + */ + nostd::variant createSession( + const google::protobuf::Message &message, + std::function &&result_callback) noexcept; + /** * Add http session and hold it's lifetime. - * @param session the session to add - * @param event_handle the event handle of this session + * @param session_data the session to add */ - void addSession( - std::shared_ptr session, - std::shared_ptr event_handle) noexcept; + void addSession(HttpSessionData &&session_data) noexcept; /** * @brief Real delete all sessions and event handles. @@ -165,28 +212,6 @@ class OtlpHttpClient OtlpHttpClient(OtlpHttpClientOptions &&options, std::shared_ptr http_client); - struct HttpSessionData - { - std::shared_ptr session; - std::shared_ptr event_handle; - - inline HttpSessionData() = default; - - inline explicit HttpSessionData( - std::shared_ptr &&input_session, - std::shared_ptr &&input_handle) - { - session.swap(input_session); - event_handle.swap(input_handle); - } - - inline explicit HttpSessionData(HttpSessionData &&other) - { - session.swap(other.session); - event_handle.swap(other.event_handle); - } - }; - // Stores if this HTTP client had its Shutdown() method called bool is_shutdown_; diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter.h index 3e2a783eff..8884c6c307 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter.h @@ -52,8 +52,9 @@ struct OtlpHttpExporterOptions // Additional HTTP headers OtlpHeaders http_headers = GetOtlpDefaultHeaders(); - // Concurrent sessions - std::size_t concurrent_sessions = 8; + // Concurrent requests + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlpgrpc-concurrent-requests + std::size_t max_concurrent_requests = 8; }; /** diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_exporter.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_exporter.h index 320a9bb963..aecacb2686 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_exporter.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_exporter.h @@ -52,8 +52,9 @@ struct OtlpHttpLogExporterOptions // Additional HTTP headers OtlpHeaders http_headers = GetOtlpDefaultLogHeaders(); - // Concurrent sessions - std::size_t concurrent_sessions = 8; + // Concurrent requests + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlpgrpc-concurrent-requests + std::size_t max_concurrent_requests = 8; }; /** diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index 79c0b69fd8..1e5f128aaf 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -17,6 +17,7 @@ #include "google/protobuf/message.h" #include "google/protobuf/reflection.h" #include "google/protobuf/stubs/common.h" +#include "google/protobuf/stubs/stringpiece.h" #include "nlohmann/json.hpp" #if defined(GOOGLE_PROTOBUF_VERSION) && GOOGLE_PROTOBUF_VERSION >= 3007000 @@ -71,7 +72,9 @@ class ResponseHandler : public http_client::EventHandler /** * Creates a response handler, that by default doesn't display to console */ - ResponseHandler(bool console_debug = false) : console_debug_{console_debug} + ResponseHandler(std::function &&callback, + bool console_debug = false) + : result_callback_{std::move(callback)}, console_debug_{console_debug} { stoping_.store(false); } @@ -110,7 +113,7 @@ class ResponseHandler : public http_client::EventHandler bool expected = false; if (stoping_.compare_exchange_strong(expected, true, std::memory_order_release)) { - Unbind(); + Unbind(sdk::common::ExportResult::kSuccess); } } } @@ -293,12 +296,12 @@ class ResponseHandler : public http_client::EventHandler bool expected = false; if (stoping_.compare_exchange_strong(expected, true, std::memory_order_release)) { - Unbind(); + Unbind(sdk::common::ExportResult::kFailure); } } } - void Unbind() + void Unbind(sdk::common::ExportResult result) { // ReleaseSession may destroy this object, so we need to move owner and session into stack // first. @@ -312,6 +315,11 @@ class ResponseHandler : public http_client::EventHandler { // Release the session at last owner->ReleaseSession(*session); + + if (result_callback_) + { + result_callback_(result); + } } } @@ -337,6 +345,9 @@ class ResponseHandler : public http_client::EventHandler // A string to store the response body std::string body_ = ""; + // Result callback when in async mode + std::function result_callback_; + // Whether to print the results from the callback bool console_debug_ = false; }; @@ -649,10 +660,19 @@ OtlpHttpClient::~OtlpHttpClient() // Wait for all the sessions to finish std::unique_lock lock(session_waker_lock_); - session_waker_.wait(lock, [this] { - std::lock_guard guard{session_manager_lock_}; - return running_sessions_.empty(); - }); + while (true) + { + { + std::lock_guard guard{session_manager_lock_}; + if (running_sessions_.empty()) + { + break; + } + } + // When changes of running_sessions_ and notify_one/notify_all happen between predicate + // checking and waiting, we should not wait forever. + session_waker_.wait_for(lock, options_.timeout); + } // And then remove all session datas while (cleanupGCSessions()) @@ -668,106 +688,65 @@ OtlpHttpClient::OtlpHttpClient(OtlpHttpClientOptions &&options, opentelemetry::sdk::common::ExportResult OtlpHttpClient::Export( const google::protobuf::Message &message) noexcept { - // Parse uri and store it to cache - if (http_uri_.empty()) + opentelemetry::sdk::common::ExportResult result = + opentelemetry::sdk::common::ExportResult::kSuccess; + auto session = + createSession(message, [&result](opentelemetry::sdk::common::ExportResult export_result) { + result = export_result; + return export_result == opentelemetry::sdk::common::ExportResult::kSuccess; + }); + + if (opentelemetry::nostd::holds_alternative(session)) { - auto parse_url = opentelemetry::ext::http::common::UrlParser(std::string(options_.url)); - if (!parse_url.success_) - { - std::string error_message = "[OTLP HTTP Client] Export failed, invalid url: " + options_.url; - if (options_.console_debug) - { - std::cerr << error_message << std::endl; - } - OTEL_INTERNAL_LOG_ERROR(error_message.c_str()); - - return opentelemetry::sdk::common::ExportResult::kFailure; - } - - if (!parse_url.path_.empty() && parse_url.path_[0] == '/') - { - http_uri_ = parse_url.path_.substr(1); - } - else - { - http_uri_ = parse_url.path_; - } + return opentelemetry::nostd::get(session); } - http_client::Body body_vec; - std::string content_type; - if (options_.content_type == HttpRequestContentType::kBinary) + // Wait for the response to be received + if (options_.console_debug) { - if (SerializeToHttpBody(body_vec, message)) - { - if (options_.console_debug) - { - OTEL_INTERNAL_LOG_DEBUG( - "[OTLP HTTP Client] Request body(Binary): " << message.Utf8DebugString()); - } - } - else - { - if (options_.console_debug) - { - OTEL_INTERNAL_LOG_DEBUG("[OTLP HTTP Client] Serialize body failed(Binary):" - << message.InitializationErrorString()); - } - return opentelemetry::sdk::common::ExportResult::kFailure; - } - content_type = kHttpBinaryContentType; + OTEL_INTERNAL_LOG_DEBUG( + "[OTLP HTTP Client] DEBUG: Waiting for response from " + << options_.url << " (timeout = " + << std::chrono::duration_cast(options_.timeout).count() + << " milliseconds)"); } - else - { - nlohmann::json json_request; - // Convert from proto into json object - ConvertGenericMessageToJson(json_request, message, options_); - - std::string post_body_json = - json_request.dump(-1, ' ', false, nlohmann::detail::error_handler_t::replace); - if (options_.console_debug) - { - OTEL_INTERNAL_LOG_DEBUG("[OTLP HTTP Client] Request body(Json)" << post_body_json); - } - body_vec.assign(post_body_json.begin(), post_body_json.end()); - content_type = kHttpJsonContentType; - } + addSession(std::move(opentelemetry::nostd::get(session))); - // Send the request - { + // Wait for any session to finish if there are to many sessions + std::unique_lock lock(session_waker_lock_); + bool wait_successful = session_waker_.wait_for(lock, options_.timeout, [this] { std::lock_guard guard{session_manager_lock_}; - // Return failure if this exporter has been shutdown - if (isShutdown()) - { - const char *error_message = "[OTLP HTTP Client] Export failed, exporter is shutdown"; - if (options_.console_debug) - { - std::cerr << error_message << std::endl; - } - OTEL_INTERNAL_LOG_ERROR(error_message); + return running_sessions_.empty(); + }); - return opentelemetry::sdk::common::ExportResult::kFailure; - } + cleanupGCSessions(); - auto session = http_client_->CreateSession(options_.url); - auto request = session->CreateRequest(); + // If an error occurred with the HTTP request + if (!wait_successful) + { + return opentelemetry::sdk::common::ExportResult::kFailure; + } + + return result; +} - for (auto &header : options_.http_headers) +void OtlpHttpClient::Export( + const google::protobuf::Message &message, + std::function &&result_callback) noexcept +{ + auto session = createSession(message, std::move(result_callback)); + if (opentelemetry::nostd::holds_alternative(session)) + { + if (result_callback) { - request->AddHeader(header.first, header.second); + result_callback(opentelemetry::nostd::get(session)); } - request->SetUri(http_uri_); - request->SetTimeoutMs(std::chrono::duration_cast(options_.timeout)); - request->SetMethod(http_client::Method::Post); - request->SetBody(body_vec); - request->ReplaceHeader("Content-Type", content_type); - - // Send the request - addSession(std::move(session), std::shared_ptr{ - new ResponseHandler(options_.console_debug)}); + return; } + addSession(std::move(opentelemetry::nostd::get(session))); + // Wait for the response to be received if (options_.console_debug) { @@ -780,20 +759,12 @@ opentelemetry::sdk::common::ExportResult OtlpHttpClient::Export( // Wait for any session to finish if there are to many sessions std::unique_lock lock(session_waker_lock_); - bool wait_successful = session_waker_.wait_for(lock, options_.timeout, [this] { + session_waker_.wait_for(lock, options_.timeout, [this] { std::lock_guard guard{session_manager_lock_}; - return running_sessions_.size() <= options_.concurrent_sessions; + return running_sessions_.size() <= options_.max_concurrent_requests; }); cleanupGCSessions(); - - // If an error occurred with the HTTP request - if (!wait_successful) - { - return opentelemetry::sdk::common::ExportResult::kFailure; - } - - return opentelemetry::sdk::common::ExportResult::kSuccess; } bool OtlpHttpClient::Shutdown(std::chrono::microseconds timeout) noexcept @@ -816,10 +787,19 @@ bool OtlpHttpClient::Shutdown(std::chrono::microseconds timeout) noexcept std::unique_lock lock(session_waker_lock_); if (timeout <= std::chrono::microseconds::zero()) { - session_waker_.wait(lock, [this] { - std::lock_guard guard{session_manager_lock_}; - return running_sessions_.empty(); - }); + while (true) + { + { + std::lock_guard guard{session_manager_lock_}; + if (running_sessions_.empty()) + { + break; + } + } + // When changes of running_sessions_ and notify_one/notify_all happen between predicate + // checking and waiting, we should not wait forever. + session_waker_.wait_for(lock, options_.timeout); + } } else { @@ -842,12 +822,12 @@ void OtlpHttpClient::ReleaseSession( { std::lock_guard guard{session_manager_lock_}; - auto seesion_iter = running_sessions_.find(&session); - if (seesion_iter != running_sessions_.end()) + auto session_iter = running_sessions_.find(&session); + if (session_iter != running_sessions_.end()) { // Move session and handle into gc list, and they will be destroyed later - gc_sessions_.emplace_back(std::move(seesion_iter->second)); - running_sessions_.erase(seesion_iter); + gc_sessions_.emplace_back(std::move(session_iter->second)); + running_sessions_.erase(session_iter); has_session = true; } @@ -859,26 +839,130 @@ void OtlpHttpClient::ReleaseSession( } } -void OtlpHttpClient::addSession( - std::shared_ptr session, - std::shared_ptr event_handle) noexcept +opentelemetry::nostd::variant +OtlpHttpClient::createSession( + const google::protobuf::Message &message, + std::function &&result_callback) noexcept +{ + // Parse uri and store it to cache + if (http_uri_.empty()) + { + auto parse_url = opentelemetry::ext::http::common::UrlParser(std::string(options_.url)); + if (!parse_url.success_) + { + std::string error_message = "[OTLP HTTP Client] Export failed, invalid url: " + options_.url; + if (options_.console_debug) + { + std::cerr << error_message << std::endl; + } + OTEL_INTERNAL_LOG_ERROR(error_message.c_str()); + + return opentelemetry::sdk::common::ExportResult::kFailure; + } + + if (!parse_url.path_.empty() && parse_url.path_[0] == '/') + { + http_uri_ = parse_url.path_.substr(1); + } + else + { + http_uri_ = parse_url.path_; + } + } + + http_client::Body body_vec; + std::string content_type; + if (options_.content_type == HttpRequestContentType::kBinary) + { + if (SerializeToHttpBody(body_vec, message)) + { + if (options_.console_debug) + { + OTEL_INTERNAL_LOG_DEBUG( + "[OTLP HTTP Client] Request body(Binary): " << message.Utf8DebugString()); + } + } + else + { + if (options_.console_debug) + { + OTEL_INTERNAL_LOG_DEBUG("[OTLP HTTP Client] Serialize body failed(Binary):" + << message.InitializationErrorString()); + } + return opentelemetry::sdk::common::ExportResult::kFailure; + } + content_type = kHttpBinaryContentType; + } + else + { + nlohmann::json json_request; + + // Convert from proto into json object + ConvertGenericMessageToJson(json_request, message, options_); + + std::string post_body_json = + json_request.dump(-1, ' ', false, nlohmann::detail::error_handler_t::replace); + if (options_.console_debug) + { + OTEL_INTERNAL_LOG_DEBUG("[OTLP HTTP Client] Request body(Json)" << post_body_json); + } + body_vec.assign(post_body_json.begin(), post_body_json.end()); + content_type = kHttpJsonContentType; + } + + // Send the request + std::lock_guard guard{session_manager_lock_}; + // Return failure if this exporter has been shutdown + if (isShutdown()) + { + const char *error_message = "[OTLP HTTP Client] Export failed, exporter is shutdown"; + if (options_.console_debug) + { + std::cerr << error_message << std::endl; + } + OTEL_INTERNAL_LOG_ERROR(error_message); + + return opentelemetry::sdk::common::ExportResult::kFailure; + } + + auto session = http_client_->CreateSession(options_.url); + auto request = session->CreateRequest(); + + for (auto &header : options_.http_headers) + { + request->AddHeader(header.first, header.second); + } + request->SetUri(http_uri_); + request->SetTimeoutMs(std::chrono::duration_cast(options_.timeout)); + request->SetMethod(http_client::Method::Post); + request->SetBody(body_vec); + request->ReplaceHeader("Content-Type", content_type); + + // Send the request + return HttpSessionData{ + std::move(session), + std::shared_ptr{ + new ResponseHandler(std::move(result_callback), options_.console_debug)}}; +} + +void OtlpHttpClient::addSession(HttpSessionData &&session_data) noexcept { - if (!session || !event_handle) + if (!session_data.session || !session_data.event_handle) { return; } - opentelemetry::ext::http::client::Session *key = session.get(); - ResponseHandler *handle = static_cast(event_handle.get()); + opentelemetry::ext::http::client::Session *key = session_data.session.get(); + ResponseHandler *handle = static_cast(session_data.event_handle.get()); handle->Bind(this, *key); - HttpSessionData &session_data = running_sessions_[key]; - session_data.session.swap(session); - session_data.event_handle.swap(event_handle); + HttpSessionData &store_session_data = running_sessions_[key]; + store_session_data = std::move(session_data); // Send request after the session is added - key->SendRequest(session_data.event_handle); + key->SendRequest(store_session_data.event_handle); } bool OtlpHttpClient::cleanupGCSessions() noexcept diff --git a/exporters/otlp/src/otlp_http_exporter.cc b/exporters/otlp/src/otlp_http_exporter.cc index d59d004c6c..a3c3354997 100644 --- a/exporters/otlp/src/otlp_http_exporter.cc +++ b/exporters/otlp/src/otlp_http_exporter.cc @@ -32,7 +32,7 @@ OtlpHttpExporter::OtlpHttpExporter(const OtlpHttpExporterOptions &options) options.console_debug, options.timeout, options.http_headers, - options.concurrent_sessions))) + options.max_concurrent_requests))) {} OtlpHttpExporter::OtlpHttpExporter(std::unique_ptr http_client) @@ -63,9 +63,14 @@ void OtlpHttpExporter::Export( const nostd::span> &spans, std::function &&result_callback) noexcept { - OTEL_INTERNAL_LOG_WARN(" async not supported. Making sync interface call"); - auto status = Export(spans); - result_callback(status); + if (spans.empty()) + { + return; + } + + proto::collector::trace::v1::ExportTraceServiceRequest service_request; + OtlpRecordableUtils::PopulateRequest(spans, &service_request); + http_client_->Export(service_request, std::move(result_callback)); } bool OtlpHttpExporter::Shutdown(std::chrono::microseconds timeout) noexcept diff --git a/exporters/otlp/src/otlp_http_log_exporter.cc b/exporters/otlp/src/otlp_http_log_exporter.cc index 8116a25301..c66ab228f2 100644 --- a/exporters/otlp/src/otlp_http_log_exporter.cc +++ b/exporters/otlp/src/otlp_http_log_exporter.cc @@ -34,7 +34,7 @@ OtlpHttpLogExporter::OtlpHttpLogExporter(const OtlpHttpLogExporterOptions &optio options.console_debug, options.timeout, options.http_headers, - options.concurrent_sessions))) + options.max_concurrent_requests))) {} OtlpHttpLogExporter::OtlpHttpLogExporter(std::unique_ptr http_client) @@ -64,9 +64,13 @@ void OtlpHttpLogExporter::Export( const nostd::span> &logs, std::function &&result_callback) noexcept { - OTEL_INTERNAL_LOG_WARN(" async not supported. Making sync interface call"); - auto status = Export(logs); - result_callback(status); + if (logs.empty()) + { + return; + } + proto::collector::logs::v1::ExportLogsServiceRequest service_request; + OtlpRecordableUtils::PopulateRequest(logs, &service_request); + http_client_->Export(service_request, std::move(result_callback)); } bool OtlpHttpLogExporter::Shutdown(std::chrono::microseconds timeout) noexcept diff --git a/exporters/otlp/test/otlp_http_exporter_test.cc b/exporters/otlp/test/otlp_http_exporter_test.cc index 12c7494b2e..8f38685963 100644 --- a/exporters/otlp/test/otlp_http_exporter_test.cc +++ b/exporters/otlp/test/otlp_http_exporter_test.cc @@ -3,6 +3,9 @@ #ifndef HAVE_CPP_STDLIB +# include +# include + # include "opentelemetry/exporters/otlp/otlp_http_exporter.h" # include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" @@ -81,170 +84,218 @@ class OtlpHttpExporterTestPeer : public ::testing::Test auto http_client = http_client::HttpClientFactory::CreateNoSend(); return {new OtlpHttpClient(MakeOtlpHttpClientOptions(content_type), http_client), http_client}; } + + void ExportJsonIntegrationTest(bool is_async) + { + auto mock_otlp_client = + OtlpHttpExporterTestPeer::GetMockOtlpHttpClient(HttpRequestContentType::kJson); + auto mock_otlp_http_client = mock_otlp_client.first; + auto client = mock_otlp_client.second; + auto exporter = GetExporter(std::unique_ptr{mock_otlp_http_client}); + + resource::ResourceAttributes resource_attributes = {{"service.name", "unit_test_service"}, + {"tenant.id", "test_user"}}; + resource_attributes["bool_value"] = true; + resource_attributes["int32_value"] = static_cast(1); + resource_attributes["uint32_value"] = static_cast(2); + resource_attributes["int64_value"] = static_cast(0x1100000000LL); + resource_attributes["uint64_value"] = static_cast(0x1200000000ULL); + resource_attributes["double_value"] = static_cast(3.1); + resource_attributes["vec_bool_value"] = std::vector{true, false, true}; + resource_attributes["vec_int32_value"] = std::vector{1, 2}; + resource_attributes["vec_uint32_value"] = std::vector{3, 4}; + resource_attributes["vec_int64_value"] = std::vector{5, 6}; + resource_attributes["vec_uint64_value"] = std::vector{7, 8}; + resource_attributes["vec_double_value"] = std::vector{3.2, 3.3}; + resource_attributes["vec_string_value"] = std::vector{"vector", "string"}; + auto resource = resource::Resource::Create(resource_attributes); + + auto processor_opts = sdk::trace::BatchSpanProcessorOptions(); + processor_opts.max_export_batch_size = 5; + processor_opts.max_queue_size = 5; + processor_opts.schedule_delay_millis = std::chrono::milliseconds(256); + processor_opts.is_export_async = is_async; + auto processor = std::unique_ptr( + new sdk::trace::BatchSpanProcessor(std::move(exporter), processor_opts)); + auto provider = nostd::shared_ptr( + new sdk::trace::TracerProvider(std::move(processor), resource)); + + std::string report_trace_id; + + char trace_id_hex[2 * trace_api::TraceId::kSize] = {0}; + auto tracer = provider->GetTracer("test"); + auto parent_span = tracer->StartSpan("Test parent span"); + + trace_api::StartSpanOptions child_span_opts = {}; + child_span_opts.parent = parent_span->GetContext(); + + auto child_span = tracer->StartSpan("Test child span", child_span_opts); + + nostd::get(child_span_opts.parent) + .trace_id() + .ToLowerBase16(MakeSpan(trace_id_hex)); + report_trace_id.assign(trace_id_hex, sizeof(trace_id_hex)); + + auto no_send_client = std::static_pointer_cast(client); + auto mock_session = + std::static_pointer_cast(no_send_client->session_); + EXPECT_CALL(*mock_session, SendRequest) + .WillOnce([&mock_session, report_trace_id, is_async]( + std::shared_ptr callback) { + auto check_json = + nlohmann::json::parse(mock_session->GetRequest()->body_, nullptr, false); + auto resource_span = *check_json["resource_spans"].begin(); + auto instrumentation_library_span = + *resource_span["instrumentation_library_spans"].begin(); + auto span = *instrumentation_library_span["spans"].begin(); + auto received_trace_id = span["trace_id"].get(); + EXPECT_EQ(received_trace_id, report_trace_id); + + auto custom_header = mock_session->GetRequest()->headers_.find("Custom-Header-Key"); + ASSERT_TRUE(custom_header != mock_session->GetRequest()->headers_.end()); + if (custom_header != mock_session->GetRequest()->headers_.end()) + { + EXPECT_EQ("Custom-Header-Value", custom_header->second); + } + + // let the otlp_http_client to continue + if (is_async) + { + std::thread async_finish{[callback]() { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + http_client::nosend::Response response; + response.Finish(*callback.get()); + }}; + async_finish.detach(); + } + else + { + http_client::nosend::Response response; + response.Finish(*callback.get()); + } + }); + + child_span->End(); + parent_span->End(); + + static_cast(provider.get())->ForceFlush(); + } + + void ExportBinaryIntegrationTest(bool is_async) + { + auto mock_otlp_client = + OtlpHttpExporterTestPeer::GetMockOtlpHttpClient(HttpRequestContentType::kBinary); + auto mock_otlp_http_client = mock_otlp_client.first; + auto client = mock_otlp_client.second; + auto exporter = GetExporter(std::unique_ptr{mock_otlp_http_client}); + + resource::ResourceAttributes resource_attributes = {{"service.name", "unit_test_service"}, + {"tenant.id", "test_user"}}; + resource_attributes["bool_value"] = true; + resource_attributes["int32_value"] = static_cast(1); + resource_attributes["uint32_value"] = static_cast(2); + resource_attributes["int64_value"] = static_cast(0x1100000000LL); + resource_attributes["uint64_value"] = static_cast(0x1200000000ULL); + resource_attributes["double_value"] = static_cast(3.1); + resource_attributes["vec_bool_value"] = std::vector{true, false, true}; + resource_attributes["vec_int32_value"] = std::vector{1, 2}; + resource_attributes["vec_uint32_value"] = std::vector{3, 4}; + resource_attributes["vec_int64_value"] = std::vector{5, 6}; + resource_attributes["vec_uint64_value"] = std::vector{7, 8}; + resource_attributes["vec_double_value"] = std::vector{3.2, 3.3}; + resource_attributes["vec_string_value"] = std::vector{"vector", "string"}; + auto resource = resource::Resource::Create(resource_attributes); + + auto processor_opts = sdk::trace::BatchSpanProcessorOptions(); + processor_opts.max_export_batch_size = 5; + processor_opts.max_queue_size = 5; + processor_opts.schedule_delay_millis = std::chrono::milliseconds(256); + processor_opts.is_export_async = is_async; + + auto processor = std::unique_ptr( + new sdk::trace::BatchSpanProcessor(std::move(exporter), processor_opts)); + auto provider = nostd::shared_ptr( + new sdk::trace::TracerProvider(std::move(processor), resource)); + + std::string report_trace_id; + + uint8_t trace_id_binary[trace_api::TraceId::kSize] = {0}; + auto tracer = provider->GetTracer("test"); + auto parent_span = tracer->StartSpan("Test parent span"); + + trace_api::StartSpanOptions child_span_opts = {}; + child_span_opts.parent = parent_span->GetContext(); + + auto child_span = tracer->StartSpan("Test child span", child_span_opts); + nostd::get(child_span_opts.parent) + .trace_id() + .CopyBytesTo(MakeSpan(trace_id_binary)); + report_trace_id.assign(reinterpret_cast(trace_id_binary), sizeof(trace_id_binary)); + + auto no_send_client = std::static_pointer_cast(client); + auto mock_session = + std::static_pointer_cast(no_send_client->session_); + EXPECT_CALL(*mock_session, SendRequest) + .WillOnce([&mock_session, report_trace_id, is_async]( + std::shared_ptr callback) { + opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest request_body; + request_body.ParseFromArray(&mock_session->GetRequest()->body_[0], + static_cast(mock_session->GetRequest()->body_.size())); + auto received_trace_id = + request_body.resource_spans(0).instrumentation_library_spans(0).spans(0).trace_id(); + EXPECT_EQ(received_trace_id, report_trace_id); + + auto custom_header = mock_session->GetRequest()->headers_.find("Custom-Header-Key"); + ASSERT_TRUE(custom_header != mock_session->GetRequest()->headers_.end()); + if (custom_header != mock_session->GetRequest()->headers_.end()) + { + EXPECT_EQ("Custom-Header-Value", custom_header->second); + } + + // let the otlp_http_client to continue + if (is_async) + { + std::thread async_finish{[callback]() { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + http_client::nosend::Response response; + response.Finish(*callback.get()); + }}; + async_finish.detach(); + } + else + { + http_client::nosend::Response response; + response.Finish(*callback.get()); + } + }); + + child_span->End(); + parent_span->End(); + + static_cast(provider.get())->ForceFlush(); + } }; // Create spans, let processor call Export() -TEST_F(OtlpHttpExporterTestPeer, ExportJsonIntegrationTest) +TEST_F(OtlpHttpExporterTestPeer, ExportJsonIntegrationTestSync) { - auto mock_otlp_client = - OtlpHttpExporterTestPeer::GetMockOtlpHttpClient(HttpRequestContentType::kJson); - auto mock_otlp_http_client = mock_otlp_client.first; - auto client = mock_otlp_client.second; - auto exporter = GetExporter(std::unique_ptr{mock_otlp_http_client}); - - resource::ResourceAttributes resource_attributes = {{"service.name", "unit_test_service"}, - {"tenant.id", "test_user"}}; - resource_attributes["bool_value"] = true; - resource_attributes["int32_value"] = static_cast(1); - resource_attributes["uint32_value"] = static_cast(2); - resource_attributes["int64_value"] = static_cast(0x1100000000LL); - resource_attributes["uint64_value"] = static_cast(0x1200000000ULL); - resource_attributes["double_value"] = static_cast(3.1); - resource_attributes["vec_bool_value"] = std::vector{true, false, true}; - resource_attributes["vec_int32_value"] = std::vector{1, 2}; - resource_attributes["vec_uint32_value"] = std::vector{3, 4}; - resource_attributes["vec_int64_value"] = std::vector{5, 6}; - resource_attributes["vec_uint64_value"] = std::vector{7, 8}; - resource_attributes["vec_double_value"] = std::vector{3.2, 3.3}; - resource_attributes["vec_string_value"] = std::vector{"vector", "string"}; - auto resource = resource::Resource::Create(resource_attributes); - - auto processor_opts = sdk::trace::BatchSpanProcessorOptions(); - processor_opts.max_export_batch_size = 5; - processor_opts.max_queue_size = 5; - processor_opts.schedule_delay_millis = std::chrono::milliseconds(256); - auto processor = std::unique_ptr( - new sdk::trace::BatchSpanProcessor(std::move(exporter), processor_opts)); - auto provider = nostd::shared_ptr( - new sdk::trace::TracerProvider(std::move(processor), resource)); - - std::string report_trace_id; - - char trace_id_hex[2 * trace_api::TraceId::kSize] = {0}; - auto tracer = provider->GetTracer("test"); - auto parent_span = tracer->StartSpan("Test parent span"); - - trace_api::StartSpanOptions child_span_opts = {}; - child_span_opts.parent = parent_span->GetContext(); - - auto child_span = tracer->StartSpan("Test child span", child_span_opts); - - nostd::get(child_span_opts.parent) - .trace_id() - .ToLowerBase16(MakeSpan(trace_id_hex)); - report_trace_id.assign(trace_id_hex, sizeof(trace_id_hex)); - - auto no_send_client = std::static_pointer_cast(client); - auto mock_session = - std::static_pointer_cast(no_send_client->session_); - EXPECT_CALL(*mock_session, SendRequest) - .WillOnce([&mock_session, report_trace_id]( - std::shared_ptr callback) { - auto check_json = nlohmann::json::parse(mock_session->GetRequest()->body_, nullptr, false); - auto resource_span = *check_json["resource_spans"].begin(); - auto instrumentation_library_span = *resource_span["instrumentation_library_spans"].begin(); - auto span = *instrumentation_library_span["spans"].begin(); - auto received_trace_id = span["trace_id"].get(); - EXPECT_EQ(received_trace_id, report_trace_id); - - auto custom_header = mock_session->GetRequest()->headers_.find("Custom-Header-Key"); - ASSERT_TRUE(custom_header != mock_session->GetRequest()->headers_.end()); - if (custom_header != mock_session->GetRequest()->headers_.end()) - { - EXPECT_EQ("Custom-Header-Value", custom_header->second); - } - // let the otlp_http_client to continue - http_client::nosend::Response response; - callback->OnResponse(response); - }); - - child_span->End(); - parent_span->End(); - - static_cast(provider.get())->ForceFlush(); + ExportJsonIntegrationTest(false); +} + +TEST_F(OtlpHttpExporterTestPeer, ExportJsonIntegrationTestAsync) +{ + ExportJsonIntegrationTest(true); } // Create spans, let processor call Export() -TEST_F(OtlpHttpExporterTestPeer, ExportBinaryIntegrationTest) +TEST_F(OtlpHttpExporterTestPeer, ExportBinaryIntegrationTestSync) +{ + ExportBinaryIntegrationTest(false); +} + +TEST_F(OtlpHttpExporterTestPeer, ExportBinaryIntegrationTestAsync) { - auto mock_otlp_client = - OtlpHttpExporterTestPeer::GetMockOtlpHttpClient(HttpRequestContentType::kBinary); - auto mock_otlp_http_client = mock_otlp_client.first; - auto client = mock_otlp_client.second; - auto exporter = GetExporter(std::unique_ptr{mock_otlp_http_client}); - - resource::ResourceAttributes resource_attributes = {{"service.name", "unit_test_service"}, - {"tenant.id", "test_user"}}; - resource_attributes["bool_value"] = true; - resource_attributes["int32_value"] = static_cast(1); - resource_attributes["uint32_value"] = static_cast(2); - resource_attributes["int64_value"] = static_cast(0x1100000000LL); - resource_attributes["uint64_value"] = static_cast(0x1200000000ULL); - resource_attributes["double_value"] = static_cast(3.1); - resource_attributes["vec_bool_value"] = std::vector{true, false, true}; - resource_attributes["vec_int32_value"] = std::vector{1, 2}; - resource_attributes["vec_uint32_value"] = std::vector{3, 4}; - resource_attributes["vec_int64_value"] = std::vector{5, 6}; - resource_attributes["vec_uint64_value"] = std::vector{7, 8}; - resource_attributes["vec_double_value"] = std::vector{3.2, 3.3}; - resource_attributes["vec_string_value"] = std::vector{"vector", "string"}; - auto resource = resource::Resource::Create(resource_attributes); - - auto processor_opts = sdk::trace::BatchSpanProcessorOptions(); - processor_opts.max_export_batch_size = 5; - processor_opts.max_queue_size = 5; - processor_opts.schedule_delay_millis = std::chrono::milliseconds(256); - - auto processor = std::unique_ptr( - new sdk::trace::BatchSpanProcessor(std::move(exporter), processor_opts)); - auto provider = nostd::shared_ptr( - new sdk::trace::TracerProvider(std::move(processor), resource)); - - std::string report_trace_id; - - uint8_t trace_id_binary[trace_api::TraceId::kSize] = {0}; - auto tracer = provider->GetTracer("test"); - auto parent_span = tracer->StartSpan("Test parent span"); - - trace_api::StartSpanOptions child_span_opts = {}; - child_span_opts.parent = parent_span->GetContext(); - - auto child_span = tracer->StartSpan("Test child span", child_span_opts); - nostd::get(child_span_opts.parent) - .trace_id() - .CopyBytesTo(MakeSpan(trace_id_binary)); - report_trace_id.assign(reinterpret_cast(trace_id_binary), sizeof(trace_id_binary)); - - auto no_send_client = std::static_pointer_cast(client); - auto mock_session = - std::static_pointer_cast(no_send_client->session_); - EXPECT_CALL(*mock_session, SendRequest) - .WillOnce([&mock_session, report_trace_id]( - std::shared_ptr callback) { - opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest request_body; - request_body.ParseFromArray(&mock_session->GetRequest()->body_[0], - static_cast(mock_session->GetRequest()->body_.size())); - auto received_trace_id = - request_body.resource_spans(0).instrumentation_library_spans(0).spans(0).trace_id(); - EXPECT_EQ(received_trace_id, report_trace_id); - - auto custom_header = mock_session->GetRequest()->headers_.find("Custom-Header-Key"); - ASSERT_TRUE(custom_header != mock_session->GetRequest()->headers_.end()); - if (custom_header != mock_session->GetRequest()->headers_.end()) - { - EXPECT_EQ("Custom-Header-Value", custom_header->second); - } - - // let the otlp_http_client to continue - http_client::nosend::Response response; - - response.Finish(*callback.get()); - }); - - child_span->End(); - parent_span->End(); - - static_cast(provider.get())->ForceFlush(); + ExportBinaryIntegrationTest(true); } // Test exporter configuration options diff --git a/exporters/otlp/test/otlp_http_log_exporter_test.cc b/exporters/otlp/test/otlp_http_log_exporter_test.cc index e49824f0b7..50034ed12d 100644 --- a/exporters/otlp/test/otlp_http_log_exporter_test.cc +++ b/exporters/otlp/test/otlp_http_log_exporter_test.cc @@ -4,6 +4,9 @@ #ifndef HAVE_CPP_STDLIB # ifdef ENABLE_LOGS_PREVIEW +# include +# include + # include "opentelemetry/exporters/otlp/otlp_http_log_exporter.h" # include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" @@ -82,188 +85,241 @@ class OtlpHttpLogExporterTestPeer : public ::testing::Test auto http_client = http_client::HttpClientFactory::CreateNoSend(); return {new OtlpHttpClient(MakeOtlpHttpClientOptions(content_type), http_client), http_client}; } + + void ExportJsonIntegrationTest(bool is_async) + { + auto mock_otlp_client = + OtlpHttpLogExporterTestPeer::GetMockOtlpHttpClient(HttpRequestContentType::kJson); + auto mock_otlp_http_client = mock_otlp_client.first; + auto client = mock_otlp_client.second; + auto exporter = GetExporter(std::unique_ptr{mock_otlp_http_client}); + + bool attribute_storage_bool_value[] = {true, false, true}; + int32_t attribute_storage_int32_value[] = {1, 2}; + uint32_t attribute_storage_uint32_value[] = {3, 4}; + int64_t attribute_storage_int64_value[] = {5, 6}; + uint64_t attribute_storage_uint64_value[] = {7, 8}; + double attribute_storage_double_value[] = {3.2, 3.3}; + std::string attribute_storage_string_value[] = {"vector", "string"}; + + auto provider = nostd::shared_ptr(new sdk::logs::LoggerProvider()); + provider->AddProcessor( + std::unique_ptr(new sdk::logs::BatchLogProcessor( + std::move(exporter), 5, std::chrono::milliseconds(256), 5, is_async))); + + std::string report_trace_id; + std::string report_span_id; + uint8_t trace_id_bin[opentelemetry::trace::TraceId::kSize] = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + char trace_id_hex[2 * opentelemetry::trace::TraceId::kSize] = {0}; + opentelemetry::trace::TraceId trace_id{trace_id_bin}; + uint8_t span_id_bin[opentelemetry::trace::SpanId::kSize] = {'7', '6', '5', '4', + '3', '2', '1', '0'}; + char span_id_hex[2 * opentelemetry::trace::SpanId::kSize] = {0}; + opentelemetry::trace::SpanId span_id{span_id_bin}; + + const std::string schema_url{"https://opentelemetry.io/schemas/1.2.0"}; + auto logger = provider->GetLogger("test", "", "opentelelemtry_library", "", schema_url); + + trace_id.ToLowerBase16(MakeSpan(trace_id_hex)); + report_trace_id.assign(trace_id_hex, sizeof(trace_id_hex)); + + span_id.ToLowerBase16(MakeSpan(span_id_hex)); + report_span_id.assign(span_id_hex, sizeof(span_id_hex)); + + auto no_send_client = std::static_pointer_cast(client); + auto mock_session = + std::static_pointer_cast(no_send_client->session_); + EXPECT_CALL(*mock_session, SendRequest) + .WillOnce([&mock_session, report_trace_id, report_span_id, is_async]( + std::shared_ptr callback) { + auto check_json = + nlohmann::json::parse(mock_session->GetRequest()->body_, nullptr, false); + auto resource_logs = *check_json["resource_logs"].begin(); + auto instrumentation_library_span = + *resource_logs["instrumentation_library_logs"].begin(); + auto log = *instrumentation_library_span["logs"].begin(); + auto received_trace_id = log["trace_id"].get(); + auto received_span_id = log["span_id"].get(); + EXPECT_EQ(received_trace_id, report_trace_id); + EXPECT_EQ(received_span_id, report_span_id); + EXPECT_EQ("Log name", log["name"].get()); + EXPECT_EQ("Log message", log["body"]["string_value"].get()); + EXPECT_LE(15, log["attributes"].size()); + auto custom_header = mock_session->GetRequest()->headers_.find("Custom-Header-Key"); + ASSERT_TRUE(custom_header != mock_session->GetRequest()->headers_.end()); + if (custom_header != mock_session->GetRequest()->headers_.end()) + { + EXPECT_EQ("Custom-Header-Value", custom_header->second); + } + + // let the otlp_http_client to continue + if (is_async) + { + std::thread async_finish{[callback]() { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + http_client::nosend::Response response; + response.Finish(*callback.get()); + }}; + async_finish.detach(); + } + else + { + http_client::nosend::Response response; + response.Finish(*callback.get()); + } + }); + + logger->Log(opentelemetry::logs::Severity::kInfo, "Log name", "Log message", + {{"service.name", "unit_test_service"}, + {"tenant.id", "test_user"}, + {"bool_value", true}, + {"int32_value", static_cast(1)}, + {"uint32_value", static_cast(2)}, + {"int64_value", static_cast(0x1100000000LL)}, + {"uint64_value", static_cast(0x1200000000ULL)}, + {"double_value", static_cast(3.1)}, + {"vec_bool_value", attribute_storage_bool_value}, + {"vec_int32_value", attribute_storage_int32_value}, + {"vec_uint32_value", attribute_storage_uint32_value}, + {"vec_int64_value", attribute_storage_int64_value}, + {"vec_uint64_value", attribute_storage_uint64_value}, + {"vec_double_value", attribute_storage_double_value}, + {"vec_string_value", attribute_storage_string_value}}, + trace_id, span_id, + opentelemetry::trace::TraceFlags{opentelemetry::trace::TraceFlags::kIsSampled}, + std::chrono::system_clock::now()); + + provider->ForceFlush(); + } + + void ExportBinaryIntegrationTest(bool is_async) + { + auto mock_otlp_client = + OtlpHttpLogExporterTestPeer::GetMockOtlpHttpClient(HttpRequestContentType::kBinary); + auto mock_otlp_http_client = mock_otlp_client.first; + auto client = mock_otlp_client.second; + auto exporter = GetExporter(std::unique_ptr{mock_otlp_http_client}); + + bool attribute_storage_bool_value[] = {true, false, true}; + int32_t attribute_storage_int32_value[] = {1, 2}; + uint32_t attribute_storage_uint32_value[] = {3, 4}; + int64_t attribute_storage_int64_value[] = {5, 6}; + uint64_t attribute_storage_uint64_value[] = {7, 8}; + double attribute_storage_double_value[] = {3.2, 3.3}; + std::string attribute_storage_string_value[] = {"vector", "string"}; + + sdk::logs::BatchLogProcessorOptions processor_options; + processor_options.max_export_batch_size = 5; + processor_options.max_queue_size = 5; + processor_options.schedule_delay_millis = std::chrono::milliseconds(256); + processor_options.is_export_async = is_async; + auto provider = nostd::shared_ptr(new sdk::logs::LoggerProvider()); + provider->AddProcessor(std::unique_ptr( + new sdk::logs::BatchLogProcessor(std::move(exporter), processor_options))); + + std::string report_trace_id; + std::string report_span_id; + uint8_t trace_id_bin[opentelemetry::trace::TraceId::kSize] = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + opentelemetry::trace::TraceId trace_id{trace_id_bin}; + uint8_t span_id_bin[opentelemetry::trace::SpanId::kSize] = {'7', '6', '5', '4', + '3', '2', '1', '0'}; + opentelemetry::trace::SpanId span_id{span_id_bin}; + + const std::string schema_url{"https://opentelemetry.io/schemas/1.2.0"}; + auto logger = provider->GetLogger("test", "", "opentelelemtry_library", "", schema_url); + + report_trace_id.assign(reinterpret_cast(trace_id_bin), sizeof(trace_id_bin)); + report_span_id.assign(reinterpret_cast(span_id_bin), sizeof(span_id_bin)); + + auto no_send_client = std::static_pointer_cast(client); + auto mock_session = + std::static_pointer_cast(no_send_client->session_); + EXPECT_CALL(*mock_session, SendRequest) + .WillOnce([&mock_session, report_trace_id, report_span_id, is_async]( + std::shared_ptr callback) { + opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest request_body; + request_body.ParseFromArray(&mock_session->GetRequest()->body_[0], + static_cast(mock_session->GetRequest()->body_.size())); + auto received_log = request_body.resource_logs(0).instrumentation_library_logs(0).logs(0); + EXPECT_EQ(received_log.trace_id(), report_trace_id); + EXPECT_EQ(received_log.span_id(), report_span_id); + EXPECT_EQ("Log name", received_log.name()); + EXPECT_EQ("Log message", received_log.body().string_value()); + EXPECT_LE(15, received_log.attributes_size()); + bool check_service_name = false; + for (auto &attribute : received_log.attributes()) + { + if ("service.name" == attribute.key()) + { + check_service_name = true; + EXPECT_EQ("unit_test_service", attribute.value().string_value()); + } + } + ASSERT_TRUE(check_service_name); + + // let the otlp_http_client to continue + if (is_async) + { + std::thread async_finish{[callback]() { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + http_client::nosend::Response response; + response.Finish(*callback.get()); + }}; + async_finish.detach(); + } + else + { + http_client::nosend::Response response; + response.Finish(*callback.get()); + } + }); + + logger->Log(opentelemetry::logs::Severity::kInfo, "Log name", "Log message", + {{"service.name", "unit_test_service"}, + {"tenant.id", "test_user"}, + {"bool_value", true}, + {"int32_value", static_cast(1)}, + {"uint32_value", static_cast(2)}, + {"int64_value", static_cast(0x1100000000LL)}, + {"uint64_value", static_cast(0x1200000000ULL)}, + {"double_value", static_cast(3.1)}, + {"vec_bool_value", attribute_storage_bool_value}, + {"vec_int32_value", attribute_storage_int32_value}, + {"vec_uint32_value", attribute_storage_uint32_value}, + {"vec_int64_value", attribute_storage_int64_value}, + {"vec_uint64_value", attribute_storage_uint64_value}, + {"vec_double_value", attribute_storage_double_value}, + {"vec_string_value", attribute_storage_string_value}}, + trace_id, span_id, + opentelemetry::trace::TraceFlags{opentelemetry::trace::TraceFlags::kIsSampled}, + std::chrono::system_clock::now()); + + provider->ForceFlush(); + } }; // Create log records, let processor call Export() -TEST_F(OtlpHttpLogExporterTestPeer, ExportJsonIntegrationTest) +TEST_F(OtlpHttpLogExporterTestPeer, ExportJsonIntegrationTestSync) +{ + ExportJsonIntegrationTest(false); +} + +TEST_F(OtlpHttpLogExporterTestPeer, ExportJsonIntegrationTestAsync) { - auto mock_otlp_client = - OtlpHttpLogExporterTestPeer::GetMockOtlpHttpClient(HttpRequestContentType::kJson); - auto mock_otlp_http_client = mock_otlp_client.first; - auto client = mock_otlp_client.second; - auto exporter = GetExporter(std::unique_ptr{mock_otlp_http_client}); - - bool attribute_storage_bool_value[] = {true, false, true}; - int32_t attribute_storage_int32_value[] = {1, 2}; - uint32_t attribute_storage_uint32_value[] = {3, 4}; - int64_t attribute_storage_int64_value[] = {5, 6}; - uint64_t attribute_storage_uint64_value[] = {7, 8}; - double attribute_storage_double_value[] = {3.2, 3.3}; - std::string attribute_storage_string_value[] = {"vector", "string"}; - - auto provider = nostd::shared_ptr(new sdk::logs::LoggerProvider()); - provider->AddProcessor(std::unique_ptr( - new sdk::logs::BatchLogProcessor(std::move(exporter), 5, std::chrono::milliseconds(256), 5))); - - std::string report_trace_id; - std::string report_span_id; - uint8_t trace_id_bin[opentelemetry::trace::TraceId::kSize] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - char trace_id_hex[2 * opentelemetry::trace::TraceId::kSize] = {0}; - opentelemetry::trace::TraceId trace_id{trace_id_bin}; - uint8_t span_id_bin[opentelemetry::trace::SpanId::kSize] = {'7', '6', '5', '4', - '3', '2', '1', '0'}; - char span_id_hex[2 * opentelemetry::trace::SpanId::kSize] = {0}; - opentelemetry::trace::SpanId span_id{span_id_bin}; - - const std::string schema_url{"https://opentelemetry.io/schemas/1.2.0"}; - auto logger = provider->GetLogger("test", "", "opentelelemtry_library", "", schema_url); - - trace_id.ToLowerBase16(MakeSpan(trace_id_hex)); - report_trace_id.assign(trace_id_hex, sizeof(trace_id_hex)); - - span_id.ToLowerBase16(MakeSpan(span_id_hex)); - report_span_id.assign(span_id_hex, sizeof(span_id_hex)); - - auto no_send_client = std::static_pointer_cast(client); - auto mock_session = - std::static_pointer_cast(no_send_client->session_); - EXPECT_CALL(*mock_session, SendRequest) - .WillOnce([&mock_session, report_trace_id, report_span_id]( - std::shared_ptr callback) { - auto check_json = nlohmann::json::parse(mock_session->GetRequest()->body_, nullptr, false); - auto resource_logs = *check_json["resource_logs"].begin(); - auto instrumentation_library_span = *resource_logs["instrumentation_library_logs"].begin(); - auto log = *instrumentation_library_span["logs"].begin(); - auto received_trace_id = log["trace_id"].get(); - auto received_span_id = log["span_id"].get(); - EXPECT_EQ(received_trace_id, report_trace_id); - EXPECT_EQ(received_span_id, report_span_id); - EXPECT_EQ("Log name", log["name"].get()); - EXPECT_EQ("Log message", log["body"]["string_value"].get()); - EXPECT_LE(15, log["attributes"].size()); - auto custom_header = mock_session->GetRequest()->headers_.find("Custom-Header-Key"); - ASSERT_TRUE(custom_header != mock_session->GetRequest()->headers_.end()); - if (custom_header != mock_session->GetRequest()->headers_.end()) - { - EXPECT_EQ("Custom-Header-Value", custom_header->second); - } - - // let the otlp_http_client to continue - http_client::nosend::Response response; - - response.Finish(*callback.get()); - }); - - logger->Log(opentelemetry::logs::Severity::kInfo, "Log name", "Log message", - {{"service.name", "unit_test_service"}, - {"tenant.id", "test_user"}, - {"bool_value", true}, - {"int32_value", static_cast(1)}, - {"uint32_value", static_cast(2)}, - {"int64_value", static_cast(0x1100000000LL)}, - {"uint64_value", static_cast(0x1200000000ULL)}, - {"double_value", static_cast(3.1)}, - {"vec_bool_value", attribute_storage_bool_value}, - {"vec_int32_value", attribute_storage_int32_value}, - {"vec_uint32_value", attribute_storage_uint32_value}, - {"vec_int64_value", attribute_storage_int64_value}, - {"vec_uint64_value", attribute_storage_uint64_value}, - {"vec_double_value", attribute_storage_double_value}, - {"vec_string_value", attribute_storage_string_value}}, - trace_id, span_id, - opentelemetry::trace::TraceFlags{opentelemetry::trace::TraceFlags::kIsSampled}, - std::chrono::system_clock::now()); - - provider->ForceFlush(); + ExportJsonIntegrationTest(true); } // Create log records, let processor call Export() -TEST_F(OtlpHttpLogExporterTestPeer, ExportBinaryIntegrationTest) +TEST_F(OtlpHttpLogExporterTestPeer, ExportBinaryIntegrationTestSync) { - auto mock_otlp_client = - OtlpHttpLogExporterTestPeer::GetMockOtlpHttpClient(HttpRequestContentType::kBinary); - auto mock_otlp_http_client = mock_otlp_client.first; - auto client = mock_otlp_client.second; - auto exporter = GetExporter(std::unique_ptr{mock_otlp_http_client}); - - bool attribute_storage_bool_value[] = {true, false, true}; - int32_t attribute_storage_int32_value[] = {1, 2}; - uint32_t attribute_storage_uint32_value[] = {3, 4}; - int64_t attribute_storage_int64_value[] = {5, 6}; - uint64_t attribute_storage_uint64_value[] = {7, 8}; - double attribute_storage_double_value[] = {3.2, 3.3}; - std::string attribute_storage_string_value[] = {"vector", "string"}; - - auto provider = nostd::shared_ptr(new sdk::logs::LoggerProvider()); - provider->AddProcessor(std::unique_ptr( - new sdk::logs::BatchLogProcessor(std::move(exporter), 5, std::chrono::milliseconds(256), 5))); - - std::string report_trace_id; - std::string report_span_id; - uint8_t trace_id_bin[opentelemetry::trace::TraceId::kSize] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - opentelemetry::trace::TraceId trace_id{trace_id_bin}; - uint8_t span_id_bin[opentelemetry::trace::SpanId::kSize] = {'7', '6', '5', '4', - '3', '2', '1', '0'}; - opentelemetry::trace::SpanId span_id{span_id_bin}; - - const std::string schema_url{"https://opentelemetry.io/schemas/1.2.0"}; - auto logger = provider->GetLogger("test", "", "opentelelemtry_library", "", schema_url); - - report_trace_id.assign(reinterpret_cast(trace_id_bin), sizeof(trace_id_bin)); - report_span_id.assign(reinterpret_cast(span_id_bin), sizeof(span_id_bin)); - - auto no_send_client = std::static_pointer_cast(client); - auto mock_session = - std::static_pointer_cast(no_send_client->session_); - EXPECT_CALL(*mock_session, SendRequest) - .WillOnce([&mock_session, report_trace_id, report_span_id]( - std::shared_ptr callback) { - opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest request_body; - request_body.ParseFromArray(&mock_session->GetRequest()->body_[0], - static_cast(mock_session->GetRequest()->body_.size())); - auto received_log = request_body.resource_logs(0).instrumentation_library_logs(0).logs(0); - EXPECT_EQ(received_log.trace_id(), report_trace_id); - EXPECT_EQ(received_log.span_id(), report_span_id); - EXPECT_EQ("Log name", received_log.name()); - EXPECT_EQ("Log message", received_log.body().string_value()); - EXPECT_LE(15, received_log.attributes_size()); - bool check_service_name = false; - for (auto &attribute : received_log.attributes()) - { - if ("service.name" == attribute.key()) - { - check_service_name = true; - EXPECT_EQ("unit_test_service", attribute.value().string_value()); - } - } - ASSERT_TRUE(check_service_name); - http_client::nosend::Response response; - callback->OnResponse(response); - }); - - logger->Log(opentelemetry::logs::Severity::kInfo, "Log name", "Log message", - {{"service.name", "unit_test_service"}, - {"tenant.id", "test_user"}, - {"bool_value", true}, - {"int32_value", static_cast(1)}, - {"uint32_value", static_cast(2)}, - {"int64_value", static_cast(0x1100000000LL)}, - {"uint64_value", static_cast(0x1200000000ULL)}, - {"double_value", static_cast(3.1)}, - {"vec_bool_value", attribute_storage_bool_value}, - {"vec_int32_value", attribute_storage_int32_value}, - {"vec_uint32_value", attribute_storage_uint32_value}, - {"vec_int64_value", attribute_storage_int64_value}, - {"vec_uint64_value", attribute_storage_uint64_value}, - {"vec_double_value", attribute_storage_double_value}, - {"vec_string_value", attribute_storage_string_value}}, - trace_id, span_id, - opentelemetry::trace::TraceFlags{opentelemetry::trace::TraceFlags::kIsSampled}, - std::chrono::system_clock::now()); - - provider->ForceFlush(); + ExportBinaryIntegrationTest(false); +} + +TEST_F(OtlpHttpLogExporterTestPeer, ExportBinaryIntegrationTestAsync) +{ + ExportBinaryIntegrationTest(true); } // Test exporter configuration options diff --git a/sdk/CMakeLists.txt b/sdk/CMakeLists.txt index 9aa45883e5..6421e3ad15 100644 --- a/sdk/CMakeLists.txt +++ b/sdk/CMakeLists.txt @@ -13,6 +13,12 @@ install( LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +install( + DIRECTORY include/opentelemetry/ + DESTINATION include/opentelemetry/ + FILES_MATCHING + PATTERN "*config.h") + set(LOGS_EXCLUDE_PATTERN "") if(NOT WITH_LOGS_PREVIEW) set(LOGS_EXCLUDE_PATTERN "logs") diff --git a/sdk/include/opentelemetry/sdk/logs/batch_log_processor.h b/sdk/include/opentelemetry/sdk/logs/batch_log_processor.h index 1387c9c82b..9a8448a119 100644 --- a/sdk/include/opentelemetry/sdk/logs/batch_log_processor.h +++ b/sdk/include/opentelemetry/sdk/logs/batch_log_processor.h @@ -10,6 +10,8 @@ # include # include +# include +# include # include OPENTELEMETRY_BEGIN_NAMESPACE @@ -19,6 +21,33 @@ namespace sdk namespace logs { +/** + * Struct to hold batch SpanProcessor options. + */ +struct BatchLogProcessorOptions +{ + /** + * The maximum buffer/queue size. After the size is reached, spans are + * dropped. + */ + size_t max_queue_size = 2048; + + /* The time interval between two consecutive exports. */ + std::chrono::milliseconds schedule_delay_millis = std::chrono::milliseconds(5000); + + /** + * The maximum batch size of every export. It must be smaller or + * equal to max_queue_size. + */ + size_t max_export_batch_size = 512; + + /** + * Determines whether the export happens asynchronously. + * Default implementation is synchronous. + */ + bool is_export_async = false; +}; + /** * This is an implementation of the LogProcessor which creates batches of finished logs and passes * the export-friendly log data representations to the configured LogExporter. @@ -44,6 +73,16 @@ class BatchLogProcessor : public LogProcessor const size_t max_export_batch_size = 512, const bool is_export_async = false); + /** + * Creates a batch log processor by configuring the specified exporter and other parameters + * as per the official, language-agnostic opentelemetry specs. + * + * @param exporter - The backend exporter to pass the logs to + * @param options - The batch SpanProcessor options. + */ + explicit BatchLogProcessor(std::unique_ptr &&exporter, + const BatchLogProcessorOptions &options); + /** Makes a new recordable **/ std::unique_ptr MakeRecordable() noexcept override; @@ -86,12 +125,8 @@ class BatchLogProcessor : public LogProcessor /** * Exports all logs to the configured exporter. * - * @param was_force_flush_called - A flag to check if the current export is the result - * of a call to ForceFlush method. If true, then we have to - * notify the main thread to wake it up in the ForceFlush - * method. */ - void Export(const bool was_for_flush_called); + void Export(); /** * Called when Shutdown() is invoked. Completely drains the queue of all log records and @@ -99,18 +134,32 @@ class BatchLogProcessor : public LogProcessor */ void DrainQueue(); - /** - * Should be called from Export to notify the main thread on Force Flush Completion - * @param was_force_flush_called - A flag to check if the current export is the result - * of a call to ForceFlush method. If true, then we have to - * notify the main thread to wake it up in the ForceFlush - * method. - */ - void NotifyForceFlushCompletion(const bool was_for_flush_called); - /* In case of async export, wait and notify for shutdown to be completed.*/ void WaitForShutdownCompletion(); - void NotifyShutdownCompletion(); + + struct SynchronizationData + { + /* Synchronization primitives */ + std::condition_variable cv, force_flush_cv, async_shutdown_cv; + std::mutex cv_m, force_flush_cv_m, shutdown_m, async_shutdown_m; + + /* Important boolean flags to handle the workflow of the processor */ + std::atomic is_force_wakeup_background_worker; + std::atomic is_force_flush_pending; + std::atomic is_force_flush_notified; + std::atomic is_shutdown; + std::atomic is_async_shutdown_notified; + }; + + /** + * @brief Notify completion of shutdown and force flush. This may be called from the any thread at + * any time + * + * @param notify_force_flush Flag to indicate whether to notify force flush completion. + * @param synchronization_data Synchronization data to be notified. + */ + static void NotifyCompletion(bool notify_force_flush, + const std::shared_ptr &synchronization_data); /* The configured backend log exporter */ std::unique_ptr exporter_; @@ -121,18 +170,10 @@ class BatchLogProcessor : public LogProcessor const size_t max_export_batch_size_; const bool is_export_async_; - /* Synchronization primitives */ - std::condition_variable cv_, force_flush_cv_, async_shutdown_cv_; - std::mutex cv_m_, force_flush_cv_m_, shutdown_m_, async_shutdown_m_; - /* The buffer/queue to which the ended logs are added */ common::CircularBuffer buffer_; - /* Important boolean flags to handle the workflow of the processor */ - std::atomic is_shutdown_; - std::atomic is_force_flush_; - std::atomic is_force_flush_notified_; - std::atomic is_async_shutdown_notified_; + std::shared_ptr synchronization_data_; /* The background worker thread */ std::thread worker_thread_; diff --git a/sdk/include/opentelemetry/sdk/metrics/aggregation/sum_aggregation.h b/sdk/include/opentelemetry/sdk/metrics/aggregation/sum_aggregation.h index c8b2eff33c..ff99cec733 100644 --- a/sdk/include/opentelemetry/sdk/metrics/aggregation/sum_aggregation.h +++ b/sdk/include/opentelemetry/sdk/metrics/aggregation/sum_aggregation.h @@ -25,7 +25,7 @@ static inline void PopulateSumPointData(SumPointData &sum, sum.end_epoch_nanos_ = end_ts; sum.value_ = value; sum.is_monotonic_ = is_monotonic; - sum.aggregation_temporarily_ = AggregationTemporarily::kDelta; + sum.aggregation_temporality_ = AggregationTemporality::kDelta; } class LongSumAggregation : public Aggregation, InstrumentMonotonicityAwareAggregation diff --git a/sdk/include/opentelemetry/sdk/metrics/data/point_data.h b/sdk/include/opentelemetry/sdk/metrics/data/point_data.h index 603bcc30bc..5bbcbd0b1e 100644 --- a/sdk/include/opentelemetry/sdk/metrics/data/point_data.h +++ b/sdk/include/opentelemetry/sdk/metrics/data/point_data.h @@ -25,7 +25,7 @@ class SumPointData opentelemetry::common::SystemTimestamp start_epoch_nanos_; opentelemetry::common::SystemTimestamp end_epoch_nanos_; ValueType value_; - AggregationTemporarily aggregation_temporarily_; + AggregationTemporality aggregation_temporality_; bool is_monotonic_; }; diff --git a/sdk/include/opentelemetry/sdk/metrics/exemplar/always_sample_filter.h b/sdk/include/opentelemetry/sdk/metrics/exemplar/always_sample_filter.h new file mode 100644 index 0000000000..5e7f0436eb --- /dev/null +++ b/sdk/include/opentelemetry/sdk/metrics/exemplar/always_sample_filter.h @@ -0,0 +1,43 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once +#ifndef ENABLE_METRICS_PREVIEW +# include "opentelemetry/sdk/metrics/exemplar/filter.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace metrics +{ + +class AlwaysSampleFilter final : public ExemplarFilter +{ +public: + static nostd::shared_ptr GetAlwaysSampleFilter() + { + static nostd::shared_ptr alwaysSampleFilter{new AlwaysSampleFilter{}}; + return alwaysSampleFilter; + } + + bool ShouldSampleMeasurement(long value, + const MetricAttributes &attributes, + const opentelemetry::context::Context &context) noexcept override + { + return true; + } + + bool ShouldSampleMeasurement(double value, + const MetricAttributes &attributes, + const opentelemetry::context::Context &context) noexcept override + { + return true; + } + +private: + explicit AlwaysSampleFilter() = default; +}; +} // namespace metrics +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE +#endif diff --git a/sdk/include/opentelemetry/sdk/metrics/exemplar/data.h b/sdk/include/opentelemetry/sdk/metrics/exemplar/data.h new file mode 100644 index 0000000000..14eac62499 --- /dev/null +++ b/sdk/include/opentelemetry/sdk/metrics/exemplar/data.h @@ -0,0 +1,45 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once +#ifndef ENABLE_METRICS_PREVIEW +# include "opentelemetry/common/timestamp.h" +# include "opentelemetry/context/context.h" +# include "opentelemetry/sdk/common/attribute_utils.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace metrics +{ +using MetricAttributes = opentelemetry::sdk::common::OrderedAttributeMap; +/** + * A sample input measurement. + * + * Exemplars also hold information about the environment when the measurement was recorded, for + * example the span and trace ID of the active span when the exemplar was recorded. + */ +class ExemplarData +{ +public: + /** + * The set of key/value pairs that were filtered out by the aggregator, but recorded alongside the + * original measurement. Only key/value pairs that were filtered out by the aggregator should be + * included + */ + MetricAttributes GetFilteredAttributes(); + + /** Returns the timestamp in nanos when measurement was collected. */ + opentelemetry::common::SystemTimestamp GetEpochNanos(); + + /** + * Returns the SpanContext associated with this exemplar. If the exemplar was not recorded + * inside a sampled trace, the Context will be invalid. + */ + opentelemetry::context::Context GetSpanContext(); +}; + +} // namespace metrics +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE +#endif diff --git a/sdk/include/opentelemetry/sdk/metrics/exemplar/filter.h b/sdk/include/opentelemetry/sdk/metrics/exemplar/filter.h new file mode 100644 index 0000000000..4b512e1317 --- /dev/null +++ b/sdk/include/opentelemetry/sdk/metrics/exemplar/filter.h @@ -0,0 +1,39 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once +#ifndef ENABLE_METRICS_PREVIEW +# include "opentelemetry/context/context.h" +# include "opentelemetry/sdk/common/attribute_utils.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace metrics +{ +using MetricAttributes = opentelemetry::sdk::common::OrderedAttributeMap; + +/** + * Exemplar filters are used to pre-filter measurements before attempting to store them in a + * reservoir. + */ +class ExemplarFilter +{ +public: + // Returns whether or not a reservoir should attempt to filter a measurement. + virtual bool ShouldSampleMeasurement(long value, + const MetricAttributes &attributes, + const opentelemetry::context::Context &context) noexcept = 0; + + // Returns whether or not a reservoir should attempt to filter a measurement. + virtual bool ShouldSampleMeasurement(double value, + const MetricAttributes &attributes, + const opentelemetry::context::Context &context) noexcept = 0; + + virtual ~ExemplarFilter() = default; +}; + +} // namespace metrics +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE +#endif diff --git a/sdk/include/opentelemetry/sdk/metrics/exemplar/never_sample_filter.h b/sdk/include/opentelemetry/sdk/metrics/exemplar/never_sample_filter.h new file mode 100644 index 0000000000..38f51778ce --- /dev/null +++ b/sdk/include/opentelemetry/sdk/metrics/exemplar/never_sample_filter.h @@ -0,0 +1,43 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once +#ifndef ENABLE_METRICS_PREVIEW +# include "opentelemetry/sdk/metrics/exemplar/filter.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace metrics +{ + +class NeverSampleFilter final : public ExemplarFilter +{ +public: + static nostd::shared_ptr GetNeverSampleFilter() + { + nostd::shared_ptr neverSampleFilter{new NeverSampleFilter{}}; + return neverSampleFilter; + } + + bool ShouldSampleMeasurement(long value, + const MetricAttributes &attributes, + const opentelemetry::context::Context &context) noexcept override + { + return false; + } + + bool ShouldSampleMeasurement(double value, + const MetricAttributes &attributes, + const opentelemetry::context::Context &context) noexcept override + { + return false; + } + +private: + explicit NeverSampleFilter() = default; +}; +} // namespace metrics +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE +#endif diff --git a/sdk/include/opentelemetry/sdk/metrics/exemplar/no_exemplar_reservoir.h b/sdk/include/opentelemetry/sdk/metrics/exemplar/no_exemplar_reservoir.h new file mode 100644 index 0000000000..1fe0586d28 --- /dev/null +++ b/sdk/include/opentelemetry/sdk/metrics/exemplar/no_exemplar_reservoir.h @@ -0,0 +1,55 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once +#ifndef ENABLE_METRICS_PREVIEW +# include +# include "opentelemetry/context/context.h" +# include "opentelemetry/nostd/shared_ptr.h" +# include "opentelemetry/sdk/common/attribute_utils.h" +# include "opentelemetry/sdk/metrics/exemplar/reservoir.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace metrics +{ +class NoExemplarReservoir final : public ExemplarReservoir +{ + +public: + static nostd::shared_ptr GetNoExemplarReservoir() + { + return nostd::shared_ptr{new NoExemplarReservoir{}}; + } + + void OfferMeasurement(long value, + const MetricAttributes &attributes, + const opentelemetry::context::Context &context, + const opentelemetry::common::SystemTimestamp ×tamp) noexcept override + { + // Stores nothing + } + + void OfferMeasurement(double value, + const MetricAttributes &attributes, + const opentelemetry::context::Context &context, + const opentelemetry::common::SystemTimestamp ×tamp) noexcept override + { + // Stores nothing. + } + + std::vector CollectAndReset( + const MetricAttributes &pointAttributes) noexcept override + { + return std::vector{}; + } + +private: + explicit NoExemplarReservoir() = default; +}; + +} // namespace metrics +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE +#endif diff --git a/sdk/include/opentelemetry/sdk/metrics/exemplar/reservoir.h b/sdk/include/opentelemetry/sdk/metrics/exemplar/reservoir.h new file mode 100644 index 0000000000..25e8421d6b --- /dev/null +++ b/sdk/include/opentelemetry/sdk/metrics/exemplar/reservoir.h @@ -0,0 +1,55 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once +#ifndef ENABLE_METRICS_PREVIEW +# include +# include "opentelemetry/sdk/metrics/exemplar/data.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace metrics +{ +/** + * An interface for an exemplar reservoir of samples. + * + *

This represents a reservoir for a specific "point" of metric data. + */ +class ExemplarReservoir +{ +public: + virtual ~ExemplarReservoir() = default; + + /** Offers a long measurement to be sampled. */ + virtual void OfferMeasurement( + long value, + const MetricAttributes &attributes, + const opentelemetry::context::Context &context, + const opentelemetry::common::SystemTimestamp ×tamp) noexcept = 0; + + /** Offers a double measurement to be sampled. */ + virtual void OfferMeasurement( + double value, + const MetricAttributes &attributes, + const opentelemetry::context::Context &context, + const opentelemetry::common::SystemTimestamp ×tamp) noexcept = 0; + + /** + * Builds vector of Exemplars for exporting from the current reservoir. + * + *

Additionally, clears the reservoir for the next sampling period. + * + * @param pointAttributes the Attributes associated with the metric point. + * ExemplarDatas should filter these out of their final data state. + * @return A vector of sampled exemplars for this point. Implementers are expected to + * filter out pointAttributes from the original recorded attributes. + */ + virtual std::vector CollectAndReset( + const MetricAttributes &pointAttributes) noexcept = 0; +}; + +} // namespace metrics +} // namespace sdk +OPENTELEMETRY_END_NAMESPACE +#endif diff --git a/sdk/include/opentelemetry/sdk/metrics/instruments.h b/sdk/include/opentelemetry/sdk/metrics/instruments.h index ad64ce718b..15e6a25705 100644 --- a/sdk/include/opentelemetry/sdk/metrics/instruments.h +++ b/sdk/include/opentelemetry/sdk/metrics/instruments.h @@ -36,7 +36,7 @@ enum class AggregationType kDefault }; -enum class AggregationTemporarily +enum class AggregationTemporality { kUnspecified, kDelta, diff --git a/sdk/include/opentelemetry/sdk/metrics/measurement_processor.h b/sdk/include/opentelemetry/sdk/metrics/measurement_processor.h deleted file mode 100644 index ffd47c5c92..0000000000 --- a/sdk/include/opentelemetry/sdk/metrics/measurement_processor.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -#pragma once -#ifndef ENABLE_METRICS_PREVIEW - -# include -# include "opentelemetry/common/key_value_iterable_view.h" -# include "opentelemetry/sdk/metrics/instruments.h" -# include "opentelemetry/sdk/metrics/metric_reader.h" -# include "opentelemetry/sdk/metrics/state/sync_metric_storage.h" -# include "opentelemetry/sdk/metrics/view/attributes_processor.h" - -OPENTELEMETRY_BEGIN_NAMESPACE -namespace sdk -{ -namespace metrics -{ - -static std::size_t MakeKey(const MetricReader &metric_reader) -{ - return reinterpret_cast(&metric_reader); -} - -class MeasurementProcessor -{ -public: - virtual void RecordLong(long value) noexcept = 0; - - virtual void RecordLong(long value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept = 0; - - virtual void RecordDouble(double value) noexcept = 0; - - virtual void RecordDouble(double value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept = 0; - - virtual bool Collect(MetricReader &reader, - AggregationTemporarily aggregation_temporarily, - nostd::function_ref callback) noexcept = 0; -}; - -class DefaultMeasurementProcessor : public MeasurementProcessor -{ -public: - bool AddMetricStorage(const MetricReader &reader) - { - // TBD Check if already present - // pass intrumentation type, and aggregation type instead of hardcodig below. - InstrumentDescriptor instr_desc = {"name", "desc", "1unit", InstrumentType::kCounter, - InstrumentValueType::kLong}; - metric_storages_[MakeKey(reader)] = std::unique_ptr( - new SyncMetricStorage(instr_desc, AggregationType::kSum, new DefaultAttributesProcessor())); - return true; - } - - virtual void RecordLong(long value) noexcept override - { - for (const auto &kv : metric_storages_) - { - kv.second->RecordLong(value); - } - } - - virtual void RecordLong( - long value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override - { - for (const auto &kv : metric_storages_) - { - kv.second->RecordLong(value, attributes); - } - } - - virtual void RecordDouble(double value) noexcept override - { - for (const auto &kv : metric_storages_) - { - kv.second->RecordDouble(value); - } - } - - virtual void RecordDouble( - double value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override - { - for (const auto &kv : metric_storages_) - { - kv.second->RecordDouble(value, attributes); - } - } - - bool Collect(MetricReader &reader, - AggregationTemporarily aggregation_temporarily, - nostd::function_ref callback) noexcept override - { - auto i = metric_storages_.find(MakeKey(reader)); - - // TBD - Remove hardcodings below - std::vector collectors; - if (i != metric_storages_.end()) - { - - return i->second->Collect(nullptr, collectors, nullptr, nullptr, callback); - } - return false; - } - -private: - std::map> metric_storages_; -}; - -} // namespace metrics -} // namespace sdk -OPENTELEMETRY_END_NAMESPACE -#endif \ No newline at end of file diff --git a/sdk/include/opentelemetry/sdk/metrics/metric_exporter.h b/sdk/include/opentelemetry/sdk/metrics/metric_exporter.h index 96898c9ad1..3769259472 100644 --- a/sdk/include/opentelemetry/sdk/metrics/metric_exporter.h +++ b/sdk/include/opentelemetry/sdk/metrics/metric_exporter.h @@ -51,7 +51,7 @@ class MetricExporter std::chrono::microseconds timeout = std::chrono::microseconds(0)) noexcept = 0; private: - AggregationTemporarily aggregation_temporarily; + AggregationTemporality aggregation_temporality_; }; } // namespace metrics } // namespace sdk diff --git a/sdk/include/opentelemetry/sdk/metrics/metric_reader.h b/sdk/include/opentelemetry/sdk/metrics/metric_reader.h index fdd5d41291..1f7cee8d30 100644 --- a/sdk/include/opentelemetry/sdk/metrics/metric_reader.h +++ b/sdk/include/opentelemetry/sdk/metrics/metric_reader.h @@ -27,7 +27,7 @@ class MetricReader { public: MetricReader( - AggregationTemporarily aggregation_temporarily = AggregationTemporarily::kCummulative); + AggregationTemporality aggregation_temporality = AggregationTemporality::kCummulative); void SetMetricProducer(MetricProducer *metric_producer); @@ -37,7 +37,7 @@ class MetricReader */ bool Collect(nostd::function_ref callback) noexcept; - AggregationTemporarily GetAggregationTemporarily() const noexcept; + AggregationTemporality GetAggregationTemporality() const noexcept; /** * Shutdown the meter reader. @@ -62,7 +62,7 @@ class MetricReader private: MetricProducer *metric_producer_; - AggregationTemporarily aggregation_temporarily_; + AggregationTemporality aggregation_temporality_; mutable opentelemetry::common::SpinLockMutex lock_; bool shutdown_; }; diff --git a/sdk/include/opentelemetry/sdk/metrics/state/metric_collector.h b/sdk/include/opentelemetry/sdk/metrics/state/metric_collector.h index 3049440d9d..51c9cf6eff 100644 --- a/sdk/include/opentelemetry/sdk/metrics/state/metric_collector.h +++ b/sdk/include/opentelemetry/sdk/metrics/state/metric_collector.h @@ -18,7 +18,7 @@ class MeterContext; class CollectorHandle { public: - virtual AggregationTemporarily GetAggregationTemporarily() noexcept = 0; + virtual AggregationTemporality GetAggregationTemporality() noexcept = 0; }; /** @@ -33,7 +33,7 @@ class MetricCollector : public MetricProducer, public CollectorHandle MetricCollector(std::shared_ptr &&context, std::unique_ptr metric_reader); - AggregationTemporarily GetAggregationTemporarily() noexcept override; + AggregationTemporality GetAggregationTemporality() noexcept override; /** * The callback to be called for each metric exporter. This will only be those diff --git a/sdk/include/opentelemetry/sdk/metrics/state/metric_storage.h b/sdk/include/opentelemetry/sdk/metrics/state/metric_storage.h index 8ae50554f1..20f7d56140 100644 --- a/sdk/include/opentelemetry/sdk/metrics/state/metric_storage.h +++ b/sdk/include/opentelemetry/sdk/metrics/state/metric_storage.h @@ -5,6 +5,7 @@ #ifndef ENABLE_METRICS_PREVIEW # include "opentelemetry/common/key_value_iterable_view.h" # include "opentelemetry/common/timestamp.h" +# include "opentelemetry/context/context.h" # include "opentelemetry/sdk/metrics/data/metric_data.h" # include "opentelemetry/sdk/metrics/instruments.h" OPENTELEMETRY_BEGIN_NAMESPACE @@ -30,15 +31,19 @@ class MetricStorage class WritableMetricStorage { public: - virtual void RecordLong(long value) noexcept = 0; + virtual void RecordLong(long value, const opentelemetry::context::Context &context) noexcept = 0; virtual void RecordLong(long value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept = 0; + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept = 0; - virtual void RecordDouble(double value) noexcept = 0; + virtual void RecordDouble(double value, + const opentelemetry::context::Context &context) noexcept = 0; virtual void RecordDouble(double value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept = 0; + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept = 0; + virtual ~WritableMetricStorage() = default; }; @@ -63,16 +68,19 @@ class NoopMetricStorage : public MetricStorage class NoopWritableMetricStorage : public WritableMetricStorage { public: - void RecordLong(long value) noexcept = 0; + void RecordLong(long value, const opentelemetry::context::Context &context) noexcept = 0; void RecordLong(long value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override {} - void RecordDouble(double value) noexcept override {} + void RecordDouble(double value, const opentelemetry::context::Context &context) noexcept override + {} void RecordDouble(double value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override {} }; diff --git a/sdk/include/opentelemetry/sdk/metrics/state/multi_metric_storage.h b/sdk/include/opentelemetry/sdk/metrics/state/multi_metric_storage.h index d9a66f732a..ceeafa0406 100644 --- a/sdk/include/opentelemetry/sdk/metrics/state/multi_metric_storage.h +++ b/sdk/include/opentelemetry/sdk/metrics/state/multi_metric_storage.h @@ -20,39 +20,41 @@ class MultiMetricStorage : public WritableMetricStorage public: void AddStorage(std::shared_ptr storage) { storages_.push_back(storage); } - virtual void RecordLong(long value) noexcept override + virtual void RecordLong(long value, + const opentelemetry::context::Context &context) noexcept override { for (auto &s : storages_) { - s->RecordLong(value); + s->RecordLong(value, context); } } - virtual void RecordLong( - long value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override + virtual void RecordLong(long value, + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override { for (auto &s : storages_) { - s->RecordLong(value, attributes); + s->RecordLong(value, attributes, context); } } - virtual void RecordDouble(double value) noexcept override + virtual void RecordDouble(double value, + const opentelemetry::context::Context &context) noexcept override { for (auto &s : storages_) { - s->RecordDouble(value); + s->RecordDouble(value, context); } } - virtual void RecordDouble( - double value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override + virtual void RecordDouble(double value, + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override { for (auto &s : storages_) { - s->RecordDouble(value, attributes); + s->RecordDouble(value, attributes, context); } } diff --git a/sdk/include/opentelemetry/sdk/metrics/state/sync_metric_storage.h b/sdk/include/opentelemetry/sdk/metrics/state/sync_metric_storage.h index f8c2a86aa4..bfe50b152d 100644 --- a/sdk/include/opentelemetry/sdk/metrics/state/sync_metric_storage.h +++ b/sdk/include/opentelemetry/sdk/metrics/state/sync_metric_storage.h @@ -7,6 +7,7 @@ # include "opentelemetry/sdk/common/attributemap_hash.h" # include "opentelemetry/sdk/instrumentationlibrary/instrumentation_library.h" # include "opentelemetry/sdk/metrics/aggregation/default_aggregation.h" +# include "opentelemetry/sdk/metrics/exemplar/reservoir.h" # include "opentelemetry/sdk/metrics/state/attributes_hashmap.h" # include "opentelemetry/sdk/metrics/state/metric_storage.h" # include "opentelemetry/sdk/metrics/view/attributes_processor.h" @@ -27,11 +28,13 @@ class SyncMetricStorage : public MetricStorage, public WritableMetricStorage public: SyncMetricStorage(InstrumentDescriptor instrument_descriptor, const AggregationType aggregation_type, - const AttributesProcessor *attributes_processor) + const AttributesProcessor *attributes_processor, + nostd::shared_ptr &&exemplar_reservoir) : instrument_descriptor_(instrument_descriptor), aggregation_type_{aggregation_type}, attributes_hashmap_(new AttributesHashMap()), - attributes_processor_{attributes_processor} + attributes_processor_{attributes_processor}, + exemplar_reservoir_(exemplar_reservoir) { create_default_aggregation_ = [&]() -> std::unique_ptr { return std::move( @@ -39,45 +42,55 @@ class SyncMetricStorage : public MetricStorage, public WritableMetricStorage }; } - void RecordLong(long value) noexcept override + void RecordLong(long value, const opentelemetry::context::Context &context) noexcept override { if (instrument_descriptor_.value_type_ != InstrumentValueType::kLong) { return; } + exemplar_reservoir_->OfferMeasurement(value, {}, context, std::chrono::system_clock::now()); attributes_hashmap_->GetOrSetDefault({}, create_default_aggregation_)->Aggregate(value); } void RecordLong(long value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override { if (instrument_descriptor_.value_type_ != InstrumentValueType::kLong) { return; } + exemplar_reservoir_->OfferMeasurement(value, attributes, context, + std::chrono::system_clock::now()); auto attr = attributes_processor_->process(attributes); attributes_hashmap_->GetOrSetDefault(attr, create_default_aggregation_)->Aggregate(value); } - void RecordDouble(double value) noexcept override + void RecordDouble(double value, const opentelemetry::context::Context &context) noexcept override { if (instrument_descriptor_.value_type_ != InstrumentValueType::kDouble) { return; } + exemplar_reservoir_->OfferMeasurement(value, {}, context, std::chrono::system_clock::now()); attributes_hashmap_->GetOrSetDefault({}, create_default_aggregation_)->Aggregate(value); } void RecordDouble(double value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override { + exemplar_reservoir_->OfferMeasurement(value, attributes, context, + std::chrono::system_clock::now()); if (instrument_descriptor_.value_type_ != InstrumentValueType::kDouble) { return; } + exemplar_reservoir_->OfferMeasurement(value, attributes, context, + std::chrono::system_clock::now()); auto attr = attributes_processor_->process(attributes); attributes_hashmap_->GetOrSetDefault(attr, create_default_aggregation_)->Aggregate(value); } @@ -102,6 +115,7 @@ class SyncMetricStorage : public MetricStorage, public WritableMetricStorage std::unique_ptr attributes_hashmap_; const AttributesProcessor *attributes_processor_; std::function()> create_default_aggregation_; + nostd::shared_ptr exemplar_reservoir_; }; } // namespace metrics diff --git a/sdk/include/opentelemetry/sdk/metrics/sync_instruments.h b/sdk/include/opentelemetry/sdk/metrics/sync_instruments.h index 3e8c5c8623..f4c9bae323 100644 --- a/sdk/include/opentelemetry/sdk/metrics/sync_instruments.h +++ b/sdk/include/opentelemetry/sdk/metrics/sync_instruments.h @@ -39,8 +39,12 @@ class LongCounter : public Synchronous, public opentelemetry::metrics::Counter storage); void Add(long value, const opentelemetry::common::KeyValueIterable &attributes) noexcept override; + void Add(long value, + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override; void Add(long value) noexcept override; + void Add(long value, const opentelemetry::context::Context &context) noexcept override; }; class DoubleCounter : public Synchronous, public opentelemetry::metrics::Counter @@ -52,8 +56,12 @@ class DoubleCounter : public Synchronous, public opentelemetry::metrics::Counter void Add(double value, const opentelemetry::common::KeyValueIterable &attributes) noexcept override; + void Add(double value, + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override; void Add(double value) noexcept override; + void Add(double value, const opentelemetry::context::Context &context) noexcept override; }; class LongUpDownCounter : public Synchronous, public opentelemetry::metrics::UpDownCounter @@ -63,8 +71,12 @@ class LongUpDownCounter : public Synchronous, public opentelemetry::metrics::UpD std::unique_ptr storage); void Add(long value, const opentelemetry::common::KeyValueIterable &attributes) noexcept override; + void Add(long value, + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override; void Add(long value) noexcept override; + void Add(long value, const opentelemetry::context::Context &context) noexcept override; }; class DoubleUpDownCounter : public Synchronous, public opentelemetry::metrics::UpDownCounter @@ -75,8 +87,12 @@ class DoubleUpDownCounter : public Synchronous, public opentelemetry::metrics::U void Add(double value, const opentelemetry::common::KeyValueIterable &attributes) noexcept override; + void Add(double value, + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override; void Add(double value) noexcept override; + void Add(double value, const opentelemetry::context::Context &context) noexcept override; }; class LongHistogram : public Synchronous, public opentelemetry::metrics::Histogram @@ -86,9 +102,10 @@ class LongHistogram : public Synchronous, public opentelemetry::metrics::Histogr std::unique_ptr storage); void Record(long value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override; + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override; - void Record(long value) noexcept override; + void Record(long value, const opentelemetry::context::Context &context) noexcept override; }; class DoubleHistogram : public Synchronous, public opentelemetry::metrics::Histogram @@ -98,9 +115,10 @@ class DoubleHistogram : public Synchronous, public opentelemetry::metrics::Histo std::unique_ptr storage); void Record(double value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override; + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override; - void Record(double value) noexcept override; + void Record(double value, const opentelemetry::context::Context &context) noexcept override; }; } // namespace metrics diff --git a/sdk/include/opentelemetry/sdk/metrics/view/attributes_processor.h b/sdk/include/opentelemetry/sdk/metrics/view/attributes_processor.h index d82607357f..fdc4e35c53 100644 --- a/sdk/include/opentelemetry/sdk/metrics/view/attributes_processor.h +++ b/sdk/include/opentelemetry/sdk/metrics/view/attributes_processor.h @@ -23,8 +23,6 @@ class AttributesProcessor // @returns The processed attributes virtual MetricAttributes process( const opentelemetry::common::KeyValueIterable &attributes) const noexcept = 0; - - virtual ~AttributesProcessor() = default; }; /** diff --git a/sdk/include/opentelemetry/sdk/trace/batch_span_processor.h b/sdk/include/opentelemetry/sdk/trace/batch_span_processor.h index ba923dd7e7..4233653d93 100644 --- a/sdk/include/opentelemetry/sdk/trace/batch_span_processor.h +++ b/sdk/include/opentelemetry/sdk/trace/batch_span_processor.h @@ -122,12 +122,8 @@ class BatchSpanProcessor : public SpanProcessor /** * Exports all ended spans to the configured exporter. * - * @param was_force_flush_called - A flag to check if the current export is the result - * of a call to ForceFlush method. If true, then we have to - * notify the main thread to wake it up in the ForceFlush - * method. */ - void Export(const bool was_for_flush_called); + void Export(); /** * Called when Shutdown() is invoked. Completely drains the queue of all its ended spans and @@ -135,18 +131,32 @@ class BatchSpanProcessor : public SpanProcessor */ void DrainQueue(); - /** - * Should be called from Export to notify the main thread on Force Flush Completion - * @param was_force_flush_called - A flag to check if the current export is the result - * of a call to ForceFlush method. If true, then we have to - * notify the main thread to wake it up in the ForceFlush - * method. - */ - void NotifyForceFlushCompletion(const bool was_for_flush_called); - /* In case of async export, wait and notify for shutdown to be completed.*/ void WaitForShutdownCompletion(); - void NotifyShutdownCompletion(); + + struct SynchronizationData + { + /* Synchronization primitives */ + std::condition_variable cv, force_flush_cv, async_shutdown_cv; + std::mutex cv_m, force_flush_cv_m, shutdown_m, async_shutdown_m; + + /* Important boolean flags to handle the workflow of the processor */ + std::atomic is_force_wakeup_background_worker; + std::atomic is_force_flush_pending; + std::atomic is_force_flush_notified; + std::atomic is_shutdown; + std::atomic is_async_shutdown_notified; + }; + + /** + * @brief Notify completion of shutdown and force flush. This may be called from the any thread at + * any time + * + * @param notify_force_flush Flag to indicate whether to notify force flush completion. + * @param synchronization_data Synchronization data to be notified. + */ + static void NotifyCompletion(bool notify_force_flush, + const std::shared_ptr &synchronization_data); /* The configured backend exporter */ std::unique_ptr exporter_; @@ -157,18 +167,10 @@ class BatchSpanProcessor : public SpanProcessor const size_t max_export_batch_size_; const bool is_export_async_; - /* Synchronization primitives */ - std::condition_variable cv_, force_flush_cv_, async_shutdown_cv_; - std::mutex cv_m_, force_flush_cv_m_, shutdown_m_, async_shutdown_m_; - /* The buffer/queue to which the ended spans are added */ common::CircularBuffer buffer_; - /* Important boolean flags to handle the workflow of the processor */ - std::atomic is_shutdown_; - std::atomic is_force_flush_; - std::atomic is_force_flush_notified_; - std::atomic is_async_shutdown_notified_; + std::shared_ptr synchronization_data_; /* The background worker thread */ std::thread worker_thread_; diff --git a/sdk/src/logs/batch_log_processor.cc b/sdk/src/logs/batch_log_processor.cc index b8db44f861..6e6c7319b0 100644 --- a/sdk/src/logs/batch_log_processor.cc +++ b/sdk/src/logs/batch_log_processor.cc @@ -3,6 +3,7 @@ #ifdef ENABLE_LOGS_PREVIEW # include "opentelemetry/sdk/logs/batch_log_processor.h" +# include "opentelemetry/common/spin_lock_mutex.h" # include using opentelemetry::sdk::common::AtomicUniquePtr; @@ -22,14 +23,34 @@ BatchLogProcessor::BatchLogProcessor(std::unique_ptr &&exporter, max_queue_size_(max_queue_size), scheduled_delay_millis_(scheduled_delay_millis), max_export_batch_size_(max_export_batch_size), - buffer_(max_queue_size_), is_export_async_(is_export_async), + buffer_(max_queue_size_), + synchronization_data_(std::make_shared()), + worker_thread_(&BatchLogProcessor::DoBackgroundWork, this) +{ + synchronization_data_->is_force_wakeup_background_worker.store(false); + synchronization_data_->is_force_flush_pending.store(false); + synchronization_data_->is_force_flush_notified.store(false); + synchronization_data_->is_shutdown.store(false); + synchronization_data_->is_async_shutdown_notified.store(false); +} + +BatchLogProcessor::BatchLogProcessor(std::unique_ptr &&exporter, + const BatchLogProcessorOptions &options) + : exporter_(std::move(exporter)), + max_queue_size_(options.max_queue_size), + scheduled_delay_millis_(options.schedule_delay_millis), + max_export_batch_size_(options.max_export_batch_size), + is_export_async_(options.is_export_async), + buffer_(options.max_queue_size), + synchronization_data_(std::make_shared()), worker_thread_(&BatchLogProcessor::DoBackgroundWork, this) { - is_shutdown_.store(false); - is_force_flush_.store(false); - is_force_flush_notified_.store(false); - is_async_shutdown_notified_.store(false); + synchronization_data_->is_force_wakeup_background_worker.store(false); + synchronization_data_->is_force_flush_pending.store(false); + synchronization_data_->is_force_flush_notified.store(false); + synchronization_data_->is_shutdown.store(false); + synchronization_data_->is_async_shutdown_notified.store(false); } std::unique_ptr BatchLogProcessor::MakeRecordable() noexcept @@ -39,7 +60,7 @@ std::unique_ptr BatchLogProcessor::MakeRecordable() noexcept void BatchLogProcessor::OnReceive(std::unique_ptr &&record) noexcept { - if (is_shutdown_.load() == true) + if (synchronization_data_->is_shutdown.load() == true) { return; } @@ -55,48 +76,78 @@ void BatchLogProcessor::OnReceive(std::unique_ptr &&record) noexcept if (buffer_size >= max_queue_size_ / 2 || buffer_size >= max_export_batch_size_) { // signal the worker thread - cv_.notify_one(); + synchronization_data_->is_force_wakeup_background_worker.store(true, std::memory_order_release); + synchronization_data_->cv.notify_one(); } } bool BatchLogProcessor::ForceFlush(std::chrono::microseconds timeout) noexcept { - if (is_shutdown_.load() == true) + if (synchronization_data_->is_shutdown.load() == true) { return false; } // Now wait for the worker thread to signal back from the Export method - std::unique_lock lk(force_flush_cv_m_); + std::unique_lock lk_cv(synchronization_data_->force_flush_cv_m); - is_force_flush_notified_.store(false, std::memory_order_release); + synchronization_data_->is_force_flush_pending.store(true, std::memory_order_release); auto break_condition = [this]() { - if (is_shutdown_.load() == true) + if (synchronization_data_->is_shutdown.load() == true) { return true; } // Wake up the worker thread once. - if (is_force_flush_.exchange(true) == false) + if (synchronization_data_->is_force_flush_pending.load(std::memory_order_acquire)) { - cv_.notify_one(); + synchronization_data_->cv.notify_one(); } - return is_force_flush_notified_.load(std::memory_order_acquire); + return synchronization_data_->is_force_flush_notified.load(std::memory_order_acquire); }; // Fix timeout to meet requirement of wait_for timeout = opentelemetry::common::DurationUtil::AdjustWaitForTimeout( timeout, std::chrono::microseconds::zero()); + bool result; if (timeout <= std::chrono::microseconds::zero()) { - force_flush_cv_.wait(lk, break_condition); - return true; + bool wait_result = false; + while (!wait_result) + { + // When is_force_flush_notified.store(true) and force_flush_cv.notify_all() is called + // between is_force_flush_pending.load() and force_flush_cv.wait(). We must not wait + // for ever + wait_result = synchronization_data_->force_flush_cv.wait_for(lk_cv, scheduled_delay_millis_, + break_condition); + } + result = true; } else { - return force_flush_cv_.wait_for(lk, timeout, break_condition); + result = synchronization_data_->force_flush_cv.wait_for(lk_cv, timeout, break_condition); } + + // If it's already signaled, we must wait util notified. + // We use a spin lock here + if (false == + synchronization_data_->is_force_flush_pending.exchange(false, std::memory_order_acq_rel)) + { + for (int retry_waiting_times = 0; + false == synchronization_data_->is_force_flush_notified.load(std::memory_order_acquire); + ++retry_waiting_times) + { + opentelemetry::common::SpinLockMutex::fast_yield(); + if ((retry_waiting_times & 127) == 127) + { + std::this_thread::yield(); + } + } + } + synchronization_data_->is_force_flush_notified.store(false, std::memory_order_release); + + return result; } void BatchLogProcessor::DoBackgroundWork() @@ -106,34 +157,26 @@ void BatchLogProcessor::DoBackgroundWork() while (true) { // Wait for `timeout` milliseconds - std::unique_lock lk(cv_m_); - cv_.wait_for(lk, timeout); + std::unique_lock lk(synchronization_data_->cv_m); + synchronization_data_->cv.wait_for(lk, timeout, [this] { + if (synchronization_data_->is_force_wakeup_background_worker.load(std::memory_order_acquire)) + { + return true; + } + + return !buffer_.empty(); + }); + synchronization_data_->is_force_wakeup_background_worker.store(false, + std::memory_order_release); - if (is_shutdown_.load() == true) + if (synchronization_data_->is_shutdown.load() == true) { - // Break loop if another thread call ForceFlush - is_force_flush_ = false; DrainQueue(); return; } - bool was_force_flush_called = is_force_flush_.exchange(false); - - // Check if this export was the result of a force flush. - if (!was_force_flush_called) - { - // If the buffer was empty during the entire `timeout` time interval, - // go back to waiting. If this was a spurious wake-up, we export only if - // `buffer_` is not empty. This is acceptable because batching is a best - // mechanism effort here. - if (buffer_.empty() == true) - { - continue; - } - } - auto start = std::chrono::steady_clock::now(); - Export(was_force_flush_called); + Export(); auto end = std::chrono::steady_clock::now(); auto duration = std::chrono::duration_cast(end - start); @@ -142,14 +185,17 @@ void BatchLogProcessor::DoBackgroundWork() } } -void BatchLogProcessor::Export(const bool was_force_flush_called) +void BatchLogProcessor::Export() { + uint64_t current_pending; + uint64_t current_notified; do { std::vector> records_arr; size_t num_records_to_export; - - if (was_force_flush_called == true) + bool notify_force_flush = + synchronization_data_->is_force_flush_pending.exchange(false, std::memory_order_acq_rel); + if (notify_force_flush) { num_records_to_export = buffer_.size(); } @@ -161,6 +207,7 @@ void BatchLogProcessor::Export(const bool was_force_flush_called) if (num_records_to_export == 0) { + NotifyCompletion(notify_force_flush, synchronization_data_); break; } @@ -178,38 +225,25 @@ void BatchLogProcessor::Export(const bool was_force_flush_called) { exporter_->Export( nostd::span>(records_arr.data(), records_arr.size())); + NotifyCompletion(notify_force_flush, synchronization_data_); } else { + std::weak_ptr synchronization_data_watcher = synchronization_data_; exporter_->Export( nostd::span>(records_arr.data(), records_arr.size()), - [this, was_force_flush_called](sdk::common::ExportResult result) { + [notify_force_flush, synchronization_data_watcher](sdk::common::ExportResult result) { // TODO: Print result - NotifyForceFlushCompletion(was_force_flush_called); + if (synchronization_data_watcher.expired()) + { + return true; + } - // Notify the thread which is waiting on shutdown to complete. - NotifyShutdownCompletion(); + NotifyCompletion(notify_force_flush, synchronization_data_watcher.lock()); return true; }); } - } while (was_force_flush_called); - - if (is_export_async_ == false) - { - NotifyForceFlushCompletion(was_force_flush_called); - } -} - -void BatchLogProcessor::NotifyForceFlushCompletion(const bool was_force_flush_called) -{ - // Notify the main thread in case this export was the result of a force flush. - if (was_force_flush_called == true) - { - if (is_force_flush_notified_.exchange(true, std::memory_order_acq_rel) == false) - { - force_flush_cv_.notify_all(); - } - } + } while (true); } void BatchLogProcessor::WaitForShutdownCompletion() @@ -218,29 +252,56 @@ void BatchLogProcessor::WaitForShutdownCompletion() // for async thread to complete. if (is_export_async_) { - std::unique_lock lk(async_shutdown_m_); - while (is_async_shutdown_notified_.load() == false) + std::unique_lock lk(synchronization_data_->async_shutdown_m); + while (true) { - async_shutdown_cv_.wait(lk); + if (synchronization_data_->is_async_shutdown_notified.load()) + { + break; + } + + // When is_async_shutdown_notified.store(true) and async_shutdown_cv.notify_all() is called + // between is_async_shutdown_notified.load() and async_shutdown_cv.wait(). We must not wait + // for ever + synchronization_data_->async_shutdown_cv.wait_for(lk, scheduled_delay_millis_); } } } -void BatchLogProcessor::NotifyShutdownCompletion() +void BatchLogProcessor::NotifyCompletion( + bool notify_force_flush, + const std::shared_ptr &synchronization_data) { + if (!synchronization_data) + { + return; + } + + if (notify_force_flush) + { + synchronization_data->is_force_flush_notified.store(true, std::memory_order_release); + synchronization_data->force_flush_cv.notify_one(); + } + // Notify the thread which is waiting on shutdown to complete. - if (is_shutdown_.load() == true) + if (synchronization_data->is_shutdown.load() == true) { - is_async_shutdown_notified_.store(true); - async_shutdown_cv_.notify_one(); + synchronization_data->is_async_shutdown_notified.store(true); + synchronization_data->async_shutdown_cv.notify_all(); } } void BatchLogProcessor::DrainQueue() { - while (buffer_.empty() == false) + while (true) { - Export(false); + if (buffer_.empty() && + false == synchronization_data_->is_force_flush_pending.load(std::memory_order_acquire)) + { + break; + } + + Export(); // Since async export is invoked due to shutdown, need to wait // for async thread to complete. @@ -252,12 +313,13 @@ bool BatchLogProcessor::Shutdown(std::chrono::microseconds timeout) noexcept { auto start_time = std::chrono::system_clock::now(); - std::lock_guard shutdown_guard{shutdown_m_}; - bool already_shutdown = is_shutdown_.exchange(true); + std::lock_guard shutdown_guard{synchronization_data_->shutdown_m}; + bool already_shutdown = synchronization_data_->is_shutdown.exchange(true); if (worker_thread_.joinable()) { - cv_.notify_one(); + synchronization_data_->is_force_wakeup_background_worker.store(true, std::memory_order_release); + synchronization_data_->cv.notify_one(); worker_thread_.join(); } @@ -287,7 +349,7 @@ bool BatchLogProcessor::Shutdown(std::chrono::microseconds timeout) noexcept BatchLogProcessor::~BatchLogProcessor() { - if (is_shutdown_.load() == false) + if (synchronization_data_->is_shutdown.load() == false) { Shutdown(); } diff --git a/sdk/src/metrics/meter.cc b/sdk/src/metrics/meter.cc index 360630e786..20e3b02768 100644 --- a/sdk/src/metrics/meter.cc +++ b/sdk/src/metrics/meter.cc @@ -6,6 +6,7 @@ # include "opentelemetry/metrics/noop.h" # include "opentelemetry/nostd/shared_ptr.h" # include "opentelemetry/sdk/metrics/async_instruments.h" +# include "opentelemetry/sdk/metrics/exemplar/no_exemplar_reservoir.h" # include "opentelemetry/sdk/metrics/state/multi_metric_storage.h" # include "opentelemetry/sdk/metrics/state/sync_metric_storage.h" # include "opentelemetry/sdk/metrics/sync_instruments.h" @@ -192,7 +193,8 @@ std::unique_ptr Meter::RegisterMetricStorage( view_instr_desc.name_ = view.GetName(); view_instr_desc.description_ = view.GetDescription(); auto storage = std::shared_ptr(new SyncMetricStorage( - view_instr_desc, view.GetAggregationType(), &view.GetAttributesProcessor())); + view_instr_desc, view.GetAggregationType(), &view.GetAttributesProcessor(), + NoExemplarReservoir::GetNoExemplarReservoir())); storage_registry_[instrument_descriptor.name_] = storage; auto multi_storage = static_cast(storages.get()); multi_storage->AddStorage(storage); diff --git a/sdk/src/metrics/metric_reader.cc b/sdk/src/metrics/metric_reader.cc index 71aedc227f..8238ad2a55 100644 --- a/sdk/src/metrics/metric_reader.cc +++ b/sdk/src/metrics/metric_reader.cc @@ -13,8 +13,8 @@ namespace sdk namespace metrics { -MetricReader::MetricReader(AggregationTemporarily aggregation_temporarily) - : aggregation_temporarily_(aggregation_temporarily) +MetricReader::MetricReader(AggregationTemporality aggregation_temporality) + : aggregation_temporality_(aggregation_temporality) {} void MetricReader::SetMetricProducer(MetricProducer *metric_producer) @@ -22,9 +22,9 @@ void MetricReader::SetMetricProducer(MetricProducer *metric_producer) metric_producer_ = metric_producer; } -AggregationTemporarily MetricReader::GetAggregationTemporarily() const noexcept +AggregationTemporality MetricReader::GetAggregationTemporality() const noexcept { - return aggregation_temporarily_; + return aggregation_temporality_; } bool MetricReader::Collect(nostd::function_ref callback) noexcept diff --git a/sdk/src/metrics/state/metric_collector.cc b/sdk/src/metrics/state/metric_collector.cc index a52c6e9b1c..5b9fc4ab70 100644 --- a/sdk/src/metrics/state/metric_collector.cc +++ b/sdk/src/metrics/state/metric_collector.cc @@ -24,9 +24,9 @@ MetricCollector::MetricCollector( metric_reader_->SetMetricProducer(this); } -AggregationTemporarily MetricCollector::GetAggregationTemporarily() noexcept +AggregationTemporality MetricCollector::GetAggregationTemporality() noexcept { - return metric_reader_->GetAggregationTemporarily(); + return metric_reader_->GetAggregationTemporality(); } bool MetricCollector::Collect(nostd::function_ref callback) noexcept diff --git a/sdk/src/metrics/sync_instruments.cc b/sdk/src/metrics/sync_instruments.cc index 863e5b6323..5c94ae6f24 100644 --- a/sdk/src/metrics/sync_instruments.cc +++ b/sdk/src/metrics/sync_instruments.cc @@ -18,12 +18,26 @@ LongCounter::LongCounter(InstrumentDescriptor instrument_descriptor, void LongCounter::Add(long value, const opentelemetry::common::KeyValueIterable &attributes) noexcept { - return storage_->RecordLong(value, attributes); + auto context = opentelemetry::context::Context{}; + return storage_->RecordLong(value, attributes, context); +} + +void LongCounter::Add(long value, + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept +{ + return storage_->RecordLong(value, attributes, context); } void LongCounter::Add(long value) noexcept { - return storage_->RecordLong(value); + auto context = opentelemetry::context::Context{}; + return storage_->RecordLong(value, context); +} + +void LongCounter::Add(long value, const opentelemetry::context::Context &context) noexcept +{ + return storage_->RecordLong(value, context); } DoubleCounter::DoubleCounter(InstrumentDescriptor instrument_descriptor, @@ -34,12 +48,26 @@ DoubleCounter::DoubleCounter(InstrumentDescriptor instrument_descriptor, void DoubleCounter::Add(double value, const opentelemetry::common::KeyValueIterable &attributes) noexcept { - return storage_->RecordDouble(value, attributes); + auto context = opentelemetry::context::Context{}; + return storage_->RecordDouble(value, attributes, context); +} + +void DoubleCounter::Add(double value, + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept +{ + return storage_->RecordDouble(value, attributes, context); } void DoubleCounter::Add(double value) noexcept { - return storage_->RecordDouble(value); + auto context = opentelemetry::context::Context{}; + return storage_->RecordDouble(value, context); +} + +void DoubleCounter::Add(double value, const opentelemetry::context::Context &context) noexcept +{ + return storage_->RecordDouble(value, context); } LongUpDownCounter::LongUpDownCounter(InstrumentDescriptor instrument_descriptor, @@ -50,12 +78,26 @@ LongUpDownCounter::LongUpDownCounter(InstrumentDescriptor instrument_descriptor, void LongUpDownCounter::Add(long value, const opentelemetry::common::KeyValueIterable &attributes) noexcept { - return storage_->RecordLong(value, attributes); + auto context = opentelemetry::context::Context{}; + return storage_->RecordLong(value, attributes, context); +} + +void LongUpDownCounter::Add(long value, + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept +{ + return storage_->RecordLong(value, attributes, context); } void LongUpDownCounter::Add(long value) noexcept { - return storage_->RecordLong(value); + auto context = opentelemetry::context::Context{}; + return storage_->RecordLong(value, context); +} + +void LongUpDownCounter::Add(long value, const opentelemetry::context::Context &context) noexcept +{ + return storage_->RecordLong(value, context); } DoubleUpDownCounter::DoubleUpDownCounter(InstrumentDescriptor instrument_descriptor, @@ -66,12 +108,26 @@ DoubleUpDownCounter::DoubleUpDownCounter(InstrumentDescriptor instrument_descrip void DoubleUpDownCounter::Add(double value, const opentelemetry::common::KeyValueIterable &attributes) noexcept { - return storage_->RecordDouble(value, attributes); + auto context = opentelemetry::context::Context{}; + return storage_->RecordDouble(value, attributes, context); +} + +void DoubleUpDownCounter::Add(double value, + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept +{ + return storage_->RecordDouble(value, attributes, context); } void DoubleUpDownCounter::Add(double value) noexcept { - return storage_->RecordDouble(value); + auto context = opentelemetry::context::Context{}; + return storage_->RecordDouble(value, context); +} + +void DoubleUpDownCounter::Add(double value, const opentelemetry::context::Context &context) noexcept +{ + return storage_->RecordDouble(value, context); } LongHistogram::LongHistogram(InstrumentDescriptor instrument_descriptor, @@ -80,14 +136,15 @@ LongHistogram::LongHistogram(InstrumentDescriptor instrument_descriptor, {} void LongHistogram::Record(long value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept { - return storage_->RecordLong(value, attributes); + return storage_->RecordLong(value, attributes, context); } -void LongHistogram::Record(long value) noexcept +void LongHistogram::Record(long value, const opentelemetry::context::Context &context) noexcept { - return storage_->RecordLong(value); + return storage_->RecordLong(value, context); } DoubleHistogram::DoubleHistogram(InstrumentDescriptor instrument_descriptor, @@ -96,14 +153,15 @@ DoubleHistogram::DoubleHistogram(InstrumentDescriptor instrument_descriptor, {} void DoubleHistogram::Record(double value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept { - return storage_->RecordDouble(value, attributes); + return storage_->RecordDouble(value, attributes, context); } -void DoubleHistogram::Record(double value) noexcept +void DoubleHistogram::Record(double value, const opentelemetry::context::Context &context) noexcept { - return storage_->RecordDouble(value); + return storage_->RecordDouble(value, context); } } // namespace metrics diff --git a/sdk/src/trace/batch_span_processor.cc b/sdk/src/trace/batch_span_processor.cc index a02e3fcaee..8aaa6d0095 100644 --- a/sdk/src/trace/batch_span_processor.cc +++ b/sdk/src/trace/batch_span_processor.cc @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "opentelemetry/sdk/trace/batch_span_processor.h" +#include "opentelemetry/common/spin_lock_mutex.h" #include using opentelemetry::sdk::common::AtomicUniquePtr; @@ -14,20 +15,23 @@ namespace sdk { namespace trace { + BatchSpanProcessor::BatchSpanProcessor(std::unique_ptr &&exporter, const BatchSpanProcessorOptions &options) : exporter_(std::move(exporter)), max_queue_size_(options.max_queue_size), schedule_delay_millis_(options.schedule_delay_millis), max_export_batch_size_(options.max_export_batch_size), - buffer_(max_queue_size_), is_export_async_(options.is_export_async), + buffer_(max_queue_size_), + synchronization_data_(std::make_shared()), worker_thread_(&BatchSpanProcessor::DoBackgroundWork, this) { - is_shutdown_.store(false); - is_force_flush_.store(false); - is_force_flush_notified_.store(false); - is_async_shutdown_notified_.store(false); + synchronization_data_->is_force_wakeup_background_worker.store(false); + synchronization_data_->is_force_flush_pending.store(false); + synchronization_data_->is_force_flush_notified.store(false); + synchronization_data_->is_shutdown.store(false); + synchronization_data_->is_async_shutdown_notified.store(false); } std::unique_ptr BatchSpanProcessor::MakeRecordable() noexcept @@ -42,7 +46,7 @@ void BatchSpanProcessor::OnStart(Recordable &, const SpanContext &) noexcept void BatchSpanProcessor::OnEnd(std::unique_ptr &&span) noexcept { - if (is_shutdown_.load() == true) + if (synchronization_data_->is_shutdown.load() == true) { return; } @@ -58,48 +62,79 @@ void BatchSpanProcessor::OnEnd(std::unique_ptr &&span) noexcept if (buffer_size >= max_queue_size_ / 2 || buffer_size >= max_export_batch_size_) { // signal the worker thread - cv_.notify_one(); + synchronization_data_->cv.notify_one(); } } bool BatchSpanProcessor::ForceFlush(std::chrono::microseconds timeout) noexcept { - if (is_shutdown_.load() == true) + if (synchronization_data_->is_shutdown.load() == true) { return false; } // Now wait for the worker thread to signal back from the Export method - std::unique_lock lk(force_flush_cv_m_); + std::unique_lock lk_cv(synchronization_data_->force_flush_cv_m); - is_force_flush_notified_.store(false, std::memory_order_release); + synchronization_data_->is_force_flush_pending.store(true, std::memory_order_release); auto break_condition = [this]() { - if (is_shutdown_.load() == true) + if (synchronization_data_->is_shutdown.load() == true) { return true; } // Wake up the worker thread once. - if (is_force_flush_.exchange(true) == false) + if (synchronization_data_->is_force_flush_pending.load(std::memory_order_acquire)) { - cv_.notify_one(); + synchronization_data_->is_force_wakeup_background_worker.store(true, + std::memory_order_release); + synchronization_data_->cv.notify_one(); } - return is_force_flush_notified_.load(std::memory_order_acquire); + return synchronization_data_->is_force_flush_notified.load(std::memory_order_acquire); }; // Fix timeout to meet requirement of wait_for timeout = opentelemetry::common::DurationUtil::AdjustWaitForTimeout( timeout, std::chrono::microseconds::zero()); + bool result; if (timeout <= std::chrono::microseconds::zero()) { - force_flush_cv_.wait(lk, break_condition); - return true; + bool wait_result = false; + while (!wait_result) + { + // When is_force_flush_notified.store(true) and force_flush_cv.notify_all() is called + // between is_force_flush_pending.load() and force_flush_cv.wait(). We must not wait + // for ever + wait_result = synchronization_data_->force_flush_cv.wait_for(lk_cv, schedule_delay_millis_, + break_condition); + } + result = true; } else { - return force_flush_cv_.wait_for(lk, timeout, break_condition); + result = synchronization_data_->force_flush_cv.wait_for(lk_cv, timeout, break_condition); } + + // If it will be already signaled, we must wait util notified. + // We use a spin lock here + if (false == + synchronization_data_->is_force_flush_pending.exchange(false, std::memory_order_acq_rel)) + { + for (int retry_waiting_times = 0; + false == synchronization_data_->is_force_flush_notified.load(std::memory_order_acquire); + ++retry_waiting_times) + { + opentelemetry::common::SpinLockMutex::fast_yield(); + if ((retry_waiting_times & 127) == 127) + { + std::this_thread::yield(); + } + } + } + synchronization_data_->is_force_flush_notified.store(false, std::memory_order_release); + + return result; } void BatchSpanProcessor::DoBackgroundWork() @@ -109,35 +144,26 @@ void BatchSpanProcessor::DoBackgroundWork() while (true) { // Wait for `timeout` milliseconds - std::unique_lock lk(cv_m_); - cv_.wait_for(lk, timeout); + std::unique_lock lk(synchronization_data_->cv_m); + synchronization_data_->cv.wait_for(lk, timeout, [this] { + if (synchronization_data_->is_force_wakeup_background_worker.load(std::memory_order_acquire)) + { + return true; + } + + return !buffer_.empty(); + }); + synchronization_data_->is_force_wakeup_background_worker.store(false, + std::memory_order_release); - if (is_shutdown_.load() == true) + if (synchronization_data_->is_shutdown.load() == true) { - // Break loop if another thread call ForceFlush - is_force_flush_ = false; DrainQueue(); return; } - bool was_force_flush_called = is_force_flush_.exchange(false); - - // Check if this export was the result of a force flush. - if (!was_force_flush_called) - { - // If the buffer was empty during the entire `timeout` time interval, - // go back to waiting. If this was a spurious wake-up, we export only if - // `buffer_` is not empty. This is acceptable because batching is a best - // mechanism effort here. - if (buffer_.empty() == true) - { - timeout = schedule_delay_millis_; - continue; - } - } - auto start = std::chrono::steady_clock::now(); - Export(was_force_flush_called); + Export(); auto end = std::chrono::steady_clock::now(); auto duration = std::chrono::duration_cast(end - start); @@ -146,28 +172,30 @@ void BatchSpanProcessor::DoBackgroundWork() } } -void BatchSpanProcessor::Export(const bool was_force_flush_called) +void BatchSpanProcessor::Export() { do { std::vector> spans_arr; - size_t num_spans_to_export; - - if (was_force_flush_called == true) + size_t num_records_to_export; + bool notify_force_flush = + synchronization_data_->is_force_flush_pending.exchange(false, std::memory_order_acq_rel); + if (notify_force_flush) { - num_spans_to_export = buffer_.size(); + num_records_to_export = buffer_.size(); } else { - num_spans_to_export = + num_records_to_export = buffer_.size() >= max_export_batch_size_ ? max_export_batch_size_ : buffer_.size(); } - if (num_spans_to_export == 0) + if (num_records_to_export == 0) { + NotifyCompletion(notify_force_flush, synchronization_data_); break; } - buffer_.Consume(num_spans_to_export, + buffer_.Consume(num_records_to_export, [&](CircularBufferRange> range) noexcept { range.ForEach([&](AtomicUniquePtr &ptr) { std::unique_ptr swap_ptr = std::unique_ptr(nullptr); @@ -184,37 +212,25 @@ void BatchSpanProcessor::Export(const bool was_force_flush_called) { exporter_->Export( nostd::span>(spans_arr.data(), spans_arr.size())); + NotifyCompletion(notify_force_flush, synchronization_data_); } else { + std::weak_ptr synchronization_data_watcher = synchronization_data_; exporter_->Export( nostd::span>(spans_arr.data(), spans_arr.size()), - [this, was_force_flush_called](sdk::common::ExportResult result) { + [notify_force_flush, synchronization_data_watcher](sdk::common::ExportResult result) { // TODO: Print result - NotifyForceFlushCompletion(was_force_flush_called); - // If export was called due to shutdown, notify the worker thread - NotifyShutdownCompletion(); + if (synchronization_data_watcher.expired()) + { + return true; + } + + NotifyCompletion(notify_force_flush, synchronization_data_watcher.lock()); return true; }); } - } while (was_force_flush_called); - - if (is_export_async_ == false) - { - NotifyForceFlushCompletion(was_force_flush_called); - } -} - -void BatchSpanProcessor::NotifyForceFlushCompletion(const bool was_force_flush_called) -{ - // Notify the main thread in case this export was the result of a force flush. - if (was_force_flush_called == true) - { - if (is_force_flush_notified_.exchange(true, std::memory_order_acq_rel) == false) - { - force_flush_cv_.notify_all(); - } - } + } while (true); } void BatchSpanProcessor::WaitForShutdownCompletion() @@ -223,29 +239,56 @@ void BatchSpanProcessor::WaitForShutdownCompletion() // for async thread to complete. if (is_export_async_) { - std::unique_lock lk(async_shutdown_m_); - while (is_async_shutdown_notified_.load() == false) + std::unique_lock lk(synchronization_data_->async_shutdown_m); + while (true) { - async_shutdown_cv_.wait(lk); + if (synchronization_data_->is_async_shutdown_notified.load()) + { + break; + } + + // When is_async_shutdown_notified.store(true) and async_shutdown_cv.notify_all() is called + // between is_async_shutdown_notified.load() and async_shutdown_cv.wait(). We must not wait + // for ever + synchronization_data_->async_shutdown_cv.wait_for(lk, schedule_delay_millis_); } } } -void BatchSpanProcessor::NotifyShutdownCompletion() +void BatchSpanProcessor::NotifyCompletion( + bool notify_force_flush, + const std::shared_ptr &synchronization_data) { + if (!synchronization_data) + { + return; + } + + if (notify_force_flush) + { + synchronization_data->is_force_flush_notified.store(true, std::memory_order_release); + synchronization_data->force_flush_cv.notify_one(); + } + // Notify the thread which is waiting on shutdown to complete. - if (is_shutdown_.load() == true) + if (synchronization_data->is_shutdown.load() == true) { - is_async_shutdown_notified_.store(true); - async_shutdown_cv_.notify_one(); + synchronization_data->is_async_shutdown_notified.store(true); + synchronization_data->async_shutdown_cv.notify_all(); } } void BatchSpanProcessor::DrainQueue() { - while (buffer_.empty() == false) + while (true) { - Export(false); + if (buffer_.empty() && + false == synchronization_data_->is_force_flush_pending.load(std::memory_order_acquire)) + { + break; + } + + Export(); WaitForShutdownCompletion(); } } @@ -253,12 +296,13 @@ void BatchSpanProcessor::DrainQueue() bool BatchSpanProcessor::Shutdown(std::chrono::microseconds timeout) noexcept { auto start_time = std::chrono::system_clock::now(); - std::lock_guard shutdown_guard{shutdown_m_}; - bool already_shutdown = is_shutdown_.exchange(true); + std::lock_guard shutdown_guard{synchronization_data_->shutdown_m}; + bool already_shutdown = synchronization_data_->is_shutdown.exchange(true); if (worker_thread_.joinable()) { - cv_.notify_one(); + synchronization_data_->is_force_wakeup_background_worker.store(true, std::memory_order_release); + synchronization_data_->cv.notify_one(); worker_thread_.join(); } @@ -289,7 +333,7 @@ bool BatchSpanProcessor::Shutdown(std::chrono::microseconds timeout) noexcept BatchSpanProcessor::~BatchSpanProcessor() { - if (is_shutdown_.load() == false) + if (synchronization_data_->is_shutdown.load() == false) { Shutdown(); } diff --git a/sdk/src/trace/span.cc b/sdk/src/trace/span.cc index dff84a22b7..13ea99126c 100644 --- a/sdk/src/trace/span.cc +++ b/sdk/src/trace/span.cc @@ -94,6 +94,10 @@ Span::~Span() void Span::SetAttribute(nostd::string_view key, const common::AttributeValue &value) noexcept { std::lock_guard lock_guard{mu_}; + if (recordable_ == nullptr) + { + return; + } recordable_->SetAttribute(key, value); } diff --git a/sdk/test/logs/batch_log_processor_test.cc b/sdk/test/logs/batch_log_processor_test.cc index 5fdda13ca7..8902f23d15 100644 --- a/sdk/test/logs/batch_log_processor_test.cc +++ b/sdk/test/logs/batch_log_processor_test.cc @@ -9,6 +9,8 @@ # include # include +# include +# include # include using namespace opentelemetry::sdk::logs; @@ -59,17 +61,31 @@ class MockLogExporter final : public LogExporter std::function &&result_callback) noexcept override { - auto th = std::thread([this, records, result_callback]() { - auto result = Export(records); - result_callback(result); - }); - th.join(); + // We should keep the order of test records + auto result = Export(records); + async_threads_.emplace_back(std::make_shared( + [result](std::function &&result_callback) { + result_callback(result); + }, + std::move(result_callback))); } // toggles the boolean flag marking this exporter as shut down bool Shutdown( std::chrono::microseconds timeout = std::chrono::microseconds::max()) noexcept override { + while (!async_threads_.empty()) + { + std::list> async_threads; + async_threads.swap(async_threads_); + for (auto &async_thread : async_threads) + { + if (async_thread && async_thread->joinable()) + { + async_thread->join(); + } + } + } *is_shutdown_ = true; return true; } @@ -79,6 +95,7 @@ class MockLogExporter final : public LogExporter std::shared_ptr> is_shutdown_; std::shared_ptr> is_export_completed_; const std::chrono::milliseconds export_delay_; + std::list> async_threads_; }; /** diff --git a/sdk/test/metrics/BUILD b/sdk/test/metrics/BUILD index 9ca191574f..819a8d225f 100644 --- a/sdk/test/metrics/BUILD +++ b/sdk/test/metrics/BUILD @@ -80,6 +80,22 @@ cc_test( ], ) +cc_test( + name = "sync_instruments_test", + srcs = [ + "sync_instruments_test.cc", + ], + tags = [ + "metrics", + "test", + ], + deps = [ + "//sdk/src/metrics", + "//sdk/src/resource", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "async_metric_storage_test", srcs = [ diff --git a/sdk/test/metrics/CMakeLists.txt b/sdk/test/metrics/CMakeLists.txt index 3ee2be6552..fa1f22c73a 100644 --- a/sdk/test/metrics/CMakeLists.txt +++ b/sdk/test/metrics/CMakeLists.txt @@ -29,3 +29,5 @@ target_link_libraries(attributes_processor_benchmark benchmark::benchmark add_executable(attributes_hashmap_benchmark attributes_hashmap_benchmark.cc) target_link_libraries(attributes_hashmap_benchmark benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT} opentelemetry_common) + +add_subdirectory(exemplar) diff --git a/sdk/test/metrics/async_metric_storage_test.cc b/sdk/test/metrics/async_metric_storage_test.cc index fd7d24c6f3..0527c7f0ca 100644 --- a/sdk/test/metrics/async_metric_storage_test.cc +++ b/sdk/test/metrics/async_metric_storage_test.cc @@ -21,7 +21,7 @@ using namespace opentelemetry::sdk::resource; class MockMetricReader : public MetricReader { public: - MockMetricReader(AggregationTemporarily aggr_temporarily) : MetricReader(aggr_temporarily) {} + MockMetricReader(AggregationTemporality aggr_temporality) : MetricReader(aggr_temporality) {} virtual bool OnForceFlush(std::chrono::microseconds timeout) noexcept override { return true; } @@ -44,7 +44,7 @@ TEST(AsyncMetricStorageTest, BasicTests) std::vector> exporters; std::shared_ptr meter_context(new MeterContext(std::move(exporters))); - std::unique_ptr metric_reader(new MockMetricReader(AggregationTemporarily::kDelta)); + std::unique_ptr metric_reader(new MockMetricReader(AggregationTemporality::kDelta)); std::shared_ptr collector = std::shared_ptr( new MetricCollector(std::move(meter_context), std::move(metric_reader))); diff --git a/sdk/test/metrics/exemplar/BUILD b/sdk/test/metrics/exemplar/BUILD new file mode 100644 index 0000000000..6481f679d2 --- /dev/null +++ b/sdk/test/metrics/exemplar/BUILD @@ -0,0 +1,47 @@ +cc_test( + name = "no_exemplar_reservoir_test", + srcs = [ + "no_exemplar_reservoir_test.cc", + ], + tags = [ + "metrics", + "test", + ], + deps = [ + "//api", + "//sdk:headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "never_sample_filter_test", + srcs = [ + "never_sample_filter_test.cc", + ], + tags = [ + "metrics", + "test", + ], + deps = [ + "//api", + "//sdk:headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "always_sample_filter_test", + srcs = [ + "always_sample_filter_test.cc", + ], + tags = [ + "metrics", + "test", + ], + deps = [ + "//api", + "//sdk:headers", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/sdk/test/metrics/exemplar/CMakeLists.txt b/sdk/test/metrics/exemplar/CMakeLists.txt new file mode 100644 index 0000000000..303294761a --- /dev/null +++ b/sdk/test/metrics/exemplar/CMakeLists.txt @@ -0,0 +1,10 @@ +foreach(testname no_exemplar_reservoir_test never_sample_filter_test + always_sample_filter_test) + add_executable(${testname} "${testname}.cc") + target_link_libraries(${testname} ${GTEST_BOTH_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} opentelemetry_metrics) + gtest_add_tests( + TARGET ${testname} + TEST_PREFIX metrics. + TEST_LIST ${testname}) +endforeach() diff --git a/sdk/test/metrics/exemplar/always_sample_filter_test.cc b/sdk/test/metrics/exemplar/always_sample_filter_test.cc new file mode 100644 index 0000000000..cf4e449957 --- /dev/null +++ b/sdk/test/metrics/exemplar/always_sample_filter_test.cc @@ -0,0 +1,19 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#ifndef ENABLE_METRICS_PREVIEW +# include "opentelemetry/sdk/metrics/exemplar/always_sample_filter.h" +# include + +using namespace opentelemetry::sdk::metrics; + +TEST(AlwaysSampleFilter, SampleMeasurement) +{ + auto filter = opentelemetry::sdk::metrics::AlwaysSampleFilter::GetAlwaysSampleFilter(); + ASSERT_TRUE( + filter->ShouldSampleMeasurement(1.0, MetricAttributes{}, opentelemetry::context::Context{})); + ASSERT_TRUE( + filter->ShouldSampleMeasurement(1l, MetricAttributes{}, opentelemetry::context::Context{})); +} + +#endif diff --git a/sdk/test/metrics/exemplar/never_sample_filter_test.cc b/sdk/test/metrics/exemplar/never_sample_filter_test.cc new file mode 100644 index 0000000000..930c572205 --- /dev/null +++ b/sdk/test/metrics/exemplar/never_sample_filter_test.cc @@ -0,0 +1,20 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/context/context.h" +#ifndef ENABLE_METRICS_PREVIEW +# include +# include "opentelemetry/sdk/metrics/exemplar/never_sample_filter.h" + +using namespace opentelemetry::sdk::metrics; + +TEST(NeverSampleFilter, SampleMeasurement) +{ + auto filter = opentelemetry::sdk::metrics::NeverSampleFilter::GetNeverSampleFilter(); + ASSERT_FALSE( + filter->ShouldSampleMeasurement(1.0, MetricAttributes{}, opentelemetry::context::Context{})); + ASSERT_FALSE( + filter->ShouldSampleMeasurement(1l, MetricAttributes{}, opentelemetry::context::Context{})); +} + +#endif diff --git a/sdk/test/metrics/exemplar/no_exemplar_reservoir_test.cc b/sdk/test/metrics/exemplar/no_exemplar_reservoir_test.cc new file mode 100644 index 0000000000..3e16940ff4 --- /dev/null +++ b/sdk/test/metrics/exemplar/no_exemplar_reservoir_test.cc @@ -0,0 +1,22 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#ifndef ENABLE_METRICS_PREVIEW +# include "opentelemetry/sdk/metrics/exemplar/no_exemplar_reservoir.h" +# include + +using namespace opentelemetry::sdk::metrics; + +TEST(NoExemplarReservoir, OfferMeasurement) +{ + auto reservoir = opentelemetry::sdk::metrics::NoExemplarReservoir::GetNoExemplarReservoir(); + EXPECT_NO_THROW(reservoir->OfferMeasurement(1.0, MetricAttributes{}, + opentelemetry::context::Context{}, + std::chrono::system_clock::now())); + EXPECT_NO_THROW(reservoir->OfferMeasurement( + 1l, MetricAttributes{}, opentelemetry::context::Context{}, std::chrono::system_clock::now())); + auto exemplar_data = reservoir->CollectAndReset(MetricAttributes{}); + ASSERT_TRUE(exemplar_data.empty()); +} + +#endif diff --git a/sdk/test/metrics/metric_reader_test.cc b/sdk/test/metrics/metric_reader_test.cc index d0f7c14981..214f522118 100644 --- a/sdk/test/metrics/metric_reader_test.cc +++ b/sdk/test/metrics/metric_reader_test.cc @@ -14,7 +14,7 @@ using namespace opentelemetry::sdk::metrics; class MockMetricReader : public MetricReader { public: - MockMetricReader(AggregationTemporarily aggr_temporarily) : MetricReader(aggr_temporarily) {} + MockMetricReader(AggregationTemporality aggr_temporality) : MetricReader(aggr_temporality) {} virtual bool OnForceFlush(std::chrono::microseconds timeout) noexcept override { return true; } @@ -25,15 +25,15 @@ class MockMetricReader : public MetricReader TEST(MetricReaderTest, BasicTests) { - AggregationTemporarily aggr_temporarily = AggregationTemporarily::kDelta; - std::unique_ptr metric_reader1(new MockMetricReader(aggr_temporarily)); - EXPECT_EQ(metric_reader1->GetAggregationTemporarily(), aggr_temporarily); + AggregationTemporality aggr_temporality = AggregationTemporality::kDelta; + std::unique_ptr metric_reader1(new MockMetricReader(aggr_temporality)); + EXPECT_EQ(metric_reader1->GetAggregationTemporality(), aggr_temporality); std::vector> exporters; std::shared_ptr meter_context1(new MeterContext(std::move(exporters))); EXPECT_NO_THROW(meter_context1->AddMetricReader(std::move(metric_reader1))); - std::unique_ptr metric_reader2(new MockMetricReader(aggr_temporarily)); + std::unique_ptr metric_reader2(new MockMetricReader(aggr_temporality)); std::shared_ptr meter_context2(new MeterContext(std::move(exporters))); MetricProducer *metric_producer = new MetricCollector(std::move(meter_context2), std::move(metric_reader2)); diff --git a/sdk/test/metrics/multi_metric_storage_test.cc b/sdk/test/metrics/multi_metric_storage_test.cc index 6696c7bb0a..d88946485c 100644 --- a/sdk/test/metrics/multi_metric_storage_test.cc +++ b/sdk/test/metrics/multi_metric_storage_test.cc @@ -4,6 +4,7 @@ #ifndef ENABLE_METRICS_PREVIEW # include "opentelemetry/sdk/metrics/state/multi_metric_storage.h" # include "opentelemetry/common/key_value_iterable_view.h" +# include "opentelemetry/sdk/metrics/exemplar/no_exemplar_reservoir.h" # include "opentelemetry/sdk/metrics/instruments.h" # include @@ -15,18 +16,26 @@ using namespace opentelemetry::sdk::metrics; class TestMetricStorage : public WritableMetricStorage { public: - void RecordLong(long value) noexcept override { num_calls_long++; } + void RecordLong(long value, const opentelemetry::context::Context &context) noexcept override + { + num_calls_long++; + } void RecordLong(long value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override { num_calls_long++; } - void RecordDouble(double value) noexcept override { num_calls_double++; } + void RecordDouble(double value, const opentelemetry::context::Context &context) noexcept override + { + num_calls_double++; + } void RecordDouble(double value, - const opentelemetry::common::KeyValueIterable &attributes) noexcept override + const opentelemetry::common::KeyValueIterable &attributes, + const opentelemetry::context::Context &context) noexcept override { num_calls_double++; } @@ -39,13 +48,13 @@ TEST(MultiMetricStorageTest, BasicTests) { std::shared_ptr storage( new TestMetricStorage()); - MultiMetricStorage storages; + MultiMetricStorage storages{}; storages.AddStorage(storage); - EXPECT_NO_THROW(storages.RecordLong(10l)); - EXPECT_NO_THROW(storages.RecordLong(20l)); + EXPECT_NO_THROW(storages.RecordLong(10l, opentelemetry::context::Context{})); + EXPECT_NO_THROW(storages.RecordLong(20l, opentelemetry::context::Context{})); - EXPECT_NO_THROW(storages.RecordDouble(10.0)); - EXPECT_NO_THROW(storages.RecordLong(30l)); + EXPECT_NO_THROW(storages.RecordDouble(10.0, opentelemetry::context::Context{})); + EXPECT_NO_THROW(storages.RecordLong(30l, opentelemetry::context::Context{})); EXPECT_EQ(static_cast(storage.get())->num_calls_long, 3); EXPECT_EQ(static_cast(storage.get())->num_calls_double, 1); diff --git a/sdk/test/metrics/sync_instruments_test.cc b/sdk/test/metrics/sync_instruments_test.cc index 27af8711d1..e029821d41 100644 --- a/sdk/test/metrics/sync_instruments_test.cc +++ b/sdk/test/metrics/sync_instruments_test.cc @@ -3,7 +3,9 @@ #ifndef ENABLE_METRICS_PREVIEW # include "opentelemetry/sdk/metrics/sync_instruments.h" +# include "opentelemetry/context/context.h" # include "opentelemetry/sdk/instrumentationlibrary/instrumentation_library.h" +# include "opentelemetry/sdk/metrics/exemplar/no_exemplar_reservoir.h" # include "opentelemetry/sdk/metrics/state/multi_metric_storage.h" # include @@ -23,11 +25,16 @@ TEST(SyncInstruments, LongCounter) std::unique_ptr metric_storage(new MultiMetricStorage()); LongCounter counter(instrument_descriptor, std::move(metric_storage)); EXPECT_NO_THROW(counter.Add(10l)); - EXPECT_NO_THROW(counter.Add(10l)); + EXPECT_NO_THROW(counter.Add(10l, opentelemetry::context::Context{})); EXPECT_NO_THROW(counter.Add( 10l, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}))); + EXPECT_NO_THROW(counter.Add( + 10l, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}), + opentelemetry::context::Context{})); EXPECT_NO_THROW(counter.Add(10l, opentelemetry::common::KeyValueIterableView({}))); + EXPECT_NO_THROW(counter.Add(10l, opentelemetry::common::KeyValueIterableView({}), + opentelemetry::context::Context{})); } TEST(SyncInstruments, DoubleCounter) @@ -37,11 +44,16 @@ TEST(SyncInstruments, DoubleCounter) std::unique_ptr metric_storage(new MultiMetricStorage()); DoubleCounter counter(instrument_descriptor, std::move(metric_storage)); EXPECT_NO_THROW(counter.Add(10.10)); - EXPECT_NO_THROW(counter.Add(10.10)); + EXPECT_NO_THROW(counter.Add(10.10, opentelemetry::context::Context{})); EXPECT_NO_THROW(counter.Add( 10.10, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}))); + EXPECT_NO_THROW(counter.Add( + 10.10, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}), + opentelemetry::context::Context{})); EXPECT_NO_THROW(counter.Add(10.10, opentelemetry::common::KeyValueIterableView({}))); + EXPECT_NO_THROW(counter.Add(10.10, opentelemetry::common::KeyValueIterableView({}), + opentelemetry::context::Context{})); } TEST(SyncInstruments, LongUpDownCounter) @@ -52,11 +64,16 @@ TEST(SyncInstruments, LongUpDownCounter) std::unique_ptr metric_storage(new MultiMetricStorage()); LongUpDownCounter counter(instrument_descriptor, std::move(metric_storage)); EXPECT_NO_THROW(counter.Add(10l)); - EXPECT_NO_THROW(counter.Add(10l)); + EXPECT_NO_THROW(counter.Add(10l, opentelemetry::context::Context{})); EXPECT_NO_THROW(counter.Add( 10l, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}))); + EXPECT_NO_THROW(counter.Add( + 10l, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}), + opentelemetry::context::Context{})); EXPECT_NO_THROW(counter.Add(10l, opentelemetry::common::KeyValueIterableView({}))); + EXPECT_NO_THROW(counter.Add(10l, opentelemetry::common::KeyValueIterableView({}), + opentelemetry::context::Context{})); } TEST(SyncInstruments, DoubleUpDownCounter) @@ -67,10 +84,15 @@ TEST(SyncInstruments, DoubleUpDownCounter) std::unique_ptr metric_storage(new MultiMetricStorage()); DoubleUpDownCounter counter(instrument_descriptor, std::move(metric_storage)); EXPECT_NO_THROW(counter.Add(10.10)); - EXPECT_NO_THROW(counter.Add(10.10)); + EXPECT_NO_THROW(counter.Add(10.10, opentelemetry::context::Context{})); + EXPECT_NO_THROW(counter.Add( + 10.10, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}), + opentelemetry::context::Context{})); EXPECT_NO_THROW(counter.Add( 10.10, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}))); + EXPECT_NO_THROW(counter.Add(10.10, opentelemetry::common::KeyValueIterableView({}), + opentelemetry::context::Context{})); EXPECT_NO_THROW(counter.Add(10.10, opentelemetry::common::KeyValueIterableView({}))); } @@ -80,12 +102,14 @@ TEST(SyncInstruments, LongHistogram) "long_histogram", "description", "1", InstrumentType::kHistogram, InstrumentValueType::kLong}; std::unique_ptr metric_storage(new MultiMetricStorage()); LongHistogram counter(instrument_descriptor, std::move(metric_storage)); - EXPECT_NO_THROW(counter.Record(10l)); - EXPECT_NO_THROW(counter.Record(10l)); + EXPECT_NO_THROW(counter.Record(10l, opentelemetry::context::Context{})); + EXPECT_NO_THROW(counter.Record(10l, opentelemetry::context::Context{})); EXPECT_NO_THROW(counter.Record( - 10l, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}))); - EXPECT_NO_THROW(counter.Record(10l, opentelemetry::common::KeyValueIterableView({}))); + 10l, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}), + opentelemetry::context::Context{})); + EXPECT_NO_THROW(counter.Record(10l, opentelemetry::common::KeyValueIterableView({}), + opentelemetry::context::Context{})); } TEST(SyncInstruments, DoubleHistogram) @@ -95,12 +119,14 @@ TEST(SyncInstruments, DoubleHistogram) InstrumentValueType::kDouble}; std::unique_ptr metric_storage(new MultiMetricStorage()); DoubleHistogram counter(instrument_descriptor, std::move(metric_storage)); - EXPECT_NO_THROW(counter.Record(10.10)); - EXPECT_NO_THROW(counter.Record(10.10)); + EXPECT_NO_THROW(counter.Record(10.10, opentelemetry::context::Context{})); + EXPECT_NO_THROW(counter.Record(10.10, opentelemetry::context::Context{})); EXPECT_NO_THROW(counter.Record( - 10.10, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}))); - EXPECT_NO_THROW(counter.Record(10.10, opentelemetry::common::KeyValueIterableView({}))); + 10.10, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}), + opentelemetry::context::Context{})); + EXPECT_NO_THROW(counter.Record(10.10, opentelemetry::common::KeyValueIterableView({}), + opentelemetry::context::Context{})); } #endif \ No newline at end of file diff --git a/sdk/test/metrics/sync_metric_storage_test.cc b/sdk/test/metrics/sync_metric_storage_test.cc index 23ef20b11d..c4bd154460 100644 --- a/sdk/test/metrics/sync_metric_storage_test.cc +++ b/sdk/test/metrics/sync_metric_storage_test.cc @@ -4,6 +4,7 @@ #ifndef ENABLE_METRICS_PREVIEW # include "opentelemetry/sdk/metrics/state/sync_metric_storage.h" # include "opentelemetry/common/key_value_iterable_view.h" +# include "opentelemetry/sdk/metrics/exemplar/no_exemplar_reservoir.h" # include "opentelemetry/sdk/metrics/instruments.h" # include "opentelemetry/sdk/metrics/view/attributes_processor.h" @@ -18,13 +19,16 @@ TEST(WritableMetricStorageTest, BasicTests) InstrumentDescriptor instr_desc = {"name", "desc", "1unit", InstrumentType::kCounter, InstrumentValueType::kLong}; - opentelemetry::sdk::metrics::SyncMetricStorage storage(instr_desc, AggregationType::kSum, - new DefaultAttributesProcessor()); - EXPECT_NO_THROW(storage.RecordLong(10l)); - EXPECT_NO_THROW(storage.RecordDouble(10.10)); + opentelemetry::sdk::metrics::SyncMetricStorage storage( + instr_desc, AggregationType::kSum, new DefaultAttributesProcessor(), + NoExemplarReservoir::GetNoExemplarReservoir()); + EXPECT_NO_THROW(storage.RecordLong(10l, opentelemetry::context::Context{})); + EXPECT_NO_THROW(storage.RecordDouble(10.10, opentelemetry::context::Context{})); EXPECT_NO_THROW(storage.RecordLong( - 10l, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}))); + 10l, opentelemetry::common::KeyValueIterableView({{"abc", "123"}, {"xyz", "456"}}), + opentelemetry::context::Context{})); - EXPECT_NO_THROW(storage.RecordDouble(10.10, opentelemetry::common::KeyValueIterableView({}))); + EXPECT_NO_THROW(storage.RecordDouble(10.10, opentelemetry::common::KeyValueIterableView({}), + opentelemetry::context::Context{})); } #endif diff --git a/sdk/test/trace/batch_span_processor_test.cc b/sdk/test/trace/batch_span_processor_test.cc index 265cd5936e..0b7e4c2e39 100644 --- a/sdk/test/trace/batch_span_processor_test.cc +++ b/sdk/test/trace/batch_span_processor_test.cc @@ -7,6 +7,8 @@ #include #include +#include +#include #include OPENTELEMETRY_BEGIN_NAMESPACE @@ -56,20 +58,34 @@ class MockSpanExporter final : public sdk::trace::SpanExporter return sdk::common::ExportResult::kSuccess; } - void Export(const nostd::span> &spans, + void Export(const nostd::span> &records, std::function &&result_callback) noexcept override { - auto th = std::thread([this, spans, result_callback]() { - auto result = Export(spans); - result_callback(result); - }); - th.join(); + // We should keep the order of test records + auto result = Export(records); + async_threads_.emplace_back(std::make_shared( + [result](std::function &&result_callback) { + result_callback(result); + }, + std::move(result_callback))); } bool Shutdown( std::chrono::microseconds timeout = std::chrono::microseconds::max()) noexcept override { + while (!async_threads_.empty()) + { + std::list> async_threads; + async_threads.swap(async_threads_); + for (auto &async_thread : async_threads) + { + if (async_thread && async_thread->joinable()) + { + async_thread->join(); + } + } + } *is_shutdown_ = true; return true; } @@ -82,6 +98,7 @@ class MockSpanExporter final : public sdk::trace::SpanExporter std::shared_ptr> is_export_completed_; // Meant exclusively to test force flush timeout const std::chrono::milliseconds export_delay_; + std::list> async_threads_; }; /** @@ -177,7 +194,7 @@ TEST_F(BatchSpanProcessorTestPeer, TestAsyncShutdown) EXPECT_TRUE(is_shutdown->load()); } -TEST_F(BatchSpanProcessorTestPeer, TestForceFlush) +TEST_F(BatchSpanProcessorTestPeer, TestAsyncForceFlush) { std::shared_ptr> is_shutdown(new std::atomic(false)); std::shared_ptr>> spans_received( @@ -230,7 +247,7 @@ TEST_F(BatchSpanProcessorTestPeer, TestForceFlush) } } -TEST_F(BatchSpanProcessorTestPeer, TestAsyncForceFlush) +TEST_F(BatchSpanProcessorTestPeer, TestForceFlush) { std::shared_ptr> is_shutdown(new std::atomic(false)); std::shared_ptr>> spans_received( diff --git a/sdk/test/trace/tracer_test.cc b/sdk/test/trace/tracer_test.cc index 9d1029cd02..dc806b079f 100644 --- a/sdk/test/trace/tracer_test.cc +++ b/sdk/test/trace/tracer_test.cc @@ -383,6 +383,34 @@ TEST(Tracer, SpanSetAttribute) ASSERT_EQ(3.1, nostd::get(cur_span_data->GetAttributes().at("abc"))); } +TEST(Tracer, TestAfterEnd) +{ + std::unique_ptr exporter(new InMemorySpanExporter()); + std::shared_ptr span_data = exporter->GetData(); + auto tracer = initTracer(std::move(exporter)); + auto span = tracer->StartSpan("span 1"); + span->SetAttribute("abc", 3.1); + + span->End(); + + // test after end + span->SetAttribute("testing null recordable", 3.1); + span->AddEvent("event 1"); + span->AddEvent("event 2", std::chrono::system_clock::now()); + span->AddEvent("event 3", std::chrono::system_clock::now(), {{"attr1", 1}}); + std::string new_name{"new name"}; + span->UpdateName(new_name); + span->SetAttribute("attr1", 3.1); + std::string description{"description"}; + span->SetStatus(opentelemetry::trace::StatusCode::kError, description); + span->End(); + + auto spans = span_data->GetSpans(); + ASSERT_EQ(1, spans.size()); + auto &cur_span_data = spans.at(0); + ASSERT_EQ(3.1, nostd::get(cur_span_data->GetAttributes().at("abc"))); +} + TEST(Tracer, SpanSetEvents) { std::unique_ptr exporter(new InMemorySpanExporter());