From 963f78b0909b2020e348cd6e704249ee7cbafcd2 Mon Sep 17 00:00:00 2001 From: Damien Mehala Date: Wed, 30 Oct 2024 15:40:54 +0100 Subject: [PATCH] [part 1] refactor!: expose telemetry module (#166) Part 1 of the telemetry module refactoring is to make the telemetry module publicly usable by other Datadog products without any dependencies on the tracer. Changes: - Replace `report_telemetry` configuration with telemetry configuration struct. - Add support for telemetry environment variables. - Implement unit tests --- .circleci/config.yml | 2 +- BUILD.bazel | 8 +- CMakeLists.txt | 15 +- cmake/compiler/msvc.cmake | 10 ++ include/datadog/environment.h | 60 ++++---- include/datadog/telemetry/configuration.h | 50 +++++++ .../datadog/telemetry}/metrics.h | 17 ++- include/datadog/telemetry/telemetry.h | 35 +++++ include/datadog/tracer_config.h | 13 +- src/datadog/datadog_agent.h | 2 +- src/datadog/telemetry/configuration.cpp | 112 +++++++++++++++ src/datadog/{ => telemetry}/metrics.cpp | 29 ++-- src/datadog/telemetry/telemetry.cpp | 43 ++++++ src/datadog/trace_segment.cpp | 2 +- src/datadog/tracer.cpp | 2 +- src/datadog/tracer_config.cpp | 21 +-- src/datadog/tracer_telemetry.cpp | 23 ++- src/datadog/tracer_telemetry.h | 75 +++++----- test/CMakeLists.txt | 5 +- test/common/environment.h | 57 ++++++++ test/telemetry/test_configuration.cpp | 136 ++++++++++++++++++ test/{ => telemetry}/test_metrics.cpp | 14 +- test/test_curl.cpp | 2 +- test/test_datadog_agent.cpp | 6 +- test/test_tracer_config.cpp | 48 +------ test/test_tracer_telemetry.cpp | 4 +- 26 files changed, 616 insertions(+), 175 deletions(-) create mode 100644 include/datadog/telemetry/configuration.h rename {src/datadog => include/datadog/telemetry}/metrics.h (81%) create mode 100644 include/datadog/telemetry/telemetry.h create mode 100644 src/datadog/telemetry/configuration.cpp rename src/datadog/{ => telemetry}/metrics.cpp (53%) create mode 100644 src/datadog/telemetry/telemetry.cpp create mode 100644 test/common/environment.h create mode 100644 test/telemetry/test_configuration.cpp rename test/{ => telemetry}/test_metrics.cpp (61%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 71cace23..96aa3abf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,7 +116,7 @@ jobs: name: Building command: | & 'C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\Common7\\Tools\\Launch-VsDevShell.ps1' -arch << parameters.arch >> - cmake -B build -DCMAKE_BUILD_TYPE=Debug -DDD_TRACE_BUILD_TESTING=1 -G Ninja . + cmake -B build -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=OFF -DDD_TRACE_STATIC_CRT=1 -DDD_TRACE_BUILD_TESTING=1 -G Ninja . cmake --build build -j $env:MAKE_JOB_COUNT -v - run: name: Testing diff --git a/BUILD.bazel b/BUILD.bazel index 30d7a2d5..1a6a1877 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,6 +1,9 @@ cc_library( name = "dd_trace_cpp", srcs = [ + "src/datadog/telemetry/configuration.cpp", + "src/datadog/telemetry/metrics.cpp", + "src/datadog/telemetry/telemetry.cpp", "src/datadog/base64.cpp", "src/datadog/cerr_logger.cpp", "src/datadog/clock.cpp", @@ -17,7 +20,6 @@ cc_library( "src/datadog/id_generator.cpp", "src/datadog/limiter.cpp", "src/datadog/logger.cpp", - "src/datadog/metrics.cpp", "src/datadog/msgpack.cpp", "src/datadog/parse_util.cpp", "src/datadog/platform_util.cpp", @@ -58,7 +60,6 @@ cc_library( "src/datadog/json.hpp", "src/datadog/json_serializer.h", "src/datadog/limiter.h", - "src/datadog/metrics.h", "src/datadog/msgpack.h", "src/datadog/parse_util.h", "src/datadog/platform_util.h", @@ -111,6 +112,9 @@ cc_library( "include/datadog/trace_sampler_config.h", "include/datadog/trace_segment.h", "include/datadog/version.h", + "include/datadog/telemetry/configuration.h", + "include/datadog/telemetry/metrics.h", + "include/datadog/telemetry/telemetry.h", "include/datadog/remote_config/capability.h", "include/datadog/remote_config/listener.h", "include/datadog/remote_config/product.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 2af61f37..8d4cab5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,14 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) option(BUILD_SHARED_LIBS "Build shared libraries" ON) option(BUILD_STATIC_LIBS "Build static libraries" ON) +if (WIN32) + option(DD_TRACE_STATIC_CRT "Build dd-trace-cpp with static CRT with MSVC" OFF) +endif () + +if (DD_TRACE_STATIC_CRT) + set(CURL_STATIC_CRT ON) +endif () + set(DD_TRACE_TRANSPORT "curl" CACHE STRING "HTTP transport that dd-trace-cpp uses to communicate with the Datadog Agent, can be either 'none' or 'curl'") # Consumer of the library using FetchContent do not need @@ -46,6 +54,9 @@ unset(DD_TRACE_VERSION_CPP_CONTENTS) message(STATUS "dd-trace-cpp transport=${DD_TRACE_TRANSPORT}") if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + if (BUILD_SHARED_LIBS AND BUILD_STATIC_LIBS) + message(FATAL_ERROR "Can't build both static and shared libary for MSVC") + endif () message(STATUS "Compiler: MSVC ${CMAKE_CXX_COMPILER_VERSION}") include(cmake/compiler/msvc.cmake) elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") @@ -87,6 +98,9 @@ target_sources(dd_trace_cpp-objects BASE_DIRS include FILES ${public_header_files} PRIVATE + src/datadog/telemetry/configuration.cpp + src/datadog/telemetry/metrics.cpp + src/datadog/telemetry/telemetry.cpp src/datadog/base64.cpp src/datadog/cerr_logger.cpp src/datadog/clock.cpp @@ -102,7 +116,6 @@ target_sources(dd_trace_cpp-objects src/datadog/id_generator.cpp src/datadog/limiter.cpp src/datadog/logger.cpp - src/datadog/metrics.cpp src/datadog/msgpack.cpp src/datadog/parse_util.cpp src/datadog/platform_util.cpp diff --git a/cmake/compiler/msvc.cmake b/cmake/compiler/msvc.cmake index b12a7c03..6b836423 100644 --- a/cmake/compiler/msvc.cmake +++ b/cmake/compiler/msvc.cmake @@ -33,6 +33,16 @@ set(WINDOWS_EXPORT_ALL_SYMBOLS ON) add_library(dd_trace_cpp-specs INTERFACE) add_library(dd_trace::specs ALIAS dd_trace_cpp-specs) +if (DD_TRACE_STATIC_CRT) + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /MT") + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /MTd") + # set_target_properties(dd_trace_cpp-specs + # PROPERTIES + # MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + # ) +endif () + target_compile_options(dd_trace_cpp-specs INTERFACE /W4 diff --git a/include/datadog/environment.h b/include/datadog/environment.h index 146979f3..c5ffb199 100644 --- a/include/datadog/environment.h +++ b/include/datadog/environment.h @@ -23,34 +23,38 @@ namespace environment { // To enforce correspondence between `enum Variable` and `variable_names`, the // preprocessor is used so that the DD_* symbols are listed exactly once. -#define LIST_ENVIRONMENT_VARIABLES(MACRO) \ - MACRO(DD_AGENT_HOST) \ - MACRO(DD_ENV) \ - MACRO(DD_INSTRUMENTATION_TELEMETRY_ENABLED) \ - MACRO(DD_PROPAGATION_STYLE_EXTRACT) \ - MACRO(DD_PROPAGATION_STYLE_INJECT) \ - MACRO(DD_REMOTE_CONFIGURATION_ENABLED) \ - MACRO(DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS) \ - MACRO(DD_SERVICE) \ - MACRO(DD_SPAN_SAMPLING_RULES) \ - MACRO(DD_SPAN_SAMPLING_RULES_FILE) \ - MACRO(DD_TRACE_DELEGATE_SAMPLING) \ - MACRO(DD_TRACE_PROPAGATION_STYLE_EXTRACT) \ - MACRO(DD_TRACE_PROPAGATION_STYLE_INJECT) \ - MACRO(DD_TRACE_PROPAGATION_STYLE) \ - MACRO(DD_TAGS) \ - MACRO(DD_TRACE_AGENT_PORT) \ - MACRO(DD_TRACE_AGENT_URL) \ - MACRO(DD_TRACE_DEBUG) \ - MACRO(DD_TRACE_ENABLED) \ - MACRO(DD_TRACE_RATE_LIMIT) \ - MACRO(DD_TRACE_REPORT_HOSTNAME) \ - MACRO(DD_TRACE_SAMPLE_RATE) \ - MACRO(DD_TRACE_SAMPLING_RULES) \ - MACRO(DD_TRACE_STARTUP_LOGS) \ - MACRO(DD_TRACE_TAGS_PROPAGATION_MAX_LENGTH) \ - MACRO(DD_VERSION) \ - MACRO(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) +#define LIST_ENVIRONMENT_VARIABLES(MACRO) \ + MACRO(DD_AGENT_HOST) \ + MACRO(DD_ENV) \ + MACRO(DD_INSTRUMENTATION_TELEMETRY_ENABLED) \ + MACRO(DD_PROPAGATION_STYLE_EXTRACT) \ + MACRO(DD_PROPAGATION_STYLE_INJECT) \ + MACRO(DD_REMOTE_CONFIGURATION_ENABLED) \ + MACRO(DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS) \ + MACRO(DD_SERVICE) \ + MACRO(DD_SPAN_SAMPLING_RULES) \ + MACRO(DD_SPAN_SAMPLING_RULES_FILE) \ + MACRO(DD_TRACE_DELEGATE_SAMPLING) \ + MACRO(DD_TRACE_PROPAGATION_STYLE_EXTRACT) \ + MACRO(DD_TRACE_PROPAGATION_STYLE_INJECT) \ + MACRO(DD_TRACE_PROPAGATION_STYLE) \ + MACRO(DD_TAGS) \ + MACRO(DD_TRACE_AGENT_PORT) \ + MACRO(DD_TRACE_AGENT_URL) \ + MACRO(DD_TRACE_DEBUG) \ + MACRO(DD_TRACE_ENABLED) \ + MACRO(DD_TRACE_RATE_LIMIT) \ + MACRO(DD_TRACE_REPORT_HOSTNAME) \ + MACRO(DD_TRACE_SAMPLE_RATE) \ + MACRO(DD_TRACE_SAMPLING_RULES) \ + MACRO(DD_TRACE_STARTUP_LOGS) \ + MACRO(DD_TRACE_TAGS_PROPAGATION_MAX_LENGTH) \ + MACRO(DD_VERSION) \ + MACRO(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) \ + MACRO(DD_TELEMETRY_HEARTBEAT_INTERVAL) \ + MACRO(DD_TELEMETRY_METRICS_ENABLED) \ + MACRO(DD_TELEMETRY_METRICS_INTERVAL_SECONDS) \ + MACRO(DD_TELEMETRY_DEBUG) #define WITH_COMMA(ARG) ARG, diff --git a/include/datadog/telemetry/configuration.h b/include/datadog/telemetry/configuration.h new file mode 100644 index 00000000..50e829b6 --- /dev/null +++ b/include/datadog/telemetry/configuration.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +#include +#include + +namespace datadog::telemetry { + +struct Configuration { + // Turn on or off telemetry module + // Enabled by default. + // Overwritten by `DD_INSTRUMENTATION_TELEMETRY_ENABLED` env var. + tracing::Optional enabled; + // Turn on or off metrics reporting + // Overwritten by `DD_TELEMETRY_METRICS_ENABLED` env var. + tracing::Optional report_metrics; + // Interval of metrics payload + // Overwritten by `DD_TELEMETRY_METRICS_INTERVAL_SECONDS` env var. + tracing::Optional metrics_interval_seconds; + // Interval of heartbeat payload + // Overwritten by `DD_TELEMETRY_HEARTBEAT_INTERVAL` env var. + tracing::Optional heartbeat_interval_seconds; + // `integration_name` is the name of the product integrating this library. + // Example: "nginx", "envoy" or "istio". + tracing::Optional integration_name; + // `integration_version` is the version of the product integrating this + // library. + // Example: "1.2.3", "6c44da20", "2020.02.13" + tracing::Optional integration_version; +}; + +struct FinalizedConfiguration { + bool debug; + bool enabled; + bool report_metrics; + std::chrono::steady_clock::duration metrics_interval; + std::chrono::steady_clock::duration heartbeat_interval; + std::string integration_name; + std::string integration_version; + + friend tracing::Expected finalize_config( + const Configuration&); +}; + +tracing::Expected finalize_config( + const Configuration& = Configuration{}); + +} // namespace datadog::telemetry diff --git a/src/datadog/metrics.h b/include/datadog/telemetry/metrics.h similarity index 81% rename from src/datadog/metrics.h rename to include/datadog/telemetry/metrics.h index 0455812e..c2943759 100644 --- a/src/datadog/metrics.h +++ b/include/datadog/telemetry/metrics.h @@ -11,7 +11,7 @@ #include namespace datadog { -namespace tracing { +namespace telemetry { class Metric { // The name of the metric that will be published. A transformation occurs @@ -20,6 +20,8 @@ class Metric { std::string name_; // The type of the metric. This will currently be count or gauge. std::string type_; + // Namespace of the metric. + std::string scope_; // Tags associated with this specific instance of the metric. std::vector tags_; // This affects the transformation of the metric name, where it can be a @@ -29,14 +31,15 @@ class Metric { protected: std::atomic value_ = 0; - Metric(std::string name, std::string type, std::vector tags, - bool common); + Metric(std::string name, std::string type, std::string scope, + std::vector tags, bool common); public: // Accessors for name, type, tags, common and capture_and_reset_value are used // when producing the JSON message for reporting metrics. std::string name(); std::string type(); + std::string scope(); std::vector tags(); bool common(); uint64_t value(); @@ -47,7 +50,8 @@ class Metric { // number of actions, or incrementing the current number of actions by 1. class CounterMetric : public Metric { public: - CounterMetric(std::string name, std::vector tags, bool common); + CounterMetric(std::string name, std::string scope, + std::vector tags, bool common); void inc(); void add(uint64_t amount); }; @@ -57,7 +61,8 @@ class CounterMetric : public Metric { // state by 1. class GaugeMetric : public Metric { public: - GaugeMetric(std::string name, std::vector tags, bool common); + GaugeMetric(std::string name, std::string scope, + std::vector tags, bool common); void set(uint64_t value); void inc(); void add(uint64_t amount); @@ -65,5 +70,5 @@ class GaugeMetric : public Metric { void sub(uint64_t amount); }; -} // namespace tracing +} // namespace telemetry } // namespace datadog diff --git a/include/datadog/telemetry/telemetry.h b/include/datadog/telemetry/telemetry.h new file mode 100644 index 00000000..ab5483bb --- /dev/null +++ b/include/datadog/telemetry/telemetry.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace datadog { + +namespace tracing { +class DatadogAgent; +class TracerTelemetry; +} // namespace tracing + +namespace telemetry { + +class Telemetry { + FinalizedConfiguration config_; + std::shared_ptr logger_; + + std::shared_ptr datadog_agent_; + std::shared_ptr tracer_telemetry_; + + public: + Telemetry(FinalizedConfiguration configuration, + std::shared_ptr logger, + std::vector> metrics); + + ~Telemetry() = default; +}; + +} // namespace telemetry +} // namespace datadog diff --git a/include/datadog/tracer_config.h b/include/datadog/tracer_config.h index 3709b936..3752cb26 100644 --- a/include/datadog/tracer_config.h +++ b/include/datadog/tracer_config.h @@ -4,6 +4,8 @@ // `Tracer`. `Tracer` is instantiated with a `FinalizedTracerConfig`, which // must be obtained from the result of a call to `finalize_config`. +#include + #include #include #include @@ -73,12 +75,9 @@ struct TracerConfig { // variable. Optional report_traces; - // `report_telemetry` indicates whether telemetry about the tracer will be - // sent to a collector (`true`) or discarded on completion (`false`). If - // `report_telemetry` is `false`, then this feature is disabled. - // `report_telemetry` is overridden by the - // `DD_INSTRUMENTATION_TELEMETRY_ENABLED` environment variable. - Optional report_telemetry; + // `telemetry` configures the telemetry module. See + // `telemetry/configuration.h` By default, the telemetry module is enabled. + telemetry::Configuration telemetry; // `delegate_trace_sampling` indicates whether the tracer will consult a child // service for a trace sampling decision, and prefer the resulting decision @@ -180,6 +179,7 @@ class FinalizedTracerConfig final { FinalizedTraceSamplerConfig trace_sampler; FinalizedSpanSamplerConfig span_sampler; + telemetry::FinalizedConfiguration telemetry; std::vector injection_styles; std::vector extraction_styles; @@ -189,7 +189,6 @@ class FinalizedTracerConfig final { std::shared_ptr logger; bool log_on_startup; bool generate_128bit_trace_ids; - bool report_telemetry; Optional runtime_id; Clock clock; std::string integration_name; diff --git a/src/datadog/datadog_agent.h b/src/datadog/datadog_agent.h index 3f013d94..40e712ba 100644 --- a/src/datadog/datadog_agent.h +++ b/src/datadog/datadog_agent.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -17,7 +18,6 @@ #include #include "config_manager.h" -#include "metrics.h" #include "remote_config/remote_config.h" #include "tracer_telemetry.h" diff --git a/src/datadog/telemetry/configuration.cpp b/src/datadog/telemetry/configuration.cpp new file mode 100644 index 00000000..d4c9c568 --- /dev/null +++ b/src/datadog/telemetry/configuration.cpp @@ -0,0 +1,112 @@ +#include +#include +#include + +#include "parse_util.h" + +using namespace datadog::tracing; + +namespace datadog::telemetry { + +namespace { + +tracing::Expected load_telemetry_env_config() { + Configuration env_cfg; + + if (auto enabled_env = + lookup(environment::DD_INSTRUMENTATION_TELEMETRY_ENABLED)) { + env_cfg.enabled = !falsy(*enabled_env); + } + + if (auto metrics_enabled = + lookup(environment::DD_TELEMETRY_METRICS_ENABLED)) { + env_cfg.report_metrics = !falsy(*metrics_enabled); + } + + if (auto metrics_interval_seconds = + lookup(environment::DD_TELEMETRY_METRICS_INTERVAL_SECONDS)) { + auto maybe_value = parse_uint64(*metrics_interval_seconds, 10); + if (auto error = maybe_value.if_error()) { + return *error; + } + env_cfg.metrics_interval_seconds = *maybe_value; + } + + if (auto heartbeat_interval_seconds = + lookup(environment::DD_TELEMETRY_HEARTBEAT_INTERVAL)) { + auto maybe_value = parse_uint64(*heartbeat_interval_seconds, 10); + if (auto error = maybe_value.if_error()) { + return *error; + } + + env_cfg.heartbeat_interval_seconds = *maybe_value; + } + + return env_cfg; +} + +} // namespace + +tracing::Expected finalize_config( + const Configuration& user_config) { + auto env_config = load_telemetry_env_config(); + if (auto error = env_config.if_error()) { + return *error; + } + + ConfigMetadata::Origin origin; + FinalizedConfiguration result; + + // enabled + std::tie(origin, result.enabled) = + pick(env_config->enabled, user_config.enabled, true); + + if (!result.enabled) { + // NOTE(@dmehala): if the telemetry module is disabled then report metrics + // is also disabled. + result.report_metrics = false; + } else { + // report_metrics + std::tie(origin, result.report_metrics) = + pick(env_config->report_metrics, user_config.report_metrics, true); + } + + // debug + if (auto enabled_debug_env = lookup(environment::DD_TELEMETRY_DEBUG)) { + result.debug = !falsy(*enabled_debug_env); + } else { + result.debug = false; + } + + // metrics_interval_seconds + auto metrics_interval = pick(env_config->metrics_interval_seconds, + user_config.metrics_interval_seconds, 60); + if (metrics_interval.second <= 0) { + // TBD + return Error{}; + } + result.metrics_interval = std::chrono::seconds(metrics_interval.second); + + // heartbeat_interval_seconds + auto heartbeat_interval = pick(env_config->heartbeat_interval_seconds, + user_config.heartbeat_interval_seconds, 10); + if (heartbeat_interval.second <= 0) { + // TBD + return Error{}; + } + result.heartbeat_interval = std::chrono::seconds(heartbeat_interval.second); + + // integration_name + std::tie(origin, result.integration_name) = + pick(env_config->integration_name, user_config.integration_name, + std::string("")); + + // integration_version + std::tie(origin, result.integration_version) = + pick(env_config->integration_version, user_config.integration_version, + std::string("")); + + return result; +} + +} // namespace datadog::telemetry diff --git a/src/datadog/metrics.cpp b/src/datadog/telemetry/metrics.cpp similarity index 53% rename from src/datadog/metrics.cpp rename to src/datadog/telemetry/metrics.cpp index 19eaa47e..8140cad3 100644 --- a/src/datadog/metrics.cpp +++ b/src/datadog/telemetry/metrics.cpp @@ -1,29 +1,32 @@ -#include "metrics.h" - -#include "json.hpp" +#include namespace datadog { -namespace tracing { +namespace telemetry { -Metric::Metric(std::string name, std::string type, +Metric::Metric(std::string name, std::string type, std::string scope, std::vector tags, bool common) - : name_(name), type_(type), tags_(tags), common_(common) {} + : name_(std::move(name)), + type_(std::move(type)), + scope_(std::move(scope)), + tags_(std::move(tags)), + common_(common) {} std::string Metric::name() { return name_; } std::string Metric::type() { return type_; } +std::string Metric::scope() { return scope_; } std::vector Metric::tags() { return tags_; } bool Metric::common() { return common_; } uint64_t Metric::value() { return value_; } uint64_t Metric::capture_and_reset_value() { return value_.exchange(0); } -CounterMetric::CounterMetric(std::string name, std::vector tags, - bool common) - : Metric(name, "count", tags, common) {} +CounterMetric::CounterMetric(std::string name, std::string scope, + std::vector tags, bool common) + : Metric(name, "count", scope, tags, common) {} void CounterMetric::inc() { add(1); } void CounterMetric::add(uint64_t amount) { value_ += amount; } -GaugeMetric::GaugeMetric(std::string name, std::vector tags, - bool common) - : Metric(name, "gauge", tags, common) {} +GaugeMetric::GaugeMetric(std::string name, std::string scope, + std::vector tags, bool common) + : Metric(name, "gauge", scope, tags, common) {} void GaugeMetric::set(uint64_t value) { value_ = value; } void GaugeMetric::inc() { add(1); } void GaugeMetric::add(uint64_t amount) { value_ += amount; } @@ -36,5 +39,5 @@ void GaugeMetric::sub(uint64_t amount) { } } -} // namespace tracing +} // namespace telemetry } // namespace datadog diff --git a/src/datadog/telemetry/telemetry.cpp b/src/datadog/telemetry/telemetry.cpp new file mode 100644 index 00000000..a7b13589 --- /dev/null +++ b/src/datadog/telemetry/telemetry.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include + +#include "datadog_agent.h" +#include "platform_util.h" +#include "tracer_telemetry.h" + +namespace datadog { +namespace telemetry { + +Telemetry::Telemetry(FinalizedConfiguration config, + std::shared_ptr logger, + std::vector> metrics) + : config_(std::move(config)), logger_(std::move(logger)) { + if (!config_.enabled) { + return; + } + + tracing::TracerSignature tracer_signature(tracing::RuntimeID::generate(), + tracing::get_process_name(), ""); + + tracer_telemetry_ = std::make_shared( + config_.enabled, tracing::default_clock, logger_, tracer_signature, + config_.integration_name, config_.integration_version, metrics); + + tracing::DatadogAgentConfig dd_config; + dd_config.remote_configuration_enabled = false; + + auto final_cfg = + tracing::finalize_config(dd_config, logger_, tracing::default_clock); + if (!final_cfg) { + return; + } + + datadog_agent_ = std::make_shared( + *final_cfg, tracer_telemetry_, logger_, tracer_signature, + std::vector>{}); +} + +} // namespace telemetry +} // namespace datadog diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 3a01cd29..c1001844 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -18,7 +19,6 @@ #include "config_manager.h" #include "hex.h" #include "json.hpp" -#include "metrics.h" #include "platform_util.h" #include "random.h" #include "span_data.h" diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index eac3ab66..321ce308 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -45,7 +45,7 @@ Tracer::Tracer(const FinalizedTracerConfig& config, signature_{runtime_id_, config.defaults.service, config.defaults.environment}, tracer_telemetry_(std::make_shared( - config.report_telemetry, config.clock, logger_, signature_, + config.telemetry.enabled, config.clock, logger_, signature_, config.integration_name, config.integration_version)), config_manager_(std::make_shared(config, signature_, tracer_telemetry_)), diff --git a/src/datadog/tracer_config.cpp b/src/datadog/tracer_config.cpp index ce6e8574..73aaeff2 100644 --- a/src/datadog/tracer_config.cpp +++ b/src/datadog/tracer_config.cpp @@ -4,11 +4,8 @@ #include #include -#include -#include #include #include -#include #include #include "cerr_logger.h" @@ -125,10 +122,6 @@ Expected load_tracer_env_config(Logger &logger) { if (auto enabled_env = lookup(environment::DD_TRACE_ENABLED)) { env_cfg.report_traces = !falsy(*enabled_env); } - if (auto enabled_env = - lookup(environment::DD_INSTRUMENTATION_TELEMETRY_ENABLED)) { - env_cfg.report_telemetry = !falsy(*enabled_env); - } if (auto trace_delegate_sampling_env = lookup(environment::DD_TRACE_DELEGATE_SAMPLING)) { env_cfg.delegate_trace_sampling = !falsy(*trace_delegate_sampling_env); @@ -333,13 +326,6 @@ Expected finalize_config(const TracerConfig &user_config, final_config.metadata[ConfigName::REPORT_TRACES] = ConfigMetadata( ConfigName::REPORT_TRACES, to_string(final_config.report_traces), origin); - // Report telemetry - std::tie(origin, final_config.report_telemetry) = - pick(env_config->report_telemetry, user_config.report_telemetry, true); - final_config.metadata[ConfigName::REPORT_TELEMETRY] = - ConfigMetadata(ConfigName::REPORT_TELEMETRY, - to_string(final_config.report_traces), origin); - // Report hostname final_config.report_hostname = value_or(env_config->report_hostname, user_config.report_hostname, false); @@ -401,6 +387,13 @@ Expected finalize_config(const TracerConfig &user_config, return std::move(span_sampler_config.error()); } + if (auto telemetry_final_config = + telemetry::finalize_config(user_config.telemetry)) { + final_config.telemetry = std::move(*telemetry_final_config); + } else { + return std::move(telemetry_final_config.error()); + } + return final_config; } diff --git a/src/datadog/tracer_telemetry.cpp b/src/datadog/tracer_telemetry.cpp index 2985657b..120ea561 100644 --- a/src/datadog/tracer_telemetry.cpp +++ b/src/datadog/tracer_telemetry.cpp @@ -53,11 +53,11 @@ std::string to_string(datadog::tracing::ConfigName name) { } // namespace -TracerTelemetry::TracerTelemetry(bool enabled, const Clock& clock, - const std::shared_ptr& logger, - const TracerSignature& tracer_signature, - const std::string& integration_name, - const std::string& integration_version) +TracerTelemetry::TracerTelemetry( + bool enabled, const Clock& clock, const std::shared_ptr& logger, + const TracerSignature& tracer_signature, + const std::string& integration_name, const std::string& integration_version, + const std::vector>& user_metrics) : enabled_(enabled), clock_(clock), logger_(logger), @@ -66,6 +66,9 @@ TracerTelemetry::TracerTelemetry(bool enabled, const Clock& clock, integration_name_(integration_name), integration_version_(integration_version) { if (enabled_) { + if (integration_name_.empty()) { + integration_name_ = "datadog"; + } // Register all the metrics that we're tracking by adding them to the // metrics_snapshots_ container. This allows for simpler iteration logic // when using the values in `generate-metrics` messages. @@ -97,6 +100,10 @@ TracerTelemetry::TracerTelemetry(bool enabled, const Clock& clock, MetricSnapshot{}); metrics_snapshots_.emplace_back(metrics_.trace_api.errors_status_code, MetricSnapshot{}); + + for (auto& m : user_metrics) { + metrics_snapshots_.emplace_back(*m, MetricSnapshot{}); + } } } @@ -258,6 +265,7 @@ std::string TracerTelemetry::heartbeat_and_telemetry() { {"tags", metric.tags()}, {"type", metric.type()}, {"points", points}, + {"namespace", metric.scope()}, {"common", metric.common()}, })); } else if (type == "gauge") { @@ -266,6 +274,7 @@ std::string TracerTelemetry::heartbeat_and_telemetry() { {"metric", metric.name()}, {"tags", metric.tags()}, {"type", metric.type()}, + {"namespace", metric.scope()}, {"interval", 10}, {"points", points}, {"common", metric.common()}, @@ -279,7 +288,6 @@ std::string TracerTelemetry::heartbeat_and_telemetry() { auto generate_metrics = nlohmann::json::object({ {"request_type", "generate-metrics"}, {"payload", nlohmann::json::object({ - {"namespace", "tracers"}, {"series", metrics}, })}, }); @@ -314,6 +322,7 @@ std::string TracerTelemetry::app_closing() { {"type", metric.type()}, {"points", points}, {"common", metric.common()}, + {"namespace", metric.scope()}, })); } else if (type == "gauge") { // gauge metrics have a interval @@ -324,6 +333,7 @@ std::string TracerTelemetry::app_closing() { {"interval", 10}, {"points", points}, {"common", metric.common()}, + {"namespace", metric.scope()}, })); } } @@ -334,7 +344,6 @@ std::string TracerTelemetry::app_closing() { auto generate_metrics = nlohmann::json::object({ {"request_type", "generate-metrics"}, {"payload", nlohmann::json::object({ - {"namespace", "tracers"}, {"series", metrics}, })}, }); diff --git a/src/datadog/tracer_telemetry.h b/src/datadog/tracer_telemetry.h index 13e6063c..1ed24e01 100644 --- a/src/datadog/tracer_telemetry.h +++ b/src/datadog/tracer_telemetry.h @@ -31,12 +31,12 @@ #include #include #include +#include #include #include #include "json.hpp" -#include "metrics.h" #include "platform_util.h" namespace datadog { @@ -62,38 +62,42 @@ class TracerTelemetry { // telemetry. struct { struct { - CounterMetric spans_created = { - "spans_created", {"integration_name:datadog"}, true}; - CounterMetric spans_finished = { - "spans_finished", {"integration_name:datadog"}, true}; + telemetry::CounterMetric spans_created = { + "spans_created", "tracers", {}, true}; + telemetry::CounterMetric spans_finished = { + "spans_finished", "tracers", {}, true}; - CounterMetric trace_segments_created_new = { - "trace_segments_created", {"new_continued:new"}, true}; - CounterMetric trace_segments_created_continued = { - "trace_segments_created", {"new_continued:continued"}, true}; - CounterMetric trace_segments_closed = { - "trace_segments_closed", {"integration_name:datadog"}, true}; + telemetry::CounterMetric trace_segments_created_new = { + "trace_segments_created", "tracers", {"new_continued:new"}, true}; + telemetry::CounterMetric trace_segments_created_continued = { + "trace_segments_created", + "tracers", + {"new_continued:continued"}, + true}; + telemetry::CounterMetric trace_segments_closed = { + "trace_segments_closed", "tracers", {}, true}; } tracer; struct { - CounterMetric requests = {"trace_api.requests", {}, true}; + telemetry::CounterMetric requests = { + "trace_api.requests", "tracers", {}, true}; - CounterMetric responses_1xx = { - "trace_api.responses", {"status_code:1xx"}, true}; - CounterMetric responses_2xx = { - "trace_api.responses", {"status_code:2xx"}, true}; - CounterMetric responses_3xx = { - "trace_api.responses", {"status_code:3xx"}, true}; - CounterMetric responses_4xx = { - "trace_api.responses", {"status_code:4xx"}, true}; - CounterMetric responses_5xx = { - "trace_api.responses", {"status_code:5xx"}, true}; + telemetry::CounterMetric responses_1xx = { + "trace_api.responses", "tracers", {"status_code:1xx"}, true}; + telemetry::CounterMetric responses_2xx = { + "trace_api.responses", "tracers", {"status_code:2xx"}, true}; + telemetry::CounterMetric responses_3xx = { + "trace_api.responses", "tracers", {"status_code:3xx"}, true}; + telemetry::CounterMetric responses_4xx = { + "trace_api.responses", "tracers", {"status_code:4xx"}, true}; + telemetry::CounterMetric responses_5xx = { + "trace_api.responses", "tracers", {"status_code:5xx"}, true}; - CounterMetric errors_timeout = { - "trace_api.errors", {"type:timeout"}, true}; - CounterMetric errors_network = { - "trace_api.errors", {"type:network"}, true}; - CounterMetric errors_status_code = { - "trace_api.errors", {"type:status_code"}, true}; + telemetry::CounterMetric errors_timeout = { + "trace_api.errors", "tracers", {"type:timeout"}, true}; + telemetry::CounterMetric errors_network = { + "trace_api.errors", "tracers", {"type:network"}, true}; + telemetry::CounterMetric errors_status_code = { + "trace_api.errors", "tracers", {"type:status_code"}, true}; } trace_api; } metrics_; @@ -103,7 +107,8 @@ class TracerTelemetry { // This uses a reference_wrapper so references to internal metric values can // be captured, and be iterated trivially when the values need to be // snapshotted and published in telemetry messages. - std::vector, MetricSnapshot>> + std::vector< + std::pair, MetricSnapshot>> metrics_snapshots_; std::vector configuration_snapshot_; @@ -114,11 +119,13 @@ class TracerTelemetry { const ConfigMetadata& config_metadata); public: - TracerTelemetry(bool enabled, const Clock& clock, - const std::shared_ptr& logger, - const TracerSignature& tracer_signature, - const std::string& integration_name, - const std::string& integration_version); + TracerTelemetry( + bool enabled, const Clock& clock, const std::shared_ptr& logger, + const TracerSignature& tracer_signature, + const std::string& integration_name, + const std::string& integration_version, + const std::vector>& user_metrics = + std::vector>{}); inline bool enabled() { return enabled_; } inline bool debug() { return debug_; } // Provides access to the telemetry metrics for updating the values. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 059f8e9d..0c99080c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -16,6 +16,10 @@ add_executable(tests # utilities matchers.cpp + # telemetry test cases + telemetry/test_configuration.cpp + telemetry/test_metrics.cpp + # test cases test_base64.cpp test_cerr_logger.cpp @@ -24,7 +28,6 @@ add_executable(tests test_datadog_agent.cpp test_glob.cpp test_limiter.cpp - test_metrics.cpp test_msgpack.cpp test_parse_util.cpp test_smoke.cpp diff --git a/test/common/environment.h b/test/common/environment.h new file mode 100644 index 00000000..089904e5 --- /dev/null +++ b/test/common/environment.h @@ -0,0 +1,57 @@ +#pragma once + +#include + +#include +#include +#include + +namespace datadog::test { + +// For the lifetime of this object, set a specified environment variable. +// Restore any previous value (or unset the value if it was unset) afterward. +class EnvGuard { + std::string name_; + tracing::Optional former_value_; + + public: + EnvGuard(std::string name, std::string value) : name_(std::move(name)) { + const char* current = std::getenv(name_.c_str()); + if (current) { + former_value_ = current; + } + set_value(value); + } + + ~EnvGuard() { + if (former_value_) { + set_value(*former_value_); + } else { + unset(); + } + } + + void set_value(const std::string& value) { +#ifdef _MSC_VER + std::string envstr{name_}; + envstr += "="; + envstr += value; + assert(_putenv(envstr.c_str()) == 0); +#else + const bool overwrite = true; + ::setenv(name_.c_str(), value.c_str(), overwrite); +#endif + } + + void unset() { +#ifdef _MSC_VER + std::string envstr{name_}; + envstr += "="; + assert(_putenv(envstr.c_str()) == 0); +#else + ::unsetenv(name_.c_str()); +#endif + } +}; + +} // namespace datadog::test diff --git a/test/telemetry/test_configuration.cpp b/test/telemetry/test_configuration.cpp new file mode 100644 index 00000000..2c22cc11 --- /dev/null +++ b/test/telemetry/test_configuration.cpp @@ -0,0 +1,136 @@ +#include +#include + +#include + +#include "../common/environment.h" +#include "../test.h" + +#define TELEMETRY_CONFIGURATION_TEST(x) \ + TEST_CASE(x, "[telemetry.configuration]") + +namespace ddtest = datadog::test; + +using namespace datadog; +using namespace std::literals; + +TELEMETRY_CONFIGURATION_TEST("defaults") { + const auto cfg = telemetry::finalize_config(); + REQUIRE(cfg); + CHECK(cfg->debug == false); + CHECK(cfg->enabled == true); + CHECK(cfg->report_metrics == true); + CHECK(cfg->metrics_interval == 60s); + CHECK(cfg->heartbeat_interval == 10s); +} + +TELEMETRY_CONFIGURATION_TEST("code override") { + telemetry::Configuration cfg; + cfg.enabled = false; + cfg.report_metrics = false; + cfg.metrics_interval_seconds = 1; + cfg.heartbeat_interval_seconds = 2; + cfg.integration_name = "test"; + cfg.integration_version = "2024.10.28"; + + auto final_cfg = finalize_config(cfg); + REQUIRE(final_cfg); + CHECK(final_cfg->enabled == false); + CHECK(final_cfg->debug == false); + CHECK(final_cfg->report_metrics == false); + CHECK(final_cfg->metrics_interval == 1s); + CHECK(final_cfg->heartbeat_interval == 2s); + CHECK(final_cfg->integration_name == "test"); + CHECK(final_cfg->integration_version == "2024.10.28"); +} + +TELEMETRY_CONFIGURATION_TEST("enabled and report metrics precedence") { + SECTION("enabled takes precedence over metrics enabled") { + telemetry::Configuration cfg; + cfg.enabled = false; + cfg.report_metrics = true; + + auto final_cfg = finalize_config(cfg); + REQUIRE(final_cfg); + CHECK(final_cfg->enabled == false); + CHECK(final_cfg->report_metrics == false); + } +} + +TELEMETRY_CONFIGURATION_TEST("environment environment override") { + telemetry::Configuration cfg; + + SECTION("Override `enabled` field") { + cfg.enabled = true; + ddtest::EnvGuard env("DD_INSTRUMENTATION_TELEMETRY_ENABLED", "false"); + auto final_cfg = telemetry::finalize_config(cfg); + REQUIRE(final_cfg); + CHECK(final_cfg->enabled == false); + } + + SECTION("Override `debug` field") { + cfg.enabled = true; + ddtest::EnvGuard env("DD_TELEMETRY_DEBUG", "true"); + auto final_cfg = telemetry::finalize_config(cfg); + REQUIRE(final_cfg); + CHECK(final_cfg->debug == true); + } + + SECTION("Override `report_metrics` field") { + cfg.report_metrics = true; + ddtest::EnvGuard env("DD_TELEMETRY_METRICS_ENABLED", "false"); + auto final_cfg = telemetry::finalize_config(cfg); + REQUIRE(final_cfg); + CHECK(final_cfg->report_metrics == false); + } + + SECTION("Override metrics interval") { + cfg.metrics_interval_seconds = 88; + ddtest::EnvGuard env("DD_TELEMETRY_METRICS_INTERVAL_SECONDS", "15"); + auto final_cfg = telemetry::finalize_config(cfg); + REQUIRE(final_cfg); + CHECK(final_cfg->metrics_interval == 15s); + } + + SECTION("Override heartbeat interval") { + cfg.heartbeat_interval_seconds = 61; + ddtest::EnvGuard env("DD_TELEMETRY_HEARTBEAT_INTERVAL", "42"); + auto final_cfg = telemetry::finalize_config(cfg); + REQUIRE(final_cfg); + CHECK(final_cfg->heartbeat_interval == 42s); + } +} + +TELEMETRY_CONFIGURATION_TEST("validation") { + SECTION("metrics interval validation") { + SECTION("code override") { + telemetry::Configuration cfg; + cfg.metrics_interval_seconds = -15; + + auto final_cfg = telemetry::finalize_config(cfg); + REQUIRE(!final_cfg); + } + + SECTION("environment variable override") { + ddtest::EnvGuard env("DD_TELEMETRY_METRICS_INTERVAL_SECONDS", "-18"); + auto final_cfg = telemetry::finalize_config(); + REQUIRE(!final_cfg); + } + } + + SECTION("heartbeat interval validation") { + SECTION("code override") { + telemetry::Configuration cfg; + cfg.heartbeat_interval_seconds = -30; + + auto final_cfg = telemetry::finalize_config(cfg); + REQUIRE(!final_cfg); + } + + SECTION("environment variable override") { + ddtest::EnvGuard env("DD_TELEMETRY_METRICS_INTERVAL_SECONDS", "-42"); + auto final_cfg = telemetry::finalize_config(); + REQUIRE(!final_cfg); + } + } +} diff --git a/test/test_metrics.cpp b/test/telemetry/test_metrics.cpp similarity index 61% rename from test/test_metrics.cpp rename to test/telemetry/test_metrics.cpp index 4fc0f705..9c0c8ca0 100644 --- a/test/test_metrics.cpp +++ b/test/telemetry/test_metrics.cpp @@ -1,13 +1,14 @@ // This test covers operations defined for metrics defined in `metrics.h`. -#include +#include #include "test.h" -using namespace datadog::tracing; +using namespace datadog::telemetry; -TEST_CASE("Counter metrics") { - CounterMetric metric = {"test.counter.metric", {"testing-testing:123"}, true}; +TEST_CASE("Counter metrics", "[telemetry.metrics]") { + CounterMetric metric = { + "test.counter.metric", "test_scope", {"testing-testing:123"}, true}; metric.inc(); metric.add(41); @@ -17,8 +18,9 @@ TEST_CASE("Counter metrics") { REQUIRE(metric.value() == 0); } -TEST_CASE("Gauge metrics") { - GaugeMetric metric = {"test.gauge.metric", {"testing-testing:123"}, true}; +TEST_CASE("Gauge metrics", "[telemetry.metrics]") { + GaugeMetric metric = { + "test.gauge.metric", "test_scope", {"testing-testing:123"}, true}; metric.set(40); metric.inc(); metric.add(10); diff --git a/test/test_curl.cpp b/test/test_curl.cpp index ad9a2900..15c4bd63 100644 --- a/test/test_curl.cpp +++ b/test/test_curl.cpp @@ -151,7 +151,7 @@ TEST_CASE("parse response headers and body", "[curl]") { config.agent.http_client = client; // The http client is a mock that only expects a single request, so // force only tracing to be sent and exclude telemetry. - config.report_telemetry = false; + config.telemetry.enabled = false; const auto finalized = finalize_config(config); REQUIRE(finalized); diff --git a/test/test_datadog_agent.cpp b/test/test_datadog_agent.cpp index da3d489e..3cc125c4 100644 --- a/test/test_datadog_agent.cpp +++ b/test/test_datadog_agent.cpp @@ -28,7 +28,7 @@ TEST_CASE("CollectorResponse", "[datadog_agent]") { // Tests currently only cover sending traces to the agent. // Submiting telemetry performs essentially the same steps, but may be added // in the future. - config.report_telemetry = false; + config.telemetry.enabled = false; auto finalized = finalize_config(config); REQUIRE(finalized); @@ -189,7 +189,7 @@ TEST_CASE("Remote Configuration", "[datadog_agent]") { config.logger = logger; config.agent.event_scheduler = event_scheduler; config.agent.http_client = http_client; - config.report_telemetry = false; + config.telemetry.enabled = false; auto finalized = finalize_config(config); REQUIRE(finalized); @@ -197,7 +197,7 @@ TEST_CASE("Remote Configuration", "[datadog_agent]") { const TracerSignature signature(RuntimeID::generate(), "testsvc", "test"); auto telemetry = std::make_shared( - finalized->report_telemetry, finalized->clock, finalized->logger, + finalized->telemetry.enabled, finalized->clock, finalized->logger, signature, "", ""); auto config_manager = diff --git a/test/test_tracer_config.cpp b/test/test_tracer_config.cpp index 46f37406..44e0b3db 100644 --- a/test/test_tracer_config.cpp +++ b/test/test_tracer_config.cpp @@ -19,6 +19,7 @@ #include #include +#include "common/environment.h" #include "mocks/collectors.h" #include "mocks/event_schedulers.h" #include "mocks/loggers.h" @@ -34,56 +35,11 @@ std::ostream& operator<<(std::ostream& stream, PropagationStyle style) { } // namespace tracing } // namespace datadog +using namespace datadog::test; using namespace datadog::tracing; namespace { -// For the lifetime of this object, set a specified environment variable. -// Restore any previous value (or unset the value if it was unset) afterward. -class EnvGuard { - std::string name_; - Optional former_value_; - - public: - EnvGuard(std::string name, std::string value) : name_(std::move(name)) { - const char* current = std::getenv(name_.c_str()); - if (current) { - former_value_ = current; - } - set_value(value); - } - - ~EnvGuard() { - if (former_value_) { - set_value(*former_value_); - } else { - unset(); - } - } - - void set_value(const std::string& value) { -#ifdef _MSC_VER - std::string envstr{name_}; - envstr += "="; - envstr += value; - assert(_putenv(envstr.c_str()) == 0); -#else - const bool overwrite = true; - ::setenv(name_.c_str(), value.c_str(), overwrite); -#endif - } - - void unset() { -#ifdef _MSC_VER - std::string envstr{name_}; - envstr += "="; - assert(_putenv(envstr.c_str()) == 0); -#else - ::unsetenv(name_.c_str()); -#endif - } -}; - // For brevity when we're tabulating a lot of test cases with parse // `Optional<...>` data members. const auto x = nullopt; diff --git a/test/test_tracer_telemetry.cpp b/test/test_tracer_telemetry.cpp index 6b983fac..87bba73b 100644 --- a/test/test_tracer_telemetry.cpp +++ b/test/test_tracer_telemetry.cpp @@ -54,7 +54,7 @@ TEST_CASE("Tracer telemetry", "[telemetry]") { auto app_started = nlohmann::json::parse(app_started_message); REQUIRE(is_valid_telemetry_payload(app_started) == true); REQUIRE(app_started["request_type"] == "message-batch"); - REQUIRE(app_started["payload"].size() == 1); + REQUIRE(app_started["payload"].size() == 2); auto& app_started_payload = app_started["payload"][0]; CHECK(app_started_payload["request_type"] == "app-started"); @@ -89,7 +89,7 @@ TEST_CASE("Tracer telemetry", "[telemetry]") { REQUIRE(is_valid_telemetry_payload(app_started) == true); REQUIRE(app_started["request_type"] == "message-batch"); REQUIRE(app_started["payload"].is_array()); - REQUIRE(app_started["payload"].size() == 1); + REQUIRE(app_started["payload"].size() == 2); auto& app_started_payload = app_started["payload"][0]; CHECK(app_started_payload["request_type"] == "app-started");