Cocurrency otlp http session (open-telemetry#1317)

owent · May 11, 2022 · 08a12b5 · 08a12b5
1 parent c614258
commit 08a12b5
Show file tree

Hide file tree

Showing 17 changed files with 1,805 additions and 547 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,8 @@ Increment the:
 
 * [SDK] Async Batch Span/Log processor with max async support ([#1306](https://github.com/open-telemetry/opentelemetry-cpp/pull/1306))
 * [EXPORTER] OTLP http exporter allow concurrency session ([#1209](https://github.com/open-telemetry/opentelemetry-cpp/pull/1209))
+* [EXT] `curl::HttpClient` use `curl_multi_handle` instead of creating a thread
+  for every request and it's able to reuse connections now. ([#1317](https://github.com/open-telemetry/opentelemetry-cpp/pull/1317))
 
 ## [1.3.0] 2022-04-11
 

diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h
@@ -79,7 +79,10 @@ struct OtlpHttpClientOptions
   OtlpHeaders http_headers = GetOtlpDefaultHeaders();
 
   // Concurrent requests
-  std::size_t max_concurrent_requests = 8;
+  std::size_t max_concurrent_requests = 64;
+
+  // Concurrent requests
+  std::size_t max_requests_per_connection = 8;
 
   inline OtlpHttpClientOptions(nostd::string_view input_url,
                                HttpRequestContentType input_content_type,
@@ -88,15 +91,17 @@ struct OtlpHttpClientOptions
                                bool input_console_debug,
                                std::chrono::system_clock::duration input_timeout,
                                const OtlpHeaders &input_http_headers,
-                               std::size_t input_concurrent_sessions = 8)
+                               std::size_t input_concurrent_sessions         = 64,
+                               std::size_t input_max_requests_per_connection = 8)
       : url(input_url),
         content_type(input_content_type),
         json_bytes_mapping(input_json_bytes_mapping),
         use_json_name(input_use_json_name),
         console_debug(input_console_debug),
         timeout(input_timeout),
         http_headers(input_http_headers),
-        max_concurrent_requests(input_concurrent_sessions)
+        max_concurrent_requests(input_concurrent_sessions),
+        max_requests_per_connection(input_max_requests_per_connection)
   {}
 };
 

diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter.h
@@ -55,7 +55,10 @@ struct OtlpHttpExporterOptions
 #ifdef ENABLE_ASYNC_EXPORT
   // Concurrent requests
   // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlpgrpc-concurrent-requests
-  std::size_t max_concurrent_requests = 8;
+  std::size_t max_concurrent_requests = 64;
+
+  // Concurrent requests
+  std::size_t max_requests_per_connection = 8;
 #endif
 };
 

diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_exporter.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_exporter.h
@@ -55,7 +55,10 @@ struct OtlpHttpLogExporterOptions
 #  ifdef ENABLE_ASYNC_EXPORT
   // Concurrent requests
   // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlpgrpc-concurrent-requests
-  std::size_t max_concurrent_requests = 8;
+  std::size_t max_concurrent_requests = 64;
+
+  // Concurrent requests
+  std::size_t max_requests_per_connection = 8;
 #  endif
 };
 

diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc
@@ -649,7 +649,9 @@ void ConvertListFieldToJson(nlohmann::json &value,
 
 OtlpHttpClient::OtlpHttpClient(OtlpHttpClientOptions &&options)
     : is_shutdown_(false), options_(options), http_client_(http_client::HttpClientFactory::Create())
-{}
+{
+  http_client_->SetMaxSessionsPerConnection(options_.max_requests_per_connection);
+}
 
 OtlpHttpClient::~OtlpHttpClient()
 {
@@ -682,7 +684,9 @@ OtlpHttpClient::~OtlpHttpClient()
 OtlpHttpClient::OtlpHttpClient(OtlpHttpClientOptions &&options,
                                std::shared_ptr<ext::http::client::HttpClient> http_client)
     : is_shutdown_(false), options_(options), http_client_(http_client)
-{}
+{
+  http_client_->SetMaxSessionsPerConnection(options_.max_requests_per_connection);
+}
 
 // ----------------------------- HTTP Client methods ------------------------------
 opentelemetry::sdk::common::ExportResult OtlpHttpClient::Export(

diff --git a/exporters/otlp/src/otlp_http_exporter.cc b/exporters/otlp/src/otlp_http_exporter.cc
@@ -34,7 +34,8 @@ OtlpHttpExporter::OtlpHttpExporter(const OtlpHttpExporterOptions &options)
                                                             options.http_headers
 #ifdef ENABLE_ASYNC_EXPORT
                                                             ,
-                                                            options.max_concurrent_requests
+                                                            options.max_concurrent_requests,
+                                                            options.max_requests_per_connection
 #endif
                                                             )))
 {}

diff --git a/exporters/otlp/src/otlp_http_log_exporter.cc b/exporters/otlp/src/otlp_http_log_exporter.cc
@@ -36,7 +36,8 @@ OtlpHttpLogExporter::OtlpHttpLogExporter(const OtlpHttpLogExporterOptions &optio
                                                             options.http_headers
 #  ifdef ENABLE_ASYNC_EXPORT
                                                             ,
-                                                            options.max_concurrent_requests
+                                                            options.max_concurrent_requests,
+                                                            options.max_requests_per_connection
 #  endif
                                                             )))
 {}