Merge branch 'master' into joey/aws-sns

DataDog · May 30, 2024 · 7e2774e · 7e2774e
2 parents c255ef1 + c2fa83f
commit 7e2774e
Show file tree

Hide file tree

Showing 70 changed files with 1,407 additions and 463 deletions.
diff --git a/communication/build.gradle b/communication/build.gradle
@@ -41,12 +41,29 @@ ext {
     'datadog.communication.monitor.DDAgentStatsDConnection',
     'datadog.communication.monitor.DDAgentStatsDConnection.*',
     'datadog.communication.monitor.LoggingStatsDClient',
+    'datadog.communication.BackendApiFactory',
+    'datadog.communication.BackendApiFactory.Intake',
+    'datadog.communication.EvpProxyApi',
+    'datadog.communication.IntakeApi',
+    'datadog.communication.util.IOUtils',
+    'datadog.communication.util.IOUtils.1',
+  ]
+  excludedClassesBranchCoverage = [
+    'datadog.communication.ddagent.TracerVersion',
+    'datadog.communication.BackendApiFactory',
+    'datadog.communication.EvpProxyApi',
+    'datadog.communication.IntakeApi',
   ]
-  excludedClassesBranchCoverage = ['datadog.communication.ddagent.TracerVersion',]
   excludedClassesInstructionCoverage = [
     // can't reach the error condition now
     'datadog.communication.fleet.FleetServiceImpl',
     'datadog.communication.ddagent.SharedCommunicationObjects',
     'datadog.communication.ddagent.TracerVersion',
+    'datadog.communication.BackendApiFactory',
+    'datadog.communication.BackendApiFactory.Intake',
+    'datadog.communication.EvpProxyApi',
+    'datadog.communication.IntakeApi',
+    'datadog.communication.util.IOUtils',
+    'datadog.communication.util.IOUtils.1',
   ]
 }
diff --git a/...ivisibility/communication/BackendApi.java → ...ava/datadog/communication/BackendApi.java b/...ivisibility/communication/BackendApi.java → ...ava/datadog/communication/BackendApi.java
@@ -1,7 +1,7 @@
-package datadog.trace.civisibility.communication;
+package datadog.communication;
 
 import datadog.communication.http.OkHttpUtils;
-import datadog.trace.civisibility.utils.IOThrowingFunction;
+import datadog.communication.util.IOThrowingFunction;
 import java.io.IOException;
 import java.io.InputStream;
 import javax.annotation.Nullable;
@@ -14,6 +14,7 @@ <T> T post(
       String uri,
       RequestBody requestBody,
       IOThrowingFunction<InputStream, T> responseParser,
-      @Nullable OkHttpUtils.CustomListener requestListener)
+      @Nullable OkHttpUtils.CustomListener requestListener,
+      boolean requestCompression)
       throws IOException;
 }
diff --git a/...lity/communication/BackendApiFactory.java → ...adog/communication/BackendApiFactory.java b/...lity/communication/BackendApiFactory.java → ...adog/communication/BackendApiFactory.java
@@ -1,10 +1,11 @@
-package datadog.trace.civisibility.communication;
+package datadog.communication;
 
 import datadog.communication.ddagent.DDAgentFeaturesDiscovery;
 import datadog.communication.ddagent.SharedCommunicationObjects;
 import datadog.communication.http.HttpRetryPolicy;
 import datadog.trace.api.Config;
 import datadog.trace.util.throwable.FatalAgentMisconfigurationError;
+import java.util.function.Function;
 import javax.annotation.Nullable;
 import okhttp3.HttpUrl;
 import org.slf4j.Logger;
@@ -22,19 +23,19 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
     this.sharedCommunicationObjects = sharedCommunicationObjects;
   }
 
-  public @Nullable BackendApi createBackendApi() {
+  public @Nullable BackendApi createBackendApi(Intake intake) {
     HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true);
 
-    if (config.isCiVisibilityAgentlessEnabled()) {
-      HttpUrl agentlessUrl = getAgentlessUrl();
+    if (intake.agentlessModeEnabled.apply(config)) {
+      HttpUrl agentlessUrl = getAgentlessUrl(intake);
       String apiKey = config.getApiKey();
       if (apiKey == null || apiKey.isEmpty()) {
         throw new FatalAgentMisconfigurationError(
             "Agentless mode is enabled and api key is not set. Please set application key");
       }
       String traceId = config.getIdGenerationStrategy().generateTraceId().toString();
       long timeoutMillis = config.getCiVisibilityBackendApiTimeoutMillis();
-      return new IntakeApi(agentlessUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory);
+      return new IntakeApi(agentlessUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory, true);
     }
 
     DDAgentFeaturesDiscovery featuresDiscovery =
@@ -45,7 +46,7 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
       String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint();
       HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint);
       return new EvpProxyApi(
-          traceId, evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient);
+          traceId, evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient, true);
     }
 
     log.warn(
@@ -54,14 +55,34 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
     return null;
   }
 
-  private HttpUrl getAgentlessUrl() {
-    final String ciVisibilityAgentlessUrlStr = config.getCiVisibilityAgentlessUrl();
-    if (ciVisibilityAgentlessUrlStr != null && !ciVisibilityAgentlessUrlStr.isEmpty()) {
-      return HttpUrl.get(
-          String.format("%s/api/%s/", ciVisibilityAgentlessUrlStr, IntakeApi.API_VERSION));
+  private HttpUrl getAgentlessUrl(Intake intake) {
+    String customUrl = intake.customUrl.apply(config);
+    if (customUrl != null && !customUrl.isEmpty()) {
+      return HttpUrl.get(String.format("%s/api/%s/", customUrl, intake.version));
     } else {
       String site = config.getSite();
-      return HttpUrl.get(String.format("https://api.%s/api/%s/", site, IntakeApi.API_VERSION));
+      return HttpUrl.get(
+          String.format("https://%s.%s/api/%s/", intake.urlPrefix, site, intake.version));
+    }
+  }
+
+  public enum Intake {
+    API("api", "v2", Config::isCiVisibilityAgentlessEnabled, Config::getCiVisibilityAgentlessUrl);
+
+    public final String urlPrefix;
+    public final String version;
+    public final Function<Config, Boolean> agentlessModeEnabled;
+    public final Function<Config, String> customUrl;
+
+    Intake(
+        String urlPrefix,
+        String version,
+        Function<Config, Boolean> agentlessModeEnabled,
+        Function<Config, String> customUrl) {
+      this.urlPrefix = urlPrefix;
+      this.version = version;
+      this.agentlessModeEnabled = agentlessModeEnabled;
+      this.customUrl = customUrl;
     }
   }
 }
diff --git a/...visibility/communication/EvpProxyApi.java → ...va/datadog/communication/EvpProxyApi.java b/...visibility/communication/EvpProxyApi.java → ...va/datadog/communication/EvpProxyApi.java
@@ -1,8 +1,8 @@
-package datadog.trace.civisibility.communication;
+package datadog.communication;
 
 import datadog.communication.http.HttpRetryPolicy;
 import datadog.communication.http.OkHttpUtils;
-import datadog.trace.civisibility.utils.IOThrowingFunction;
+import datadog.communication.util.IOThrowingFunction;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.zip.GZIPInputStream;
@@ -32,35 +32,28 @@ public class EvpProxyApi implements BackendApi {
   private final HttpRetryPolicy.Factory retryPolicyFactory;
   private final HttpUrl evpProxyUrl;
   private final OkHttpClient httpClient;
-  private final boolean gzipEnabled;
-
-  public EvpProxyApi(
-      String traceId,
-      HttpUrl evpProxyUrl,
-      HttpRetryPolicy.Factory retryPolicyFactory,
-      OkHttpClient httpClient) {
-    this(traceId, evpProxyUrl, retryPolicyFactory, httpClient, true);
-  }
+  private final boolean responseCompression;
 
   public EvpProxyApi(
       String traceId,
       HttpUrl evpProxyUrl,
       HttpRetryPolicy.Factory retryPolicyFactory,
       OkHttpClient httpClient,
-      boolean gzipEnabled) {
+      boolean responseCompression) {
     this.traceId = traceId;
     this.evpProxyUrl = evpProxyUrl.resolve(String.format("api/%s/", API_VERSION));
     this.retryPolicyFactory = retryPolicyFactory;
     this.httpClient = httpClient;
-    this.gzipEnabled = gzipEnabled;
+    this.responseCompression = responseCompression;
   }
 
   @Override
   public <T> T post(
       String uri,
       RequestBody requestBody,
       IOThrowingFunction<InputStream, T> responseParser,
-      @Nullable OkHttpUtils.CustomListener requestListener)
+      @Nullable OkHttpUtils.CustomListener requestListener,
+      boolean requestCompression)
       throws IOException {
     final HttpUrl url = evpProxyUrl.resolve(uri);
 
@@ -75,7 +68,11 @@ public <T> T post(
       requestBuilder.tag(OkHttpUtils.CustomListener.class, requestListener);
     }
 
-    if (gzipEnabled) {
+    if (requestCompression) {
+      requestBuilder.addHeader(CONTENT_ENCODING_HEADER, GZIP_ENCODING);
+    }
+
+    if (responseCompression) {
       requestBuilder.addHeader(ACCEPT_ENCODING_HEADER, GZIP_ENCODING);
     }
 

diff --git a/...civisibility/communication/IntakeApi.java → ...java/datadog/communication/IntakeApi.java b/...civisibility/communication/IntakeApi.java → ...java/datadog/communication/IntakeApi.java
@@ -1,8 +1,8 @@
-package datadog.trace.civisibility.communication;
+package datadog.communication;
 
 import datadog.communication.http.HttpRetryPolicy;
 import datadog.communication.http.OkHttpUtils;
-import datadog.trace.civisibility.utils.IOThrowingFunction;
+import datadog.communication.util.IOThrowingFunction;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.zip.GZIPInputStream;
@@ -19,8 +19,6 @@ public class IntakeApi implements BackendApi {
 
   private static final Logger log = LoggerFactory.getLogger(IntakeApi.class);
 
-  public static final String API_VERSION = "v2";
-
   private static final String DD_API_KEY_HEADER = "dd-api-key";
   private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id";
   private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id";
@@ -31,31 +29,22 @@ public class IntakeApi implements BackendApi {
   private final String apiKey;
   private final String traceId;
   private final HttpRetryPolicy.Factory retryPolicyFactory;
-  private final boolean gzipEnabled;
+  private final boolean responseCompression;
   private final HttpUrl hostUrl;
   private final OkHttpClient httpClient;
 
-  public IntakeApi(
-      HttpUrl hostUrl,
-      String apiKey,
-      String traceId,
-      long timeoutMillis,
-      HttpRetryPolicy.Factory retryPolicyFactory) {
-    this(hostUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory, true);
-  }
-
   public IntakeApi(
       HttpUrl hostUrl,
       String apiKey,
       String traceId,
       long timeoutMillis,
       HttpRetryPolicy.Factory retryPolicyFactory,
-      boolean gzipEnabled) {
+      boolean responseCompression) {
     this.hostUrl = hostUrl;
     this.apiKey = apiKey;
     this.traceId = traceId;
     this.retryPolicyFactory = retryPolicyFactory;
-    this.gzipEnabled = gzipEnabled;
+    this.responseCompression = responseCompression;
 
     httpClient = OkHttpUtils.buildHttpClient(hostUrl, timeoutMillis);
   }
@@ -65,7 +54,8 @@ public <T> T post(
       String uri,
       RequestBody requestBody,
       IOThrowingFunction<InputStream, T> responseParser,
-      @Nullable OkHttpUtils.CustomListener requestListener)
+      @Nullable OkHttpUtils.CustomListener requestListener,
+      boolean requestCompression)
       throws IOException {
     HttpUrl url = hostUrl.resolve(uri);
     Request.Builder requestBuilder =
@@ -80,7 +70,11 @@ public <T> T post(
       requestBuilder.tag(OkHttpUtils.CustomListener.class, requestListener);
     }
 
-    if (gzipEnabled) {
+    if (requestCompression) {
+      requestBuilder.addHeader(CONTENT_ENCODING_HEADER, GZIP_ENCODING);
+    }
+
+    if (responseCompression) {
       requestBuilder.addHeader(ACCEPT_ENCODING_HEADER, GZIP_ENCODING);
     }
 

diff --git a/communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java b/communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java
@@ -29,7 +29,7 @@ public void createRemaining(Config config) {
       monitoring = Monitoring.DISABLED;
     }
     if (agentUrl == null) {
-      agentUrl = HttpUrl.parse(config.getAgentUrl());
+      agentUrl = parseAgentUrl(config);
       if (agentUrl == null) {
         throw new IllegalArgumentException("Bad agent URL: " + config.getAgentUrl());
       }
@@ -43,6 +43,15 @@ public void createRemaining(Config config) {
     }
   }
 
+  private static HttpUrl parseAgentUrl(Config config) {
+    String agentUrl = config.getAgentUrl();
+    if (agentUrl.startsWith("unix:")) {
+      // provide placeholder agent URL, in practice we'll be tunnelling over UDS
+      agentUrl = "http://" + config.getAgentHost() + ":" + config.getAgentPort();
+    }
+    return HttpUrl.parse(agentUrl);
+  }
+
   private static long getHttpClientTimeout(Config config) {
     if (!config.isCiVisibilityEnabled()) {
       return TimeUnit.SECONDS.toMillis(config.getAgentTimeout());

diff --git a/communication/src/main/java/datadog/communication/monitor/DDAgentStatsDConnection.java b/communication/src/main/java/datadog/communication/monitor/DDAgentStatsDConnection.java
@@ -5,7 +5,7 @@
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
 
-import com.timgroup.statsd.NoOpStatsDClient;
+import com.timgroup.statsd.NoOpDirectStatsDClient;
 import com.timgroup.statsd.NonBlockingStatsDClientBuilder;
 import com.timgroup.statsd.StatsDClientErrorHandler;
 import datadog.trace.api.Config;
@@ -23,7 +23,7 @@ final class DDAgentStatsDConnection implements StatsDClientErrorHandler {
   private static final Logger log = LoggerFactory.getLogger(DDAgentStatsDConnection.class);
   private static final IOLogger ioLogger = new IOLogger(log);
 
-  private static final com.timgroup.statsd.StatsDClient NO_OP = new NoOpStatsDClient();
+  private static final com.timgroup.statsd.StatsDClient NO_OP = new NoOpDirectStatsDClient();
 
   private static final String UNIX_DOMAIN_SOCKET_PREFIX = "unix://";
 

diff --git a/...ivisibility/utils/IOThrowingFunction.java → ...ommunication/util/IOThrowingFunction.java b/...ivisibility/utils/IOThrowingFunction.java → ...ommunication/util/IOThrowingFunction.java
@@ -1,4 +1,4 @@
-package datadog.trace.civisibility.utils;
+package datadog.communication.util;
 
 import java.io.IOException;
 

diff --git a/...dog/trace/civisibility/utils/IOUtils.java → ...a/datadog/communication/util/IOUtils.java b/...dog/trace/civisibility/utils/IOUtils.java → ...a/datadog/communication/util/IOUtils.java
@@ -1,4 +1,4 @@
-package datadog.trace.civisibility.utils;
+package datadog.communication.util;
 
 import edu.umd.cs.findbugs.annotations.NonNull;
 import java.io.BufferedReader;

diff --git a/dd-java-agent/agent-builder/build.gradle b/dd-java-agent/agent-builder/build.gradle
@@ -7,6 +7,8 @@ excludedClassesCoverage += ['datadog.trace.agent.tooling.*']
 dependencies {
   api project(':dd-java-agent:agent-tooling')
 
+  implementation project(':dd-java-agent:agent-otel:otel-tooling')
+
   testImplementation project(':dd-java-agent:testing')
 }
 

diff --git a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/ExtensionFinder.java b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/ExtensionFinder.java
@@ -1,5 +1,6 @@
 package datadog.trace.agent.tooling;
 
+import static datadog.opentelemetry.tooling.OtelExtensionHandler.OPENTELEMETRY;
 import static datadog.trace.agent.tooling.ExtensionHandler.DATADOG;
 
 import de.thetaphi.forbiddenapis.SuppressForbidden;
@@ -23,7 +24,7 @@
 public final class ExtensionFinder {
   private static final Logger log = LoggerFactory.getLogger(ExtensionFinder.class);
 
-  private static final ExtensionHandler[] handlers = {DATADOG};
+  private static final ExtensionHandler[] handlers = {OPENTELEMETRY, DATADOG};
 
   /**
    * Discovers extensions on the configured path and creates a classloader for each extension.

diff --git a/...gent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java b/...gent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java
@@ -1,5 +1,6 @@
 package datadog.trace.civisibility;
 
+import datadog.communication.BackendApi;
 import datadog.trace.api.Config;
 import datadog.trace.api.civisibility.config.ModuleExecutionSettings;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
@@ -10,7 +11,6 @@
 import datadog.trace.civisibility.codeowners.Codeowners;
 import datadog.trace.civisibility.codeowners.CodeownersProvider;
 import datadog.trace.civisibility.codeowners.NoCodeowners;
-import datadog.trace.civisibility.communication.BackendApi;
 import datadog.trace.civisibility.config.CachingModuleExecutionSettingsFactory;
 import datadog.trace.civisibility.config.ConfigurationApi;
 import datadog.trace.civisibility.config.ConfigurationApiImpl;

diff --git a/...nt/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java b/...nt/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java
@@ -1,12 +1,12 @@
 package datadog.trace.civisibility;
 
+import datadog.communication.BackendApi;
+import datadog.communication.BackendApiFactory;
 import datadog.communication.ddagent.SharedCommunicationObjects;
 import datadog.trace.api.Config;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
 import datadog.trace.api.git.GitInfoProvider;
 import datadog.trace.civisibility.ci.CIProviderInfoFactory;
-import datadog.trace.civisibility.communication.BackendApi;
-import datadog.trace.civisibility.communication.BackendApiFactory;
 import datadog.trace.civisibility.config.CachingJvmInfoFactory;
 import datadog.trace.civisibility.config.JvmInfoFactory;
 import datadog.trace.civisibility.config.JvmInfoFactoryImpl;
@@ -61,7 +61,8 @@ public class CiVisibilityServices {
       GitInfoProvider gitInfoProvider) {
     this.config = config;
     this.metricCollector = metricCollector;
-    this.backendApi = new BackendApiFactory(config, sco).createBackendApi();
+    this.backendApi =
+        new BackendApiFactory(config, sco).createBackendApi(BackendApiFactory.Intake.API);
     this.jvmInfoFactory = new CachingJvmInfoFactory(config, new JvmInfoFactoryImpl());
     this.gitClientFactory = new GitClient.Factory(config, metricCollector);
     this.ciProviderInfoFactory = new CIProviderInfoFactory(config);