Implement Git data upload (#5408)

communication/src/main/java/datadog/communication/http/OkHttpUtils.java

@@ -11,6 +11,7 @@
 import datadog.trace.util.AgentProxySelector;
 import java.io.File;
 import java.io.IOException;
+import java.net.ConnectException;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.nio.ByteBuffer;
@@ -26,6 +27,7 @@
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import okhttp3.RequestBody;
+import okhttp3.Response;
 import okio.BufferedSink;
 import okio.GzipSink;
 import okio.Okio;
@@ -300,4 +302,41 @@ public void writeTo(BufferedSink sink) throws IOException {
       gzipSink.close();
     }
   }
+
+  public static Response sendWithRetries(
+      OkHttpClient httpClient, HttpRetryPolicy retryPolicy, Request request) throws IOException {
+    while (true) {
+      try {
+        okhttp3.Response response = httpClient.newCall(request).execute();
+        if (response.isSuccessful()) {
+          return response;
+        }
+        if (!retryPolicy.shouldRetry(response)) {
+          return response;
+        } else {
+          closeQuietly(response);
+        }
+      } catch (ConnectException ex) {
+        if (!retryPolicy.shouldRetry(null)) {
+          throw ex;
+        }
+      }
+      // If we get here, there has been an error, and we still have retries left
+      long backoffMs = retryPolicy.backoff();
+      try {
+        Thread.sleep(backoffMs);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        throw new IOException(e);
+      }
+    }
+  }
+
+  private static void closeQuietly(Response response) {
+    try {
+      response.close();
+    } catch (Exception e) {
+      // ignore
+    }
+  }
 }

dd-java-agent/agent-ci-visibility/build.gradle

@@ -17,6 +17,7 @@ excludedClassesCoverage += [
   "datadog.trace.civisibility.DDTestImpl",
   "datadog.trace.civisibility.ci.CIInfo",
   "datadog.trace.civisibility.ci.CIInfo.Builder",
+  "datadog.trace.civisibility.communication.*",
   "datadog.trace.civisibility.context.AbstractTestContext",
   "datadog.trace.civisibility.context.EmptyTestContext",
   "datadog.trace.civisibility.context.ParentProcessTestContext",
@@ -29,7 +30,10 @@ excludedClassesCoverage += [
   "datadog.trace.civisibility.events.TestModuleDescriptor",
   "datadog.trace.civisibility.events.TestSuiteDescriptor",
   "datadog.trace.civisibility.git.GitObject",
+  "datadog.trace.civisibility.git.tree.*",
   "datadog.trace.civisibility.source.MethodLinesResolver.MethodLines",
+  "datadog.trace.civisibility.utils.ShellCommandExecutor",
+  "datadog.trace.civisibility.utils.ShellCommandExecutor.OutputParser",
 ]
 
 dependencies {
@@ -39,6 +43,7 @@ dependencies {
   implementation deps.asmcommons
   implementation group: 'org.jacoco', name: 'org.jacoco.core', version: '0.8.9'
 
+  implementation project(':communication')
   implementation project(':internal-api')
   implementation project(':internal-api:internal-api-9')
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApi.java

@@ -0,0 +1,13 @@
+package datadog.trace.civisibility.communication;
+
+import datadog.trace.civisibility.utils.IOThrowingFunction;
+import java.io.IOException;
+import java.io.InputStream;
+import okhttp3.RequestBody;
+
+/** API for posting HTTP requests to backend */
+public interface BackendApi {
+
+  <T> T post(String uri, RequestBody requestBody, IOThrowingFunction<InputStream, T> responseParser)
+      throws IOException;
+}

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java

@@ -0,0 +1,57 @@
+package datadog.trace.civisibility.communication;
+
+import datadog.communication.ddagent.DDAgentFeaturesDiscovery;
+import datadog.communication.ddagent.SharedCommunicationObjects;
+import datadog.communication.http.HttpRetryPolicy;
+import datadog.trace.api.Config;
+import datadog.trace.util.throwable.FatalAgentMisconfigurationError;
+import javax.annotation.Nullable;
+import okhttp3.HttpUrl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class BackendApiFactory {
+
+  private static final Logger log = LoggerFactory.getLogger(BackendApiFactory.class);
+
+  private final Config config;
+  private final SharedCommunicationObjects sharedCommunicationObjects;
+
+  public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommunicationObjects) {
+    this.config = config;
+    this.sharedCommunicationObjects = sharedCommunicationObjects;
+  }
+
+  public @Nullable BackendApi createBackendApi() {
+    long timeoutMillis = config.getCiVisibilityBackendApiTimeoutMillis();
+    HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0);
+
+    if (config.isCiVisibilityAgentlessEnabled()) {
+      String site = config.getSite();
+      String apiKey = config.getApiKey();
+      if (apiKey == null || apiKey.isEmpty()) {
+        throw new FatalAgentMisconfigurationError(
+            "Agentless mode is enabled and api key is not set. Please set application key");
+      }
+      String applicationKey = config.getApplicationKey();
+      if (applicationKey == null || applicationKey.isEmpty()) {
+        log.warn(
+            "Agentless mode is enabled and application key is not set. Some CI Visibility features will be unavailable");
+      }
+      return new IntakeApi(site, apiKey, applicationKey, timeoutMillis, retryPolicyFactory);
+    }
+
+    DDAgentFeaturesDiscovery featuresDiscovery =
+        sharedCommunicationObjects.featuresDiscovery(config);
+    if (featuresDiscovery.supportsEvpProxy()) {
+      String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint();
+      HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint);
+      return new EvpProxyApi(evpProxyUrl, timeoutMillis, retryPolicyFactory);
+    }
+
+    log.warn(
+        "Cannot create backend API client since agentless mode is disabled, "
+            + "and agent does not support EVP proxy");
+    return null;
+  }
+}

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java

@@ -0,0 +1,66 @@
+package datadog.trace.civisibility.communication;
+
+import datadog.communication.http.HttpRetryPolicy;
+import datadog.communication.http.OkHttpUtils;
+import datadog.trace.civisibility.utils.IOThrowingFunction;
+import java.io.IOException;
+import java.io.InputStream;
+import okhttp3.HttpUrl;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** API that uses DD Agent as a proxy to post request to backend. */
+public class EvpProxyApi implements BackendApi {
+
+  private static final Logger log = LoggerFactory.getLogger(EvpProxyApi.class);
+
+  private static final String API_VERSION = "v2";
+  private static final String X_DATADOG_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain";
+  private static final String API_SUBDOMAIN = "api";
+
+  private final HttpRetryPolicy.Factory retryPolicyFactory;
+  private final HttpUrl evpProxyUrl;
+  private final OkHttpClient httpClient;
+
+  public EvpProxyApi(
+      HttpUrl evpProxyUrl, long timeoutMillis, HttpRetryPolicy.Factory retryPolicyFactory) {
+    this.evpProxyUrl = evpProxyUrl.resolve(String.format("api/%s/", API_VERSION));
+    this.retryPolicyFactory = retryPolicyFactory;
+    httpClient = OkHttpUtils.buildHttpClient(evpProxyUrl, timeoutMillis);
+  }
+
+  @Override
+  public <T> T post(
+      String uri, RequestBody requestBody, IOThrowingFunction<InputStream, T> responseParser)
+      throws IOException {
+    final HttpUrl url = evpProxyUrl.resolve(uri);
+    final Request request =
+        new Request.Builder()
+            .url(url)
+            .addHeader(X_DATADOG_EVP_SUBDOMAIN_HEADER, API_SUBDOMAIN)
+            .post(requestBody)
+            .build();
+
+    HttpRetryPolicy retryPolicy = retryPolicyFactory.create();
+    try (okhttp3.Response response =
+        OkHttpUtils.sendWithRetries(httpClient, retryPolicy, request)) {
+      if (response.isSuccessful()) {
+        log.debug("Request to {} returned successful response: {}", uri, response.code());
+        return responseParser.apply(response.body().byteStream());
+      } else {
+        throw new IOException(
+            "Request to "
+                + uri
+                + " returned error response "
+                + response.code()
+                + ": "
+                + response.message()
+                + "; "
+                + (response.body() != null ? response.body().string() : ""));
+      }
+    }
+  }
+}

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java

@@ -0,0 +1,83 @@
+package datadog.trace.civisibility.communication;
+
+import datadog.communication.http.HttpRetryPolicy;
+import datadog.communication.http.OkHttpUtils;
+import datadog.trace.api.Config;
+import datadog.trace.civisibility.utils.IOThrowingFunction;
+import java.io.IOException;
+import java.io.InputStream;
+import okhttp3.HttpUrl;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** API for posting HTTP requests directly to backend, without the need for DD Agent */
+public class IntakeApi implements BackendApi {
+
+  private static final Logger log = LoggerFactory.getLogger(IntakeApi.class);
+
+  private static final String API_VERSION = "v2";
+  private static final String DD_API_KEY_HEADER = "dd-api-key";
+  private static final String DD_APPLICATION_KEY_HEADER = "dd-application-key";
+
+  private final String apiKey;
+  private final String applicationKey;
+  private final HttpRetryPolicy.Factory retryPolicyFactory;
+  private final HttpUrl hostUrl;
+  private final OkHttpClient httpClient;
+
+  public IntakeApi(
+      String site,
+      String apiKey,
+      String applicationKey,
+      long timeoutMillis,
+      HttpRetryPolicy.Factory retryPolicyFactory) {
+    this.apiKey = apiKey;
+    this.applicationKey = applicationKey;
+    this.retryPolicyFactory = retryPolicyFactory;
+
+    final String ciVisibilityAgentlessUrlStr = Config.get().getCiVisibilityAgentlessUrl();
+    if (ciVisibilityAgentlessUrlStr != null && !ciVisibilityAgentlessUrlStr.isEmpty()) {
+      hostUrl = HttpUrl.get(String.format("%s/api/%s/", ciVisibilityAgentlessUrlStr, API_VERSION));
+    } else {
+      hostUrl = HttpUrl.get(String.format("https://api.%s/api/%s/", site, API_VERSION));
+    }
+
+    httpClient = OkHttpUtils.buildHttpClient(hostUrl, timeoutMillis);
+  }
+
+  @Override
+  public <T> T post(
+      String uri, RequestBody requestBody, IOThrowingFunction<InputStream, T> responseParser)
+      throws IOException {
+    HttpUrl url = hostUrl.resolve(uri);
+    Request.Builder requestBuilder =
+        new Request.Builder().url(url).post(requestBody).addHeader(DD_API_KEY_HEADER, apiKey);
+
+    if (applicationKey != null) {
+      requestBuilder.addHeader(DD_APPLICATION_KEY_HEADER, applicationKey);
+    }
+
+    Request request = requestBuilder.build();
+    HttpRetryPolicy retryPolicy = retryPolicyFactory.create();
+    try (okhttp3.Response response =
+        OkHttpUtils.sendWithRetries(httpClient, retryPolicy, request)) {
+      if (response.isSuccessful()) {
+        log.debug("Request to {} returned successful response: {}", uri, response.code());
+        return responseParser.apply(response.body().byteStream());
+      } else {
+        throw new IOException(
+            "Request to "
+                + uri
+                + " returned error response "
+                + response.code()
+                + ": "
+                + response.message()
+                + "; "
+                + (response.body() != null ? response.body().string() : ""));
+      }
+    }
+  }
+}