Merge branch 'master' into andrew/lib-injection-images

.gitlab/benchmarks.yml

@@ -10,6 +10,7 @@ variables:
   image: $BASE_CI_IMAGE
   script:
     - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
+    - export CIRCLE_CI_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.circleci_token --with-decryption --query "Parameter.Value" --out text)
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
     - git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
   artifacts:

.gitlab/single-step-instrumentation-tests.yml

@@ -10,6 +10,7 @@
     - export DD_APP_KEY_ONBOARDING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.dd-app-key-onboarding --with-decryption --query "Parameter.Value" --out text)
     - export ONBOARDING_AWS_INFRA_SUBNET_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.aws-infra-subnet-id --with-decryption --query "Parameter.Value" --out text)
     - export ONBOARDING_AWS_INFRA_SECURITY_GROUPS_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.aws-infra-securiy-groups-id --with-decryption --query "Parameter.Value" --out text)
+    - export ONBOARDING_AWS_INFRA_IAM_INSTANCE_PROFILE=ec2InstanceRole
     - export PULUMI_CONFIG_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.pulumi-config-passphrase --with-decryption --query "Parameter.Value" --out text) 
     #Install plugins for PULUMI you need connect to gh. Sometimes this problem arises: GitHub rate limit exceeded
     - export GITHUB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.gh-token --with-decryption --query "Parameter.Value" --out text) 
@@ -33,10 +34,34 @@
       paths:
         - system-tests/reports/
 
+oci-internal-publish:
+  extends: .oci-internal-publish
+  stage: package
+  needs: [ package-oci ]
+  rules:
+    - when: on_success
+  variables: 
+    FLAVOR: datadog-apm-library-java
+
+oci-internal-test-ecr-publish:
+  stage: package
+  needs: [ oci-internal-publish ]
+  rules:
+    - when: on_success
+  trigger:
+    project: DataDog/public-images
+    branch: main
+    strategy: depend
+  variables:
+    IMG_SOURCES: registry.ddbuild.io/ci/remote-updates/datadog-apm-library-java:pipeline-${CI_PIPELINE_ID}-1
+    IMG_DESTINATIONS: apm-library-java-package:pipeline-${CI_PIPELINE_ID}
+    IMG_REGISTRIES: agent-qa
+
+
 onboarding_tests:
   extends: .base_job_onboarding_tests
   stage: single-step-instrumentation-tests
-  needs: [ package, package-arm]
+  needs: [ package, package-arm, oci-internal-test-ecr-publish ]
   rules:
     - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "master"
       when: on_success
@@ -50,11 +75,14 @@ onboarding_tests:
           SCENARIO: [SIMPLE_HOST_AUTO_INJECTION]
         - ONBOARDING_FILTER_WEBLOG: [test-app-java-container,test-app-java-buildpack,test-app-java-alpine]
           SCENARIO: [SIMPLE_CONTAINER_AUTO_INJECTION]
+        - ONBOARDING_FILTER_WEBLOG: [test-app-java,test-app-java-container,test-app-java-buildpack,test-app-java-alpine]
+          SCENARIO: [INSTALLER_AUTO_INJECTION]
   script:
     - git clone https://git@github.com/DataDog/system-tests.git system-tests
     - cp packaging/*.rpm system-tests/binaries
     - cp packaging/*.deb system-tests/binaries
+    - export DD_INSTALLER_LIBRARY_VERSION="pipeline-${CI_PIPELINE_ID}"
     - ls system-tests/binaries
     - cd system-tests
     - ./build.sh -i runner
-    - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env prod --vm-library ${TEST_LIBRARY} --vm-provider aws --vm-skip-branches ubuntu18_amd64
+    - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env prod --vm-library ${TEST_LIBRARY} --vm-provider aws --vm-skip-branches ubuntu18_amd64

buildSrc/src/main/groovy/InstrumentPlugin.groovy

@@ -178,17 +178,20 @@ interface InstrumentWorkParameters extends WorkParameters {
 abstract class InstrumentAction implements WorkAction<InstrumentWorkParameters> {
   private static final Object lock = new Object()
   private static ClassLoader pluginCL
-  private static volatile long lastBuildStamp
+  private static String cachedPluginPath
+  private static volatile long cachedBuildStamp
 
   @Override
   void execute() {
     // reset shared class-loaders each time a new build starts
     long buildStamp = parameters.buildStartedTime.get()
-    if (lastBuildStamp < buildStamp || !pluginCL) {
+    String pluginPath = parameters.pluginClassPath.join(':')
+    if (rebuildSharedClassLoader(buildStamp, pluginPath)) {
       synchronized (lock) {
-        if (lastBuildStamp < buildStamp || !pluginCL) {
+        if (rebuildSharedClassLoader(buildStamp, pluginPath)) {
           pluginCL = createClassLoader(parameters.pluginClassPath)
-          lastBuildStamp = buildStamp
+          cachedPluginPath = pluginPath
+          cachedBuildStamp = buildStamp
         }
       }
     }
@@ -199,6 +202,10 @@ abstract class InstrumentAction implements WorkAction<InstrumentWorkParameters>
     InstrumentingPlugin.instrumentClasses(plugins, instrumentingCL, sourceDirectory, targetDirectory)
   }
 
+  static boolean rebuildSharedClassLoader(long buildStamp, String pluginPath) {
+    return cachedBuildStamp < buildStamp || cachedPluginPath != pluginPath
+  }
+
   static ClassLoader createClassLoader(cp, parent = InstrumentAction.classLoader) {
     return new URLClassLoader(cp*.toURI()*.toURL() as URL[], parent as ClassLoader)
   }

communication/build.gradle

@@ -41,12 +41,29 @@ ext {
     'datadog.communication.monitor.DDAgentStatsDConnection',
     'datadog.communication.monitor.DDAgentStatsDConnection.*',
     'datadog.communication.monitor.LoggingStatsDClient',
+    'datadog.communication.BackendApiFactory',
+    'datadog.communication.BackendApiFactory.Intake',
+    'datadog.communication.EvpProxyApi',
+    'datadog.communication.IntakeApi',
+    'datadog.communication.util.IOUtils',
+    'datadog.communication.util.IOUtils.1',
+  ]
+  excludedClassesBranchCoverage = [
+    'datadog.communication.ddagent.TracerVersion',
+    'datadog.communication.BackendApiFactory',
+    'datadog.communication.EvpProxyApi',
+    'datadog.communication.IntakeApi',
   ]
-  excludedClassesBranchCoverage = ['datadog.communication.ddagent.TracerVersion',]
   excludedClassesInstructionCoverage = [
     // can't reach the error condition now
     'datadog.communication.fleet.FleetServiceImpl',
     'datadog.communication.ddagent.SharedCommunicationObjects',
     'datadog.communication.ddagent.TracerVersion',
+    'datadog.communication.BackendApiFactory',
+    'datadog.communication.BackendApiFactory.Intake',
+    'datadog.communication.EvpProxyApi',
+    'datadog.communication.IntakeApi',
+    'datadog.communication.util.IOUtils',
+    'datadog.communication.util.IOUtils.1',
   ]
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApi.java → communication/src/main/java/datadog/communication/BackendApi.java

@@ -1,7 +1,7 @@
-package datadog.trace.civisibility.communication;
+package datadog.communication;
 
 import datadog.communication.http.OkHttpUtils;
-import datadog.trace.civisibility.utils.IOThrowingFunction;
+import datadog.communication.util.IOThrowingFunction;
 import java.io.IOException;
 import java.io.InputStream;
 import javax.annotation.Nullable;
@@ -14,6 +14,7 @@ <T> T post(
       String uri,
       RequestBody requestBody,
       IOThrowingFunction<InputStream, T> responseParser,
-      @Nullable OkHttpUtils.CustomListener requestListener)
+      @Nullable OkHttpUtils.CustomListener requestListener,
+      boolean requestCompression)
       throws IOException;
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java → communication/src/main/java/datadog/communication/BackendApiFactory.java

@@ -1,10 +1,11 @@
-package datadog.trace.civisibility.communication;
+package datadog.communication;
 
 import datadog.communication.ddagent.DDAgentFeaturesDiscovery;
 import datadog.communication.ddagent.SharedCommunicationObjects;
 import datadog.communication.http.HttpRetryPolicy;
 import datadog.trace.api.Config;
 import datadog.trace.util.throwable.FatalAgentMisconfigurationError;
+import java.util.function.Function;
 import javax.annotation.Nullable;
 import okhttp3.HttpUrl;
 import org.slf4j.Logger;
@@ -22,19 +23,19 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
     this.sharedCommunicationObjects = sharedCommunicationObjects;
   }
 
-  public @Nullable BackendApi createBackendApi() {
+  public @Nullable BackendApi createBackendApi(Intake intake) {
     HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true);
 
-    if (config.isCiVisibilityAgentlessEnabled()) {
-      HttpUrl agentlessUrl = getAgentlessUrl();
+    if (intake.agentlessModeEnabled.apply(config)) {
+      HttpUrl agentlessUrl = getAgentlessUrl(intake);
       String apiKey = config.getApiKey();
       if (apiKey == null || apiKey.isEmpty()) {
         throw new FatalAgentMisconfigurationError(
             "Agentless mode is enabled and api key is not set. Please set application key");
       }
       String traceId = config.getIdGenerationStrategy().generateTraceId().toString();
       long timeoutMillis = config.getCiVisibilityBackendApiTimeoutMillis();
-      return new IntakeApi(agentlessUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory);
+      return new IntakeApi(agentlessUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory, true);
     }
 
     DDAgentFeaturesDiscovery featuresDiscovery =
@@ -45,7 +46,7 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
       String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint();
       HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint);
       return new EvpProxyApi(
-          traceId, evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient);
+          traceId, evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient, true);
     }
 
     log.warn(
@@ -54,14 +55,34 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
     return null;
   }
 
-  private HttpUrl getAgentlessUrl() {
-    final String ciVisibilityAgentlessUrlStr = config.getCiVisibilityAgentlessUrl();
-    if (ciVisibilityAgentlessUrlStr != null && !ciVisibilityAgentlessUrlStr.isEmpty()) {
-      return HttpUrl.get(
-          String.format("%s/api/%s/", ciVisibilityAgentlessUrlStr, IntakeApi.API_VERSION));
+  private HttpUrl getAgentlessUrl(Intake intake) {
+    String customUrl = intake.customUrl.apply(config);
+    if (customUrl != null && !customUrl.isEmpty()) {
+      return HttpUrl.get(String.format("%s/api/%s/", customUrl, intake.version));
     } else {
       String site = config.getSite();
-      return HttpUrl.get(String.format("https://api.%s/api/%s/", site, IntakeApi.API_VERSION));
+      return HttpUrl.get(
+          String.format("https://%s.%s/api/%s/", intake.urlPrefix, site, intake.version));
+    }
+  }
+
+  public enum Intake {
+    API("api", "v2", Config::isCiVisibilityAgentlessEnabled, Config::getCiVisibilityAgentlessUrl);
+
+    public final String urlPrefix;
+    public final String version;
+    public final Function<Config, Boolean> agentlessModeEnabled;
+    public final Function<Config, String> customUrl;
+
+    Intake(
+        String urlPrefix,
+        String version,
+        Function<Config, Boolean> agentlessModeEnabled,
+        Function<Config, String> customUrl) {
+      this.urlPrefix = urlPrefix;
+      this.version = version;
+      this.agentlessModeEnabled = agentlessModeEnabled;
+      this.customUrl = customUrl;
     }
   }
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java → communication/src/main/java/datadog/communication/EvpProxyApi.java

@@ -1,8 +1,8 @@
-package datadog.trace.civisibility.communication;
+package datadog.communication;
 
 import datadog.communication.http.HttpRetryPolicy;
 import datadog.communication.http.OkHttpUtils;
-import datadog.trace.civisibility.utils.IOThrowingFunction;
+import datadog.communication.util.IOThrowingFunction;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.zip.GZIPInputStream;
@@ -32,35 +32,28 @@ public class EvpProxyApi implements BackendApi {
   private final HttpRetryPolicy.Factory retryPolicyFactory;
   private final HttpUrl evpProxyUrl;
   private final OkHttpClient httpClient;
-  private final boolean gzipEnabled;
-
-  public EvpProxyApi(
-      String traceId,
-      HttpUrl evpProxyUrl,
-      HttpRetryPolicy.Factory retryPolicyFactory,
-      OkHttpClient httpClient) {
-    this(traceId, evpProxyUrl, retryPolicyFactory, httpClient, true);
-  }
+  private final boolean responseCompression;
 
   public EvpProxyApi(
       String traceId,
       HttpUrl evpProxyUrl,
       HttpRetryPolicy.Factory retryPolicyFactory,
       OkHttpClient httpClient,
-      boolean gzipEnabled) {
+      boolean responseCompression) {
     this.traceId = traceId;
     this.evpProxyUrl = evpProxyUrl.resolve(String.format("api/%s/", API_VERSION));
     this.retryPolicyFactory = retryPolicyFactory;
     this.httpClient = httpClient;
-    this.gzipEnabled = gzipEnabled;
+    this.responseCompression = responseCompression;
   }
 
   @Override
   public <T> T post(
       String uri,
       RequestBody requestBody,
       IOThrowingFunction<InputStream, T> responseParser,
-      @Nullable OkHttpUtils.CustomListener requestListener)
+      @Nullable OkHttpUtils.CustomListener requestListener,
+      boolean requestCompression)
       throws IOException {
     final HttpUrl url = evpProxyUrl.resolve(uri);
 
@@ -75,7 +68,11 @@ public <T> T post(
       requestBuilder.tag(OkHttpUtils.CustomListener.class, requestListener);
     }
 
-    if (gzipEnabled) {
+    if (requestCompression) {
+      requestBuilder.addHeader(CONTENT_ENCODING_HEADER, GZIP_ENCODING);
+    }
+
+    if (responseCompression) {
       requestBuilder.addHeader(ACCEPT_ENCODING_HEADER, GZIP_ENCODING);
     }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java → communication/src/main/java/datadog/communication/IntakeApi.java

@@ -1,8 +1,8 @@
-package datadog.trace.civisibility.communication;
+package datadog.communication;
 
 import datadog.communication.http.HttpRetryPolicy;
 import datadog.communication.http.OkHttpUtils;
-import datadog.trace.civisibility.utils.IOThrowingFunction;
+import datadog.communication.util.IOThrowingFunction;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.zip.GZIPInputStream;
@@ -19,8 +19,6 @@ public class IntakeApi implements BackendApi {
 
   private static final Logger log = LoggerFactory.getLogger(IntakeApi.class);
 
-  public static final String API_VERSION = "v2";
-
   private static final String DD_API_KEY_HEADER = "dd-api-key";
   private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id";
   private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id";
@@ -31,31 +29,22 @@ public class IntakeApi implements BackendApi {
   private final String apiKey;
   private final String traceId;
   private final HttpRetryPolicy.Factory retryPolicyFactory;
-  private final boolean gzipEnabled;
+  private final boolean responseCompression;
   private final HttpUrl hostUrl;
   private final OkHttpClient httpClient;
 
-  public IntakeApi(
-      HttpUrl hostUrl,
-      String apiKey,
-      String traceId,
-      long timeoutMillis,
-      HttpRetryPolicy.Factory retryPolicyFactory) {
-    this(hostUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory, true);
-  }
-
   public IntakeApi(
       HttpUrl hostUrl,
       String apiKey,
       String traceId,
       long timeoutMillis,
       HttpRetryPolicy.Factory retryPolicyFactory,
-      boolean gzipEnabled) {
+      boolean responseCompression) {
     this.hostUrl = hostUrl;
     this.apiKey = apiKey;
     this.traceId = traceId;
     this.retryPolicyFactory = retryPolicyFactory;
-    this.gzipEnabled = gzipEnabled;
+    this.responseCompression = responseCompression;
 
     httpClient = OkHttpUtils.buildHttpClient(hostUrl, timeoutMillis);
   }
@@ -65,7 +54,8 @@ public <T> T post(
       String uri,
       RequestBody requestBody,
       IOThrowingFunction<InputStream, T> responseParser,
-      @Nullable OkHttpUtils.CustomListener requestListener)
+      @Nullable OkHttpUtils.CustomListener requestListener,
+      boolean requestCompression)
       throws IOException {
     HttpUrl url = hostUrl.resolve(uri);
     Request.Builder requestBuilder =
@@ -80,7 +70,11 @@ public <T> T post(
       requestBuilder.tag(OkHttpUtils.CustomListener.class, requestListener);
     }
 
-    if (gzipEnabled) {
+    if (requestCompression) {
+      requestBuilder.addHeader(CONTENT_ENCODING_HEADER, GZIP_ENCODING);
+    }
+
+    if (responseCompression) {
       requestBuilder.addHeader(ACCEPT_ENCODING_HEADER, GZIP_ENCODING);
     }