diff --git a/communication/build.gradle b/communication/build.gradle index b00ca74d4900..02c24dae0724 100644 --- a/communication/build.gradle +++ b/communication/build.gradle @@ -41,12 +41,29 @@ ext { 'datadog.communication.monitor.DDAgentStatsDConnection', 'datadog.communication.monitor.DDAgentStatsDConnection.*', 'datadog.communication.monitor.LoggingStatsDClient', + 'datadog.communication.BackendApiFactory', + 'datadog.communication.BackendApiFactory.Intake', + 'datadog.communication.EvpProxyApi', + 'datadog.communication.IntakeApi', + 'datadog.communication.util.IOUtils', + 'datadog.communication.util.IOUtils.1', + ] + excludedClassesBranchCoverage = [ + 'datadog.communication.ddagent.TracerVersion', + 'datadog.communication.BackendApiFactory', + 'datadog.communication.EvpProxyApi', + 'datadog.communication.IntakeApi', ] - excludedClassesBranchCoverage = ['datadog.communication.ddagent.TracerVersion',] excludedClassesInstructionCoverage = [ // can't reach the error condition now 'datadog.communication.fleet.FleetServiceImpl', 'datadog.communication.ddagent.SharedCommunicationObjects', 'datadog.communication.ddagent.TracerVersion', + 'datadog.communication.BackendApiFactory', + 'datadog.communication.BackendApiFactory.Intake', + 'datadog.communication.EvpProxyApi', + 'datadog.communication.IntakeApi', + 'datadog.communication.util.IOUtils', + 'datadog.communication.util.IOUtils.1', ] } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApi.java b/communication/src/main/java/datadog/communication/BackendApi.java similarity index 68% rename from dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApi.java rename to communication/src/main/java/datadog/communication/BackendApi.java index 52aa8f706414..aa4385d6d757 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApi.java +++ b/communication/src/main/java/datadog/communication/BackendApi.java @@ -1,7 +1,7 @@ -package datadog.trace.civisibility.communication; +package datadog.communication; import datadog.communication.http.OkHttpUtils; -import datadog.trace.civisibility.utils.IOThrowingFunction; +import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; import javax.annotation.Nullable; @@ -14,6 +14,7 @@ T post( String uri, RequestBody requestBody, IOThrowingFunction responseParser, - @Nullable OkHttpUtils.CustomListener requestListener) + @Nullable OkHttpUtils.CustomListener requestListener, + boolean requestCompression) throws IOException; } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java similarity index 63% rename from dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java rename to communication/src/main/java/datadog/communication/BackendApiFactory.java index 5fe30e1a9e8f..1b32de9ac30f 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -1,10 +1,11 @@ -package datadog.trace.civisibility.communication; +package datadog.communication; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.communication.http.HttpRetryPolicy; import datadog.trace.api.Config; import datadog.trace.util.throwable.FatalAgentMisconfigurationError; +import java.util.function.Function; import javax.annotation.Nullable; import okhttp3.HttpUrl; import org.slf4j.Logger; @@ -22,11 +23,11 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni this.sharedCommunicationObjects = sharedCommunicationObjects; } - public @Nullable BackendApi createBackendApi() { + public @Nullable BackendApi createBackendApi(Intake intake) { HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true); - if (config.isCiVisibilityAgentlessEnabled()) { - HttpUrl agentlessUrl = getAgentlessUrl(); + if (intake.agentlessModeEnabled.apply(config)) { + HttpUrl agentlessUrl = getAgentlessUrl(intake); String apiKey = config.getApiKey(); if (apiKey == null || apiKey.isEmpty()) { throw new FatalAgentMisconfigurationError( @@ -34,7 +35,7 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni } String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); long timeoutMillis = config.getCiVisibilityBackendApiTimeoutMillis(); - return new IntakeApi(agentlessUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory); + return new IntakeApi(agentlessUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory, true); } DDAgentFeaturesDiscovery featuresDiscovery = @@ -45,7 +46,7 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint(); HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint); return new EvpProxyApi( - traceId, evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient); + traceId, evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient, true); } log.warn( @@ -54,14 +55,34 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni return null; } - private HttpUrl getAgentlessUrl() { - final String ciVisibilityAgentlessUrlStr = config.getCiVisibilityAgentlessUrl(); - if (ciVisibilityAgentlessUrlStr != null && !ciVisibilityAgentlessUrlStr.isEmpty()) { - return HttpUrl.get( - String.format("%s/api/%s/", ciVisibilityAgentlessUrlStr, IntakeApi.API_VERSION)); + private HttpUrl getAgentlessUrl(Intake intake) { + String customUrl = intake.customUrl.apply(config); + if (customUrl != null && !customUrl.isEmpty()) { + return HttpUrl.get(String.format("%s/api/%s/", customUrl, intake.version)); } else { String site = config.getSite(); - return HttpUrl.get(String.format("https://api.%s/api/%s/", site, IntakeApi.API_VERSION)); + return HttpUrl.get( + String.format("https://%s.%s/api/%s/", intake.urlPrefix, site, intake.version)); + } + } + + public enum Intake { + API("api", "v2", Config::isCiVisibilityAgentlessEnabled, Config::getCiVisibilityAgentlessUrl); + + public final String urlPrefix; + public final String version; + public final Function agentlessModeEnabled; + public final Function customUrl; + + Intake( + String urlPrefix, + String version, + Function agentlessModeEnabled, + Function customUrl) { + this.urlPrefix = urlPrefix; + this.version = version; + this.agentlessModeEnabled = agentlessModeEnabled; + this.customUrl = customUrl; } } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java b/communication/src/main/java/datadog/communication/EvpProxyApi.java similarity index 86% rename from dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java rename to communication/src/main/java/datadog/communication/EvpProxyApi.java index fcc482ece680..06e230fa4ee1 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java +++ b/communication/src/main/java/datadog/communication/EvpProxyApi.java @@ -1,8 +1,8 @@ -package datadog.trace.civisibility.communication; +package datadog.communication; import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; -import datadog.trace.civisibility.utils.IOThrowingFunction; +import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; import java.util.zip.GZIPInputStream; @@ -32,27 +32,19 @@ public class EvpProxyApi implements BackendApi { private final HttpRetryPolicy.Factory retryPolicyFactory; private final HttpUrl evpProxyUrl; private final OkHttpClient httpClient; - private final boolean gzipEnabled; - - public EvpProxyApi( - String traceId, - HttpUrl evpProxyUrl, - HttpRetryPolicy.Factory retryPolicyFactory, - OkHttpClient httpClient) { - this(traceId, evpProxyUrl, retryPolicyFactory, httpClient, true); - } + private final boolean responseCompression; public EvpProxyApi( String traceId, HttpUrl evpProxyUrl, HttpRetryPolicy.Factory retryPolicyFactory, OkHttpClient httpClient, - boolean gzipEnabled) { + boolean responseCompression) { this.traceId = traceId; this.evpProxyUrl = evpProxyUrl.resolve(String.format("api/%s/", API_VERSION)); this.retryPolicyFactory = retryPolicyFactory; this.httpClient = httpClient; - this.gzipEnabled = gzipEnabled; + this.responseCompression = responseCompression; } @Override @@ -60,7 +52,8 @@ public T post( String uri, RequestBody requestBody, IOThrowingFunction responseParser, - @Nullable OkHttpUtils.CustomListener requestListener) + @Nullable OkHttpUtils.CustomListener requestListener, + boolean requestCompression) throws IOException { final HttpUrl url = evpProxyUrl.resolve(uri); @@ -75,7 +68,11 @@ public T post( requestBuilder.tag(OkHttpUtils.CustomListener.class, requestListener); } - if (gzipEnabled) { + if (requestCompression) { + requestBuilder.addHeader(CONTENT_ENCODING_HEADER, GZIP_ENCODING); + } + + if (responseCompression) { requestBuilder.addHeader(ACCEPT_ENCODING_HEADER, GZIP_ENCODING); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java b/communication/src/main/java/datadog/communication/IntakeApi.java similarity index 85% rename from dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java rename to communication/src/main/java/datadog/communication/IntakeApi.java index 067969c27895..ff29dc891248 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java +++ b/communication/src/main/java/datadog/communication/IntakeApi.java @@ -1,8 +1,8 @@ -package datadog.trace.civisibility.communication; +package datadog.communication; import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; -import datadog.trace.civisibility.utils.IOThrowingFunction; +import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; import java.util.zip.GZIPInputStream; @@ -19,8 +19,6 @@ public class IntakeApi implements BackendApi { private static final Logger log = LoggerFactory.getLogger(IntakeApi.class); - public static final String API_VERSION = "v2"; - private static final String DD_API_KEY_HEADER = "dd-api-key"; private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id"; private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id"; @@ -31,31 +29,22 @@ public class IntakeApi implements BackendApi { private final String apiKey; private final String traceId; private final HttpRetryPolicy.Factory retryPolicyFactory; - private final boolean gzipEnabled; + private final boolean responseCompression; private final HttpUrl hostUrl; private final OkHttpClient httpClient; - public IntakeApi( - HttpUrl hostUrl, - String apiKey, - String traceId, - long timeoutMillis, - HttpRetryPolicy.Factory retryPolicyFactory) { - this(hostUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory, true); - } - public IntakeApi( HttpUrl hostUrl, String apiKey, String traceId, long timeoutMillis, HttpRetryPolicy.Factory retryPolicyFactory, - boolean gzipEnabled) { + boolean responseCompression) { this.hostUrl = hostUrl; this.apiKey = apiKey; this.traceId = traceId; this.retryPolicyFactory = retryPolicyFactory; - this.gzipEnabled = gzipEnabled; + this.responseCompression = responseCompression; httpClient = OkHttpUtils.buildHttpClient(hostUrl, timeoutMillis); } @@ -65,7 +54,8 @@ public T post( String uri, RequestBody requestBody, IOThrowingFunction responseParser, - @Nullable OkHttpUtils.CustomListener requestListener) + @Nullable OkHttpUtils.CustomListener requestListener, + boolean requestCompression) throws IOException { HttpUrl url = hostUrl.resolve(uri); Request.Builder requestBuilder = @@ -80,7 +70,11 @@ public T post( requestBuilder.tag(OkHttpUtils.CustomListener.class, requestListener); } - if (gzipEnabled) { + if (requestCompression) { + requestBuilder.addHeader(CONTENT_ENCODING_HEADER, GZIP_ENCODING); + } + + if (responseCompression) { requestBuilder.addHeader(ACCEPT_ENCODING_HEADER, GZIP_ENCODING); } diff --git a/communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java b/communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java index 243795008367..7c24dccad4bd 100644 --- a/communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java +++ b/communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java @@ -29,7 +29,7 @@ public void createRemaining(Config config) { monitoring = Monitoring.DISABLED; } if (agentUrl == null) { - agentUrl = HttpUrl.parse(config.getAgentUrl()); + agentUrl = parseAgentUrl(config); if (agentUrl == null) { throw new IllegalArgumentException("Bad agent URL: " + config.getAgentUrl()); } @@ -43,6 +43,15 @@ public void createRemaining(Config config) { } } + private static HttpUrl parseAgentUrl(Config config) { + String agentUrl = config.getAgentUrl(); + if (agentUrl.startsWith("unix:")) { + // provide placeholder agent URL, in practice we'll be tunnelling over UDS + agentUrl = "http://" + config.getAgentHost() + ":" + config.getAgentPort(); + } + return HttpUrl.parse(agentUrl); + } + private static long getHttpClientTimeout(Config config) { if (!config.isCiVisibilityEnabled()) { return TimeUnit.SECONDS.toMillis(config.getAgentTimeout()); diff --git a/communication/src/main/java/datadog/communication/monitor/DDAgentStatsDConnection.java b/communication/src/main/java/datadog/communication/monitor/DDAgentStatsDConnection.java index e4dbe099ca24..7f2dbd1525ef 100644 --- a/communication/src/main/java/datadog/communication/monitor/DDAgentStatsDConnection.java +++ b/communication/src/main/java/datadog/communication/monitor/DDAgentStatsDConnection.java @@ -5,7 +5,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; -import com.timgroup.statsd.NoOpStatsDClient; +import com.timgroup.statsd.NoOpDirectStatsDClient; import com.timgroup.statsd.NonBlockingStatsDClientBuilder; import com.timgroup.statsd.StatsDClientErrorHandler; import datadog.trace.api.Config; @@ -23,7 +23,7 @@ final class DDAgentStatsDConnection implements StatsDClientErrorHandler { private static final Logger log = LoggerFactory.getLogger(DDAgentStatsDConnection.class); private static final IOLogger ioLogger = new IOLogger(log); - private static final com.timgroup.statsd.StatsDClient NO_OP = new NoOpStatsDClient(); + private static final com.timgroup.statsd.StatsDClient NO_OP = new NoOpDirectStatsDClient(); private static final String UNIX_DOMAIN_SOCKET_PREFIX = "unix://"; diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/IOThrowingFunction.java b/communication/src/main/java/datadog/communication/util/IOThrowingFunction.java similarity index 76% rename from dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/IOThrowingFunction.java rename to communication/src/main/java/datadog/communication/util/IOThrowingFunction.java index 50ce43d29fa7..e6cd18d96edd 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/IOThrowingFunction.java +++ b/communication/src/main/java/datadog/communication/util/IOThrowingFunction.java @@ -1,4 +1,4 @@ -package datadog.trace.civisibility.utils; +package datadog.communication.util; import java.io.IOException; diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/IOUtils.java b/communication/src/main/java/datadog/communication/util/IOUtils.java similarity index 98% rename from dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/IOUtils.java rename to communication/src/main/java/datadog/communication/util/IOUtils.java index 320a61ac5e25..23b41fd6c4d0 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/IOUtils.java +++ b/communication/src/main/java/datadog/communication/util/IOUtils.java @@ -1,4 +1,4 @@ -package datadog.trace.civisibility.utils; +package datadog.communication.util; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.BufferedReader; diff --git a/dd-java-agent/agent-builder/build.gradle b/dd-java-agent/agent-builder/build.gradle index 6dce229a347b..d014ee79196c 100644 --- a/dd-java-agent/agent-builder/build.gradle +++ b/dd-java-agent/agent-builder/build.gradle @@ -7,6 +7,8 @@ excludedClassesCoverage += ['datadog.trace.agent.tooling.*'] dependencies { api project(':dd-java-agent:agent-tooling') + implementation project(':dd-java-agent:agent-otel:otel-tooling') + testImplementation project(':dd-java-agent:testing') } diff --git a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/ExtensionFinder.java b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/ExtensionFinder.java index 2bc4c63fe1da..925c8093e229 100644 --- a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/ExtensionFinder.java +++ b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/ExtensionFinder.java @@ -1,5 +1,6 @@ package datadog.trace.agent.tooling; +import static datadog.opentelemetry.tooling.OtelExtensionHandler.OPENTELEMETRY; import static datadog.trace.agent.tooling.ExtensionHandler.DATADOG; import de.thetaphi.forbiddenapis.SuppressForbidden; @@ -23,7 +24,7 @@ public final class ExtensionFinder { private static final Logger log = LoggerFactory.getLogger(ExtensionFinder.class); - private static final ExtensionHandler[] handlers = {DATADOG}; + private static final ExtensionHandler[] handlers = {OPENTELEMETRY, DATADOG}; /** * Discovers extensions on the configured path and creates a classloader for each extension. diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java index 453f7ea05d21..0ea74131542e 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java @@ -1,5 +1,6 @@ package datadog.trace.civisibility; +import datadog.communication.BackendApi; import datadog.trace.api.Config; import datadog.trace.api.civisibility.config.ModuleExecutionSettings; import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; @@ -10,7 +11,6 @@ import datadog.trace.civisibility.codeowners.Codeowners; import datadog.trace.civisibility.codeowners.CodeownersProvider; import datadog.trace.civisibility.codeowners.NoCodeowners; -import datadog.trace.civisibility.communication.BackendApi; import datadog.trace.civisibility.config.CachingModuleExecutionSettingsFactory; import datadog.trace.civisibility.config.ConfigurationApi; import datadog.trace.civisibility.config.ConfigurationApiImpl; diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java index c6f3ad580cf9..8f0d52b90eb5 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java @@ -1,12 +1,12 @@ package datadog.trace.civisibility; +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; import datadog.trace.api.git.GitInfoProvider; import datadog.trace.civisibility.ci.CIProviderInfoFactory; -import datadog.trace.civisibility.communication.BackendApi; -import datadog.trace.civisibility.communication.BackendApiFactory; import datadog.trace.civisibility.config.CachingJvmInfoFactory; import datadog.trace.civisibility.config.JvmInfoFactory; import datadog.trace.civisibility.config.JvmInfoFactoryImpl; @@ -61,7 +61,8 @@ public class CiVisibilityServices { GitInfoProvider gitInfoProvider) { this.config = config; this.metricCollector = metricCollector; - this.backendApi = new BackendApiFactory(config, sco).createBackendApi(); + this.backendApi = + new BackendApiFactory(config, sco).createBackendApi(BackendApiFactory.Intake.API); this.jvmInfoFactory = new CachingJvmInfoFactory(config, new JvmInfoFactoryImpl()); this.gitClientFactory = new GitClient.Factory(config, metricCollector); this.ciProviderInfoFactory = new CIProviderInfoFactory(config); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java index 551409a50ab1..62ebbb3572e5 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java @@ -3,6 +3,7 @@ import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; +import datadog.communication.BackendApi; import datadog.communication.http.OkHttpUtils; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; @@ -13,7 +14,6 @@ import datadog.trace.api.civisibility.telemetry.tag.ItrEnabled; import datadog.trace.api.civisibility.telemetry.tag.ItrSkipEnabled; import datadog.trace.api.civisibility.telemetry.tag.RequireGit; -import datadog.trace.civisibility.communication.BackendApi; import datadog.trace.civisibility.communication.TelemetryListener; import java.io.IOException; import java.lang.reflect.ParameterizedType; @@ -109,7 +109,8 @@ public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) thr SETTINGS_URI, requestBody, is -> settingsResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data.attributes, - telemetryListener); + telemetryListener, + false); metricCollector.add( CiVisibilityCountMetric.GIT_REQUESTS_SETTINGS_RESPONSE, @@ -145,7 +146,8 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) thr SKIPPABLE_TESTS_URI, requestBody, is -> testIdentifiersResponseAdapter.fromJson(Okio.buffer(Okio.source(is))), - telemetryListener); + telemetryListener, + false); List testIdentifiers = response.data.stream().map(DataDto::getAttributes).collect(Collectors.toList()); @@ -170,7 +172,8 @@ public Collection getFlakyTests(TracerEnvironment tracerEnvironm FLAKY_TESTS_URI, requestBody, is -> testIdentifiersResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data, - null); + null, + false); return response.stream().map(DataDto::getAttributes).collect(Collectors.toList()); } @@ -196,7 +199,8 @@ public Map> getKnownTestsByModuleName( requestBody, is -> testFullNamesResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data.attributes, - telemetryListener); + telemetryListener, + false); return parseTestIdentifiers(knownTests); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitClient.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitClient.java index 794c673a73b6..0966bf8602ed 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitClient.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitClient.java @@ -1,12 +1,12 @@ package datadog.trace.civisibility.git.tree; +import datadog.communication.util.IOUtils; import datadog.trace.api.Config; import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; import datadog.trace.api.civisibility.telemetry.CiVisibilityDistributionMetric; import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; import datadog.trace.api.civisibility.telemetry.tag.Command; import datadog.trace.api.civisibility.telemetry.tag.ExitCode; -import datadog.trace.civisibility.utils.IOUtils; import datadog.trace.civisibility.utils.ShellCommandExecutor; import datadog.trace.util.Strings; import edu.umd.cs.findbugs.annotations.NonNull; diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitDataApi.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitDataApi.java index 11448c6f3d53..5b2f6f6e8237 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitDataApi.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitDataApi.java @@ -3,13 +3,13 @@ import com.squareup.moshi.Json; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; +import datadog.communication.BackendApi; import datadog.communication.http.OkHttpUtils; +import datadog.communication.util.IOUtils; import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; import datadog.trace.api.civisibility.telemetry.CiVisibilityDistributionMetric; import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; -import datadog.trace.civisibility.communication.BackendApi; import datadog.trace.civisibility.communication.TelemetryListener; -import datadog.trace.civisibility.utils.IOUtils; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -78,7 +78,8 @@ public Collection searchCommits(String gitRemoteUrl, List commit SEARCH_COMMITS_URI, requestBody, is -> searchCommitsResponseAdapter.fromJson(Okio.buffer(Okio.source(is))), - telemetryListener); + telemetryListener, + false); return response.data.stream().map(Commit::getId).collect(Collectors.toSet()); } @@ -119,7 +120,8 @@ public void uploadPackFile(String gitRemoteUrl, String currentCommitHash, Path p .build(); String response = - backendApi.post(UPLOAD_PACKFILES_URI, requestBody, IOUtils::readFully, telemetryListener); + backendApi.post( + UPLOAD_PACKFILES_URI, requestBody, IOUtils::readFully, telemetryListener, false); LOGGER.debug("Uploading pack file {} returned response {}", packFile, response); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/ShellCommandExecutor.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/ShellCommandExecutor.java index ece37891d6b8..5943e734f6ff 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/ShellCommandExecutor.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/ShellCommandExecutor.java @@ -1,5 +1,6 @@ package datadog.trace.civisibility.utils; +import datadog.communication.util.IOUtils; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.context.TraceScope; import datadog.trace.util.AgentThreadFactory; diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy index 5b9b325922ef..bce44ee785c7 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy @@ -1,15 +1,16 @@ package datadog.trace.civisibility.config import com.squareup.moshi.Moshi +import datadog.communication.BackendApi +import datadog.communication.BackendApiFactory +import datadog.communication.EvpProxyApi +import datadog.communication.IntakeApi import datadog.communication.http.HttpRetryPolicy import datadog.communication.http.OkHttpUtils import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.civisibility.config.Configurations import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector -import datadog.trace.civisibility.communication.BackendApi -import datadog.trace.civisibility.communication.EvpProxyApi -import datadog.trace.civisibility.communication.IntakeApi import okhttp3.HttpUrl import okhttp3.OkHttpClient import org.apache.commons.io.IOUtils @@ -354,22 +355,22 @@ class ConfigurationApiImplTest extends Specification { where: api | displayName - givenEvpProxy(false) | "EVP proxy, compression disabled" - givenEvpProxy(true) | "EVP proxy, compression enabled" - givenIntakeApi(false) | "intake, compression disabled" - givenIntakeApi(true) | "intake, compression enabled" + givenEvpProxy(false) | "EVP proxy, response compression disabled" + givenEvpProxy(true) | "EVP proxy, response compression enabled" + givenIntakeApi(false) | "intake, response compression disabled" + givenIntakeApi(true) | "intake, response compression enabled" } - private BackendApi givenEvpProxy(boolean enableGzip) { + private BackendApi givenEvpProxy(boolean responseCompression) { String traceId = "a-trace-id" HttpUrl proxyUrl = HttpUrl.get(intakeServer.address) HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0) OkHttpClient client = OkHttpUtils.buildHttpClient(proxyUrl, REQUEST_TIMEOUT_MILLIS) - return new EvpProxyApi(traceId, proxyUrl, retryPolicyFactory, client, enableGzip) + return new EvpProxyApi(traceId, proxyUrl, retryPolicyFactory, client, responseCompression) } - private BackendApi givenIntakeApi(boolean enableGzip) { - HttpUrl intakeUrl = HttpUrl.get(String.format("%s/api/%s/", intakeServer.address.toString(), IntakeApi.API_VERSION)) + private BackendApi givenIntakeApi(boolean responseCompression) { + HttpUrl intakeUrl = HttpUrl.get(String.format("%s/api/%s/", intakeServer.address.toString(), BackendApiFactory.Intake.API.version)) String apiKey = "api-key" String traceId = "a-trace-id" @@ -381,7 +382,7 @@ class ConfigurationApiImplTest extends Specification { HttpRetryPolicy.Factory retryPolicyFactory = Stub(HttpRetryPolicy.Factory) retryPolicyFactory.create() >> retryPolicy - return new IntakeApi(intakeUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory, enableGzip) + return new IntakeApi(intakeUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory, responseCompression) } private static TracerEnvironment givenTracerEnvironment() { diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/GitClientGitInfoBuilderTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/GitClientGitInfoBuilderTest.groovy index 0d03e2434d06..62ecf600bec9 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/GitClientGitInfoBuilderTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/GitClientGitInfoBuilderTest.groovy @@ -3,7 +3,7 @@ package datadog.trace.civisibility.git import datadog.trace.api.Config import datadog.trace.civisibility.telemetry.CiVisibilityMetricCollectorImpl import datadog.trace.civisibility.git.tree.GitClient -import datadog.trace.civisibility.utils.IOUtils +import datadog.communication.util.IOUtils import spock.lang.Specification import spock.lang.TempDir diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitClientTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitClientTest.groovy index eb9edf3f9e0d..88147a4c9344 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitClientTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitClientTest.groovy @@ -3,7 +3,7 @@ package datadog.trace.civisibility.git.tree import datadog.trace.civisibility.telemetry.CiVisibilityMetricCollectorImpl import datadog.trace.civisibility.git.GitObject import datadog.trace.civisibility.git.pack.V2PackGitInfoExtractor -import datadog.trace.civisibility.utils.IOUtils +import datadog.communication.util.IOUtils import spock.lang.Specification import spock.lang.TempDir diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataApiTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataApiTest.groovy index 9b15c9d7b15c..99c49e8dc46b 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataApiTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataApiTest.groovy @@ -5,8 +5,8 @@ import datadog.communication.http.HttpRetryPolicy import datadog.communication.http.OkHttpUtils import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector -import datadog.trace.civisibility.communication.BackendApi -import datadog.trace.civisibility.communication.EvpProxyApi +import datadog.communication.BackendApi +import datadog.communication.EvpProxyApi import datadog.trace.test.util.MultipartRequestParser import okhttp3.HttpUrl import okhttp3.OkHttpClient @@ -121,7 +121,7 @@ class GitDataApiTest extends Specification { HttpUrl proxyUrl = HttpUrl.get(intakeServer.address) HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0) OkHttpClient client = OkHttpUtils.buildHttpClient(proxyUrl, REQUEST_TIMEOUT_MILLIS) - return new EvpProxyApi(traceId, proxyUrl, retryPolicyFactory, client) + return new EvpProxyApi(traceId, proxyUrl, retryPolicyFactory, client, true) } private Path givenPackFile() { diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataUploaderImplTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataUploaderImplTest.groovy index 8d18ffab8a19..31010087c96b 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataUploaderImplTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataUploaderImplTest.groovy @@ -4,7 +4,7 @@ import datadog.trace.api.Config import datadog.trace.civisibility.telemetry.CiVisibilityMetricCollectorImpl import datadog.trace.api.git.GitInfo import datadog.trace.api.git.GitInfoProvider -import datadog.trace.civisibility.utils.IOUtils +import datadog.communication.util.IOUtils import org.hamcrest.Description import org.hamcrest.Matcher import org.hamcrest.TypeSafeMatcher diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/utils/ShellCommandExecutorTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/utils/ShellCommandExecutorTest.groovy index 27bb24f024a2..5b9dd7527ba5 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/utils/ShellCommandExecutorTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/utils/ShellCommandExecutorTest.groovy @@ -1,5 +1,6 @@ package datadog.trace.civisibility.utils +import datadog.communication.util.IOUtils import spock.lang.Specification import spock.lang.TempDir diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy index 1ab3a12f8eeb..645cc1965d9a 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy @@ -1,181 +1,11 @@ package datadog.trace.civisibility -import com.fasterxml.jackson.databind.ObjectMapper -import datadog.trace.agent.test.server.http.TestHttpServer -import datadog.trace.test.util.MultipartRequestParser -import org.apache.commons.io.IOUtils -import org.msgpack.jackson.dataformat.MessagePackFactory -import spock.lang.AutoCleanup -import spock.lang.Shared -import spock.lang.Specification -import spock.util.concurrent.PollingConditions - -import java.util.concurrent.ConcurrentLinkedQueue -import java.util.zip.GZIPInputStream -import java.util.zip.GZIPOutputStream -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer +import spock.lang.Specification abstract class CiVisibilitySmokeTest extends Specification { - @Shared - ObjectMapper msgPackMapper = new ObjectMapper(new MessagePackFactory()) - - @Shared - Queue> receivedTraces = new ConcurrentLinkedQueue<>() - - @Shared - Queue> receivedCoverages = new ConcurrentLinkedQueue<>() - - @Shared - Queue> receivedTelemetryMetrics = new ConcurrentLinkedQueue<>() - - @Shared - Queue> receivedTelemetryDistributions = new ConcurrentLinkedQueue<>() - - @Shared - boolean codeCoverageEnabled = true - - @Shared - boolean testsSkippingEnabled = true - - @Shared - boolean flakyRetriesEnabled = false - - def setup() { - receivedTraces.clear() - receivedCoverages.clear() - receivedTelemetryMetrics.clear() - receivedTelemetryDistributions.clear() - } - - @Shared - @AutoCleanup - protected TestHttpServer intakeServer = httpServer { - handlers { - prefix("/api/v2/citestcycle") { - def contentEncodingHeader = request.getHeader("Content-Encoding") - def gzipEnabled = contentEncodingHeader != null && contentEncodingHeader.contains("gzip") - def requestBody = gzipEnabled ? CiVisibilitySmokeTest.decompress(request.body) : request.body - def decodedEvent = msgPackMapper.readValue(requestBody, Map) - receivedTraces.add(decodedEvent) - - response.status(200).send() - } - - prefix("/api/v2/citestcov") { - def contentEncodingHeader = request.getHeader("Content-Encoding") - def gzipEnabled = contentEncodingHeader != null && contentEncodingHeader.contains("gzip") - def requestBody = gzipEnabled ? CiVisibilitySmokeTest.decompress(request.body) : request.body - def parsed = MultipartRequestParser.parseRequest(requestBody, request.headers.get("Content-Type")) - def coverages = parsed.get("coverage1") - for (def coverage : coverages) { - def decodedCoverage = msgPackMapper.readValue(coverage.get(), Map) - receivedCoverages.add(decodedCoverage) - } - - response.status(202).send() - } - - prefix("/api/v2/libraries/tests/services/setting") { - // not compressing settings response on purpose, to better mimic real backend behavior: - // it may choose to compress the response or not based on its size, - // so smaller responses (like those of /setting endpoint) are uncompressed, - // while the larger ones (skippable and flaky test lists) are compressed - response.status(200).send(('{ "data": { "type": "ci_app_tracers_test_service_settings", "id": "uuid", "attributes": { ' - + '"code_coverage": ' + codeCoverageEnabled - + ', "tests_skipping": ' + testsSkippingEnabled - + ', "flaky_test_retries_enabled": ' + flakyRetriesEnabled + '} } }').bytes) - } - - prefix("/api/v2/ci/tests/skippable") { - response.status(200).addHeader("Content-Encoding", "gzip").send(CiVisibilitySmokeTest.compress(('{ "data": [{' + - ' "id": "d230520a0561ee2f",' + - ' "type": "test",' + - ' "attributes": {' + - ' "configurations": {' + - ' "test.bundle": "Maven Smoke Tests Project maven-surefire-plugin default-test"' + - ' },' + - ' "name": "test_to_skip_with_itr",' + - ' "suite": "datadog.smoke.TestSucceed"' + - ' }' + - '}, {' + - ' "id": "d230520a0561ee2g",' + - ' "type": "test",' + - ' "attributes": {' + - ' "configurations": {' + - ' "test.bundle": ":test"' + - ' },' + - ' "name": "test_to_skip_with_itr",' + - ' "suite": "datadog.smoke.TestSucceed"' + - ' }' + - '}] ' + - '}').bytes)) - } - - prefix("/api/v2/ci/libraries/tests/flaky") { - response.status(200).addHeader("Content-Encoding", "gzip").send(CiVisibilitySmokeTest.compress(('{ "data": [{' + - ' "id": "d230520a0561ee2f",' + - ' "type": "test",' + - ' "attributes": {' + - ' "configurations": {' + - ' "test.bundle": "Maven Smoke Tests Project maven-surefire-plugin default-test"' + - ' },' + - ' "name": "test_failed",' + - ' "suite": "datadog.smoke.TestFailed"' + - ' }' + - '}, {' + - ' "id": "d230520a0561ee2g",' + - ' "type": "test",' + - ' "attributes": {' + - ' "configurations": {' + - ' "test.bundle": ":test"' + - ' },' + - ' "name": "test_failed",' + - ' "suite": "datadog.smoke.TestFailed"' + - ' }' + - '}] ' + - '}').bytes)) - } - - prefix("/api/v2/apmtelemetry") { - def telemetryRequest = CiVisibilityTestUtils.JSON_MAPPER.readerFor(Map.class).readValue(request.body) - def requestType = telemetryRequest["request_type"] - if (requestType == "message-batch") { - for (def message : telemetryRequest["payload"]) { - def payload = message["payload"] - if (message["request_type"] == 'generate-metrics') { - receivedTelemetryMetrics.addAll((List) payload["series"]) - } else if (message["request_type"] == 'distributions') { - receivedTelemetryDistributions.addAll((List) payload["series"]) - } - } - } - response.status(202).send() - } - } - } - - private static byte[] compress(byte[] bytes) { - def baos = new ByteArrayOutputStream() - try (GZIPOutputStream zip = new GZIPOutputStream(baos)) { - IOUtils.copy(new ByteArrayInputStream(bytes), zip) - } - return baos.toByteArray() - } - - private static byte[] decompress(byte[] bytes) { - def baos = new ByteArrayOutputStream() - try (GZIPInputStream zip = new GZIPInputStream(new ByteArrayInputStream(bytes))) { - IOUtils.copy(zip, baos) - } - return baos.toByteArray() - } - - protected verifyEventsAndCoverages(String projectName, String toolchain, String toolchainVersion, int expectedEventsCount, int expectedCoveragesCount) { - def events = waitForEvents(expectedEventsCount) - def coverages = waitForCoverages(expectedCoveragesCount) - + protected verifyEventsAndCoverages(String projectName, String toolchain, String toolchainVersion, List> events, List> coverages) { def additionalReplacements = ["content.meta.['test.toolchain']": "$toolchain:$toolchainVersion"] // uncomment to generate expected data templates @@ -192,7 +22,7 @@ abstract class CiVisibilitySmokeTest extends Specification { * Currently the check is not performed for Gradle builds: * Gradle daemon started with Gradle TestKit outlives the test, so the final telemetry flush happens after the assertions. */ - protected verifyTelemetryMetrics( int expectedEventsCount) { + protected verifyTelemetryMetrics(List> receivedTelemetryMetrics, List> receivedTelemetryDistributions, int expectedEventsCount) { int eventsCreated = 0, eventsFinished = 0 for (Map metric : receivedTelemetryMetrics) { if (metric["metric"] == "event_created") { @@ -212,36 +42,4 @@ abstract class CiVisibilitySmokeTest extends Specification { // an even more basic smoke check for distributions: assert that we received some assert !receivedTelemetryDistributions.isEmpty() } - - protected List> waitForEvents(int expectedEventsSize) { - def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1) - traceReceiveConditions.eventually { - int eventsSize = 0 - for (Map trace : receivedTraces) { - eventsSize += trace["events"].size() - } - assert eventsSize >= expectedEventsSize - } - - List> events = new ArrayList<>() - while (!receivedTraces.isEmpty()) { - def trace = receivedTraces.poll() - events.addAll((List>) trace["events"]) - } - return events - } - - protected List> waitForCoverages(int traceSize) { - def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1) - traceReceiveConditions.eventually { - assert receivedCoverages.size() == traceSize - } - - List> coverages = new ArrayList<>() - while (!receivedCoverages.isEmpty()) { - def trace = receivedCoverages.poll() - coverages.addAll((List>) trace["coverages"]) - } - return coverages - } } diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java index f91b20798ebd..c1e01b13fc75 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java @@ -142,7 +142,9 @@ public static synchronized void run( private static String getDiagnosticEndpoint( Config config, DDAgentFeaturesDiscovery ddAgentFeaturesDiscovery) { if (ddAgentFeaturesDiscovery.supportsDebuggerDiagnostics()) { - return config.getAgentUrl() + "/" + DDAgentFeaturesDiscovery.DEBUGGER_DIAGNOSTICS_ENDPOINT; + return ddAgentFeaturesDiscovery + .buildUrl(DDAgentFeaturesDiscovery.DEBUGGER_DIAGNOSTICS_ENDPOINT) + .toString(); } return config.getFinalDebuggerSnapshotUrl(); } diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/ExceptionProbeManager.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/ExceptionProbeManager.java index 912c8009be52..34c9f6ce8723 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/ExceptionProbeManager.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/ExceptionProbeManager.java @@ -6,6 +6,7 @@ import com.datadog.debugger.util.ClassNameFiltering; import com.datadog.debugger.util.ExceptionHelper; import com.datadog.debugger.util.WeakIdentityHashMap; +import datadog.trace.api.Config; import datadog.trace.bootstrap.debugger.ProbeId; import java.time.Clock; import java.time.Duration; @@ -33,20 +34,33 @@ public class ExceptionProbeManager { Collections.synchronizedMap(new WeakIdentityHashMap<>()); private final long captureIntervalS; private final Clock clock; + private final int maxCapturedFrames; public ExceptionProbeManager(ClassNameFiltering classNameFiltering, Duration captureInterval) { - this(classNameFiltering, captureInterval, Clock.systemUTC()); + this( + classNameFiltering, + captureInterval, + Clock.systemUTC(), + Config.get().getDebuggerExceptionMaxCapturedFrames()); } ExceptionProbeManager(ClassNameFiltering classNameFiltering) { - this(classNameFiltering, Duration.ofHours(1), Clock.systemUTC()); + this( + classNameFiltering, + Duration.ofHours(1), + Clock.systemUTC(), + Config.get().getDebuggerExceptionMaxCapturedFrames()); } ExceptionProbeManager( - ClassNameFiltering classNameFiltering, Duration captureInterval, Clock clock) { + ClassNameFiltering classNameFiltering, + Duration captureInterval, + Clock clock, + int maxCapturedFrames) { this.classNameFiltering = classNameFiltering; this.captureIntervalS = captureInterval.getSeconds(); this.clock = clock; + this.maxCapturedFrames = maxCapturedFrames; } public ClassNameFiltering getClassNameFiltering() { @@ -55,7 +69,11 @@ public ClassNameFiltering getClassNameFiltering() { public boolean createProbesForException(StackTraceElement[] stackTraceElements) { boolean created = false; + int maxFrames = maxCapturedFrames; for (StackTraceElement stackTraceElement : stackTraceElements) { + if (maxFrames <= 0) { + break; + } if (stackTraceElement.isNativeMethod() || stackTraceElement.getLineNumber() < 0) { // Skip native methods and lines without line numbers // TODO log? @@ -73,6 +91,7 @@ public boolean createProbesForException(StackTraceElement[] stackTraceElements) ExceptionProbe probe = createMethodProbe(this, where); created = true; probes.putIfAbsent(probe.getId(), probe); + maxFrames--; } return created; } diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java index c1ac05b892f3..26572352260a 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java @@ -95,9 +95,13 @@ public void doubleHandleException() { @Test public void nestedException() { RuntimeException exception = createNestException(); + String fingerprint = Fingerprinter.fingerprint(exception, classNameFiltering); AgentSpan span = mock(AgentSpan.class); doAnswer(this::recordTags).when(span).setTag(anyString(), anyString()); exceptionDebugger.handleException(exception, span); + assertWithTimeout( + () -> exceptionDebugger.getExceptionProbeManager().isAlreadyInstrumented(fingerprint), + Duration.ofSeconds(30)); generateSnapshots(exception); exception.printStackTrace(); exceptionDebugger.handleException(exception, span); @@ -144,7 +148,9 @@ public void nestedException() { @Test public void doubleNestedException() { RuntimeException nestedException = createNestException(); + String nestedFingerprint = Fingerprinter.fingerprint(nestedException, classNameFiltering); RuntimeException simpleException = new RuntimeException("test"); + String simpleFingerprint = Fingerprinter.fingerprint(simpleException, classNameFiltering); AgentSpan span = mock(AgentSpan.class); doAnswer(this::recordTags).when(span).setTag(anyString(), anyString()); when(span.getTag(anyString())) @@ -154,6 +160,12 @@ public void doubleNestedException() { exceptionDebugger.handleException(nestedException, span); // instrument first simple Exception exceptionDebugger.handleException(simpleException, span); + assertWithTimeout( + () -> exceptionDebugger.getExceptionProbeManager().isAlreadyInstrumented(nestedFingerprint), + Duration.ofSeconds(30)); + assertWithTimeout( + () -> exceptionDebugger.getExceptionProbeManager().isAlreadyInstrumented(simpleFingerprint), + Duration.ofSeconds(30)); generateSnapshots(nestedException); generateSnapshots(simpleException); exceptionDebugger.handleException(simpleException, span); diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/ExceptionProbeInstrumentationTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/ExceptionProbeInstrumentationTest.java index f3ad0373fa54..cd6f11076b43 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/ExceptionProbeInstrumentationTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/ExceptionProbeInstrumentationTest.java @@ -190,7 +190,8 @@ public void differentExceptionsSameStack() throws Exception { disabledReason = "Bug in J9: no LocalVariableTable for ClassFileTransformer") public void recursive() throws Exception { Config config = createConfig(); - ExceptionProbeManager exceptionProbeManager = new ExceptionProbeManager(classNameFiltering); + ExceptionProbeManager exceptionProbeManager = + new ExceptionProbeManager(classNameFiltering, Duration.ofHours(1), Clock.systemUTC(), 20); TestSnapshotListener listener = setupExceptionDebugging(config, exceptionProbeManager, classNameFiltering); final String CLASS_NAME = "com.datadog.debugger.CapturedSnapshot20"; @@ -227,7 +228,7 @@ public void captureOncePerHour() throws Exception { Clock clockMock = mock(Clock.class); when(clockMock.instant()).thenReturn(Instant.now()); ExceptionProbeManager exceptionProbeManager = - new ExceptionProbeManager(classNameFiltering, Duration.ofHours(1), clockMock); + new ExceptionProbeManager(classNameFiltering, Duration.ofHours(1), clockMock, 3); TestSnapshotListener listener = setupExceptionDebugging(config, exceptionProbeManager, classNameFiltering); final String CLASS_NAME = "com.datadog.debugger.CapturedSnapshot20"; diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/ExceptionProbeManagerTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/ExceptionProbeManagerTest.java index 30faf4e424d6..7e4bebca0fce 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/ExceptionProbeManagerTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/ExceptionProbeManagerTest.java @@ -60,7 +60,6 @@ void filterAllFrames() { when(config.getThirdPartyIncludes()) .thenReturn( Stream.of( - ",", "org.gradle.", "worker.org.gradle.", "org.junit.", @@ -88,4 +87,36 @@ void lastCapture() { Clock.fixed(Instant.now().plus(Duration.ofMinutes(61)), Clock.systemUTC().getZone()); assertTrue(exceptionProbeManager.shouldCaptureException(fingerprint, clock)); } + + @Test + void maxFrames() { + RuntimeException deepException = level1(); + ClassNameFiltering classNameFiltering = + new ClassNameFiltering( + Stream.of( + "java.", + "jdk.", + "sun.", + "com.sun.", + "org.gradle.", + "worker.org.gradle.", + "org.junit.") + .collect(Collectors.toSet())); + ExceptionProbeManager exceptionProbeManager = + new ExceptionProbeManager(classNameFiltering, Duration.ofHours(1), Clock.systemUTC(), 3); + exceptionProbeManager.createProbesForException(deepException.getStackTrace()); + assertEquals(3, exceptionProbeManager.getProbes().size()); + } + + RuntimeException level1() { + return level2(); + } + + RuntimeException level2() { + return level3(); + } + + RuntimeException level3() { + return new RuntimeException("3 level deep exception"); + } } diff --git a/dd-java-agent/agent-logging/src/main/java/datadog/trace/logging/ddlogger/DDLoggerFactory.java b/dd-java-agent/agent-logging/src/main/java/datadog/trace/logging/ddlogger/DDLoggerFactory.java index efd8ef2293f7..2a9f9ab5d8c3 100644 --- a/dd-java-agent/agent-logging/src/main/java/datadog/trace/logging/ddlogger/DDLoggerFactory.java +++ b/dd-java-agent/agent-logging/src/main/java/datadog/trace/logging/ddlogger/DDLoggerFactory.java @@ -13,7 +13,7 @@ public class DDLoggerFactory implements ILoggerFactory, LogLevelSwitcher { - private final boolean telemetryLogCollectionEnabled = getLogCollectionEnabled(false); + private final boolean telemetryLogCollectionEnabled = isLogCollectionEnabled(); private volatile LoggerHelperFactory helperFactory = null; private volatile LogLevel override = null; @@ -88,17 +88,30 @@ public void reinitialize() { helperFactory = null; } - // DDLoggerFactory can be called at very early stage, before Config loaded + // DDLoggerFactory can be called at very early stage, before Config is loaded // So to get property/env we use this custom function - private static boolean getLogCollectionEnabled(boolean defaultValue) { - String value = System.getProperty("dd.telemetry.log-collection.enabled"); + private static boolean isLogCollectionEnabled() { + // FIXME: For the initial rollout, we default log collection to true for IAST and CI Visibility + // users. This should be removed once we default to true. + final boolean defaultValue = + isFlagEnabled("dd.iast.enabled", "DD_IAST_ENABLED", false) + || isFlagEnabled("dd.civisibility.enabled", "DD_CIVISIBILITY_ENABLED", false) + || isFlagEnabled( + "dd.dynamic.instrumentation.enabled", "DD_DYNAMIC_INSTRUMENTATION_ENABLED", false); + return isFlagEnabled( + "dd.telemetry.log-collection.enabled", "DD_TELEMETRY_LOG_COLLECTION_ENABLED", defaultValue); + } + + private static boolean isFlagEnabled( + final String systemProperty, final String envVar, final boolean defaultValue) { + String value = System.getProperty(systemProperty); if ("true".equalsIgnoreCase(value)) { return true; } if ("false".equalsIgnoreCase(value)) { return false; } - value = System.getenv("DD_TELEMETRY_LOG_COLLECTION_ENABLED"); + value = System.getenv(envVar); if ("true".equalsIgnoreCase(value)) { return true; } diff --git a/dd-java-agent/agent-otel/otel-bootstrap/build.gradle b/dd-java-agent/agent-otel/otel-bootstrap/build.gradle index ae5a4aed26a0..c57766ed84f4 100644 --- a/dd-java-agent/agent-otel/otel-bootstrap/build.gradle +++ b/dd-java-agent/agent-otel/otel-bootstrap/build.gradle @@ -11,6 +11,7 @@ configurations { canBeConsumed = false canBeResolved = true } + compileClasspath.extendsFrom(embeddedClasspath) instrumentPluginClasspath { visible = false canBeConsumed = false @@ -18,7 +19,7 @@ configurations { } } -instrument.plugins = ['datadog.opentelemetry.tooling.OtelShimGradlePlugin'] +instrument.plugins = ['datadog.opentelemetry.tooling.shim.OtelShimGradlePlugin'] minimumInstructionCoverage = 0.0 minimumBranchCoverage = 0.0 @@ -36,7 +37,7 @@ dependencies { implementation project(':dd-java-agent:agent-otel:otel-shim') - instrumentPluginClasspath project(':dd-java-agent:agent-otel:otel-tooling') + instrumentPluginClasspath project(path: ':dd-java-agent:agent-otel:otel-tooling', configuration: 'instrumentPluginClasspath') } // unpack embeddedClasspath to same path as compiled classes so it can get instrumented diff --git a/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/Java8BytecodeBridge.java b/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/Java8BytecodeBridge.java new file mode 100644 index 000000000000..ffc369acd088 --- /dev/null +++ b/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/Java8BytecodeBridge.java @@ -0,0 +1,33 @@ +package datadog.trace.bootstrap.otel; + +import io.opentelemetry.api.baggage.Baggage; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; + +/** Support references to OpenTelemetry's Java8BytecodeBridge in external extensions. */ +public final class Java8BytecodeBridge { + + // Static helpers that will redirect to our embedded copy of the OpenTelemetry API + + public static Context currentContext() { + return Context.current(); + } + + public static Context rootContext() { + return Context.root(); + } + + public static Span currentSpan() { + return Span.current(); + } + + public static Span spanFromContext(Context context) { + return Span.fromContext(context); + } + + public static Baggage baggageFromContext(Context context) { + return Baggage.fromContext(context); + } + + private Java8BytecodeBridge() {} +} diff --git a/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/package-info.java b/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/package-info.java deleted file mode 100644 index 3f00de3c82c4..000000000000 --- a/dd-java-agent/agent-otel/otel-bootstrap/src/main/java/package-info.java +++ /dev/null @@ -1 +0,0 @@ -// placeholder to activate the compiler task diff --git a/dd-java-agent/agent-otel/otel-tooling/build.gradle b/dd-java-agent/agent-otel/otel-tooling/build.gradle index e3511f9f60c9..34055c1e8296 100644 --- a/dd-java-agent/agent-otel/otel-tooling/build.gradle +++ b/dd-java-agent/agent-otel/otel-tooling/build.gradle @@ -3,7 +3,17 @@ apply from: "$rootDir/gradle/java.gradle" minimumInstructionCoverage = 0.0 minimumBranchCoverage = 0.0 +configurations { + instrumentPluginClasspath { + canBeConsumed = true + canBeResolved = false + extendsFrom runtimeElements + } +} + dependencies { api deps.bytebuddy api deps.bytebuddyagent + + compileOnly project(':dd-java-agent:agent-tooling') } diff --git a/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelExtensionHandler.java b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelExtensionHandler.java new file mode 100644 index 000000000000..06a6f3168829 --- /dev/null +++ b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelExtensionHandler.java @@ -0,0 +1,39 @@ +package datadog.opentelemetry.tooling; + +import datadog.trace.agent.tooling.ExtensionHandler; +import java.net.URL; +import java.net.URLConnection; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +/** Handles OpenTelemetry instrumentations, so they can be loaded into the Datadog tracer. */ +public final class OtelExtensionHandler extends ExtensionHandler { + + /** Handler for loading externally built OpenTelemetry extensions. */ + public static final OtelExtensionHandler OPENTELEMETRY = new OtelExtensionHandler(); + + private static final String OPENTELEMETRY_MODULE_DESCRIPTOR = + "META-INF/services/io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule"; + + private static final String DATADOG_MODULE_DESCRIPTOR = + "META-INF/services/datadog.trace.agent.tooling.InstrumenterModule"; + + @Override + public JarEntry mapEntry(JarFile jar, String file) { + if (DATADOG_MODULE_DESCRIPTOR.equals(file)) { + // redirect request to include OpenTelemetry instrumentations + return super.mapEntry(jar, OPENTELEMETRY_MODULE_DESCRIPTOR); + } else { + return super.mapEntry(jar, file); + } + } + + @Override + public URLConnection mapContent(URL url, JarFile jar, JarEntry entry) { + if (entry.getName().endsWith(".class")) { + return new ClassMappingConnection(url, jar, entry, OtelInstrumentationMapper::new); + } else { + return new JarFileConnection(url, jar, entry); + } + } +} diff --git a/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelInstrumentationMapper.java b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelInstrumentationMapper.java new file mode 100644 index 000000000000..3a047579ba2d --- /dev/null +++ b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelInstrumentationMapper.java @@ -0,0 +1,109 @@ +package datadog.opentelemetry.tooling; + +import static datadog.trace.agent.tooling.ExtensionHandler.MAP_LOGGING; + +import datadog.trace.agent.tooling.InstrumenterModule; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import net.bytebuddy.jar.asm.ClassVisitor; +import net.bytebuddy.jar.asm.MethodVisitor; +import net.bytebuddy.jar.asm.commons.ClassRemapper; +import net.bytebuddy.jar.asm.commons.Remapper; + +/** Maps OpenTelemetry instrumentations to use the Datadog {@link InstrumenterModule} API. */ +public final class OtelInstrumentationMapper extends ClassRemapper { + + private static final Set UNSUPPORTED_TYPES = + new HashSet<>( + Arrays.asList("io/opentelemetry/javaagent/tooling/muzzle/InstrumentationModuleMuzzle")); + + private static final Set UNSUPPORTED_METHODS = + new HashSet<>( + Arrays.asList( + "getMuzzleReferences", "getMuzzleHelperClassNames", "registerMuzzleVirtualFields")); + + public OtelInstrumentationMapper(ClassVisitor classVisitor) { + super(classVisitor, Renamer.INSTANCE); + } + + @Override + public void visit( + int version, + int access, + String name, + String signature, + String superName, + String[] interfaces) { + super.visit(version, access, name, signature, superName, removeUnsupportedTypes(interfaces)); + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + if (!UNSUPPORTED_METHODS.contains(name)) { + return super.visitMethod(access, name, descriptor, signature, exceptions); + } else { + return null; // remove unsupported method + } + } + + private String[] removeUnsupportedTypes(String[] interfaces) { + List filtered = null; + for (int i = interfaces.length - 1; i >= 0; i--) { + if (UNSUPPORTED_TYPES.contains(interfaces[i])) { + if (null == filtered) { + filtered = new ArrayList<>(Arrays.asList(interfaces)); + } + filtered.remove(i); // remove unsupported interface + } + } + return null != filtered ? filtered.toArray(new String[0]) : interfaces; + } + + static final class Renamer extends Remapper { + static final Renamer INSTANCE = new Renamer(); + + private static final String OTEL_JAVAAGENT_SHADED_PREFIX = + "io/opentelemetry/javaagent/shaded/io/opentelemetry/"; + + /** Datadog equivalent of OpenTelemetry instrumentation classes. */ + private static final Map RENAMED_TYPES = new HashMap<>(); + + static { + RENAMED_TYPES.put( + "io/opentelemetry/javaagent/extension/instrumentation/InstrumentationModule", + "datadog/opentelemetry/tooling/OtelInstrumenterModule"); + RENAMED_TYPES.put( + "io/opentelemetry/javaagent/extension/instrumentation/TypeInstrumentation", + "datadog/opentelemetry/tooling/OtelInstrumenter"); + RENAMED_TYPES.put( + "io/opentelemetry/javaagent/extension/instrumentation/TypeTransformer", + "datadog/opentelemetry/tooling/OtelTransformer"); + RENAMED_TYPES.put( + "io/opentelemetry/javaagent/bootstrap/Java8BytecodeBridge", + "datadog/trace/bootstrap/otel/Java8BytecodeBridge"); + RENAMED_TYPES.put( + "io/opentelemetry/javaagent/extension/matcher/AgentElementMatchers", + "datadog/trace/agent/tooling/bytebuddy/matcher/HierarchyMatchers"); + } + + @Override + public String map(String internalName) { + String rename = RENAMED_TYPES.get(internalName); + if (null != rename) { + return rename; + } + // map OpenTelemetry's shaded API to our embedded copy + if (internalName.startsWith(OTEL_JAVAAGENT_SHADED_PREFIX)) { + return "datadog/trace/bootstrap/otel/" + + internalName.substring(OTEL_JAVAAGENT_SHADED_PREFIX.length()); + } + return MAP_LOGGING.apply(internalName); + } + } +} diff --git a/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelInstrumenter.java b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelInstrumenter.java new file mode 100644 index 000000000000..47f6bcbb6bf2 --- /dev/null +++ b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelInstrumenter.java @@ -0,0 +1,41 @@ +package datadog.opentelemetry.tooling; + +import datadog.trace.agent.tooling.Instrumenter; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** Replaces OpenTelemetry's {@code TypeInstrumentation} callback when mapping extensions. */ +public interface OtelInstrumenter + extends Instrumenter.ForTypeHierarchy, + Instrumenter.HasMethodAdvice, + Instrumenter.HasTypeAdvice { + + @Override + default String hierarchyMarkerType() { + return null; // no hint available + } + + @Override + default ElementMatcher hierarchyMatcher() { + return typeMatcher(); + } + + @Override + default void methodAdvice(MethodTransformer methodTransformer) { + OtelTransformerState.capture(this).with(methodTransformer); + } + + @Override + default void typeAdvice(TypeTransformer typeTransformer) { + OtelTransformerState.capture(this).with(typeTransformer); + } + + ElementMatcher typeMatcher(); + + /** + * Once both transformers have been captured in {@link #methodAdvice} and {@link #typeAdvice} the + * {@code #transform} method will be called. This allows the extension to register method and type + * advice at the same time, using the single interface expected by OpenTelemetry. + */ + void transform(OtelTransformer transformer); +} diff --git a/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelInstrumenterModule.java b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelInstrumenterModule.java new file mode 100644 index 000000000000..e88d2c09264a --- /dev/null +++ b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelInstrumenterModule.java @@ -0,0 +1,33 @@ +package datadog.opentelemetry.tooling; + +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.InstrumenterConfig; + +/** + * Replaces OpenTelemetry's {@code InstrumentationModule} when mapping extensions. + * + *

Original instrumentation names and aliases are prefixed with {@literal "otel."}. + */ +public abstract class OtelInstrumenterModule extends InstrumenterModule.Tracing { + + public OtelInstrumenterModule(String instrumentationName, String... additionalNames) { + super(namespace(instrumentationName), namespace(additionalNames)); + } + + @Override + protected boolean defaultEnabled() { + return InstrumenterConfig.get().isTraceOtelEnabled() && super.defaultEnabled(); + } + + private static String namespace(String name) { + return "otel." + name; + } + + private static String[] namespace(String[] names) { + String[] namespaced = new String[names.length]; + for (int i = 0; i < names.length; i++) { + namespaced[i] = namespace(names[i]); + } + return namespaced; + } +} diff --git a/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelTransformer.java b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelTransformer.java new file mode 100644 index 000000000000..f57bf7534817 --- /dev/null +++ b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelTransformer.java @@ -0,0 +1,14 @@ +package datadog.opentelemetry.tooling; + +import net.bytebuddy.agent.builder.AgentBuilder; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** Replaces OpenTelemetry's {@code TypeTransformer} callback when mapping extensions. */ +public interface OtelTransformer { + + void applyAdviceToMethod( + ElementMatcher methodMatcher, String adviceClassName); + + void applyTransformer(AgentBuilder.Transformer transformer); +} diff --git a/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelTransformerState.java b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelTransformerState.java new file mode 100644 index 000000000000..04bfe4949a64 --- /dev/null +++ b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelTransformerState.java @@ -0,0 +1,73 @@ +package datadog.opentelemetry.tooling; + +import datadog.trace.agent.tooling.Instrumenter; +import net.bytebuddy.agent.builder.AgentBuilder; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** + * {@link OtelTransformer} state captured when processing OpenTelemetry extensions with {@code + * CombiningTransformerBuilder}. Assumes that the builder is single threaded. + * + *

OpenTelemetry has a single transformer callback with methods to register method advice and + * type transformations at the same time, whereas the Datadog tracer has separate {@link + * Instrumenter.MethodTransformer} and {@link Instrumenter.TypeTransformer} callbacks. + * + *

To map between the two we capture the Datadog method and type transformers here, from calls to + * {@link OtelInstrumenter}. Once we have captured both transformers we trigger the single transform + * request through the mapped OpenTelemetry callback. + */ +final class OtelTransformerState implements OtelTransformer { + private static final OtelTransformerState CURRENT = new OtelTransformerState(); + + private OtelInstrumenter instrumenter; + private Instrumenter.MethodTransformer methodTransformer; + private Instrumenter.TypeTransformer typeTransformer; + + static OtelTransformerState capture(OtelInstrumenter instrumenter) { + if (instrumenter != CURRENT.instrumenter) { + CURRENT.reset(); + CURRENT.instrumenter = instrumenter; + } + return CURRENT; + } + + void with(Instrumenter.MethodTransformer methodTransformer) { + this.methodTransformer = methodTransformer; + if (null != this.typeTransformer) { + triggerTransform(); + } + } + + void with(Instrumenter.TypeTransformer typeTransformer) { + this.typeTransformer = typeTransformer; + if (null != this.methodTransformer) { + triggerTransform(); + } + } + + private void triggerTransform() { + try { + instrumenter.transform(this); + } finally { + reset(); + } + } + + private void reset() { + this.instrumenter = null; + this.methodTransformer = null; + this.typeTransformer = null; + } + + @Override + public void applyAdviceToMethod( + ElementMatcher methodMatcher, String adviceClassName) { + methodTransformer.applyAdvice(methodMatcher, adviceClassName); + } + + @Override + public void applyTransformer(AgentBuilder.Transformer transformer) { + typeTransformer.applyAdvice(transformer::transform); + } +} diff --git a/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelShimGradlePlugin.java b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/shim/OtelShimGradlePlugin.java similarity index 68% rename from dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelShimGradlePlugin.java rename to dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/shim/OtelShimGradlePlugin.java index ca891d259925..f23f446983d5 100644 --- a/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelShimGradlePlugin.java +++ b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/shim/OtelShimGradlePlugin.java @@ -1,8 +1,5 @@ -package datadog.opentelemetry.tooling; +package datadog.opentelemetry.tooling.shim; -import static datadog.opentelemetry.tooling.OtelShimInjector.OTEL_CONTEXT_CLASSES; -import static datadog.opentelemetry.tooling.OtelShimInjector.OTEL_CONTEXT_STORAGE_CLASSES; -import static datadog.opentelemetry.tooling.OtelShimInjector.OTEL_ENTRYPOINT_CLASSES; import static net.bytebuddy.matcher.ElementMatchers.namedOneOf; import de.thetaphi.forbiddenapis.SuppressForbidden; @@ -21,12 +18,17 @@ public class OtelShimGradlePlugin extends Plugin.ForElementMatcher { private final File targetDir; + static final String[] OTEL_SHIM_INJECTED_CLASSES = { + "io.opentelemetry.api.DefaultOpenTelemetry", + "io.opentelemetry.api.GlobalOpenTelemetry$ObfuscatedOpenTelemetry", + "io.opentelemetry.context.ThreadLocalContextStorage", + "io.opentelemetry.context.StrictContextStorage", + "io.opentelemetry.context.ArrayBasedContext", + }; + @SuppressForbidden public OtelShimGradlePlugin(File targetDir) { - super( - namedOneOf(OTEL_ENTRYPOINT_CLASSES) - .or(namedOneOf(OTEL_CONTEXT_STORAGE_CLASSES)) - .or(namedOneOf(OTEL_CONTEXT_CLASSES))); + super(namedOneOf(OTEL_SHIM_INJECTED_CLASSES)); this.targetDir = targetDir; } diff --git a/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelShimInjector.java b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/shim/OtelShimInjector.java similarity index 89% rename from dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelShimInjector.java rename to dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/shim/OtelShimInjector.java index 9d4c63ed0af0..01390fc8bcd6 100644 --- a/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/OtelShimInjector.java +++ b/dd-java-agent/agent-otel/otel-tooling/src/main/java/datadog/opentelemetry/tooling/shim/OtelShimInjector.java @@ -1,4 +1,4 @@ -package datadog.opentelemetry.tooling; +package datadog.opentelemetry.tooling.shim; import net.bytebuddy.asm.AsmVisitorWrapper; import net.bytebuddy.description.field.FieldDescription; @@ -16,20 +16,6 @@ public final class OtelShimInjector implements AsmVisitorWrapper { static final OtelShimInjector INSTANCE = new OtelShimInjector(); - static final String[] OTEL_ENTRYPOINT_CLASSES = { - "io.opentelemetry.api.DefaultOpenTelemetry", - "io.opentelemetry.api.GlobalOpenTelemetry$ObfuscatedOpenTelemetry", - }; - - static final String[] OTEL_CONTEXT_STORAGE_CLASSES = { - "io.opentelemetry.context.ThreadLocalContextStorage", - "io.opentelemetry.context.StrictContextStorage", - }; - - static final String[] OTEL_CONTEXT_CLASSES = { - "io.opentelemetry.context.ArrayBasedContext", - }; - static final String TRACER_PROVIDER_DESCRIPTOR = "Lio/opentelemetry/api/trace/TracerProvider;"; static final String CONTEXT_PROPAGATORS_DESCRIPTOR = diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/ExtensionHandler.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/ExtensionHandler.java index 4dd23c26dd39..45c9f2760d4f 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/ExtensionHandler.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/ExtensionHandler.java @@ -28,7 +28,7 @@ public class ExtensionHandler { new Remapper() { @Override public String map(String internalName) { - return SHADE_LOGGING.apply(internalName); + return MAP_LOGGING.apply(internalName); } }); @@ -113,7 +113,7 @@ private byte[] mapBytecode() throws IOException { } /** Maps logging references in the extension to use the tracer's embedded logger. */ - protected static final Function SHADE_LOGGING = + public static final Function MAP_LOGGING = new Function() { // substring stops string literal from being changed by shadow plugin private final String ORG_SLF4J = "_org/slf4j/".substring(1); diff --git a/dd-java-agent/instrumentation/java-concurrent/build.gradle b/dd-java-agent/instrumentation/java-concurrent/build.gradle index 802c60ab1f0d..9160ba471375 100644 --- a/dd-java-agent/instrumentation/java-concurrent/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/build.gradle @@ -1,7 +1,3 @@ -ext { - latestDepTestMinJavaVersionForTests = JavaVersion.VERSION_21 -} - muzzle { pass { coreJdk() @@ -10,11 +6,6 @@ muzzle { apply from: "$rootDir/gradle/java.gradle" -addTestSuite('latestDepTest') - -compileLatestDepTestGroovy.configure { - javaLauncher = getJavaLauncherFor(21) -} dependencies { testImplementation project(':dd-java-agent:instrumentation:trace-annotation') @@ -22,5 +13,4 @@ dependencies { testImplementation 'org.apache.tomcat.embed:tomcat-embed-core:7.0.0' testImplementation deps.guava testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.9.Final' - latestDepTestImplementation group: 'io.netty', name: 'netty-all', version: '4.+' } diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle new file mode 100644 index 000000000000..a47a59025ac6 --- /dev/null +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -0,0 +1,55 @@ +ext { + minJavaVersionForTests = JavaVersion.VERSION_21 +} + +apply from: "$rootDir/gradle/java.gradle" +apply plugin: 'idea' + +muzzle { + pass { + coreJdk('21') + } +} + +idea { + module { + jdkName = '21' + } +} + +/* + * Declare previewTest, a test suite that requires the Javac/Java --enable-preview feature flag + */ +addTestSuite('previewTest') +// Configure groovy test file compilation +compilePreviewTestGroovy.configure { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(21) + } + options.compilerArgs.add("--enable-preview") +} +// Configure Java test files compilation +compilePreviewTestJava.configure { + options.compilerArgs.add("--enable-preview") +} +// Configure tests execution +previewTest.configure { + jvmArgs = ['--enable-preview'] +} +// Require the preview test suite to run as part of module check +tasks.named("check").configure { + dependsOn "previewTest" +} + +dependencies { + testImplementation project(':dd-java-agent:instrumentation:trace-annotation') +} + +// Set all compile tasks to use JDK21 but let instrumentation code targets 1.8 compatibility +project.tasks.withType(AbstractCompile).configureEach { + setJavaVersion(it, 21) +} +compileJava.configure { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/main/java/datadog/trace/instrumentation/java/concurrent/structuredconcurrency/StructuredTaskScopeInstrumentation.java b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/main/java/datadog/trace/instrumentation/java/concurrent/structuredconcurrency/StructuredTaskScopeInstrumentation.java new file mode 100644 index 000000000000..2edb1eab5026 --- /dev/null +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/main/java/datadog/trace/instrumentation/java/concurrent/structuredconcurrency/StructuredTaskScopeInstrumentation.java @@ -0,0 +1,77 @@ +package datadog.trace.instrumentation.java.concurrent.structuredconcurrency; + +import static datadog.trace.bootstrap.instrumentation.java.concurrent.AdviceUtils.capture; +import static datadog.trace.bootstrap.instrumentation.java.concurrent.ExcludeFilter.ExcludeType.FORK_JOIN_TASK; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonMap; +import static net.bytebuddy.matcher.ElementMatchers.isConstructor; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.ExcludeFilterProvider; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.Platform; +import datadog.trace.bootstrap.ContextStore; +import datadog.trace.bootstrap.InstrumentationContext; +import datadog.trace.bootstrap.instrumentation.java.concurrent.ExcludeFilter; +import datadog.trace.bootstrap.instrumentation.java.concurrent.State; +import java.util.Collection; +import java.util.Map; +import net.bytebuddy.asm.Advice; + +/** + * This instrumentation captures the active span scope at StructuredTaskScope task creation + * (SubtaskImpl). The scope is then activate and close through the Runnable instrumentation + * (SubtaskImpl implementation Runnable). + */ +@SuppressWarnings("unused") +@AutoService(InstrumenterModule.class) +public class StructuredTaskScopeInstrumentation extends InstrumenterModule.Tracing + implements Instrumenter.ForBootstrap, Instrumenter.ForSingleType, ExcludeFilterProvider { + + public StructuredTaskScopeInstrumentation() { + super("java_concurrent", "structured_task_scope"); + } + + @Override + public String instrumentedType() { + return "java.util.concurrent.StructuredTaskScope$SubtaskImpl"; + } + + @Override + public boolean isEnabled() { + return Platform.isJavaVersionAtLeast(21) && super.isEnabled(); + } + + @Override + public Map contextStore() { + return singletonMap( + "java.util.concurrent.StructuredTaskScope.SubtaskImpl", State.class.getName()); + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice(isConstructor(), getClass().getName() + "$ConstructorAdvice"); + } + + @Override + public Map> excludedClasses() { + // Prevent the ForkJoinPool instrumentation to enable the task scope too early on the carrier + // thread rather than on the expected running thread, which is virtual by default. + return singletonMap( + FORK_JOIN_TASK, singleton("java.util.concurrent.ForkJoinTask$RunnableExecuteAction")); + } + + public static final class ConstructorAdvice { + @Advice.OnMethodExit + public static void captureScope( + @Advice.This Object task // StructuredTaskScope.SubtaskImpl (can't use the type) + ) { + ContextStore contextStore = + InstrumentationContext.get( + "java.util.concurrent.StructuredTaskScope.SubtaskImpl", + "datadog.trace.bootstrap.instrumentation.java.concurrent.State"); + capture(contextStore, task, true); + } + } +} diff --git a/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/TaskRunnerInstrumentation.java b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/main/java/datadog/trace/instrumentation/java/concurrent/virtualthread/TaskRunnerInstrumentation.java similarity index 85% rename from dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/TaskRunnerInstrumentation.java rename to dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/main/java/datadog/trace/instrumentation/java/concurrent/virtualthread/TaskRunnerInstrumentation.java index 41b0e439f6f1..6999a0b6b878 100644 --- a/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/TaskRunnerInstrumentation.java +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/main/java/datadog/trace/instrumentation/java/concurrent/virtualthread/TaskRunnerInstrumentation.java @@ -1,4 +1,4 @@ -package datadog.trace.instrumentation.java.concurrent; +package datadog.trace.instrumentation.java.concurrent.virtualthread; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static datadog.trace.bootstrap.instrumentation.java.concurrent.AdviceUtils.capture; @@ -11,12 +11,17 @@ import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.Platform; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.java.concurrent.State; import java.util.Map; import net.bytebuddy.asm.Advice; +/** + * Instruments {@code TaskRunner}, internal runnable for {@code ThreadPerTaskExecutor} (JDK 19+ as + * preview, 21+ as stable), the executor with default virtual thread factory. + */ @AutoService(InstrumenterModule.class) public final class TaskRunnerInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForBootstrap, Instrumenter.ForSingleType { @@ -29,6 +34,11 @@ public String instrumentedType() { return "java.util.concurrent.ThreadPerTaskExecutor$TaskRunner"; } + @Override + public boolean isEnabled() { + return Platform.isJavaVersionAtLeast(19) && super.isEnabled(); + } + @Override public Map contextStore() { return singletonMap("java.lang.Runnable", State.class.getName()); diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy new file mode 100644 index 000000000000..7661b813a703 --- /dev/null +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -0,0 +1,167 @@ +import datadog.trace.agent.test.AgentTestRunner +import datadog.trace.api.Trace + +import java.util.concurrent.Callable +import java.util.concurrent.StructuredTaskScope + +import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace +import static datadog.trace.agent.test.utils.TraceUtils.runnableUnderTrace +import static java.time.Instant.now + +class StructuredConcurrencyTest extends AgentTestRunner { + /** + * Tests the structured task scope with a single task. + */ + def "test single task"() { + setup: + def taskScope = new StructuredTaskScope.ShutdownOnFailure() + def result = false + + when: + runUnderTrace("parent") { + def task = taskScope.fork(new Callable() { + @Trace(operationName = "child") + @Override + Boolean call() throws Exception { + return true + } + }) + taskScope.joinUntil(now() + 1) // Wait for a second at maximum + result = task.get() + } + taskScope.close() + + then: + result + assertTraces(1) { + sortSpansByStart() + trace(2) { + span(0) { + parent() + operationName "parent" + } + span(1) { + childOfPrevious() + operationName "child" + } + } + } + } + + /** + * Tests the structured task scope with a multiple tasks. + * Here is the expected task/span structure: + * 

+   *   parent
+   *   |-- child1
+   *   |-- child2
+   *   \-- child3
+   *
+ */ + def "test multiple tasks"() { + setup: + def taskScope = new StructuredTaskScope.ShutdownOnFailure() + + when: + runUnderTrace("parent") { + taskScope.fork { + runnableUnderTrace("child1") {} + } + taskScope.fork { + runnableUnderTrace("child2") {} + } + taskScope.fork { + runnableUnderTrace("child3") {} + } + taskScope.joinUntil(now() + 2) // Wait for two seconds at maximum + } + taskScope.close() + + then: + assertTraces(1) { + sortSpansByStart() + trace(4) { + span { + parent() + operationName "parent" + } + def parent = span(0) + span { + childOf(parent) + assert span.operationName.toString().startsWith("child") + } + span { + childOf(parent) + assert span.operationName.toString().startsWith("child") + } + span { + childOf(parent) + assert span.operationName.toString().startsWith("child") + } + } + } + } + + /** + * Tests the structured task scope with a multiple nested tasks. + * Here is the expected task/span structure: + * 
+   *   parent
+   *   |-- child1
+   *   |   |-- great-child1-1
+   *   |   \-- great-child1-2
+   *   \-- child2
+   *
+ */ + def "test nested tasks"() { + setup: + def taskScope = new StructuredTaskScope.ShutdownOnFailure() + + when: + runUnderTrace("parent") { + taskScope.fork { + runnableUnderTrace("child1") { + taskScope.fork { + runnableUnderTrace("great-child1-1") {} + } + taskScope.fork { + runnableUnderTrace("great-child1-2") {} + } + } + } + taskScope.fork { + runnableUnderTrace("child2") {} + } + taskScope.joinUntil(now() + 2) // Wait for two seconds at maximum + } + taskScope.close() + + then: + assertTraces(1) { + sortSpansByStart() + trace(5) { + // Check parent span + span { + parent() + operationName "parent" + } + def parent = span(0) + // Check child and great child spans + def child1 = null + for (i in 0..<4) { + span { + def name = span.operationName.toString() + if (name.startsWith("child")) { + childOf(parent) + if (name == "child1") { + child1 = span + } + } else if (name.startsWith("great-child1")) { + childOf(child1) // We can assume child1 will be set as spans are sorted by start time + } + } + } + } + } + } +} diff --git a/dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/groovy/VirtualThreadTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/test/groovy/VirtualThreadTest.groovy similarity index 99% rename from dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/groovy/VirtualThreadTest.groovy rename to dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/test/groovy/VirtualThreadTest.groovy index 15ba703c93ec..a19b5c926237 100644 --- a/dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/groovy/VirtualThreadTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/test/groovy/VirtualThreadTest.groovy @@ -11,7 +11,6 @@ import java.util.concurrent.TimeUnit import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeScope class VirtualThreadTest extends AgentTestRunner { - @Shared def executeRunnable = { e, c -> e.execute((Runnable) c) } @Shared diff --git a/dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/java/JavaAsyncChild.java b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/test/java/JavaAsyncChild.java similarity index 69% rename from dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/java/JavaAsyncChild.java rename to dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/test/java/JavaAsyncChild.java index 67266dcb122e..b93e43a5fa09 100644 --- a/dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/java/JavaAsyncChild.java +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/test/java/JavaAsyncChild.java @@ -1,9 +1,8 @@ import datadog.trace.api.Trace; import java.util.concurrent.Callable; -import java.util.concurrent.ForkJoinTask; import java.util.concurrent.atomic.AtomicBoolean; -public class JavaAsyncChild extends ForkJoinTask implements Runnable, Callable { +public class JavaAsyncChild implements Runnable, Callable { private final AtomicBoolean blockThread; private final boolean doTraceableWork; @@ -11,20 +10,6 @@ public JavaAsyncChild() { this(true, false); } - @Override - public Object getRawResult() { - return null; - } - - @Override - protected void setRawResult(final Object value) {} - - @Override - protected boolean exec() { - runImpl(); - return true; - } - public JavaAsyncChild(final boolean doTraceableWork, final boolean blockThread) { this.doTraceableWork = doTraceableWork; this.blockThread = new AtomicBoolean(blockThread); @@ -40,7 +25,7 @@ public void run() { } @Override - public Object call() throws Exception { + public Void call() { runImpl(); return null; } diff --git a/dd-java-agent/instrumentation/java-concurrent/src/test/groovy/NettyExecutorInstrumentationTest.groovy b/dd-java-agent/instrumentation/java-concurrent/src/test/groovy/NettyExecutorInstrumentationTest.groovy index 4ccbb001aced..01f2767af1fc 100644 --- a/dd-java-agent/instrumentation/java-concurrent/src/test/groovy/NettyExecutorInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/src/test/groovy/NettyExecutorInstrumentationTest.groovy @@ -195,7 +195,7 @@ class NettyExecutorInstrumentationTest extends AgentTestRunner { "schedule Runnable" | scheduleRunnable | localEventLoopGroup "schedule Callable" | scheduleCallable | localEventLoopGroup - poolName = poolImpl.class.simpleName + poolName = poolImpl?.class?.simpleName } def "#poolName '#name' reports after canceled jobs"() { @@ -274,7 +274,7 @@ class NettyExecutorInstrumentationTest extends AgentTestRunner { "schedule Runnable" | scheduleRunnable | localEventLoopGroup.next() "schedule Callable" | scheduleCallable | localEventLoopGroup.next() - poolName = poolImpl.class.simpleName + poolName = poolImpl?.class?.simpleName } def epollExecutor() { diff --git a/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenUtils.java b/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenUtils.java index 0af804a208c0..67f5950acd2d 100644 --- a/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenUtils.java +++ b/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenUtils.java @@ -290,6 +290,8 @@ public static String getUniqueModuleName(MavenProject project, MojoExecution moj private static final MethodHandle SESSION_FIELD = METHOD_HANDLES.privateFieldGetter(MavenSession.class, "session"); private static final MethodHandle LOOKUP_FIELD = + METHOD_HANDLES.privateFieldGetter("org.apache.maven.internal.impl.AbstractSession", "lookup"); + private static final MethodHandle ALTERNATIVE_LOOKUP_FIELD = METHOD_HANDLES.privateFieldGetter("org.apache.maven.internal.impl.DefaultSession", "lookup"); private static final MethodHandle LOOKUP_METHOD = METHOD_HANDLES.method("org.apache.maven.api.services.Lookup", "lookup", Class.class); @@ -301,8 +303,12 @@ public static PlexusContainer getContainer(MavenSession mavenSession) { } Object /* org.apache.maven.internal.impl.DefaultSession */ session = METHOD_HANDLES.invoke(SESSION_FIELD, mavenSession); - Object /* org.apache.maven.api.services.Lookup */ lookup = - METHOD_HANDLES.invoke(LOOKUP_FIELD, session); + Object /* org.apache.maven.api.services.Lookup */ lookup; + if (LOOKUP_FIELD != null) { + lookup = METHOD_HANDLES.invoke(LOOKUP_FIELD, session); + } else { + lookup = METHOD_HANDLES.invoke(ALTERNATIVE_LOOKUP_FIELD, session); + } return METHOD_HANDLES.invoke(LOOKUP_METHOD, lookup, PlexusContainer.class); } } diff --git a/dd-java-agent/instrumentation/selenium/src/testFixtures/groovy/datadog/trace/instrumentation/selenium/AbstractSeleniumTest.groovy b/dd-java-agent/instrumentation/selenium/src/testFixtures/groovy/datadog/trace/instrumentation/selenium/AbstractSeleniumTest.groovy index 3429b0f1d787..095a719e1762 100644 --- a/dd-java-agent/instrumentation/selenium/src/testFixtures/groovy/datadog/trace/instrumentation/selenium/AbstractSeleniumTest.groovy +++ b/dd-java-agent/instrumentation/selenium/src/testFixtures/groovy/datadog/trace/instrumentation/selenium/AbstractSeleniumTest.groovy @@ -3,7 +3,7 @@ package datadog.trace.instrumentation.selenium import com.fasterxml.jackson.databind.ObjectMapper import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.civisibility.CiVisibilityInstrumentationTest -import datadog.trace.civisibility.utils.IOUtils +import datadog.communication.util.IOUtils import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder import org.junit.jupiter.engine.JupiterTestEngine import org.junit.platform.engine.DiscoverySelector diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy index 0d3006742d00..27f81e86cffe 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy @@ -1638,7 +1638,7 @@ abstract class HttpServerTest extends WithHttpServer { } } - @Flaky(value = "https://github.com/DataDog/dd-trace-java/issues/7061", suites = ["JettyContinuationHandlerV0ForkedTest"]) + @Flaky(value = "https://github.com/DataDog/dd-trace-java/issues/7061", suites = ["JettyContinuationHandlerV0ForkedTest", "JettyContinuationHandlerV1ForkedTest"]) def 'test blocking of request for request body variant #variant'() { setup: assumeTrue(testBlocking()) diff --git a/dd-smoke-tests/backend-mock/build.gradle b/dd-smoke-tests/backend-mock/build.gradle new file mode 100644 index 000000000000..76f6e1754a64 --- /dev/null +++ b/dd-smoke-tests/backend-mock/build.gradle @@ -0,0 +1,7 @@ +apply from: "$rootDir/gradle/java.gradle" +description = 'Mock Datadog backend used by smoke tests.' + +dependencies { + api project(':dd-smoke-tests') + api testFixtures(project(':dd-java-agent:agent-ci-visibility')) +} diff --git a/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy b/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy new file mode 100644 index 000000000000..b3beb4f7c17d --- /dev/null +++ b/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy @@ -0,0 +1,275 @@ +package datadog.smoketest + +import com.fasterxml.jackson.databind.ObjectMapper +import datadog.trace.agent.test.server.http.TestHttpServer +import datadog.trace.test.util.MultipartRequestParser +import org.apache.commons.io.IOUtils +import org.msgpack.jackson.dataformat.MessagePackFactory +import spock.util.concurrent.PollingConditions + +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CopyOnWriteArrayList +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream + +import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer + +class MockBackend implements AutoCloseable { + + private static final ObjectMapper MSG_PACK_MAPPER = new ObjectMapper(new MessagePackFactory()) + private static final ObjectMapper JSON_MAPPER = new ObjectMapper() + + private final Queue> receivedTraces = new ConcurrentLinkedQueue<>() + + private final Queue> receivedCoverages = new ConcurrentLinkedQueue<>() + + private final Queue> receivedTelemetryMetrics = new ConcurrentLinkedQueue<>() + + private final Queue> receivedTelemetryDistributions = new ConcurrentLinkedQueue<>() + private final Queue> receivedLogs = new ConcurrentLinkedQueue<>() + + private final Collection> skippableTests = new CopyOnWriteArrayList<>() + private final Collection> flakyTests = new CopyOnWriteArrayList<>() + + private boolean codeCoverageEnabled = true + + private boolean testsSkippingEnabled = true + + private boolean flakyRetriesEnabled = false + + void reset() { + receivedTraces.clear() + receivedCoverages.clear() + receivedTelemetryMetrics.clear() + receivedTelemetryDistributions.clear() + receivedLogs.clear() + + skippableTests.clear() + flakyTests.clear() + } + + @Override + void close() throws Exception { + intakeServer.close() + } + + void givenFlakyRetries(boolean flakyRetries) { + this.flakyRetriesEnabled = flakyRetries + } + + void givenFlakyTest(String module, String suite, String name) { + flakyTests.add(["module": module, "suite": suite, "name": name]) + } + + void givenSkippableTest(String module, String suite, String name) { + skippableTests.add(["module": module, "suite": suite, "name": name]) + } + + String getIntakeUrl() { + return intakeServer.address.toString() + } + + private final TestHttpServer intakeServer = httpServer { + handlers { + prefix("/api/v2/citestcycle") { + def contentEncodingHeader = request.getHeader("Content-Encoding") + def gzipEnabled = contentEncodingHeader != null && contentEncodingHeader.contains("gzip") + def requestBody = gzipEnabled ? MockBackend.decompress(request.body) : request.body + def decodedEvent = MSG_PACK_MAPPER.readValue(requestBody, Map) + receivedTraces.add(decodedEvent) + + response.status(200).send() + } + + prefix("/api/v2/citestcov") { + def contentEncodingHeader = request.getHeader("Content-Encoding") + def gzipEnabled = contentEncodingHeader != null && contentEncodingHeader.contains("gzip") + def requestBody = gzipEnabled ? MockBackend.decompress(request.body) : request.body + def parsed = MultipartRequestParser.parseRequest(requestBody, request.headers.get("Content-Type")) + def coverages = parsed.get("coverage1") + for (def coverage : coverages) { + def decodedCoverage = MSG_PACK_MAPPER.readValue(coverage.get(), Map) + receivedCoverages.add(decodedCoverage) + } + + response.status(202).send() + } + + prefix("/api/v2/libraries/tests/services/setting") { + // not compressing settings response on purpose, to better mimic real backend behavior: + // it may choose to compress the response or not based on its size, + // so smaller responses (like those of /setting endpoint) are uncompressed, + // while the larger ones (skippable and flaky test lists) are compressed + response.status(200).send(('{ "data": { "type": "ci_app_tracers_test_service_settings", "id": "uuid", "attributes": { ' + + '"code_coverage": ' + codeCoverageEnabled + + ', "tests_skipping": ' + testsSkippingEnabled + + ', "flaky_test_retries_enabled": ' + flakyRetriesEnabled + '} } }').bytes) + } + + prefix("/api/v2/ci/tests/skippable") { + StringBuilder skippableTestsResponse = new StringBuilder("[") + def i = skippableTests.iterator() + while (i.hasNext()) { + def test = i.next() + skippableTestsResponse.append(""" + { + "id": "${UUID.randomUUID().toString()}", + "type": "test", + "attributes": { + "configurations": { + "test.bundle": "$test.module" + }, + "name": "$test.name", + "suite": "$test.suite" + } + } + """) + if (i.hasNext()) { + skippableTestsResponse.append(',') + } + } + skippableTestsResponse.append("]") + + response.status(200) + .addHeader("Content-Encoding", "gzip") + .send(MockBackend.compress((""" { "data": $skippableTestsResponse } """).bytes)) + } + + prefix("/api/v2/ci/libraries/tests/flaky") { + StringBuilder flakyTestsResponse = new StringBuilder("[") + def i = flakyTests.iterator() + while (i.hasNext()) { + def test = i.next() + flakyTestsResponse.append(""" + { + "id": "${UUID.randomUUID().toString()}", + "type": "test", + "attributes": { + "configurations": { + "test.bundle": "$test.module" + }, + "name": "$test.name", + "suite": "$test.suite" + } + } + """) + if (i.hasNext()) { + flakyTestsResponse.append(',') + } + } + flakyTestsResponse.append("]") + + response.status(200) + .addHeader("Content-Encoding", "gzip") + .send(MockBackend.compress((""" { "data": $flakyTestsResponse } """).bytes)) + } + + prefix("/api/v2/apmtelemetry") { + def telemetryRequest = JSON_MAPPER.readerFor(Map.class).readValue(request.body) + def requestType = telemetryRequest["request_type"] + if (requestType == "message-batch") { + for (def message : telemetryRequest["payload"]) { + def payload = message["payload"] + if (message["request_type"] == 'generate-metrics') { + receivedTelemetryMetrics.addAll((List) payload["series"]) + } else if (message["request_type"] == 'distributions') { + receivedTelemetryDistributions.addAll((List) payload["series"]) + } + } + } + response.status(202).send() + } + + prefix("/api/v2/logs") { + def contentEncodingHeader = request.getHeader("Content-Encoding") + def gzipEnabled = contentEncodingHeader != null && contentEncodingHeader.contains("gzip") + def requestBody = gzipEnabled ? MockBackend.decompress(request.body) : request.body + def decodedEvent = JSON_MAPPER.readValue(requestBody, List) + receivedLogs.addAll(decodedEvent) + + response.status(200).send() + } + } + } + + private static byte[] compress(byte[] bytes) { + def baos = new ByteArrayOutputStream() + try (GZIPOutputStream zip = new GZIPOutputStream(baos)) { + IOUtils.copy(new ByteArrayInputStream(bytes), zip) + } + return baos.toByteArray() + } + + private static byte[] decompress(byte[] bytes) { + def baos = new ByteArrayOutputStream() + try (GZIPInputStream zip = new GZIPInputStream(new ByteArrayInputStream(bytes))) { + IOUtils.copy(zip, baos) + } + return baos.toByteArray() + } + + List> waitForEvents(int expectedEventsSize) { + def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1) + try { + traceReceiveConditions.eventually { + int eventsSize = 0 + for (Map trace : receivedTraces) { + eventsSize += trace["events"].size() + } + assert eventsSize >= expectedEventsSize + } + } catch (AssertionError e) { + throw new AssertionError("Error while waiting for $expectedEventsSize trace events, received: $receivedTraces", e) + } + + List> events = new ArrayList<>() + while (!receivedTraces.isEmpty()) { + def trace = receivedTraces.poll() + events.addAll((List>) trace["events"]) + } + return events + } + + List> waitForCoverages(int expectedCoveragesSize) { + def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1) + try { + traceReceiveConditions.eventually { + int coveragesSize = 0 + for (Map trace : receivedCoverages) { + coveragesSize += trace["coverages"].size() + } + assert coveragesSize >= expectedCoveragesSize + } + } catch (AssertionError e) { + throw new AssertionError("Error while waiting for $expectedCoveragesSize coverages, received: $receivedCoverages", e) + } + + List> coverages = new ArrayList<>() + while (!receivedCoverages.isEmpty()) { + def trace = receivedCoverages.poll() + coverages.addAll((List>) trace["coverages"]) + } + return coverages + } + + List> waitForLogs(int expectedCount) { + def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1) + traceReceiveConditions.eventually { + assert receivedLogs.size() == expectedCount + } + + List> logs = new ArrayList<>() + while (!receivedLogs.isEmpty()) { + logs.add(receivedLogs.poll()) + } + return logs + } + + List> getAllReceivedTelemetryMetrics() { + return new ArrayList(receivedTelemetryMetrics) + } + + List> getAllReceivedTelemetryDistributions() { + return new ArrayList(receivedTelemetryDistributions) + } +} diff --git a/dd-smoke-tests/gradle/build.gradle b/dd-smoke-tests/gradle/build.gradle index a16b49b635ea..8063eccd959c 100644 --- a/dd-smoke-tests/gradle/build.gradle +++ b/dd-smoke-tests/gradle/build.gradle @@ -6,9 +6,8 @@ apply from: "$rootDir/gradle/java.gradle" description = 'Gradle Daemon Instrumentation Smoke Tests.' dependencies { - testImplementation project(':dd-smoke-tests') testImplementation gradleTestKit() - testImplementation testFixtures(project(':dd-java-agent:agent-ci-visibility')) + testImplementation project(':dd-smoke-tests:backend-mock') } test { diff --git a/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy b/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy index 29c686d2e08f..46eb4175a9aa 100644 --- a/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy +++ b/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy @@ -7,7 +7,6 @@ import datadog.trace.api.Platform import datadog.trace.api.config.CiVisibilityConfig import datadog.trace.api.config.GeneralConfig import datadog.trace.civisibility.CiVisibilitySmokeTest -import datadog.trace.test.util.Flaky import datadog.trace.util.Strings import okhttp3.OkHttpClient import okhttp3.Request @@ -24,6 +23,8 @@ import org.gradle.wrapper.Install import org.gradle.wrapper.PathAssembler import org.gradle.wrapper.WrapperConfiguration import org.junit.jupiter.api.Assumptions +import spock.lang.AutoCleanup +import spock.lang.IgnoreIf import spock.lang.Shared import spock.lang.TempDir import spock.util.environment.Jvm @@ -60,24 +61,70 @@ class GradleDaemonSmokeTest extends CiVisibilitySmokeTest { @TempDir Path projectFolder + @Shared + @AutoCleanup + MockBackend mockBackend = new MockBackend() + def setupSpec() { givenGradleProperties() } - @Flaky("https://github.com/DataDog/dd-trace-java/issues/7024") + def setup() { + mockBackend.reset() + } + + @IgnoreIf(reason = "Jacoco plugin does not work with OpenJ9 in older Gradle versions", value = { + Platform.isJ9() + }) + def "test legacy #projectName, v#gradleVersion, configCache: #configurationCache"() { + runGradleTest(gradleVersion, projectName, configurationCache, successExpected, flakyRetries, expectedTraces, expectedCoverages) + + where: + gradleVersion | projectName | configurationCache | successExpected | flakyRetries | expectedTraces | expectedCoverages + "3.0" | "test-succeed-old-gradle" | false | true | false | 5 | 1 + "7.6.4" | "test-succeed-legacy-instrumentation" | false | true | false | 5 | 1 + "7.6.4" | "test-succeed-multi-module-legacy-instrumentation" | false | true | false | 7 | 2 + "7.6.4" | "test-succeed-multi-forks-legacy-instrumentation" | false | true | false | 6 | 2 + "7.6.4" | "test-skip-legacy-instrumentation" | false | true | false | 2 | 0 + "7.6.4" | "test-failed-legacy-instrumentation" | false | false | false | 4 | 0 + "7.6.4" | "test-corrupted-config-legacy-instrumentation" | false | false | false | 1 | 0 + } + def "test #projectName, v#gradleVersion, configCache: #configurationCache"() { + runGradleTest(gradleVersion, projectName, configurationCache, successExpected, flakyRetries, expectedTraces, expectedCoverages) + + where: + gradleVersion | projectName | configurationCache | successExpected | flakyRetries | expectedTraces | expectedCoverages + "8.3" | "test-succeed-new-instrumentation" | false | true | false | 5 | 1 + LATEST_GRADLE_VERSION | "test-succeed-new-instrumentation" | false | true | false | 5 | 1 + "8.3" | "test-succeed-new-instrumentation" | true | true | false | 5 | 1 + LATEST_GRADLE_VERSION | "test-succeed-new-instrumentation" | true | true | false | 5 | 1 + LATEST_GRADLE_VERSION | "test-succeed-multi-module-new-instrumentation" | false | true | false | 7 | 2 + LATEST_GRADLE_VERSION | "test-succeed-multi-forks-new-instrumentation" | false | true | false | 6 | 2 + LATEST_GRADLE_VERSION | "test-skip-new-instrumentation" | false | true | false | 2 | 0 + LATEST_GRADLE_VERSION | "test-failed-new-instrumentation" | false | false | false | 4 | 0 + LATEST_GRADLE_VERSION | "test-corrupted-config-new-instrumentation" | false | false | false | 1 | 0 + LATEST_GRADLE_VERSION | "test-succeed-junit-5" | false | true | false | 5 | 1 + LATEST_GRADLE_VERSION | "test-failed-flaky-retries" | false | false | true | 8 | 0 + } + + private runGradleTest(String gradleVersion, String projectName, boolean configurationCache, boolean successExpected, boolean flakyRetries, int expectedTraces, int expectedCoverages) { givenGradleVersionIsCompatibleWithCurrentJvm(gradleVersion) givenConfigurationCacheIsCompatibleWithCurrentPlatform(configurationCache) givenGradleProjectFiles(projectName) - givenFlakyRetriesEnabled(flakyRetries) ensureDependenciesDownloaded(gradleVersion) + mockBackend.givenFlakyRetries(flakyRetries) + mockBackend.givenFlakyTest(":test", "datadog.smoke.TestFailed", "test_failed") + mockBackend.givenSkippableTest(":test", "datadog.smoke.TestSucceed", "test_to_skip_with_itr") + BuildResult buildResult = runGradleTests(gradleVersion, successExpected, configurationCache) if (successExpected) { assertBuildSuccessful(buildResult) } - verifyEventsAndCoverages(projectName, "gradle", gradleVersion, expectedTraces, expectedCoverages) + + verifyEventsAndCoverages(projectName, "gradle", gradleVersion, mockBackend.waitForEvents(expectedTraces), mockBackend.waitForCoverages(expectedCoverages)) if (configurationCache) { // if configuration cache is enabled, run the build one more time @@ -85,32 +132,8 @@ class GradleDaemonSmokeTest extends CiVisibilitySmokeTest { BuildResult buildResultWithConfigCacheEntry = runGradleTests(gradleVersion, successExpected, configurationCache) assertBuildSuccessful(buildResultWithConfigCacheEntry) - verifyEventsAndCoverages(projectName, "gradle", gradleVersion, expectedTraces, expectedCoverages) + verifyEventsAndCoverages(projectName, "gradle", gradleVersion, mockBackend.waitForEvents(expectedTraces), mockBackend.waitForCoverages(expectedCoverages)) } - - where: - gradleVersion | projectName | configurationCache | successExpected | flakyRetries | expectedTraces | expectedCoverages - "3.0" | "test-succeed-old-gradle" | false | true | false | 5 | 1 - "4.0" | "test-succeed-legacy-instrumentation" | false | true | false | 5 | 1 - "5.0" | "test-succeed-legacy-instrumentation" | false | true | false | 5 | 1 - "6.0" | "test-succeed-legacy-instrumentation" | false | true | false | 5 | 1 - "7.6.4" | "test-succeed-legacy-instrumentation" | false | true | false | 5 | 1 - "8.3" | "test-succeed-new-instrumentation" | false | true | false | 5 | 1 - LATEST_GRADLE_VERSION | "test-succeed-new-instrumentation" | false | true | false | 5 | 1 - "8.3" | "test-succeed-new-instrumentation" | true | true | false | 5 | 1 - LATEST_GRADLE_VERSION | "test-succeed-new-instrumentation" | true | true | false | 5 | 1 - "7.6.4" | "test-succeed-multi-module-legacy-instrumentation" | false | true | false | 7 | 2 - LATEST_GRADLE_VERSION | "test-succeed-multi-module-new-instrumentation" | false | true | false | 7 | 2 - "7.6.4" | "test-succeed-multi-forks-legacy-instrumentation" | false | true | false | 6 | 2 - LATEST_GRADLE_VERSION | "test-succeed-multi-forks-new-instrumentation" | false | true | false | 6 | 2 - "7.6.4" | "test-skip-legacy-instrumentation" | false | true | false | 2 | 0 - LATEST_GRADLE_VERSION | "test-skip-new-instrumentation" | false | true | false | 2 | 0 - "7.6.4" | "test-failed-legacy-instrumentation" | false | false | false | 4 | 0 - LATEST_GRADLE_VERSION | "test-failed-new-instrumentation" | false | false | false | 4 | 0 - "7.6.4" | "test-corrupted-config-legacy-instrumentation" | false | false | false | 1 | 0 - LATEST_GRADLE_VERSION | "test-corrupted-config-new-instrumentation" | false | false | false | 1 | 0 - LATEST_GRADLE_VERSION | "test-succeed-junit-5" | false | true | false | 5 | 1 - LATEST_GRADLE_VERSION | "test-failed-flaky-retries" | false | false | true | 8 | 0 } private void givenGradleProperties() { @@ -127,6 +150,7 @@ class GradleDaemonSmokeTest extends CiVisibilitySmokeTest { "-javaagent:${agentShadowJar}=" + // for convenience when debugging locally (System.getenv("DD_CIVISIBILITY_SMOKETEST_DEBUG_CHILD") != null ? "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_DEBUG_PORT)}=5055," : "") + + "${Strings.propertyNameToSystemPropertyName(GeneralConfig.TRACE_DEBUG)}=true," + "${Strings.propertyNameToSystemPropertyName(GeneralConfig.ENV)}=${TEST_ENVIRONMENT_NAME}," + "${Strings.propertyNameToSystemPropertyName(GeneralConfig.SERVICE_NAME)}=${TEST_SERVICE_NAME}," + "${Strings.propertyNameToSystemPropertyName(GeneralConfig.API_KEY_FILE)}=${ddApiKeyPath.toAbsolutePath().toString()}," + @@ -136,7 +160,7 @@ class GradleDaemonSmokeTest extends CiVisibilitySmokeTest { "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_CIPROVIDER_INTEGRATION_ENABLED)}=false," + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_JACOCO_PLUGIN_VERSION)}=$JACOCO_PLUGIN_VERSION," + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_SEGMENTS_ENABLED)}=true," + - "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL)}=${intakeServer.address.toString()}" + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL)}=${mockBackend.intakeUrl}" Files.write(testKitFolder.resolve("gradle.properties"), gradleProperties.getBytes()) } @@ -290,8 +314,4 @@ class GradleDaemonSmokeTest extends CiVisibilitySmokeTest { return root.get("version").asText() } } - - private void givenFlakyRetriesEnabled(boolean flakyRetries) { - this.flakyRetriesEnabled = flakyRetries - } } diff --git a/dd-smoke-tests/maven/build.gradle b/dd-smoke-tests/maven/build.gradle index 4a39d9a60b8f..2d352f03411e 100644 --- a/dd-smoke-tests/maven/build.gradle +++ b/dd-smoke-tests/maven/build.gradle @@ -8,8 +8,7 @@ description = 'Maven Instrumentation Smoke Tests.' dependencies { implementation group: 'org.apache.maven.wrapper', name: 'maven-wrapper', version: '3.2.0' - testImplementation project(':dd-smoke-tests') - testImplementation testFixtures(project(':dd-java-agent:agent-ci-visibility')) + testImplementation project(':dd-smoke-tests:backend-mock') } jar { diff --git a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy index 765ceb2bf5a1..7c3b44a43dd2 100644 --- a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy +++ b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy @@ -1,21 +1,23 @@ package datadog.smoketest - import datadog.trace.api.Config import datadog.trace.api.config.CiVisibilityConfig import datadog.trace.api.config.GeneralConfig import datadog.trace.civisibility.CiVisibilitySmokeTest -import datadog.trace.test.util.Flaky import datadog.trace.util.Strings import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response import org.apache.maven.wrapper.MavenWrapperMain +import org.junit.jupiter.api.Assumptions import org.slf4j.Logger import org.slf4j.LoggerFactory import org.w3c.dom.Document import org.w3c.dom.NodeList +import spock.lang.AutoCleanup +import spock.lang.Shared import spock.lang.TempDir +import spock.util.environment.Jvm import javax.xml.parsers.DocumentBuilder import javax.xml.parsers.DocumentBuilderFactory @@ -46,12 +48,25 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { @TempDir Path projectHome - @Flaky("https://github.com/DataDog/dd-trace-java/issues/7025") + @Shared + @AutoCleanup + MockBackend mockBackend = new MockBackend() + + def setup() { + mockBackend.reset() + } + def "test #projectName, v#mavenVersion"() { + Assumptions.assumeTrue(Jvm.current.isJavaVersionCompatible(minSupportedJavaVersion), + "Current JVM " + Jvm.current.javaVersion + " is not compatible with minimum required version " + minSupportedJavaVersion) + givenWrapperPropertiesFile(mavenVersion) givenMavenProjectFiles(projectName) givenMavenDependenciesAreLoaded(projectName, mavenVersion) - givenFlakyRetries(flakyRetries) + + mockBackend.givenFlakyRetries(flakyRetries) + mockBackend.givenFlakyTest("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestFailed", "test_failed") + mockBackend.givenSkippableTest("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestSucceed", "test_to_skip_with_itr") def exitCode = whenRunningMavenBuild(jacocoCoverage) @@ -60,24 +75,25 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { } else { assert exitCode != 0 } - verifyEventsAndCoverages(projectName, "maven", mavenVersion, expectedEvents, expectedCoverages) - verifyTelemetryMetrics(expectedEvents) + + verifyEventsAndCoverages(projectName, "maven", mavenVersion, mockBackend.waitForEvents(expectedEvents), mockBackend.waitForCoverages(expectedCoverages)) + verifyTelemetryMetrics(mockBackend.getAllReceivedTelemetryMetrics(), mockBackend.getAllReceivedTelemetryDistributions(), expectedEvents) where: - projectName | mavenVersion | expectedEvents | expectedCoverages | expectSuccess | flakyRetries | jacocoCoverage - "test_successful_maven_run" | "3.2.1" | 5 | 1 | true | false | true - "test_successful_maven_run" | "3.5.4" | 5 | 1 | true | false | true - "test_successful_maven_run" | "3.6.3" | 5 | 1 | true | false | true - "test_successful_maven_run" | "3.8.8" | 5 | 1 | true | false | true - "test_successful_maven_run" | "3.9.6" | 5 | 1 | true | false | true - "test_successful_maven_run_surefire_3_0_0" | "3.9.6" | 5 | 1 | true | false | true - "test_successful_maven_run_surefire_3_0_0" | LATEST_MAVEN_VERSION | 5 | 1 | true | false | true - "test_successful_maven_run_builtin_coverage" | "3.9.6" | 5 | 1 | true | false | false - "test_successful_maven_run_with_jacoco_and_argline" | "3.9.6" | 5 | 1 | true | false | true + projectName | mavenVersion | expectedEvents | expectedCoverages | expectSuccess | flakyRetries | jacocoCoverage | minSupportedJavaVersion + "test_successful_maven_run" | "3.2.1" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run" | "3.5.4" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run" | "3.6.3" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run" | "3.8.8" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run" | "3.9.7" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run_surefire_3_0_0" | "3.9.7" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run_surefire_3_0_0" | LATEST_MAVEN_VERSION | 5 | 1 | true | false | true | 17 + "test_successful_maven_run_builtin_coverage" | "3.9.7" | 5 | 1 | true | false | false | 8 + "test_successful_maven_run_with_jacoco_and_argline" | "3.9.7" | 5 | 1 | true | false | true | 8 // "expectedEvents" count for this test case does not include the spans that correspond to Cucumber steps - "test_successful_maven_run_with_cucumber" | "3.9.6" | 4 | 1 | true | false | true - "test_failed_maven_run_flaky_retries" | "3.9.6" | 8 | 1 | false | true | true - "test_successful_maven_run_junit_platform_runner" | "3.9.6" | 4 | 0 | true | false | false + "test_successful_maven_run_with_cucumber" | "3.9.7" | 4 | 1 | true | false | true | 8 + "test_failed_maven_run_flaky_retries" | "3.9.7" | 8 | 5 | false | true | true | 8 + "test_successful_maven_run_junit_platform_runner" | "3.9.7" | 4 | 0 | true | false | false | 8 } private void givenWrapperPropertiesFile(String mavenVersion) { @@ -215,6 +231,7 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { def agentArgument = "-javaagent:${agentShadowJar}=" + // for convenience when debugging locally (System.getenv("DD_CIVISIBILITY_SMOKETEST_DEBUG_CHILD") != null ? "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_DEBUG_PORT)}=5055," : "") + + "${Strings.propertyNameToSystemPropertyName(GeneralConfig.TRACE_DEBUG)}=true," + "${Strings.propertyNameToSystemPropertyName(GeneralConfig.ENV)}=${TEST_ENVIRONMENT_NAME}," + "${Strings.propertyNameToSystemPropertyName(GeneralConfig.SERVICE_NAME)}=${TEST_SERVICE_NAME}," + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_ENABLED)}=true," + @@ -223,8 +240,9 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_SOURCE_DATA_ROOT_CHECK_ENABLED)}=false," + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_GIT_UPLOAD_ENABLED)}=false," + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_COMPILER_PLUGIN_VERSION)}=${JAVAC_PLUGIN_VERSION}," + - "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL)}=${intakeServer.address.toString()}," + - "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_ONLY_KNOWN_FLAKES)}=true," // to cover flaky test retrieval logic + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL)}=${mockBackend.intakeUrl}," + + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_ONLY_KNOWN_FLAKES)}=true," + // to cover flaky test retrieval logic if (injectJacoco) { agentArgument += "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_SEGMENTS_ENABLED)}=true," + @@ -240,10 +258,6 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { return [projectHome.toAbsolutePath().toString()] } - void givenFlakyRetries(boolean flakyRetries) { - this.flakyRetriesEnabled = flakyRetries - } - private static class StreamConsumer extends Thread { final InputStream is final String messagePrefix @@ -280,7 +294,7 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { NodeList versionList = doc.getElementsByTagName("latest") if (versionList.getLength() > 0) { def version = versionList.item(0).getTextContent() - if (!version.contains('alpha')) { + if (!version.contains('alpha') && !version.contains('beta')) { LOGGER.info("Will run the 'latest' tests with version ${version}") return version } @@ -291,7 +305,7 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { } catch (Exception e) { LOGGER.warn("Could not get latest maven version", e) } - def hardcodedLatestVersion = "4.0.0-alpha-12" // latest alpha that is known to work + def hardcodedLatestVersion = "4.0.0-beta-3" // latest version that is known to work LOGGER.info("Will run the 'latest' tests with hard-coded version ${hardcodedLatestVersion}") return hardcodedLatestVersion } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 6ff2a2272900..3c3741893a0f 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -176,6 +176,7 @@ public final class ConfigDefaults { static final boolean DEFAULT_DEBUGGER_EXCEPTION_ENABLED = false; static final int DEFAULT_DEBUGGER_MAX_EXCEPTION_PER_SECOND = 100; static final boolean DEFAULT_DEBUGGER_EXCEPTION_ONLY_LOCAL_ROOT = true; + static final int DEFAULT_DEBUGGER_EXCEPTION_MAX_CAPTURED_FRAMES = 3; static final boolean DEFAULT_TRACE_REPORT_HOSTNAME = false; static final String DEFAULT_TRACE_ANNOTATIONS = null; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/DebuggerConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/DebuggerConfig.java index ebfd3afb14fb..b3037387c08d 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/DebuggerConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/DebuggerConfig.java @@ -35,6 +35,8 @@ public final class DebuggerConfig { "exception.replay.max.exception.analysis.limit"; public static final String DEBUGGER_EXCEPTION_ONLY_LOCAL_ROOT = "internal.exception.replay.only.local.root"; + public static final String DEBUGGER_EXCEPTION_MAX_CAPTURED_FRAMES = + "exception.replay.max.frames.to.capture"; public static final String THIRD_PARTY_INCLUDES = "third.party.includes"; public static final String THIRD_PARTY_EXCLUDES = "third.party.excludes"; diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java index c0fea2444284..d7521591efb8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java @@ -71,7 +71,7 @@ public ConflatingMetricsAggregator( sharedCommunicationObjects.featuresDiscovery(config), new OkHttpSink( sharedCommunicationObjects.okHttpClient, - config.getAgentUrl(), + sharedCommunicationObjects.agentUrl.toString(), V6_METRICS_ENDPOINT, config.isTracerMetricsBufferingEnabled(), false, diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.groovy index cb6b9e6d6a1d..fc47ba0ecfa9 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.groovy @@ -3,6 +3,7 @@ package datadog.trace.common.metrics import datadog.communication.ddagent.SharedCommunicationObjects import datadog.trace.api.Config import datadog.trace.test.util.DDSpecification +import okhttp3.HttpUrl class MetricsAggregatorFactoryTest extends DDSpecification { @@ -10,8 +11,10 @@ class MetricsAggregatorFactoryTest extends DDSpecification { setup: Config config = Mock(Config) config.isTracerMetricsEnabled() >> false + def sco = Mock(SharedCommunicationObjects) + sco.agentUrl = HttpUrl.parse("http://localhost:8126") expect: - def aggregator = MetricsAggregatorFactory.createMetricsAggregator(config, Mock(SharedCommunicationObjects)) + def aggregator = MetricsAggregatorFactory.createMetricsAggregator(config, sco) assert aggregator instanceof NoOpMetricsAggregator } @@ -19,8 +22,10 @@ class MetricsAggregatorFactoryTest extends DDSpecification { setup: Config config = Spy(Config.get()) config.isTracerMetricsEnabled() >> true + def sco = Mock(SharedCommunicationObjects) + sco.agentUrl = HttpUrl.parse("http://localhost:8126") expect: - def aggregator = MetricsAggregatorFactory.createMetricsAggregator(config, Mock(SharedCommunicationObjects)) + def aggregator = MetricsAggregatorFactory.createMetricsAggregator(config, sco) assert aggregator instanceof ConflatingMetricsAggregator } } diff --git a/dd-trace-ot/build.gradle b/dd-trace-ot/build.gradle index 78e010d7494d..5cf60505d8ea 100644 --- a/dd-trace-ot/build.gradle +++ b/dd-trace-ot/build.gradle @@ -45,7 +45,7 @@ dependencies { api group: 'io.opentracing', name: 'opentracing-api', version: '0.32.0' api group: 'io.opentracing', name: 'opentracing-noop', version: '0.32.0' api group: 'io.opentracing', name: 'opentracing-util', version: '0.32.0' - api group: 'io.opentracing.contrib', name: 'opentracing-tracerresolver', version: '0.1.0' + api group: 'io.opentracing.contrib', name: 'opentracing-tracerresolver', version: '0.1.6' api deps.slf4j implementation project(':dd-trace-ot:correlation-id-injection') diff --git a/docs/how_instrumentations_work.md b/docs/how_instrumentations_work.md index be7cefa7e550..2fcb8c48bde5 100644 --- a/docs/how_instrumentations_work.md +++ b/docs/how_instrumentations_work.md @@ -176,6 +176,10 @@ classpath. For example, [`ShutdownInstrumentation`](https://github.com/DataDog/dd-trace-java/blob/3e81c006b54f73aae61f88c39b52a7267267075b/dd-java-agent/instrumentation/shutdown/src/main/java/datadog/trace/instrumentation/shutdown/ShutdownInstrumentation.java#L18) or [`UrlInstrumentation`](https://github.com/DataDog/dd-trace-java/blob/3e81c006b54f73aae61f88c39b52a7267267075b/dd-java-agent/instrumentation/http-url-connection/src/main/java/datadog/trace/instrumentation/http_url_connection/UrlInstrumentation.java#L21). +> [!NOTE] +> Without classloader available, helper classes for bootstrap instrumentation must be place into the +> `:dd-java-agent:agent-bootstrap` module rather than loaded using [the default mechanism](#helper-classes). + ### Method Matching After the type is selected, the type’s target members(e.g., methods) must next be selected using the Instrumentation diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 5fd289e6f2bc..fcd72559ed51 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -23,7 +23,7 @@ final class CachedData { truth : "1.1.3", kotlin : "1.6.21", coroutines : "1.3.0", - dogstatsd : "4.3.0", + dogstatsd : "4.4.0", jnr_unixsocket: "0.38.22", jnr_posix : '3.1.19', commons : "3.2", diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 201dfcdafd6c..764c8db1dc04 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -45,6 +45,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_DEBUGGER_DIAGNOSTICS_INTERVAL; import static datadog.trace.api.ConfigDefaults.DEFAULT_DEBUGGER_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_DEBUGGER_EXCEPTION_ENABLED; +import static datadog.trace.api.ConfigDefaults.DEFAULT_DEBUGGER_EXCEPTION_MAX_CAPTURED_FRAMES; import static datadog.trace.api.ConfigDefaults.DEFAULT_DEBUGGER_EXCEPTION_ONLY_LOCAL_ROOT; import static datadog.trace.api.ConfigDefaults.DEFAULT_DEBUGGER_INSTRUMENT_THE_WORLD; import static datadog.trace.api.ConfigDefaults.DEFAULT_DEBUGGER_MAX_EXCEPTION_PER_SECOND; @@ -211,6 +212,7 @@ import static datadog.trace.api.config.DebuggerConfig.DEBUGGER_DIAGNOSTICS_INTERVAL; import static datadog.trace.api.config.DebuggerConfig.DEBUGGER_ENABLED; import static datadog.trace.api.config.DebuggerConfig.DEBUGGER_EXCEPTION_ENABLED; +import static datadog.trace.api.config.DebuggerConfig.DEBUGGER_EXCEPTION_MAX_CAPTURED_FRAMES; import static datadog.trace.api.config.DebuggerConfig.DEBUGGER_EXCEPTION_ONLY_LOCAL_ROOT; import static datadog.trace.api.config.DebuggerConfig.DEBUGGER_EXCLUDE_FILES; import static datadog.trace.api.config.DebuggerConfig.DEBUGGER_INSTRUMENT_THE_WORLD; @@ -841,6 +843,7 @@ static class HostNameHolder { private final boolean debuggerExceptionEnabled; private final int debuggerMaxExceptionPerSecond; private final boolean debuggerExceptionOnlyLocalRoot; + private final int debuggerExceptionMaxCapturedFrames; private final Set debuggerThirdPartyIncludes; private final Set debuggerThirdPartyExcludes; @@ -1087,14 +1090,17 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins } } - if (agentHostFromEnvironment == null) { - agentHostFromEnvironment = configProvider.getString(AGENT_HOST); - rebuildAgentUrl = true; - } - - if (agentPortFromEnvironment < 0) { - agentPortFromEnvironment = configProvider.getInteger(TRACE_AGENT_PORT, -1, AGENT_PORT_LEGACY); - rebuildAgentUrl = true; + // avoid merging in supplementary host/port settings when dealing with unix: URLs + if (unixSocketFromEnvironment == null) { + if (agentHostFromEnvironment == null) { + agentHostFromEnvironment = configProvider.getString(AGENT_HOST); + rebuildAgentUrl = true; + } + if (agentPortFromEnvironment < 0) { + agentPortFromEnvironment = + configProvider.getInteger(TRACE_AGENT_PORT, -1, AGENT_PORT_LEGACY); + rebuildAgentUrl = true; + } } if (agentHostFromEnvironment == null) { @@ -1592,10 +1598,6 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getBoolean( TELEMETRY_DEPENDENCY_COLLECTION_ENABLED, DEFAULT_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED); - - isTelemetryLogCollectionEnabled = - configProvider.getBoolean( - TELEMETRY_LOG_COLLECTION_ENABLED, DEFAULT_TELEMETRY_LOG_COLLECTION_ENABLED); telemetryDependencyResolutionQueueSize = configProvider.getInteger( TELEMETRY_DEPENDENCY_RESOLUTION_QUEUE_SIZE, @@ -1900,10 +1902,26 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) debuggerExceptionOnlyLocalRoot = configProvider.getBoolean( DEBUGGER_EXCEPTION_ONLY_LOCAL_ROOT, DEFAULT_DEBUGGER_EXCEPTION_ONLY_LOCAL_ROOT); + debuggerExceptionMaxCapturedFrames = + configProvider.getInteger( + DEBUGGER_EXCEPTION_MAX_CAPTURED_FRAMES, DEFAULT_DEBUGGER_EXCEPTION_MAX_CAPTURED_FRAMES); debuggerThirdPartyIncludes = tryMakeImmutableSet(configProvider.getList(THIRD_PARTY_INCLUDES)); debuggerThirdPartyExcludes = tryMakeImmutableSet(configProvider.getList(THIRD_PARTY_EXCLUDES)); + // FIXME: For the initial rollout, we default log collection to true for IAST and CI Visibility + // users. This should be removed once we default to true, and then it can also be moved up + // together with the rest of telemetry ocnfig. + final boolean telemetryLogCollectionEnabledDefault = + instrumenterConfig.isTelemetryEnabled() + && (instrumenterConfig.getIastActivation() == ProductActivation.FULLY_ENABLED + || instrumenterConfig.isCiVisibilityEnabled() + || debuggerEnabled) + || DEFAULT_TELEMETRY_LOG_COLLECTION_ENABLED; + isTelemetryLogCollectionEnabled = + configProvider.getBoolean( + TELEMETRY_LOG_COLLECTION_ENABLED, telemetryLogCollectionEnabledDefault); + awsPropagationEnabled = isPropagationEnabled(true, "aws", "aws-sdk"); sqsPropagationEnabled = isPropagationEnabled(true, "sqs"); @@ -3229,6 +3247,10 @@ public boolean isDebuggerExceptionOnlyLocalRoot() { return debuggerExceptionOnlyLocalRoot; } + public int getDebuggerExceptionMaxCapturedFrames() { + return debuggerExceptionMaxCapturedFrames; + } + public Set getThirdPartyIncludes() { return debuggerThirdPartyIncludes; } diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 544b1f52584b..232352b40cb5 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -2036,8 +2036,45 @@ class ConfigTest extends DDSpecification { !config.perfMetricsEnabled } - def "trace_agent_url overrides either host and port or unix domain"() { + def "trace_agent_url overrides default host and port or unix domain"() { setup: + if (configuredUrl != null) { + System.setProperty(PREFIX + TRACE_AGENT_URL, configuredUrl) + } else { + System.clearProperty(PREFIX + TRACE_AGENT_URL) + } + + when: + def config = new Config() + + then: + config.agentUrl == expectedUrl + config.agentHost == expectedHost + config.agentPort == expectedPort + config.agentUnixDomainSocket == expectedUnixDomainSocket + + where: + // spotless:off + configuredUrl | expectedUrl | expectedHost | expectedPort | expectedUnixDomainSocket + null | "http://localhost:8126" | "localhost" | 8126 | null + "" | "http://localhost:8126" | "localhost" | 8126 | null + "http://localhost:1234" | "http://localhost:1234" | "localhost" | 1234 | null + "http://somehost" | "http://somehost:8126" | "somehost" | 8126 | null + "http://somehost:80" | "http://somehost:80" | "somehost" | 80 | null + "https://somehost:8143" | "https://somehost:8143" | "somehost" | 8143 | null + "unix:///another/socket/path" | "unix:///another/socket/path" | "localhost" | 8126 | "/another/socket/path" + "unix:///another%2Fsocket%2Fpath" | "unix:///another%2Fsocket%2Fpath" | "localhost" | 8126 | "/another/socket/path" + "http:" | "http://localhost:8126" | "localhost" | 8126 | null + "unix:" | "http://localhost:8126" | "localhost" | 8126 | null + "1234" | "http://localhost:8126" | "localhost" | 8126 | null + ":1234" | "http://localhost:8126" | "localhost" | 8126 | null + // spotless:on + } + + def "trace_agent_url overrides configured host and port or unix domain"() { + setup: + System.setProperty(PREFIX + AGENT_HOST, "test-host") + System.setProperty(PREFIX + TRACE_AGENT_PORT, "8888") System.setProperty(PREFIX + AGENT_UNIX_DOMAIN_SOCKET, "/path/to/socket") if (configuredUrl != null) { System.setProperty(PREFIX + TRACE_AGENT_URL, configuredUrl) @@ -2056,19 +2093,19 @@ class ConfigTest extends DDSpecification { where: // spotless:off - configuredUrl | expectedUrl | expectedHost | expectedPort | expectedUnixDomainSocket - null | "http://localhost:8126" | "localhost" | 8126 | "/path/to/socket" - "" | "http://localhost:8126" | "localhost" | 8126 | "/path/to/socket" - "http://localhost:1234" | "http://localhost:1234" | "localhost" | 1234 | "/path/to/socket" - "http://somehost" | "http://somehost:8126" | "somehost" | 8126 | "/path/to/socket" - "http://somehost:80" | "http://somehost:80" | "somehost" | 80 | "/path/to/socket" - "https://somehost:8143" | "https://somehost:8143" | "somehost" | 8143 | "/path/to/socket" - "unix:///another/socket/path" | "http://localhost:8126" | "localhost" | 8126 | "/another/socket/path" - "unix:///another%2Fsocket%2Fpath" | "http://localhost:8126" | "localhost" | 8126 | "/another/socket/path" - "http:" | "http://localhost:8126" | "localhost" | 8126 | "/path/to/socket" - "unix:" | "http://localhost:8126" | "localhost" | 8126 | "/path/to/socket" - "1234" | "http://localhost:8126" | "localhost" | 8126 | "/path/to/socket" - ":1234" | "http://localhost:8126" | "localhost" | 8126 | "/path/to/socket" + configuredUrl | expectedUrl | expectedHost | expectedPort | expectedUnixDomainSocket + null | "http://test-host:8888" | "test-host" | 8888 | "/path/to/socket" + "" | "http://test-host:8888" | "test-host" | 8888 | "/path/to/socket" + "http://localhost:1234" | "http://localhost:1234" | "localhost" | 1234 | "/path/to/socket" + "http://somehost" | "http://somehost:8888" | "somehost" | 8888 | "/path/to/socket" + "http://somehost:80" | "http://somehost:80" | "somehost" | 80 | "/path/to/socket" + "https://somehost:8143" | "https://somehost:8143" | "somehost" | 8143 | "/path/to/socket" + "unix:///another/socket/path" | "unix:///another/socket/path" | "localhost" | 8126 | "/another/socket/path" + "unix:///another%2Fsocket%2Fpath" | "unix:///another%2Fsocket%2Fpath" | "localhost" | 8126 | "/another/socket/path" + "http:" | "http://test-host:8888" | "test-host" | 8888 | "/path/to/socket" + "unix:" | "http://test-host:8888" | "test-host" | 8888 | "/path/to/socket" + "1234" | "http://test-host:8888" | "test-host" | 8888 | "/path/to/socket" + ":1234" | "http://test-host:8888" | "test-host" | 8888 | "/path/to/socket" // spotless:on } diff --git a/settings.gradle b/settings.gradle index 46eccb007860..8632adc8d4c8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -90,6 +90,7 @@ include ':utils:version-utils' // smoke tests include ':dd-smoke-tests:armeria-grpc' +include ':dd-smoke-tests:backend-mock' include ':dd-smoke-tests:cli' include ':dd-smoke-tests:custom-systemloader' include ':dd-smoke-tests:dynamic-config' @@ -247,6 +248,7 @@ include ':dd-java-agent:instrumentation:jakarta-jms' include ':dd-java-agent:instrumentation:jakarta-rs-annotations-3' include ':dd-java-agent:instrumentation:java-concurrent' include ':dd-java-agent:instrumentation:java-concurrent:java-completablefuture' +include ':dd-java-agent:instrumentation:java-concurrent:java-concurrent-21' include ':dd-java-agent:instrumentation:java-concurrent:lambda-testing' include ':dd-java-agent:instrumentation:java-directbytebuffer' include ':dd-java-agent:instrumentation:java-http-client'