Skip to content

Commit

Permalink
Merge branch 'master' into joey/aws-sns
Browse files Browse the repository at this point in the history
  • Loading branch information
joeyzhao2018 authored May 28, 2024
2 parents 4c00b5e + 4cdcb05 commit c255ef1
Show file tree
Hide file tree
Showing 58 changed files with 929 additions and 186 deletions.
1 change: 1 addition & 0 deletions .gitlab/benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ variables:
image: $BASE_CI_IMAGE
script:
- export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
- export CIRCLE_CI_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.circleci_token --with-decryption --query "Parameter.Value" --out text)
- git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
- git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
artifacts:
Expand Down
32 changes: 30 additions & 2 deletions .gitlab/single-step-instrumentation-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
- export DD_APP_KEY_ONBOARDING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.dd-app-key-onboarding --with-decryption --query "Parameter.Value" --out text)
- export ONBOARDING_AWS_INFRA_SUBNET_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.aws-infra-subnet-id --with-decryption --query "Parameter.Value" --out text)
- export ONBOARDING_AWS_INFRA_SECURITY_GROUPS_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.aws-infra-securiy-groups-id --with-decryption --query "Parameter.Value" --out text)
- export ONBOARDING_AWS_INFRA_IAM_INSTANCE_PROFILE=ec2InstanceRole
- export PULUMI_CONFIG_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.pulumi-config-passphrase --with-decryption --query "Parameter.Value" --out text)
#Install plugins for PULUMI you need connect to gh. Sometimes this problem arises: GitHub rate limit exceeded
- export GITHUB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.gh-token --with-decryption --query "Parameter.Value" --out text)
Expand All @@ -33,10 +34,34 @@
paths:
- system-tests/reports/

oci-internal-publish:
extends: .oci-internal-publish
stage: package
needs: [ package-oci ]
rules:
- when: on_success
variables:
FLAVOR: datadog-apm-library-java

oci-internal-test-ecr-publish:
stage: package
needs: [ oci-internal-publish ]
rules:
- when: on_success
trigger:
project: DataDog/public-images
branch: main
strategy: depend
variables:
IMG_SOURCES: registry.ddbuild.io/ci/remote-updates/datadog-apm-library-java:pipeline-${CI_PIPELINE_ID}-1
IMG_DESTINATIONS: apm-library-java-package:pipeline-${CI_PIPELINE_ID}
IMG_REGISTRIES: agent-qa


onboarding_tests:
extends: .base_job_onboarding_tests
stage: single-step-instrumentation-tests
needs: [ package, package-arm]
needs: [ package, package-arm, oci-internal-test-ecr-publish ]
rules:
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "master"
when: on_success
Expand All @@ -50,11 +75,14 @@ onboarding_tests:
SCENARIO: [SIMPLE_HOST_AUTO_INJECTION]
- ONBOARDING_FILTER_WEBLOG: [test-app-java-container,test-app-java-buildpack,test-app-java-alpine]
SCENARIO: [SIMPLE_CONTAINER_AUTO_INJECTION]
- ONBOARDING_FILTER_WEBLOG: [test-app-java,test-app-java-container,test-app-java-buildpack,test-app-java-alpine]
SCENARIO: [INSTALLER_AUTO_INJECTION]
script:
- git clone https://git@github.com/DataDog/system-tests.git system-tests
- cp packaging/*.rpm system-tests/binaries
- cp packaging/*.deb system-tests/binaries
- export DD_INSTALLER_LIBRARY_VERSION="pipeline-${CI_PIPELINE_ID}"
- ls system-tests/binaries
- cd system-tests
- ./build.sh -i runner
- timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env prod --vm-library ${TEST_LIBRARY} --vm-provider aws --vm-skip-branches ubuntu18_amd64
- timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env prod --vm-library ${TEST_LIBRARY} --vm-provider aws --vm-skip-branches ubuntu18_amd64
15 changes: 11 additions & 4 deletions buildSrc/src/main/groovy/InstrumentPlugin.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -178,17 +178,20 @@ interface InstrumentWorkParameters extends WorkParameters {
abstract class InstrumentAction implements WorkAction<InstrumentWorkParameters> {
private static final Object lock = new Object()
private static ClassLoader pluginCL
private static volatile long lastBuildStamp
private static String cachedPluginPath
private static volatile long cachedBuildStamp

@Override
void execute() {
// reset shared class-loaders each time a new build starts
long buildStamp = parameters.buildStartedTime.get()
if (lastBuildStamp < buildStamp || !pluginCL) {
String pluginPath = parameters.pluginClassPath.join(':')
if (rebuildSharedClassLoader(buildStamp, pluginPath)) {
synchronized (lock) {
if (lastBuildStamp < buildStamp || !pluginCL) {
if (rebuildSharedClassLoader(buildStamp, pluginPath)) {
pluginCL = createClassLoader(parameters.pluginClassPath)
lastBuildStamp = buildStamp
cachedPluginPath = pluginPath
cachedBuildStamp = buildStamp
}
}
}
Expand All @@ -199,6 +202,10 @@ abstract class InstrumentAction implements WorkAction<InstrumentWorkParameters>
InstrumentingPlugin.instrumentClasses(plugins, instrumentingCL, sourceDirectory, targetDirectory)
}

static boolean rebuildSharedClassLoader(long buildStamp, String pluginPath) {
return cachedBuildStamp < buildStamp || cachedPluginPath != pluginPath
}

static ClassLoader createClassLoader(cp, parent = InstrumentAction.classLoader) {
return new URLClassLoader(cp*.toURI()*.toURL() as URL[], parent as ClassLoader)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,10 @@ public static synchronized void run(
if (config.isDebuggerExceptionEnabled()) {
defaultExceptionDebugger =
new DefaultExceptionDebugger(
configurationUpdater, classNameFiltering, EXCEPTION_CAPTURE_INTERVAL);
configurationUpdater,
classNameFiltering,
EXCEPTION_CAPTURE_INTERVAL,
config.getDebuggerMaxExceptionPerSecond());
DebuggerContext.initExceptionDebugger(defaultExceptionDebugger);
}
if (config.isDebuggerInstrumentTheWorld()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.datadog.debugger.agent.DebuggerAgent;
import com.datadog.debugger.exception.ExceptionProbeManager.ThrowableState;
import com.datadog.debugger.sink.Snapshot;
import com.datadog.debugger.util.CircuitBreaker;
import com.datadog.debugger.util.ClassNameFiltering;
import com.datadog.debugger.util.ExceptionHelper;
import datadog.trace.bootstrap.debugger.DebuggerContext;
Expand All @@ -31,24 +32,29 @@ public class DefaultExceptionDebugger implements DebuggerContext.ExceptionDebugg
private final ExceptionProbeManager exceptionProbeManager;
private final ConfigurationUpdater configurationUpdater;
private final ClassNameFiltering classNameFiltering;
private final CircuitBreaker circuitBreaker;

public DefaultExceptionDebugger(
ConfigurationUpdater configurationUpdater,
ClassNameFiltering classNameFiltering,
Duration captureInterval) {
Duration captureInterval,
int maxExceptionPerSecond) {
this(
new ExceptionProbeManager(classNameFiltering, captureInterval),
configurationUpdater,
classNameFiltering);
classNameFiltering,
maxExceptionPerSecond);
}

DefaultExceptionDebugger(
ExceptionProbeManager exceptionProbeManager,
ConfigurationUpdater configurationUpdater,
ClassNameFiltering classNameFiltering) {
ClassNameFiltering classNameFiltering,
int maxExceptionPerSecond) {
this.exceptionProbeManager = exceptionProbeManager;
this.configurationUpdater = configurationUpdater;
this.classNameFiltering = classNameFiltering;
this.circuitBreaker = new CircuitBreaker(maxExceptionPerSecond, Duration.ofSeconds(1));
}

@Override
Expand All @@ -59,6 +65,9 @@ public void handleException(Throwable t, AgentSpan span) {
}
return;
}
if (!circuitBreaker.trip()) {
return;
}
String fingerprint = Fingerprinter.fingerprint(t, classNameFiltering);
if (fingerprint == null) {
LOGGER.debug("Unable to fingerprint exception", t);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package com.datadog.debugger.util;

import java.time.Duration;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

// CircuitBreaker is a simple circuit breaker implementation that allows a certain number of trips
// within a time window. If the number of trips exceeds the limit, the circuit breaker will trip and
// return false until the time window has passed.
public class CircuitBreaker {
private static final Logger LOGGER = LoggerFactory.getLogger(CircuitBreaker.class);

private final int maxTrips;
private final Duration timeWindow;
private AtomicInteger count = new AtomicInteger(0);
private volatile long lastResetTime = System.currentTimeMillis();
private volatile long lastLoggingTime = System.currentTimeMillis();

public CircuitBreaker(int maxTrips, Duration timeWindow) {
this.maxTrips = maxTrips;
this.timeWindow = timeWindow;
}

public boolean trip() {
int localCount = count.incrementAndGet();
if (localCount > maxTrips) {
long currentTime = System.currentTimeMillis();
if (currentTime - lastLoggingTime > Duration.ofMinutes(1).toMillis()) {
lastLoggingTime = currentTime;
LOGGER.debug("Circuit breaker opened");
}
if (currentTime - lastResetTime > timeWindow.toMillis()) {
lastResetTime = currentTime;
localCount = 1;
count.set(localCount);
}
}
return localCount <= maxTrips;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ public void setUp() {
configurationUpdater = mock(ConfigurationUpdater.class);
classNameFiltering = new ClassNameFiltering(emptySet());
exceptionDebugger =
new DefaultExceptionDebugger(configurationUpdater, classNameFiltering, Duration.ofHours(1));
new DefaultExceptionDebugger(
configurationUpdater, classNameFiltering, Duration.ofHours(1), 100);
listener = new TestSnapshotListener(createConfig(), mock(ProbeStatusSink.class));
DebuggerAgentHelper.injectSink(listener);
}
Expand Down Expand Up @@ -203,7 +204,7 @@ public void doubleNestedException() {
public void filteringOutErrors() {
ExceptionProbeManager manager = mock(ExceptionProbeManager.class);
exceptionDebugger =
new DefaultExceptionDebugger(manager, configurationUpdater, classNameFiltering);
new DefaultExceptionDebugger(manager, configurationUpdater, classNameFiltering, 100);
exceptionDebugger.handleException(new AssertionError("test"), mock(AgentSpan.class));
verify(manager, times(0)).isAlreadyInstrumented(any());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,7 @@ private TestSnapshotListener setupExceptionDebugging(
DebuggerContext.initValueSerializer(new JsonSnapshotSerializer());
DefaultExceptionDebugger exceptionDebugger =
new DefaultExceptionDebugger(
exceptionProbeManager, configurationUpdater, classNameFiltering);
exceptionProbeManager, configurationUpdater, classNameFiltering, 100);
DebuggerContext.initExceptionDebugger(exceptionDebugger);
configurationUpdater.accept(REMOTE_CONFIG, null);
return listener;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package com.datadog.debugger.util;

import static org.junit.jupiter.api.Assertions.*;

import java.time.Duration;
import java.util.concurrent.locks.LockSupport;
import org.junit.jupiter.api.Test;

class CircuitBreakerTest {

@Test
void noBreaker() {
CircuitBreaker cb = new CircuitBreaker(3, Duration.ofMillis(10));
for (int i = 0; i < 10; i++) {
assertTrue(cb.trip());
LockSupport.parkNanos(Duration.ofMillis(50).toNanos());
}
}

@Test
void breaker() {
CircuitBreaker cb = new CircuitBreaker(3, Duration.ofMillis(200));
for (int i = 0; i < 3; i++) {
assertTrue(cb.trip());
}
for (int i = 0; i < 100; i++) {
assertFalse(cb.trip());
}
LockSupport.parkNanos(Duration.ofMillis(250).toNanos());
for (int i = 0; i < 3; i++) {
assertTrue(cb.trip());
}
for (int i = 0; i < 100; i++) {
assertFalse(cb.trip());
}
}
}
66 changes: 66 additions & 0 deletions dd-java-agent/agent-otel/otel-bootstrap/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
plugins {
id "com.github.johnrengelman.shadow"
}

apply from: "$rootDir/gradle/java.gradle"
apply plugin: 'instrument'

configurations {
embeddedClasspath {
visible = false
canBeConsumed = false
canBeResolved = true
}
instrumentPluginClasspath {
visible = false
canBeConsumed = false
canBeResolved = true
}
}

instrument.plugins = ['datadog.opentelemetry.tooling.OtelShimGradlePlugin']

minimumInstructionCoverage = 0.0
minimumBranchCoverage = 0.0

forbiddenApis {
ignoreFailures = true
}
spotbugs {
onlyAnalyze = ['none']
}

dependencies {
// latest OpenTelemetry API for drop-in support; instrumented at build-time with our shim
embeddedClasspath group: 'io.opentelemetry', name: 'opentelemetry-api', version: '1.38.0'

implementation project(':dd-java-agent:agent-otel:otel-shim')

instrumentPluginClasspath project(':dd-java-agent:agent-otel:otel-tooling')
}

// unpack embeddedClasspath to same path as compiled classes so it can get instrumented
tasks.register('unpackJars', Copy) {
dependsOn configurations.embeddedClasspath
exclude 'META-INF/'
from {
configurations.embeddedClasspath.collect { zipTree(it) }
}
into compileJava.destinationDirectory
}
tasks.named('compileJava') {
dependsOn 'unpackJars'
}

shadowJar {
dependencies deps.excludeShared

exclude 'io/opentelemetry/context/internal/shaded/**'

relocate 'io.opentelemetry', 'datadog.trace.bootstrap.otel'
relocate 'datadog.opentelemetry.shim', 'datadog.trace.bootstrap.otel.shim'
}

jar {
archiveClassifier = 'unbundled'
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// placeholder to activate the compiler task
11 changes: 11 additions & 0 deletions dd-java-agent/agent-otel/otel-shim/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
apply from: "$rootDir/gradle/java.gradle"

minimumInstructionCoverage = 0.0
minimumBranchCoverage = 0.0

dependencies {
// minimum OpenTelemetry API version this shim is compatible with
compileOnly group: 'io.opentelemetry', name: 'opentelemetry-api', version: '1.4.0'

implementation project(':internal-api')
}
Loading

0 comments on commit c255ef1

Please sign in to comment.