Proposed Internal Datadog Module Component (#24650)

Component(s):
processor/datadog

The Datadog processor is a component the needs to be deprecated in favor
of the Datadog connector. The migration from one component to the other
will take effect after two minor releases. Therefore, during the
deprecation process, code that is used by the Datadog processor and
connector can be shared instead of duplicated.

[Link to relevant
issue](#19740)

---------

Co-authored-by: Yang Song <songy23@users.noreply.github.com>

.github/CODEOWNERS

@@ -112,6 +112,7 @@ extension/storage/filestorage/                           @open-telemetry/collect
 
 internal/aws/                                            @open-telemetry/collector-contrib-approvers @Aneurysm9 @mxiamxia
 internal/coreinternal/                                   @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers
+internal/datadog/                                        @open-telemetry/collector-contrib-approvers @mx-psi @gbbr @dineshg13
 internal/docker/                                         @open-telemetry/collector-contrib-approvers @rmfitzpatrick @jamesmoessis
 internal/filter/                                         @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers
 internal/k8sconfig/                                      @open-telemetry/collector-contrib-approvers @dmitryax

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -103,6 +103,7 @@ body:
       - extension/storage/filestorage
       - internal/aws
       - internal/core
+      - internal/datadog
       - internal/docker
       - internal/filter
       - internal/k8sconfig

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -97,6 +97,7 @@ body:
       - extension/storage/filestorage
       - internal/aws
       - internal/core
+      - internal/datadog
       - internal/docker
       - internal/filter
       - internal/k8sconfig

.github/ISSUE_TEMPLATE/other.yaml

@@ -97,6 +97,7 @@ body:
       - extension/storage/filestorage
       - internal/aws
       - internal/core
+      - internal/datadog
       - internal/docker
       - internal/filter
       - internal/k8sconfig

.github/dependabot.yml

@@ -472,6 +472,11 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
+  - package-ecosystem: "gomod"
+    directory: "/internal/datadog"
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
   - package-ecosystem: "gomod"
     directory: "/internal/docker"
     schedule:
@@ -1097,8 +1102,3 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
-  - package-ecosystem: "gomod"
-    directory: "/receiver/sqlqueryreceiver"
-    schedule:
-      interval: "weekly"
-      day: "wednesday"

cmd/configschema/go.mod

@@ -509,6 +509,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.82.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/winperfcounters v0.82.0 // indirect
@@ -1119,4 +1120,6 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/obse
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8stest => ../../internal/k8stest
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ../../internal/datadog
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/jaegerremotesampling => ../../extension/jaegerremotesampling

cmd/otelcontribcol/builder-config.yaml

@@ -241,6 +241,7 @@ replaces:
   - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/cassandraexporter => ../../exporter/cassandraexporter
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/udplogreceiver => ../../receiver/udplogreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk => ../../internal/splunk
+  - github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ../../internal/datadog
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/podmanreceiver => ../../receiver/podmanreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/flinkmetricsreceiver => ../../receiver/flinkmetricsreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver => ../../receiver/awsecscontainermetricsreceiver

cmd/otelcontribcol/go.mod

@@ -518,6 +518,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray v0.82.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/docker v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.82.0 // indirect
@@ -782,6 +783,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/udplo
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk => ../../internal/splunk
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ../../internal/datadog
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/podmanreceiver => ../../receiver/podmanreceiver
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/flinkmetricsreceiver => ../../receiver/flinkmetricsreceiver

go.mod

@@ -499,6 +499,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.82.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/docker v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.82.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.82.0 // indirect
@@ -1093,6 +1094,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zooke
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/dockerobserver => ./extension/observer/dockerobserver
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog => ./internal/datadog
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/jaegerremotesampling => ./extension/jaegerremotesampling
 
 // see https://github.com/google/gnostic/issues/262

internal/datadog/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

processor/datadogprocessor/agent.go → internal/datadog/agent.go

@@ -1,7 +1,7 @@
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
-package datadogprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/datadogprocessor"
+package datadog // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog"
 
 import (
 	"context"
@@ -24,8 +24,8 @@ import (
 // computed for the resource spans.
 const keyStatsComputed = "_dd.stats_computed"
 
-// traceagent specifies a minimal trace agent instance that is able to process traces and output stats.
-type traceagent struct {
+// TraceAgent specifies a minimal trace agent instance that is able to process traces and output stats.
+type TraceAgent struct {
 	*agent.Agent
 
 	// pchan specifies the channel that will be used to output Datadog Trace Agent API Payloads
@@ -41,12 +41,12 @@ type traceagent struct {
 
 // newAgent creates a new unstarted traceagent using the given context. Call Start to start the traceagent.
 // The out channel will receive outoing stats payloads resulting from spans ingested using the Ingest method.
-func newAgent(ctx context.Context, out chan pb.StatsPayload) *traceagent {
-	return newAgentWithConfig(ctx, traceconfig.New(), out)
+func NewAgent(ctx context.Context, out chan pb.StatsPayload) *TraceAgent {
+	return NewAgentWithConfig(ctx, traceconfig.New(), out)
 }
 
 // newAgentWithConfig creates a new traceagent with the given config cfg. Used in tests; use newAgent instead.
-func newAgentWithConfig(ctx context.Context, cfg *traceconfig.AgentConfig, out chan pb.StatsPayload) *traceagent {
+func NewAgentWithConfig(ctx context.Context, cfg *traceconfig.AgentConfig, out chan pb.StatsPayload) *TraceAgent {
 	// disable the HTTP receiver
 	cfg.ReceiverPort = 0
 	// set the API key to succeed startup; it is never used nor needed
@@ -68,15 +68,15 @@ func newAgentWithConfig(ctx context.Context, cfg *traceconfig.AgentConfig, out c
 	// lastly, start the OTLP receiver, which will be used to introduce ResourceSpans into the traceagent,
 	// so that we can transform them to Datadog spans and receive stats.
 	a.OTLPReceiver = api.NewOTLPReceiver(pchan, cfg)
-	return &traceagent{
+	return &TraceAgent{
 		Agent: a,
 		exit:  make(chan struct{}),
 		pchan: pchan,
 	}
 }
 
 // Start starts the traceagent, making it ready to ingest spans.
-func (p *traceagent) Start() {
+func (p *TraceAgent) Start() {
 	// we don't need to start the full agent, so we only start a set of minimal
 	// components needed to compute stats:
 	for _, starter := range []interface{ Start() }{
@@ -98,7 +98,7 @@ func (p *traceagent) Start() {
 }
 
 // Stop stops the traceagent, making it unable to ingest spans. Do not call Ingest after Stop.
-func (p *traceagent) Stop() {
+func (p *TraceAgent) Stop() {
 	for _, stopper := range []interface{ Stop() }{
 		p.Concentrator,
 		p.ClientStatsAggregator,
@@ -115,7 +115,7 @@ func (p *traceagent) Stop() {
 
 // goDrain drains the TraceWriter channel, ensuring it won't block. We don't need the traces,
 // nor do we have a running TraceWrite. We just want the outgoing stats.
-func (p *traceagent) goDrain() {
+func (p *TraceAgent) goDrain() {
 	p.wg.Add(1)
 	go func() {
 		defer p.wg.Done()
@@ -132,7 +132,7 @@ func (p *traceagent) goDrain() {
 
 // Ingest processes the given spans within the traceagent and outputs stats through the output channel
 // provided to newAgent. Do not call Ingest on an unstarted or stopped traceagent.
-func (p *traceagent) Ingest(ctx context.Context, traces ptrace.Traces) {
+func (p *TraceAgent) Ingest(ctx context.Context, traces ptrace.Traces) {
 	rspanss := traces.ResourceSpans()
 	for i := 0; i < rspanss.Len(); i++ {
 		rspans := rspanss.At(i)
@@ -148,7 +148,7 @@ func (p *traceagent) Ingest(ctx context.Context, traces ptrace.Traces) {
 
 // goProcesses runs the main loop which takes incoming payloads, processes them and generates stats.
 // It then picks up those stats and converts them to metrics.
-func (p *traceagent) goProcess() {
+func (p *TraceAgent) goProcess() {
 	for i := 0; i < runtime.NumCPU(); i++ {
 		p.wg.Add(1)
 		go func() {
@@ -167,10 +167,10 @@ func (p *traceagent) goProcess() {
 	}
 }
 
-var _ ingester = (*traceagent)(nil)
+var _ Ingester = (*TraceAgent)(nil)
 
-// An ingester is able to ingest traces. Implemented by traceagent.
-type ingester interface {
+// An Ingester is able to ingest traces. Implemented by traceagent.
+type Ingester interface {
 	// Start starts the ingester.
 	Start()
 

processor/datadogprocessor/agent_test.go → internal/datadog/agent_test.go

@@ -1,7 +1,7 @@
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
-package datadogprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/datadogprocessor"
+package datadog // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/datadog"
 
 import (
 	"context"
@@ -21,7 +21,7 @@ func TestTraceAgentConfig(t *testing.T) {
 	require.NotZero(t, cfg.ReceiverPort)
 
 	out := make(chan pb.StatsPayload)
-	agnt := newAgentWithConfig(context.Background(), cfg, out)
+	agnt := NewAgentWithConfig(context.Background(), cfg, out)
 	require.Zero(t, cfg.ReceiverPort)
 	require.NotEmpty(t, cfg.Endpoints[0].APIKey)
 	require.Equal(t, metrics.UnsetHostnamePlaceholder, cfg.Hostname)
@@ -33,7 +33,7 @@ func TestTraceAgent(t *testing.T) {
 	cfg.BucketInterval = 50 * time.Millisecond
 	out := make(chan pb.StatsPayload, 10)
 	ctx := context.Background()
-	a := newAgentWithConfig(ctx, cfg, out)
+	a := NewAgentWithConfig(ctx, cfg, out)
 	a.Start()
 	defer a.Stop()
 

internal/datadog/go.mod

@@ -0,0 +1,72 @@
+module github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog
+
+go 1.19
+
+require (
+	github.com/DataDog/datadog-agent/pkg/trace v0.48.0-devel
+	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics v0.5.2
+	github.com/stretchr/testify v1.8.4
+	go.opentelemetry.io/collector/pdata v1.0.0-rcv0014
+)
+
+require (
+	github.com/DataDog/datadog-agent/pkg/obfuscate v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/cgroups v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/log v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/pointer v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/scrubber v0.47.0-rc.3 // indirect
+	github.com/DataDog/datadog-go/v5 v5.1.1 // indirect
+	github.com/DataDog/go-tuf v0.3.0--fix-localmeta-fork // indirect
+	github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.5.2 // indirect
+	github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.5.2 // indirect
+	github.com/DataDog/sketches-go v1.4.2 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect
+	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/godbus/dbus/v5 v5.0.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/karrick/godirwalk v1.17.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
+	github.com/outcaste-io/ristretto v0.2.1 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.5.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.2 // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.opentelemetry.io/collector/semconv v0.82.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
+	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/grpc v1.56.2 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)