From 4a106fbc8fd802d9c94c9ac35867e5eb5e79b3c9 Mon Sep 17 00:00:00 2001 From: Mohamed Osman Date: Wed, 14 Dec 2022 21:15:40 -0500 Subject: [PATCH] improving unit test for otel --- .../proctelemetry/process_telemetry.go | 41 ++++--- .../proctelemetry/process_telemetry_test.go | 105 ++++++++++++++++-- 2 files changed, 113 insertions(+), 33 deletions(-) diff --git a/service/internal/proctelemetry/process_telemetry.go b/service/internal/proctelemetry/process_telemetry.go index 8006ec2d27c..34aa5f0b2e9 100644 --- a/service/internal/proctelemetry/process_telemetry.go +++ b/service/internal/proctelemetry/process_telemetry.go @@ -27,8 +27,8 @@ import ( "go.opentelemetry.io/otel/attribute" otelmetric "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/metric/instrument" - "go.opentelemetry.io/otel/metric/instrument/asyncint64" "go.opentelemetry.io/otel/metric/instrument/asyncfloat64" + "go.opentelemetry.io/otel/metric/instrument/asyncint64" "go.opentelemetry.io/otel/metric/unit" "go.uber.org/multierr" "go.uber.org/zap" @@ -38,9 +38,8 @@ import ( ) const ( - scopeName = "go.opentelemetry.io/collector/service/internal/proctelemetry" + scopeName = "go.opentelemetry.io/collector/service/internal/proctelemetry" processNameKey = "process_name" - ) // processMetrics is a struct that contains views related to process metrics (cpu, mem, etc) @@ -64,10 +63,10 @@ type processMetrics struct { otelCpuSeconds asyncfloat64.Counter otelRssMemory asyncint64.Counter - meter otelmetric.Meter - logger *zap.Logger - useOtelForMetrics bool - otelAttrs []attribute.KeyValue + meter otelmetric.Meter + logger *zap.Logger + useOtelForMetrics bool + otelAttrs []attribute.KeyValue // mu protects everything bellow. mu sync.Mutex @@ -80,7 +79,7 @@ func NewProcessMetrics(logger *zap.Logger, registry *featuregate.Registry, meter startTimeUnixNano: time.Now().UnixNano(), ballastSizeBytes: ballastSizeBytes, ms: &runtime.MemStats{}, - useOtelForMetrics: registry.IsEnabled(obsreportconfig.UseOtelForInternalMetricsfeatureGateID), + useOtelForMetrics: registry.IsEnabled(obsreportconfig.UseOtelForInternalMetricsfeatureGateID), logger: logger, } @@ -106,7 +105,7 @@ func (pm *processMetrics) RegisterProcessMetrics(ctx context.Context, ocRegistry return err } - err = pm.recordWithOtel(context.Background()) + err = pm.recordWithOtel(ctx) if err != nil { return err } @@ -129,37 +128,37 @@ func (pm *processMetrics) createOtelMetrics() error { var errors, err error pm.otelProcessUptime, err = pm.meter.AsyncFloat64().Counter( - "process/uptime", + "process_uptime", instrument.WithDescription("Uptime of the process"), instrument.WithUnit(unit.Milliseconds)) errors = multierr.Append(errors, err) pm.otelAllocMem, err = pm.meter.AsyncInt64().Counter( - "process/runtime/heap_alloc_bytes", + "process_runtime_heap_alloc_bytes", instrument.WithDescription("Bytes of allocated heap objects (see 'go doc runtime.MemStats.HeapAlloc')"), instrument.WithUnit(unit.Bytes)) errors = multierr.Append(errors, err) pm.otelTotalAllocMem, err = pm.meter.AsyncInt64().Counter( - "process/runtime/total_alloc_bytes", + "process_runtime_total_alloc_bytes", instrument.WithDescription("Cumulative bytes allocated for heap objects (see 'go doc runtime.MemStats.TotalAlloc')"), instrument.WithUnit(unit.Bytes)) errors = multierr.Append(errors, err) pm.otelSysMem, err = pm.meter.AsyncInt64().Counter( - "process/runtime/total_sys_memory_bytes", + "process_runtime_total_sys_memory_bytes", instrument.WithDescription("Total bytes of memory obtained from the OS (see 'go doc runtime.MemStats.Sys')"), instrument.WithUnit(unit.Bytes)) errors = multierr.Append(errors, err) pm.otelCpuSeconds, err = pm.meter.AsyncFloat64().Counter( - "process/cpu_seconds", + "process_cpu_seconds", instrument.WithDescription("Total CPU user and system time in seconds"), instrument.WithUnit(unit.Milliseconds)) errors = multierr.Append(errors, err) pm.otelRssMemory, err = pm.meter.AsyncInt64().Counter( - "process/memory/rss", + "process_memory_rss", instrument.WithDescription("Total physical memory (resident set size)"), instrument.WithUnit(unit.Bytes)) errors = multierr.Append(errors, err) @@ -171,33 +170,33 @@ func (pm *processMetrics) recordWithOtel(ctx context.Context) error { var errors, err error err = pm.meter.RegisterCallback([]instrument.Asynchronous{pm.otelProcessUptime}, func(ctx context.Context) { processTimeMs := 1000 * pm.updateProcessUptime() - pm.otelProcessUptime.Observe(ctx, processTimeMs, pm.otelAttrs...) + pm.otelProcessUptime.Observe(ctx, processTimeMs) }) errors = multierr.Append(errors, err) err = pm.meter.RegisterCallback([]instrument.Asynchronous{pm.otelAllocMem}, func(ctx context.Context) { - pm.otelAllocMem.Observe(ctx, pm.updateAllocMem(), pm.otelAttrs...) + pm.otelAllocMem.Observe(ctx, pm.updateAllocMem()) }) errors = multierr.Append(errors, err) err = pm.meter.RegisterCallback([]instrument.Asynchronous{pm.otelTotalAllocMem}, func(ctx context.Context) { - pm.otelTotalAllocMem.Observe(ctx, pm.updateTotalAllocMem(), pm.otelAttrs...) + pm.otelTotalAllocMem.Observe(ctx, pm.updateTotalAllocMem()) }) errors = multierr.Append(errors, err) err = pm.meter.RegisterCallback([]instrument.Asynchronous{pm.otelSysMem}, func(ctx context.Context) { - pm.otelSysMem.Observe(ctx, pm.updateSysMem(), pm.otelAttrs...) + pm.otelSysMem.Observe(ctx, pm.updateSysMem()) }) errors = multierr.Append(errors, err) err = pm.meter.RegisterCallback([]instrument.Asynchronous{pm.otelCpuSeconds}, func(ctx context.Context) { cpuMs := 1000 * pm.updateCPUSeconds() - pm.otelCpuSeconds.Observe(ctx, cpuMs, pm.otelAttrs...) + pm.otelCpuSeconds.Observe(ctx, cpuMs) }) errors = multierr.Append(errors, err) err = pm.meter.RegisterCallback([]instrument.Asynchronous{pm.otelRssMemory}, func(ctx context.Context) { - pm.otelRssMemory.Observe(ctx, pm.updateRSSMemory(), pm.otelAttrs...) + pm.otelRssMemory.Observe(ctx, pm.updateRSSMemory()) }) errors = multierr.Append(errors, err) diff --git a/service/internal/proctelemetry/process_telemetry_test.go b/service/internal/proctelemetry/process_telemetry_test.go index 1b3a16cc61a..b2775e50a42 100644 --- a/service/internal/proctelemetry/process_telemetry_test.go +++ b/service/internal/proctelemetry/process_telemetry_test.go @@ -16,17 +16,39 @@ package proctelemetry import ( "context" + "net/http" + "net/http/httptest" "runtime" "testing" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + io_prometheus_client "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opencensus.io/metric" "go.opencensus.io/metric/metricdata" - otelmetric "go.opentelemetry.io/otel/metric" + "go.opencensus.io/stats/view" + otelprom "go.opentelemetry.io/otel/exporters/prometheus" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/configtelemetry" + "go.opentelemetry.io/collector/internal/obsreportconfig" ) +type testTelemetry struct { + component.TelemetrySettings + views []*view.View + promHandler http.Handler + meterProvider *sdkmetric.MeterProvider + expectedMetrics []string +} + var expectedMetrics = []string{ // Changing a metric name is a breaking change. // Adding new metrics is ok as long it follows the conventions described at @@ -39,23 +61,82 @@ var expectedMetrics = []string{ "process/memory/rss", } +var otelExpectedMetrics = []string{ + // OTel Go adds `_total` suffix + "process_uptime_total", + "process_runtime_heap_alloc_bytes_total", + "process_runtime_total_alloc_bytes_total", + "process_runtime_total_sys_memory_bytes_total", + "process_cpu_seconds_total", + "process_memory_rss_total", +} + +func setupTelemetry(t *testing.T) testTelemetry { + settings := testTelemetry{ + TelemetrySettings: componenttest.NewNopTelemetrySettings(), + expectedMetrics: otelExpectedMetrics, + } + settings.TelemetrySettings.MetricsLevel = configtelemetry.LevelNormal + + obsMetrics := obsreportconfig.Configure(configtelemetry.LevelNormal) + settings.views = obsMetrics.Views + err := view.Register(settings.views...) + require.NoError(t, err) + + promReg := prometheus.NewRegistry() + exporter, err := otelprom.New(otelprom.WithRegisterer(promReg), otelprom.WithoutUnits()) + require.NoError(t, err) + + settings.meterProvider = sdkmetric.NewMeterProvider( + sdkmetric.WithResource(resource.Empty()), + sdkmetric.WithReader(exporter), + ) + settings.TelemetrySettings.MeterProvider = settings.meterProvider + + settings.promHandler = promhttp.HandlerFor(promReg, promhttp.HandlerOpts{}) + + t.Cleanup(func() { assert.NoError(t, settings.meterProvider.Shutdown(context.Background())) }) + + return settings +} + +func fetchPrometheusMetrics(handler http.Handler) (map[string]*io_prometheus_client.MetricFamily, error) { + req, err := http.NewRequest("GET", "/metrics", nil) + if err != nil { + return nil, err + } + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + var parser expfmt.TextParser + return parser.TextToMetricFamilies(rr.Body) +} + func TestOtelProcessTelemetry(t *testing.T) { + tel := setupTelemetry(t) + pm := &processMetrics{ startTimeUnixNano: time.Now().UnixNano(), ballastSizeBytes: 0, ms: &runtime.MemStats{}, - useOtelForMetrics: true, + useOtelForMetrics: true, } - meterProvider := otelmetric.NewNoopMeterProvider() - pm.meter = meterProvider.Meter("test") + + pm.meter = tel.MeterProvider.Meter("test") require.NoError(t, pm.RegisterProcessMetrics(context.Background(), nil)) - assert.NotNil(t, pm.otelProcessUptime) - assert.NotNil(t, pm.otelAllocMem) - assert.NotNil(t, pm.otelTotalAllocMem) - assert.NotNil(t, pm.otelSysMem) - assert.NotNil(t, pm.otelCpuSeconds) - assert.NotNil(t, pm.otelRssMemory) + mp, _ := fetchPrometheusMetrics(tel.promHandler) + + for _, metricName := range tel.expectedMetrics { + _, _ = view.RetrieveData(metricName) + metric, ok := mp[metricName] + require.True(t, ok) + require.True(t, len(metric.Metric) == 1) + require.True(t, metric.GetType() == io_prometheus_client.MetricType_COUNTER) + metricValue := metric.Metric[0].GetCounter().GetValue() + assert.True(t, metricValue > 0) + } } func TestOCProcessTelemetry(t *testing.T) { @@ -64,7 +145,7 @@ func TestOCProcessTelemetry(t *testing.T) { startTimeUnixNano: time.Now().UnixNano(), ballastSizeBytes: 0, ms: &runtime.MemStats{}, - useOtelForMetrics: false, + useOtelForMetrics: false, } require.NoError(t, pm.RegisterProcessMetrics(context.Background(), registry)) @@ -101,7 +182,7 @@ func TestOCProcessTelemetry(t *testing.T) { func TestProcessTelemetryFailToRegister(t *testing.T) { pm := &processMetrics{ - useOtelForMetrics: false, + useOtelForMetrics: false, } for _, metricName := range expectedMetrics { t.Run(metricName, func(t *testing.T) {