feat: extract sharedotlp common to go-libs (#309)

cmd/container.go

@@ -17,8 +17,8 @@ import (
 	"github.com/numary/go-libs/sharedauth"
 	"github.com/numary/go-libs/sharedlogging"
 	"github.com/numary/go-libs/sharedlogging/sharedlogginglogrus"
-	"github.com/numary/go-libs/sharedotlp/sharedotlpmetrics"
-	"github.com/numary/go-libs/sharedotlp/sharedotlptraces"
+	"github.com/numary/go-libs/sharedotlp/pkg/sharedotlpmetrics"
+	"github.com/numary/go-libs/sharedotlp/pkg/sharedotlptraces"
 	"github.com/numary/go-libs/sharedpublish"
 	"github.com/numary/go-libs/sharedpublish/sharedpublishhttp"
 	"github.com/numary/go-libs/sharedpublish/sharedpublishkafka"
@@ -113,46 +113,11 @@ func NewContainer(v *viper.Viper, userOptions ...fx.Option) *fx.App {
 	}
 
 	// Handle OpenTelemetry
-	if v.GetBool(otelTracesFlag) {
-		options = append(options, sharedotlptraces.TracesModule(sharedotlptraces.ModuleConfig{
-			Batch:    v.GetBool(otelTracesBatchFlag),
-			Exporter: v.GetString(otelTracesExporterFlag),
-			JaegerConfig: func() *sharedotlptraces.JaegerConfig {
-				if v.GetString(otelTracesExporterFlag) != sharedotlptraces.JaegerExporter {
-					return nil
-				}
-				return &sharedotlptraces.JaegerConfig{
-					Endpoint: v.GetString(otelTracesExporterJaegerEndpointFlag),
-					User:     v.GetString(otelTracesExporterJaegerUserFlag),
-					Password: v.GetString(otelTracesExporterJaegerPasswordFlag),
-				}
-			}(),
-			OTLPConfig: func() *sharedotlptraces.OTLPConfig {
-				if v.GetString(otelTracesExporterFlag) != sharedotlptraces.OTLPExporter {
-					return nil
-				}
-				return &sharedotlptraces.OTLPConfig{
-					Mode:     v.GetString(otelTracesExporterOTLPModeFlag),
-					Endpoint: v.GetString(otelTracesExporterOTLPEndpointFlag),
-					Insecure: v.GetBool(otelTracesExporterOTLPInsecureFlag),
-				}
-			}(),
-		}))
+	if m := sharedotlptraces.CLITracesModule(v); m != nil {
+		options = append(options, m)
 	}
-	if v.GetBool(otelMetricsFlag) {
-		options = append(options, sharedotlpmetrics.MetricsModule(sharedotlpmetrics.MetricsModuleConfig{
-			Exporter: v.GetString(otelMetricsExporterFlag),
-			OTLPConfig: func() *sharedotlpmetrics.OTLPMetricsConfig {
-				if v.GetString(otelMetricsExporterFlag) != sharedotlpmetrics.OTLPMetricsExporter {
-					return nil
-				}
-				return &sharedotlpmetrics.OTLPMetricsConfig{
-					Mode:     v.GetString(otelMetricsExporterOTLPModeFlag),
-					Endpoint: v.GetString(otelMetricsExporterOTLPEndpointFlag),
-					Insecure: v.GetBool(otelMetricsExporterOTLPInsecureFlag),
-				}
-			}(),
-		}))
+	if m := sharedotlpmetrics.CLIMetricsModule(v); m != nil {
+		options = append(options, m)
 	}
 
 	switch v.GetString(lockStrategyFlag) {
@@ -226,10 +191,10 @@ func NewContainer(v *viper.Viper, userOptions ...fx.Option) *fx.App {
 
 	options = append(options,
 		fx.Decorate(fx.Annotate(func(driver storage.Driver, mp metric.MeterProvider) storage.Driver {
-			if v.GetBool(otelTracesFlag) {
+			if v.GetBool(sharedotlptraces.OtelTracesFlag) {
 				driver = opentelemetrytraces.WrapStorageDriver(driver)
 			}
-			if v.GetBool(otelMetricsFlag) {
+			if v.GetBool(sharedotlpmetrics.OtelMetricsFlag) {
 				driver = opentelemetrymetrics.WrapStorageDriver(driver, mp)
 			}
 			return driver
@@ -280,7 +245,7 @@ func NewContainer(v *viper.Viper, userOptions ...fx.Option) *fx.App {
 		cc.AddAllowHeaders("authorization")
 
 		res = append(res, cors.New(cc))
-		if v.GetBool(otelTracesFlag) {
+		if v.GetBool(sharedotlptraces.OtelTracesFlag) {
 			res = append(res, otelgin.Middleware(ServiceName, otelgin.WithTracerProvider(tp)))
 		} else {
 			res = append(res, func(context *gin.Context) {
@@ -292,7 +257,7 @@ func NewContainer(v *viper.Viper, userOptions ...fx.Option) *fx.App {
 		}
 		res = append(res, middlewares.Log())
 		var writer io.Writer = os.Stderr
-		if v.GetBool(otelTracesFlag) {
+		if v.GetBool(sharedotlptraces.OtelTracesFlag) {
 			writer = ioutil.Discard
 			res = append(res, opentelemetrytraces.Middleware())
 		}

cmd/container_test.go

@@ -11,7 +11,8 @@ import (
 	"time"
 
 	"github.com/ThreeDotsLabs/watermill/pubsub/gochannel"
-	"github.com/numary/go-libs/sharedotlp/sharedotlptraces"
+	"github.com/numary/go-libs/sharedotlp/pkg/sharedotlpmetrics"
+	"github.com/numary/go-libs/sharedotlp/pkg/sharedotlptraces"
 	"github.com/numary/ledger/internal/pgtesting"
 	"github.com/numary/ledger/pkg/bus"
 	"github.com/numary/ledger/pkg/core"
@@ -55,8 +56,8 @@ func TestContainers(t *testing.T) {
 			name: "default-with-opentelemetry-traces-on-stdout",
 			init: func(v *viper.Viper) {
 				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
-				v.Set(otelTracesFlag, true)
-				v.Set(otelTracesExporterFlag, "stdout")
+				v.Set(sharedotlptraces.OtelTracesFlag, true)
+				v.Set(sharedotlptraces.OtelTracesExporterFlag, "stdout")
 			},
 			options: []fx.Option{
 				fx.Invoke(fx.Annotate(func(lc fx.Lifecycle, t *testing.T, exp trace.SpanExporter, options ...trace.TracerProviderOption) {
@@ -90,9 +91,9 @@ func TestContainers(t *testing.T) {
 			name: "default-with-opentelemetry-traces-on-stdout-and-batch",
 			init: func(v *viper.Viper) {
 				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
-				v.Set(otelTracesFlag, true)
-				v.Set(otelTracesExporterFlag, "stdout")
-				v.Set(otelTracesBatchFlag, true)
+				v.Set(sharedotlptraces.OtelTracesFlag, true)
+				v.Set(sharedotlptraces.OtelTracesExporterFlag, "stdout")
+				v.Set(sharedotlptraces.OtelTracesBatchFlag, true)
 			},
 			options: []fx.Option{
 				fx.Invoke(fx.Annotate(func(lc fx.Lifecycle, t *testing.T, exp trace.SpanExporter, options ...trace.TracerProviderOption) {
@@ -128,32 +129,32 @@ func TestContainers(t *testing.T) {
 			name: "default-with-opentelemetry-traces-on-otlp",
 			init: func(v *viper.Viper) {
 				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
-				v.Set(otelTracesFlag, true)
-				v.Set(otelTracesExporterFlag, "otlp")
+				v.Set(sharedotlptraces.OtelTracesFlag, true)
+				v.Set(sharedotlptraces.OtelTracesExporterFlag, "otlp")
 			},
 		},
 		{
 			name: "default-with-opentelemetry-traces-on-jaeger",
 			init: func(v *viper.Viper) {
 				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
-				v.Set(otelTracesFlag, true)
-				v.Set(otelTracesExporterFlag, "jaeger")
+				v.Set(sharedotlptraces.OtelTracesFlag, true)
+				v.Set(sharedotlptraces.OtelTracesExporterFlag, "jaeger")
 			},
 		},
 		{
 			name: "default-with-opentelemetry-metrics-on-noop",
 			init: func(v *viper.Viper) {
 				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
-				v.Set(otelMetricsFlag, true)
-				v.Set(otelMetricsExporterFlag, "noop")
+				v.Set(sharedotlpmetrics.OtelMetricsFlag, true)
+				v.Set(sharedotlpmetrics.OtelMetricsExporterFlag, "noop")
 			},
 		},
 		{
 			name: "default-with-opentelemetry-metrics-on-otlp",
 			init: func(v *viper.Viper) {
 				v.Set(storageDriverFlag, sqlstorage.SQLite.String())
-				v.Set(otelMetricsFlag, true)
-				v.Set(otelMetricsExporterFlag, "otlp")
+				v.Set(sharedotlpmetrics.OtelMetricsFlag, true)
+				v.Set(sharedotlpmetrics.OtelMetricsExporterFlag, "otlp")
 			},
 		},
 		{

cmd/root.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"path"
 
+	"github.com/numary/go-libs/sharedotlp/pkg/sharedotlpmetrics"
+	"github.com/numary/go-libs/sharedotlp/pkg/sharedotlptraces"
 	"github.com/numary/ledger/cmd/internal"
 	"github.com/numary/ledger/pkg/redis"
 	_ "github.com/numary/ledger/pkg/storage/sqlstorage/migrates/9-add-pre-post-volumes"
@@ -20,39 +22,26 @@ const (
 	storageSQLiteDBNameFlag             = "storage.sqlite.db_name"
 	storagePostgresConnectionStringFlag = "storage.postgres.conn_string"
 	// Deprecated
-	storageCacheFlag                     = "storage.cache"
-	serverHttpBindAddressFlag            = "server.http.bind_address"
-	uiHttpBindAddressFlag                = "ui.http.bind_address"
-	lockStrategyFlag                     = "lock-strategy"
-	lockStrategyRedisUrlFlag             = "lock-strategy-redis-url"
-	lockStrategyRedisDurationFlag        = "lock-strategy-redis-duration"
-	lockStrategyRedisRetryFlag           = "lock-strategy-redis-retry"
-	lockStrategyRedisTLSEnabledFlag      = "lock-strategy-redis-tls-enabled"
-	lockStrategyRedisTLSInsecureFlag     = "lock-strategy-redis-tls-insecure"
-	otelTracesFlag                       = "otel-traces"
-	otelTracesBatchFlag                  = "otel-traces-batch"
-	otelTracesExporterFlag               = "otel-traces-exporter"
-	otelTracesExporterJaegerEndpointFlag = "otel-traces-exporter-jaeger-endpoint"
-	otelTracesExporterJaegerUserFlag     = "otel-traces-exporter-jaeger-user"
-	otelTracesExporterJaegerPasswordFlag = "otel-traces-exporter-jaeger-password"
-	otelTracesExporterOTLPModeFlag       = "otel-traces-exporter-otlp-mode"
-	otelTracesExporterOTLPEndpointFlag   = "otel-traces-exporter-otlp-endpoint"
-	otelTracesExporterOTLPInsecureFlag   = "otel-traces-exporter-otlp-insecure"
-	otelMetricsFlag                      = "otel-metrics"
-	otelMetricsExporterFlag              = "otel-metrics-exporter"
-	otelMetricsExporterOTLPModeFlag      = "otel-metrics-exporter-otlp-mode"
-	otelMetricsExporterOTLPEndpointFlag  = "otel-metrics-exporter-otlp-endpoint"
-	otelMetricsExporterOTLPInsecureFlag  = "otel-metrics-exporter-otlp-insecure"
-	publisherKafkaEnabledFlag            = "publisher-kafka-enabled"
-	publisherKafkaBrokerFlag             = "publisher-kafka-broker"
-	publisherKafkaSASLEnabled            = "publisher-kafka-sasl-enabled"
-	publisherKafkaSASLUsername           = "publisher-kafka-sasl-username"
-	publisherKafkaSASLPassword           = "publisher-kafka-sasl-password"
-	publisherKafkaSASLMechanism          = "publisher-kafka-sasl-mechanism"
-	publisherKafkaSASLScramSHASize       = "publisher-kafka-sasl-scram-sha-size"
-	publisherKafkaTLSEnabled             = "publisher-kafka-tls-enabled"
-	publisherTopicMappingFlag            = "publisher-topic-mapping"
-	publisherHttpEnabledFlag             = "publisher-http-enabled"
+	storageCacheFlag                 = "storage.cache"
+	serverHttpBindAddressFlag        = "server.http.bind_address"
+	uiHttpBindAddressFlag            = "ui.http.bind_address"
+	lockStrategyFlag                 = "lock-strategy"
+	lockStrategyRedisUrlFlag         = "lock-strategy-redis-url"
+	lockStrategyRedisDurationFlag    = "lock-strategy-redis-duration"
+	lockStrategyRedisRetryFlag       = "lock-strategy-redis-retry"
+	lockStrategyRedisTLSEnabledFlag  = "lock-strategy-redis-tls-enabled"
+	lockStrategyRedisTLSInsecureFlag = "lock-strategy-redis-tls-insecure"
+
+	publisherKafkaEnabledFlag      = "publisher-kafka-enabled"
+	publisherKafkaBrokerFlag       = "publisher-kafka-broker"
+	publisherKafkaSASLEnabled      = "publisher-kafka-sasl-enabled"
+	publisherKafkaSASLUsername     = "publisher-kafka-sasl-username"
+	publisherKafkaSASLPassword     = "publisher-kafka-sasl-password"
+	publisherKafkaSASLMechanism    = "publisher-kafka-sasl-mechanism"
+	publisherKafkaSASLScramSHASize = "publisher-kafka-sasl-scram-sha-size"
+	publisherKafkaTLSEnabled       = "publisher-kafka-tls-enabled"
+	publisherTopicMappingFlag      = "publisher-topic-mapping"
+	publisherHttpEnabledFlag       = "publisher-http-enabled"
 
 	authBearerEnabledFlag           = "auth-bearer-enabled"
 	authBearerIntrospectUrlFlag     = "auth-bearer-introspect-url"
@@ -127,20 +116,6 @@ func NewRootCommand() *cobra.Command {
 	root.PersistentFlags().Bool(storageCacheFlag, true, "Storage cache")
 	root.PersistentFlags().String(serverHttpBindAddressFlag, "localhost:3068", "API bind address")
 	root.PersistentFlags().String(uiHttpBindAddressFlag, "localhost:3068", "UI bind address")
-	root.PersistentFlags().Bool(otelTracesFlag, false, "Enable OpenTelemetry traces support")
-	root.PersistentFlags().Bool(otelTracesBatchFlag, false, "Use OpenTelemetry batching")
-	root.PersistentFlags().String(otelTracesExporterFlag, "stdout", "OpenTelemetry traces exporter")
-	root.PersistentFlags().String(otelTracesExporterJaegerEndpointFlag, "", "OpenTelemetry traces Jaeger exporter endpoint")
-	root.PersistentFlags().String(otelTracesExporterJaegerUserFlag, "", "OpenTelemetry traces Jaeger exporter user")
-	root.PersistentFlags().String(otelTracesExporterJaegerPasswordFlag, "", "OpenTelemetry traces Jaeger exporter password")
-	root.PersistentFlags().String(otelTracesExporterOTLPModeFlag, "grpc", "OpenTelemetry traces OTLP exporter mode (grpc|http)")
-	root.PersistentFlags().String(otelTracesExporterOTLPEndpointFlag, "", "OpenTelemetry traces grpc endpoint")
-	root.PersistentFlags().Bool(otelTracesExporterOTLPInsecureFlag, false, "OpenTelemetry traces grpc insecure")
-	root.PersistentFlags().Bool(otelMetricsFlag, false, "Enable OpenTelemetry metrics support")
-	root.PersistentFlags().String(otelMetricsExporterFlag, "stdout", "OpenTelemetry metrics exporter")
-	root.PersistentFlags().String(otelMetricsExporterOTLPModeFlag, "grpc", "OpenTelemetry metrics OTLP exporter mode (grpc|http)")
-	root.PersistentFlags().String(otelMetricsExporterOTLPEndpointFlag, "", "OpenTelemetry metrics grpc endpoint")
-	root.PersistentFlags().Bool(otelMetricsExporterOTLPInsecureFlag, false, "OpenTelemetry metrics grpc insecure")
 	root.PersistentFlags().String(lockStrategyFlag, "memory", "Lock strategy (memory, none, redis)")
 	root.PersistentFlags().String(lockStrategyRedisUrlFlag, "", "Redis url when using redis locking strategy")
 	root.PersistentFlags().Duration(lockStrategyRedisDurationFlag, redis.DefaultLockDuration, "Lock duration")
@@ -164,6 +139,8 @@ func NewRootCommand() *cobra.Command {
 	root.PersistentFlags().Bool(authBearerUseScopesFlag, false, "Use scopes as defined by rfc https://datatracker.ietf.org/doc/html/rfc8693")
 	root.PersistentFlags().String(commitPolicyFlag, "", "Transaction commit policy (default or allow-past-timestamps)")
 
+	sharedotlptraces.InitOTLPTracesFlags(root.PersistentFlags())
+	sharedotlpmetrics.InitOTLPMetricsFlags(root.PersistentFlags())
 	internal.InitHTTPBasicFlags(root)
 	internal.InitAnalyticsFlags(root, DefaultSegmentWriteKey)
 

cmd/server_start.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/numary/go-libs/sharedlogging"
 	"github.com/numary/go-libs/sharedlogging/sharedlogginglogrus"
+	"github.com/numary/go-libs/sharedotlp/pkg/sharedotlptraces"
 	"github.com/numary/ledger/pkg/api"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -23,7 +24,7 @@ func NewServerStart() *cobra.Command {
 			if viper.GetBool(debugFlag) {
 				l.Level = logrus.DebugLevel
 			}
-			if viper.GetBool(otelTracesFlag) {
+			if viper.GetBool(sharedotlptraces.OtelTracesFlag) {
 				l.AddHook(otellogrus.NewHook(otellogrus.WithLevels(
 					logrus.PanicLevel,
 					logrus.FatalLevel,