diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index ac6c4d5237e..1b74911b141 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -60,54 +60,40 @@ type Indices struct { // Configuration describes the configuration properties needed to connect to an ElasticSearch cluster type Configuration struct { - Servers []string `mapstructure:"server_urls" valid:"required,url"` - RemoteReadClusters []string `mapstructure:"remote_read_clusters"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password" json:"-"` - TokenFilePath string `mapstructure:"token_file"` - PasswordFilePath string `mapstructure:"password_file"` - AllowTokenFromContext bool `mapstructure:"-"` - Sniffer bool `mapstructure:"sniffer"` // https://github.com/olivere/elastic/wiki/Sniffing - SnifferTLSEnabled bool `mapstructure:"sniffer_tls_enabled"` - MaxDocCount int `mapstructure:"-"` // Defines maximum number of results to fetch from storage per query - MaxSpanAge time.Duration `mapstructure:"-"` // configures the maximum lookback on span reads - NumShards int64 `mapstructure:"num_shards"` - NumReplicas int64 `mapstructure:"num_replicas"` - PrioritySpanTemplate int64 `mapstructure:"priority_span_template"` - PriorityServiceTemplate int64 `mapstructure:"priority_service_template"` - PriorityDependenciesTemplate int64 `mapstructure:"priority_dependencies_template"` - Timeout time.Duration `mapstructure:"-"` - BulkSize int `mapstructure:"-"` - BulkWorkers int `mapstructure:"-"` - BulkActions int `mapstructure:"-"` - BulkFlushInterval time.Duration `mapstructure:"-"` - IndexPrefix string `mapstructure:"index_prefix"` - // Deprecated: use Indices and IndexOptions instead. - IndexDateLayoutSpans string `mapstructure:"-"` - // Deprecated: use Indices and IndexOptions instead. - IndexDateLayoutServices string `mapstructure:"-"` - // Deprecated: use Indices and IndexOptions instead. - IndexDateLayoutSampling string `mapstructure:"-"` - // Deprecated: use Indices and IndexOptions instead. - IndexDateLayoutDependencies string `mapstructure:"-"` - // Deprecated: use Indices and IndexOptions instead. - IndexRolloverFrequencySpans string `mapstructure:"-"` - // Deprecated: use Indices and IndexOptions instead. - IndexRolloverFrequencyServices string `mapstructure:"-"` - // Deprecated: use Indices and IndexOptions instead. - IndexRolloverFrequencySampling string `mapstructure:"-"` - Indices Indices `mapstructure:"indices"` - ServiceCacheTTL time.Duration `mapstructure:"service_cache_ttl"` - AdaptiveSamplingLookback time.Duration `mapstructure:"-"` - Tags TagsAsFields `mapstructure:"tags_as_fields"` - Enabled bool `mapstructure:"-"` - TLS tlscfg.Options `mapstructure:"tls"` - UseReadWriteAliases bool `mapstructure:"use_aliases"` - CreateIndexTemplates bool `mapstructure:"create_mappings"` - UseILM bool `mapstructure:"use_ilm"` - Version uint `mapstructure:"version"` - LogLevel string `mapstructure:"log_level"` - SendGetBodyAs string `mapstructure:"send_get_body_as"` + Servers []string `mapstructure:"server_urls" valid:"required,url"` + RemoteReadClusters []string `mapstructure:"remote_read_clusters"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password" json:"-"` + TokenFilePath string `mapstructure:"token_file"` + PasswordFilePath string `mapstructure:"password_file"` + AllowTokenFromContext bool `mapstructure:"-"` + Sniffer bool `mapstructure:"sniffer"` // https://github.com/olivere/elastic/wiki/Sniffing + SnifferTLSEnabled bool `mapstructure:"sniffer_tls_enabled"` + MaxDocCount int `mapstructure:"-"` // Defines maximum number of results to fetch from storage per query + MaxSpanAge time.Duration `mapstructure:"-"` // configures the maximum lookback on span reads + NumShards int64 `mapstructure:"num_shards"` + NumReplicas int64 `mapstructure:"num_replicas"` + PrioritySpanTemplate int64 `mapstructure:"priority_span_template"` + PriorityServiceTemplate int64 `mapstructure:"priority_service_template"` + PriorityDependenciesTemplate int64 `mapstructure:"priority_dependencies_template"` + Timeout time.Duration `mapstructure:"-"` + BulkSize int `mapstructure:"-"` + BulkWorkers int `mapstructure:"-"` + BulkActions int `mapstructure:"-"` + BulkFlushInterval time.Duration `mapstructure:"-"` + IndexPrefix string `mapstructure:"index_prefix"` + Indices Indices `mapstructure:"indices"` + ServiceCacheTTL time.Duration `mapstructure:"service_cache_ttl"` + AdaptiveSamplingLookback time.Duration `mapstructure:"-"` + Tags TagsAsFields `mapstructure:"tags_as_fields"` + Enabled bool `mapstructure:"-"` + TLS tlscfg.Options `mapstructure:"tls"` + UseReadWriteAliases bool `mapstructure:"use_aliases"` + CreateIndexTemplates bool `mapstructure:"create_mappings"` + UseILM bool `mapstructure:"use_ilm"` + Version uint `mapstructure:"version"` + LogLevel string `mapstructure:"log_level"` + SendGetBodyAs string `mapstructure:"send_get_body_as"` } // TagsAsFields holds configuration for tag schema. @@ -315,17 +301,17 @@ func (c *Configuration) ApplyDefaults(source *Configuration) { // GetIndexRolloverFrequencySpansDuration returns jaeger-span index rollover frequency duration func (c *Configuration) GetIndexRolloverFrequencySpansDuration() time.Duration { - return getIndexRolloverFrequencyDuration(c.IndexRolloverFrequencySpans) + return getIndexRolloverFrequencyDuration(c.Indices.Spans.RolloverFrequency) } // GetIndexRolloverFrequencyServicesDuration returns jaeger-service index rollover frequency duration func (c *Configuration) GetIndexRolloverFrequencyServicesDuration() time.Duration { - return getIndexRolloverFrequencyDuration(c.IndexRolloverFrequencyServices) + return getIndexRolloverFrequencyDuration(c.Indices.Services.RolloverFrequency) } // GetIndexRolloverFrequencySamplingDuration returns jaeger-sampling index rollover frequency duration func (c *Configuration) GetIndexRolloverFrequencySamplingDuration() time.Duration { - return getIndexRolloverFrequencyDuration(c.IndexRolloverFrequencySampling) + return getIndexRolloverFrequencyDuration(c.Indices.Sampling.RolloverFrequency) } // GetIndexRolloverFrequencyDuration returns the index rollover frequency duration for the given frequency string diff --git a/plugin/storage/es/factory.go b/plugin/storage/es/factory.go index d9c7f624e8d..cc662c00f87 100644 --- a/plugin/storage/es/factory.go +++ b/plugin/storage/es/factory.go @@ -236,8 +236,8 @@ func createSpanReader( MaxDocCount: cfg.MaxDocCount, MaxSpanAge: cfg.MaxSpanAge, IndexPrefix: cfg.IndexPrefix, - SpanIndexDateLayout: cfg.IndexDateLayoutSpans, - ServiceIndexDateLayout: cfg.IndexDateLayoutServices, + SpanIndexDateLayout: cfg.Indices.Spans.DateLayout, + ServiceIndexDateLayout: cfg.Indices.Services.DateLayout, SpanIndexRolloverFrequency: cfg.GetIndexRolloverFrequencySpansDuration(), ServiceIndexRolloverFrequency: cfg.GetIndexRolloverFrequencyServicesDuration(), TagDotReplacement: cfg.Tags.DotReplacement, @@ -270,8 +270,8 @@ func createSpanWriter( writer := esSpanStore.NewSpanWriter(esSpanStore.SpanWriterParams{ Client: clientFn, IndexPrefix: cfg.IndexPrefix, - SpanIndexDateLayout: cfg.IndexDateLayoutSpans, - ServiceIndexDateLayout: cfg.IndexDateLayoutServices, + SpanIndexDateLayout: cfg.Indices.Spans.DateLayout, + ServiceIndexDateLayout: cfg.Indices.Services.DateLayout, AllTagsAsFields: cfg.Tags.AllAsFields, TagKeysAsFields: tags, TagDotReplacement: cfg.Tags.DotReplacement, @@ -301,7 +301,7 @@ func (f *Factory) CreateSamplingStore(int /* maxBuckets */) (samplingstore.Store Client: f.getPrimaryClient, Logger: f.logger, IndexPrefix: f.primaryConfig.IndexPrefix, - IndexDateLayout: f.primaryConfig.IndexDateLayoutSampling, + IndexDateLayout: f.primaryConfig.Indices.Sampling.DateLayout, IndexRolloverFrequency: f.primaryConfig.GetIndexRolloverFrequencySamplingDuration(), Lookback: f.primaryConfig.AdaptiveSamplingLookback, MaxDocCount: f.primaryConfig.MaxDocCount, @@ -345,7 +345,7 @@ func createDependencyReader( Client: clientFn, Logger: logger, IndexPrefix: cfg.IndexPrefix, - IndexDateLayout: cfg.IndexDateLayoutDependencies, + IndexDateLayout: cfg.Indices.Dependencies.RolloverFrequency, MaxDocCount: cfg.MaxDocCount, UseReadWriteAliases: cfg.UseReadWriteAliases, }) diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 3c14177681a..f9d96d7b6ad 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -363,17 +363,17 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { cfg.RemoteReadClusters = strings.Split(remoteReadClusters, ",") } - cfg.IndexRolloverFrequencySpans = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencySpans)) - cfg.IndexRolloverFrequencyServices = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencyServices)) - cfg.IndexRolloverFrequencySampling = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencySampling)) + cfg.Indices.Spans.RolloverFrequency = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencySpans)) + cfg.Indices.Services.RolloverFrequency = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencyServices)) + cfg.Indices.Sampling.RolloverFrequency = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencySampling)) separator := v.GetString(cfg.namespace + suffixIndexDateSeparator) - cfg.IndexDateLayoutSpans = initDateLayout(cfg.IndexRolloverFrequencySpans, separator) - cfg.IndexDateLayoutServices = initDateLayout(cfg.IndexRolloverFrequencyServices, separator) - cfg.IndexDateLayoutSampling = initDateLayout(cfg.IndexRolloverFrequencySampling, separator) + cfg.Indices.Spans.DateLayout = initDateLayout(cfg.Indices.Spans.RolloverFrequency, separator) + cfg.Indices.Services.DateLayout = initDateLayout(cfg.Indices.Services.RolloverFrequency, separator) + cfg.Indices.Sampling.DateLayout = initDateLayout(cfg.Indices.Sampling.RolloverFrequency, separator) // Dependencies calculation should be daily, and this index size is very small - cfg.IndexDateLayoutDependencies = initDateLayout(defaultIndexRolloverFrequency, separator) + cfg.Indices.Dependencies.DateLayout = initDateLayout(defaultIndexRolloverFrequency, separator) var err error cfg.TLS, err = cfg.getTLSFlagsConfig().InitFromViper(v) if err != nil { @@ -434,22 +434,16 @@ func DefaultConfig() config.Configuration { Tags: config.TagsAsFields{ DotReplacement: "@", }, - Enabled: true, - CreateIndexTemplates: true, - Version: 0, - UseReadWriteAliases: false, - UseILM: false, - Servers: []string{defaultServerURL}, - RemoteReadClusters: []string{}, - MaxDocCount: defaultMaxDocCount, - LogLevel: "error", - SendGetBodyAs: defaultSendGetBodyAs, - IndexDateLayoutSpans: defaultIndexOptions.DateLayout, - IndexDateLayoutServices: defaultIndexOptions.DateLayout, - IndexDateLayoutSampling: defaultIndexOptions.DateLayout, - IndexRolloverFrequencySpans: defaultIndexOptions.RolloverFrequency, - IndexRolloverFrequencyServices: defaultIndexOptions.RolloverFrequency, - IndexRolloverFrequencySampling: defaultIndexOptions.RolloverFrequency, + Enabled: true, + CreateIndexTemplates: true, + Version: 0, + UseReadWriteAliases: false, + UseILM: false, + Servers: []string{defaultServerURL}, + RemoteReadClusters: []string{}, + MaxDocCount: defaultMaxDocCount, + LogLevel: "error", + SendGetBodyAs: defaultSendGetBodyAs, Indices: config.Indices{ Spans: defaultIndexOptions, Services: defaultIndexOptions, diff --git a/plugin/storage/es/options_test.go b/plugin/storage/es/options_test.go index ed9bbbefab3..3f12ed384e0 100644 --- a/plugin/storage/es/options_test.go +++ b/plugin/storage/es/options_test.go @@ -98,8 +98,8 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, "!", primary.Tags.DotReplacement) assert.Equal(t, "./file.txt", primary.Tags.File) assert.Equal(t, "test,tags", primary.Tags.Include) - assert.Equal(t, "20060102", primary.IndexDateLayoutServices) - assert.Equal(t, "2006010215", primary.IndexDateLayoutSpans) + assert.Equal(t, "20060102", primary.Indices.Services.DateLayout) + assert.Equal(t, "2006010215", primary.Indices.Spans.DateLayout) aux := opts.Get("es.aux") assert.Equal(t, []string{"3.3.3.3", "4.4.4.4"}, aux.Servers) assert.Equal(t, "hello", aux.Username) @@ -112,8 +112,8 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, "@", aux.Tags.DotReplacement) assert.Equal(t, "./file.txt", aux.Tags.File) assert.Equal(t, "test,tags", aux.Tags.Include) - assert.Equal(t, "2006.01.02", aux.IndexDateLayoutServices) - assert.Equal(t, "2006.01.02.15", aux.IndexDateLayoutSpans) + assert.Equal(t, "2006.01.02", aux.Indices.Services.DateLayout) + assert.Equal(t, "2006.01.02.15", aux.Indices.Spans.DateLayout) assert.True(t, primary.UseILM) assert.Equal(t, "POST", aux.SendGetBodyAs) } @@ -182,7 +182,7 @@ func TestIndexDateSeparator(t *testing.T) { opts.InitFromViper(v) primary := opts.GetPrimary() - assert.Equal(t, tc.wantDateLayout, primary.IndexDateLayoutSpans) + assert.Equal(t, tc.wantDateLayout, primary.Indices.Spans.DateLayout) }) } } @@ -236,8 +236,8 @@ func TestIndexRollover(t *testing.T) { command.ParseFlags(tc.flags) opts.InitFromViper(v) primary := opts.GetPrimary() - assert.Equal(t, tc.wantSpanDateLayout, primary.IndexDateLayoutSpans) - assert.Equal(t, tc.wantServiceDateLayout, primary.IndexDateLayoutServices) + assert.Equal(t, tc.wantSpanDateLayout, primary.Indices.Spans.DateLayout) + assert.Equal(t, tc.wantServiceDateLayout, primary.Indices.Services.DateLayout) assert.Equal(t, tc.wantSpanIndexRolloverFrequency, primary.GetIndexRolloverFrequencySpansDuration()) assert.Equal(t, tc.wantServiceIndexRolloverFrequency, primary.GetIndexRolloverFrequencyServicesDuration()) })