Skip to content

Commit

Permalink
polish
Browse files Browse the repository at this point in the history 
Signed-off-by: Jared Tan <jian.tan@daocloud.io>
  • Loading branch information
@JaredTan95
JaredTan95 committed Aug 14, 2024
1 parent 6f95325 commit 48e8a0d
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 87 deletions.
88 changes: 37 additions & 51 deletions pkg/es/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,54 +60,40 @@ type Indices struct {

// Configuration describes the configuration properties needed to connect to an ElasticSearch cluster
type Configuration struct {
Servers []string `mapstructure:"server_urls" valid:"required,url"`
RemoteReadClusters []string `mapstructure:"remote_read_clusters"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password" json:"-"`
TokenFilePath string `mapstructure:"token_file"`
PasswordFilePath string `mapstructure:"password_file"`
AllowTokenFromContext bool `mapstructure:"-"`
Sniffer bool `mapstructure:"sniffer"` // https://github.com/olivere/elastic/wiki/Sniffing
SnifferTLSEnabled bool `mapstructure:"sniffer_tls_enabled"`
MaxDocCount int `mapstructure:"-"` // Defines maximum number of results to fetch from storage per query
MaxSpanAge time.Duration `mapstructure:"-"` // configures the maximum lookback on span reads
NumShards int64 `mapstructure:"num_shards"`
NumReplicas int64 `mapstructure:"num_replicas"`
PrioritySpanTemplate int64 `mapstructure:"priority_span_template"`
PriorityServiceTemplate int64 `mapstructure:"priority_service_template"`
PriorityDependenciesTemplate int64 `mapstructure:"priority_dependencies_template"`
Timeout time.Duration `mapstructure:"-"`
BulkSize int `mapstructure:"-"`
BulkWorkers int `mapstructure:"-"`
BulkActions int `mapstructure:"-"`
BulkFlushInterval time.Duration `mapstructure:"-"`
IndexPrefix string `mapstructure:"index_prefix"`
// Deprecated: use Indices and IndexOptions instead.
IndexDateLayoutSpans string `mapstructure:"-"`
// Deprecated: use Indices and IndexOptions instead.
IndexDateLayoutServices string `mapstructure:"-"`
// Deprecated: use Indices and IndexOptions instead.
IndexDateLayoutSampling string `mapstructure:"-"`
// Deprecated: use Indices and IndexOptions instead.
IndexDateLayoutDependencies string `mapstructure:"-"`
// Deprecated: use Indices and IndexOptions instead.
IndexRolloverFrequencySpans string `mapstructure:"-"`
// Deprecated: use Indices and IndexOptions instead.
IndexRolloverFrequencyServices string `mapstructure:"-"`
// Deprecated: use Indices and IndexOptions instead.
IndexRolloverFrequencySampling string `mapstructure:"-"`
Indices Indices `mapstructure:"indices"`
ServiceCacheTTL time.Duration `mapstructure:"service_cache_ttl"`
AdaptiveSamplingLookback time.Duration `mapstructure:"-"`
Tags TagsAsFields `mapstructure:"tags_as_fields"`
Enabled bool `mapstructure:"-"`
TLS tlscfg.Options `mapstructure:"tls"`
UseReadWriteAliases bool `mapstructure:"use_aliases"`
CreateIndexTemplates bool `mapstructure:"create_mappings"`
UseILM bool `mapstructure:"use_ilm"`
Version uint `mapstructure:"version"`
LogLevel string `mapstructure:"log_level"`
SendGetBodyAs string `mapstructure:"send_get_body_as"`
Servers []string `mapstructure:"server_urls" valid:"required,url"`
RemoteReadClusters []string `mapstructure:"remote_read_clusters"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password" json:"-"`
TokenFilePath string `mapstructure:"token_file"`
PasswordFilePath string `mapstructure:"password_file"`
AllowTokenFromContext bool `mapstructure:"-"`
Sniffer bool `mapstructure:"sniffer"` // https://github.com/olivere/elastic/wiki/Sniffing
SnifferTLSEnabled bool `mapstructure:"sniffer_tls_enabled"`
MaxDocCount int `mapstructure:"-"` // Defines maximum number of results to fetch from storage per query
MaxSpanAge time.Duration `mapstructure:"-"` // configures the maximum lookback on span reads
NumShards int64 `mapstructure:"num_shards"`
NumReplicas int64 `mapstructure:"num_replicas"`
PrioritySpanTemplate int64 `mapstructure:"priority_span_template"`
PriorityServiceTemplate int64 `mapstructure:"priority_service_template"`
PriorityDependenciesTemplate int64 `mapstructure:"priority_dependencies_template"`
Timeout time.Duration `mapstructure:"-"`
BulkSize int `mapstructure:"-"`
BulkWorkers int `mapstructure:"-"`
BulkActions int `mapstructure:"-"`
BulkFlushInterval time.Duration `mapstructure:"-"`
IndexPrefix string `mapstructure:"index_prefix"`
Indices Indices `mapstructure:"indices"`
ServiceCacheTTL time.Duration `mapstructure:"service_cache_ttl"`
AdaptiveSamplingLookback time.Duration `mapstructure:"-"`
Tags TagsAsFields `mapstructure:"tags_as_fields"`
Enabled bool `mapstructure:"-"`
TLS tlscfg.Options `mapstructure:"tls"`
UseReadWriteAliases bool `mapstructure:"use_aliases"`
CreateIndexTemplates bool `mapstructure:"create_mappings"`
UseILM bool `mapstructure:"use_ilm"`
Version uint `mapstructure:"version"`
LogLevel string `mapstructure:"log_level"`
SendGetBodyAs string `mapstructure:"send_get_body_as"`
}

// TagsAsFields holds configuration for tag schema.
Expand Down Expand Up @@ -315,17 +301,17 @@ func (c *Configuration) ApplyDefaults(source *Configuration) {

// GetIndexRolloverFrequencySpansDuration returns jaeger-span index rollover frequency duration
func (c *Configuration) GetIndexRolloverFrequencySpansDuration() time.Duration {
return getIndexRolloverFrequencyDuration(c.IndexRolloverFrequencySpans)
return getIndexRolloverFrequencyDuration(c.Indices.Spans.RolloverFrequency)
}

// GetIndexRolloverFrequencyServicesDuration returns jaeger-service index rollover frequency duration
func (c *Configuration) GetIndexRolloverFrequencyServicesDuration() time.Duration {
return getIndexRolloverFrequencyDuration(c.IndexRolloverFrequencyServices)
return getIndexRolloverFrequencyDuration(c.Indices.Services.RolloverFrequency)
}

// GetIndexRolloverFrequencySamplingDuration returns jaeger-sampling index rollover frequency duration
func (c *Configuration) GetIndexRolloverFrequencySamplingDuration() time.Duration {
return getIndexRolloverFrequencyDuration(c.IndexRolloverFrequencySampling)
return getIndexRolloverFrequencyDuration(c.Indices.Sampling.RolloverFrequency)
}

// GetIndexRolloverFrequencyDuration returns the index rollover frequency duration for the given frequency string
Expand Down
12 changes: 6 additions & 6 deletions plugin/storage/es/factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,8 @@ func createSpanReader(
MaxDocCount: cfg.MaxDocCount,
MaxSpanAge: cfg.MaxSpanAge,
IndexPrefix: cfg.IndexPrefix,
SpanIndexDateLayout: cfg.IndexDateLayoutSpans,
ServiceIndexDateLayout: cfg.IndexDateLayoutServices,
SpanIndexDateLayout: cfg.Indices.Spans.DateLayout,
ServiceIndexDateLayout: cfg.Indices.Services.DateLayout,
SpanIndexRolloverFrequency: cfg.GetIndexRolloverFrequencySpansDuration(),
ServiceIndexRolloverFrequency: cfg.GetIndexRolloverFrequencyServicesDuration(),
TagDotReplacement: cfg.Tags.DotReplacement,
Expand Down Expand Up @@ -270,8 +270,8 @@ func createSpanWriter(
writer := esSpanStore.NewSpanWriter(esSpanStore.SpanWriterParams{
Client: clientFn,
IndexPrefix: cfg.IndexPrefix,
SpanIndexDateLayout: cfg.IndexDateLayoutSpans,
ServiceIndexDateLayout: cfg.IndexDateLayoutServices,
SpanIndexDateLayout: cfg.Indices.Spans.DateLayout,
ServiceIndexDateLayout: cfg.Indices.Services.DateLayout,
AllTagsAsFields: cfg.Tags.AllAsFields,
TagKeysAsFields: tags,
TagDotReplacement: cfg.Tags.DotReplacement,
Expand Down Expand Up @@ -301,7 +301,7 @@ func (f *Factory) CreateSamplingStore(int /* maxBuckets */) (samplingstore.Store
Client: f.getPrimaryClient,
Logger: f.logger,
IndexPrefix: f.primaryConfig.IndexPrefix,
IndexDateLayout: f.primaryConfig.IndexDateLayoutSampling,
IndexDateLayout: f.primaryConfig.Indices.Sampling.DateLayout,
IndexRolloverFrequency: f.primaryConfig.GetIndexRolloverFrequencySamplingDuration(),
Lookback: f.primaryConfig.AdaptiveSamplingLookback,
MaxDocCount: f.primaryConfig.MaxDocCount,
Expand Down Expand Up @@ -345,7 +345,7 @@ func createDependencyReader(
Client: clientFn,
Logger: logger,
IndexPrefix: cfg.IndexPrefix,
IndexDateLayout: cfg.IndexDateLayoutDependencies,
IndexDateLayout: cfg.Indices.Dependencies.RolloverFrequency,
MaxDocCount: cfg.MaxDocCount,
UseReadWriteAliases: cfg.UseReadWriteAliases,
})
Expand Down
40 changes: 17 additions & 23 deletions plugin/storage/es/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,17 +363,17 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) {
cfg.RemoteReadClusters = strings.Split(remoteReadClusters, ",")
}

cfg.IndexRolloverFrequencySpans = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencySpans))
cfg.IndexRolloverFrequencyServices = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencyServices))
cfg.IndexRolloverFrequencySampling = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencySampling))
cfg.Indices.Spans.RolloverFrequency = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencySpans))
cfg.Indices.Services.RolloverFrequency = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencyServices))
cfg.Indices.Sampling.RolloverFrequency = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencySampling))

separator := v.GetString(cfg.namespace + suffixIndexDateSeparator)
cfg.IndexDateLayoutSpans = initDateLayout(cfg.IndexRolloverFrequencySpans, separator)
cfg.IndexDateLayoutServices = initDateLayout(cfg.IndexRolloverFrequencyServices, separator)
cfg.IndexDateLayoutSampling = initDateLayout(cfg.IndexRolloverFrequencySampling, separator)
cfg.Indices.Spans.DateLayout = initDateLayout(cfg.Indices.Spans.RolloverFrequency, separator)
cfg.Indices.Services.DateLayout = initDateLayout(cfg.Indices.Services.RolloverFrequency, separator)
cfg.Indices.Sampling.DateLayout = initDateLayout(cfg.Indices.Sampling.RolloverFrequency, separator)

// Dependencies calculation should be daily, and this index size is very small
cfg.IndexDateLayoutDependencies = initDateLayout(defaultIndexRolloverFrequency, separator)
cfg.Indices.Dependencies.DateLayout = initDateLayout(defaultIndexRolloverFrequency, separator)
var err error
cfg.TLS, err = cfg.getTLSFlagsConfig().InitFromViper(v)
if err != nil {
Expand Down Expand Up @@ -434,22 +434,16 @@ func DefaultConfig() config.Configuration {
Tags: config.TagsAsFields{
DotReplacement: "@",
},
Enabled: true,
CreateIndexTemplates: true,
Version: 0,
UseReadWriteAliases: false,
UseILM: false,
Servers: []string{defaultServerURL},
RemoteReadClusters: []string{},
MaxDocCount: defaultMaxDocCount,
LogLevel: "error",
SendGetBodyAs: defaultSendGetBodyAs,
IndexDateLayoutSpans: defaultIndexOptions.DateLayout,
IndexDateLayoutServices: defaultIndexOptions.DateLayout,
IndexDateLayoutSampling: defaultIndexOptions.DateLayout,
IndexRolloverFrequencySpans: defaultIndexOptions.RolloverFrequency,
IndexRolloverFrequencyServices: defaultIndexOptions.RolloverFrequency,
IndexRolloverFrequencySampling: defaultIndexOptions.RolloverFrequency,
Enabled: true,
CreateIndexTemplates: true,
Version: 0,
UseReadWriteAliases: false,
UseILM: false,
Servers: []string{defaultServerURL},
RemoteReadClusters: []string{},
MaxDocCount: defaultMaxDocCount,
LogLevel: "error",
SendGetBodyAs: defaultSendGetBodyAs,
Indices: config.Indices{
Spans: defaultIndexOptions,
Services: defaultIndexOptions,
Expand Down
14 changes: 7 additions & 7 deletions plugin/storage/es/options_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ func TestOptionsWithFlags(t *testing.T) {
assert.Equal(t, "!", primary.Tags.DotReplacement)
assert.Equal(t, "./file.txt", primary.Tags.File)
assert.Equal(t, "test,tags", primary.Tags.Include)
assert.Equal(t, "20060102", primary.IndexDateLayoutServices)
assert.Equal(t, "2006010215", primary.IndexDateLayoutSpans)
assert.Equal(t, "20060102", primary.Indices.Services.DateLayout)
assert.Equal(t, "2006010215", primary.Indices.Spans.DateLayout)
aux := opts.Get("es.aux")
assert.Equal(t, []string{"3.3.3.3", "4.4.4.4"}, aux.Servers)
assert.Equal(t, "hello", aux.Username)
Expand All @@ -112,8 +112,8 @@ func TestOptionsWithFlags(t *testing.T) {
assert.Equal(t, "@", aux.Tags.DotReplacement)
assert.Equal(t, "./file.txt", aux.Tags.File)
assert.Equal(t, "test,tags", aux.Tags.Include)
assert.Equal(t, "2006.01.02", aux.IndexDateLayoutServices)
assert.Equal(t, "2006.01.02.15", aux.IndexDateLayoutSpans)
assert.Equal(t, "2006.01.02", aux.Indices.Services.DateLayout)
assert.Equal(t, "2006.01.02.15", aux.Indices.Spans.DateLayout)
assert.True(t, primary.UseILM)
assert.Equal(t, "POST", aux.SendGetBodyAs)
}
Expand Down Expand Up @@ -182,7 +182,7 @@ func TestIndexDateSeparator(t *testing.T) {
opts.InitFromViper(v)

primary := opts.GetPrimary()
assert.Equal(t, tc.wantDateLayout, primary.IndexDateLayoutSpans)
assert.Equal(t, tc.wantDateLayout, primary.Indices.Spans.DateLayout)
})
}
}
Expand Down Expand Up @@ -236,8 +236,8 @@ func TestIndexRollover(t *testing.T) {
command.ParseFlags(tc.flags)
opts.InitFromViper(v)
primary := opts.GetPrimary()
assert.Equal(t, tc.wantSpanDateLayout, primary.IndexDateLayoutSpans)
assert.Equal(t, tc.wantServiceDateLayout, primary.IndexDateLayoutServices)
assert.Equal(t, tc.wantSpanDateLayout, primary.Indices.Spans.DateLayout)
assert.Equal(t, tc.wantServiceDateLayout, primary.Indices.Services.DateLayout)
assert.Equal(t, tc.wantSpanIndexRolloverFrequency, primary.GetIndexRolloverFrequencySpansDuration())
assert.Equal(t, tc.wantServiceIndexRolloverFrequency, primary.GetIndexRolloverFrequencyServicesDuration())
})
Expand Down

0 comments on commit 48e8a0d

Please sign in to comment.