From e22fb12aae6a1b7f43301e0fe16c78949d548572 Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Mon, 11 Jan 2021 09:59:15 -0500 Subject: [PATCH 1/6] [aggregator] Add ActivePlacementVersion to tcp client (#3071) --- src/aggregator/client/tcp_client.go | 12 ++++++++++++ src/aggregator/client/tcp_client_test.go | 9 +++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/aggregator/client/tcp_client.go b/src/aggregator/client/tcp_client.go index 40a519457c..38209e20f7 100644 --- a/src/aggregator/client/tcp_client.go +++ b/src/aggregator/client/tcp_client.go @@ -239,6 +239,18 @@ func (c *TCPClient) ActivePlacement() (placement.Placement, int, error) { return placement.Clone(), stagedPlacement.Version(), nil } +// ActivePlacementVersion returns a copy of the currently active placement version. It is a far less expensive call +// than ActivePlacement, as it does not clone the placement. +func (c *TCPClient) ActivePlacementVersion() (int, error) { + stagedPlacement, onStagedPlacementDoneFn, err := c.placementWatcher.ActiveStagedPlacement() + if err != nil { + return 0, err + } + defer onStagedPlacementDoneFn() + + return stagedPlacement.Version(), nil +} + // Flush flushes any remaining data buffered by the client. func (c *TCPClient) Flush() error { c.metrics.flush.Inc(1) diff --git a/src/aggregator/client/tcp_client_test.go b/src/aggregator/client/tcp_client_test.go index b1e5db411d..a6f4a4af36 100644 --- a/src/aggregator/client/tcp_client_test.go +++ b/src/aggregator/client/tcp_client_test.go @@ -799,8 +799,8 @@ func TestTCPClientActivePlacement(t *testing.T) { ) c.placementWatcher = watcher - watcher.EXPECT().ActiveStagedPlacement().Return(stagedPlacement, func() { doneCalls++ }, nil) - stagedPlacement.EXPECT().Version().Return(42) + watcher.EXPECT().ActiveStagedPlacement().Return(stagedPlacement, func() { doneCalls++ }, nil).Times(2) + stagedPlacement.EXPECT().Version().Return(42).Times(2) stagedPlacement.EXPECT().ActivePlacement().Return(mockPl, func() { doneCalls++ }, nil) mockPl.EXPECT().Clone().Return(emptyPl) @@ -809,6 +809,11 @@ func TestTCPClientActivePlacement(t *testing.T) { assert.Equal(t, 42, v) assert.Equal(t, 2, doneCalls) assert.Equal(t, emptyPl, pl) + + v, err = c.ActivePlacementVersion() + assert.NoError(t, err) + assert.Equal(t, 42, v) + assert.Equal(t, 3, doneCalls) } func TestTCPClientInitAndClose(t *testing.T) { From f26cceaf6fc72fbb9410bf03d2ce080c4f4d30bc Mon Sep 17 00:00:00 2001 From: Wesley Kim Date: Tue, 12 Jan 2021 16:53:41 -0500 Subject: [PATCH 2/6] [coordinator] Rollout augmentM3Tags flag to true by default (#3082) * [coordinator] Rollout augmentM3Tags flag to true by default * Fixup operator docs link * Fixup doc site links --- site/content/m3query/architecture/_index.md | 2 +- .../m3query/config/annotated_config.yaml | 2 +- site/content/operator/api.md | 2 +- .../m3coordinator/downsample/downsampler.go | 1 - .../downsample/metrics_appender.go | 61 +++++-------------- .../downsample/metrics_appender_test.go | 5 +- .../m3coordinator/downsample/options.go | 37 +++-------- 7 files changed, 30 insertions(+), 80 deletions(-) diff --git a/site/content/m3query/architecture/_index.md b/site/content/m3query/architecture/_index.md index b9747a458e..11dda3cda7 100644 --- a/site/content/m3query/architecture/_index.md +++ b/site/content/m3query/architecture/_index.md @@ -9,4 +9,4 @@ chapter: true ## Overview -M3 Query and M3 Coordinator are written entirely in Go, M3 Query is as a query engine for [M3DB](https://m3db.github.io/m3/) and M3 Coordinator is a remote read/write endpoint for Prometheus and M3DB. To learn more about Prometheus's remote endpoints and storage, [see here](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage). +M3 Query and M3 Coordinator are written entirely in Go, M3 Query is as a query engine for [M3DB](https://m3db.io/) and M3 Coordinator is a remote read/write endpoint for Prometheus and M3DB. To learn more about Prometheus's remote endpoints and storage, [see here](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage). diff --git a/site/content/m3query/config/annotated_config.yaml b/site/content/m3query/config/annotated_config.yaml index 3b5da983d2..e29e868a5f 100644 --- a/site/content/m3query/config/annotated_config.yaml +++ b/site/content/m3query/config/annotated_config.yaml @@ -63,7 +63,7 @@ writeWorkerPoolPolicy: size: tagOptions: - # See here for more information: http://m3db.github.io/m3/how_to/query/#id-generation + # See here for more information under ID generation: https://m3db.io/docs/how_to/query/ idScheme: # lookbackDuration defines, at each step, how long we lookback until we see a non-NaN value. diff --git a/site/content/operator/api.md b/site/content/operator/api.md index 3c18b884b7..fc6c4f7d2b 100644 --- a/site/content/operator/api.md +++ b/site/content/operator/api.md @@ -220,7 +220,7 @@ Namespace defines an M3DB namespace or points to a preset M3DB namespace. ## NamespaceOptions -NamespaceOptions defines parameters for an M3DB namespace. See https://m3db.github.io/m3/operational_guide/namespace_configuration/ for more details. +NamespaceOptions defines parameters for an M3DB namespace. See https://m3db.io/docs/operational_guide/namespace_configuration/ for more details. | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | diff --git a/src/cmd/services/m3coordinator/downsample/downsampler.go b/src/cmd/services/m3coordinator/downsample/downsampler.go index 9a635758cc..fef7abe4d9 100644 --- a/src/cmd/services/m3coordinator/downsample/downsampler.go +++ b/src/cmd/services/m3coordinator/downsample/downsampler.go @@ -135,7 +135,6 @@ func defaultMetricsAppenderOptions(opts DownsamplerOptions, agg agg) metricsAppe metricTagsIteratorPool: agg.pools.metricTagsIteratorPool, debugLogging: debugLogging, logger: logger, - augmentM3Tags: agg.augmentM3Tags, } } diff --git a/src/cmd/services/m3coordinator/downsample/metrics_appender.go b/src/cmd/services/m3coordinator/downsample/metrics_appender.go index 72d295f200..288404a68c 100644 --- a/src/cmd/services/m3coordinator/downsample/metrics_appender.go +++ b/src/cmd/services/m3coordinator/downsample/metrics_appender.go @@ -102,7 +102,6 @@ type metricsAppenderOptions struct { matcher matcher.Matcher tagEncoderPool serialize.TagEncoderPool metricTagsIteratorPool serialize.MetricTagsIteratorPool - augmentM3Tags bool clockOpts clock.Options debugLogging bool @@ -149,19 +148,16 @@ func (a *metricsAppender) SamplesAppender(opts SampleAppenderOptions) (SamplesAp } tags := a.originalTags - // Augment tags if necessary. - if a.augmentM3Tags { - // NB (@shreyas): Add the metric type tag. The tag has the prefix - // __m3_. All tags with that prefix are only used for the purpose of - // filter match and then stripped off before we actually send to the aggregator. - switch opts.MetricType { - case ts.M3MetricTypeCounter: - tags.append(metric.M3TypeTag, metric.M3CounterValue) - case ts.M3MetricTypeGauge: - tags.append(metric.M3TypeTag, metric.M3GaugeValue) - case ts.M3MetricTypeTimer: - tags.append(metric.M3TypeTag, metric.M3TimerValue) - } + // NB (@shreyas): Add the metric type tag. The tag has the prefix + // __m3_. All tags with that prefix are only used for the purpose of + // filter match and then stripped off before we actually send to the aggregator. + switch opts.MetricType { + case ts.M3MetricTypeCounter: + tags.append(metric.M3TypeTag, metric.M3CounterValue) + case ts.M3MetricTypeGauge: + tags.append(metric.M3TypeTag, metric.M3GaugeValue) + case ts.M3MetricTypeTimer: + tags.append(metric.M3TypeTag, metric.M3TimerValue) } // Sort tags @@ -190,11 +186,8 @@ func (a *metricsAppender) SamplesAppender(opts SampleAppenderOptions) (SamplesAp matchResult := a.matcher.ForwardMatch(id, fromNanos, toNanos) id.Close() - // If we augmented metrics tags before running the forward match, then - // filter them out. - if a.augmentM3Tags { - tags.filterPrefix(metric.M3MetricsPrefix) - } + // filter out augmented metrics tags + tags.filterPrefix(metric.M3MetricsPrefix) var dropApplyResult metadata.ApplyOrRemoveDropPoliciesResult if opts.Override { @@ -215,7 +208,7 @@ func (a *metricsAppender) SamplesAppender(opts SampleAppenderOptions) (SamplesAp append(a.curr.Pipelines, pipelines.Pipelines...) } - if err := a.addSamplesAppenders(tags, a.curr, unownedID); err != nil { + if err := a.addSamplesAppenders(tags, a.curr); err != nil { return SamplesAppenderResult{}, err } } else { @@ -358,7 +351,7 @@ func (a *metricsAppender) SamplesAppender(opts SampleAppenderOptions) (SamplesAp a.debugLogMatch("downsampler using built mapping staged metadatas", debugLogMatchOptions{Meta: []metadata.StagedMetadata{a.curr}}) - if err := a.addSamplesAppenders(tags, a.curr, unownedID); err != nil { + if err := a.addSamplesAppenders(tags, a.curr); err != nil { return SamplesAppenderResult{}, err } } @@ -472,31 +465,7 @@ func (a *metricsAppender) resetTags() { a.originalTags = nil } -func (a *metricsAppender) addSamplesAppenders( - originalTags *tags, - stagedMetadata metadata.StagedMetadata, - unownedID []byte, -) error { - // Check if any of the pipelines have tags or a graphite prefix to set. - var tagsExist bool - for _, pipeline := range stagedMetadata.Pipelines { - if len(pipeline.Tags) > 0 || len(pipeline.GraphitePrefix) > 0 { - tagsExist = true - break - } - } - - // If we do not need to do any tag augmentation then just return. - if !a.augmentM3Tags && !tagsExist { - a.multiSamplesAppender.addSamplesAppender(samplesAppender{ - agg: a.agg, - clientRemote: a.clientRemote, - unownedID: unownedID, - stagedMetadatas: []metadata.StagedMetadata{stagedMetadata}, - }) - return nil - } - +func (a *metricsAppender) addSamplesAppenders(originalTags *tags, stagedMetadata metadata.StagedMetadata) error { var ( pipelines []metadata.PipelineMetadata ) diff --git a/src/cmd/services/m3coordinator/downsample/metrics_appender_test.go b/src/cmd/services/m3coordinator/downsample/metrics_appender_test.go index 8f2907d29b..003906e57a 100644 --- a/src/cmd/services/m3coordinator/downsample/metrics_appender_test.go +++ b/src/cmd/services/m3coordinator/downsample/metrics_appender_test.go @@ -119,8 +119,9 @@ func TestSamplesAppenderPoolResetsTagsAcrossSamples(t *testing.T) { } // NB: expected ID is generated into human-readable form - // from tags in ForwardMatch mock above. - expected := fmt.Sprintf("foo%d-bar%d", i, i) + // from tags in ForwardMatch mock above. Also include the m3 type, which is included when matching. + // nolint:scopelint + expected := fmt.Sprintf("__m3_type__-gauge,foo%d-bar%d", i, i) if expected != u.ID.String() { // NB: if this fails, appender is holding state after Finalize. return fmt.Errorf("expected ID %s, got %s", expected, u.ID.String()) diff --git a/src/cmd/services/m3coordinator/downsample/options.go b/src/cmd/services/m3coordinator/downsample/options.go index 07ea13d37f..03bf33026a 100644 --- a/src/cmd/services/m3coordinator/downsample/options.go +++ b/src/cmd/services/m3coordinator/downsample/options.go @@ -24,7 +24,6 @@ import ( "errors" "fmt" "runtime" - "strings" "time" "github.com/m3db/m3/src/aggregator/aggregator" @@ -225,10 +224,9 @@ type agg struct { aggregator aggregator.Aggregator clientRemote client.Client - clockOpts clock.Options - matcher matcher.Matcher - pools aggPools - augmentM3Tags bool + clockOpts clock.Options + matcher matcher.Matcher + pools aggPools } // Configuration configurates a downsampler. @@ -262,14 +260,6 @@ type Configuration struct { // EntryTTL determines how long an entry remains alive before it may be expired due to inactivity. EntryTTL time.Duration `yaml:"entryTTL"` - - // AugmentM3Tags will augment the metric type to aggregated metrics - // to be used within the filter for rules. If enabled, for example, - // your filter can specify '__m3_type__:gauge' to filter by gauges. - // This is particularly useful for Graphite metrics today. - // Furthermore, the option is automatically enabled if static rules are - // used and any filter contain an __m3_type__ tag. - AugmentM3Tags bool `yaml:"augmentM3Tags"` } // MatcherConfiguration is the configuration for the rule matcher. @@ -658,7 +648,6 @@ func (cfg Configuration) newAggregator(o DownsamplerOptions) (agg, error) { scope = instrumentOpts.MetricsScope() logger = instrumentOpts.Logger() openTimeout = defaultOpenTimeout - augmentM3Tags = cfg.AugmentM3Tags namespaceTag = defaultNamespaceTag ) if o.StorageFlushConcurrency > 0 { @@ -717,9 +706,6 @@ func (cfg Configuration) newAggregator(o DownsamplerOptions) (agg, error) { rs := rules.NewEmptyRuleSet(defaultConfigInMemoryNamespace, updateMetadata) for _, mappingRule := range cfg.Rules.MappingRules { - if strings.Contains(mappingRule.Filter, metric.M3MetricsPrefixString) { - augmentM3Tags = true - } rule, err := mappingRule.Rule() if err != nil { return agg{}, err @@ -732,9 +718,6 @@ func (cfg Configuration) newAggregator(o DownsamplerOptions) (agg, error) { } for _, rollupRule := range cfg.Rules.RollupRules { - if strings.Contains(rollupRule.Filter, metric.M3MetricsPrefixString) { - augmentM3Tags = true - } rule, err := rollupRule.Rule() if err != nil { return agg{}, err @@ -788,10 +771,9 @@ func (cfg Configuration) newAggregator(o DownsamplerOptions) (agg, error) { } return agg{ - clientRemote: client, - matcher: matcher, - pools: pools, - augmentM3Tags: augmentM3Tags, + clientRemote: client, + matcher: matcher, + pools: pools, }, nil } @@ -953,10 +935,9 @@ func (cfg Configuration) newAggregator(o DownsamplerOptions) (agg, error) { } return agg{ - aggregator: aggregatorInstance, - matcher: matcher, - pools: pools, - augmentM3Tags: augmentM3Tags, + aggregator: aggregatorInstance, + matcher: matcher, + pools: pools, }, nil } From 60e8e7009c9139a8584887e50ab4be65ac006d13 Mon Sep 17 00:00:00 2001 From: Wesley Kim Date: Tue, 12 Jan 2021 17:31:24 -0500 Subject: [PATCH 3/6] [matcher/coordinator] Add latency metrics to rule matching (#3083) --- src/metrics/matcher/match.go | 34 ++++++++++++++++++++++--- src/metrics/matcher/match_test.go | 41 +++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/src/metrics/matcher/match.go b/src/metrics/matcher/match.go index 776428feb2..ddadaa82a8 100644 --- a/src/metrics/matcher/match.go +++ b/src/metrics/matcher/match.go @@ -21,6 +21,10 @@ package matcher import ( + "time" + + "github.com/uber-go/tally" + "github.com/m3db/m3/src/metrics/matcher/cache" "github.com/m3db/m3/src/metrics/metric/id" "github.com/m3db/m3/src/metrics/rules" @@ -40,6 +44,7 @@ type matcher struct { namespaceResolver namespaceResolver namespaces Namespaces cache cache.Cache + metrics matcherMetrics } type namespaceResolver struct { @@ -89,6 +94,7 @@ func NewMatcher(cache cache.Cache, opts Options) (Matcher, error) { return &noCacheMatcher{ namespaceResolver: nsResolver, namespaces: namespaces, + metrics: newMatcherMetrics(scope.SubScope("matcher")), }, nil } @@ -96,6 +102,7 @@ func NewMatcher(cache cache.Cache, opts Options) (Matcher, error) { namespaceResolver: nsResolver, namespaces: namespaces, cache: cache, + metrics: newMatcherMetrics(scope.SubScope("cached-matcher")), }, nil } @@ -103,8 +110,9 @@ func (m *matcher) ForwardMatch( id id.ID, fromNanos, toNanos int64, ) rules.MatchResult { - return m.cache.ForwardMatch(m.namespaceResolver.Resolve(id), - id.Bytes(), fromNanos, toNanos) + sw := m.metrics.matchLatency.Start() + defer sw.Stop() + return m.cache.ForwardMatch(m.namespaceResolver.Resolve(id), id.Bytes(), fromNanos, toNanos) } func (m *matcher) Close() error { @@ -115,14 +123,32 @@ func (m *matcher) Close() error { type noCacheMatcher struct { namespaces Namespaces namespaceResolver namespaceResolver + metrics matcherMetrics +} + +type matcherMetrics struct { + matchLatency tally.Histogram +} + +func newMatcherMetrics(scope tally.Scope) matcherMetrics { + return matcherMetrics{ + matchLatency: scope.Histogram( + "match-latency", + append( + tally.DurationBuckets{0}, + tally.MustMakeExponentialDurationBuckets(time.Millisecond, 1.5, 15)..., + ), + ), + } } func (m *noCacheMatcher) ForwardMatch( id id.ID, fromNanos, toNanos int64, ) rules.MatchResult { - return m.namespaces.ForwardMatch(m.namespaceResolver.Resolve(id), - id.Bytes(), fromNanos, toNanos) + sw := m.metrics.matchLatency.Start() + defer sw.Stop() + return m.namespaces.ForwardMatch(m.namespaceResolver.Resolve(id), id.Bytes(), fromNanos, toNanos) } func (m *noCacheMatcher) Close() error { diff --git a/src/metrics/matcher/match_test.go b/src/metrics/matcher/match_test.go index cce42bab56..71ad1035f1 100644 --- a/src/metrics/matcher/match_test.go +++ b/src/metrics/matcher/match_test.go @@ -25,6 +25,10 @@ import ( "testing" "time" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "github.com/uber-go/tally" + "github.com/m3db/m3/src/cluster/kv" "github.com/m3db/m3/src/cluster/kv/mem" "github.com/m3db/m3/src/metrics/aggregation" @@ -42,9 +46,6 @@ import ( "github.com/m3db/m3/src/x/instrument" xtest "github.com/m3db/m3/src/x/test" "github.com/m3db/m3/src/x/watch" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/require" ) func TestMatcherCreateWatchError(t *testing.T) { @@ -82,10 +83,12 @@ func TestMatcherMatchDoesNotExist(t *testing.T) { tagValueFn: func(tagName []byte) ([]byte, bool) { return nil, false }, } now := time.Now() - matcher := testMatcher(t, testMatcherOptions{ + matcher, testScope := testMatcher(t, testMatcherOptions{ cache: newMemCache(), }) require.Equal(t, rules.EmptyMatchResult, matcher.ForwardMatch(id, now.UnixNano(), now.UnixNano())) + + requireLatencyMetrics(t, "cached-matcher", testScope) } func TestMatcherMatchExists(t *testing.T) { @@ -100,7 +103,7 @@ func TestMatcherMatchExists(t *testing.T) { memRes = memResults{results: map[string]rules.MatchResult{"foo": res}} ) cache := newMemCache() - matcher := testMatcher(t, testMatcherOptions{ + matcher, _ := testMatcher(t, testMatcherOptions{ cache: cache, }) c := cache.(*memCache) @@ -125,7 +128,7 @@ func TestMatcherMatchExistsNoCache(t *testing.T) { } now = time.Now() ) - matcher := testMatcher(t, testMatcherOptions{ + matcher, testScope := testMatcher(t, testMatcherOptions{ tagFilterOptions: filters.TagsFilterOptions{ NameAndTagsFn: func(id []byte) (name []byte, tags []byte, err error) { name = metric.id @@ -210,10 +213,13 @@ func TestMatcherMatchExistsNoCache(t *testing.T) { result := matcher.ForwardMatch(metric, now.UnixNano(), now.UnixNano()) require.Equal(t, expected, result) + + // Check that latency was measured + requireLatencyMetrics(t, "matcher", testScope) } func TestMatcherClose(t *testing.T) { - matcher := testMatcher(t, testMatcherOptions{ + matcher, _ := testMatcher(t, testMatcherOptions{ cache: newMemCache(), }) require.NoError(t, matcher.Close()) @@ -225,12 +231,13 @@ type testMatcherOptions struct { tagFilterOptions filters.TagsFilterOptions } -func testMatcher(t *testing.T, opts testMatcherOptions) Matcher { +func testMatcher(t *testing.T, opts testMatcherOptions) (Matcher, tally.TestScope) { + scope := tally.NewTestScope("", nil) var ( store = mem.NewStore() matcherOpts = NewOptions(). SetClockOptions(clock.NewOptions()). - SetInstrumentOptions(instrument.NewOptions()). + SetInstrumentOptions(instrument.NewOptions().SetMetricsScope(scope)). SetInitWatchTimeout(100 * time.Millisecond). SetKVStore(store). SetNamespacesKey(testNamespacesKey). @@ -264,7 +271,21 @@ func testMatcher(t *testing.T, opts testMatcherOptions) Matcher { m, err := NewMatcher(opts.cache, matcherOpts) require.NoError(t, err) - return m + return m, scope +} + +func requireLatencyMetrics(t *testing.T, metricScope string, testScope tally.TestScope) { + // Check that latency was measured + values, found := testScope.Snapshot().Histograms()[metricScope+".match-latency+"] + require.True(t, found) + latencyMeasured := false + for _, valuesInBucket := range values.Durations() { + if valuesInBucket > 0 { + latencyMeasured = true + break + } + } + require.True(t, latencyMeasured) } type tagValueFn func(tagName []byte) ([]byte, bool) From c46f488e0e87f1cc886f090d6e949437b96664ce Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 12 Jan 2021 20:57:23 -0500 Subject: [PATCH 4/6] [dbnode][m3ninx] Use new doc.Document in query results to reduce slice allocations (#3057) --- src/cmd/services/m3dbnode/config/bootstrap.go | 4 +- .../tools/query_index_segments/main/main.go | 10 +- src/dbnode/generated-source-files.mk | 5 +- src/dbnode/integration/integration.go | 4 +- .../integration/write_tagged_quorum_test.go | 11 +- .../server/tchannelthrift/node/service.go | 38 ++++- .../tchannelthrift/node/service_test.go | 145 ++++++++++++++---- .../bootstrap/bootstrapper/fs/source.go | 12 +- .../bootstrapper/fs/source_data_test.go | 4 +- .../bootstrapper/peers/peers_test.go | 4 +- .../bootstrap/bootstrapper/peers/source.go | 12 +- .../bootstrapper/peers/source_data_test.go | 4 +- src/dbnode/storage/index/aggregate_results.go | 19 ++- .../storage/index/aggregate_results_test.go | 61 ++++---- src/dbnode/storage/index/block.go | 6 +- src/dbnode/storage/index/block_test.go | 94 +++++++----- .../storage/index/compaction/compactor.go | 12 +- .../index/compaction/compactor_test.go | 16 +- src/dbnode/storage/index/index_mock.go | 36 ++++- src/dbnode/storage/index/mutable_segments.go | 12 +- src/dbnode/storage/index/options.go | 44 +++++- src/dbnode/storage/index/results.go | 39 ++--- src/dbnode/storage/index/results_map_gen.go | 60 +++++--- src/dbnode/storage/index/results_new_map.go | 20 ++- src/dbnode/storage/index/results_test.go | 84 ++++++---- src/dbnode/storage/index/types.go | 8 +- .../storage/index/wide_query_results.go | 16 +- .../storage/index/wide_query_results_test.go | 23 +-- src/dbnode/storage/index_block_test.go | 9 +- .../storage/index_query_concurrent_test.go | 13 +- .../storage/index_queue_forward_write_test.go | 9 +- src/dbnode/storage/index_queue_test.go | 9 +- src/dbnode/test/util.go | 43 ++++++ src/m3ninx/doc/doc_arraypool_gen.go | 28 ++-- src/m3ninx/doc/document.go | 7 +- src/m3ninx/doc/metadata_arraypool_gen.go | 127 +++++++++++++++ src/m3ninx/generated-source-files.mk | 29 +++- .../index/segment/fst/encoding/docs/data.go | 4 +- src/m3ninx/search/executor/executor.go | 4 +- src/m3ninx/search/executor/executor_test.go | 4 +- src/m3ninx/search/executor/iterator.go | 16 +- src/m3ninx/search/executor/iterator_test.go | 37 +---- src/m3ninx/search/proptest/concurrent_test.go | 2 +- src/m3ninx/search/proptest/issue865_test.go | 30 ++-- src/m3ninx/search/proptest/prop_test.go | 4 +- src/m3ninx/search/proptest/segment_gen.go | 4 +- src/m3ninx/search/proptest/util.go | 41 +++-- src/m3ninx/search/search_mock.go | 6 +- src/m3ninx/search/types.go | 2 +- 49 files changed, 860 insertions(+), 371 deletions(-) create mode 100644 src/dbnode/test/util.go create mode 100644 src/m3ninx/doc/metadata_arraypool_gen.go diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index 09d618b82c..fdc163116b 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -279,8 +279,8 @@ func (bsc BootstrapConfiguration) New( adminClient client.AdminClient, ) (bootstrap.ProcessProvider, error) { idxOpts := opts.IndexOptions() - compactor, err := compaction.NewCompactor(idxOpts.DocumentArrayPool(), - index.DocumentArrayPoolCapacity, + compactor, err := compaction.NewCompactor(idxOpts.MetadataArrayPool(), + index.MetadataArrayPoolCapacity, idxOpts.SegmentBuilderOptions(), idxOpts.FSTSegmentOptions(), compaction.CompactorOptions{ diff --git a/src/cmd/tools/query_index_segments/main/main.go b/src/cmd/tools/query_index_segments/main/main.go index 1556a4add4..dcd540b0b8 100644 --- a/src/cmd/tools/query_index_segments/main/main.go +++ b/src/cmd/tools/query_index_segments/main/main.go @@ -32,6 +32,7 @@ import ( "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/m3ninx/index" + "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" "github.com/m3db/m3/src/m3ninx/search/executor" "github.com/m3db/m3/src/query/generated/proto/prompb" "github.com/m3db/m3/src/query/parser/promql" @@ -189,11 +190,16 @@ func run(opts runOptions) { log.Fatal("search execute error", zap.Error(err)) } + reader := docs.NewEncodedDocumentReader() fields := make(map[string]string) for iter.Next() { d := iter.Current() + m, err := docs.MetadataFromDocument(d, reader) + if err != nil { + log.Fatal("error retrieve document metadata", zap.Error(err)) + } - key := string(d.ID) + key := string(m.ID) resultsLock.Lock() _, ok := results[key] @@ -209,7 +215,7 @@ func run(opts runOptions) { for k := range fields { delete(fields, k) } - for _, field := range d.Fields { + for _, field := range m.Fields { // nolint:gocritic fields[string(field.Name)] = string(field.Value) } diff --git a/src/dbnode/generated-source-files.mk b/src/dbnode/generated-source-files.mk index 748dfeaea9..6ca17180f5 100644 --- a/src/dbnode/generated-source-files.mk +++ b/src/dbnode/generated-source-files.mk @@ -149,10 +149,9 @@ genny-map-persist-fs: # Map generation rule for storage/index/ResultsMap .PHONY: genny-map-storage-index-results genny-map-storage-index-results: - cd $(m3x_package_path) && make hashmap-gen \ + cd $(m3x_package_path) && make byteshashmap-gen \ pkg=index \ - key_type=ident.ID \ - value_type=ident.TagIterator \ + value_type=doc.Document \ target_package=$(m3db_package)/src/dbnode/storage/index \ rename_nogen_key=true \ rename_nogen_value=true \ diff --git a/src/dbnode/integration/integration.go b/src/dbnode/integration/integration.go index 3c61c170d3..1aa2d56d13 100644 --- a/src/dbnode/integration/integration.go +++ b/src/dbnode/integration/integration.go @@ -430,8 +430,8 @@ func newCompactor( } func newCompactorWithErr(opts index.Options) (*compaction.Compactor, error) { - return compaction.NewCompactor(opts.DocumentArrayPool(), - index.DocumentArrayPoolCapacity, + return compaction.NewCompactor(opts.MetadataArrayPool(), + index.MetadataArrayPoolCapacity, opts.SegmentBuilderOptions(), opts.FSTSegmentOptions(), compaction.CompactorOptions{ diff --git a/src/dbnode/integration/write_tagged_quorum_test.go b/src/dbnode/integration/write_tagged_quorum_test.go index 5a95d8729b..91122c6d6c 100644 --- a/src/dbnode/integration/write_tagged_quorum_test.go +++ b/src/dbnode/integration/write_tagged_quorum_test.go @@ -30,6 +30,7 @@ import ( "github.com/m3db/m3/src/cluster/shard" "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/storage/index" + "github.com/m3db/m3/src/dbnode/test" "github.com/m3db/m3/src/dbnode/topology" "github.com/m3db/m3/src/dbnode/x/xio" m3ninxidx "github.com/m3db/m3/src/m3ninx/idx" @@ -303,9 +304,13 @@ func nodeHasTaggedWrite(t *testing.T, s TestSetup) bool { require.NoError(t, err) results := res.Results require.Equal(t, nsCtx.ID.String(), results.Namespace().String()) - tags, ok := results.Map().Get(ident.StringID("quorumTest")) - idxFound := ok && ident.NewTagIterMatcher(ident.MustNewTagStringsIterator( - "foo", "bar", "boo", "baz")).Matches(tags) + doc, ok := results.Map().Get(ident.BytesID("quorumTest")) + idxFound := false + if ok { + tags := test.DocumentToTagIter(t, doc) + idxFound = ident.NewTagIterMatcher(ident.MustNewTagStringsIterator( + "foo", "bar", "boo", "baz")).Matches(tags) + } if !idxFound { return false diff --git a/src/dbnode/network/server/tchannelthrift/node/service.go b/src/dbnode/network/server/tchannelthrift/node/service.go index 61ac91688f..8a7d7a0f6a 100644 --- a/src/dbnode/network/server/tchannelthrift/node/service.go +++ b/src/dbnode/network/server/tchannelthrift/node/service.go @@ -37,11 +37,13 @@ import ( "github.com/m3db/m3/src/dbnode/storage" "github.com/m3db/m3/src/dbnode/storage/block" "github.com/m3db/m3/src/dbnode/storage/index" + idxconvert "github.com/m3db/m3/src/dbnode/storage/index/convert" "github.com/m3db/m3/src/dbnode/storage/limits" "github.com/m3db/m3/src/dbnode/tracepoint" "github.com/m3db/m3/src/dbnode/ts/writes" "github.com/m3db/m3/src/dbnode/x/xio" "github.com/m3db/m3/src/dbnode/x/xpool" + "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" "github.com/m3db/m3/src/x/checked" "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" @@ -514,10 +516,19 @@ func (s *service) query(ctx context.Context, db storage.Database, req *rpc.Query if req.NoData != nil && *req.NoData { fetchData = false } + // Re-use reader and id for more memory-efficient processing of + // tags from doc.Metadata + reader := docs.NewEncodedDocumentReader() + id := ident.NewReusableBytesID() for _, entry := range queryResult.Results.Map().Iter() { - tags := entry.Value() + d := entry.Value() + metadata, err := docs.MetadataFromDocument(d, reader) + if err != nil { + return nil, err + } + tags := idxconvert.ToSeriesTags(metadata, idxconvert.Opts{NoClone: true}) elem := &rpc.QueryResultElement{ - ID: entry.Key().String(), + ID: string(entry.Key()), Tags: make([]*rpc.Tag, 0, tags.Remaining()), } result.Results = append(result.Results, elem) @@ -535,8 +546,8 @@ func (s *service) query(ctx context.Context, db storage.Database, req *rpc.Query if !fetchData { continue } - tsID := entry.Key() - datapoints, err := s.readDatapoints(ctx, db, nsID, tsID, start, end, + id.Reset(entry.Key()) + datapoints, err := s.readDatapoints(ctx, db, nsID, id, start, end, req.ResultTimeType) if err != nil { return nil, convert.ToRPCError(err) @@ -794,12 +805,23 @@ func (s *service) fetchReadEncoded(ctx context.Context, defer sp.Finish() i := 0 + // Re-use reader and id for more memory-efficient processing of + // tags from doc.Metadata + reader := docs.NewEncodedDocumentReader() + id := ident.NewReusableBytesID() for _, entry := range results.Map().Iter() { idx := i i++ - tsID := entry.Key() - tags := entry.Value() + id.Reset(entry.Key()) + + d := entry.Value() + metadata, err := docs.MetadataFromDocument(d, reader) + if err != nil { + return err + } + tags := idxconvert.ToSeriesTags(metadata, idxconvert.Opts{NoClone: true}) + enc := s.pools.tagEncoder.Get() ctx.RegisterFinalizer(enc) encodedTags, err := s.encodeTags(enc, tags) @@ -809,7 +831,7 @@ func (s *service) fetchReadEncoded(ctx context.Context, elem := &rpc.FetchTaggedIDResult_{ NameSpace: nsIDBytes, - ID: tsID.Bytes(), + ID: id.Bytes(), EncodedTags: encodedTags.Bytes(), } response.Elements = append(response.Elements, elem) @@ -817,7 +839,7 @@ func (s *service) fetchReadEncoded(ctx context.Context, continue } - encoded, err := db.ReadEncoded(ctx, nsID, tsID, + encoded, err := db.ReadEncoded(ctx, nsID, id, opts.StartInclusive, opts.EndExclusive) if err != nil { return convert.ToRPCError(err) diff --git a/src/dbnode/network/server/tchannelthrift/node/service_test.go b/src/dbnode/network/server/tchannelthrift/node/service_test.go index 637a19612c..1996795609 100644 --- a/src/dbnode/network/server/tchannelthrift/node/service_test.go +++ b/src/dbnode/network/server/tchannelthrift/node/service_test.go @@ -44,6 +44,7 @@ import ( "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/dbnode/ts/writes" "github.com/m3db/m3/src/dbnode/x/xio" + "github.com/m3db/m3/src/m3ninx/doc" "github.com/m3db/m3/src/m3ninx/idx" "github.com/m3db/m3/src/x/checked" "github.com/m3db/m3/src/x/ident" @@ -286,16 +287,37 @@ func TestServiceQuery(t *testing.T) { require.NoError(t, err) qry := index.Query{Query: req} + md1 := doc.Metadata{ + ID: ident.BytesID("foo"), + Fields: []doc.Field{ + { + Name: []byte("foo"), + Value: []byte("bar"), + }, + { + Name: []byte("baz"), + Value: []byte("dxk"), + }, + }, + } + md2 := doc.Metadata{ + ID: ident.BytesID("bar"), + Fields: []doc.Field{ + { + Name: []byte("foo"), + Value: []byte("bar"), + }, + { + Name: []byte("dzk"), + Value: []byte("baz"), + }, + }, + } + resMap := index.NewQueryResults(ident.StringID(nsID), index.QueryResultsOptions{}, testIndexOptions) - resMap.Map().Set(ident.StringID("foo"), ident.NewTagsIterator(ident.NewTags( - ident.StringTag(tags["foo"][0].name, tags["foo"][0].value), - ident.StringTag(tags["foo"][1].name, tags["foo"][1].value), - ))) - resMap.Map().Set(ident.StringID("bar"), ident.NewTagsIterator(ident.NewTags( - ident.StringTag(tags["bar"][0].name, tags["bar"][0].value), - ident.StringTag(tags["bar"][1].name, tags["bar"][1].value), - ))) + resMap.Map().Set(md1.ID, doc.NewDocumentFromMetadata(md1)) + resMap.Map().Set(md2.ID, doc.NewDocumentFromMetadata(md2)) mockDB.EXPECT().QueryIDs( ctx, @@ -1579,16 +1601,37 @@ func TestServiceFetchTagged(t *testing.T) { require.NoError(t, err) qry := index.Query{Query: req} + md1 := doc.Metadata{ + ID: ident.BytesID("foo"), + Fields: []doc.Field{ + { + Name: []byte("foo"), + Value: []byte("bar"), + }, + { + Name: []byte("baz"), + Value: []byte("dxk"), + }, + }, + } + md2 := doc.Metadata{ + ID: ident.BytesID("bar"), + Fields: []doc.Field{ + { + Name: []byte("foo"), + Value: []byte("bar"), + }, + { + Name: []byte("dzk"), + Value: []byte("baz"), + }, + }, + } + resMap := index.NewQueryResults(ident.StringID(nsID), index.QueryResultsOptions{}, testIndexOptions) - resMap.Map().Set(ident.StringID("foo"), ident.NewTagsIterator(ident.NewTags( - ident.StringTag("foo", "bar"), - ident.StringTag("baz", "dxk"), - ))) - resMap.Map().Set(ident.StringID("bar"), ident.NewTagsIterator(ident.NewTags( - ident.StringTag("foo", "bar"), - ident.StringTag("dzk", "baz"), - ))) + resMap.Map().Set(md1.ID, doc.NewDocumentFromMetadata(md1)) + resMap.Map().Set(md2.ID, doc.NewDocumentFromMetadata(md2)) mockDB.EXPECT().QueryIDs( gomock.Any(), @@ -1685,16 +1728,37 @@ func TestServiceFetchTaggedIsOverloaded(t *testing.T) { req, err := idx.NewRegexpQuery([]byte("foo"), []byte("b.*")) require.NoError(t, err) + md1 := doc.Metadata{ + ID: ident.BytesID("foo"), + Fields: []doc.Field{ + { + Name: []byte("foo"), + Value: []byte("bar"), + }, + { + Name: []byte("baz"), + Value: []byte("dxk"), + }, + }, + } + md2 := doc.Metadata{ + ID: ident.BytesID("bar"), + Fields: []doc.Field{ + { + Name: []byte("foo"), + Value: []byte("bar"), + }, + { + Name: []byte("dzk"), + Value: []byte("baz"), + }, + }, + } + resMap := index.NewQueryResults(ident.StringID(nsID), index.QueryResultsOptions{}, testIndexOptions) - resMap.Map().Set(ident.StringID("foo"), ident.NewTagsIterator(ident.NewTags( - ident.StringTag("foo", "bar"), - ident.StringTag("baz", "dxk"), - ))) - resMap.Map().Set(ident.StringID("bar"), ident.NewTagsIterator(ident.NewTags( - ident.StringTag("foo", "bar"), - ident.StringTag("dzk", "baz"), - ))) + resMap.Map().Set(md1.ID, doc.NewDocumentFromMetadata(md1)) + resMap.Map().Set(md2.ID, doc.NewDocumentFromMetadata(md2)) startNanos, err := convert.ToValue(start, rpc.TimeType_UNIX_NANOSECONDS) require.NoError(t, err) @@ -1779,10 +1843,20 @@ func TestServiceFetchTaggedNoData(t *testing.T) { require.NoError(t, err) qry := index.Query{Query: req} + md1 := doc.Metadata{ + ID: ident.BytesID("foo"), + Fields: []doc.Field{}, + } + md2 := doc.Metadata{ + ID: ident.BytesID("bar"), + Fields: []doc.Field{}, + } + resMap := index.NewQueryResults(ident.StringID(nsID), index.QueryResultsOptions{}, testIndexOptions) - resMap.Map().Set(ident.StringID("foo"), ident.NewTagsIterator(ident.Tags{})) - resMap.Map().Set(ident.StringID("bar"), ident.NewTagsIterator(ident.Tags{})) + resMap.Map().Set(md1.ID, doc.NewDocumentFromMetadata(md1)) + resMap.Map().Set(md2.ID, doc.NewDocumentFromMetadata(md2)) + mockDB.EXPECT().QueryIDs( ctx, ident.NewIDMatcher(nsID), @@ -1931,12 +2005,23 @@ func TestServiceFetchTaggedReturnOnFirstErr(t *testing.T) { require.NoError(t, err) qry := index.Query{Query: req} + md1 := doc.Metadata{ + ID: ident.BytesID("foo"), + Fields: []doc.Field{ + { + Name: []byte("foo"), + Value: []byte("bar"), + }, + { + Name: []byte("baz"), + Value: []byte("dxk"), + }, + }, + } + resMap := index.NewQueryResults(ident.StringID(nsID), index.QueryResultsOptions{}, testIndexOptions) - resMap.Map().Set(ident.StringID("foo"), ident.NewTagsIterator(ident.NewTags( - ident.StringTag("foo", "bar"), - ident.StringTag("baz", "dxk"), - ))) + resMap.Map().Set(md1.ID, doc.NewDocumentFromMetadata(md1)) mockDB.EXPECT().QueryIDs( gomock.Any(), diff --git a/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go b/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go index 8848be7ed3..79b0d58398 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go @@ -401,12 +401,12 @@ func (s *fileSystemSource) loadShardReadersDataIntoShardResult( seriesCachePolicy = ropts.SeriesCachePolicy() timesWithErrors []time.Time nsCtx = namespace.NewContextFrom(ns) - docsPool = s.opts.IndexOptions().DocumentArrayPool() - batch = docsPool.Get() + metadataPool = s.opts.IndexOptions().MetadataArrayPool() + batch = metadataPool.Get() totalEntries int totalFulfilledRanges = result.NewShardTimeRanges() ) - defer docsPool.Put(batch) + defer metadataPool.Put(batch) requestedRanges := timeWindowReaders.Ranges remainingRanges := requestedRanges.Copy() @@ -740,7 +740,7 @@ func (s *fileSystemSource) readNextEntryAndMaybeIndex( batch = append(batch, d) - if len(batch) >= index.DocumentArrayPoolCapacity { + if len(batch) >= index.MetadataArrayPoolCapacity { return builder.FlushBatch(batch) } @@ -857,8 +857,8 @@ func (s *fileSystemSource) read( builder := result.NewIndexBuilder(segBuilder) indexOpts := s.opts.IndexOptions() - compactor, err := compaction.NewCompactor(indexOpts.DocumentArrayPool(), - index.DocumentArrayPoolCapacity, + compactor, err := compaction.NewCompactor(indexOpts.MetadataArrayPool(), + index.MetadataArrayPoolCapacity, indexOpts.SegmentBuilderOptions(), indexOpts.FSTSegmentOptions(), compaction.CompactorOptions{ diff --git a/src/dbnode/storage/bootstrap/bootstrapper/fs/source_data_test.go b/src/dbnode/storage/bootstrap/bootstrapper/fs/source_data_test.go index 6e2b509339..89f0cd6809 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/fs/source_data_test.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/fs/source_data_test.go @@ -79,8 +79,8 @@ var ( func newTestOptions(t require.TestingT, filePathPrefix string) Options { idxOpts := index.NewOptions() - compactor, err := compaction.NewCompactor(idxOpts.DocumentArrayPool(), - index.DocumentArrayPoolCapacity, + compactor, err := compaction.NewCompactor(idxOpts.MetadataArrayPool(), + index.MetadataArrayPoolCapacity, idxOpts.SegmentBuilderOptions(), idxOpts.FSTSegmentOptions(), compaction.CompactorOptions{ diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/peers_test.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/peers_test.go index d30b675457..d223877d8f 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/peers_test.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/peers_test.go @@ -46,8 +46,8 @@ func TestNewPeersBootstrapper(t *testing.T) { defer ctrl.Finish() idxOpts := index.NewOptions() - compactor, err := compaction.NewCompactor(idxOpts.DocumentArrayPool(), - index.DocumentArrayPoolCapacity, + compactor, err := compaction.NewCompactor(idxOpts.MetadataArrayPool(), + index.MetadataArrayPoolCapacity, idxOpts.SegmentBuilderOptions(), idxOpts.FSTSegmentOptions(), compaction.CompactorOptions{ diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go index d25dd4b7d7..c8d409cfad 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go @@ -750,8 +750,8 @@ func (s *peersSource) readIndex( builder := result.NewIndexBuilder(segBuilder) indexOpts := s.opts.IndexOptions() - compactor, err := compaction.NewCompactor(indexOpts.DocumentArrayPool(), - index.DocumentArrayPoolCapacity, + compactor, err := compaction.NewCompactor(indexOpts.MetadataArrayPool(), + index.MetadataArrayPoolCapacity, indexOpts.SegmentBuilderOptions(), indexOpts.FSTSegmentOptions(), compaction.CompactorOptions{ @@ -831,13 +831,13 @@ func (s *peersSource) processReaders( resultLock *sync.Mutex, ) (result.ShardTimeRanges, []time.Time) { var ( - docsPool = s.opts.IndexOptions().DocumentArrayPool() - batch = docsPool.Get() + metadataPool = s.opts.IndexOptions().MetadataArrayPool() + batch = metadataPool.Get() timesWithErrors []time.Time totalEntries int ) defer func() { - docsPool.Put(batch) + metadataPool.Put(batch) // Return readers to pool. for _, shardReaders := range timeWindowReaders.Readers { for _, r := range shardReaders.Readers { @@ -1031,7 +1031,7 @@ func (s *peersSource) readNextEntryAndMaybeIndex( batch = append(batch, d) - if len(batch) >= index.DocumentArrayPoolCapacity { + if len(batch) >= index.MetadataArrayPoolCapacity { return builder.FlushBatch(batch) } diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/source_data_test.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/source_data_test.go index 25a6d565bc..7242a47fe7 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/source_data_test.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/source_data_test.go @@ -85,8 +85,8 @@ type namespaceOption func(namespace.Options) namespace.Options func newTestDefaultOpts(t *testing.T, ctrl *gomock.Controller) Options { idxOpts := index.NewOptions() - compactor, err := compaction.NewCompactor(idxOpts.DocumentArrayPool(), - index.DocumentArrayPoolCapacity, + compactor, err := compaction.NewCompactor(idxOpts.MetadataArrayPool(), + index.MetadataArrayPoolCapacity, idxOpts.SegmentBuilderOptions(), idxOpts.FSTSegmentOptions(), compaction.CompactorOptions{ diff --git a/src/dbnode/storage/index/aggregate_results.go b/src/dbnode/storage/index/aggregate_results.go index adfe613600..b2bd4ea908 100644 --- a/src/dbnode/storage/index/aggregate_results.go +++ b/src/dbnode/storage/index/aggregate_results.go @@ -25,6 +25,7 @@ import ( "sync" "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" ) @@ -49,6 +50,8 @@ type aggregatedResults struct { pool AggregateResultsPool valuesPool AggregateValuesPool + + encodedDocReader docs.EncodedDocumentReader } // NewAggregateResults returns a new AggregateResults object. @@ -104,7 +107,7 @@ func (r *aggregatedResults) Reset( r.Unlock() } -func (r *aggregatedResults) AddDocuments(batch []doc.Metadata) (int, int, error) { +func (r *aggregatedResults) AddDocuments(batch []doc.Document) (int, int, error) { r.Lock() err := r.addDocumentsBatchWithLock(batch) size := r.resultsMap.Len() @@ -162,7 +165,7 @@ func (r *aggregatedResults) AddFields(batch []AggregateResultsEntry) (int, int) } func (r *aggregatedResults) addDocumentsBatchWithLock( - batch []doc.Metadata, + batch []doc.Document, ) error { for _, doc := range batch { switch r.aggregateOpts.Type { @@ -193,8 +196,12 @@ func (r *aggregatedResults) addDocumentsBatchWithLock( } func (r *aggregatedResults) addDocumentTermsWithLock( - document doc.Metadata, + container doc.Document, ) error { + document, err := docs.MetadataFromDocument(container, &r.encodedDocReader) + if err != nil { + return fmt.Errorf("unable to decode encoded document; %w", err) + } for _, field := range document.Fields { if err := r.addTermWithLock(field.Name); err != nil { return fmt.Errorf("unable to add document terms [%+v]: %v", document, err) @@ -233,8 +240,12 @@ func (r *aggregatedResults) addTermWithLock( } func (r *aggregatedResults) addDocumentWithLock( - document doc.Metadata, + container doc.Document, ) error { + document, err := docs.MetadataFromDocument(container, &r.encodedDocReader) + if err != nil { + return fmt.Errorf("unable to decode encoded document; %w", err) + } for _, field := range document.Fields { if err := r.addFieldWithLock(field.Name, field.Value); err != nil { return fmt.Errorf("unable to add document [%+v]: %v", document, err) diff --git a/src/dbnode/storage/index/aggregate_results_test.go b/src/dbnode/storage/index/aggregate_results_test.go index 4a92a9f900..7ca2954598 100644 --- a/src/dbnode/storage/index/aggregate_results_test.go +++ b/src/dbnode/storage/index/aggregate_results_test.go @@ -32,7 +32,7 @@ import ( "github.com/stretchr/testify/require" ) -func genDoc(strs ...string) doc.Metadata { +func genDoc(strs ...string) doc.Document { if len(strs)%2 != 0 { panic("invalid test setup; need even str length") } @@ -45,15 +45,15 @@ func genDoc(strs ...string) doc.Metadata { } } - return doc.Metadata{Fields: fields} + return doc.NewDocumentFromMetadata(doc.Metadata{Fields: fields}) } func TestAggResultsInsertInvalid(t *testing.T) { res := NewAggregateResults(nil, AggregateResultsOptions{}, testOpts) assert.True(t, res.EnforceLimits()) - dInvalid := doc.Metadata{Fields: []doc.Field{{}}} - size, docsCount, err := res.AddDocuments([]doc.Metadata{dInvalid}) + dInvalid := doc.NewDocumentFromMetadata(doc.Metadata{Fields: []doc.Field{{}}}) + size, docsCount, err := res.AddDocuments([]doc.Document{dInvalid}) require.Error(t, err) require.Equal(t, 0, size) require.Equal(t, 1, docsCount) @@ -62,7 +62,7 @@ func TestAggResultsInsertInvalid(t *testing.T) { require.Equal(t, 1, res.TotalDocsCount()) dInvalid = genDoc("", "foo") - size, docsCount, err = res.AddDocuments([]doc.Metadata{dInvalid}) + size, docsCount, err = res.AddDocuments([]doc.Document{dInvalid}) require.Error(t, err) require.Equal(t, 0, size) require.Equal(t, 2, docsCount) @@ -74,7 +74,7 @@ func TestAggResultsInsertInvalid(t *testing.T) { func TestAggResultsInsertEmptyTermValue(t *testing.T) { res := NewAggregateResults(nil, AggregateResultsOptions{}, testOpts) dValidEmptyTerm := genDoc("foo", "") - size, docsCount, err := res.AddDocuments([]doc.Metadata{dValidEmptyTerm}) + size, docsCount, err := res.AddDocuments([]doc.Document{dValidEmptyTerm}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) @@ -87,7 +87,7 @@ func TestAggResultsInsertBatchOfTwo(t *testing.T) { res := NewAggregateResults(nil, AggregateResultsOptions{}, testOpts) d1 := genDoc("d1", "") d2 := genDoc("d2", "") - size, docsCount, err := res.AddDocuments([]doc.Metadata{d1, d2}) + size, docsCount, err := res.AddDocuments([]doc.Document{d1, d2}) require.NoError(t, err) require.Equal(t, 2, size) require.Equal(t, 2, docsCount) @@ -100,8 +100,8 @@ func TestAggResultsTermOnlyInsert(t *testing.T) { res := NewAggregateResults(nil, AggregateResultsOptions{ Type: AggregateTagNames, }, testOpts) - dInvalid := doc.Metadata{Fields: []doc.Field{{}}} - size, docsCount, err := res.AddDocuments([]doc.Metadata{dInvalid}) + dInvalid := doc.NewDocumentFromMetadata(doc.Metadata{Fields: []doc.Field{{}}}) + size, docsCount, err := res.AddDocuments([]doc.Document{dInvalid}) require.Error(t, err) require.Equal(t, 0, size) require.Equal(t, 1, docsCount) @@ -110,7 +110,7 @@ func TestAggResultsTermOnlyInsert(t *testing.T) { require.Equal(t, 1, res.TotalDocsCount()) dInvalid = genDoc("", "foo") - size, docsCount, err = res.AddDocuments([]doc.Metadata{dInvalid}) + size, docsCount, err = res.AddDocuments([]doc.Document{dInvalid}) require.Error(t, err) require.Equal(t, 0, size) require.Equal(t, 2, docsCount) @@ -119,7 +119,7 @@ func TestAggResultsTermOnlyInsert(t *testing.T) { require.Equal(t, 2, res.TotalDocsCount()) valid := genDoc("foo", "") - size, docsCount, err = res.AddDocuments([]doc.Metadata{valid}) + size, docsCount, err = res.AddDocuments([]doc.Document{valid}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 3, docsCount) @@ -130,7 +130,7 @@ func TestAggResultsTermOnlyInsert(t *testing.T) { func testAggResultsInsertIdempotency(t *testing.T, res AggregateResults) { dValid := genDoc("foo", "bar") - size, docsCount, err := res.AddDocuments([]doc.Metadata{dValid}) + size, docsCount, err := res.AddDocuments([]doc.Document{dValid}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) @@ -138,7 +138,7 @@ func testAggResultsInsertIdempotency(t *testing.T, res AggregateResults) { require.Equal(t, 1, res.Size()) require.Equal(t, 1, res.TotalDocsCount()) - size, docsCount, err = res.AddDocuments([]doc.Metadata{dValid}) + size, docsCount, err = res.AddDocuments([]doc.Document{dValid}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 2, docsCount) @@ -164,7 +164,7 @@ func TestInvalidAggregateType(t *testing.T) { Type: 100, }, testOpts) dValid := genDoc("foo", "bar") - size, docsCount, err := res.AddDocuments([]doc.Metadata{dValid}) + size, docsCount, err := res.AddDocuments([]doc.Document{dValid}) require.Error(t, err) require.Equal(t, 0, size) require.Equal(t, 1, docsCount) @@ -173,7 +173,7 @@ func TestInvalidAggregateType(t *testing.T) { func TestAggResultsSameName(t *testing.T) { res := NewAggregateResults(nil, AggregateResultsOptions{}, testOpts) d1 := genDoc("foo", "bar") - size, docsCount, err := res.AddDocuments([]doc.Metadata{d1}) + size, docsCount, err := res.AddDocuments([]doc.Document{d1}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) @@ -185,7 +185,7 @@ func TestAggResultsSameName(t *testing.T) { assert.True(t, aggVals.Map().Contains(ident.StringID("bar"))) d2 := genDoc("foo", "biz") - size, docsCount, err = res.AddDocuments([]doc.Metadata{d2}) + size, docsCount, err = res.AddDocuments([]doc.Document{d2}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 2, docsCount) @@ -212,7 +212,7 @@ func TestAggResultsTermOnlySameName(t *testing.T) { Type: AggregateTagNames, }, testOpts) d1 := genDoc("foo", "bar") - size, docsCount, err := res.AddDocuments([]doc.Metadata{d1}) + size, docsCount, err := res.AddDocuments([]doc.Document{d1}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) @@ -223,7 +223,7 @@ func TestAggResultsTermOnlySameName(t *testing.T) { assertNoValuesInNameOnlyAggregate(t, aggVals) d2 := genDoc("foo", "biz") - size, docsCount, err = res.AddDocuments([]doc.Metadata{d2}) + size, docsCount, err = res.AddDocuments([]doc.Document{d2}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 2, docsCount) @@ -235,20 +235,20 @@ func TestAggResultsTermOnlySameName(t *testing.T) { } func addMultipleDocuments(t *testing.T, res AggregateResults) (int, int) { - _, _, err := res.AddDocuments([]doc.Metadata{ + _, _, err := res.AddDocuments([]doc.Document{ genDoc("foo", "bar"), genDoc("fizz", "bar"), genDoc("buzz", "bar"), }) require.NoError(t, err) - _, _, err = res.AddDocuments([]doc.Metadata{ + _, _, err = res.AddDocuments([]doc.Document{ genDoc("foo", "biz"), genDoc("fizz", "bar"), }) require.NoError(t, err) - size, docsCount, err := res.AddDocuments([]doc.Metadata{ + size, docsCount, err := res.AddDocuments([]doc.Document{ genDoc("foo", "baz", "buzz", "bag", "qux", "qaz"), }) @@ -376,9 +376,12 @@ func TestAggResultsMergeNameOnly(t *testing.T) { func TestAggResultsInsertCopies(t *testing.T) { res := NewAggregateResults(nil, AggregateResultsOptions{}, testOpts) dValid := genDoc("foo", "bar") - name := dValid.Fields[0].Name - value := dValid.Fields[0].Value - size, docsCount, err := res.AddDocuments([]doc.Metadata{dValid}) + + d, ok := dValid.Metadata() + require.True(t, ok) + name := d.Fields[0].Name + value := d.Fields[0].Value + size, docsCount, err := res.AddDocuments([]doc.Document{dValid}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) @@ -419,8 +422,10 @@ func TestAggResultsNameOnlyInsertCopies(t *testing.T) { Type: AggregateTagNames, }, testOpts) dValid := genDoc("foo", "bar") - name := dValid.Fields[0].Name - size, docsCount, err := res.AddDocuments([]doc.Metadata{dValid}) + d, ok := dValid.Metadata() + require.True(t, ok) + name := d.Fields[0].Name + size, docsCount, err := res.AddDocuments([]doc.Document{dValid}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) @@ -450,7 +455,7 @@ func TestAggResultsReset(t *testing.T) { res := NewAggregateResults(ident.StringID("qux"), AggregateResultsOptions{}, testOpts) d1 := genDoc("foo", "bar") - size, docsCount, err := res.AddDocuments([]doc.Metadata{d1}) + size, docsCount, err := res.AddDocuments([]doc.Document{d1}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) @@ -500,7 +505,7 @@ func TestAggResultFinalize(t *testing.T) { // Create a Results and insert some data. res := NewAggregateResults(nil, AggregateResultsOptions{}, testOpts) d1 := genDoc("foo", "bar") - size, docsCount, err := res.AddDocuments([]doc.Metadata{d1}) + size, docsCount, err := res.AddDocuments([]doc.Document{d1}) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) diff --git a/src/dbnode/storage/index/block.go b/src/dbnode/storage/index/block.go index 94892d232f..cbb641e6e5 100644 --- a/src/dbnode/storage/index/block.go +++ b/src/dbnode/storage/index/block.go @@ -533,9 +533,9 @@ func (b *block) closeAsync(closer io.Closer) { func (b *block) addQueryResults( cancellable *xresource.CancellableLifetime, results BaseResults, - batch []doc.Metadata, + batch []doc.Document, source []byte, -) ([]doc.Metadata, int, int, error) { +) ([]doc.Document, int, int, error) { // update recently queried docs to monitor memory. if results.EnforceLimits() { if err := b.docsLimit.Inc(len(batch), source); err != nil { @@ -557,7 +557,7 @@ func (b *block) addQueryResults( cancellable.ReleaseCheckout() // reset batch. - var emptyDoc doc.Metadata + var emptyDoc doc.Document for i := range batch { batch[i] = emptyDoc } diff --git a/src/dbnode/storage/index/block_test.go b/src/dbnode/storage/index/block_test.go index f84914962f..f33c6b71e9 100644 --- a/src/dbnode/storage/index/block_test.go +++ b/src/dbnode/storage/index/block_test.go @@ -30,6 +30,7 @@ import ( "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" "github.com/m3db/m3/src/dbnode/storage/index/compaction" + "github.com/m3db/m3/src/dbnode/test" "github.com/m3db/m3/src/dbnode/tracepoint" "github.com/m3db/m3/src/m3ninx/doc" "github.com/m3db/m3/src/m3ninx/idx" @@ -518,11 +519,11 @@ func TestBlockMockQueryExecutorExecIterErr(t *testing.T) { return exec, nil } - dIter := doc.NewMockMetadataIterator(ctrl) + dIter := doc.NewMockIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), - dIter.EXPECT().Current().Return(testDoc1()), + dIter.EXPECT().Current().Return(doc.NewDocumentFromMetadata(testDoc1())), dIter.EXPECT().Next().Return(false), dIter.EXPECT().Err().Return(fmt.Errorf("randomerr")), dIter.EXPECT().Close(), @@ -559,11 +560,11 @@ func TestBlockMockQueryExecutorExecLimit(t *testing.T) { return exec, nil } - dIter := doc.NewMockMetadataIterator(ctrl) + dIter := doc.NewMockIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), - dIter.EXPECT().Current().Return(testDoc1()), + dIter.EXPECT().Current().Return(doc.NewDocumentFromMetadata(testDoc1())), dIter.EXPECT().Next().Return(true), dIter.EXPECT().Err().Return(nil), dIter.EXPECT().Close().Return(nil), @@ -581,8 +582,10 @@ func TestBlockMockQueryExecutorExecLimit(t *testing.T) { require.False(t, exhaustive) require.Equal(t, 1, results.Map().Len()) - t1, ok := results.Map().Get(ident.StringID(string(testDoc1().ID))) + d, ok := results.Map().Get(testDoc1().ID) require.True(t, ok) + + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) @@ -610,7 +613,7 @@ func TestBlockMockQueryExecutorExecIterCloseErr(t *testing.T) { return exec, nil } - dIter := doc.NewMockMetadataIterator(ctrl) + dIter := doc.NewMockIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(false), @@ -649,11 +652,11 @@ func TestBlockMockQuerySeriesLimitNonExhaustive(t *testing.T) { return exec, nil } - dIter := doc.NewMockMetadataIterator(ctrl) + dIter := doc.NewMockIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), - dIter.EXPECT().Current().Return(testDoc1()), + dIter.EXPECT().Current().Return(doc.NewDocumentFromMetadata(testDoc1())), dIter.EXPECT().Next().Return(true), dIter.EXPECT().Err().Return(nil), dIter.EXPECT().Close().Return(nil), @@ -670,8 +673,11 @@ func TestBlockMockQuerySeriesLimitNonExhaustive(t *testing.T) { require.False(t, exhaustive) require.Equal(t, 1, results.Map().Len()) - t1, ok := results.Map().Get(ident.StringID(string(testDoc1().ID))) + + d, ok := results.Map().Get(testDoc1().ID) require.True(t, ok) + + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) @@ -699,11 +705,11 @@ func TestBlockMockQuerySeriesLimitExhaustive(t *testing.T) { return exec, nil } - dIter := doc.NewMockMetadataIterator(ctrl) + dIter := doc.NewMockIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), - dIter.EXPECT().Current().Return(testDoc1()), + dIter.EXPECT().Current().Return(doc.NewDocumentFromMetadata(testDoc1())), dIter.EXPECT().Next().Return(false), dIter.EXPECT().Err().Return(nil), dIter.EXPECT().Close().Return(nil), @@ -722,8 +728,9 @@ func TestBlockMockQuerySeriesLimitExhaustive(t *testing.T) { rMap := results.Map() require.Equal(t, 1, rMap.Len()) - t1, ok := rMap.Get(ident.StringID(string(testDoc1().ID))) + d, ok := rMap.Get(testDoc1().ID) require.True(t, ok) + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) @@ -751,11 +758,11 @@ func TestBlockMockQueryDocsLimitNonExhaustive(t *testing.T) { return exec, nil } - dIter := doc.NewMockMetadataIterator(ctrl) + dIter := doc.NewMockIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), - dIter.EXPECT().Current().Return(testDoc1()), + dIter.EXPECT().Current().Return(doc.NewDocumentFromMetadata(testDoc1())), dIter.EXPECT().Next().Return(true), dIter.EXPECT().Err().Return(nil), dIter.EXPECT().Close().Return(nil), @@ -772,8 +779,9 @@ func TestBlockMockQueryDocsLimitNonExhaustive(t *testing.T) { require.False(t, exhaustive) require.Equal(t, 1, results.Map().Len()) - t1, ok := results.Map().Get(ident.StringID(string(testDoc1().ID))) + d, ok := results.Map().Get(testDoc1().ID) require.True(t, ok) + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) @@ -801,11 +809,11 @@ func TestBlockMockQueryDocsLimitExhaustive(t *testing.T) { return exec, nil } - dIter := doc.NewMockMetadataIterator(ctrl) + dIter := doc.NewMockIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), - dIter.EXPECT().Current().Return(testDoc1()), + dIter.EXPECT().Current().Return(doc.NewDocumentFromMetadata(testDoc1())), dIter.EXPECT().Next().Return(false), dIter.EXPECT().Err().Return(nil), dIter.EXPECT().Close().Return(nil), @@ -824,8 +832,9 @@ func TestBlockMockQueryDocsLimitExhaustive(t *testing.T) { rMap := results.Map() require.Equal(t, 1, rMap.Len()) - t1, ok := rMap.Get(ident.StringID(string(testDoc1().ID))) + d, ok := rMap.Get(testDoc1().ID) require.True(t, ok) + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) @@ -857,10 +866,12 @@ func TestBlockMockQueryMergeResultsMapLimit(t *testing.T) { limit := 1 results := NewQueryResults(nil, QueryResultsOptions{SizeLimit: limit}, testOpts) - _, _, err = results.AddDocuments([]doc.Metadata{testDoc1()}) + _, _, err = results.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(testDoc1()), + }) require.NoError(t, err) - dIter := doc.NewMockMetadataIterator(ctrl) + dIter := doc.NewMockIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), @@ -878,8 +889,9 @@ func TestBlockMockQueryMergeResultsMapLimit(t *testing.T) { rMap := results.Map() require.Equal(t, 1, rMap.Len()) - t1, ok := rMap.Get(ident.StringID(string(testDoc1().ID))) + d, ok := rMap.Get(testDoc1().ID) require.True(t, ok) + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) @@ -908,16 +920,18 @@ func TestBlockMockQueryMergeResultsDupeID(t *testing.T) { } results := NewQueryResults(nil, QueryResultsOptions{}, testOpts) - _, _, err = results.AddDocuments([]doc.Metadata{testDoc1()}) + _, _, err = results.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(testDoc1()), + }) require.NoError(t, err) - dIter := doc.NewMockMetadataIterator(ctrl) + dIter := doc.NewMockIterator(ctrl) gomock.InOrder( exec.EXPECT().Execute(gomock.Any()).Return(dIter, nil), dIter.EXPECT().Next().Return(true), - dIter.EXPECT().Current().Return(testDoc1DupeID()), + dIter.EXPECT().Current().Return(doc.NewDocumentFromMetadata(testDoc1DupeID())), dIter.EXPECT().Next().Return(true), - dIter.EXPECT().Current().Return(testDoc2()), + dIter.EXPECT().Current().Return(doc.NewDocumentFromMetadata(testDoc2())), dIter.EXPECT().Next().Return(false), dIter.EXPECT().Err().Return(nil), dIter.EXPECT().Close().Return(nil), @@ -933,14 +947,16 @@ func TestBlockMockQueryMergeResultsDupeID(t *testing.T) { rMap := results.Map() require.Equal(t, 2, rMap.Len()) - t1, ok := rMap.Get(ident.StringID(string(testDoc1().ID))) + d, ok := rMap.Get(testDoc1().ID) require.True(t, ok) + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) - t2, ok := rMap.Get(ident.StringID(string(testDoc2().ID))) + d, ok = rMap.Get(testDoc2().ID) require.True(t, ok) + t2 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz", "some", "more")).Matches( t2)) @@ -1402,14 +1418,16 @@ func TestBlockE2EInsertQuery(t *testing.T) { require.Equal(t, 2, results.Size()) rMap := results.Map() - t1, ok := rMap.Get(ident.StringID(string(testDoc1().ID))) + d, ok := rMap.Get(testDoc1().ID) require.True(t, ok) + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) - t2, ok := rMap.Get(ident.StringID(string(testDoc2().ID))) + d, ok = rMap.Get(testDoc2().ID) require.True(t, ok) + t2 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz", "some", "more")).Matches( t2)) @@ -1479,17 +1497,19 @@ func TestBlockE2EInsertQueryLimit(t *testing.T) { rMap := results.Map() numFound := 0 - t1, ok := rMap.Get(ident.StringID(string(testDoc1().ID))) + d, ok := rMap.Get(testDoc1().ID) if ok { numFound++ + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) } - t2, ok := rMap.Get(ident.StringID(string(testDoc2().ID))) + d, ok = rMap.Get(testDoc2().ID) if ok { numFound++ + t2 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz", "some", "more")).Matches( t2)) @@ -1567,14 +1587,16 @@ func TestBlockE2EInsertAddResultsQuery(t *testing.T) { require.Equal(t, 2, results.Size()) rMap := results.Map() - t1, ok := rMap.Get(ident.StringID(string(testDoc1().ID))) + d, ok := rMap.Get(testDoc1().ID) require.True(t, ok) + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) - t2, ok := rMap.Get(ident.StringID(string(testDoc2().ID))) + d, ok = rMap.Get(testDoc2().ID) require.True(t, ok) + t2 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz", "some", "more")).Matches( t2)) @@ -1646,14 +1668,16 @@ func TestBlockE2EInsertAddResultsMergeQuery(t *testing.T) { require.Equal(t, 2, results.Size()) rMap := results.Map() - t1, ok := results.Map().Get(ident.StringID(string(testDoc1().ID))) + d, ok := results.Map().Get(testDoc1().ID) require.True(t, ok) + t1 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz")).Matches( t1)) - t2, ok := rMap.Get(ident.StringID(string(testDoc2().ID))) + d, ok = rMap.Get(testDoc2().ID) require.True(t, ok) + t2 := test.DocumentToTagIter(t, d) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("bar", "baz", "some", "more")).Matches( t2)) diff --git a/src/dbnode/storage/index/compaction/compactor.go b/src/dbnode/storage/index/compaction/compactor.go index 7ec097fcc0..b69d8908f6 100644 --- a/src/dbnode/storage/index/compaction/compactor.go +++ b/src/dbnode/storage/index/compaction/compactor.go @@ -47,7 +47,7 @@ type Compactor struct { opts CompactorOptions writer fst.Writer - docsPool doc.DocumentArrayPool + metadataPool doc.MetadataArrayPool docsMaxBatch int fstOpts fst.Options builder segment.SegmentsBuilder @@ -71,7 +71,7 @@ type CompactorOptions struct { // NewCompactor returns a new compactor which reuses buffers // to avoid allocating intermediate buffers when compacting. func NewCompactor( - docsPool doc.DocumentArrayPool, + metadataPool doc.MetadataArrayPool, docsMaxBatch int, builderOpts builder.Options, fstOpts fst.Options, @@ -88,7 +88,7 @@ func NewCompactor( return &Compactor{ opts: opts, writer: writer, - docsPool: docsPool, + metadataPool: metadataPool, docsMaxBatch: docsMaxBatch, builder: builder.NewBuilderFromSegments(builderOpts), fstOpts: fstOpts, @@ -147,9 +147,9 @@ func (c *Compactor) CompactUsingBuilder( } // Need to combine segments first - batch := c.docsPool.Get() + batch := c.metadataPool.Get() defer func() { - c.docsPool.Put(batch) + c.metadataPool.Put(batch) }() // flushBatch is declared to reuse the same code from the @@ -374,7 +374,7 @@ func (c *Compactor) Close() error { c.closed = true c.writer = nil - c.docsPool = nil + c.metadataPool = nil c.fstOpts = nil c.builder = nil c.buff = nil diff --git a/src/dbnode/storage/index/compaction/compactor_test.go b/src/dbnode/storage/index/compaction/compactor_test.go index 631d8c892b..e0cb38249c 100644 --- a/src/dbnode/storage/index/compaction/compactor_test.go +++ b/src/dbnode/storage/index/compaction/compactor_test.go @@ -69,15 +69,15 @@ var ( }, } - testDocsMaxBatch = 8 - testDocsPool = doc.NewDocumentArrayPool(doc.DocumentArrayPoolOpts{ + testMetadataMaxBatch = 8 + testMetadataPool = doc.NewMetadataArrayPool(doc.MetadataArrayPoolOpts{ Options: pool.NewObjectPoolOptions().SetSize(1), - Capacity: testDocsMaxBatch, + Capacity: testMetadataMaxBatch, }) ) func init() { - testDocsPool.Init() + testMetadataPool.Init() } func TestCompactorSingleMutableSegment(t *testing.T) { @@ -90,7 +90,7 @@ func TestCompactorSingleMutableSegment(t *testing.T) { _, err = seg.Insert(testDocuments[1]) require.NoError(t, err) - compactor, err := NewCompactor(testDocsPool, testDocsMaxBatch, + compactor, err := NewCompactor(testMetadataPool, testMetadataMaxBatch, testBuilderSegmentOptions, testFSTSegmentOptions, CompactorOptions{}) require.NoError(t, err) @@ -114,7 +114,7 @@ func TestCompactorSingleMutableSegmentWithMmapDocsData(t *testing.T) { _, err = seg.Insert(testDocuments[1]) require.NoError(t, err) - compactor, err := NewCompactor(testDocsPool, testDocsMaxBatch, + compactor, err := NewCompactor(testMetadataPool, testMetadataMaxBatch, testBuilderSegmentOptions, testFSTSegmentOptions, CompactorOptions{ MmapDocsData: true, }) @@ -143,7 +143,7 @@ func TestCompactorManySegments(t *testing.T) { _, err = seg2.Insert(testDocuments[1]) require.NoError(t, err) - compactor, err := NewCompactor(testDocsPool, testDocsMaxBatch, + compactor, err := NewCompactor(testMetadataPool, testMetadataMaxBatch, testBuilderSegmentOptions, testFSTSegmentOptions, CompactorOptions{}) require.NoError(t, err) @@ -174,7 +174,7 @@ func TestCompactorCompactDuplicateIDsNoError(t *testing.T) { _, err = seg2.Insert(testDocuments[1]) require.NoError(t, err) - compactor, err := NewCompactor(testDocsPool, testDocsMaxBatch, + compactor, err := NewCompactor(testMetadataPool, testMetadataMaxBatch, testBuilderSegmentOptions, testFSTSegmentOptions, CompactorOptions{}) require.NoError(t, err) diff --git a/src/dbnode/storage/index/index_mock.go b/src/dbnode/storage/index/index_mock.go index 41e14ef338..0c44cf9614 100644 --- a/src/dbnode/storage/index/index_mock.go +++ b/src/dbnode/storage/index/index_mock.go @@ -1,7 +1,7 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/m3db/m3/src/dbnode/storage/index/types.go -// Copyright (c) 2020 Uber Technologies, Inc. +// Copyright (c) 2021 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -129,7 +129,7 @@ func (mr *MockBaseResultsMockRecorder) EnforceLimits() *gomock.Call { } // AddDocuments mocks base method -func (m *MockBaseResults) AddDocuments(batch []doc.Metadata) (int, int, error) { +func (m *MockBaseResults) AddDocuments(batch []doc.Document) (int, int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AddDocuments", batch) ret0, _ := ret[0].(int) @@ -236,7 +236,7 @@ func (mr *MockQueryResultsMockRecorder) EnforceLimits() *gomock.Call { } // AddDocuments mocks base method -func (m *MockQueryResults) AddDocuments(batch []doc.Metadata) (int, int, error) { +func (m *MockQueryResults) AddDocuments(batch []doc.Document) (int, int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AddDocuments", batch) ret0, _ := ret[0].(int) @@ -430,7 +430,7 @@ func (mr *MockAggregateResultsMockRecorder) EnforceLimits() *gomock.Call { } // AddDocuments mocks base method -func (m *MockAggregateResults) AddDocuments(batch []doc.Metadata) (int, int, error) { +func (m *MockAggregateResults) AddDocuments(batch []doc.Document) (int, int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AddDocuments", batch) ret0, _ := ret[0].(int) @@ -1485,6 +1485,34 @@ func (mr *MockOptionsMockRecorder) DocumentArrayPool() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DocumentArrayPool", reflect.TypeOf((*MockOptions)(nil).DocumentArrayPool)) } +// SetMetadataArrayPool mocks base method +func (m *MockOptions) SetMetadataArrayPool(value doc.MetadataArrayPool) Options { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetMetadataArrayPool", value) + ret0, _ := ret[0].(Options) + return ret0 +} + +// SetMetadataArrayPool indicates an expected call of SetMetadataArrayPool +func (mr *MockOptionsMockRecorder) SetMetadataArrayPool(value interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetMetadataArrayPool", reflect.TypeOf((*MockOptions)(nil).SetMetadataArrayPool), value) +} + +// MetadataArrayPool mocks base method +func (m *MockOptions) MetadataArrayPool() doc.MetadataArrayPool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MetadataArrayPool") + ret0, _ := ret[0].(doc.MetadataArrayPool) + return ret0 +} + +// MetadataArrayPool indicates an expected call of MetadataArrayPool +func (mr *MockOptionsMockRecorder) MetadataArrayPool() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MetadataArrayPool", reflect.TypeOf((*MockOptions)(nil).MetadataArrayPool)) +} + // SetAggregateResultsEntryArrayPool mocks base method func (m *MockOptions) SetAggregateResultsEntryArrayPool(value AggregateResultsEntryArrayPool) Options { m.ctrl.T.Helper() diff --git a/src/dbnode/storage/index/mutable_segments.go b/src/dbnode/storage/index/mutable_segments.go index 256bb4a313..4722f8e77f 100644 --- a/src/dbnode/storage/index/mutable_segments.go +++ b/src/dbnode/storage/index/mutable_segments.go @@ -781,8 +781,8 @@ func (m *mutableSegmentsCompact) allocLazyBuilderAndCompactorsWithLock( opts Options, ) error { var ( - err error - docsPool = opts.DocumentArrayPool() + err error + metadataPool = opts.MetadataArrayPool() ) if m.segmentBuilder == nil { builderOpts := opts.SegmentBuilderOptions(). @@ -795,8 +795,8 @@ func (m *mutableSegmentsCompact) allocLazyBuilderAndCompactorsWithLock( } if m.foregroundCompactor == nil { - m.foregroundCompactor, err = compaction.NewCompactor(docsPool, - DocumentArrayPoolCapacity, + m.foregroundCompactor, err = compaction.NewCompactor(metadataPool, + MetadataArrayPoolCapacity, opts.SegmentBuilderOptions(), opts.FSTSegmentOptions(), compaction.CompactorOptions{ @@ -814,8 +814,8 @@ func (m *mutableSegmentsCompact) allocLazyBuilderAndCompactorsWithLock( } if m.backgroundCompactor == nil { - m.backgroundCompactor, err = compaction.NewCompactor(docsPool, - DocumentArrayPoolCapacity, + m.backgroundCompactor, err = compaction.NewCompactor(metadataPool, + MetadataArrayPoolCapacity, opts.SegmentBuilderOptions(), opts.FSTSegmentOptions(), compaction.CompactorOptions{ diff --git a/src/dbnode/storage/index/options.go b/src/dbnode/storage/index/options.go index 2c5ca34829..055ab95cac 100644 --- a/src/dbnode/storage/index/options.go +++ b/src/dbnode/storage/index/options.go @@ -40,18 +40,26 @@ const ( // defaultIndexInsertMode sets the default indexing mode to synchronous. defaultIndexInsertMode = InsertSync - // documentArrayPool size in general: 256*256*sizeof(doc.Metadata) - // = 256 * 256 * 16 - // = 1mb (but with Go's heap probably 2mb) + // metadataArrayPool size in general: 256*256*sizeof(doc.Metadata) + // = 256 * 256 * 48 + // =~ 3mb // TODO(r): Make this configurable in a followup change. + metadataArrayPoolSize = 256 + // MetadataArrayPoolCapacity is the capacity of the metadata array pool. + MetadataArrayPoolCapacity = 256 + metadataArrayPoolMaxCapacity = 256 // Do not allow grows, since we know the size + + // documentArrayPool size in general: 256*256*sizeof(doc.Document) + // = 256 * 256 * 80 + // =~ 5mb documentArrayPoolSize = 256 - // DocumentArrayPoolCapacity is the capacity of the document array pool. + // DocumentArrayPoolCapacity is the capacity of the encoded document array pool. DocumentArrayPoolCapacity = 256 documentArrayPoolMaxCapacity = 256 // Do not allow grows, since we know the size // aggregateResultsEntryArrayPool size in general: 256*256*sizeof(doc.Field) - // = 256 * 256 * 16 - // = 1mb (but with Go's heap probably 2mb) + // = 256 * 256 * 48 + // =~ 3mb // TODO(prateek): Make this configurable in a followup change. aggregateResultsEntryArrayPoolSize = 256 aggregateResultsEntryArrayPoolCapacity = 256 @@ -65,6 +73,7 @@ var ( errOptionsAggResultsPoolUnspecified = errors.New("aggregate results pool is unset") errOptionsAggValuesPoolUnspecified = errors.New("aggregate values pool is unset") errOptionsDocPoolUnspecified = errors.New("docs array pool is unset") + errOptionsDocContainerPoolUnspecified = errors.New("doc container array pool is unset") errOptionsAggResultsEntryPoolUnspecified = errors.New("aggregate results entry array pool is unset") errIDGenerationDisabled = errors.New("id generation is disabled") errPostingsListCacheUnspecified = errors.New("postings list cache is unset") @@ -118,6 +127,7 @@ type opts struct { aggResultsPool AggregateResultsPool aggValuesPool AggregateValuesPool docArrayPool doc.DocumentArrayPool + metadataArrayPool doc.MetadataArrayPool aggResultsEntryArrayPool AggregateResultsEntryArrayPool foregroundCompactionPlannerOpts compaction.PlannerOptions backgroundCompactionPlannerOpts compaction.PlannerOptions @@ -150,6 +160,14 @@ func NewOptions() Options { }) docArrayPool.Init() + metadataArrayPool := doc.NewMetadataArrayPool(doc.MetadataArrayPoolOpts{ + Options: pool.NewObjectPoolOptions(). + SetSize(metadataArrayPoolSize), + Capacity: MetadataArrayPoolCapacity, + MaxCapacity: metadataArrayPoolMaxCapacity, + }) + metadataArrayPool.Init() + aggResultsEntryArrayPool := NewAggregateResultsEntryArrayPool(AggregateResultsEntryArrayPoolOpts{ Options: pool.NewObjectPoolOptions(). SetSize(aggregateResultsEntryArrayPoolSize), @@ -172,6 +190,7 @@ func NewOptions() Options { aggResultsPool: aggResultsPool, aggValuesPool: aggValuesPool, docArrayPool: docArrayPool, + metadataArrayPool: metadataArrayPool, aggResultsEntryArrayPool: aggResultsEntryArrayPool, foregroundCompactionPlannerOpts: defaultForegroundCompactionOpts, backgroundCompactionPlannerOpts: defaultBackgroundCompactionOpts, @@ -206,6 +225,9 @@ func (o *opts) Validate() error { if o.docArrayPool == nil { return errOptionsDocPoolUnspecified } + if o.metadataArrayPool == nil { + return errOptionsDocContainerPoolUnspecified + } if o.aggResultsEntryArrayPool == nil { return errOptionsAggResultsEntryPoolUnspecified } @@ -339,6 +361,16 @@ func (o *opts) DocumentArrayPool() doc.DocumentArrayPool { return o.docArrayPool } +func (o *opts) SetMetadataArrayPool(value doc.MetadataArrayPool) Options { + opts := *o // nolint:govet + opts.metadataArrayPool = value + return &opts +} + +func (o *opts) MetadataArrayPool() doc.MetadataArrayPool { + return o.metadataArrayPool +} + func (o *opts) SetAggregateResultsEntryArrayPool(value AggregateResultsEntryArrayPool) Options { opts := *o opts.aggResultsEntryArrayPool = value diff --git a/src/dbnode/storage/index/results.go b/src/dbnode/storage/index/results.go index c4d09520e1..65a0875c04 100644 --- a/src/dbnode/storage/index/results.go +++ b/src/dbnode/storage/index/results.go @@ -24,8 +24,8 @@ import ( "errors" "sync" - "github.com/m3db/m3/src/dbnode/storage/index/convert" "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" ) @@ -44,6 +44,7 @@ type results struct { nsID ident.ID opts QueryResultsOptions + reusableID *ident.ReusableBytesID resultsMap *ResultsMap totalDocsCount int @@ -63,10 +64,11 @@ func NewQueryResults( return &results{ nsID: namespaceID, opts: opts, - resultsMap: newResultsMap(indexOpts.IdentifierPool()), + resultsMap: newResultsMap(), idPool: indexOpts.IdentifierPool(), bytesPool: indexOpts.CheckedBytesPool(), pool: indexOpts.QueryResultsPool(), + reusableID: ident.NewReusableBytesID(), } } @@ -84,19 +86,11 @@ func (r *results) Reset(nsID ident.ID, opts QueryResultsOptions) { nsID = r.idPool.Clone(nsID) } r.nsID = nsID - // Reset all values from map first if they are present. - for _, entry := range r.resultsMap.Iter() { - tags := entry.Value() - tags.Close() - } // Reset all keys in the map next, this will finalize the keys. r.resultsMap.Reset() r.totalDocsCount = 0 - // NB: could do keys+value in one step but I'm trying to avoid - // using an internal method of a code-gen'd type. - r.opts = opts r.Unlock() @@ -104,7 +98,7 @@ func (r *results) Reset(nsID ident.ID, opts QueryResultsOptions) { // NB: If documents with duplicate IDs are added, they are simply ignored and // the first document added with an ID is returned. -func (r *results) AddDocuments(batch []doc.Metadata) (int, int, error) { +func (r *results) AddDocuments(batch []doc.Document) (int, int, error) { r.Lock() err := r.addDocumentsBatchWithLock(batch) size := r.resultsMap.Len() @@ -114,7 +108,7 @@ func (r *results) AddDocuments(batch []doc.Metadata) (int, int, error) { return size, docsCount, err } -func (r *results) addDocumentsBatchWithLock(batch []doc.Metadata) error { +func (r *results) addDocumentsBatchWithLock(batch []doc.Document) error { for i := range batch { _, size, err := r.addDocumentWithLock(batch[i]) if err != nil { @@ -128,29 +122,30 @@ func (r *results) addDocumentsBatchWithLock(batch []doc.Metadata) error { return nil } -func (r *results) addDocumentWithLock(d doc.Metadata) (bool, int, error) { - if len(d.ID) == 0 { - return false, r.resultsMap.Len(), errUnableToAddResultMissingID +func (r *results) addDocumentWithLock(w doc.Document) (bool, int, error) { + id, err := docs.ReadIDFromDocument(w) + if err != nil { + return false, r.resultsMap.Len(), err } - // NB: can cast the []byte -> ident.ID to avoid an alloc - // before we're sure we need it. - tsID := ident.BytesID(d.ID) + if len(id) == 0 { + return false, r.resultsMap.Len(), errUnableToAddResultMissingID + } // Need to apply filter if set first. - if r.opts.FilterID != nil && !r.opts.FilterID(tsID) { + r.reusableID.Reset(id) + if r.opts.FilterID != nil && !r.opts.FilterID(r.reusableID) { return false, r.resultsMap.Len(), nil } // check if it already exists in the map. - if r.resultsMap.Contains(tsID) { + if r.resultsMap.Contains(id) { return false, r.resultsMap.Len(), nil } - tags := convert.ToSeriesTags(d, convert.Opts{NoClone: true}) // It is assumed that the document is valid for the lifetime of the index // results. - r.resultsMap.SetUnsafe(tsID, tags, resultMapNoFinalizeOpts) + r.resultsMap.SetUnsafe(id, w, resultMapNoFinalizeOpts) return true, r.resultsMap.Len(), nil } diff --git a/src/dbnode/storage/index/results_map_gen.go b/src/dbnode/storage/index/results_map_gen.go index 9ef643dc5d..748acfbe3b 100644 --- a/src/dbnode/storage/index/results_map_gen.go +++ b/src/dbnode/storage/index/results_map_gen.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019 Uber Technologies, Inc. +// Copyright (c) 2021 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -25,9 +25,33 @@ package index import ( - "github.com/m3db/m3/src/x/ident" + "github.com/m3db/m3/src/m3ninx/doc" ) +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// This file was automatically generated by genny. +// Any changes will be lost if this file is regenerated. +// see https://github.com/mauricelam/genny + // Copyright (c) 2018 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy @@ -53,16 +77,16 @@ import ( type ResultsMapHash uint64 // ResultsMapHashFn is the hash function to execute when hashing a key. -type ResultsMapHashFn func(ident.ID) ResultsMapHash +type ResultsMapHashFn func([]byte) ResultsMapHash // ResultsMapEqualsFn is the equals key function to execute when detecting equality of a key. -type ResultsMapEqualsFn func(ident.ID, ident.ID) bool +type ResultsMapEqualsFn func([]byte, []byte) bool // ResultsMapCopyFn is the copy key function to execute when copying the key. -type ResultsMapCopyFn func(ident.ID) ident.ID +type ResultsMapCopyFn func([]byte) []byte // ResultsMapFinalizeFn is the finalize key function to execute when finished with a key. -type ResultsMapFinalizeFn func(ident.ID) +type ResultsMapFinalizeFn func([]byte) // ResultsMap uses the genny package to provide a generic hash map that can be specialized // by running the following command from this root of the repository: @@ -115,21 +139,21 @@ type ResultsMapEntry struct { // key is used to check equality on lookups to resolve collisions key _ResultsMapKey // value type stored - value ident.TagIterator + value doc.Document } type _ResultsMapKey struct { - key ident.ID + key []byte finalize bool } // Key returns the map entry key. -func (e ResultsMapEntry) Key() ident.ID { +func (e ResultsMapEntry) Key() []byte { return e.key.key } // Value returns the map entry value. -func (e ResultsMapEntry) Value() ident.TagIterator { +func (e ResultsMapEntry) Value() doc.Document { return e.value } @@ -143,7 +167,7 @@ func _ResultsMapAlloc(opts _ResultsMapOptions) *ResultsMap { return m } -func (m *ResultsMap) newMapKey(k ident.ID, opts _ResultsMapKeyOptions) _ResultsMapKey { +func (m *ResultsMap) newMapKey(k []byte, opts _ResultsMapKeyOptions) _ResultsMapKey { key := _ResultsMapKey{key: k, finalize: opts.finalizeKey} if !opts.copyKey { return key @@ -161,7 +185,7 @@ func (m *ResultsMap) removeMapKey(hash ResultsMapHash, key _ResultsMapKey) { } // Get returns a value in the map for an identifier if found. -func (m *ResultsMap) Get(k ident.ID) (ident.TagIterator, bool) { +func (m *ResultsMap) Get(k []byte) (doc.Document, bool) { hash := m.hash(k) for entry, ok := m.lookup[hash]; ok; entry, ok = m.lookup[hash] { if m.equals(entry.key.key, k) { @@ -170,12 +194,12 @@ func (m *ResultsMap) Get(k ident.ID) (ident.TagIterator, bool) { // Linear probe to "next" to this entry (really a rehash) hash++ } - var empty ident.TagIterator + var empty doc.Document return empty, false } // Set will set the value for an identifier. -func (m *ResultsMap) Set(k ident.ID, v ident.TagIterator) { +func (m *ResultsMap) Set(k []byte, v doc.Document) { m.set(k, v, _ResultsMapKeyOptions{ copyKey: true, finalizeKey: m.finalize != nil, @@ -191,7 +215,7 @@ type ResultsMapSetUnsafeOptions struct { // SetUnsafe will set the value for an identifier with unsafe options for how // the map treats the key. -func (m *ResultsMap) SetUnsafe(k ident.ID, v ident.TagIterator, opts ResultsMapSetUnsafeOptions) { +func (m *ResultsMap) SetUnsafe(k []byte, v doc.Document, opts ResultsMapSetUnsafeOptions) { m.set(k, v, _ResultsMapKeyOptions{ copyKey: !opts.NoCopyKey, finalizeKey: !opts.NoFinalizeKey, @@ -203,7 +227,7 @@ type _ResultsMapKeyOptions struct { finalizeKey bool } -func (m *ResultsMap) set(k ident.ID, v ident.TagIterator, opts _ResultsMapKeyOptions) { +func (m *ResultsMap) set(k []byte, v doc.Document, opts _ResultsMapKeyOptions) { hash := m.hash(k) for entry, ok := m.lookup[hash]; ok; entry, ok = m.lookup[hash] { if m.equals(entry.key.key, k) { @@ -237,13 +261,13 @@ func (m *ResultsMap) Len() int { // Contains returns true if value exists for key, false otherwise, it is // shorthand for a call to Get that doesn't return the value. -func (m *ResultsMap) Contains(k ident.ID) bool { +func (m *ResultsMap) Contains(k []byte) bool { _, ok := m.Get(k) return ok } // Delete will remove a value set in the map for the specified key. -func (m *ResultsMap) Delete(k ident.ID) { +func (m *ResultsMap) Delete(k []byte) { hash := m.hash(k) for entry, ok := m.lookup[hash]; ok; entry, ok = m.lookup[hash] { if m.equals(entry.key.key, k) { diff --git a/src/dbnode/storage/index/results_new_map.go b/src/dbnode/storage/index/results_new_map.go index 6620400594..e0aac29c8b 100644 --- a/src/dbnode/storage/index/results_new_map.go +++ b/src/dbnode/storage/index/results_new_map.go @@ -21,7 +21,7 @@ package index import ( - "github.com/m3db/m3/src/x/ident" + "bytes" "github.com/cespare/xxhash/v2" ) @@ -30,19 +30,17 @@ const ( defaultInitialResultsMapSize = 10 ) -func newResultsMap(idPool ident.Pool) *ResultsMap { +func newResultsMap() *ResultsMap { return _ResultsMapAlloc(_ResultsMapOptions{ - hash: func(k ident.ID) ResultsMapHash { - return ResultsMapHash(xxhash.Sum64(k.Bytes())) + hash: func(k []byte) ResultsMapHash { + return ResultsMapHash(xxhash.Sum64(k)) }, - equals: func(x, y ident.ID) bool { - return x.Equal(y) + equals: bytes.Equal, + copy: func(k []byte) []byte { + return append(make([]byte, 0, len(k)), k...) }, - copy: func(k ident.ID) ident.ID { - return idPool.Clone(k) - }, - finalize: func(k ident.ID) { - k.Finalize() + finalize: func(k []byte) { + // NB(nate): no-op for bytes IDs }, initialSize: defaultInitialResultsMapSize, }) diff --git a/src/dbnode/storage/index/results_test.go b/src/dbnode/storage/index/results_test.go index 3dc91b5947..864a8bf50f 100644 --- a/src/dbnode/storage/index/results_test.go +++ b/src/dbnode/storage/index/results_test.go @@ -24,7 +24,10 @@ import ( "bytes" "testing" + idxconvert "github.com/m3db/m3/src/dbnode/storage/index/convert" + "github.com/m3db/m3/src/dbnode/test" "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" xtest "github.com/m3db/m3/src/x/test" @@ -59,7 +62,9 @@ func TestResultsInsertInvalid(t *testing.T) { assert.True(t, res.EnforceLimits()) dInvalid := doc.Metadata{ID: nil} - size, docsCount, err := res.AddDocuments([]doc.Metadata{dInvalid}) + size, docsCount, err := res.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(dInvalid), + }) require.Error(t, err) require.Equal(t, 0, size) require.Equal(t, 1, docsCount) @@ -71,7 +76,9 @@ func TestResultsInsertInvalid(t *testing.T) { func TestResultsInsertIdempotency(t *testing.T) { res := NewQueryResults(nil, QueryResultsOptions{}, testOpts) dValid := doc.Metadata{ID: []byte("abc")} - size, docsCount, err := res.AddDocuments([]doc.Metadata{dValid}) + size, docsCount, err := res.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(dValid), + }) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) @@ -79,7 +86,9 @@ func TestResultsInsertIdempotency(t *testing.T) { require.Equal(t, 1, res.Size()) require.Equal(t, 1, res.TotalDocsCount()) - size, docsCount, err = res.AddDocuments([]doc.Metadata{dValid}) + size, docsCount, err = res.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(dValid), + }) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 2, docsCount) @@ -92,7 +101,10 @@ func TestResultsInsertBatchOfTwo(t *testing.T) { res := NewQueryResults(nil, QueryResultsOptions{}, testOpts) d1 := doc.Metadata{ID: []byte("d1")} d2 := doc.Metadata{ID: []byte("d2")} - size, docsCount, err := res.AddDocuments([]doc.Metadata{d1, d2}) + size, docsCount, err := res.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(d1), + doc.NewDocumentFromMetadata(d2), + }) require.NoError(t, err) require.Equal(t, 2, size) require.Equal(t, 2, docsCount) @@ -104,7 +116,9 @@ func TestResultsInsertBatchOfTwo(t *testing.T) { func TestResultsFirstInsertWins(t *testing.T) { res := NewQueryResults(nil, QueryResultsOptions{}, testOpts) d1 := doc.Metadata{ID: []byte("abc")} - size, docsCount, err := res.AddDocuments([]doc.Metadata{d1}) + size, docsCount, err := res.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(d1), + }) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) @@ -112,8 +126,10 @@ func TestResultsFirstInsertWins(t *testing.T) { require.Equal(t, 1, res.Size()) require.Equal(t, 1, res.TotalDocsCount()) - tags, ok := res.Map().Get(ident.StringID("abc")) + d, ok := res.Map().Get(d1.ID) require.True(t, ok) + + tags := test.DocumentToTagIter(t, d) require.Equal(t, 0, tags.Remaining()) d2 := doc.Metadata{ @@ -124,7 +140,9 @@ func TestResultsFirstInsertWins(t *testing.T) { Value: []byte("bar"), }, }} - size, docsCount, err = res.AddDocuments([]doc.Metadata{d2}) + size, docsCount, err = res.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(d2), + }) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 2, docsCount) @@ -132,21 +150,27 @@ func TestResultsFirstInsertWins(t *testing.T) { require.Equal(t, 1, res.Size()) require.Equal(t, 2, res.TotalDocsCount()) - tags, ok = res.Map().Get(ident.StringID("abc")) + d, ok = res.Map().Get([]byte("abc")) require.True(t, ok) + + tags = test.DocumentToTagIter(t, d) require.Equal(t, 0, tags.Remaining()) } func TestResultsInsertContains(t *testing.T) { res := NewQueryResults(nil, QueryResultsOptions{}, testOpts) dValid := doc.Metadata{ID: []byte("abc")} - size, docsCount, err := res.AddDocuments([]doc.Metadata{dValid}) + size, docsCount, err := res.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(dValid), + }) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) - tags, ok := res.Map().Get(ident.StringID("abc")) + d, ok := res.Map().Get([]byte("abc")) require.True(t, ok) + + tags := test.DocumentToTagIter(t, d) require.Equal(t, 0, tags.Remaining()) } @@ -155,7 +179,9 @@ func TestResultsInsertDoesNotCopy(t *testing.T) { dValid := doc.Metadata{ID: []byte("abc"), Fields: []doc.Field{ {Name: []byte("name"), Value: []byte("value")}, }} - size, docsCount, err := res.AddDocuments([]doc.Metadata{dValid}) + size, docsCount, err := res.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(dValid), + }) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) @@ -165,9 +191,10 @@ func TestResultsInsertDoesNotCopy(t *testing.T) { // Our genny generated maps don't provide access to MapEntry directly, // so we iterate over the map to find the added entry. Could avoid this // in the future if we expose `func (m *Map) Entry(k Key) Entry {}`. + reader := docs.NewEncodedDocumentReader() for _, entry := range res.Map().Iter() { // see if this key has the same value as the added document's ID. - key := entry.Key().Bytes() + key := entry.Key() if !bytes.Equal(dValid.ID, key) { continue } @@ -175,7 +202,11 @@ func TestResultsInsertDoesNotCopy(t *testing.T) { // Ensure the underlying []byte for ID/Fields is the same. require.True(t, xtest.ByteSlicesBackedBySameData(key, dValid.ID)) - tags := entry.Value() + d := entry.Value() + m, err := docs.MetadataFromDocument(d, reader) + require.NoError(t, err) + + tags := idxconvert.ToSeriesTags(m, idxconvert.Opts{NoClone: true}) for _, f := range dValid.Fields { fName := f.Name fValue := f.Value @@ -200,17 +231,21 @@ func TestResultsInsertDoesNotCopy(t *testing.T) { func TestResultsReset(t *testing.T) { res := NewQueryResults(nil, QueryResultsOptions{}, testOpts) d1 := doc.Metadata{ID: []byte("abc")} - size, docsCount, err := res.AddDocuments([]doc.Metadata{d1}) + size, docsCount, err := res.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(d1), + }) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) - tags, ok := res.Map().Get(ident.StringID("abc")) + d, ok := res.Map().Get([]byte("abc")) require.True(t, ok) + + tags := test.DocumentToTagIter(t, d) require.Equal(t, 0, tags.Remaining()) res.Reset(nil, QueryResultsOptions{}) - _, ok = res.Map().Get(ident.StringID("abc")) + _, ok = res.Map().Get([]byte("abc")) require.False(t, ok) require.Equal(t, 0, tags.Remaining()) require.Equal(t, 0, res.Size()) @@ -234,29 +269,26 @@ func TestFinalize(t *testing.T) { // Create a Results and insert some data. res := NewQueryResults(nil, QueryResultsOptions{}, testOpts) d1 := doc.Metadata{ID: []byte("abc")} - size, docsCount, err := res.AddDocuments([]doc.Metadata{d1}) + size, docsCount, err := res.AddDocuments([]doc.Document{ + doc.NewDocumentFromMetadata(d1), + }) require.NoError(t, err) require.Equal(t, 1, size) require.Equal(t, 1, docsCount) // Ensure the data is present. - tags, ok := res.Map().Get(ident.StringID("abc")) + d, ok := res.Map().Get([]byte("abc")) require.True(t, ok) + + tags := test.DocumentToTagIter(t, d) require.Equal(t, 0, tags.Remaining()) // Call Finalize() to reset the Results. res.Finalize() // Ensure data was removed by call to Finalize(). - tags, ok = res.Map().Get(ident.StringID("abc")) + _, ok = res.Map().Get([]byte("abc")) require.False(t, ok) require.Equal(t, 0, res.Size()) require.Equal(t, 0, res.TotalDocsCount()) - - for _, entry := range res.Map().Iter() { - id, _ := entry.Key(), entry.Value() - require.False(t, id.IsNoFinalize()) - // TODO(rartoul): Could verify tags are NoFinalize() as well if - // they had that method. - } } diff --git a/src/dbnode/storage/index/types.go b/src/dbnode/storage/index/types.go index a37b7a6849..9e5e07ec07 100644 --- a/src/dbnode/storage/index/types.go +++ b/src/dbnode/storage/index/types.go @@ -164,7 +164,7 @@ type BaseResults interface { // modified after this function returns without affecting the results map. // TODO(r): We will need to change this behavior once index fields are // mutable and the most recent need to shadow older entries. - AddDocuments(batch []doc.Metadata) (size, docsCount int, err error) + AddDocuments(batch []doc.Document) (size, docsCount int, err error) // Finalize releases any resources held by the Results object, // including returning it to a backing pool. @@ -937,6 +937,12 @@ type Options interface { // DocumentArrayPool returns the document array pool. DocumentArrayPool() doc.DocumentArrayPool + // SetMetadataArrayPool sets the document container array pool. + SetMetadataArrayPool(value doc.MetadataArrayPool) Options + + // MetadataArrayPool returns the document container array pool. + MetadataArrayPool() doc.MetadataArrayPool + // SetAggregateResultsEntryArrayPool sets the aggregate results entry array pool. SetAggregateResultsEntryArrayPool(value AggregateResultsEntryArrayPool) Options diff --git a/src/dbnode/storage/index/wide_query_results.go b/src/dbnode/storage/index/wide_query_results.go index 4cf8a0011e..af6b707584 100644 --- a/src/dbnode/storage/index/wide_query_results.go +++ b/src/dbnode/storage/index/wide_query_results.go @@ -22,9 +22,11 @@ package index import ( "errors" + "fmt" "sync" "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" "github.com/m3db/m3/src/x/ident" ) @@ -94,7 +96,7 @@ func (r *wideResults) EnforceLimits() bool { return false } -func (r *wideResults) AddDocuments(batch []doc.Metadata) (int, int, error) { +func (r *wideResults) AddDocuments(batch []doc.Document) (int, int, error) { var size, totalDocsCount int r.RLock() size, totalDocsCount = r.size, r.totalDocsCount @@ -124,7 +126,7 @@ func (r *wideResults) AddDocuments(batch []doc.Metadata) (int, int, error) { return size, totalDocsCount, err } -func (r *wideResults) addDocumentsBatchWithLock(batch []doc.Metadata) error { +func (r *wideResults) addDocumentsBatchWithLock(batch []doc.Document) error { for i := range batch { if err := r.addDocumentWithLock(batch[i]); err != nil { return err @@ -134,12 +136,16 @@ func (r *wideResults) addDocumentsBatchWithLock(batch []doc.Metadata) error { return nil } -func (r *wideResults) addDocumentWithLock(d doc.Metadata) error { - if len(d.ID) == 0 { +func (r *wideResults) addDocumentWithLock(w doc.Document) error { + docID, err := docs.ReadIDFromDocument(w) + if err != nil { + return fmt.Errorf("unable to decode document ID: %w", err) + } + if len(docID) == 0 { return errUnableToAddResultMissingID } - var tsID ident.ID = ident.BytesID(d.ID) + var tsID ident.ID = ident.BytesID(docID) documentShard, documentShardOwned := r.shardFilter(tsID) if !documentShardOwned { diff --git a/src/dbnode/storage/index/wide_query_results_test.go b/src/dbnode/storage/index/wide_query_results_test.go index 2f57a729ee..4160f32414 100644 --- a/src/dbnode/storage/index/wide_query_results_test.go +++ b/src/dbnode/storage/index/wide_query_results_test.go @@ -29,6 +29,7 @@ import ( "time" "github.com/m3db/m3/src/m3ninx/doc" + encoding "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" @@ -52,18 +53,19 @@ func init() { bytesPool.Init() } -func buildDocs(documentCount int, batchSize int) [][]doc.Metadata { +func buildDocs(documentCount int, batchSize int) [][]doc.Document { docBatches := int(math.Ceil(float64(documentCount) / float64(batchSize))) - docs := make([][]doc.Metadata, 0, docBatches) + docs := make([][]doc.Document, 0, docBatches) for i := 0; i < docBatches; i++ { - batch := make([]doc.Metadata, 0, batchSize) + batch := make([]doc.Document, 0, batchSize) for j := 0; j < batchSize; j++ { val := i*batchSize + j if val < documentCount { val := fmt.Sprintf("foo%d", i*batchSize+j) - batch = append(batch, doc.Metadata{ - ID: []byte(val), - }) + batch = append(batch, doc.NewDocumentFromMetadata( + doc.Metadata{ + ID: []byte(val), + })) } } @@ -73,12 +75,15 @@ func buildDocs(documentCount int, batchSize int) [][]doc.Metadata { return docs } -func buildExpected(_ *testing.T, docs [][]doc.Metadata) [][]string { +func buildExpected(t *testing.T, docs [][]doc.Document) [][]string { expected := make([][]string, 0, len(docs)) + reader := encoding.NewEncodedDocumentReader() for _, batch := range docs { idBatch := make([]string, 0, len(batch)) - for _, doc := range batch { - idBatch = append(idBatch, string(doc.ID)) + for _, document := range batch { // nolint:gocritic + m, err := encoding.MetadataFromDocument(document, reader) + require.NoError(t, err) + idBatch = append(idBatch, string(m.ID)) } expected = append(expected, idBatch) diff --git a/src/dbnode/storage/index_block_test.go b/src/dbnode/storage/index_block_test.go index 53bade47c0..a4cbadcdfa 100644 --- a/src/dbnode/storage/index_block_test.go +++ b/src/dbnode/storage/index_block_test.go @@ -831,10 +831,13 @@ func TestLimits(t *testing.T) { opts interface{}, results index.BaseResults, logFields interface{}) (bool, error) { - _, _, err = results.AddDocuments([]doc.Metadata{ + _, _, err = results.AddDocuments([]doc.Document{ // Results in size=1 and docs=2. - {ID: []byte("A")}, - {ID: []byte("A")}, + // Byte array represents ID encoded as bytes. + // 1 represents the ID length in bytes, 49 is the ID itself which is + // the ASCII value for A + doc.NewDocumentFromMetadata(doc.Metadata{ID: []byte("A")}), + doc.NewDocumentFromMetadata(doc.Metadata{ID: []byte("A")}), }) require.NoError(t, err) return false, nil diff --git a/src/dbnode/storage/index_query_concurrent_test.go b/src/dbnode/storage/index_query_concurrent_test.go index d27d442a92..3c2e3ff27b 100644 --- a/src/dbnode/storage/index_query_concurrent_test.go +++ b/src/dbnode/storage/index_query_concurrent_test.go @@ -31,9 +31,11 @@ import ( "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/dbnode/storage/index/convert" + testutil "github.com/m3db/m3/src/dbnode/test" "github.com/m3db/m3/src/m3ninx/doc" "github.com/m3db/m3/src/m3ninx/idx" "github.com/m3db/m3/src/x/context" + "github.com/m3db/m3/src/x/ident" xresource "github.com/m3db/m3/src/x/resource" xsync "github.com/m3db/m3/src/x/sync" xtest "github.com/m3db/m3/src/x/test" @@ -344,23 +346,24 @@ func testNamespaceIndexHighConcurrentQueries( // Read the results concurrently too hits := make(map[string]struct{}, results.Results.Size()) + id := ident.NewReusableBytesID() for _, entry := range results.Results.Map().Iter() { - id := entry.Key().String() - - doc, err := convert.FromSeriesIDAndTagIter(entry.Key(), entry.Value()) + id.Reset(entry.Key()) + tags := testutil.DocumentToTagIter(t, entry.Value()) + doc, err := convert.FromSeriesIDAndTagIter(id, tags) require.NoError(t, err) if err != nil { continue // this will fail the test anyway, but don't want to panic } - expectedDoc, ok := expectedResults[id] + expectedDoc, ok := expectedResults[id.String()] require.True(t, ok) if !ok { continue // this will fail the test anyway, but don't want to panic } require.Equal(t, expectedDoc, doc) - hits[id] = struct{}{} + hits[id.String()] = struct{}{} } expectedHits := idsPerBlock * (k + 1) require.Equal(t, expectedHits, len(hits)) diff --git a/src/dbnode/storage/index_queue_forward_write_test.go b/src/dbnode/storage/index_queue_forward_write_test.go index 05ddd534e4..7099951c36 100644 --- a/src/dbnode/storage/index_queue_forward_write_test.go +++ b/src/dbnode/storage/index_queue_forward_write_test.go @@ -29,11 +29,13 @@ import ( "github.com/m3db/m3/src/dbnode/namespace" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/storage/index" + idxconvert "github.com/m3db/m3/src/dbnode/storage/index/convert" "github.com/m3db/m3/src/dbnode/storage/series" "github.com/m3db/m3/src/dbnode/ts/writes" xmetrics "github.com/m3db/m3/src/dbnode/x/metrics" "github.com/m3db/m3/src/m3ninx/doc" m3ninxidx "github.com/m3db/m3/src/m3ninx/idx" + "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" @@ -143,6 +145,7 @@ func TestNamespaceForwardIndexInsertQuery(t *testing.T) { // write was correctly indexed to both. nextBlockTime := now.Add(blockSize) queryTimes := []time.Time{now, nextBlockTime} + reader := docs.NewEncodedDocumentReader() for _, ts := range queryTimes { res, err := idx.Query(ctx, index.Query{Query: reQuery}, index.QueryOptions{ StartInclusive: ts.Add(-1 * time.Minute), @@ -154,7 +157,11 @@ func TestNamespaceForwardIndexInsertQuery(t *testing.T) { results := res.Results require.Equal(t, "testns1", results.Namespace().String()) - tags, ok := results.Map().Get(ident.StringID("foo")) + d, ok := results.Map().Get(ident.BytesID("foo")) + md, err := docs.MetadataFromDocument(d, reader) + require.NoError(t, err) + tags := idxconvert.ToSeriesTags(md, idxconvert.Opts{NoClone: true}) + require.True(t, ok) require.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("name", "value")).Matches( diff --git a/src/dbnode/storage/index_queue_test.go b/src/dbnode/storage/index_queue_test.go index 64d1e6bf1f..14fbc09d5c 100644 --- a/src/dbnode/storage/index_queue_test.go +++ b/src/dbnode/storage/index_queue_test.go @@ -29,8 +29,10 @@ import ( "github.com/m3db/m3/src/dbnode/namespace" m3dberrors "github.com/m3db/m3/src/dbnode/storage/errors" "github.com/m3db/m3/src/dbnode/storage/index" + idxconvert "github.com/m3db/m3/src/dbnode/storage/index/convert" "github.com/m3db/m3/src/m3ninx/doc" m3ninxidx "github.com/m3db/m3/src/m3ninx/idx" + "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/context" "github.com/m3db/m3/src/x/ident" @@ -347,7 +349,12 @@ func TestNamespaceIndexInsertQuery(t *testing.T) { results := res.Results assert.Equal(t, "testns1", results.Namespace().String()) - tags, ok := results.Map().Get(ident.StringID("foo")) + reader := docs.NewEncodedDocumentReader() + d, ok := results.Map().Get(ident.BytesID("foo")) + md, err := docs.MetadataFromDocument(d, reader) + require.NoError(t, err) + tags := idxconvert.ToSeriesTags(md, idxconvert.Opts{NoClone: true}) + assert.True(t, ok) assert.True(t, ident.NewTagIterMatcher( ident.MustNewTagStringsIterator("name", "value")).Matches( diff --git a/src/dbnode/test/util.go b/src/dbnode/test/util.go new file mode 100644 index 0000000000..034c6d82f8 --- /dev/null +++ b/src/dbnode/test/util.go @@ -0,0 +1,43 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package test is a package for shared test helpers. +package test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/m3db/m3/src/dbnode/storage/index/convert" + "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" + "github.com/m3db/m3/src/x/ident" +) + +// DocumentToTagIter is a help for converting a doc.Document into an +// ident.TagIterator. +func DocumentToTagIter(t *testing.T, doc doc.Document) ident.TagIterator { + reader := docs.NewEncodedDocumentReader() + m, err := docs.MetadataFromDocument(doc, reader) + require.NoError(t, err) + + return convert.ToSeriesTags(m, convert.Opts{NoClone: true}) +} diff --git a/src/m3ninx/doc/doc_arraypool_gen.go b/src/m3ninx/doc/doc_arraypool_gen.go index c52d4f4305..cc6dab1a25 100644 --- a/src/m3ninx/doc/doc_arraypool_gen.go +++ b/src/m3ninx/doc/doc_arraypool_gen.go @@ -48,20 +48,20 @@ import ( // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -// DocumentArrayPool provides a pool for metadata slices. +// DocumentArrayPool provides a pool for document slices. type DocumentArrayPool interface { // Init initializes the array pool, it needs to be called // before Get/Put use. Init() // Get returns the a slice from the pool. - Get() []Metadata + Get() []Document // Put returns the provided slice to the pool. - Put(elems []Metadata) + Put(elems []Document) } -type DocumentFinalizeFn func([]Metadata) []Metadata +type DocumentFinalizeFn func([]Document) []Document type DocumentArrayPoolOpts struct { Options pool.ObjectPoolOptions @@ -85,15 +85,15 @@ func NewDocumentArrayPool(opts DocumentArrayPoolOpts) DocumentArrayPool { func (p *DocumentArrPool) Init() { p.pool.Init(func() interface{} { - return make([]Metadata, 0, p.opts.Capacity) + return make([]Document, 0, p.opts.Capacity) }) } -func (p *DocumentArrPool) Get() []Metadata { - return p.pool.Get().([]Metadata) +func (p *DocumentArrPool) Get() []Document { + return p.pool.Get().([]Document) } -func (p *DocumentArrPool) Put(arr []Metadata) { +func (p *DocumentArrPool) Put(arr []Document) { arr = p.opts.FinalizeFn(arr) if max := p.opts.MaxCapacity; max > 0 && cap(arr) > max { return @@ -101,8 +101,8 @@ func (p *DocumentArrPool) Put(arr []Metadata) { p.pool.Put(arr) } -func defaultDocumentFinalizerFn(elems []Metadata) []Metadata { - var empty Metadata +func defaultDocumentFinalizerFn(elems []Document) []Document { + var empty Document for i := range elems { elems[i] = empty } @@ -110,16 +110,16 @@ func defaultDocumentFinalizerFn(elems []Metadata) []Metadata { return elems } -type DocumentArr []Metadata +type DocumentArr []Document -func (elems DocumentArr) grow(n int) []Metadata { +func (elems DocumentArr) grow(n int) []Document { if cap(elems) < n { - elems = make([]Metadata, n) + elems = make([]Document, n) } elems = elems[:n] // following compiler optimized memcpy impl // https://github.com/golang/go/wiki/CompilerOptimizations#optimized-memclr - var empty Metadata + var empty Document for i := range elems { elems[i] = empty } diff --git a/src/m3ninx/doc/document.go b/src/m3ninx/doc/document.go index 7dc3f04a36..4eb6bc0f9c 100644 --- a/src/m3ninx/doc/document.go +++ b/src/m3ninx/doc/document.go @@ -221,9 +221,10 @@ type Encoded struct { // Document contains either metadata or an encoded metadata // but never both. type Document struct { - metadata Metadata encoded Encoded + metadata Metadata + hasEncoded bool hasMetadata bool } @@ -234,7 +235,7 @@ func NewDocumentFromMetadata(m Metadata) Document { // NewDocumentFromEncoded creates a Document from an Encoded. func NewDocumentFromEncoded(e Encoded) Document { - return Document{encoded: e} + return Document{encoded: e, hasEncoded: true} } // Metadata returns the metadata it contains, if it has one. Otherwise returns an empty metadata @@ -250,7 +251,7 @@ func (d *Document) Metadata() (Metadata, bool) { // Encoded returns the encoded metadata it contains, if it has one. Otherwise returns an // empty encoded metadata and false. func (d *Document) Encoded() (Encoded, bool) { - if !d.hasMetadata { + if d.hasEncoded { return d.encoded, true } diff --git a/src/m3ninx/doc/metadata_arraypool_gen.go b/src/m3ninx/doc/metadata_arraypool_gen.go new file mode 100644 index 0000000000..09aab94011 --- /dev/null +++ b/src/m3ninx/doc/metadata_arraypool_gen.go @@ -0,0 +1,127 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// This file was automatically generated by genny. +// Any changes will be lost if this file is regenerated. +// see https://github.com/mauricelam/genny + +package doc + +import ( + "github.com/m3db/m3/src/x/pool" +) + +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// MetadataArrayPool provides a pool for metadata slices. +type MetadataArrayPool interface { + // Init initializes the array pool, it needs to be called + // before Get/Put use. + Init() + + // Get returns the a slice from the pool. + Get() []Metadata + + // Put returns the provided slice to the pool. + Put(elems []Metadata) +} + +type MetadataFinalizeFn func([]Metadata) []Metadata + +type MetadataArrayPoolOpts struct { + Options pool.ObjectPoolOptions + Capacity int + MaxCapacity int + FinalizeFn MetadataFinalizeFn +} + +type MetadataArrPool struct { + opts MetadataArrayPoolOpts + pool pool.ObjectPool +} + +func NewMetadataArrayPool(opts MetadataArrayPoolOpts) MetadataArrayPool { + if opts.FinalizeFn == nil { + opts.FinalizeFn = defaultMetadataFinalizerFn + } + p := pool.NewObjectPool(opts.Options) + return &MetadataArrPool{opts, p} +} + +func (p *MetadataArrPool) Init() { + p.pool.Init(func() interface{} { + return make([]Metadata, 0, p.opts.Capacity) + }) +} + +func (p *MetadataArrPool) Get() []Metadata { + return p.pool.Get().([]Metadata) +} + +func (p *MetadataArrPool) Put(arr []Metadata) { + arr = p.opts.FinalizeFn(arr) + if max := p.opts.MaxCapacity; max > 0 && cap(arr) > max { + return + } + p.pool.Put(arr) +} + +func defaultMetadataFinalizerFn(elems []Metadata) []Metadata { + var empty Metadata + for i := range elems { + elems[i] = empty + } + elems = elems[:0] + return elems +} + +type MetadataArr []Metadata + +func (elems MetadataArr) grow(n int) []Metadata { + if cap(elems) < n { + elems = make([]Metadata, n) + } + elems = elems[:n] + // following compiler optimized memcpy impl + // https://github.com/golang/go/wiki/CompilerOptimizations#optimized-memclr + var empty Metadata + for i := range elems { + elems[i] = empty + } + return elems +} diff --git a/src/m3ninx/generated-source-files.mk b/src/m3ninx/generated-source-files.mk index 4b85df122b..bfd8364e91 100644 --- a/src/m3ninx/generated-source-files.mk +++ b/src/m3ninx/generated-source-files.mk @@ -98,9 +98,10 @@ genny-map-segment-mem-fieldsmap: # generation rule for all generated arraypools .PHONY: genny-arraypool-all -genny-arraypool-all: \ - genny-arraypool-bytes-slice-array-pool \ - genny-arraypool-document-array-pool \ +genny-arraypool-all: \ + genny-arraypool-bytes-slice-array-pool \ + genny-arraypool-document-array-pool \ + genny-arraypool-metadata-array-pool \ # arraypool generation rule for ./x/bytes.SliceArrayPool .PHONY: genny-arraypool-bytes-slice-array-pool @@ -117,13 +118,25 @@ genny-arraypool-bytes-slice-array-pool: # arraypool generation rule for ./doc.DocumentArrayPool .PHONY: genny-arraypool-document-array-pool genny-arraypool-document-array-pool: + cd $(m3x_package_path) && make genny-arraypool \ + pkg=doc \ + elem_type=Document \ + target_package=$(m3ninx_package)/doc \ + out_file=doc_arraypool_gen.go \ + rename_type_prefix=Document \ + rename_type_middle=Document \ + rename_constructor=NewDocumentArrayPool \ + rename_gen_types=true \ + +# arraypool generation rule for ./doc.MetadataArrayPool +.PHONY: genny-arraypool-metadata-array-pool +genny-arraypool-metadata-array-pool: cd $(m3x_package_path) && make genny-arraypool \ pkg=doc \ elem_type=Metadata \ target_package=$(m3ninx_package)/doc \ - out_file=doc_arraypool_gen.go \ - rename_type_prefix=Document \ - rename_type_middle=Document \ - rename_constructor=NewDocumentArrayPool \ + out_file=metadata_arraypool_gen.go \ + rename_type_prefix=Metadata \ + rename_type_middle=Metadata \ + rename_constructor=NewMetadataArrayPool \ rename_gen_types=true \ - diff --git a/src/m3ninx/index/segment/fst/encoding/docs/data.go b/src/m3ninx/index/segment/fst/encoding/docs/data.go index 00092152b3..aa2d44fd59 100644 --- a/src/m3ninx/index/segment/fst/encoding/docs/data.go +++ b/src/m3ninx/index/segment/fst/encoding/docs/data.go @@ -213,8 +213,8 @@ func ReadEncodedDocumentID(encoded doc.Encoded) ([]byte, error) { return id, err } -// GetFromDocument retrieves a doc.Metadata from a doc.Document. -func GetFromDocument(document doc.Document, reader *EncodedDocumentReader) (doc.Metadata, error) { +// MetadataFromDocument retrieves a doc.Metadata from a doc.Document. +func MetadataFromDocument(document doc.Document, reader *EncodedDocumentReader) (doc.Metadata, error) { if d, ok := document.Metadata(); ok { return d, nil } diff --git a/src/m3ninx/search/executor/executor.go b/src/m3ninx/search/executor/executor.go index 79eca270fe..e5f606a82f 100644 --- a/src/m3ninx/search/executor/executor.go +++ b/src/m3ninx/search/executor/executor.go @@ -33,7 +33,7 @@ var ( errExecutorClosed = errors.New("executor is closed") ) -type newIteratorFn func(s search.Searcher, rs index.Readers) (doc.MetadataIterator, error) +type newIteratorFn func(s search.Searcher, rs index.Readers) (doc.Iterator, error) type executor struct { sync.RWMutex @@ -52,7 +52,7 @@ func NewExecutor(rs index.Readers) search.Executor { } } -func (e *executor) Execute(q search.Query) (doc.MetadataIterator, error) { +func (e *executor) Execute(q search.Query) (doc.Iterator, error) { e.RLock() defer e.RUnlock() if e.closed { diff --git a/src/m3ninx/search/executor/executor_test.go b/src/m3ninx/search/executor/executor_test.go index ce8ef85a04..25e611efc0 100644 --- a/src/m3ninx/search/executor/executor_test.go +++ b/src/m3ninx/search/executor/executor_test.go @@ -36,7 +36,7 @@ type testIterator struct{} func newTestIterator() testIterator { return testIterator{} } func (it testIterator) Next() bool { return false } -func (it testIterator) Current() doc.Metadata { return doc.Metadata{} } +func (it testIterator) Current() doc.Document { return doc.Document{} } func (it testIterator) Err() error { return nil } func (it testIterator) Close() error { return nil } @@ -58,7 +58,7 @@ func TestExecutor(t *testing.T) { e := NewExecutor(rs).(*executor) // Override newIteratorFn to return test iterator. - e.newIteratorFn = func(_ search.Searcher, _ index.Readers) (doc.MetadataIterator, error) { + e.newIteratorFn = func(_ search.Searcher, _ index.Readers) (doc.Iterator, error) { return newTestIterator(), nil } diff --git a/src/m3ninx/search/executor/iterator.go b/src/m3ninx/search/executor/iterator.go index c1cc471ef9..3ca5e82781 100644 --- a/src/m3ninx/search/executor/iterator.go +++ b/src/m3ninx/search/executor/iterator.go @@ -31,14 +31,14 @@ type iterator struct { readers index.Readers idx int - currDoc doc.Metadata - currIter doc.MetadataIterator + currDoc doc.Document + currIter doc.Iterator err error closed bool } -func newIterator(s search.Searcher, rs index.Readers) (doc.MetadataIterator, error) { +func newIterator(s search.Searcher, rs index.Readers) (doc.Iterator, error) { it := &iterator{ searcher: s, readers: rs, @@ -91,7 +91,7 @@ func (it *iterator) Next() bool { return true } -func (it *iterator) Current() doc.Metadata { +func (it *iterator) Current() doc.Document { return it.currDoc } @@ -108,9 +108,9 @@ func (it *iterator) Close() error { } // nextIter gets the next document iterator by getting the next postings list from -// the it's searcher and then getting the documents for that postings list from the -// corresponding reader associated with that postings list. -func (it *iterator) nextIter() (doc.MetadataIterator, bool, error) { +// the it's searcher and then getting the encoded documents for that postings list from +// the corresponding reader associated with that postings list. +func (it *iterator) nextIter() (doc.Iterator, bool, error) { it.idx++ if it.idx >= len(it.readers) { return nil, false, nil @@ -122,7 +122,7 @@ func (it *iterator) nextIter() (doc.MetadataIterator, bool, error) { return nil, false, err } - iter, err := reader.MetadataIterator(pl) + iter, err := reader.Docs(pl) if err != nil { return nil, false, err } diff --git a/src/m3ninx/search/executor/iterator_test.go b/src/m3ninx/search/executor/iterator_test.go index 28cf95f021..6ebcfe9bcb 100644 --- a/src/m3ninx/search/executor/iterator_test.go +++ b/src/m3ninx/search/executor/iterator_test.go @@ -44,35 +44,14 @@ func TestIterator(t *testing.T) { require.NoError(t, secondPL.Insert(67)) // Set up Readers. - docs := []doc.Metadata{ - { - Fields: []doc.Field{ - { - Name: []byte("apple"), - Value: []byte("red"), - }, - }, - }, - { - Fields: []doc.Field{ - { - Name: []byte("banana"), - Value: []byte("yellow"), - }, - }, - }, - { - Fields: []doc.Field{ - { - Name: []byte("carrot"), - Value: []byte("orange"), - }, - }, - }, + docs := []doc.Document{ + doc.NewDocumentFromEncoded(doc.Encoded{Bytes: []byte("encodedbytes1")}), + doc.NewDocumentFromEncoded(doc.Encoded{Bytes: []byte("encodedbytes2")}), + doc.NewDocumentFromEncoded(doc.Encoded{Bytes: []byte("encodedbytes3")}), } - firstDocIter := doc.NewMockMetadataIterator(mockCtrl) - secondDocIter := doc.NewMockMetadataIterator(mockCtrl) + firstDocIter := doc.NewMockIterator(mockCtrl) + secondDocIter := doc.NewMockIterator(mockCtrl) gomock.InOrder( firstDocIter.EXPECT().Next().Return(true), firstDocIter.EXPECT().Current().Return(docs[0]), @@ -92,8 +71,8 @@ func TestIterator(t *testing.T) { firstReader := index.NewMockReader(mockCtrl) secondReader := index.NewMockReader(mockCtrl) gomock.InOrder( - firstReader.EXPECT().MetadataIterator(firstPL).Return(firstDocIter, nil), - secondReader.EXPECT().MetadataIterator(secondPL).Return(secondDocIter, nil), + firstReader.EXPECT().Docs(firstPL).Return(firstDocIter, nil), + secondReader.EXPECT().Docs(secondPL).Return(secondDocIter, nil), ) searcher := search.NewMockSearcher(mockCtrl) diff --git a/src/m3ninx/search/proptest/concurrent_test.go b/src/m3ninx/search/proptest/concurrent_test.go index 9d728ab3a1..02009512c8 100644 --- a/src/m3ninx/search/proptest/concurrent_test.go +++ b/src/m3ninx/search/proptest/concurrent_test.go @@ -61,7 +61,7 @@ func TestConcurrentQueries(t *testing.T) { require.NoError(t, err) matchedDocs, err := collectDocs(dOrg) require.NoError(t, err) - docMatcher, err := newDocumentIteratorMatcher(matchedDocs...) + docMatcher, err := newDocumentIteratorMatcher(t, matchedDocs...) require.NoError(t, err) var ( diff --git a/src/m3ninx/search/proptest/issue865_test.go b/src/m3ninx/search/proptest/issue865_test.go index 3a257a50e3..ad72b7ca67 100644 --- a/src/m3ninx/search/proptest/issue865_test.go +++ b/src/m3ninx/search/proptest/issue865_test.go @@ -45,35 +45,35 @@ var ( doc1 = doc.Metadata{ ID: []byte("__name__=node_cpu_seconds_total,cpu=1,instance=m3db-node01:9100,job=node-exporter,mode=system,"), Fields: []doc.Field{ - doc.Field{[]byte("cpu"), []byte("1")}, - doc.Field{[]byte("__name__"), []byte("node_cpu_seconds_total")}, - doc.Field{[]byte("instance"), []byte("m3db-node01:9100")}, - doc.Field{[]byte("job"), []byte("node-exporter")}, - doc.Field{[]byte("mode"), []byte("system")}, + {[]byte("cpu"), []byte("1")}, + {[]byte("__name__"), []byte("node_cpu_seconds_total")}, + {[]byte("instance"), []byte("m3db-node01:9100")}, + {[]byte("job"), []byte("node-exporter")}, + {[]byte("mode"), []byte("system")}, }, } doc2 = doc.Metadata{ ID: []byte("__name__=node_memory_SwapTotal_bytes,instance=m3db-node01:9100,job=node-exporter,"), Fields: []doc.Field{ - doc.Field{[]byte("__name__"), []byte("node_memory_SwapTotal_bytes")}, - doc.Field{[]byte("instance"), []byte("m3db-node01:9100")}, - doc.Field{[]byte("job"), []byte("node-exporter")}, + {[]byte("__name__"), []byte("node_memory_SwapTotal_bytes")}, + {[]byte("instance"), []byte("m3db-node01:9100")}, + {[]byte("job"), []byte("node-exporter")}, }, } doc3 = doc.Metadata{ ID: []byte("__name__=node_memory_SwapTotal_bytes,instance=alertmanager03:9100,job=node-exporter,"), Fields: []doc.Field{ - doc.Field{[]byte("__name__"), []byte("node_memory_SwapTotal_bytes")}, - doc.Field{[]byte("instance"), []byte("alertmanager03:9100")}, - doc.Field{[]byte("job"), []byte("node-exporter")}, + {[]byte("__name__"), []byte("node_memory_SwapTotal_bytes")}, + {[]byte("instance"), []byte("alertmanager03:9100")}, + {[]byte("job"), []byte("node-exporter")}, }, } doc4 = doc.Metadata{ ID: []byte("__name__=node_memory_SwapTotal_bytes,instance=prometheus01:9100,job=node-exporter,"), Fields: []doc.Field{ - doc.Field{[]byte("__name__"), []byte("node_memory_SwapTotal_bytes")}, - doc.Field{[]byte("instance"), []byte("prometheus01:9100")}, - doc.Field{[]byte("job"), []byte("node-exporter")}, + {[]byte("__name__"), []byte("node_memory_SwapTotal_bytes")}, + {[]byte("instance"), []byte("prometheus01:9100")}, + {[]byte("job"), []byte("node-exporter")}, }, } simpleTestDocs = []doc.Metadata{doc1, doc2, doc3, doc4} @@ -87,7 +87,7 @@ func TestAnyDistributionOfDocsDoesNotAffectQuery(t *testing.T) { parameters.Rng = rand.New(rand.NewSource(seed)) properties := gopter.NewProperties(parameters) - docMatcher, err := newDocumentIteratorMatcher(doc2) + docMatcher, err := newDocumentIteratorMatcher(t, doc.NewDocumentFromMetadata(doc2)) require.NoError(t, err) properties.Property("Any distribution of simple documents does not affect query results", prop.ForAll( func(i propTestInput) (bool, error) { diff --git a/src/m3ninx/search/proptest/prop_test.go b/src/m3ninx/search/proptest/prop_test.go index 0ecc89440e..a58a34d843 100644 --- a/src/m3ninx/search/proptest/prop_test.go +++ b/src/m3ninx/search/proptest/prop_test.go @@ -58,7 +58,7 @@ func TestSegmentDistributionDoesNotAffectQuery(t *testing.T) { } matchedDocs, err := collectDocs(dOrg) require.NoError(t, err) - docMatcher, err := newDocumentIteratorMatcher(matchedDocs...) + docMatcher, err := newDocumentIteratorMatcher(t, matchedDocs...) require.NoError(t, err) segments := i.generate(t, lotsTestDocuments) @@ -115,7 +115,7 @@ func TestFSTSimpleSegmentsQueryTheSame(t *testing.T) { } matchedDocs, err := collectDocs(dOrg) require.NoError(t, err) - docMatcher, err := newDocumentIteratorMatcher(matchedDocs...) + docMatcher, err := newDocumentIteratorMatcher(t, matchedDocs...) require.NoError(t, err) rFst, err := fstSeg.Reader() diff --git a/src/m3ninx/search/proptest/segment_gen.go b/src/m3ninx/search/proptest/segment_gen.go index aa3bf83c4e..2b62319c6f 100644 --- a/src/m3ninx/search/proptest/segment_gen.go +++ b/src/m3ninx/search/proptest/segment_gen.go @@ -40,8 +40,8 @@ var ( fstOptions = fst.NewOptions() ) -func collectDocs(iter doc.MetadataIterator) ([]doc.Metadata, error) { - var docs []doc.Metadata +func collectDocs(iter doc.Iterator) ([]doc.Document, error) { + var docs []doc.Document for iter.Next() { docs = append(docs, iter.Current()) } diff --git a/src/m3ninx/search/proptest/util.go b/src/m3ninx/search/proptest/util.go index 60dd08e341..58a25fa873 100644 --- a/src/m3ninx/search/proptest/util.go +++ b/src/m3ninx/search/proptest/util.go @@ -22,40 +22,52 @@ package proptest import ( "fmt" + "testing" + + "github.com/stretchr/testify/require" "github.com/m3db/m3/src/m3ninx/doc" + idxdocs "github.com/m3db/m3/src/m3ninx/index/segment/fst/encoding/docs" ) type documentIteratorMatcher struct { - expectedDocs map[string]doc.Metadata + expectedDocs map[string]doc.Document + t *testing.T } -func newDocumentIteratorMatcher(docs ...doc.Metadata) (*documentIteratorMatcher, error) { - docMap := make(map[string]doc.Metadata, len(docs)) +func newDocumentIteratorMatcher(t *testing.T, docs ...doc.Document) (*documentIteratorMatcher, error) { + docMap := make(map[string]doc.Document, len(docs)) for _, d := range docs { - id := string(d.ID) + rawID, err := idxdocs.ReadIDFromDocument(d) + id := string(rawID) + require.NoError(t, err) if _, ok := docMap[id]; ok { return nil, fmt.Errorf("received document with duplicate id: %v", d) } docMap[id] = d } - return &documentIteratorMatcher{docMap}, nil + return &documentIteratorMatcher{ + expectedDocs: docMap, + t: t, + }, nil } // Matches returns whether the provided iterator matches the collection of provided docs. -func (m *documentIteratorMatcher) Matches(i doc.MetadataIterator) error { - pendingDocIDs := make(map[string]doc.Metadata, len(m.expectedDocs)) +func (m *documentIteratorMatcher) Matches(i doc.Iterator) error { + pendingDocIDs := make(map[string]doc.Document, len(m.expectedDocs)) for id := range m.expectedDocs { pendingDocIDs[id] = m.expectedDocs[id] } for i.Next() { d := i.Current() - id := string(d.ID) + rawID, err := idxdocs.ReadIDFromDocument(d) + require.NoError(m.t, err) + id := string(rawID) expectedDoc, ok := m.expectedDocs[id] if !ok { return fmt.Errorf("received un-expected document: %+v", d) } - if !expectedDoc.Equal(d) { + if !m.compareDocs(expectedDoc, d) { return fmt.Errorf("received document: %+v did not match expected doc %+v", d, expectedDoc) } delete(pendingDocIDs, id) @@ -71,3 +83,14 @@ func (m *documentIteratorMatcher) Matches(i doc.MetadataIterator) error { } return nil } + +func (m *documentIteratorMatcher) compareDocs(d1 doc.Document, d2 doc.Document) bool { + docReader := idxdocs.NewEncodedDocumentReader() + d1Metadata, err := idxdocs.MetadataFromDocument(d1, docReader) + require.NoError(m.t, err) + + d2Metadata, err := idxdocs.MetadataFromDocument(d2, docReader) + require.NoError(m.t, err) + + return d1Metadata.Equal(d2Metadata) +} diff --git a/src/m3ninx/search/search_mock.go b/src/m3ninx/search/search_mock.go index b070619f5f..3c327a7bba 100644 --- a/src/m3ninx/search/search_mock.go +++ b/src/m3ninx/search/search_mock.go @@ -1,7 +1,7 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/m3db/m3/src/m3ninx/search/types.go -// Copyright (c) 2018 Uber Technologies, Inc. +// Copyright (c) 2021 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -59,10 +59,10 @@ func (m *MockExecutor) EXPECT() *MockExecutorMockRecorder { } // Execute mocks base method -func (m *MockExecutor) Execute(q Query) (doc.MetadataIterator, error) { +func (m *MockExecutor) Execute(q Query) (doc.Iterator, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Execute", q) - ret0, _ := ret[0].(doc.MetadataIterator) + ret0, _ := ret[0].(doc.Iterator) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/src/m3ninx/search/types.go b/src/m3ninx/search/types.go index cb21903fda..d4fc76b4ab 100644 --- a/src/m3ninx/search/types.go +++ b/src/m3ninx/search/types.go @@ -32,7 +32,7 @@ import ( // Executor is responsible for executing queries over a snapshot. type Executor interface { // Execute executes a query over the Executor's snapshot. - Execute(q Query) (doc.MetadataIterator, error) + Execute(q Query) (doc.Iterator, error) // Close closes the iterator. Close() error From c217be9fedcef704f7ce68921f0d6d43c1379fcc Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Tue, 12 Jan 2021 22:35:07 -0500 Subject: [PATCH 5/6] [coordinator] Disable downsampler matcher cache by default (#3080) --- src/cmd/services/m3coordinator/downsample/options.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/cmd/services/m3coordinator/downsample/options.go b/src/cmd/services/m3coordinator/downsample/options.go index 03bf33026a..711b061d4d 100644 --- a/src/cmd/services/m3coordinator/downsample/options.go +++ b/src/cmd/services/m3coordinator/downsample/options.go @@ -81,7 +81,13 @@ const ( defaultOpenTimeout = 10 * time.Second defaultBufferFutureTimedMetric = time.Minute defaultVerboseErrors = true - defaultMatcherCacheCapacity = 100000 + // defaultMatcherCacheCapacity sets the default matcher cache + // capacity to zero so that the cache is turned off. + // This is due to discovering that there is a lot of contention + // used by the cache and the fact that most coordinators are used + // in a stateless manner with a central deployment which in turn + // leads to an extremely low cache hit ratio anyway. + defaultMatcherCacheCapacity = 0 ) var ( @@ -527,7 +533,7 @@ func (r RollupRuleConfiguration) Rule() (view.RollupRule, error) { targetPipeline := pipeline.NewPipeline(ops) targets := []view.RollupTarget{ - view.RollupTarget{ + { Pipeline: targetPipeline, StoragePolicies: storagePolicies, }, From 482c510a4ae5991f4e7b11959862504cfa3bd7ca Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Wed, 13 Jan 2021 18:38:46 +1100 Subject: [PATCH 6/6] [dtest] Run tests on an existing cluster launched with docker compose (#3067) --- scripts/dtest/docker-compose.yml | 24 ++++++++++++++++++ .../dtest}/m3coordinator.yml | 0 .../config => scripts/dtest}/m3dbnode.yml | 0 scripts/dtest/run.sh | 16 ++++++++++++ .../dtest/docker/harness/harness_test.go | 6 ++--- .../dtest/docker/harness/resources/common.go | 11 -------- .../resources/config/m3coordinator.Dockerfile | 12 --------- .../resources/config/m3dbnode.Dockerfile | 12 --------- .../docker/harness/resources/coordinator.go | 6 ++--- .../dtest/docker/harness/resources/dbnode.go | 2 -- .../harness/resources/docker_resource.go | 16 ++++++------ .../dtest/docker/harness/resources/harness.go | 25 +++++++++++-------- .../dtest/docker/harness/resources/options.go | 14 +++++++++++ 13 files changed, 80 insertions(+), 64 deletions(-) create mode 100755 scripts/dtest/docker-compose.yml rename {src/cmd/tools/dtest/docker/harness/resources/config => scripts/dtest}/m3coordinator.yml (100%) rename {src/cmd/tools/dtest/docker/harness/resources/config => scripts/dtest}/m3dbnode.yml (100%) create mode 100755 scripts/dtest/run.sh delete mode 100644 src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.Dockerfile delete mode 100644 src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.Dockerfile diff --git a/scripts/dtest/docker-compose.yml b/scripts/dtest/docker-compose.yml new file mode 100755 index 0000000000..ba8502c42e --- /dev/null +++ b/scripts/dtest/docker-compose.yml @@ -0,0 +1,24 @@ +version: "3.5" +services: + dbnode01: + networks: + - dtest + image: m3dbnode:dev + container_name: dbnode01 + ports: + - "0.0.0.0:2379:2379" + - "0.0.0.0:9000:9000" + volumes: + - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + coord01: + networks: + - dtest + image: m3coordinator:dev + container_name: coord01 + ports: + - "0.0.0.0:7201:7201" + - "0.0.0.0:7204:7204" + volumes: + - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" +networks: + dtest: diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml b/scripts/dtest/m3coordinator.yml similarity index 100% rename from src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml rename to scripts/dtest/m3coordinator.yml diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml b/scripts/dtest/m3dbnode.yml similarity index 100% rename from src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml rename to scripts/dtest/m3dbnode.yml diff --git a/scripts/dtest/run.sh b/scripts/dtest/run.sh new file mode 100755 index 0000000000..d1be90f6f9 --- /dev/null +++ b/scripts/dtest/run.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -ex +set -o pipefail + +COMPOSE_FILE=./scripts/dtest/docker-compose.yml + +function defer { + docker-compose -f "${COMPOSE_FILE}" down +} + +trap defer EXIT + +docker-compose -f "${COMPOSE_FILE}" up --detach + +go test -v -tags=dtest ./src/cmd/tools/dtest/docker/harness diff --git a/src/cmd/tools/dtest/docker/harness/harness_test.go b/src/cmd/tools/dtest/docker/harness/harness_test.go index 138a997959..109bcfa22d 100644 --- a/src/cmd/tools/dtest/docker/harness/harness_test.go +++ b/src/cmd/tools/dtest/docker/harness/harness_test.go @@ -34,7 +34,9 @@ var singleDBNodeDockerResources resources.DockerResources func TestMain(m *testing.M) { var err error - singleDBNodeDockerResources, err = resources.SetupSingleM3DBNode() + singleDBNodeDockerResources, err = resources.SetupSingleM3DBNode( + resources.WithExistingCluster("dbnode01", "coord01"), + ) if err != nil { fmt.Println("could not set up db docker containers", err) @@ -42,12 +44,10 @@ func TestMain(m *testing.M) { } if l := len(singleDBNodeDockerResources.Nodes()); l != 1 { - singleDBNodeDockerResources.Cleanup() //nolint:errcheck fmt.Println("should only have a single node, have", l) os.Exit(1) } code := m.Run() - singleDBNodeDockerResources.Cleanup() //nolint:errcheck os.Exit(code) } diff --git a/src/cmd/tools/dtest/docker/harness/resources/common.go b/src/cmd/tools/dtest/docker/harness/resources/common.go index e6b2e4a286..99f00b780d 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/common.go +++ b/src/cmd/tools/dtest/docker/harness/resources/common.go @@ -26,7 +26,6 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "github.com/m3db/m3/src/x/instrument" @@ -52,7 +51,6 @@ type dockerResourceOptions struct { source string containerName string image dockerImage - dockerFile string portList []int mounts []string iOpts instrument.Options @@ -77,10 +75,6 @@ func (o dockerResourceOptions) withDefaults( o.image = defaultOpts.image } - if len(o.dockerFile) == 0 { - o.dockerFile = defaultOpts.dockerFile - } - if len(o.portList) == 0 { o.portList = defaultOpts.portList } @@ -176,11 +170,6 @@ func exposePorts( return opts } -func getDockerfile(file string) string { - src, _ := os.Getwd() - return fmt.Sprintf("%s/%s", src, file) -} - func toResponse( resp *http.Response, response proto.Message, diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.Dockerfile b/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.Dockerfile deleted file mode 100644 index 994ed23c8b..0000000000 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM alpine:latest AS builder -LABEL maintainer="The M3DB Authors " - -RUN mkdir -p /bin -RUN mkdir -p /etc/m3coordinator -ADD ./m3coordinator /bin/ -ADD ./m3coordinator.yml /etc/m3coordinator.yml - -EXPOSE 7201/tcp 7203/tcp 7204/tcp - -ENTRYPOINT [ "/bin/m3coordinator" ] -CMD [ "-f", "/etc/m3coordinator.yml" ] diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.Dockerfile b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.Dockerfile deleted file mode 100644 index b5bf62d1a4..0000000000 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM alpine:latest AS builder -LABEL maintainer="The M3DB Authors " - -RUN mkdir -p /bin -RUN mkdir -p /etc/m3dbnode -ADD ./m3dbnode /bin/ -ADD ./m3dbnode.yml /etc/m3dbnode/m3dbnode.yml - -EXPOSE 2379/tcp 2380/tcp 7201/tcp 7203/tcp 9000-9004/tcp - -ENTRYPOINT [ "/bin/m3dbnode" ] -CMD [ "-f", "/etc/m3dbnode/m3dbnode.yml" ] diff --git a/src/cmd/tools/dtest/docker/harness/resources/coordinator.go b/src/cmd/tools/dtest/docker/harness/resources/coordinator.go index 2c7f2cba1c..768ceed7fd 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/coordinator.go +++ b/src/cmd/tools/dtest/docker/harness/resources/coordinator.go @@ -42,9 +42,8 @@ import ( ) const ( - defaultCoordinatorSource = "coordinator" - defaultCoordinatorName = "coord01" - defaultCoordinatorDockerfile = "resources/config/m3coordinator.Dockerfile" + defaultCoordinatorSource = "coordinator" + defaultCoordinatorName = "coord01" ) var ( @@ -53,7 +52,6 @@ var ( defaultCoordinatorOptions = dockerResourceOptions{ source: defaultCoordinatorSource, containerName: defaultCoordinatorName, - dockerFile: defaultCoordinatorDockerfile, portList: defaultCoordinatorList, } ) diff --git a/src/cmd/tools/dtest/docker/harness/resources/dbnode.go b/src/cmd/tools/dtest/docker/harness/resources/dbnode.go index ef6adf2774..d7eca22f4b 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/dbnode.go +++ b/src/cmd/tools/dtest/docker/harness/resources/dbnode.go @@ -36,7 +36,6 @@ import ( const ( defaultDBNodeSource = "dbnode" defaultDBNodeContainerName = "dbnode01" - defaultDBNodeDockerfile = "resources/config/m3dbnode.Dockerfile" ) var ( @@ -45,7 +44,6 @@ var ( defaultDBNodeOptions = dockerResourceOptions{ source: defaultDBNodeSource, containerName: defaultDBNodeContainerName, - dockerFile: getDockerfile(defaultDBNodeDockerfile), portList: defaultDBNodePortList, } ) diff --git a/src/cmd/tools/dtest/docker/harness/resources/docker_resource.go b/src/cmd/tools/dtest/docker/harness/resources/docker_resource.go index a8b9f8ae79..7cfa5357d3 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/docker_resource.go +++ b/src/cmd/tools/dtest/docker/harness/resources/docker_resource.go @@ -50,7 +50,6 @@ func newDockerResource( source = resourceOpts.source image = resourceOpts.image containerName = resourceOpts.containerName - dockerFile = resourceOpts.dockerFile iOpts = resourceOpts.iOpts portList = resourceOpts.portList @@ -60,11 +59,6 @@ func newDockerResource( ) ) - if err := pool.RemoveContainerByName(containerName); err != nil { - logger.Error("could not remove container from pool", zap.Error(err)) - return nil, err - } - opts := exposePorts(newOptions(containerName), portList) hostConfigOpts := func(c *dc.HostConfig) { @@ -83,9 +77,13 @@ func newDockerResource( var resource *dockertest.Resource var err error if image.name == "" { - logger.Info("building and running container with options", - zap.String("dockerFile", dockerFile), zap.Any("options", opts)) - resource, err = pool.BuildAndRunWithOptions(dockerFile, opts, hostConfigOpts) + logger.Info("connecting to existing container", zap.String("container", containerName)) + var ok bool + resource, ok = pool.ContainerByName(containerName) + if !ok { + logger.Error("could not find container", zap.Error(err)) + return nil, fmt.Errorf("could not find container %v", containerName) + } } else { opts = useImage(opts, image) imageWithTag := fmt.Sprintf("%v:%v", image.name, image.tag) diff --git a/src/cmd/tools/dtest/docker/harness/resources/harness.go b/src/cmd/tools/dtest/docker/harness/resources/harness.go index 6ad245027d..74ec12257d 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/harness.go +++ b/src/cmd/tools/dtest/docker/harness/resources/harness.go @@ -77,20 +77,22 @@ func SetupSingleM3DBNode(opts ...SetupOptions) (DockerResources, error) { // nol } pool.MaxWait = timeout - err = setupNetwork(pool) - if err != nil { - return nil, err - } - err = setupVolume(pool) - if err != nil { - return nil, err + if !options.existingCluster { + if err := setupNetwork(pool); err != nil { + return nil, err + } + + if err := setupVolume(pool); err != nil { + return nil, err + } } iOpts := instrument.NewOptions() dbNode, err := newDockerHTTPNode(pool, dockerResourceOptions{ - image: options.dbNodeImage, - iOpts: iOpts, + image: options.dbNodeImage, + containerName: options.dbNodeContainerName, + iOpts: iOpts, }) success := false @@ -112,8 +114,9 @@ func SetupSingleM3DBNode(opts ...SetupOptions) (DockerResources, error) { // nol } coordinator, err := newDockerHTTPCoordinator(pool, dockerResourceOptions{ - image: options.coordinatorImage, - iOpts: iOpts, + image: options.coordinatorImage, + containerName: options.coordinatorContainerName, + iOpts: iOpts, }) defer func() { diff --git a/src/cmd/tools/dtest/docker/harness/resources/options.go b/src/cmd/tools/dtest/docker/harness/resources/options.go index 7f1f970333..886aebb746 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/options.go +++ b/src/cmd/tools/dtest/docker/harness/resources/options.go @@ -28,6 +28,10 @@ type dockerImage struct { type setupOptions struct { dbNodeImage dockerImage coordinatorImage dockerImage + + existingCluster bool + dbNodeContainerName string + coordinatorContainerName string } // SetupOptions is a setup option. @@ -46,3 +50,13 @@ func WithCoordinatorImage(name, tag string) SetupOptions { o.coordinatorImage = dockerImage{name: name, tag: tag} } } + +// WithExistingCluster sets the names of already running containers dbnode and coordinator +// containers that should be used for tests. +func WithExistingCluster(dbNodeContainerName, coordinatorContainerName string) SetupOptions { + return func(o *setupOptions) { + o.existingCluster = true + o.dbNodeContainerName = dbNodeContainerName + o.coordinatorContainerName = coordinatorContainerName + } +}