From 28911bb24a926f3357a11cd42893a3d5c244e26a Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 6 Mar 2023 15:32:25 +0530 Subject: [PATCH] Sync small miscellaneous changes from sparsehistogram branch Signed-off-by: Ganesh Vernekar --- pkg/storegateway/bucket_test.go | 4 +- pkg/storegateway/testhelper/testhelper.go | 5 +- pkg/util/merger.go | 43 ------------- pkg/util/merger_test.go | 74 ----------------------- tools/tsdb-chunks/main_test.go | 7 +-- tools/tsdb-index-health/main_test.go | 14 ++--- tools/tsdb-print-chunk/main_test.go | 7 +-- 7 files changed, 18 insertions(+), 136 deletions(-) diff --git a/pkg/storegateway/bucket_test.go b/pkg/storegateway/bucket_test.go index 09d0effdf62..42615d9aeaa 100644 --- a/pkg/storegateway/bucket_test.go +++ b/pkg/storegateway/bucket_test.go @@ -1863,7 +1863,7 @@ func TestBucketStore_Series_BlockWithMultipleChunks(t *testing.T) { } func TestBucketStore_Series_BlockWithMultipleHistogramChunks(t *testing.T) { - histograms := tsdb.GenerateTestHistograms(10000) + histograms := test.GenerateTestHistograms(10000) appendF := func(app storage.Appender, lset labels.Labels, ts int64) error { _, err := app.AppendHistogram(0, lset, ts, histograms[ts], nil) return err @@ -1872,7 +1872,7 @@ func TestBucketStore_Series_BlockWithMultipleHistogramChunks(t *testing.T) { } func TestBucketStore_Series_BlockWithMultipleFloatHistogramChunks(t *testing.T) { - histograms := tsdb.GenerateTestFloatHistograms(10000) + histograms := test.GenerateTestFloatHistograms(10000) appendF := func(app storage.Appender, lset labels.Labels, ts int64) error { _, err := app.AppendHistogram(0, lset, ts, nil, histograms[ts]) return err diff --git a/pkg/storegateway/testhelper/testhelper.go b/pkg/storegateway/testhelper/testhelper.go index 1b0cedc1c9c..b82109c21ba 100644 --- a/pkg/storegateway/testhelper/testhelper.go +++ b/pkg/storegateway/testhelper/testhelper.go @@ -21,6 +21,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/grafana/mimir/pkg/storage/tsdb/metadata" + "github.com/grafana/mimir/pkg/util/test" ) // CreateBlock writes a block with the given series and numSamples samples each. @@ -60,8 +61,8 @@ func CreateBlock( // For calculating the next series value type (float, histogram, etc) batchOffset := 0 // prep some histogram test data - testHistograms := tsdb.GenerateTestHistograms(numSamples) - testFloatHistograms := tsdb.GenerateTestFloatHistograms(numSamples) + testHistograms := test.GenerateTestHistograms(numSamples) + testFloatHistograms := test.GenerateTestFloatHistograms(numSamples) for len(series) > 0 { l := batchSize diff --git a/pkg/util/merger.go b/pkg/util/merger.go index d816132890d..2e85f2452c8 100644 --- a/pkg/util/merger.go +++ b/pkg/util/merger.go @@ -7,49 +7,6 @@ package util -import ( - "github.com/prometheus/common/model" -) - -// MergeSampleSets merges and dedupes two sets of already sorted sample pairs. -func MergeSampleSets(a, b []model.SamplePair) []model.SamplePair { - result := make([]model.SamplePair, 0, len(a)+len(b)) - i, j := 0, 0 - for i < len(a) && j < len(b) { - if a[i].Timestamp < b[j].Timestamp { - result = append(result, a[i]) - i++ - } else if a[i].Timestamp > b[j].Timestamp { - result = append(result, b[j]) - j++ - } else { - result = append(result, a[i]) - i++ - j++ - } - } - // Add the rest of a or b. One of them is empty now. - result = append(result, a[i:]...) - result = append(result, b[j:]...) - return result -} - -// MergeNSampleSets merges and dedupes n sets of already sorted sample pairs. -func MergeNSampleSets(sampleSets ...[]model.SamplePair) []model.SamplePair { - l := len(sampleSets) - switch l { - case 0: - return []model.SamplePair{} - case 1: - return sampleSets[0] - } - - n := l / 2 - left := MergeNSampleSets(sampleSets[:n]...) - right := MergeNSampleSets(sampleSets[n:]...) - return MergeSampleSets(left, right) -} - // MergeSlices merges a set of sorted string slices into a single ones // while removing all duplicates. func MergeSlices(a ...[]string) []string { diff --git a/pkg/util/merger_test.go b/pkg/util/merger_test.go index f3caddefb35..e12c363ebe9 100644 --- a/pkg/util/merger_test.go +++ b/pkg/util/merger_test.go @@ -7,84 +7,10 @@ package util import ( "testing" - "time" - "github.com/prometheus/common/model" "github.com/stretchr/testify/require" ) -func TestMergeSampleSets(t *testing.T) { - now := model.Now() - sample1 := model.SamplePair{Timestamp: now, Value: 1} - sample2 := model.SamplePair{Timestamp: now.Add(1 * time.Second), Value: 2} - sample3 := model.SamplePair{Timestamp: now.Add(4 * time.Second), Value: 3} - sample4 := model.SamplePair{Timestamp: now.Add(8 * time.Second), Value: 7} - - for _, c := range []struct { - samplesA []model.SamplePair - samplesB []model.SamplePair - expected []model.SamplePair - }{ - { - samplesA: []model.SamplePair{}, - samplesB: []model.SamplePair{}, - expected: []model.SamplePair{}, - }, - { - samplesA: []model.SamplePair{sample1}, - samplesB: []model.SamplePair{}, - expected: []model.SamplePair{sample1}, - }, - { - samplesA: []model.SamplePair{}, - samplesB: []model.SamplePair{sample1}, - expected: []model.SamplePair{sample1}, - }, - { - samplesA: []model.SamplePair{sample1}, - samplesB: []model.SamplePair{sample1}, - expected: []model.SamplePair{sample1}, - }, - { - samplesA: []model.SamplePair{sample1, sample2, sample3}, - samplesB: []model.SamplePair{sample1, sample3, sample4}, - expected: []model.SamplePair{sample1, sample2, sample3, sample4}, - }, - } { - samples := MergeSampleSets(c.samplesA, c.samplesB) - require.Equal(t, c.expected, samples) - } -} - -func TestMergeNSampleSets(t *testing.T) { - now := model.Now() - sample1 := model.SamplePair{Timestamp: now, Value: 1} - sample2 := model.SamplePair{Timestamp: now.Add(1 * time.Second), Value: 2} - sample3 := model.SamplePair{Timestamp: now.Add(4 * time.Second), Value: 3} - sample4 := model.SamplePair{Timestamp: now.Add(8 * time.Second), Value: 7} - - for _, c := range []struct { - sampleSets [][]model.SamplePair - expected []model.SamplePair - }{ - { - sampleSets: [][]model.SamplePair{{}, {}, {}}, - expected: []model.SamplePair{}, - }, - { - sampleSets: [][]model.SamplePair{ - {sample1, sample2}, - {sample2}, - {sample1, sample3, sample4}, - }, - expected: []model.SamplePair{sample1, sample2, sample3, sample4}, - }, - } { - samples := MergeNSampleSets(c.sampleSets...) - require.Equal(t, c.expected, samples) - } -} - func TestMergeSlices(t *testing.T) { tests := []struct { name string diff --git a/tools/tsdb-chunks/main_test.go b/tools/tsdb-chunks/main_test.go index a2082486e47..606a5b6b054 100644 --- a/tools/tsdb-chunks/main_test.go +++ b/tools/tsdb-chunks/main_test.go @@ -9,7 +9,6 @@ import ( "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/tsdbutil" @@ -32,9 +31,9 @@ func TestTSDBChunks(t *testing.T) { sample{30, 13, nil, nil}, }), tsdbutil.ChunkFromSamples([]tsdbutil.Sample{ - sample{40, 0, tsdb.GenerateTestHistogram(1), nil}, - sample{50, 0, tsdb.GenerateTestHistogram(2), nil}, - sample{60, 0, tsdb.GenerateTestHistogram(3), nil}, + sample{40, 0, test.GenerateTestHistogram(1), nil}, + sample{50, 0, test.GenerateTestHistogram(2), nil}, + sample{60, 0, test.GenerateTestHistogram(3), nil}, }), }, } diff --git a/tools/tsdb-index-health/main_test.go b/tools/tsdb-index-health/main_test.go index 8f2760f3961..2310eba8967 100644 --- a/tools/tsdb-index-health/main_test.go +++ b/tools/tsdb-index-health/main_test.go @@ -9,13 +9,13 @@ import ( "github.com/go-kit/log" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/stretchr/testify/require" "github.com/grafana/mimir/pkg/storage/tsdb/testutil" + "github.com/grafana/mimir/pkg/util/test" ) func TestGatherIndexHealthStats(t *testing.T) { @@ -36,14 +36,14 @@ func TestGatherIndexHealthStats(t *testing.T) { Labels: labels.FromStrings(labels.MetricName, "zxcv", "foo", "bar"), Chunks: []chunks.Meta{ tsdbutil.ChunkFromSamples([]tsdbutil.Sample{ - sample{40, 0, tsdb.GenerateTestHistogram(1), nil}, - sample{50, 0, tsdb.GenerateTestHistogram(2), nil}, - sample{60, 0, tsdb.GenerateTestHistogram(3), nil}, + sample{40, 0, test.GenerateTestHistogram(1), nil}, + sample{50, 0, test.GenerateTestHistogram(2), nil}, + sample{60, 0, test.GenerateTestHistogram(3), nil}, }), tsdbutil.ChunkFromSamples([]tsdbutil.Sample{ - sample{70, 0, tsdb.GenerateTestHistogram(4), nil}, - sample{80, 0, tsdb.GenerateTestHistogram(5), nil}, - sample{90, 0, tsdb.GenerateTestHistogram(6), nil}, + sample{70, 0, test.GenerateTestHistogram(4), nil}, + sample{80, 0, test.GenerateTestHistogram(5), nil}, + sample{90, 0, test.GenerateTestHistogram(6), nil}, }), }, } diff --git a/tools/tsdb-print-chunk/main_test.go b/tools/tsdb-print-chunk/main_test.go index 86753a6aa39..f8522695b0c 100644 --- a/tools/tsdb-print-chunk/main_test.go +++ b/tools/tsdb-print-chunk/main_test.go @@ -9,7 +9,6 @@ import ( "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/tsdbutil" @@ -32,9 +31,9 @@ func TestTSDBPrintChunk(t *testing.T) { sample{30, 13, nil, nil}, }), tsdbutil.ChunkFromSamples([]tsdbutil.Sample{ - sample{40, 0, tsdb.GenerateTestHistogram(1), nil}, - sample{50, 0, tsdb.GenerateTestHistogram(2), nil}, - sample{60, 0, tsdb.GenerateTestHistogram(3), nil}, + sample{40, 0, test.GenerateTestHistogram(1), nil}, + sample{50, 0, test.GenerateTestHistogram(2), nil}, + sample{60, 0, test.GenerateTestHistogram(3), nil}, }), }, }