Skip to content
This repository has been archived by the owner on Jul 19, 2023. It is now read-only.

Commit

Permalink
Refine ingester/store-gateway time range split (#742)
Browse files Browse the repository at this point in the history
  • Loading branch information
@kolesnikovae
kolesnikovae authored Jun 1, 2023
1 parent 766d0b4 commit d9eb816
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 47 deletions.
58 changes: 11 additions & 47 deletions pkg/querier/querier.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/grafana/phlare/pkg/iter"
phlaremodel "github.com/grafana/phlare/pkg/model"
"github.com/grafana/phlare/pkg/util"
"github.com/grafana/phlare/pkg/util/math"
)

type Config struct {
Expand Down Expand Up @@ -391,55 +392,18 @@ func (sq storeQueries) Log(logger log.Logger) {
// todo(ctovena): Later we should try to deduplicate blocks between ingesters and store gateways (prefer) and simply query both
func splitQueryToStores(start, end model.Time, now model.Time, queryStoreAfter time.Duration) (queries storeQueries) {
queries.queryStoreAfter = queryStoreAfter
// If the start time is in the future, there is nothing to query.
if start > now {
queries.storeGateway.shouldQuery = false
queries.ingester.shouldQuery = false
return
}

// If we do not have a query store after duration, then the only store we
// need to query is the store gateway.
if queryStoreAfter == 0 {
queries.storeGateway.shouldQuery = true
queries.storeGateway.start = start
queries.storeGateway.end = end

queries.ingester.shouldQuery = false
return
}

cutOff := now.Add(-queryStoreAfter)
if start >= cutOff {
queries.storeGateway.shouldQuery = false

queries.ingester.shouldQuery = true
queries.ingester.start = start
queries.ingester.end = end
return
}
// If the cut off is in the middle of the query, then we need to query both
// the store gateway and the ingester.
if cutOff < end && cutOff > start {
queries.storeGateway.shouldQuery = true
queries.storeGateway.start = start
queries.storeGateway.end = cutOff

queries.ingester.shouldQuery = true
queries.ingester.start = cutOff + 1
queries.ingester.end = end

return
if start.Before(cutOff) {
queries.storeGateway = storeQuery{shouldQuery: true, start: start, end: math.Min(cutOff, end)}
}
if end.After(cutOff) {
queries.ingester = storeQuery{shouldQuery: true, start: math.Max(cutOff, start), end: end}
// Note that the ranges must not overlap.
if queries.storeGateway.shouldQuery {
queries.ingester.start++
}
}

// If the cut off is not in the query, then we only need to query the store
// gateway.
queries.storeGateway.shouldQuery = true
queries.storeGateway.start = start
queries.storeGateway.end = end

queries.ingester.shouldQuery = false
return
return queries
}

func (q *Querier) SelectMergeProfile(ctx context.Context, req *connect.Request[querierv1.SelectMergeProfileRequest]) (*connect.Response[googlev1.Profile], error) {
Expand Down
73 changes: 73 additions & 0 deletions pkg/querier/querier_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -874,8 +874,58 @@ func Test_splitQueryToStores(t *testing.T) {
shouldQuery: false,
},
ingester: storeQuery{
shouldQuery: true,
start: model.TimeFromUnixNano(int64(time.Hour)),
end: model.TimeFromUnixNano(int64(2 * time.Hour)),
},
},
},
{
// ----|-------|-----|----|----
// ^ ^ ^ ^
// cutoff start now end
//
name: "end is in the future and start is after the cutoff",
now: model.TimeFromUnixNano(int64(time.Hour)),
start: model.TimeFromUnixNano(int64(45 * time.Minute)),
end: model.TimeFromUnixNano(int64(2 * time.Hour)),
queryStoreAfter: 30 * time.Minute,

expected: storeQueries{
queryStoreAfter: 30 * time.Minute,
storeGateway: storeQuery{
shouldQuery: false,
},
ingester: storeQuery{
shouldQuery: true,
start: model.TimeFromUnixNano(int64(45 * time.Minute)),
end: model.TimeFromUnixNano(int64(2 * time.Hour)),
},
},
},
{
// ----|-------|-----|----|----
// ^ ^ ^ ^
// start cutoff now end
//
name: "end is in the future and start is before the cutoff",
now: model.TimeFromUnixNano(int64(time.Hour)),
start: model.TimeFromUnixNano(int64(15 * time.Minute)),
end: model.TimeFromUnixNano(int64(2 * time.Hour)),
queryStoreAfter: 30 * time.Minute,

expected: storeQueries{
queryStoreAfter: 30 * time.Minute,
storeGateway: storeQuery{
shouldQuery: true,
start: model.TimeFromUnixNano(int64(15 * time.Minute)),
end: model.TimeFromUnixNano(int64(30 * time.Minute)),
},
ingester: storeQuery{
shouldQuery: true,
start: model.TimeFromUnixNano(int64(30*time.Minute)) + 1,
end: model.TimeFromUnixNano(int64(2 * time.Hour)),
},
},
},
{
Expand Down Expand Up @@ -949,6 +999,29 @@ func Test_splitQueryToStores(t *testing.T) {
},
},
},
{
// ----|------|--------|----
// ^ ^ ^
// start end=cutoff now
//
name: "end is exactly at cutoff",
now: model.TimeFromUnixNano(int64(15 * time.Hour)),
start: model.TimeFromUnixNano(int64(60 * time.Minute)),
end: model.TimeFromUnixNano(int64(30 * time.Minute)),
queryStoreAfter: 30 * time.Minute,

expected: storeQueries{
queryStoreAfter: 30 * time.Minute,
storeGateway: storeQuery{
shouldQuery: true,
start: model.TimeFromUnixNano(int64(60 * time.Minute)),
end: model.TimeFromUnixNano(int64(30 * time.Minute)),
},
ingester: storeQuery{
shouldQuery: false,
},
},
},
{
// ----|------|-----|----|----
// ^ ^ ^ ^
Expand Down

0 comments on commit d9eb816

Please sign in to comment.