Skip to content

Commit

Permalink
fix: make the tsdb filenames correctly reproducible from the identifi…
Browse files Browse the repository at this point in the history
…er (#12536)
  • Loading branch information
sandeepsukhani authored Apr 9, 2024
1 parent 82f6548 commit ec888ec
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 20 deletions.
28 changes: 20 additions & 8 deletions pkg/storage/stores/shipper/indexshipper/tsdb/identifier.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,14 @@ func (p prefixedIdentifier) Name() string {
// Identifier has all the information needed to resolve a TSDB index
// Notably this abstracts away OS path separators, etc.
type SingleTenantTSDBIdentifier struct {
TS time.Time
From, Through model.Time
Checksum uint32
// exportTSInSecs tells whether creation timestamp should be exported in unix seconds instead of nanoseconds.
// timestamp in filename could be a unix second or a unix nanosecond so
// helps us to be able to reproduce filename back from parsed identifier.
// Should be true ideally for older files with creation timestamp in seconds.
exportTSInSecs bool
TS time.Time
From, Through model.Time
Checksum uint32
}

// implement Hash
Expand All @@ -77,9 +82,15 @@ func (i SingleTenantTSDBIdentifier) Hash(h hash.Hash32) (err error) {

// str builds filename with format <file-creation-ts> + `-` + `compactor` + `-` + <oldest-chunk-start-ts> + `-` + <latest-chunk-end-ts> `-` + <index-checksum>
func (i SingleTenantTSDBIdentifier) str() string {
ts := int64(0)
if i.exportTSInSecs {
ts = i.TS.Unix()
} else {
ts = i.TS.UnixNano()
}
return fmt.Sprintf(
"%d-%s-%d-%d-%x.tsdb",
i.TS.UnixNano(),
ts,
compactedFileUploader,
i.From,
i.Through,
Expand Down Expand Up @@ -140,10 +151,11 @@ func ParseSingleTenantTSDBPath(p string) (id SingleTenantTSDBIdentifier, ok bool
parsedTS = time.Unix(0, ts)
}
return SingleTenantTSDBIdentifier{
TS: parsedTS,
From: model.Time(from),
Through: model.Time(through),
Checksum: uint32(checksum),
exportTSInSecs: len(elems[0]) <= 10,
TS: parsedTS,
From: model.Time(from),
Through: model.Time(through),
Checksum: uint32(checksum),
}, true

}
Expand Down
30 changes: 18 additions & 12 deletions pkg/storage/stores/shipper/indexshipper/tsdb/identifier_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,32 +20,35 @@ func TestParseSingleTenantTSDBPath(t *testing.T) {
desc: "simple_works",
input: "1-compactor-1-10-ff.tsdb",
id: SingleTenantTSDBIdentifier{
TS: time.Unix(1, 0),
From: 1,
Through: 10,
Checksum: 255,
exportTSInSecs: true,
TS: time.Unix(1, 0),
From: 1,
Through: 10,
Checksum: 255,
},
ok: true,
},
{
desc: "simple_works_with_nanosecond",
input: "1712534400000000000-compactor-1-10-ff.tsdb",
id: SingleTenantTSDBIdentifier{
TS: time.Unix(0, 1712534400000000000),
From: 1,
Through: 10,
Checksum: 255,
exportTSInSecs: false,
TS: time.Unix(0, 1712534400000000000),
From: 1,
Through: 10,
Checksum: 255,
},
ok: true,
},
{
desc: "uint32_max_checksum_works",
input: fmt.Sprintf("1-compactor-1-10-%x.tsdb", math.MaxUint32),
id: SingleTenantTSDBIdentifier{
TS: time.Unix(1, 0),
From: 1,
Through: 10,
Checksum: math.MaxUint32,
exportTSInSecs: true,
TS: time.Unix(1, 0),
From: 1,
Through: 10,
Checksum: math.MaxUint32,
},
ok: true,
},
Expand All @@ -69,6 +72,9 @@ func TestParseSingleTenantTSDBPath(t *testing.T) {
id, ok := ParseSingleTenantTSDBPath(tc.input)
require.Equal(t, tc.ok, ok)
require.Equal(t, tc.id, id)
if ok {
require.Equal(t, tc.input, id.Name())
}
})
}
}

0 comments on commit ec888ec

Please sign in to comment.