Problem: memiavl snapshot rewriting is not triggered #1034
Merged
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Solution: - add configs to trigger the snapshot rewriting.
yihuang
requested review from
JayT106,
calvinaco and
mmsqe
and removed request for
a team
yihuang
commented
May 23, 2023
Signed-off-by: yihuang <huang@crypto.com>
![github-advanced-security[bot]](https://avatars.githubusercontent.com/in/57789?s=60&v=4){kind=small}
| opts := rs.opts | ||
| opts.CreateIfMissing = true | ||
| opts.InitialStores = initialStores | ||
| opts.TargetVersion = uint32(version) |
Check failure
Code scanning / gosec
Potential integer overflow by integer type conversion
Codecov Report
@@ Coverage Diff @@
## main #1034 +/- ##
===========================================
+ Coverage 22.43% 46.55% +24.11%
===========================================
Files 50 82 +32
Lines 3013 7091 +4078
===========================================
+ Hits 676 3301 +2625
- Misses 2272 3443 +1171
- Partials 65 347 +282
|
mmsqe
reviewed
May 23, 2023
| @@ -372,18 +404,26 @@ | |||
| return nil | |||
| } | |||
|
|
|||
| // rewriteIfApplicable execute the snapshot rewrite strategy according to current height | |||
| func (db *DB) rewriteIfApplicable(height int64) { | |||
| if height%int64(db.snapshotInterval) != 0 { | |||
Check failure
Code scanning / gosec
Potential integer overflow by integer type conversion
mmsqe
reviewed
May 24, 2023
Comment on lines
+92
to
+100
| for key := range rs.stores { | ||
| store := rs.stores[key] | ||
| if store.GetStoreType() == types.StoreTypeIAVL { | ||
| rs.stores[key], err = rs.loadCommitStoreFromParams(rs.db, key, rs.storesParams[key]) | ||
| if err != nil { | ||
| panic(fmt.Errorf("inconsistent store map, store %s not found", key.Name())) | ||
| } | ||
| } | ||
| } |
Check failure
Code scanning / gosec
the value in the range statement should be _ unless copying a map: want: for key := range m
mmsqe
approved these changes
May 24, 2023
Comment on lines
+92
to
+100
| for key := range rs.stores { | ||
| store := rs.stores[key] | ||
| if store.GetStoreType() == types.StoreTypeIAVL { | ||
| rs.stores[key], err = rs.loadCommitStoreFromParams(rs.db, key, rs.storesParams[key]) | ||
| if err != nil { | ||
| panic(fmt.Errorf("inconsistent store map, store %s not found", key.Name())) | ||
| } | ||
| } | ||
| } |
Check warning
Code scanning / CodeQL
Iteration over map
| if initialVersion > 1 { | ||
| return int64(index) + int64(initialVersion) - 1 | ||
| } | ||
| return int64(index) |
Check failure
Code scanning / gosec
Potential integer overflow by integer type conversion
| // walVersion converts wal index to version, reverse of walIndex | ||
| func walVersion(index uint64, initialVersion uint32) int64 { | ||
| if initialVersion > 1 { | ||
| return int64(index) + int64(initialVersion) - 1 |
Check failure
Code scanning / gosec
Potential integer overflow by integer type conversion
| // walVersion converts wal index to version, reverse of walIndex | ||
| func walVersion(index uint64, initialVersion uint32) int64 { | ||
| if initialVersion > 1 { | ||
| return int64(index) + int64(initialVersion) - 1 |
Check failure
Code scanning / gosec
Potential integer overflow by integer type conversion
yihuang
force-pushed
the
memiavl-snapshot-strategy
branch
from
12c2522
to
46b7a07
Compare
mmsqe
reviewed
May 25, 2023
where did you see that error? |
I was trying ibc test |
|
|
||
| // truncate WAL until the earliest remaining snapshot | ||
| earliestVersion, err := firstSnapshotVersion(db.dir) | ||
| if err := db.wal.TruncateFront(uint64(earliestVersion + 1)); err != nil { |
Check failure
Code scanning / gosec
Potential integer overflow by integer type conversion
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if err := mtree.CatchupWAL(wal, int64(opts.TargetVersion)); err != nil { | ||
| return nil, err | ||
| if opts.TargetVersion == 0 || int64(opts.TargetVersion) > mtree.Version() { |
Check failure
Code scanning / gosec
Potential integer overflow by integer type conversion
| return nil, fmt.Errorf("failed to load current version: %w", err) | ||
| } | ||
|
|
||
| if int64(opts.TargetVersion) < version { |
Check failure
Code scanning / gosec
Potential integer overflow by integer type conversion
is it when quitting the node? it seems don't reproduce here. |
| if err := mtree.CatchupWAL(wal, int64(opts.TargetVersion)); err != nil { | ||
| return nil, err | ||
| if opts.TargetVersion == 0 || int64(opts.TargetVersion) > mtree.Version() { | ||
| if err := mtree.CatchupWAL(wal, int64(opts.TargetVersion)); err != nil { |
Check failure
Code scanning / gosec
Potential integer overflow by integer type conversion
Comment on lines
+271
to
+318
| go func() { | ||
| defer db.pruneSnapshotLock.Unlock() | ||
|
|
||
| currentName, err := os.Readlink(currentPath(db.dir)) | ||
| if err != nil { | ||
| db.logger.Error("failed to read current snapshot name", "err", err) | ||
| return | ||
| } | ||
|
|
||
| entries, err := os.ReadDir(db.dir) | ||
| if err != nil { | ||
| db.logger.Error("failed to read db dir", "err", err) | ||
| return | ||
| } | ||
|
|
||
| counter := db.snapshotKeepRecent | ||
| for i := len(entries) - 1; i >= 0; i-- { | ||
| name := entries[i].Name() | ||
| if !strings.HasPrefix(name, SnapshotPrefix) { | ||
| continue | ||
| } | ||
|
|
||
| if name >= currentName { | ||
| // ignore any newer snapshot directories, there could be ongoning snapshot rewrite. | ||
| continue | ||
| } | ||
|
|
||
| if counter > 0 { | ||
| counter-- | ||
| continue | ||
| } | ||
|
|
||
| db.logger.Info("prune snapshot", "name", name) | ||
| if err := os.RemoveAll(filepath.Join(db.dir, name)); err != nil { | ||
| db.logger.Error("failed to prune snapshot", "err", err) | ||
| } | ||
| } | ||
|
|
||
| // truncate WAL until the earliest remaining snapshot | ||
| earliestVersion, err := firstSnapshotVersion(db.dir) | ||
| if err != nil { | ||
| db.logger.Error("failed to find first snapshot", "err", err) | ||
| } | ||
|
|
||
| if err := db.wal.TruncateFront(uint64(earliestVersion + 1)); err != nil { | ||
| db.logger.Error("failed to truncate wal", "err", err, "version", earliestVersion+1) | ||
| } | ||
| }() |
Check notice
Code scanning / CodeQL
Spawning a Go routine
Strange that I only see once in e2e, maybe I can add some unit test later |
github-merge-queue
bot
removed this pull request from the merge queue due to no response for status checks
github-merge-queue
bot
removed this pull request from the merge queue due to failed status checks
This was referenced May 26, 2023
13 tasks
This was referenced Apr 26, 2024
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Solution:
👮🏻👮🏻👮🏻 !!!! REFERENCE THE PROBLEM YOUR ARE SOLVING IN THE PR TITLE AND DESCRIBE YOUR SOLUTION HERE !!!! DO NOT FORGET !!!! 👮🏻👮🏻👮🏻
PR Checklist:
make)make test)go fmt)golangci-lint run)go list -json -m all | nancy sleuth)Thank you for your code, it's appreciated! :)