From 9c1ca44188c3361ae6fe8a82105493400f014af1 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Fri, 17 Feb 2023 14:56:04 +0800 Subject: [PATCH 1/4] core/rawdb: find smallest block stored in key-value store --- core/rawdb/database.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index ef80c251a457..5da2589d0aac 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -242,8 +242,17 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 { // Subsequent header after the freezer limit is missing from the database. // Reject startup if the database has a more recent head. - if ldbNum := *ReadHeaderNumber(db, ReadHeadHeaderHash(db)); ldbNum > frozen-1 { - return nil, fmt.Errorf("gap in the chain between ancients (#%d) and leveldb (#%d) ", frozen, ldbNum) + if head := *ReadHeaderNumber(db, ReadHeadHeaderHash(db)); head > frozen-1 { + // Find the smallest block stored in the key-value store + // in range of [frozen, head] + var number uint64 + for number = head; number >= frozen; number -= 1 { + data, _ := db.Get(headerHashKey(number)) + if len(data) == 0 { + break + } + } + return nil, fmt.Errorf("gap in the chain between ancients (#%d) and leveldb (#%d) ", frozen, number) } // Database contains only older data than the freezer, this happens if the // state was wiped and reinited from an existing freezer. From 71e67279835593acfa53c617e2a38828e2cc013b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 17 Feb 2023 10:08:03 +0100 Subject: [PATCH 2/4] cmd/geth, core/rawdb: print out chain metadata on startup failure --- cmd/geth/dbcmd.go | 28 +++----------------------- core/rawdb/database.go | 45 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 4deb081ed8f6..b409b19260e0 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -694,41 +694,19 @@ func showMetaData(ctx *cli.Context) error { if err != nil { fmt.Fprintf(os.Stderr, "Error accessing ancients: %v", err) } - pp := func(val *uint64) string { - if val == nil { - return "" - } - return fmt.Sprintf("%d (%#x)", *val, *val) - } - data := [][]string{ - {"databaseVersion", pp(rawdb.ReadDatabaseVersion(db))}, - {"headBlockHash", fmt.Sprintf("%v", rawdb.ReadHeadBlockHash(db))}, - {"headFastBlockHash", fmt.Sprintf("%v", rawdb.ReadHeadFastBlockHash(db))}, - {"headHeaderHash", fmt.Sprintf("%v", rawdb.ReadHeadHeaderHash(db))}} + data := rawdb.ReadChainMetadata(db) + data = append(data, []string{"frozen", fmt.Sprintf("%d items", ancients)}) + data = append(data, []string{"snapshotGenerator", snapshot.ParseGeneratorStatus(rawdb.ReadSnapshotGenerator(db))}) if b := rawdb.ReadHeadBlock(db); b != nil { data = append(data, []string{"headBlock.Hash", fmt.Sprintf("%v", b.Hash())}) data = append(data, []string{"headBlock.Root", fmt.Sprintf("%v", b.Root())}) data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (%#x)", b.Number(), b.Number())}) } - if b := rawdb.ReadSkeletonSyncStatus(db); b != nil { - data = append(data, []string{"SkeletonSyncStatus", string(b)}) - } if h := rawdb.ReadHeadHeader(db); h != nil { data = append(data, []string{"headHeader.Hash", fmt.Sprintf("%v", h.Hash())}) data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)}) data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (%#x)", h.Number, h.Number)}) } - data = append(data, [][]string{{"frozen", fmt.Sprintf("%d items", ancients)}, - {"lastPivotNumber", pp(rawdb.ReadLastPivotNumber(db))}, - {"len(snapshotSyncStatus)", fmt.Sprintf("%d bytes", len(rawdb.ReadSnapshotSyncStatus(db)))}, - {"snapshotGenerator", snapshot.ParseGeneratorStatus(rawdb.ReadSnapshotGenerator(db))}, - {"snapshotDisabled", fmt.Sprintf("%v", rawdb.ReadSnapshotDisabled(db))}, - {"snapshotJournal", fmt.Sprintf("%d bytes", len(rawdb.ReadSnapshotJournal(db)))}, - {"snapshotRecoveryNumber", pp(rawdb.ReadSnapshotRecoveryNumber(db))}, - {"snapshotRoot", fmt.Sprintf("%v", rawdb.ReadSnapshotRoot(db))}, - {"txIndexTail", pp(rawdb.ReadTxIndexTail(db))}, - {"fastTxLookupLimit", pp(rawdb.ReadFastTxLookupLimit(db))}, - }...) table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Field", "Value"}) table.AppendBulk(data) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 5da2589d0aac..c8fd910e89a2 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -202,6 +202,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st // Create the idle freezer instance frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, freezerTableSize, chainFreezerNoSnappy) if err != nil { + PrintChainMetadata(db) return nil, err } // Since the freezer can be stored separately from the user's key-value database, @@ -233,8 +234,10 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st // the freezer and the key-value store. frgenesis, err := frdb.Ancient(chainFreezerHashTable, 0) if err != nil { + PrintChainMetadata(db) return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) } else if !bytes.Equal(kvgenesis, frgenesis) { + PrintChainMetadata(db) return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) } // Key-value store and freezer belong to the same network. Ensure that they @@ -252,6 +255,8 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st break } } + // We are about to exit on error. Print database metdata beore exiting + PrintChainMetadata(db) return nil, fmt.Errorf("gap in the chain between ancients (#%d) and leveldb (#%d) ", frozen, number) } // Database contains only older data than the freezer, this happens if the @@ -269,6 +274,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st // Key-value store contains more data than the genesis block, make sure we // didn't freeze anything yet. if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { + PrintChainMetadata(db) return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") } // Block #1 is still in the database, we're allowed to init a new freezer @@ -590,3 +596,42 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { } return nil } + +// PrintChainMetadata prints out chain metadata to stderr +func PrintChainMetadata(db ethdb.KeyValueStore) { + fmt.Fprintf(os.Stderr, "Chain metadata\n") + for _, v := range ReadChainMetadata(db) { + fmt.Fprintf(os.Stderr, " %s\n", strings.Join(v, " : ")) + } + fmt.Fprintf(os.Stderr, "\n\n") +} + +// ReadChainMetadata returns a set of key/value pairs that contains informatin +// about the database chain status. This can be used for diagnostic purposes +// when investigating the state of the node. +func ReadChainMetadata(db ethdb.KeyValueStore) [][]string { + pp := func(val *uint64) string { + if val == nil { + return "" + } + return fmt.Sprintf("%d (%#x)", *val, *val) + } + data := [][]string{ + {"databaseVersion", pp(ReadDatabaseVersion(db))}, + {"headBlockHash", fmt.Sprintf("%v", ReadHeadBlockHash(db))}, + {"headFastBlockHash", fmt.Sprintf("%v", ReadHeadFastBlockHash(db))}, + {"headHeaderHash", fmt.Sprintf("%v", ReadHeadHeaderHash(db))}, + {"lastPivotNumber", pp(ReadLastPivotNumber(db))}, + {"len(snapshotSyncStatus)", fmt.Sprintf("%d bytes", len(ReadSnapshotSyncStatus(db)))}, + {"snapshotDisabled", fmt.Sprintf("%v", ReadSnapshotDisabled(db))}, + {"snapshotJournal", fmt.Sprintf("%d bytes", len(ReadSnapshotJournal(db)))}, + {"snapshotRecoveryNumber", pp(ReadSnapshotRecoveryNumber(db))}, + {"snapshotRoot", fmt.Sprintf("%v", ReadSnapshotRoot(db))}, + {"txIndexTail", pp(ReadTxIndexTail(db))}, + {"fastTxLookupLimit", pp(ReadFastTxLookupLimit(db))}, + } + if b := ReadSkeletonSyncStatus(db); b != nil { + data = append(data, []string{"SkeletonSyncStatus", string(b)}) + } + return data +} From ebe1bd2410d0dac5447ce69592d92ad1c1545a0d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 17 Feb 2023 10:19:36 +0100 Subject: [PATCH 3/4] core/rawdb: check gap ascending --- core/rawdb/database.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index c8fd910e89a2..b8b794ef795c 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -249,15 +249,15 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st // Find the smallest block stored in the key-value store // in range of [frozen, head] var number uint64 - for number = head; number >= frozen; number -= 1 { - data, _ := db.Get(headerHashKey(number)) - if len(data) == 0 { + for number = frozen; number <= head; number++ { + if present, _ := db.Has(headerHashKey(number)); present { break } } // We are about to exit on error. Print database metdata beore exiting PrintChainMetadata(db) - return nil, fmt.Errorf("gap in the chain between ancients (#%d) and leveldb (#%d) ", frozen, number) + return nil, fmt.Errorf("gap in the chain between ancients [0 - #%d] and leveldb [#%d - #%d] ", + frozen-1, number, head) } // Database contains only older data than the freezer, this happens if the // state was wiped and reinited from an existing freezer. From 32b608649019cc915012340b333429367046c96c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 21 Feb 2023 08:41:37 +0100 Subject: [PATCH 4/4] core/rawdb: unexport method --- core/rawdb/database.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index b8b794ef795c..4804dd86ff28 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -202,7 +202,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st // Create the idle freezer instance frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, freezerTableSize, chainFreezerNoSnappy) if err != nil { - PrintChainMetadata(db) + printChainMetadata(db) return nil, err } // Since the freezer can be stored separately from the user's key-value database, @@ -234,10 +234,10 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st // the freezer and the key-value store. frgenesis, err := frdb.Ancient(chainFreezerHashTable, 0) if err != nil { - PrintChainMetadata(db) + printChainMetadata(db) return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) } else if !bytes.Equal(kvgenesis, frgenesis) { - PrintChainMetadata(db) + printChainMetadata(db) return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) } // Key-value store and freezer belong to the same network. Ensure that they @@ -255,7 +255,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st } } // We are about to exit on error. Print database metdata beore exiting - PrintChainMetadata(db) + printChainMetadata(db) return nil, fmt.Errorf("gap in the chain between ancients [0 - #%d] and leveldb [#%d - #%d] ", frozen-1, number, head) } @@ -274,7 +274,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st // Key-value store contains more data than the genesis block, make sure we // didn't freeze anything yet. if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { - PrintChainMetadata(db) + printChainMetadata(db) return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") } // Block #1 is still in the database, we're allowed to init a new freezer @@ -597,11 +597,11 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { return nil } -// PrintChainMetadata prints out chain metadata to stderr -func PrintChainMetadata(db ethdb.KeyValueStore) { +// printChainMetadata prints out chain metadata to stderr. +func printChainMetadata(db ethdb.KeyValueStore) { fmt.Fprintf(os.Stderr, "Chain metadata\n") for _, v := range ReadChainMetadata(db) { - fmt.Fprintf(os.Stderr, " %s\n", strings.Join(v, " : ")) + fmt.Fprintf(os.Stderr, " %s\n", strings.Join(v, ": ")) } fmt.Fprintf(os.Stderr, "\n\n") }