Skip to content

Commit

Permalink
Merge pull request bnb-chain#1882 from Fynnss/add_hash_trie_node_prun…
Browse files Browse the repository at this point in the history 
…e_tool

cmd/geth: add hbss to pbss convert tool
  • Loading branch information
@fynnss
fynnss authored Oct 5, 2023
2 parents 528d97b + b8bad31 commit c26a303
Show file tree
Hide file tree
Showing 4 changed files with 537 additions and 0 deletions.
267 changes: 267 additions & 0 deletions cmd/geth/dbcmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package main
import (
"bytes"
"fmt"
"math"
"os"
"os/signal"
"path/filepath"
Expand Down Expand Up @@ -72,6 +73,10 @@ Remove blockchain and state databases`,
// no legacy stored receipts for bsc
// dbMigrateFreezerCmd,
dbCheckStateContentCmd,
dbHbss2PbssCmd,
dbPruneHashTrieCmd,
dbTrieGetCmd,
dbTrieDeleteCmd,
},
}
dbInspectCmd = &cli.Command{
Expand All @@ -94,6 +99,54 @@ Remove blockchain and state databases`,
For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates
a data corruption.`,
}
dbHbss2PbssCmd = &cli.Command{
Action: hbss2pbss,
Name: "hbss-to-pbss",
ArgsUsage: "<jobnum (optional)>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
},
Usage: "Convert Hash-Base to Path-Base trie node.",
Description: `This command iterates the entire trie node database and convert the hash-base node to path-base node.`,
}
dbTrieGetCmd = &cli.Command{
Action: dbTrieGet,
Name: "trie-get",
Usage: "Show the value of a trie node path key",
ArgsUsage: "[trie owner] <path-base key>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.StateSchemeFlag,
},
Description: "This command looks up the specified trie node key from the database.",
}
dbTrieDeleteCmd = &cli.Command{
Action: dbTrieDelete,
Name: "trie-delete",
Usage: "delete the specify trie node",
ArgsUsage: "[trie owner] <hash-base key> | <path-base key>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.StateSchemeFlag,
},
Description: "This command delete the specify trie node from the database.",
}
dbPruneHashTrieCmd = &cli.Command{
Action: pruneHashTrie,
Name: "prune-hash-trie",
ArgsUsage: "",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
},
Usage: "[Caution]Prune all the hash trie node in diskdb",
Description: `This command iterates the entrie kv in leveldb and delete all the hash trie node.`,
}
dbStatCmd = &cli.Command{
Action: dbStats,
Name: "stats",
Expand Down Expand Up @@ -433,6 +486,133 @@ func dbGet(ctx *cli.Context) error {
return nil
}

// dbTrieGet shows the value of a given database key
func dbTrieGet(ctx *cli.Context) error {
if ctx.NArg() < 1 || ctx.NArg() > 2 {
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack, false, false)
defer db.Close()

scheme := ctx.String(utils.StateSchemeFlag.Name)
if scheme == "" {
scheme = rawdb.HashScheme
}

if scheme == rawdb.PathScheme {
var (
pathKey []byte
owner []byte
err error
)
if ctx.NArg() == 1 {
pathKey, err = hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
nodeVal, hash := rawdb.ReadAccountTrieNode(db, pathKey)
log.Info("TrieGet result ", "PathKey", common.Bytes2Hex(pathKey), "Hash: ", hash, "node: ", trie.NodeString(hash.Bytes(), nodeVal))
} else if ctx.NArg() == 2 {
owner, err = hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
pathKey, err = hexutil.Decode(ctx.Args().Get(1))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}

nodeVal, hash := rawdb.ReadStorageTrieNode(db, common.BytesToHash(owner), pathKey)
log.Info("TrieGet result ", "PathKey: ", common.Bytes2Hex(pathKey), "Owner: ", common.BytesToHash(owner), "Hash: ", hash, "node: ", trie.NodeString(hash.Bytes(), nodeVal))
}
} else if scheme == rawdb.HashScheme {
if ctx.NArg() == 1 {
hashKey, err := hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
val, err := db.Get(hashKey)
if err != nil {
log.Error("db get failed, ", "error: ", err)
return err
}
log.Info("TrieGet result ", "HashKey: ", common.BytesToHash(hashKey), "node: ", trie.NodeString(hashKey, val))
} else {
log.Error("args too much")
}
}

return nil
}

// dbTrieDelete delete the trienode of a given database key
func dbTrieDelete(ctx *cli.Context) error {
if ctx.NArg() < 1 || ctx.NArg() > 2 {
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack, false, false)
defer db.Close()

scheme := ctx.String(utils.StateSchemeFlag.Name)
if scheme == "" {
scheme = rawdb.HashScheme
}

if scheme == rawdb.PathScheme {
var (
pathKey []byte
owner []byte
err error
)
if ctx.NArg() == 1 {
pathKey, err = hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
rawdb.DeleteAccountTrieNode(db, pathKey)
} else if ctx.NArg() == 2 {
owner, err = hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
pathKey, err = hexutil.Decode(ctx.Args().Get(1))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
rawdb.DeleteStorageTrieNode(db, common.BytesToHash(owner), pathKey)
}
} else if scheme == rawdb.HashScheme {
if ctx.NArg() == 1 {
hashKey, err := hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the value", "error", err)
return err
}
err = db.Delete(hashKey)
if err != nil {
log.Error("db delete failed", "err", err)
return err
}
} else {
log.Error("args too much")
}
}
return nil
}

// dbDelete deletes a key from the database
func dbDelete(ctx *cli.Context) error {
if ctx.NArg() != 1 {
Expand Down Expand Up @@ -743,3 +923,90 @@ func showMetaData(ctx *cli.Context) error {
table.Render()
return nil
}

func hbss2pbss(ctx *cli.Context) error {
if ctx.NArg() > 1 {
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
}

var jobnum uint64
var err error
if ctx.NArg() == 1 {
jobnum, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64)
if err != nil {
return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err)
}
} else {
// by default
jobnum = 1000
}

stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack, false, false)
db.Sync()
defer db.Close()

config := trie.HashDefaults
triedb := trie.NewDatabase(db, config)
triedb.Cap(0)
log.Info("hbss2pbss triedb", "scheme", triedb.Scheme())
defer triedb.Close()

headerHash := rawdb.ReadHeadHeaderHash(db)
blockNumber := rawdb.ReadHeaderNumber(db, headerHash)
if blockNumber == nil {
log.Error("read header number failed.")
return fmt.Errorf("read header number failed")
}

log.Info("hbss2pbss converting", "HeaderHash: ", headerHash.String(), ", blockNumber: ", *blockNumber)

var headerBlockHash common.Hash
var trieRootHash common.Hash

if *blockNumber != math.MaxUint64 {
headerBlockHash = rawdb.ReadCanonicalHash(db, *blockNumber)
if headerBlockHash == (common.Hash{}) {
return fmt.Errorf("ReadHeadBlockHash empty hash")
}
blockHeader := rawdb.ReadHeader(db, headerBlockHash, *blockNumber)
trieRootHash = blockHeader.Root
fmt.Println("Canonical Hash: ", headerBlockHash.String(), ", TrieRootHash: ", trieRootHash.String())
}
if (trieRootHash == common.Hash{}) {
log.Error("Empty root hash")
return fmt.Errorf("Empty root hash.")
}

id := trie.StateTrieID(trieRootHash)
theTrie, err := trie.New(id, triedb)
if err != nil {
log.Error("fail to new trie tree", "err", err, "rootHash", err, trieRootHash.String())
return err
}

h2p, err := trie.NewHbss2Pbss(theTrie, triedb, trieRootHash, *blockNumber, jobnum)
if err != nil {
log.Error("fail to new hash2pbss", "err", err, "rootHash", err, trieRootHash.String())
return err
}
h2p.Run()

return nil
}

func pruneHashTrie(ctx *cli.Context) error {
if ctx.NArg() != 0 {
return fmt.Errorf("required none argument")
}

stack, _ := makeConfigNode(ctx)
defer stack.Close()

db := utils.MakeChainDatabase(ctx, stack, false, false)
defer db.Close()

return rawdb.PruneHashTrieNodeInDataBase(db)
}
22 changes: 22 additions & 0 deletions core/rawdb/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,28 @@ func AncientInspect(db ethdb.Database) error {
return nil
}

func PruneHashTrieNodeInDataBase(db ethdb.Database) error {
it := db.NewIterator([]byte{}, []byte{})
defer it.Release()

total_num := 0
for it.Next() {
var key = it.Key()
switch {
case IsLegacyTrieNode(key, it.Value()):
db.Delete(key)
total_num++
if total_num%100000 == 0 {
log.Info("Pruning ", "Complete progress: ", total_num, "hash-base trie nodes")
}
default:
continue
}
}
log.Info("Pruning ", "Complete progress", total_num, "hash-base trie nodes")
return nil
}

// InspectDatabase traverses the entire database and checks the size
// of all different categories of data.
func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
Expand Down
Loading

0 comments on commit c26a303

Please sign in to comment.