diff --git a/cmd/integration/commands/flags.go b/cmd/integration/commands/flags.go index bf57f18a61c..a9927ae12b9 100644 --- a/cmd/integration/commands/flags.go +++ b/cmd/integration/commands/flags.go @@ -31,6 +31,7 @@ var ( pruneTBefore, pruneCBefore uint64 experiments []string chain string // Which chain to use (mainnet, goerli, sepolia, etc.) + outputCsvFile string commitmentMode string commitmentTrie string @@ -169,6 +170,10 @@ func withTraceFromTx(cmd *cobra.Command) { cmd.Flags().Uint64Var(&traceFromTx, "txtrace.from", 0, "start tracing from tx number") } +func withOutputCsvFile(cmd *cobra.Command) { + cmd.Flags().StringVar(&outputCsvFile, "output.csv.file", "", "location to output csv data") +} + func withCommitment(cmd *cobra.Command) { cmd.Flags().StringVar(&commitmentMode, "commitment.mode", "direct", "defines the way to calculate commitments: 'direct' mode reads from state directly, 'update' accumulate updates before commitment, 'off' actually disables commitment calculation") cmd.Flags().StringVar(&commitmentTrie, "commitment.trie", "hex", "hex - use Hex Patricia Hashed Trie for commitments, bin - use of binary patricia trie") diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index f13a18a6581..f265fd35983 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "os" "strings" "sync" "time" @@ -12,7 +13,6 @@ import ( "github.com/c2h5oh/datasize" "github.com/erigontech/mdbx-go/mdbx" lru "github.com/hashicorp/golang-lru/arc/v2" - "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/secp256k1" "github.com/spf13/cobra" @@ -24,6 +24,7 @@ import ( "github.com/ledgerwatch/erigon-lib/common/cmp" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dir" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" @@ -313,6 +314,7 @@ var cmdStageTxLookup = &cobra.Command{ } }, } + var cmdPrintStages = &cobra.Command{ Use: "print_stages", Short: "", @@ -334,6 +336,72 @@ var cmdPrintStages = &cobra.Command{ }, } +var cmdPrintTableSizes = &cobra.Command{ + Use: "print_table_sizes", + Short: "", + Run: func(cmd *cobra.Command, args []string) { + logger := debug.SetupCobra(cmd, "integration") + db, err := openDB(dbCfg(kv.ChainDB, chaindata), false, logger) + if err != nil { + logger.Error("Opening DB", "error", err) + return + } + defer db.Close() + + allTablesCfg := db.AllTables() + allTables := make([]string, 0, len(allTablesCfg)) + for table := range allTablesCfg { + allTables = append(allTables, table) + } + + var tableSizes []interface{} + err = db.View(cmd.Context(), func(tx kv.Tx) error { + tableSizes = stagedsync.CollectTableSizes(db, tx, allTables) + return nil + }) + if err != nil { + logger.Error("error while collecting table sizes", "err", err) + return + } + + if len(tableSizes)%2 != 0 { + logger.Error("table sizes len not even", "len", len(tableSizes)) + return + } + + var sb strings.Builder + sb.WriteString("Table") + sb.WriteRune(',') + sb.WriteString("Size") + sb.WriteRune('\n') + for i := 0; i < len(tableSizes)/2; i++ { + sb.WriteString(tableSizes[i*2].(string)) + sb.WriteRune(',') + sb.WriteString(tableSizes[i*2+1].(string)) + sb.WriteRune('\n') + } + + if outputCsvFile == "" { + logger.Info("table sizes", "csv", sb.String()) + return + } + + f, err := os.Create(outputCsvFile) + if err != nil { + logger.Error("issue creating file", "file", outputCsvFile, "err", err) + return + } + + _, err = f.WriteString(sb.String()) + if err != nil { + logger.Error("issue writing output to file", "file", outputCsvFile, "err", err) + return + } + + logger.Info("wrote table sizes to csv output file", "file", outputCsvFile) + }, +} + var cmdPrintMigrations = &cobra.Command{ Use: "print_migrations", Short: "", @@ -476,6 +544,10 @@ func init() { withHeimdall(cmdPrintStages) rootCmd.AddCommand(cmdPrintStages) + withDataDir(cmdPrintTableSizes) + withOutputCsvFile(cmdPrintTableSizes) + rootCmd.AddCommand(cmdPrintTableSizes) + withConfig(cmdStageSenders) withIntegrityChecks(cmdStageSenders) withReset(cmdStageSenders) diff --git a/eth/stagedsync/sync.go b/eth/stagedsync/sync.go index 8af2d280283..a0a8dc366b3 100644 --- a/eth/stagedsync/sync.go +++ b/eth/stagedsync/sync.go @@ -462,16 +462,23 @@ func (s *Sync) PrintTimings() []interface{} { return logCtx } -func PrintTables(db kv.RoDB, tx kv.RwTx) []interface{} { - if tx == nil { - return nil - } - buckets := []string{ +func CollectDBMetrics(db kv.RoDB, tx kv.RwTx) []interface{} { + res := CollectTableSizes(db, tx, []string{ kv.PlainState, kv.AccountChangeSet, kv.StorageChangeSet, kv.EthTx, kv.Log, + }) + + tx.CollectMetrics() + + return res +} + +func CollectTableSizes(db kv.RoDB, tx kv.Tx, buckets []string) []interface{} { + if tx == nil { + return nil } bucketSizes := make([]interface{}, 0, 2*(len(buckets)+2)) for _, bucket := range buckets { @@ -491,7 +498,7 @@ func PrintTables(db kv.RoDB, tx kv.RwTx) []interface{} { if db != nil { bucketSizes = append(bucketSizes, "ReclaimableSpace", libcommon.ByteCount(amountOfFreePagesInDb*db.PageSize())) } - tx.CollectMetrics() + return bucketSizes } diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go index 01fee094183..1365df8a317 100644 --- a/turbo/stages/stageloop.go +++ b/turbo/stages/stageloop.go @@ -154,7 +154,7 @@ func StageLoopIteration(ctx context.Context, db kv.RwDB, txc wrap.TxContainer, s var tableSizes []interface{} var commitTime time.Duration if canRunCycleInOneTransaction && !externalTx { - tableSizes = stagedsync.PrintTables(db, txc.Tx) // Need to do this before commit to access tx + tableSizes = stagedsync.CollectDBMetrics(db, txc.Tx) // Need to do this before commit to access tx commitStart := time.Now() errTx := txc.Tx.Commit() txc.Tx = nil