From 8df4afc24be5012abe025e561e19a22ee3901555 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Mon, 16 Jan 2023 16:23:18 +0800 Subject: [PATCH 1/2] add 'bbolt surgery revert-meta-page' command Signed-off-by: Benjamin Wang --- cmd/bbolt/main.go | 3 + cmd/bbolt/surgery_commands.go | 156 +++++++++++++++++++++++++++++ cmd/bbolt/surgery_commands_test.go | 69 +++++++++++++ internal/guts_cli/guts_cli.go | 4 + 4 files changed, 232 insertions(+) create mode 100644 cmd/bbolt/surgery_commands.go create mode 100644 cmd/bbolt/surgery_commands_test.go diff --git a/cmd/bbolt/main.go b/cmd/bbolt/main.go index 409dbd5a4..a3f93e381 100644 --- a/cmd/bbolt/main.go +++ b/cmd/bbolt/main.go @@ -130,6 +130,8 @@ func (m *Main) Run(args ...string) error { return newPagesCommand(m).Run(args[1:]...) case "stats": return newStatsCommand(m).Run(args[1:]...) + case "surgery": + return newSurgeryCommand(m).Run(args[1:]...) default: return ErrUnknownCommand } @@ -159,6 +161,7 @@ The commands are: pages print list of pages with their types page-item print the key and value of a page item. stats iterate over all pages and generate usage stats + surgery perform surgery on bbolt database Use "bbolt [command] -h" for more information about a command. `, "\n") diff --git a/cmd/bbolt/surgery_commands.go b/cmd/bbolt/surgery_commands.go new file mode 100644 index 000000000..43b38acc1 --- /dev/null +++ b/cmd/bbolt/surgery_commands.go @@ -0,0 +1,156 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "os" + "strings" + + "go.etcd.io/bbolt/internal/surgeon" +) + +// SurgeryCommand represents the "surgery" command execution. +type SurgeryCommand struct { + baseCommand +} + +// newSurgeryCommand returns a SurgeryCommand. +func newSurgeryCommand(m *Main) *SurgeryCommand { + c := &SurgeryCommand{} + c.baseCommand = m.baseCommand + return c +} + +// Run executes the `surgery` program. +func (cmd *SurgeryCommand) Run(args ...string) error { + // Require a command at the beginning. + if len(args) == 0 || strings.HasPrefix(args[0], "-") { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } + + // Execute command. + switch args[0] { + case "help": + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + case "revert-meta-page": + return newRevertMetaPageCommand(cmd).Run(args[1:]...) + default: + return ErrUnknownCommand + } +} + +// Usage returns the help message. +func (cmd *SurgeryCommand) Usage() string { + return strings.TrimLeft(` +Surgery is a command for performing low level update on bbolt databases. + +Usage: + + bbolt surgery command [arguments] + +The commands are: + + help print this screen + revert-meta-page revert the meta page change made by the last transaction + +Use "bbolt surgery [command] -h" for more information about a command. +`, "\n") +} + +// RevertMetaPageCommand represents the "surgery revert-meta-page" command execution. +type RevertMetaPageCommand struct { + baseCommand + + SrcPath string + DstPath string +} + +// newRevertMetaPageCommand returns a RevertMetaPageCommand. +func newRevertMetaPageCommand(m *SurgeryCommand) *RevertMetaPageCommand { + c := &RevertMetaPageCommand{} + c.baseCommand = m.baseCommand + return c +} + +// Run executes the command. +func (cmd *RevertMetaPageCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.SetOutput(io.Discard) + fs.StringVar(&cmd.DstPath, "o", "", "") + if err := fs.Parse(args); err == flag.ErrHelp { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } else if err != nil { + return err + } else if cmd.DstPath == "" { + return errors.New("output file required") + } + + // Require database paths. + cmd.SrcPath = fs.Arg(0) + if cmd.SrcPath == "" { + return ErrPathRequired + } + + // Ensure source file exists. + fi, err := os.Stat(cmd.SrcPath) + if os.IsNotExist(err) { + return ErrFileNotFound + } else if err != nil { + return err + } + initialSize := fi.Size() + + // Ensure output file not exist. + _, err = os.Stat(cmd.DstPath) + if err == nil { + return fmt.Errorf("output file %q already exists", cmd.DstPath) + } else if !os.IsNotExist(err) { + return err + } + + // Copy database from SrcPath to DstPath + srcDB, err := os.Open(cmd.SrcPath) + if err != nil { + return fmt.Errorf("failed to open source file %q: %w", cmd.SrcPath, err) + } + defer srcDB.Close() + dstDB, err := os.Create(cmd.DstPath) + if err != nil { + return fmt.Errorf("failed to create output file %q: %w", cmd.DstPath, err) + } + defer dstDB.Close() + written, err := io.Copy(dstDB, srcDB) + if err != nil { + return fmt.Errorf("failed to copy database file from %q to %q: %w", cmd.SrcPath, cmd.DstPath, err) + } + if initialSize != written { + return fmt.Errorf("the byte copied (%q: %d) isn't equal to the initial db size (%q: %d)", cmd.DstPath, written, cmd.SrcPath, initialSize) + } + + // revert the meta page + if err = surgeon.RevertMetaPage(cmd.DstPath); err != nil { + return err + } + + fmt.Fprintln(cmd.Stdout, "The meta page is reverted.") + return nil +} + +// Usage returns the help message. +func (cmd *RevertMetaPageCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt surgery revert-meta-page -o DST SRC + +RevertMetaPage copies the database file at SRC to a newly created database +file at DST. Afterwards, it reverts the meta page on the newly created +database at DST. + +The original database is left untouched. +`, "\n") +} diff --git a/cmd/bbolt/surgery_commands_test.go b/cmd/bbolt/surgery_commands_test.go new file mode 100644 index 000000000..1176a3d32 --- /dev/null +++ b/cmd/bbolt/surgery_commands_test.go @@ -0,0 +1,69 @@ +package main_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.etcd.io/bbolt/internal/btesting" + "go.etcd.io/bbolt/internal/guts_cli" +) + +func TestSurgery_RevertMetaPage(t *testing.T) { + pageSize := os.Getpagesize() + db := btesting.MustCreateDB(t) + srcPath := db.Path() + + srcFile, err := os.Open(srcPath) + require.NoError(t, err) + defer srcFile.Close() + + // Read both meta0 and meta1 from srcFile + srcBuf0, srcBuf1 := readBothMetaPages(t, srcPath, pageSize) + meta0Page := guts_cli.LoadPageMeta(srcBuf0) + meta1Page := guts_cli.LoadPageMeta(srcBuf1) + + // Get the non-active meta page + nonActiveSrcBuf := srcBuf0 + nonActiveMetaPageId := 0 + if meta0Page.Txid() > meta1Page.Txid() { + nonActiveSrcBuf = srcBuf1 + nonActiveMetaPageId = 1 + } + t.Logf("non active meta page id: %d", nonActiveMetaPageId) + + // revert the meta page + dstPath := filepath.Join(t.TempDir(), "dstdb") + m := NewMain() + err = m.Run("surgery", "revert-meta-page", "-o", dstPath, srcPath) + require.NoError(t, err) + + // read both meta0 and meta1 from dst file + dstBuf0, dstBuf1 := readBothMetaPages(t, dstPath, pageSize) + + // check result. Note we should skip the page ID + assert.Equal(t, nonActiveSrcBuf[8:], dstBuf0[8:]) + assert.Equal(t, nonActiveSrcBuf[8:], dstBuf1[8:]) +} + +func readBothMetaPages(t *testing.T, filePath string, pageSize int) ([]byte, []byte) { + dbFile, err := os.Open(filePath) + require.NoError(t, err) + defer dbFile.Close() + + buf0 := make([]byte, pageSize) + buf1 := make([]byte, pageSize) + + meta0Len, err := dbFile.ReadAt(buf0, 0) + require.NoError(t, err) + require.Equal(t, pageSize, meta0Len) + + meta1Len, err := dbFile.ReadAt(buf1, int64(pageSize)) + require.NoError(t, err) + require.Equal(t, pageSize, meta1Len) + + return buf0, buf1 +} diff --git a/internal/guts_cli/guts_cli.go b/internal/guts_cli/guts_cli.go index d54bc3cfd..719ed453b 100644 --- a/internal/guts_cli/guts_cli.go +++ b/internal/guts_cli/guts_cli.go @@ -64,6 +64,10 @@ func (m *Meta) RootBucket() *Bucket { return &m.root } +func (m *Meta) Txid() uint64 { + return uint64(m.txid) +} + func (m *Meta) Print(w io.Writer) { fmt.Fprintf(w, "Version: %d\n", m.version) fmt.Fprintf(w, "Page Size: %d bytes\n", m.pageSize) From ff467f2ff5c460f1681f908c732e8caaaebbec54 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Mon, 16 Jan 2023 18:46:51 +0800 Subject: [PATCH 2/2] addresed review comments Signed-off-by: Benjamin Wang --- cmd/bbolt/surgery_commands.go | 57 +++++++++++++++++++----------- cmd/bbolt/surgery_commands_test.go | 15 +++++--- 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/cmd/bbolt/surgery_commands.go b/cmd/bbolt/surgery_commands.go index 43b38acc1..a98c1e517 100644 --- a/cmd/bbolt/surgery_commands.go +++ b/cmd/bbolt/surgery_commands.go @@ -80,15 +80,12 @@ func newRevertMetaPageCommand(m *SurgeryCommand) *RevertMetaPageCommand { func (cmd *RevertMetaPageCommand) Run(args ...string) error { // Parse flags. fs := flag.NewFlagSet("", flag.ContinueOnError) - fs.SetOutput(io.Discard) - fs.StringVar(&cmd.DstPath, "o", "", "") - if err := fs.Parse(args); err == flag.ErrHelp { + help := fs.Bool("h", false, "") + if err := fs.Parse(args); err != nil { + return err + } else if *help { fmt.Fprintln(cmd.Stderr, cmd.Usage()) return ErrUsage - } else if err != nil { - return err - } else if cmd.DstPath == "" { - return errors.New("output file required") } // Require database paths. @@ -97,14 +94,18 @@ func (cmd *RevertMetaPageCommand) Run(args ...string) error { return ErrPathRequired } + cmd.DstPath = fs.Arg(1) + if cmd.DstPath == "" { + return errors.New("output file required") + } + // Ensure source file exists. - fi, err := os.Stat(cmd.SrcPath) + _, err := os.Stat(cmd.SrcPath) if os.IsNotExist(err) { return ErrFileNotFound } else if err != nil { return err } - initialSize := fi.Size() // Ensure output file not exist. _, err = os.Stat(cmd.DstPath) @@ -115,30 +116,44 @@ func (cmd *RevertMetaPageCommand) Run(args ...string) error { } // Copy database from SrcPath to DstPath - srcDB, err := os.Open(cmd.SrcPath) + if err := copyFile(cmd.SrcPath, cmd.DstPath); err != nil { + return fmt.Errorf("failed to copy file: %w", err) + } + + // revert the meta page + if err = surgeon.RevertMetaPage(cmd.DstPath); err != nil { + return err + } + + fmt.Fprintln(cmd.Stdout, "The meta page is reverted.") + return nil +} + +func copyFile(srcPath, dstPath string) error { + srcDB, err := os.Open(srcPath) if err != nil { - return fmt.Errorf("failed to open source file %q: %w", cmd.SrcPath, err) + return fmt.Errorf("failed to open source file %q: %w", srcPath, err) } defer srcDB.Close() - dstDB, err := os.Create(cmd.DstPath) + dstDB, err := os.Create(dstPath) if err != nil { - return fmt.Errorf("failed to create output file %q: %w", cmd.DstPath, err) + return fmt.Errorf("failed to create output file %q: %w", dstPath, err) } defer dstDB.Close() written, err := io.Copy(dstDB, srcDB) if err != nil { - return fmt.Errorf("failed to copy database file from %q to %q: %w", cmd.SrcPath, cmd.DstPath, err) - } - if initialSize != written { - return fmt.Errorf("the byte copied (%q: %d) isn't equal to the initial db size (%q: %d)", cmd.DstPath, written, cmd.SrcPath, initialSize) + return fmt.Errorf("failed to copy database file from %q to %q: %w", srcPath, dstPath, err) } - // revert the meta page - if err = surgeon.RevertMetaPage(cmd.DstPath); err != nil { - return err + srcFi, err := srcDB.Stat() + if err != nil { + return fmt.Errorf("failed to get source file info %q: %w", srcPath, err) + } + initialSize := srcFi.Size() + if initialSize != written { + return fmt.Errorf("the byte copied (%q: %d) isn't equal to the initial db size (%q: %d)", dstPath, written, srcPath, initialSize) } - fmt.Fprintln(cmd.Stdout, "The meta page is reverted.") return nil } diff --git a/cmd/bbolt/surgery_commands_test.go b/cmd/bbolt/surgery_commands_test.go index 1176a3d32..4c95f16c8 100644 --- a/cmd/bbolt/surgery_commands_test.go +++ b/cmd/bbolt/surgery_commands_test.go @@ -1,6 +1,7 @@ package main_test import ( + bolt "go.etcd.io/bbolt" "os" "path/filepath" "testing" @@ -13,8 +14,8 @@ import ( ) func TestSurgery_RevertMetaPage(t *testing.T) { - pageSize := os.Getpagesize() - db := btesting.MustCreateDB(t) + pageSize := 4096 + db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) srcPath := db.Path() srcFile, err := os.Open(srcPath) @@ -38,15 +39,19 @@ func TestSurgery_RevertMetaPage(t *testing.T) { // revert the meta page dstPath := filepath.Join(t.TempDir(), "dstdb") m := NewMain() - err = m.Run("surgery", "revert-meta-page", "-o", dstPath, srcPath) + err = m.Run("surgery", "revert-meta-page", srcPath, dstPath) require.NoError(t, err) // read both meta0 and meta1 from dst file dstBuf0, dstBuf1 := readBothMetaPages(t, dstPath, pageSize) // check result. Note we should skip the page ID - assert.Equal(t, nonActiveSrcBuf[8:], dstBuf0[8:]) - assert.Equal(t, nonActiveSrcBuf[8:], dstBuf1[8:]) + assert.Equal(t, pageDataWithoutPageId(nonActiveSrcBuf), pageDataWithoutPageId(dstBuf0)) + assert.Equal(t, pageDataWithoutPageId(nonActiveSrcBuf), pageDataWithoutPageId(dstBuf1)) +} + +func pageDataWithoutPageId(buf []byte) []byte { + return buf[8:] } func readBothMetaPages(t *testing.T, filePath string, pageSize int) ([]byte, []byte) {