From 9832aff38a922c491f2e9934fe1d47b75f68a86e Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Wed, 15 Mar 2023 16:41:12 +0800 Subject: [PATCH 1/8] internal: add function ClearElements in surgeon package Signed-off-by: Benjamin Wang --- internal/guts_cli/guts_cli.go | 4 ++ internal/surgeon/surgeon.go | 93 +++++++++++++++++++++++++++++++++-- 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/internal/guts_cli/guts_cli.go b/internal/guts_cli/guts_cli.go index 891ddb7b4..20b74b081 100644 --- a/internal/guts_cli/guts_cli.go +++ b/internal/guts_cli/guts_cli.go @@ -50,6 +50,10 @@ func ReadPage(path string, pageID uint64) (*common.Page, []byte, error) { return nil, nil, fmt.Errorf("error: %w, Page claims to have %d overflow pages (>=hwm=%d). Interrupting to avoid risky OOM", ErrCorrupt, overflowN, hwm) } + if overflowN == 0 { + return p, buf, nil + } + // Re-read entire Page (with overflow) into buffer. buf = make([]byte, (uint64(overflowN)+1)*pageSize) if n, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { diff --git a/internal/surgeon/surgeon.go b/internal/surgeon/surgeon.go index d2220a276..d2c7be876 100644 --- a/internal/surgeon/surgeon.go +++ b/internal/surgeon/surgeon.go @@ -2,6 +2,7 @@ package surgeon import ( "fmt" + "go.etcd.io/bbolt/internal/common" "go.etcd.io/bbolt/internal/guts_cli" ) @@ -16,19 +17,105 @@ func CopyPage(path string, srcPage common.Pgid, target common.Pgid) error { } func ClearPage(path string, pgId common.Pgid) error { + return ClearPageElements(path, pgId, 0, -1) +} + +func ClearPageElements(path string, pgId common.Pgid, start, end int) error { // Read the page p, buf, err := guts_cli.ReadPage(path, uint64(pgId)) if err != nil { return fmt.Errorf("ReadPage failed: %w", err) } - // Update and rewrite the page - p.SetCount(0) - p.SetOverflow(0) + if !p.IsLeafPage() && !p.IsBranchPage() { + return fmt.Errorf("can't clear elements in %q page", p.Typ()) + } + + elementCnt := int(p.Count()) + + if elementCnt == 0 { + return nil + } + + if start < 0 || start >= elementCnt { + return fmt.Errorf("the start index (%d) is out of range [0, %d)", start, elementCnt) + } + + if (end < 0 || end > elementCnt) && end != -1 { + return fmt.Errorf("the end index (%d) is out of range [0, %d]", end, elementCnt) + } + + if start > end && end != -1 { + return fmt.Errorf("the start index (%d) is bigger than the end index (%d)", start, end) + } + + if start == end { + return fmt.Errorf("invalid: the start index (%d) is equal to the end index (%d)", start, end) + } + + preOverflow := p.Overflow() + + if end == int(p.Count()) || end == -1 { + p.SetCount(uint16(start)) + p.SetOverflow(0) + if preOverflow != 0 || p.IsBranchPage() { + if err := clearFreelist(path); err != nil { + return err + } + } + } else { + inodes := common.ReadInodeFromPage(p) + inodes = append(inodes[:start], inodes[end:]...) + + p.SetCount(uint16(len(inodes))) + dataWritten := common.WriteInodeToPage(inodes, p) + + pageSize, _, err := guts_cli.ReadPageAndHWMSize(path) + if err != nil { + return fmt.Errorf("ReadPageAndHWMSize failed: %w", err) + } + if dataWritten%uint32(pageSize) == 0 { + p.SetOverflow(dataWritten/uint32(pageSize) - 1) + } else { + p.SetOverflow(dataWritten / uint32(pageSize)) + } + } + if err := guts_cli.WritePage(path, buf); err != nil { return fmt.Errorf("WritePage failed: %w", err) } + if preOverflow != p.Overflow() || p.IsBranchPage() { + return clearFreelist(path) + } + + return nil +} + +func clearFreelist(path string) error { + if err := clearFreelistAt(path, 0); err != nil { + return fmt.Errorf("clearFreelist on meta page 0 failed: %w", err) + } + if err := clearFreelistAt(path, 1); err != nil { + return fmt.Errorf("clearFreelist on meta page 1 failed: %w", err) + } + return nil +} + +func clearFreelistAt(path string, pageId uint64) error { + _, buf, err := guts_cli.ReadPage(path, pageId) + if err != nil { + return fmt.Errorf("ReadPage %d failed: %w", pageId, err) + } + + meta := common.LoadPageMeta(buf) + meta.SetFreelist(common.PgidNoFreelist) + meta.SetChecksum(meta.Sum64()) + + if err := guts_cli.WritePage(path, buf); err != nil { + return fmt.Errorf("WritePage %d failed: %w", pageId, err) + } + return nil } From 3c3da590b120ac548d8543162160c543522905b1 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Sun, 19 Mar 2023 09:18:53 +0800 Subject: [PATCH 2/8] CMD: add cobra style 'surgery clear-page-elements' command Signed-off-by: Benjamin Wang --- cmd/bbolt/command_root.go | 24 ++++++++++ cmd/bbolt/command_surgery_cobra.go | 74 ++++++++++++++++++++++++++++++ cmd/bbolt/main.go | 14 ++++++ go.mod | 3 ++ go.sum | 9 +++- 5 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 cmd/bbolt/command_root.go create mode 100644 cmd/bbolt/command_surgery_cobra.go diff --git a/cmd/bbolt/command_root.go b/cmd/bbolt/command_root.go new file mode 100644 index 000000000..b960df898 --- /dev/null +++ b/cmd/bbolt/command_root.go @@ -0,0 +1,24 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +const ( + cliName = "bbolt" + cliDescription = "A simple command line tool for inspecting bbolt databases" +) + +func NewRootCommand() *cobra.Command { + rootCmd := &cobra.Command{ + Use: cliName, + Short: cliDescription, + Version: "dev", + } + + rootCmd.AddCommand( + newSurgeryCobraCommand(), + ) + + return rootCmd +} diff --git a/cmd/bbolt/command_surgery_cobra.go b/cmd/bbolt/command_surgery_cobra.go new file mode 100644 index 000000000..b72cc6f14 --- /dev/null +++ b/cmd/bbolt/command_surgery_cobra.go @@ -0,0 +1,74 @@ +package main + +import ( + "errors" + "fmt" + "os" + + "github.com/spf13/cobra" + + "go.etcd.io/bbolt/internal/common" + "go.etcd.io/bbolt/internal/surgeon" +) + +var ( + surgeryTargetDBFilePath string + surgeryPageId uint64 + surgeryStartElementIdx int + surgeryEndElementIdx int +) + +func newSurgeryCobraCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "surgery ", + Short: "surgery related commands", + } + + cmd.AddCommand(newSurgeryClearPageElementsCommand()) + + return cmd +} + +func newSurgeryClearPageElementsCommand() *cobra.Command { + clearElementCmd := &cobra.Command{ + Use: "clear-page-elements [options]", + Short: "Clears elements from the given page", + Args: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("db file path not provided") + } + if len(args) > 1 { + return errors.New("too many arguments") + } + return nil + }, + + RunE: surgeryClearPageElementFunc, + } + + clearElementCmd.Flags().StringVar(&surgeryTargetDBFilePath, "output", "", "path to the target db file") + clearElementCmd.Flags().Uint64VarP(&surgeryPageId, "pageId", "", 0, "page id") + clearElementCmd.Flags().IntVarP(&surgeryStartElementIdx, "from", "", 0, "start element index (included) to clear, starting from 0") + clearElementCmd.Flags().IntVarP(&surgeryEndElementIdx, "to", "", 0, "end element index (excluded) to clear, starting from 0, -1 means to the end of page") + + return clearElementCmd +} + +func surgeryClearPageElementFunc(cmd *cobra.Command, args []string) error { + srcDBPath := args[0] + + if err := copyFile(srcDBPath, surgeryTargetDBFilePath); err != nil { + return fmt.Errorf("[clear-page-element] copy file failed: %w", err) + } + + if surgeryPageId < 2 { + return fmt.Errorf("the pageId must be at least 2, but got %d", surgeryPageId) + } + + if err := surgeon.ClearPageElements(surgeryTargetDBFilePath, common.Pgid(surgeryPageId), surgeryStartElementIdx, surgeryEndElementIdx); err != nil { + return fmt.Errorf("clear-page-element command failed: %w", err) + } + + fmt.Fprintf(os.Stdout, "All elements in [%d, %d) in page %d were cleared\n", surgeryStartElementIdx, surgeryEndElementIdx, surgeryPageId) + return nil +} diff --git a/cmd/bbolt/main.go b/cmd/bbolt/main.go index 7afb059f6..e84b31bf7 100644 --- a/cmd/bbolt/main.go +++ b/cmd/bbolt/main.go @@ -60,12 +60,26 @@ func main() { m := NewMain() if err := m.Run(os.Args[1:]...); err == ErrUsage { os.Exit(2) + } else if err == ErrUnknownCommand { + execute() } else if err != nil { fmt.Println(err.Error()) os.Exit(1) } } +func execute() { + rootCmd := NewRootCommand() + if err := rootCmd.Execute(); err != nil { + if rootCmd.SilenceErrors { + fmt.Fprintln(os.Stderr, "Error:", err) + os.Exit(1) + } else { + os.Exit(1) + } + } +} + type baseCommand struct { Stdin io.Reader Stdout io.Writer diff --git a/go.mod b/go.mod index 3602e0697..4506b5af8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module go.etcd.io/bbolt go 1.19 require ( + github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.2 go.etcd.io/gofail v0.1.0 golang.org/x/sys v0.6.0 @@ -10,6 +11,8 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ed1ee3861..529183910 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,21 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg= From 95576a4f10b51e2e0383fe07b528804844e1f79c Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Sun, 19 Mar 2023 09:20:13 +0800 Subject: [PATCH 3/8] CMD: add test cases for the 'surgery clear-page-elements' command Signed-off-by: Benjamin Wang --- cmd/bbolt/command_surgery_cobra_test.go | 184 ++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 cmd/bbolt/command_surgery_cobra_test.go diff --git a/cmd/bbolt/command_surgery_cobra_test.go b/cmd/bbolt/command_surgery_cobra_test.go new file mode 100644 index 000000000..1b054333c --- /dev/null +++ b/cmd/bbolt/command_surgery_cobra_test.go @@ -0,0 +1,184 @@ +package main_test + +import ( + "fmt" + "math/rand" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + bolt "go.etcd.io/bbolt" + main "go.etcd.io/bbolt/cmd/bbolt" + "go.etcd.io/bbolt/internal/btesting" + "go.etcd.io/bbolt/internal/guts_cli" +) + +func TestSurgery_ClearPageElement(t *testing.T) { + testCases := []struct { + name string + from int + to int + setEndIdxAsCount bool + removeOnlyOneElement bool // only valid when setEndIdxAsCount == true, and startIdx = endIdx -1 in this case. + expectError bool + }{ + // normal range + { + name: "normal range: [4, 8)", + from: 4, + to: 8, + }, + { + name: "normal range: [5, -1)", + from: 4, + to: -1, + }, + { + name: "normal range: all", + from: 0, + to: -1, + }, + { + name: "normal range: [0, 7)", + from: 0, + to: 7, + }, + { + name: "normal range: [3, count)", + from: 4, + setEndIdxAsCount: true, + }, + // remove only one element + { + name: "one element: the first one", + from: 0, + to: 1, + }, + { + name: "one element: [6, 7)", + from: 6, + to: 7, + }, + { + name: "one element: the last one", + setEndIdxAsCount: true, + removeOnlyOneElement: true, + }, + // abnormal range + { + name: "abnormal range: [-1, 4)", + from: -1, + to: 4, + expectError: true, + }, + { + name: "abnormal range: [-2, 5)", + from: -1, + to: 5, + expectError: true, + }, + { + name: "abnormal range: [3, 3)", + from: 3, + to: 3, + expectError: true, + }, + { + name: "abnormal range: [5, 3)", + from: 5, + to: 3, + expectError: true, + }, + { + name: "abnormal range: [3, -2)", + from: 3, + to: -2, + expectError: true, + }, + { + name: "abnormal range: [3, 1000000)", + from: -1, + to: 4, + expectError: true, + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + testSurgeryClearPageElement(t, tc.from, tc.to, tc.setEndIdxAsCount, tc.removeOnlyOneElement, tc.expectError) + }) + } +} + +func testSurgeryClearPageElement(t *testing.T, startIdx, endIdx int, setEndIdxAsCount, removeOnlyOne, expectError bool) { + pageSize := 4096 + db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) + srcPath := db.Path() + + // Generate sample db + t.Log("Generate some sample data") + err := db.Fill([]byte("data"), 10, 200, + func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", tx*10000+k)) }, + func(tx int, k int) []byte { return make([]byte, 10) }, + ) + require.NoError(t, err) + + defer requireDBNoChange(t, dbData(t, srcPath), srcPath) + + // find a page with at least 10 elements + var ( + pageId uint64 = 2 + elementCount uint16 = 0 + ) + for { + p, _, err := guts_cli.ReadPage(srcPath, pageId) + require.NoError(t, err) + + if p.IsLeafPage() && p.Count() > 10 { + elementCount = p.Count() + break + } + pageId++ + } + t.Logf("The original element count: %d", elementCount) + + if setEndIdxAsCount { + t.Logf("Set the endIdx as the element count: %d", elementCount) + endIdx = int(elementCount) + if removeOnlyOne { + startIdx = endIdx - 1 + t.Logf("Set the startIdx as the endIdx-1: %d", startIdx) + } + } + + // clear elements [startIdx, endIdx) in the page + rootCmd := main.NewRootCommand() + output := filepath.Join(t.TempDir(), fmt.Sprintf("db_%d", rand.Intn(100))) + rootCmd.SetArgs([]string{ + "surgery", "clear-page-elements", srcPath, + "--output", output, + "--pageId", fmt.Sprintf("%d", pageId), + "--from", fmt.Sprintf("%d", startIdx), + "--to", fmt.Sprintf("%d", endIdx), + }) + err = rootCmd.Execute() + if expectError { + require.Error(t, err) + return + } + + require.NoError(t, err) + + // check the element count again + expectedCnt := 0 + if endIdx == -1 { + expectedCnt = startIdx + } else { + expectedCnt = int(elementCount) - (endIdx - startIdx) + } + p, _, err := guts_cli.ReadPage(output, pageId) + require.NoError(t, err) + assert.Equal(t, expectedCnt, int(p.Count())) +} From a9a9356ea4aa43a3ef5ce360001bae704a9a5d13 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Sun, 19 Mar 2023 09:21:32 +0800 Subject: [PATCH 4/8] test: verify the left elements after clear-page-elements operation should have the same content Signed-off-by: Benjamin Wang --- cmd/bbolt/command_surgery_cobra_test.go | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cmd/bbolt/command_surgery_cobra_test.go b/cmd/bbolt/command_surgery_cobra_test.go index 1b054333c..de4aa2b5e 100644 --- a/cmd/bbolt/command_surgery_cobra_test.go +++ b/cmd/bbolt/command_surgery_cobra_test.go @@ -181,4 +181,30 @@ func testSurgeryClearPageElement(t *testing.T, startIdx, endIdx int, setEndIdxAs p, _, err := guts_cli.ReadPage(output, pageId) require.NoError(t, err) assert.Equal(t, expectedCnt, int(p.Count())) + + compareDataAfterClearingElement(t, srcPath, output, pageId, startIdx, endIdx) +} + +func compareDataAfterClearingElement(t *testing.T, srcPath, dstPath string, pageId uint64, startIdx, endIdx int) { + srcPage, _, err := guts_cli.ReadPage(srcPath, pageId) + require.NoError(t, err) + + dstPage, _, err := guts_cli.ReadPage(dstPath, pageId) + require.NoError(t, err) + + var dstIdx uint16 + for i := uint16(0); i < srcPage.Count(); i++ { + // skip the cleared elements + if dstIdx >= uint16(startIdx) && (dstIdx < uint16(endIdx) || endIdx == -1) { + continue + } + srcElement := srcPage.LeafPageElement(i) + dstElement := dstPage.LeafPageElement(dstIdx) + + require.Equal(t, srcElement.Flags(), dstElement.Flags()) + require.Equal(t, srcElement.Key(), dstElement.Key()) + require.Equal(t, srcElement.Value(), dstElement.Value()) + + dstIdx++ + } } From 8902ef92e971732e9ede717c9119b88f3e184c2d Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Wed, 22 Mar 2023 11:29:27 +0800 Subject: [PATCH 5/8] resolve some minor review comments Signed-off-by: Benjamin Wang --- cmd/bbolt/command_surgery_cobra.go | 4 ++-- cmd/bbolt/command_surgery_cobra_test.go | 3 +-- cmd/bbolt/main.go | 4 ++-- internal/surgeon/surgeon.go | 12 +++++++++--- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/cmd/bbolt/command_surgery_cobra.go b/cmd/bbolt/command_surgery_cobra.go index b72cc6f14..e903da0e7 100644 --- a/cmd/bbolt/command_surgery_cobra.go +++ b/cmd/bbolt/command_surgery_cobra.go @@ -31,8 +31,8 @@ func newSurgeryCobraCommand() *cobra.Command { func newSurgeryClearPageElementsCommand() *cobra.Command { clearElementCmd := &cobra.Command{ - Use: "clear-page-elements [options]", - Short: "Clears elements from the given page", + Use: "clear-page-elements [options]", + Short: "Clears elements from the given page, which can be a branch or leaf page", Args: func(cmd *cobra.Command, args []string) error { if len(args) == 0 { return errors.New("db file path not provided") diff --git a/cmd/bbolt/command_surgery_cobra_test.go b/cmd/bbolt/command_surgery_cobra_test.go index de4aa2b5e..4231228f4 100644 --- a/cmd/bbolt/command_surgery_cobra_test.go +++ b/cmd/bbolt/command_surgery_cobra_test.go @@ -2,7 +2,6 @@ package main_test import ( "fmt" - "math/rand" "path/filepath" "testing" @@ -155,7 +154,7 @@ func testSurgeryClearPageElement(t *testing.T, startIdx, endIdx int, setEndIdxAs // clear elements [startIdx, endIdx) in the page rootCmd := main.NewRootCommand() - output := filepath.Join(t.TempDir(), fmt.Sprintf("db_%d", rand.Intn(100))) + output := filepath.Join(t.TempDir(), "db") rootCmd.SetArgs([]string{ "surgery", "clear-page-elements", srcPath, "--output", output, diff --git a/cmd/bbolt/main.go b/cmd/bbolt/main.go index e84b31bf7..413a49a09 100644 --- a/cmd/bbolt/main.go +++ b/cmd/bbolt/main.go @@ -61,14 +61,14 @@ func main() { if err := m.Run(os.Args[1:]...); err == ErrUsage { os.Exit(2) } else if err == ErrUnknownCommand { - execute() + cobraExecute() } else if err != nil { fmt.Println(err.Error()) os.Exit(1) } } -func execute() { +func cobraExecute() { rootCmd := NewRootCommand() if err := rootCmd.Execute(); err != nil { if rootCmd.SilenceErrors { diff --git a/internal/surgeon/surgeon.go b/internal/surgeon/surgeon.go index d2c7be876..5b915d36a 100644 --- a/internal/surgeon/surgeon.go +++ b/internal/surgeon/surgeon.go @@ -20,6 +20,12 @@ func ClearPage(path string, pgId common.Pgid) error { return ClearPageElements(path, pgId, 0, -1) } +// ClearPageElements supports clearing elements in both branch and leaf +// pages. Note the freelist may be cleaned in the meta pages in the following +// two cases, and bbolt needs to scan the db to reconstruct free list. It may +// cause some delay on next startup, depending on the db size. +// 1. Any branch elements are cleared; +// 2. An object saved in overflow pages is cleared; func ClearPageElements(path string, pgId common.Pgid, start, end int) error { // Read the page p, buf, err := guts_cli.ReadPage(path, uint64(pgId)) @@ -93,16 +99,16 @@ func ClearPageElements(path string, pgId common.Pgid, start, end int) error { } func clearFreelist(path string) error { - if err := clearFreelistAt(path, 0); err != nil { + if err := clearFreelistInMetaPage(path, 0); err != nil { return fmt.Errorf("clearFreelist on meta page 0 failed: %w", err) } - if err := clearFreelistAt(path, 1); err != nil { + if err := clearFreelistInMetaPage(path, 1); err != nil { return fmt.Errorf("clearFreelist on meta page 1 failed: %w", err) } return nil } -func clearFreelistAt(path string, pageId uint64) error { +func clearFreelistInMetaPage(path string, pageId uint64) error { _, buf, err := guts_cli.ReadPage(path, pageId) if err != nil { return fmt.Errorf("ReadPage %d failed: %w", pageId, err) From a0e5e3a1dd8b6497d08014bc9d2b64a4cfea5a0b Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Wed, 22 Mar 2023 12:25:05 +0800 Subject: [PATCH 6/8] test: add more subcases to verify 'surgery-clear-elements' on branch page Signed-off-by: Benjamin Wang --- cmd/bbolt/command_surgery_cobra_test.go | 84 +++++++++++++++++++------ 1 file changed, 66 insertions(+), 18 deletions(-) diff --git a/cmd/bbolt/command_surgery_cobra_test.go b/cmd/bbolt/command_surgery_cobra_test.go index 4231228f4..c7e03642d 100644 --- a/cmd/bbolt/command_surgery_cobra_test.go +++ b/cmd/bbolt/command_surgery_cobra_test.go @@ -19,36 +19,68 @@ func TestSurgery_ClearPageElement(t *testing.T) { name string from int to int + isBranchPage bool setEndIdxAsCount bool removeOnlyOneElement bool // only valid when setEndIdxAsCount == true, and startIdx = endIdx -1 in this case. expectError bool }{ - // normal range + // normal range in leaf page { - name: "normal range: [4, 8)", + name: "normal range in leaf page: [4, 8)", from: 4, to: 8, }, { - name: "normal range: [5, -1)", + name: "normal range in leaf page: [5, -1)", from: 4, to: -1, }, { - name: "normal range: all", + name: "normal range in leaf page: all", from: 0, to: -1, }, { - name: "normal range: [0, 7)", + name: "normal range in leaf page: [0, 7)", from: 0, to: 7, }, { - name: "normal range: [3, count)", + name: "normal range in leaf page: [3, count)", from: 4, setEndIdxAsCount: true, }, + // normal range in branch page + { + name: "normal range in branch page: [4, 8)", + from: 4, + to: 8, + isBranchPage: true, + }, + { + name: "normal range in branch page: [5, -1)", + from: 4, + to: -1, + isBranchPage: true, + }, + { + name: "normal range in branch page: all", + from: 0, + to: -1, + isBranchPage: true, + }, + { + name: "normal range in branch page: [0, 7)", + from: 0, + to: 7, + isBranchPage: true, + }, + { + name: "normal range in branch page: [3, count)", + from: 4, + isBranchPage: true, + setEndIdxAsCount: true, + }, // remove only one element { name: "one element: the first one", @@ -106,12 +138,12 @@ func TestSurgery_ClearPageElement(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - testSurgeryClearPageElement(t, tc.from, tc.to, tc.setEndIdxAsCount, tc.removeOnlyOneElement, tc.expectError) + testSurgeryClearPageElement(t, tc.from, tc.to, tc.isBranchPage, tc.setEndIdxAsCount, tc.removeOnlyOneElement, tc.expectError) }) } } -func testSurgeryClearPageElement(t *testing.T, startIdx, endIdx int, setEndIdxAsCount, removeOnlyOne, expectError bool) { +func testSurgeryClearPageElement(t *testing.T, startIdx, endIdx int, isBranchPage, setEndIdxAsCount, removeOnlyOne, expectError bool) { pageSize := 4096 db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) srcPath := db.Path() @@ -135,9 +167,16 @@ func testSurgeryClearPageElement(t *testing.T, startIdx, endIdx int, setEndIdxAs p, _, err := guts_cli.ReadPage(srcPath, pageId) require.NoError(t, err) - if p.IsLeafPage() && p.Count() > 10 { - elementCount = p.Count() - break + if isBranchPage { + if p.IsBranchPage() && p.Count() > 10 { + elementCount = p.Count() + break + } + } else { + if p.IsLeafPage() && p.Count() > 10 { + elementCount = p.Count() + break + } } pageId++ } @@ -181,10 +220,10 @@ func testSurgeryClearPageElement(t *testing.T, startIdx, endIdx int, setEndIdxAs require.NoError(t, err) assert.Equal(t, expectedCnt, int(p.Count())) - compareDataAfterClearingElement(t, srcPath, output, pageId, startIdx, endIdx) + compareDataAfterClearingElement(t, srcPath, output, pageId, isBranchPage, startIdx, endIdx) } -func compareDataAfterClearingElement(t *testing.T, srcPath, dstPath string, pageId uint64, startIdx, endIdx int) { +func compareDataAfterClearingElement(t *testing.T, srcPath, dstPath string, pageId uint64, isBranchPage bool, startIdx, endIdx int) { srcPage, _, err := guts_cli.ReadPage(srcPath, pageId) require.NoError(t, err) @@ -197,12 +236,21 @@ func compareDataAfterClearingElement(t *testing.T, srcPath, dstPath string, page if dstIdx >= uint16(startIdx) && (dstIdx < uint16(endIdx) || endIdx == -1) { continue } - srcElement := srcPage.LeafPageElement(i) - dstElement := dstPage.LeafPageElement(dstIdx) - require.Equal(t, srcElement.Flags(), dstElement.Flags()) - require.Equal(t, srcElement.Key(), dstElement.Key()) - require.Equal(t, srcElement.Value(), dstElement.Value()) + if isBranchPage { + srcElement := srcPage.BranchPageElement(i) + dstElement := dstPage.BranchPageElement(dstIdx) + + require.Equal(t, srcElement.Key(), dstElement.Key()) + require.Equal(t, srcElement.Pgid(), dstElement.Pgid()) + } else { + srcElement := srcPage.LeafPageElement(i) + dstElement := dstPage.LeafPageElement(dstIdx) + + require.Equal(t, srcElement.Flags(), dstElement.Flags()) + require.Equal(t, srcElement.Key(), dstElement.Key()) + require.Equal(t, srcElement.Value(), dstElement.Value()) + } dstIdx++ } From 87eed0ac93871accb709518440b9488a3acdc047 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Wed, 22 Mar 2023 15:59:29 +0800 Subject: [PATCH 7/8] add test case to verify 'surgery-clear-elements' on overflow page Also resolved a bug related to overflow page. Signed-off-by: Benjamin Wang --- cmd/bbolt/command_surgery_cobra_test.go | 179 +++++++++++++++++++++++- internal/common/inode.go | 10 ++ internal/surgeon/surgeon.go | 38 ++--- 3 files changed, 207 insertions(+), 20 deletions(-) diff --git a/cmd/bbolt/command_surgery_cobra_test.go b/cmd/bbolt/command_surgery_cobra_test.go index c7e03642d..653a67989 100644 --- a/cmd/bbolt/command_surgery_cobra_test.go +++ b/cmd/bbolt/command_surgery_cobra_test.go @@ -14,7 +14,7 @@ import ( "go.etcd.io/bbolt/internal/guts_cli" ) -func TestSurgery_ClearPageElement(t *testing.T) { +func TestSurgery_ClearPageElements_Without_Overflow(t *testing.T) { testCases := []struct { name string from int @@ -138,12 +138,12 @@ func TestSurgery_ClearPageElement(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - testSurgeryClearPageElement(t, tc.from, tc.to, tc.isBranchPage, tc.setEndIdxAsCount, tc.removeOnlyOneElement, tc.expectError) + testSurgeryClearPageElementsWithoutOverflow(t, tc.from, tc.to, tc.isBranchPage, tc.setEndIdxAsCount, tc.removeOnlyOneElement, tc.expectError) }) } } -func testSurgeryClearPageElement(t *testing.T, startIdx, endIdx int, isBranchPage, setEndIdxAsCount, removeOnlyOne, expectError bool) { +func testSurgeryClearPageElementsWithoutOverflow(t *testing.T, startIdx, endIdx int, isBranchPage, setEndIdxAsCount, removeOnlyOne, expectError bool) { pageSize := 4096 db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) srcPath := db.Path() @@ -255,3 +255,176 @@ func compareDataAfterClearingElement(t *testing.T, srcPath, dstPath string, page dstIdx++ } } + +func TestSurgery_ClearPageElements_With_Overflow(t *testing.T) { + testCases := []struct { + name string + from int + to int + valueSizes []int + expectedOverflow int + }{ + // big element + { + name: "remove a big element at the end", + valueSizes: []int{500, 500, 500, 2600}, + from: 3, + to: 4, + expectedOverflow: 0, + }, + { + name: "remove a big element at the begin", + valueSizes: []int{2600, 500, 500, 500}, + from: 0, + to: 1, + expectedOverflow: 0, + }, + { + name: "remove a big element in the middle", + valueSizes: []int{500, 2600, 500, 500}, + from: 1, + to: 2, + expectedOverflow: 0, + }, + // small element + { + name: "remove a small element at the end", + valueSizes: []int{500, 500, 3100, 100}, + from: 3, + to: 4, + expectedOverflow: 1, + }, + { + name: "remove a small element at the begin", + valueSizes: []int{100, 500, 3100, 500}, + from: 0, + to: 1, + expectedOverflow: 1, + }, + { + name: "remove a small element in the middle", + valueSizes: []int{500, 100, 3100, 500}, + from: 1, + to: 2, + expectedOverflow: 1, + }, + { + name: "remove a small element at the end of page with big overflow", + valueSizes: []int{500, 500, 4096 * 5, 100}, + from: 3, + to: 4, + expectedOverflow: 5, + }, + { + name: "remove a small element at the begin of page with big overflow", + valueSizes: []int{100, 500, 4096 * 6, 500}, + from: 0, + to: 1, + expectedOverflow: 6, + }, + { + name: "remove a small element in the middle of page with big overflow", + valueSizes: []int{500, 100, 4096 * 4, 500}, + from: 1, + to: 2, + expectedOverflow: 4, + }, + // huge element + { + name: "remove a huge element at the end", + valueSizes: []int{500, 500, 500, 4096 * 5}, + from: 3, + to: 4, + expectedOverflow: 0, + }, + { + name: "remove a huge element at the begin", + valueSizes: []int{4096 * 5, 500, 500, 500}, + from: 0, + to: 1, + expectedOverflow: 0, + }, + { + name: "remove a huge element in the middle", + valueSizes: []int{500, 4096 * 5, 500, 500}, + from: 1, + to: 2, + expectedOverflow: 0, + }, + } + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + testSurgeryClearPageElementsWithOverflow(t, tc.from, tc.to, tc.valueSizes, tc.expectedOverflow) + }) + } +} + +func testSurgeryClearPageElementsWithOverflow(t *testing.T, startIdx, endIdx int, valueSizes []int, expectedOverflow int) { + pageSize := 4096 + db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) + srcPath := db.Path() + + // Generate sample db + err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists([]byte("data")) + for i, valueSize := range valueSizes { + key := []byte(fmt.Sprintf("%04d", i)) + val := make([]byte, valueSize) + if putErr := b.Put(key, val); putErr != nil { + return putErr + } + } + return nil + }) + require.NoError(t, err) + + defer requireDBNoChange(t, dbData(t, srcPath), srcPath) + + // find a page with overflow pages + var ( + pageId uint64 = 2 + elementCount uint16 = 0 + ) + for { + p, _, err := guts_cli.ReadPage(srcPath, pageId) + require.NoError(t, err) + + if p.Overflow() > 0 { + elementCount = p.Count() + break + } + pageId++ + } + t.Logf("The original element count: %d", elementCount) + + // clear elements [startIdx, endIdx) in the page + rootCmd := main.NewRootCommand() + output := filepath.Join(t.TempDir(), "db") + rootCmd.SetArgs([]string{ + "surgery", "clear-page-elements", srcPath, + "--output", output, + "--pageId", fmt.Sprintf("%d", pageId), + "--from", fmt.Sprintf("%d", startIdx), + "--to", fmt.Sprintf("%d", endIdx), + }) + err = rootCmd.Execute() + require.NoError(t, err) + + // check the element count again + expectedCnt := 0 + if endIdx == -1 { + expectedCnt = startIdx + } else { + expectedCnt = int(elementCount) - (endIdx - startIdx) + } + p, _, err := guts_cli.ReadPage(output, pageId) + require.NoError(t, err) + assert.Equal(t, expectedCnt, int(p.Count())) + + assert.Equal(t, expectedOverflow, int(p.Overflow())) + + compareDataAfterClearingElement(t, srcPath, output, pageId, false, startIdx, endIdx) +} diff --git a/internal/common/inode.go b/internal/common/inode.go index 9f99937e7..080b9af78 100644 --- a/internal/common/inode.go +++ b/internal/common/inode.go @@ -103,3 +103,13 @@ func WriteInodeToPage(inodes Inodes, p *Page) uint32 { return uint32(off) } + +func UsedSpaceInPage(inodes Inodes, p *Page) uint32 { + off := unsafe.Sizeof(*p) + p.PageElementSize()*uintptr(len(inodes)) + for _, item := range inodes { + sz := len(item.Key()) + len(item.Value()) + off += uintptr(sz) + } + + return uint32(off) +} diff --git a/internal/surgeon/surgeon.go b/internal/surgeon/surgeon.go index 5b915d36a..3d556e3a2 100644 --- a/internal/surgeon/surgeon.go +++ b/internal/surgeon/surgeon.go @@ -61,33 +61,37 @@ func ClearPageElements(path string, pgId common.Pgid, start, end int) error { preOverflow := p.Overflow() + var ( + dataWritten uint32 + ) if end == int(p.Count()) || end == -1 { + inodes := common.ReadInodeFromPage(p) + inodes = inodes[:start] + p.SetCount(uint16(start)) - p.SetOverflow(0) - if preOverflow != 0 || p.IsBranchPage() { - if err := clearFreelist(path); err != nil { - return err - } - } + // no need to write inode & data again, we just need to get + // the data size which will be kept. + dataWritten = common.UsedSpaceInPage(inodes, p) } else { inodes := common.ReadInodeFromPage(p) inodes = append(inodes[:start], inodes[end:]...) p.SetCount(uint16(len(inodes))) - dataWritten := common.WriteInodeToPage(inodes, p) + dataWritten = common.WriteInodeToPage(inodes, p) + } - pageSize, _, err := guts_cli.ReadPageAndHWMSize(path) - if err != nil { - return fmt.Errorf("ReadPageAndHWMSize failed: %w", err) - } - if dataWritten%uint32(pageSize) == 0 { - p.SetOverflow(dataWritten/uint32(pageSize) - 1) - } else { - p.SetOverflow(dataWritten / uint32(pageSize)) - } + pageSize, _, err := guts_cli.ReadPageAndHWMSize(path) + if err != nil { + return fmt.Errorf("ReadPageAndHWMSize failed: %w", err) + } + if dataWritten%uint32(pageSize) == 0 { + p.SetOverflow(dataWritten/uint32(pageSize) - 1) + } else { + p.SetOverflow(dataWritten / uint32(pageSize)) } - if err := guts_cli.WritePage(path, buf); err != nil { + datasz := pageSize * (uint64(p.Overflow()) + 1) + if err := guts_cli.WritePage(path, buf[0:datasz]); err != nil { return fmt.Errorf("WritePage failed: %w", err) } From 74e26bee7782a7ddd79c2f9936ea2cccea0513c3 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Sat, 25 Mar 2023 08:16:43 +0800 Subject: [PATCH 8/8] update command 'surgery clear-page-elements' not to automatically abandon freelist Signed-off-by: Benjamin Wang --- cmd/bbolt/command_surgery_cobra.go | 12 +++++-- cmd/bbolt/command_surgery_cobra_test.go | 8 ++--- cmd/bbolt/surgery_commands.go | 9 ++++- internal/surgeon/surgeon.go | 44 +++++++++++++++---------- 4 files changed, 48 insertions(+), 25 deletions(-) diff --git a/cmd/bbolt/command_surgery_cobra.go b/cmd/bbolt/command_surgery_cobra.go index e903da0e7..407d7eccd 100644 --- a/cmd/bbolt/command_surgery_cobra.go +++ b/cmd/bbolt/command_surgery_cobra.go @@ -48,8 +48,8 @@ func newSurgeryClearPageElementsCommand() *cobra.Command { clearElementCmd.Flags().StringVar(&surgeryTargetDBFilePath, "output", "", "path to the target db file") clearElementCmd.Flags().Uint64VarP(&surgeryPageId, "pageId", "", 0, "page id") - clearElementCmd.Flags().IntVarP(&surgeryStartElementIdx, "from", "", 0, "start element index (included) to clear, starting from 0") - clearElementCmd.Flags().IntVarP(&surgeryEndElementIdx, "to", "", 0, "end element index (excluded) to clear, starting from 0, -1 means to the end of page") + clearElementCmd.Flags().IntVarP(&surgeryStartElementIdx, "from-index", "", 0, "start element index (included) to clear, starting from 0") + clearElementCmd.Flags().IntVarP(&surgeryEndElementIdx, "to-index", "", 0, "end element index (excluded) to clear, starting from 0, -1 means to the end of page") return clearElementCmd } @@ -65,10 +65,16 @@ func surgeryClearPageElementFunc(cmd *cobra.Command, args []string) error { return fmt.Errorf("the pageId must be at least 2, but got %d", surgeryPageId) } - if err := surgeon.ClearPageElements(surgeryTargetDBFilePath, common.Pgid(surgeryPageId), surgeryStartElementIdx, surgeryEndElementIdx); err != nil { + needAbandonFreelist, err := surgeon.ClearPageElements(surgeryTargetDBFilePath, common.Pgid(surgeryPageId), surgeryStartElementIdx, surgeryEndElementIdx, false) + if err != nil { return fmt.Errorf("clear-page-element command failed: %w", err) } + if needAbandonFreelist { + fmt.Fprintf(os.Stdout, "WARNING: The clearing has abandoned some pages that are not yet referenced from free list.\n") + fmt.Fprintf(os.Stdout, "Please consider executing `./bbolt surgery abandon-freelist ...`\n") + } + fmt.Fprintf(os.Stdout, "All elements in [%d, %d) in page %d were cleared\n", surgeryStartElementIdx, surgeryEndElementIdx, surgeryPageId) return nil } diff --git a/cmd/bbolt/command_surgery_cobra_test.go b/cmd/bbolt/command_surgery_cobra_test.go index 653a67989..3016a963f 100644 --- a/cmd/bbolt/command_surgery_cobra_test.go +++ b/cmd/bbolt/command_surgery_cobra_test.go @@ -198,8 +198,8 @@ func testSurgeryClearPageElementsWithoutOverflow(t *testing.T, startIdx, endIdx "surgery", "clear-page-elements", srcPath, "--output", output, "--pageId", fmt.Sprintf("%d", pageId), - "--from", fmt.Sprintf("%d", startIdx), - "--to", fmt.Sprintf("%d", endIdx), + "--from-index", fmt.Sprintf("%d", startIdx), + "--to-index", fmt.Sprintf("%d", endIdx), }) err = rootCmd.Execute() if expectError { @@ -407,8 +407,8 @@ func testSurgeryClearPageElementsWithOverflow(t *testing.T, startIdx, endIdx int "surgery", "clear-page-elements", srcPath, "--output", output, "--pageId", fmt.Sprintf("%d", pageId), - "--from", fmt.Sprintf("%d", startIdx), - "--to", fmt.Sprintf("%d", endIdx), + "--from-index", fmt.Sprintf("%d", startIdx), + "--to-index", fmt.Sprintf("%d", endIdx), }) err = rootCmd.Execute() require.NoError(t, err) diff --git a/cmd/bbolt/surgery_commands.go b/cmd/bbolt/surgery_commands.go index 652f21484..07b128819 100644 --- a/cmd/bbolt/surgery_commands.go +++ b/cmd/bbolt/surgery_commands.go @@ -4,6 +4,7 @@ import ( "errors" "flag" "fmt" + "os" "strconv" "strings" @@ -233,10 +234,16 @@ func (cmd *clearPageCommand) Run(args ...string) error { return err } - if err := surgeon.ClearPage(cmd.dstPath, common.Pgid(pageId)); err != nil { + needAbandonFreelist, err := surgeon.ClearPage(cmd.dstPath, common.Pgid(pageId)) + if err != nil { return fmt.Errorf("clearPageCommand failed: %w", err) } + if needAbandonFreelist { + fmt.Fprintf(os.Stdout, "WARNING: The clearing has abandoned some pages that are not yet referenced from free list.\n") + fmt.Fprintf(os.Stdout, "Please consider executing `./bbolt surgery abandon-freelist ...`\n") + } + fmt.Fprintf(cmd.Stdout, "Page (%d) was cleared\n", pageId) return nil } diff --git a/internal/surgeon/surgeon.go b/internal/surgeon/surgeon.go index 3d556e3a2..e69eb7234 100644 --- a/internal/surgeon/surgeon.go +++ b/internal/surgeon/surgeon.go @@ -16,47 +16,54 @@ func CopyPage(path string, srcPage common.Pgid, target common.Pgid) error { return guts_cli.WritePage(path, d1) } -func ClearPage(path string, pgId common.Pgid) error { - return ClearPageElements(path, pgId, 0, -1) +func ClearPage(path string, pgId common.Pgid) (bool, error) { + return ClearPageElements(path, pgId, 0, -1, false) } // ClearPageElements supports clearing elements in both branch and leaf -// pages. Note the freelist may be cleaned in the meta pages in the following -// two cases, and bbolt needs to scan the db to reconstruct free list. It may -// cause some delay on next startup, depending on the db size. +// pages. Note if the ${abandonFreelist} is true, the freelist may be cleaned +// in the meta pages in the following two cases, and bbolt needs to scan the +// db to reconstruct free list. It may cause some delay on next startup, +// depending on the db size. // 1. Any branch elements are cleared; // 2. An object saved in overflow pages is cleared; -func ClearPageElements(path string, pgId common.Pgid, start, end int) error { +// +// Usually ${abandonFreelist} defaults to false, it means it will not clear the +// freelist in meta pages automatically. Users will receive a warning message +// to remind them to explicitly execute `bbolt surgery abandom-freelist` +// afterwards; the first return parameter will be true in such case. But if +// the freelist isn't synced at all, no warning message will be displayed. +func ClearPageElements(path string, pgId common.Pgid, start, end int, abandonFreelist bool) (bool, error) { // Read the page p, buf, err := guts_cli.ReadPage(path, uint64(pgId)) if err != nil { - return fmt.Errorf("ReadPage failed: %w", err) + return false, fmt.Errorf("ReadPage failed: %w", err) } if !p.IsLeafPage() && !p.IsBranchPage() { - return fmt.Errorf("can't clear elements in %q page", p.Typ()) + return false, fmt.Errorf("can't clear elements in %q page", p.Typ()) } elementCnt := int(p.Count()) if elementCnt == 0 { - return nil + return false, nil } if start < 0 || start >= elementCnt { - return fmt.Errorf("the start index (%d) is out of range [0, %d)", start, elementCnt) + return false, fmt.Errorf("the start index (%d) is out of range [0, %d)", start, elementCnt) } if (end < 0 || end > elementCnt) && end != -1 { - return fmt.Errorf("the end index (%d) is out of range [0, %d]", end, elementCnt) + return false, fmt.Errorf("the end index (%d) is out of range [0, %d]", end, elementCnt) } if start > end && end != -1 { - return fmt.Errorf("the start index (%d) is bigger than the end index (%d)", start, end) + return false, fmt.Errorf("the start index (%d) is bigger than the end index (%d)", start, end) } if start == end { - return fmt.Errorf("invalid: the start index (%d) is equal to the end index (%d)", start, end) + return false, fmt.Errorf("invalid: the start index (%d) is equal to the end index (%d)", start, end) } preOverflow := p.Overflow() @@ -82,7 +89,7 @@ func ClearPageElements(path string, pgId common.Pgid, start, end int) error { pageSize, _, err := guts_cli.ReadPageAndHWMSize(path) if err != nil { - return fmt.Errorf("ReadPageAndHWMSize failed: %w", err) + return false, fmt.Errorf("ReadPageAndHWMSize failed: %w", err) } if dataWritten%uint32(pageSize) == 0 { p.SetOverflow(dataWritten/uint32(pageSize) - 1) @@ -92,14 +99,17 @@ func ClearPageElements(path string, pgId common.Pgid, start, end int) error { datasz := pageSize * (uint64(p.Overflow()) + 1) if err := guts_cli.WritePage(path, buf[0:datasz]); err != nil { - return fmt.Errorf("WritePage failed: %w", err) + return false, fmt.Errorf("WritePage failed: %w", err) } if preOverflow != p.Overflow() || p.IsBranchPage() { - return clearFreelist(path) + if abandonFreelist { + return false, clearFreelist(path) + } + return true, nil } - return nil + return false, nil } func clearFreelist(path string) error {