diff --git a/cmd/bbolt/command_root.go b/cmd/bbolt/command_root.go new file mode 100644 index 000000000..b960df898 --- /dev/null +++ b/cmd/bbolt/command_root.go @@ -0,0 +1,24 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +const ( + cliName = "bbolt" + cliDescription = "A simple command line tool for inspecting bbolt databases" +) + +func NewRootCommand() *cobra.Command { + rootCmd := &cobra.Command{ + Use: cliName, + Short: cliDescription, + Version: "dev", + } + + rootCmd.AddCommand( + newSurgeryCobraCommand(), + ) + + return rootCmd +} diff --git a/cmd/bbolt/command_surgery_cobra.go b/cmd/bbolt/command_surgery_cobra.go new file mode 100644 index 000000000..407d7eccd --- /dev/null +++ b/cmd/bbolt/command_surgery_cobra.go @@ -0,0 +1,80 @@ +package main + +import ( + "errors" + "fmt" + "os" + + "github.com/spf13/cobra" + + "go.etcd.io/bbolt/internal/common" + "go.etcd.io/bbolt/internal/surgeon" +) + +var ( + surgeryTargetDBFilePath string + surgeryPageId uint64 + surgeryStartElementIdx int + surgeryEndElementIdx int +) + +func newSurgeryCobraCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "surgery ", + Short: "surgery related commands", + } + + cmd.AddCommand(newSurgeryClearPageElementsCommand()) + + return cmd +} + +func newSurgeryClearPageElementsCommand() *cobra.Command { + clearElementCmd := &cobra.Command{ + Use: "clear-page-elements [options]", + Short: "Clears elements from the given page, which can be a branch or leaf page", + Args: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("db file path not provided") + } + if len(args) > 1 { + return errors.New("too many arguments") + } + return nil + }, + + RunE: surgeryClearPageElementFunc, + } + + clearElementCmd.Flags().StringVar(&surgeryTargetDBFilePath, "output", "", "path to the target db file") + clearElementCmd.Flags().Uint64VarP(&surgeryPageId, "pageId", "", 0, "page id") + clearElementCmd.Flags().IntVarP(&surgeryStartElementIdx, "from-index", "", 0, "start element index (included) to clear, starting from 0") + clearElementCmd.Flags().IntVarP(&surgeryEndElementIdx, "to-index", "", 0, "end element index (excluded) to clear, starting from 0, -1 means to the end of page") + + return clearElementCmd +} + +func surgeryClearPageElementFunc(cmd *cobra.Command, args []string) error { + srcDBPath := args[0] + + if err := copyFile(srcDBPath, surgeryTargetDBFilePath); err != nil { + return fmt.Errorf("[clear-page-element] copy file failed: %w", err) + } + + if surgeryPageId < 2 { + return fmt.Errorf("the pageId must be at least 2, but got %d", surgeryPageId) + } + + needAbandonFreelist, err := surgeon.ClearPageElements(surgeryTargetDBFilePath, common.Pgid(surgeryPageId), surgeryStartElementIdx, surgeryEndElementIdx, false) + if err != nil { + return fmt.Errorf("clear-page-element command failed: %w", err) + } + + if needAbandonFreelist { + fmt.Fprintf(os.Stdout, "WARNING: The clearing has abandoned some pages that are not yet referenced from free list.\n") + fmt.Fprintf(os.Stdout, "Please consider executing `./bbolt surgery abandon-freelist ...`\n") + } + + fmt.Fprintf(os.Stdout, "All elements in [%d, %d) in page %d were cleared\n", surgeryStartElementIdx, surgeryEndElementIdx, surgeryPageId) + return nil +} diff --git a/cmd/bbolt/command_surgery_cobra_test.go b/cmd/bbolt/command_surgery_cobra_test.go new file mode 100644 index 000000000..3016a963f --- /dev/null +++ b/cmd/bbolt/command_surgery_cobra_test.go @@ -0,0 +1,430 @@ +package main_test + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + bolt "go.etcd.io/bbolt" + main "go.etcd.io/bbolt/cmd/bbolt" + "go.etcd.io/bbolt/internal/btesting" + "go.etcd.io/bbolt/internal/guts_cli" +) + +func TestSurgery_ClearPageElements_Without_Overflow(t *testing.T) { + testCases := []struct { + name string + from int + to int + isBranchPage bool + setEndIdxAsCount bool + removeOnlyOneElement bool // only valid when setEndIdxAsCount == true, and startIdx = endIdx -1 in this case. + expectError bool + }{ + // normal range in leaf page + { + name: "normal range in leaf page: [4, 8)", + from: 4, + to: 8, + }, + { + name: "normal range in leaf page: [5, -1)", + from: 4, + to: -1, + }, + { + name: "normal range in leaf page: all", + from: 0, + to: -1, + }, + { + name: "normal range in leaf page: [0, 7)", + from: 0, + to: 7, + }, + { + name: "normal range in leaf page: [3, count)", + from: 4, + setEndIdxAsCount: true, + }, + // normal range in branch page + { + name: "normal range in branch page: [4, 8)", + from: 4, + to: 8, + isBranchPage: true, + }, + { + name: "normal range in branch page: [5, -1)", + from: 4, + to: -1, + isBranchPage: true, + }, + { + name: "normal range in branch page: all", + from: 0, + to: -1, + isBranchPage: true, + }, + { + name: "normal range in branch page: [0, 7)", + from: 0, + to: 7, + isBranchPage: true, + }, + { + name: "normal range in branch page: [3, count)", + from: 4, + isBranchPage: true, + setEndIdxAsCount: true, + }, + // remove only one element + { + name: "one element: the first one", + from: 0, + to: 1, + }, + { + name: "one element: [6, 7)", + from: 6, + to: 7, + }, + { + name: "one element: the last one", + setEndIdxAsCount: true, + removeOnlyOneElement: true, + }, + // abnormal range + { + name: "abnormal range: [-1, 4)", + from: -1, + to: 4, + expectError: true, + }, + { + name: "abnormal range: [-2, 5)", + from: -1, + to: 5, + expectError: true, + }, + { + name: "abnormal range: [3, 3)", + from: 3, + to: 3, + expectError: true, + }, + { + name: "abnormal range: [5, 3)", + from: 5, + to: 3, + expectError: true, + }, + { + name: "abnormal range: [3, -2)", + from: 3, + to: -2, + expectError: true, + }, + { + name: "abnormal range: [3, 1000000)", + from: -1, + to: 4, + expectError: true, + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + testSurgeryClearPageElementsWithoutOverflow(t, tc.from, tc.to, tc.isBranchPage, tc.setEndIdxAsCount, tc.removeOnlyOneElement, tc.expectError) + }) + } +} + +func testSurgeryClearPageElementsWithoutOverflow(t *testing.T, startIdx, endIdx int, isBranchPage, setEndIdxAsCount, removeOnlyOne, expectError bool) { + pageSize := 4096 + db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) + srcPath := db.Path() + + // Generate sample db + t.Log("Generate some sample data") + err := db.Fill([]byte("data"), 10, 200, + func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", tx*10000+k)) }, + func(tx int, k int) []byte { return make([]byte, 10) }, + ) + require.NoError(t, err) + + defer requireDBNoChange(t, dbData(t, srcPath), srcPath) + + // find a page with at least 10 elements + var ( + pageId uint64 = 2 + elementCount uint16 = 0 + ) + for { + p, _, err := guts_cli.ReadPage(srcPath, pageId) + require.NoError(t, err) + + if isBranchPage { + if p.IsBranchPage() && p.Count() > 10 { + elementCount = p.Count() + break + } + } else { + if p.IsLeafPage() && p.Count() > 10 { + elementCount = p.Count() + break + } + } + pageId++ + } + t.Logf("The original element count: %d", elementCount) + + if setEndIdxAsCount { + t.Logf("Set the endIdx as the element count: %d", elementCount) + endIdx = int(elementCount) + if removeOnlyOne { + startIdx = endIdx - 1 + t.Logf("Set the startIdx as the endIdx-1: %d", startIdx) + } + } + + // clear elements [startIdx, endIdx) in the page + rootCmd := main.NewRootCommand() + output := filepath.Join(t.TempDir(), "db") + rootCmd.SetArgs([]string{ + "surgery", "clear-page-elements", srcPath, + "--output", output, + "--pageId", fmt.Sprintf("%d", pageId), + "--from-index", fmt.Sprintf("%d", startIdx), + "--to-index", fmt.Sprintf("%d", endIdx), + }) + err = rootCmd.Execute() + if expectError { + require.Error(t, err) + return + } + + require.NoError(t, err) + + // check the element count again + expectedCnt := 0 + if endIdx == -1 { + expectedCnt = startIdx + } else { + expectedCnt = int(elementCount) - (endIdx - startIdx) + } + p, _, err := guts_cli.ReadPage(output, pageId) + require.NoError(t, err) + assert.Equal(t, expectedCnt, int(p.Count())) + + compareDataAfterClearingElement(t, srcPath, output, pageId, isBranchPage, startIdx, endIdx) +} + +func compareDataAfterClearingElement(t *testing.T, srcPath, dstPath string, pageId uint64, isBranchPage bool, startIdx, endIdx int) { + srcPage, _, err := guts_cli.ReadPage(srcPath, pageId) + require.NoError(t, err) + + dstPage, _, err := guts_cli.ReadPage(dstPath, pageId) + require.NoError(t, err) + + var dstIdx uint16 + for i := uint16(0); i < srcPage.Count(); i++ { + // skip the cleared elements + if dstIdx >= uint16(startIdx) && (dstIdx < uint16(endIdx) || endIdx == -1) { + continue + } + + if isBranchPage { + srcElement := srcPage.BranchPageElement(i) + dstElement := dstPage.BranchPageElement(dstIdx) + + require.Equal(t, srcElement.Key(), dstElement.Key()) + require.Equal(t, srcElement.Pgid(), dstElement.Pgid()) + } else { + srcElement := srcPage.LeafPageElement(i) + dstElement := dstPage.LeafPageElement(dstIdx) + + require.Equal(t, srcElement.Flags(), dstElement.Flags()) + require.Equal(t, srcElement.Key(), dstElement.Key()) + require.Equal(t, srcElement.Value(), dstElement.Value()) + } + + dstIdx++ + } +} + +func TestSurgery_ClearPageElements_With_Overflow(t *testing.T) { + testCases := []struct { + name string + from int + to int + valueSizes []int + expectedOverflow int + }{ + // big element + { + name: "remove a big element at the end", + valueSizes: []int{500, 500, 500, 2600}, + from: 3, + to: 4, + expectedOverflow: 0, + }, + { + name: "remove a big element at the begin", + valueSizes: []int{2600, 500, 500, 500}, + from: 0, + to: 1, + expectedOverflow: 0, + }, + { + name: "remove a big element in the middle", + valueSizes: []int{500, 2600, 500, 500}, + from: 1, + to: 2, + expectedOverflow: 0, + }, + // small element + { + name: "remove a small element at the end", + valueSizes: []int{500, 500, 3100, 100}, + from: 3, + to: 4, + expectedOverflow: 1, + }, + { + name: "remove a small element at the begin", + valueSizes: []int{100, 500, 3100, 500}, + from: 0, + to: 1, + expectedOverflow: 1, + }, + { + name: "remove a small element in the middle", + valueSizes: []int{500, 100, 3100, 500}, + from: 1, + to: 2, + expectedOverflow: 1, + }, + { + name: "remove a small element at the end of page with big overflow", + valueSizes: []int{500, 500, 4096 * 5, 100}, + from: 3, + to: 4, + expectedOverflow: 5, + }, + { + name: "remove a small element at the begin of page with big overflow", + valueSizes: []int{100, 500, 4096 * 6, 500}, + from: 0, + to: 1, + expectedOverflow: 6, + }, + { + name: "remove a small element in the middle of page with big overflow", + valueSizes: []int{500, 100, 4096 * 4, 500}, + from: 1, + to: 2, + expectedOverflow: 4, + }, + // huge element + { + name: "remove a huge element at the end", + valueSizes: []int{500, 500, 500, 4096 * 5}, + from: 3, + to: 4, + expectedOverflow: 0, + }, + { + name: "remove a huge element at the begin", + valueSizes: []int{4096 * 5, 500, 500, 500}, + from: 0, + to: 1, + expectedOverflow: 0, + }, + { + name: "remove a huge element in the middle", + valueSizes: []int{500, 4096 * 5, 500, 500}, + from: 1, + to: 2, + expectedOverflow: 0, + }, + } + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + testSurgeryClearPageElementsWithOverflow(t, tc.from, tc.to, tc.valueSizes, tc.expectedOverflow) + }) + } +} + +func testSurgeryClearPageElementsWithOverflow(t *testing.T, startIdx, endIdx int, valueSizes []int, expectedOverflow int) { + pageSize := 4096 + db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) + srcPath := db.Path() + + // Generate sample db + err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists([]byte("data")) + for i, valueSize := range valueSizes { + key := []byte(fmt.Sprintf("%04d", i)) + val := make([]byte, valueSize) + if putErr := b.Put(key, val); putErr != nil { + return putErr + } + } + return nil + }) + require.NoError(t, err) + + defer requireDBNoChange(t, dbData(t, srcPath), srcPath) + + // find a page with overflow pages + var ( + pageId uint64 = 2 + elementCount uint16 = 0 + ) + for { + p, _, err := guts_cli.ReadPage(srcPath, pageId) + require.NoError(t, err) + + if p.Overflow() > 0 { + elementCount = p.Count() + break + } + pageId++ + } + t.Logf("The original element count: %d", elementCount) + + // clear elements [startIdx, endIdx) in the page + rootCmd := main.NewRootCommand() + output := filepath.Join(t.TempDir(), "db") + rootCmd.SetArgs([]string{ + "surgery", "clear-page-elements", srcPath, + "--output", output, + "--pageId", fmt.Sprintf("%d", pageId), + "--from-index", fmt.Sprintf("%d", startIdx), + "--to-index", fmt.Sprintf("%d", endIdx), + }) + err = rootCmd.Execute() + require.NoError(t, err) + + // check the element count again + expectedCnt := 0 + if endIdx == -1 { + expectedCnt = startIdx + } else { + expectedCnt = int(elementCount) - (endIdx - startIdx) + } + p, _, err := guts_cli.ReadPage(output, pageId) + require.NoError(t, err) + assert.Equal(t, expectedCnt, int(p.Count())) + + assert.Equal(t, expectedOverflow, int(p.Overflow())) + + compareDataAfterClearingElement(t, srcPath, output, pageId, false, startIdx, endIdx) +} diff --git a/cmd/bbolt/main.go b/cmd/bbolt/main.go index 7afb059f6..413a49a09 100644 --- a/cmd/bbolt/main.go +++ b/cmd/bbolt/main.go @@ -60,12 +60,26 @@ func main() { m := NewMain() if err := m.Run(os.Args[1:]...); err == ErrUsage { os.Exit(2) + } else if err == ErrUnknownCommand { + cobraExecute() } else if err != nil { fmt.Println(err.Error()) os.Exit(1) } } +func cobraExecute() { + rootCmd := NewRootCommand() + if err := rootCmd.Execute(); err != nil { + if rootCmd.SilenceErrors { + fmt.Fprintln(os.Stderr, "Error:", err) + os.Exit(1) + } else { + os.Exit(1) + } + } +} + type baseCommand struct { Stdin io.Reader Stdout io.Writer diff --git a/cmd/bbolt/surgery_commands.go b/cmd/bbolt/surgery_commands.go index 652f21484..07b128819 100644 --- a/cmd/bbolt/surgery_commands.go +++ b/cmd/bbolt/surgery_commands.go @@ -4,6 +4,7 @@ import ( "errors" "flag" "fmt" + "os" "strconv" "strings" @@ -233,10 +234,16 @@ func (cmd *clearPageCommand) Run(args ...string) error { return err } - if err := surgeon.ClearPage(cmd.dstPath, common.Pgid(pageId)); err != nil { + needAbandonFreelist, err := surgeon.ClearPage(cmd.dstPath, common.Pgid(pageId)) + if err != nil { return fmt.Errorf("clearPageCommand failed: %w", err) } + if needAbandonFreelist { + fmt.Fprintf(os.Stdout, "WARNING: The clearing has abandoned some pages that are not yet referenced from free list.\n") + fmt.Fprintf(os.Stdout, "Please consider executing `./bbolt surgery abandon-freelist ...`\n") + } + fmt.Fprintf(cmd.Stdout, "Page (%d) was cleared\n", pageId) return nil } diff --git a/go.mod b/go.mod index 3602e0697..4506b5af8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module go.etcd.io/bbolt go 1.19 require ( + github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.2 go.etcd.io/gofail v0.1.0 golang.org/x/sys v0.6.0 @@ -10,6 +11,8 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ed1ee3861..529183910 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,21 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg= diff --git a/internal/common/inode.go b/internal/common/inode.go index 9f99937e7..080b9af78 100644 --- a/internal/common/inode.go +++ b/internal/common/inode.go @@ -103,3 +103,13 @@ func WriteInodeToPage(inodes Inodes, p *Page) uint32 { return uint32(off) } + +func UsedSpaceInPage(inodes Inodes, p *Page) uint32 { + off := unsafe.Sizeof(*p) + p.PageElementSize()*uintptr(len(inodes)) + for _, item := range inodes { + sz := len(item.Key()) + len(item.Value()) + off += uintptr(sz) + } + + return uint32(off) +} diff --git a/internal/guts_cli/guts_cli.go b/internal/guts_cli/guts_cli.go index 891ddb7b4..20b74b081 100644 --- a/internal/guts_cli/guts_cli.go +++ b/internal/guts_cli/guts_cli.go @@ -50,6 +50,10 @@ func ReadPage(path string, pageID uint64) (*common.Page, []byte, error) { return nil, nil, fmt.Errorf("error: %w, Page claims to have %d overflow pages (>=hwm=%d). Interrupting to avoid risky OOM", ErrCorrupt, overflowN, hwm) } + if overflowN == 0 { + return p, buf, nil + } + // Re-read entire Page (with overflow) into buffer. buf = make([]byte, (uint64(overflowN)+1)*pageSize) if n, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { diff --git a/internal/surgeon/surgeon.go b/internal/surgeon/surgeon.go index d2220a276..e69eb7234 100644 --- a/internal/surgeon/surgeon.go +++ b/internal/surgeon/surgeon.go @@ -2,6 +2,7 @@ package surgeon import ( "fmt" + "go.etcd.io/bbolt/internal/common" "go.etcd.io/bbolt/internal/guts_cli" ) @@ -15,18 +16,124 @@ func CopyPage(path string, srcPage common.Pgid, target common.Pgid) error { return guts_cli.WritePage(path, d1) } -func ClearPage(path string, pgId common.Pgid) error { +func ClearPage(path string, pgId common.Pgid) (bool, error) { + return ClearPageElements(path, pgId, 0, -1, false) +} + +// ClearPageElements supports clearing elements in both branch and leaf +// pages. Note if the ${abandonFreelist} is true, the freelist may be cleaned +// in the meta pages in the following two cases, and bbolt needs to scan the +// db to reconstruct free list. It may cause some delay on next startup, +// depending on the db size. +// 1. Any branch elements are cleared; +// 2. An object saved in overflow pages is cleared; +// +// Usually ${abandonFreelist} defaults to false, it means it will not clear the +// freelist in meta pages automatically. Users will receive a warning message +// to remind them to explicitly execute `bbolt surgery abandom-freelist` +// afterwards; the first return parameter will be true in such case. But if +// the freelist isn't synced at all, no warning message will be displayed. +func ClearPageElements(path string, pgId common.Pgid, start, end int, abandonFreelist bool) (bool, error) { // Read the page p, buf, err := guts_cli.ReadPage(path, uint64(pgId)) if err != nil { - return fmt.Errorf("ReadPage failed: %w", err) + return false, fmt.Errorf("ReadPage failed: %w", err) + } + + if !p.IsLeafPage() && !p.IsBranchPage() { + return false, fmt.Errorf("can't clear elements in %q page", p.Typ()) + } + + elementCnt := int(p.Count()) + + if elementCnt == 0 { + return false, nil + } + + if start < 0 || start >= elementCnt { + return false, fmt.Errorf("the start index (%d) is out of range [0, %d)", start, elementCnt) + } + + if (end < 0 || end > elementCnt) && end != -1 { + return false, fmt.Errorf("the end index (%d) is out of range [0, %d]", end, elementCnt) + } + + if start > end && end != -1 { + return false, fmt.Errorf("the start index (%d) is bigger than the end index (%d)", start, end) + } + + if start == end { + return false, fmt.Errorf("invalid: the start index (%d) is equal to the end index (%d)", start, end) + } + + preOverflow := p.Overflow() + + var ( + dataWritten uint32 + ) + if end == int(p.Count()) || end == -1 { + inodes := common.ReadInodeFromPage(p) + inodes = inodes[:start] + + p.SetCount(uint16(start)) + // no need to write inode & data again, we just need to get + // the data size which will be kept. + dataWritten = common.UsedSpaceInPage(inodes, p) + } else { + inodes := common.ReadInodeFromPage(p) + inodes = append(inodes[:start], inodes[end:]...) + + p.SetCount(uint16(len(inodes))) + dataWritten = common.WriteInodeToPage(inodes, p) + } + + pageSize, _, err := guts_cli.ReadPageAndHWMSize(path) + if err != nil { + return false, fmt.Errorf("ReadPageAndHWMSize failed: %w", err) + } + if dataWritten%uint32(pageSize) == 0 { + p.SetOverflow(dataWritten/uint32(pageSize) - 1) + } else { + p.SetOverflow(dataWritten / uint32(pageSize)) } - // Update and rewrite the page - p.SetCount(0) - p.SetOverflow(0) + datasz := pageSize * (uint64(p.Overflow()) + 1) + if err := guts_cli.WritePage(path, buf[0:datasz]); err != nil { + return false, fmt.Errorf("WritePage failed: %w", err) + } + + if preOverflow != p.Overflow() || p.IsBranchPage() { + if abandonFreelist { + return false, clearFreelist(path) + } + return true, nil + } + + return false, nil +} + +func clearFreelist(path string) error { + if err := clearFreelistInMetaPage(path, 0); err != nil { + return fmt.Errorf("clearFreelist on meta page 0 failed: %w", err) + } + if err := clearFreelistInMetaPage(path, 1); err != nil { + return fmt.Errorf("clearFreelist on meta page 1 failed: %w", err) + } + return nil +} + +func clearFreelistInMetaPage(path string, pageId uint64) error { + _, buf, err := guts_cli.ReadPage(path, pageId) + if err != nil { + return fmt.Errorf("ReadPage %d failed: %w", pageId, err) + } + + meta := common.LoadPageMeta(buf) + meta.SetFreelist(common.PgidNoFreelist) + meta.SetChecksum(meta.Sum64()) + if err := guts_cli.WritePage(path, buf); err != nil { - return fmt.Errorf("WritePage failed: %w", err) + return fmt.Errorf("WritePage %d failed: %w", pageId, err) } return nil