diff --git a/cmd/bbolt/main.go b/cmd/bbolt/main.go index b92832e0c..ff24df019 100644 --- a/cmd/bbolt/main.go +++ b/cmd/bbolt/main.go @@ -195,7 +195,10 @@ func (cmd *CheckCommand) Run(args ...string) error { } // Open database. - db, err := bolt.Open(path, 0666, &bolt.Options{ReadOnly: true}) + db, err := bolt.Open(path, 0666, &bolt.Options{ + ReadOnly: true, + PreLoadFreelist: true, + }) if err != nil { return err } @@ -644,7 +647,10 @@ func (cmd *PagesCommand) Run(args ...string) error { } // Open database. - db, err := bolt.Open(path, 0666, &bolt.Options{ReadOnly: true}) + db, err := bolt.Open(path, 0666, &bolt.Options{ + ReadOnly: true, + PreLoadFreelist: true, + }) if err != nil { return err } diff --git a/cmd/bbolt/main_test.go b/cmd/bbolt/main_test.go index 382b7fa9a..7d0cfd249 100644 --- a/cmd/bbolt/main_test.go +++ b/cmd/bbolt/main_test.go @@ -255,6 +255,37 @@ func TestGetCommand_Run(t *testing.T) { } } +// Ensure the "pages" command neither panic, nor change the db file. +func TestPagesCommand_Run(t *testing.T) { + db := btesting.MustCreateDB(t) + + err := db.Update(func(tx *bolt.Tx) error { + for _, name := range []string{"foo", "bar"} { + b, err := tx.CreateBucket([]byte(name)) + if err != nil { + return err + } + for i := 0; i < 3; i++ { + key := fmt.Sprintf("%s-%d", name, i) + val := fmt.Sprintf("val-%s-%d", name, i) + if err := b.Put([]byte(key), []byte(val)); err != nil { + return err + } + } + } + return nil + }) + require.NoError(t, err) + db.Close() + + defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) + + // Run the command. + m := NewMain() + err = m.Run("pages", db.Path()) + require.NoError(t, err) +} + // Main represents a test wrapper for main.Main that records output. type Main struct { *main.Main diff --git a/db.go b/db.go index a74c2b66f..d9749f46e 100644 --- a/db.go +++ b/db.go @@ -95,6 +95,11 @@ type DB struct { // https://github.com/boltdb/bolt/issues/284 NoGrowSync bool + // When `true`, bbolt will always load the free pages when opening the DB. + // When opening db in write mode, this flag will always automatically + // set to `true`. + PreLoadFreelist bool + // If you want to read the entire database fast, you can set MmapFlag to // syscall.MAP_POPULATE on Linux 2.6.23+ for sequential read-ahead. MmapFlags int @@ -196,6 +201,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { db.NoGrowSync = options.NoGrowSync db.MmapFlags = options.MmapFlags db.NoFreelistSync = options.NoFreelistSync + db.PreLoadFreelist = options.PreLoadFreelist db.FreelistType = options.FreelistType db.Mlock = options.Mlock @@ -208,6 +214,9 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { if options.ReadOnly { flag = os.O_RDONLY db.readOnly = true + } else { + // always load free pages in write mode + db.PreLoadFreelist = true } db.openFile = options.OpenFile @@ -277,12 +286,14 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { return nil, err } + if db.PreLoadFreelist { + db.loadFreelist() + } + if db.readOnly { return db, nil } - db.loadFreelist() - // Flush freelist when transitioning from no sync to sync so // NoFreelistSync unaware boltdb can open the db later. if !db.NoFreelistSync && !db.hasSyncedFreelist() { @@ -1163,6 +1174,11 @@ type Options struct { // under normal operation, but requires a full database re-sync during recovery. NoFreelistSync bool + // PreLoadFreelist sets whether to load the free pages when opening + // the db file. Note when opening db in write mode, bbolt will always + // load the free pages. + PreLoadFreelist bool + // FreelistType sets the backend freelist type. There are two options. Array which is simple but endures // dramatic performance degradation if database is large and framentation in freelist is common. // The alternative one is using hashmap, it is faster in almost all circumstances diff --git a/db_whitebox_test.go b/db_whitebox_test.go new file mode 100644 index 000000000..eb95155e5 --- /dev/null +++ b/db_whitebox_test.go @@ -0,0 +1,124 @@ +package bbolt + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOpenWithPreLoadFreelist(t *testing.T) { + testCases := []struct { + name string + readonly bool + preLoadFreePage bool + expectedFreePagesLoaded bool + }{ + { + name: "write mode always load free pages", + readonly: false, + preLoadFreePage: false, + expectedFreePagesLoaded: true, + }, + { + name: "readonly mode load free pages when flag set", + readonly: true, + preLoadFreePage: true, + expectedFreePagesLoaded: true, + }, + { + name: "readonly mode doesn't load free pages when flag not set", + readonly: true, + preLoadFreePage: false, + expectedFreePagesLoaded: false, + }, + } + + fileName, err := prepareData(t) + require.NoError(t, err) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + db, err := Open(fileName, 0666, &Options{ + ReadOnly: tc.readonly, + PreLoadFreelist: tc.preLoadFreePage, + }) + require.NoError(t, err) + + assert.Equal(t, tc.expectedFreePagesLoaded, db.freelist != nil) + + assert.NoError(t, db.Close()) + }) + } +} + +func TestMethodPage(t *testing.T) { + testCases := []struct { + name string + readonly bool + preLoadFreePage bool + expectedError error + }{ + { + name: "write mode", + readonly: false, + preLoadFreePage: false, + expectedError: nil, + }, + { + name: "readonly mode with preloading free pages", + readonly: true, + preLoadFreePage: true, + expectedError: nil, + }, + { + name: "readonly mode without preloading free pages", + readonly: true, + preLoadFreePage: false, + expectedError: ErrFreePagesNotLoaded, + }, + } + + fileName, err := prepareData(t) + require.NoError(t, err) + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + db, err := Open(fileName, 0666, &Options{ + ReadOnly: tc.readonly, + PreLoadFreelist: tc.preLoadFreePage, + }) + require.NoError(t, err) + defer db.Close() + + tx, err := db.Begin(!tc.readonly) + require.NoError(t, err) + + _, err = tx.Page(0) + require.Equal(t, tc.expectedError, err) + + if tc.readonly { + require.NoError(t, tx.Rollback()) + } else { + require.NoError(t, tx.Commit()) + } + + require.NoError(t, db.Close()) + }) + } +} + +func prepareData(t *testing.T) (string, error) { + fileName := filepath.Join(t.TempDir(), "db") + db, err := Open(fileName, 0666, nil) + if err != nil { + return "", err + } + if err := db.Close(); err != nil { + return "", err + } + + return fileName, nil +} diff --git a/errors.go b/errors.go index 48758ca57..2ad14f12e 100644 --- a/errors.go +++ b/errors.go @@ -41,6 +41,10 @@ var ( // ErrDatabaseReadOnly is returned when a mutating transaction is started on a // read-only database. ErrDatabaseReadOnly = errors.New("database is in read-only mode") + + // ErrFreePagesNotLoaded is returned when a readonly transaction without + // preloading the free pages is trying to access the free pages. + ErrFreePagesNotLoaded = errors.New("free pages are not pre-loaded") ) // These errors can occur when putting or deleting a value or a bucket. diff --git a/tx.go b/tx.go index f8119a0b7..97adbe762 100644 --- a/tx.go +++ b/tx.go @@ -652,6 +652,10 @@ func (tx *Tx) Page(id int) (*PageInfo, error) { return nil, nil } + if tx.db.freelist == nil { + return nil, ErrFreePagesNotLoaded + } + // Build the page info. p := tx.db.page(pgid(id)) info := &PageInfo{