Merge pull request #381 from ahrtr/loadfreepage_20230111

add PreLoadFreelist to support loading free pages in readonly mode

cmd/bbolt/main.go

@@ -195,7 +195,10 @@ func (cmd *CheckCommand) Run(args ...string) error {
 	}
 
 	// Open database.
-	db, err := bolt.Open(path, 0666, &bolt.Options{ReadOnly: true})
+	db, err := bolt.Open(path, 0666, &bolt.Options{
+		ReadOnly:        true,
+		PreLoadFreelist: true,
+	})
 	if err != nil {
 		return err
 	}
@@ -644,7 +647,10 @@ func (cmd *PagesCommand) Run(args ...string) error {
 	}
 
 	// Open database.
-	db, err := bolt.Open(path, 0666, &bolt.Options{ReadOnly: true})
+	db, err := bolt.Open(path, 0666, &bolt.Options{
+		ReadOnly:        true,
+		PreLoadFreelist: true,
+	})
 	if err != nil {
 		return err
 	}

cmd/bbolt/main_test.go

@@ -255,6 +255,37 @@ func TestGetCommand_Run(t *testing.T) {
 	}
 }
 
+// Ensure the "pages" command neither panic, nor change the db file.
+func TestPagesCommand_Run(t *testing.T) {
+	db := btesting.MustCreateDB(t)
+
+	err := db.Update(func(tx *bolt.Tx) error {
+		for _, name := range []string{"foo", "bar"} {
+			b, err := tx.CreateBucket([]byte(name))
+			if err != nil {
+				return err
+			}
+			for i := 0; i < 3; i++ {
+				key := fmt.Sprintf("%s-%d", name, i)
+				val := fmt.Sprintf("val-%s-%d", name, i)
+				if err := b.Put([]byte(key), []byte(val)); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	db.Close()
+
+	defer requireDBNoChange(t, dbData(t, db.Path()), db.Path())
+
+	// Run the command.
+	m := NewMain()
+	err = m.Run("pages", db.Path())
+	require.NoError(t, err)
+}
+
 // Main represents a test wrapper for main.Main that records output.
 type Main struct {
 	*main.Main

db.go

@@ -95,6 +95,11 @@ type DB struct {
 	// https://github.com/boltdb/bolt/issues/284
 	NoGrowSync bool
 
+	// When `true`, bbolt will always load the free pages when opening the DB.
+	// When opening db in write mode, this flag will always automatically
+	// set to `true`.
+	PreLoadFreelist bool
+
 	// If you want to read the entire database fast, you can set MmapFlag to
 	// syscall.MAP_POPULATE on Linux 2.6.23+ for sequential read-ahead.
 	MmapFlags int
@@ -196,6 +201,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	db.NoGrowSync = options.NoGrowSync
 	db.MmapFlags = options.MmapFlags
 	db.NoFreelistSync = options.NoFreelistSync
+	db.PreLoadFreelist = options.PreLoadFreelist
 	db.FreelistType = options.FreelistType
 	db.Mlock = options.Mlock
 
@@ -208,6 +214,9 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	if options.ReadOnly {
 		flag = os.O_RDONLY
 		db.readOnly = true
+	} else {
+		// always load free pages in write mode
+		db.PreLoadFreelist = true
 	}
 
 	db.openFile = options.OpenFile
@@ -277,12 +286,14 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 		return nil, err
 	}
 
+	if db.PreLoadFreelist {
+		db.loadFreelist()
+	}
+
 	if db.readOnly {
 		return db, nil
 	}
 
-	db.loadFreelist()
-
 	// Flush freelist when transitioning from no sync to sync so
 	// NoFreelistSync unaware boltdb can open the db later.
 	if !db.NoFreelistSync && !db.hasSyncedFreelist() {
@@ -1163,6 +1174,11 @@ type Options struct {
 	// under normal operation, but requires a full database re-sync during recovery.
 	NoFreelistSync bool
 
+	// PreLoadFreelist sets whether to load the free pages when opening
+	// the db file. Note when opening db in write mode, bbolt will always
+	// load the free pages.
+	PreLoadFreelist bool
+
 	// FreelistType sets the backend freelist type. There are two options. Array which is simple but endures
 	// dramatic performance degradation if database is large and framentation in freelist is common.
 	// The alternative one is using hashmap, it is faster in almost all circumstances

db_whitebox_test.go

@@ -0,0 +1,124 @@
+package bbolt
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOpenWithPreLoadFreelist(t *testing.T) {
+	testCases := []struct {
+		name                    string
+		readonly                bool
+		preLoadFreePage         bool
+		expectedFreePagesLoaded bool
+	}{
+		{
+			name:                    "write mode always load free pages",
+			readonly:                false,
+			preLoadFreePage:         false,
+			expectedFreePagesLoaded: true,
+		},
+		{
+			name:                    "readonly mode load free pages when flag set",
+			readonly:                true,
+			preLoadFreePage:         true,
+			expectedFreePagesLoaded: true,
+		},
+		{
+			name:                    "readonly mode doesn't load free pages when flag not set",
+			readonly:                true,
+			preLoadFreePage:         false,
+			expectedFreePagesLoaded: false,
+		},
+	}
+
+	fileName, err := prepareData(t)
+	require.NoError(t, err)
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			db, err := Open(fileName, 0666, &Options{
+				ReadOnly:        tc.readonly,
+				PreLoadFreelist: tc.preLoadFreePage,
+			})
+			require.NoError(t, err)
+
+			assert.Equal(t, tc.expectedFreePagesLoaded, db.freelist != nil)
+
+			assert.NoError(t, db.Close())
+		})
+	}
+}
+
+func TestMethodPage(t *testing.T) {
+	testCases := []struct {
+		name            string
+		readonly        bool
+		preLoadFreePage bool
+		expectedError   error
+	}{
+		{
+			name:            "write mode",
+			readonly:        false,
+			preLoadFreePage: false,
+			expectedError:   nil,
+		},
+		{
+			name:            "readonly mode with preloading free pages",
+			readonly:        true,
+			preLoadFreePage: true,
+			expectedError:   nil,
+		},
+		{
+			name:            "readonly mode without preloading free pages",
+			readonly:        true,
+			preLoadFreePage: false,
+			expectedError:   ErrFreePagesNotLoaded,
+		},
+	}
+
+	fileName, err := prepareData(t)
+	require.NoError(t, err)
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			db, err := Open(fileName, 0666, &Options{
+				ReadOnly:        tc.readonly,
+				PreLoadFreelist: tc.preLoadFreePage,
+			})
+			require.NoError(t, err)
+			defer db.Close()
+
+			tx, err := db.Begin(!tc.readonly)
+			require.NoError(t, err)
+
+			_, err = tx.Page(0)
+			require.Equal(t, tc.expectedError, err)
+
+			if tc.readonly {
+				require.NoError(t, tx.Rollback())
+			} else {
+				require.NoError(t, tx.Commit())
+			}
+
+			require.NoError(t, db.Close())
+		})
+	}
+}
+
+func prepareData(t *testing.T) (string, error) {
+	fileName := filepath.Join(t.TempDir(), "db")
+	db, err := Open(fileName, 0666, nil)
+	if err != nil {
+		return "", err
+	}
+	if err := db.Close(); err != nil {
+		return "", err
+	}
+
+	return fileName, nil
+}

errors.go

@@ -41,6 +41,10 @@ var (
 	// ErrDatabaseReadOnly is returned when a mutating transaction is started on a
 	// read-only database.
 	ErrDatabaseReadOnly = errors.New("database is in read-only mode")
+
+	// ErrFreePagesNotLoaded is returned when a readonly transaction without
+	// preloading the free pages is trying to access the free pages.
+	ErrFreePagesNotLoaded = errors.New("free pages are not pre-loaded")
 )
 
 // These errors can occur when putting or deleting a value or a bucket.

tx.go

@@ -652,6 +652,10 @@ func (tx *Tx) Page(id int) (*PageInfo, error) {
 		return nil, nil
 	}
 
+	if tx.db.freelist == nil {
+		return nil, ErrFreePagesNotLoaded
+	}
+
 	// Build the page info.
 	p := tx.db.page(pgid(id))
 	info := &PageInfo{