From 46cdbf5b6cf9518d604f67a62bd9fa4ec4c466d1 Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Tue, 20 Dec 2022 14:26:46 +0100 Subject: [PATCH 1/6] Factor out (bolt)testing library. Thanks to putting it outside the bbolt_test package it can be used to test other packages. The replacement of the bbolt_test variant inlined in db_test.go is to be performed in a follow up PR. Signed-off-by: Piotr Tabor --- internal/btesting/btesting.go | 204 ++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 internal/btesting/btesting.go diff --git a/internal/btesting/btesting.go b/internal/btesting/btesting.go new file mode 100644 index 000000000..fba561f59 --- /dev/null +++ b/internal/btesting/btesting.go @@ -0,0 +1,204 @@ +package btesting + +import ( + "flag" + "fmt" + bolt "go.etcd.io/bbolt" + "os" + "path/filepath" + "regexp" + "testing" + "time" +) + +var statsFlag = flag.Bool("stats", false, "show performance stats") + +// TestFreelistType is used as a env variable for test to indicate the backend type +const TestFreelistType = "TEST_FREELIST_TYPE" + +// DB is a test wrapper for bolt.DB. +type DB struct { + *bolt.DB + f string + o *bolt.Options + t testing.TB +} + +// MustCreateDB returns a new, open DB at a temporary location. +func MustCreateDB(t testing.TB) *DB { + return MustCreateDBWithOption(t, nil) +} + +// MustCreateDBWithOption returns a new, open DB at a temporary location with given options. +func MustCreateDBWithOption(t testing.TB, o *bolt.Options) *DB { + f := filepath.Join(t.TempDir(), "db") + return MustOpenDBWithOption(t, f, o) +} + +func MustOpenDBWithOption(t testing.TB, f string, o *bolt.Options) *DB { + if o == nil { + o = bolt.DefaultOptions + } + + freelistType := bolt.FreelistMapType + if env := os.Getenv(TestFreelistType); env == string(bolt.FreelistArrayType) { + freelistType = bolt.FreelistArrayType + } + o.FreelistType = freelistType + + o.FreelistType = freelistType + + db, err := bolt.Open(f, 0666, o) + if err != nil { + panic(err) + } + resDB := &DB{ + DB: db, + f: f, + o: o, + t: t, + } + t.Cleanup(resDB.PostTestCleanup) + return resDB +} + +func (db *DB) PostTestCleanup() { + // Check database consistency after every test. + if db.DB != nil { + db.MustCheck() + db.MustClose() + } +} + +// Close closes the database but does NOT delete the underlying file. +func (db *DB) Close() error { + if db.DB != nil { + // Log statistics. + if *statsFlag { + db.PrintStats() + } + + err := db.DB.Close() + if err != nil { + return err + } + db.DB = nil + } + return nil +} + +// MustClose closes the database and deletes the underlying file. Panic on error. +func (db *DB) MustClose() { + if err := db.Close(); err != nil { + panic(err) + } +} + +func (db *DB) SetOptions(o *bolt.Options) { + db.o = o +} + +// MustReopen reopen the database. Panic on error. +func (db *DB) MustReopen() { + if db.DB != nil { + panic("Please call Close() before MustReopen()") + } + indb, err := bolt.Open(db.Path(), 0666, db.o) + if err != nil { + panic(err) + } + db.DB = indb +} + +// MustCheck runs a consistency check on the database and panics if any errors are found. +func (db *DB) MustCheck() { + if err := db.Update(func(tx *bolt.Tx) error { + // Collect all the errors. + var errors []error + for err := range tx.Check() { + errors = append(errors, err) + if len(errors) > 10 { + break + } + } + + // If errors occurred, copy the DB and print the errors. + if len(errors) > 0 { + var path = filepath.Join(db.t.TempDir(), "db.backup") + if err := tx.CopyFile(path, 0600); err != nil { + panic(err) + } + + // Print errors. + fmt.Print("\n\n") + fmt.Printf("consistency check failed (%d errors)\n", len(errors)) + for _, err := range errors { + fmt.Println(err) + } + fmt.Println("") + fmt.Println("db saved to:") + fmt.Println(path) + fmt.Print("\n\n") + os.Exit(-1) + } + + return nil + }); err != nil && err != bolt.ErrDatabaseNotOpen { + panic(err) + } +} + +// Fill - fills the DB using numTx transactions and numKeysPerTx. +func (db *DB) Fill(bucket []byte, numTx int, numKeysPerTx int, + keyGen func(tx int, key int) []byte, + valueGen func(tx int, key int) []byte) error { + for tr := 0; tr < numTx; tr++ { + err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists(bucket) + for i := 0; i < numKeysPerTx; i++ { + if err := b.Put(keyGen(tr, i), valueGen(tr, i)); err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + } + return nil +} + +func (db *DB) Path() string { + return db.f +} + +// CopyTempFile copies a database to a temporary file. +func (db *DB) CopyTempFile() { + path := filepath.Join(db.t.TempDir(), "db.copy") + if err := db.View(func(tx *bolt.Tx) error { + return tx.CopyFile(path, 0600) + }); err != nil { + panic(err) + } + fmt.Println("db copied to: ", path) +} + +// PrintStats prints the database stats +func (db *DB) PrintStats() { + var stats = db.Stats() + fmt.Printf("[db] %-20s %-20s %-20s\n", + fmt.Sprintf("pg(%d/%d)", stats.TxStats.PageCount, stats.TxStats.PageAlloc), + fmt.Sprintf("cur(%d)", stats.TxStats.CursorCount), + fmt.Sprintf("node(%d/%d)", stats.TxStats.NodeCount, stats.TxStats.NodeDeref), + ) + fmt.Printf(" %-20s %-20s %-20s\n", + fmt.Sprintf("rebal(%d/%v)", stats.TxStats.Rebalance, truncDuration(stats.TxStats.RebalanceTime)), + fmt.Sprintf("spill(%d/%v)", stats.TxStats.Spill, truncDuration(stats.TxStats.SpillTime)), + fmt.Sprintf("w(%d/%v)", stats.TxStats.Write, truncDuration(stats.TxStats.WriteTime)), + ) +} + +func truncDuration(d time.Duration) string { + return regexp.MustCompile(`^(\d+)(\.\d+)`).ReplaceAllString(d.String(), "$1") +} From f3e164d8dda629b31d11fa6c764677fa209d815c Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Mon, 19 Dec 2022 16:52:00 +0100 Subject: [PATCH 2/6] Example: How to use the separated 'btesting' library. Signed-off-by: Piotr Tabor --- db_test.go | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/db_test.go b/db_test.go index 84794e51d..4bb825dd8 100644 --- a/db_test.go +++ b/db_test.go @@ -6,6 +6,7 @@ import ( "errors" "flag" "fmt" + "go.etcd.io/bbolt/internal/btesting" "hash/fnv" "log" "math/rand" @@ -218,40 +219,29 @@ func TestOpen_ErrChecksum(t *testing.T) { // https://github.com/boltdb/bolt/issues/291 func TestOpen_Size(t *testing.T) { // Open a data file. - db := MustOpenDB() - path := db.Path() - defer db.MustClose() + db := btesting.MustCreateDB(t) pagesize := db.Info().PageSize // Insert until we get above the minimum 4MB size. - if err := db.Update(func(tx *bolt.Tx) error { - b, _ := tx.CreateBucketIfNotExists([]byte("data")) - for i := 0; i < 10000; i++ { - if err := b.Put([]byte(fmt.Sprintf("%04d", i)), make([]byte, 1000)); err != nil { - t.Fatal(err) - } - } - return nil - }); err != nil { + err := db.Fill([]byte("data"), 1, 10000, + func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) }, + func(tx int, k int) []byte { return make([]byte, 1000) }, + ) + if err != nil { t.Fatal(err) } - // Close database and grab the size. - if err := db.DB.Close(); err != nil { - t.Fatal(err) - } + path := db.Path() + db.MustClose() + sz := fileSize(path) if sz == 0 { t.Fatalf("unexpected new file size: %d", sz) } - // Reopen database, update, and check size again. - db0, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } - if err := db0.Update(func(tx *bolt.Tx) error { + db.MustReopen() + if err := db.Update(func(tx *bolt.Tx) error { if err := tx.Bucket([]byte("data")).Put([]byte{0}, []byte{0}); err != nil { t.Fatal(err) } @@ -259,7 +249,7 @@ func TestOpen_Size(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := db0.Close(); err != nil { + if err := db.Close(); err != nil { t.Fatal(err) } newSz := fileSize(path) @@ -1627,11 +1617,13 @@ type DB struct { } // MustOpenDB returns a new, open DB at a temporary location. +// Deprecated: Please use btesting.MustCreateDB(...). func MustOpenDB() *DB { return MustOpenWithOption(nil) } // MustOpenDBWithOption returns a new, open DB at a temporary location with given options. +// Deprecated: Please use btesting.MustCreateWithOption(...). func MustOpenWithOption(o *bolt.Options) *DB { f := tempfile() if o == nil { @@ -1655,6 +1647,16 @@ func MustOpenWithOption(o *bolt.Options) *DB { } } +// Closes the DB without removing the file. +// Allows for Reopen(). +func (db *DB) CloseTemporarily() error { + if err := db.DB.Close(); err != nil { + return err + } + db.DB = nil + return nil +} + // Close closes the database and deletes the underlying file. func (db *DB) Close() error { // Log statistics. From 26cc5e3e8bf5dbfdb6a8b7a61d0e04ad657e485f Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Tue, 20 Dec 2022 17:12:04 +0100 Subject: [PATCH 3/6] Tests: Migrate from test DB to btesting.DB Signed-off-by: Piotr Tabor --- bucket_test.go | 144 +++++++----------- cursor_test.go | 44 ++---- db_test.go | 272 ++++++---------------------------- internal/btesting/btesting.go | 19 ++- simulation_test.go | 7 +- tx_test.go | 82 ++++------ unix_test.go | 16 +- 7 files changed, 165 insertions(+), 419 deletions(-) diff --git a/bucket_test.go b/bucket_test.go index 30b5b8828..137061232 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -15,13 +15,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + bolt "go.etcd.io/bbolt" + "go.etcd.io/bbolt/internal/btesting" ) // Ensure that a bucket that gets a non-existent key returns nil. func TestBucket_Get_NonExistent(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -39,8 +40,7 @@ func TestBucket_Get_NonExistent(t *testing.T) { // Ensure that a bucket can read a value that is not flushed yet. func TestBucket_Get_FromNode(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -61,8 +61,7 @@ func TestBucket_Get_FromNode(t *testing.T) { // Ensure that a bucket retrieved via Get() returns a nil. func TestBucket_Get_IncompatibleValue(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -87,8 +86,7 @@ func TestBucket_Get_IncompatibleValue(t *testing.T) { // // https://github.com/boltdb/bolt/issues/544 func TestBucket_Get_Capacity(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) // Write key to a bucket. if err := db.Update(func(tx *bolt.Tx) error { @@ -125,8 +123,7 @@ func TestBucket_Get_Capacity(t *testing.T) { // Ensure that a bucket can write a key/value. func TestBucket_Put(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -148,8 +145,7 @@ func TestBucket_Put(t *testing.T) { // Ensure that a bucket can rewrite a key in the same transaction. func TestBucket_Put_Repeat(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -174,8 +170,7 @@ func TestBucket_Put_Repeat(t *testing.T) { // Ensure that a bucket can write a bunch of large values. func TestBucket_Put_Large(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) count, factor := 100, 200 if err := db.Update(func(tx *bolt.Tx) error { @@ -216,8 +211,7 @@ func TestDB_Put_VeryLarge(t *testing.T) { n, batchN := 400000, 200000 ksize, vsize := 8, 500 - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) for i := 0; i < n; i += batchN { if err := db.Update(func(tx *bolt.Tx) error { @@ -241,8 +235,7 @@ func TestDB_Put_VeryLarge(t *testing.T) { // Ensure that a setting a value on a key with a bucket value returns an error. func TestBucket_Put_IncompatibleValue(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b0, err := tx.CreateBucket([]byte("widgets")) @@ -264,8 +257,7 @@ func TestBucket_Put_IncompatibleValue(t *testing.T) { // Ensure that a setting a value while the transaction is closed returns an error. func TestBucket_Put_Closed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) tx, err := db.Begin(true) if err != nil { t.Fatal(err) @@ -287,8 +279,7 @@ func TestBucket_Put_Closed(t *testing.T) { // Ensure that setting a value on a read-only bucket returns an error. func TestBucket_Put_ReadOnly(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucket([]byte("widgets")); err != nil { @@ -312,8 +303,7 @@ func TestBucket_Put_ReadOnly(t *testing.T) { // Ensure that a bucket can delete an existing key. func TestBucket_Delete(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -337,8 +327,7 @@ func TestBucket_Delete(t *testing.T) { // Ensure that deleting a large set of keys will work correctly. func TestBucket_Delete_Large(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -388,8 +377,7 @@ func TestBucket_Delete_FreelistOverflow(t *testing.T) { t.Skip("skipping test in short mode.") } - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) k := make([]byte, 16) // The bigger the pages - the more values we need to write. @@ -436,9 +424,7 @@ func TestBucket_Delete_FreelistOverflow(t *testing.T) { } // Free page count should be preserved on reopen. - if err := db.DB.Close(); err != nil { - t.Fatal(err) - } + db.MustClose() db.MustReopen() if reopenFreePages := db.Stats().FreePageN; freePages != reopenFreePages { t.Fatalf("expected %d free pages, got %+v", freePages, db.Stats()) @@ -447,8 +433,7 @@ func TestBucket_Delete_FreelistOverflow(t *testing.T) { // Ensure that deleting of non-existing key is a no-op. func TestBucket_Delete_NonExisting(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -480,8 +465,7 @@ func TestBucket_Delete_NonExisting(t *testing.T) { // Ensure that accessing and updating nested buckets is ok across transactions. func TestBucket_Nested(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { // Create a widgets bucket. @@ -567,8 +551,7 @@ func TestBucket_Nested(t *testing.T) { // Ensure that deleting a bucket using Delete() returns an error. func TestBucket_Delete_Bucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -588,8 +571,7 @@ func TestBucket_Delete_Bucket(t *testing.T) { // Ensure that deleting a key on a read-only bucket returns an error. func TestBucket_Delete_ReadOnly(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucket([]byte("widgets")); err != nil { @@ -612,8 +594,7 @@ func TestBucket_Delete_ReadOnly(t *testing.T) { // Ensure that a deleting value while the transaction is closed returns an error. func TestBucket_Delete_Closed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) tx, err := db.Begin(true) if err != nil { @@ -635,8 +616,7 @@ func TestBucket_Delete_Closed(t *testing.T) { // Ensure that deleting a bucket causes nested buckets to be deleted. func TestBucket_DeleteBucket_Nested(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { widgets, err := tx.CreateBucket([]byte("widgets")) @@ -667,8 +647,7 @@ func TestBucket_DeleteBucket_Nested(t *testing.T) { // Ensure that deleting a bucket causes nested buckets to be deleted after they have been committed. func TestBucket_DeleteBucket_Nested2(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { widgets, err := tx.CreateBucket([]byte("widgets")) @@ -734,8 +713,7 @@ func TestBucket_DeleteBucket_Nested2(t *testing.T) { // Ensure that deleting a child bucket with multiple pages causes all pages to get collected. // NOTE: Consistency check in bolt_test.DB.Close() will panic if pages not freed properly. func TestBucket_DeleteBucket_Large(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { widgets, err := tx.CreateBucket([]byte("widgets")) @@ -770,8 +748,7 @@ func TestBucket_DeleteBucket_Large(t *testing.T) { // Ensure that a simple value retrieved via Bucket() returns a nil. func TestBucket_Bucket_IncompatibleValue(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { widgets, err := tx.CreateBucket([]byte("widgets")) @@ -793,8 +770,7 @@ func TestBucket_Bucket_IncompatibleValue(t *testing.T) { // Ensure that creating a bucket on an existing non-bucket key returns an error. func TestBucket_CreateBucket_IncompatibleValue(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { widgets, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -815,8 +791,7 @@ func TestBucket_CreateBucket_IncompatibleValue(t *testing.T) { // Ensure that deleting a bucket on an existing non-bucket key returns an error. func TestBucket_DeleteBucket_IncompatibleValue(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { widgets, err := tx.CreateBucket([]byte("widgets")) @@ -837,8 +812,7 @@ func TestBucket_DeleteBucket_IncompatibleValue(t *testing.T) { // Ensure bucket can set and update its sequence number. func TestBucket_Sequence(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { bkt, err := tx.CreateBucket([]byte("0")) @@ -879,8 +853,7 @@ func TestBucket_Sequence(t *testing.T) { // Ensure that a bucket can return an autoincrementing sequence. func TestBucket_NextSequence(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { widgets, err := tx.CreateBucket([]byte("widgets")) @@ -922,8 +895,7 @@ func TestBucket_NextSequence(t *testing.T) { // the only thing updated on the bucket. // https://github.com/boltdb/bolt/issues/296 func TestBucket_NextSequence_Persist(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucket([]byte("widgets")); err != nil { @@ -958,8 +930,7 @@ func TestBucket_NextSequence_Persist(t *testing.T) { // Ensure that retrieving the next sequence on a read-only bucket returns an error. func TestBucket_NextSequence_ReadOnly(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucket([]byte("widgets")); err != nil { @@ -983,8 +954,7 @@ func TestBucket_NextSequence_ReadOnly(t *testing.T) { // Ensure that retrieving the next sequence for a bucket on a closed database return an error. func TestBucket_NextSequence_Closed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) tx, err := db.Begin(true) if err != nil { t.Fatal(err) @@ -1003,8 +973,7 @@ func TestBucket_NextSequence_Closed(t *testing.T) { // Ensure a user can loop over all key/value pairs in a bucket. func TestBucket_ForEach(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) type kv struct { k []byte @@ -1053,8 +1022,7 @@ func TestBucket_ForEach(t *testing.T) { } func TestBucket_ForEachBucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) expectedItems := [][]byte{ []byte("csubbucket"), @@ -1098,8 +1066,7 @@ func TestBucket_ForEachBucket(t *testing.T) { } func TestBucket_ForEachBucket_NoBuckets(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) verifyReads := func(b *bolt.Bucket) { var items [][]byte @@ -1137,8 +1104,7 @@ func TestBucket_ForEachBucket_NoBuckets(t *testing.T) { // Ensure a database can stop iteration early. func TestBucket_ForEach_ShortCircuit(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -1176,8 +1142,7 @@ func TestBucket_ForEach_ShortCircuit(t *testing.T) { // Ensure that looping over a bucket on a closed database returns an error. func TestBucket_ForEach_Closed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) tx, err := db.Begin(true) if err != nil { @@ -1200,8 +1165,7 @@ func TestBucket_ForEach_Closed(t *testing.T) { // Ensure that an error is returned when inserting with an empty key. func TestBucket_Put_EmptyKey(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -1222,8 +1186,7 @@ func TestBucket_Put_EmptyKey(t *testing.T) { // Ensure that an error is returned when inserting with a key that's too large. func TestBucket_Put_KeyTooLarge(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -1245,8 +1208,7 @@ func TestBucket_Put_ValueTooLarge(t *testing.T) { t.Skip("not enough RAM for test") } - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -1268,8 +1230,7 @@ func TestBucket_Stats(t *testing.T) { t.Skip("skipping test in short mode") } - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) // Add bucket with fewer keys but one big value. bigKey := []byte("really-big-value") @@ -1361,8 +1322,7 @@ func TestBucket_Stats_RandomFill(t *testing.T) { t.Skip("invalid page size for test") } - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) // Add a set of values in random order. It will be the same random // order so we can maintain consistency between test runs. @@ -1423,8 +1383,7 @@ func TestBucket_Stats_RandomFill(t *testing.T) { // Ensure a bucket can calculate stats. func TestBucket_Stats_Small(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { // Add a bucket that fits on a single root leaf. @@ -1487,8 +1446,7 @@ func TestBucket_Stats_Small(t *testing.T) { } func TestBucket_Stats_EmptyBucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { // Add a bucket that fits on a single root leaf. @@ -1547,8 +1505,7 @@ func TestBucket_Stats_EmptyBucket(t *testing.T) { // Ensure a bucket can calculate stats. func TestBucket_Stats_Nested(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("foo")) @@ -1653,8 +1610,7 @@ func TestBucket_Stats_Large(t *testing.T) { t.Skip("skipping test in short mode.") } - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) var index int for i := 0; i < 100; i++ { @@ -1731,7 +1687,7 @@ func TestBucket_Put_Single(t *testing.T) { index := 0 if err := quick.Check(func(items testdata) bool { - db := MustOpenDB() + db := btesting.MustCreateDB(t) defer db.MustClose() m := make(map[string][]byte) @@ -1788,7 +1744,7 @@ func TestBucket_Put_Multiple(t *testing.T) { } if err := quick.Check(func(items testdata) bool { - db := MustOpenDB() + db := btesting.MustCreateDB(t) defer db.MustClose() // Bulk insert all values. @@ -1841,7 +1797,7 @@ func TestBucket_Delete_Quick(t *testing.T) { } if err := quick.Check(func(items testdata) bool { - db := MustOpenDB() + db := btesting.MustCreateDB(t) defer db.MustClose() // Bulk insert all values. diff --git a/cursor_test.go b/cursor_test.go index 1038adfbf..d58a97152 100644 --- a/cursor_test.go +++ b/cursor_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "fmt" + "go.etcd.io/bbolt/internal/btesting" "log" "os" "reflect" @@ -16,8 +17,7 @@ import ( // Ensure that a cursor can return a reference to the bucket that created it. func TestCursor_Bucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -34,8 +34,7 @@ func TestCursor_Bucket(t *testing.T) { // Ensure that a Tx cursor can seek to the appropriate keys. func TestCursor_Seek(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -104,8 +103,7 @@ func TestCursor_Seek(t *testing.T) { } func TestCursor_Delete(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) const count = 1000 @@ -167,8 +165,7 @@ func TestCursor_Delete(t *testing.T) { // // Related: https://github.com/boltdb/bolt/pull/187 func TestCursor_Seek_Large(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) var count = 10000 @@ -231,8 +228,7 @@ func TestCursor_Seek_Large(t *testing.T) { // Ensure that a cursor can iterate over an empty bucket without error. func TestCursor_EmptyBucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("widgets")) return err @@ -256,8 +252,7 @@ func TestCursor_EmptyBucket(t *testing.T) { // Ensure that a Tx cursor can reverse iterate over an empty bucket without error. func TestCursor_EmptyBucketReverse(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("widgets")) @@ -281,8 +276,7 @@ func TestCursor_EmptyBucketReverse(t *testing.T) { // Ensure that a Tx cursor can iterate over a single root with a couple elements. func TestCursor_Iterate_Leaf(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -352,8 +346,7 @@ func TestCursor_Iterate_Leaf(t *testing.T) { // Ensure that a Tx cursor can iterate in reverse over a single root with a couple elements. func TestCursor_LeafRootReverse(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -416,8 +409,7 @@ func TestCursor_LeafRootReverse(t *testing.T) { // Ensure that a Tx cursor can restart from the beginning. func TestCursor_Restart(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -462,8 +454,7 @@ func TestCursor_Restart(t *testing.T) { // Ensure that a cursor can skip over empty pages that have been deleted. func TestCursor_First_EmptyPages(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) // Create 1000 keys in the "widgets" bucket. if err := db.Update(func(tx *bolt.Tx) error { @@ -509,8 +500,7 @@ func TestCursor_First_EmptyPages(t *testing.T) { // Ensure that a cursor can skip over empty pages that have been deleted. func TestCursor_Last_EmptyPages(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) // Create 1000 keys in the "widgets" bucket. if err := db.Update(func(tx *bolt.Tx) error { @@ -557,7 +547,7 @@ func TestCursor_Last_EmptyPages(t *testing.T) { // Ensure that a Tx can iterate over all elements in a bucket. func TestCursor_QuickCheck(t *testing.T) { f := func(items testdata) bool { - db := MustOpenDB() + db := btesting.MustCreateDB(t) defer db.MustClose() // Bulk insert all values. @@ -615,7 +605,7 @@ func TestCursor_QuickCheck(t *testing.T) { // Ensure that a transaction can iterate over all elements in a bucket in reverse. func TestCursor_QuickCheck_Reverse(t *testing.T) { f := func(items testdata) bool { - db := MustOpenDB() + db := btesting.MustCreateDB(t) defer db.MustClose() // Bulk insert all values. @@ -671,8 +661,7 @@ func TestCursor_QuickCheck_Reverse(t *testing.T) { // Ensure that a Tx cursor can iterate over subbuckets. func TestCursor_QuickCheck_BucketsOnly(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -713,8 +702,7 @@ func TestCursor_QuickCheck_BucketsOnly(t *testing.T) { // Ensure that a Tx cursor can reverse iterate over subbuckets. func TestCursor_QuickCheck_BucketsOnly_Reverse(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) diff --git a/db_test.go b/db_test.go index 4bb825dd8..d36d280e9 100644 --- a/db_test.go +++ b/db_test.go @@ -4,25 +4,23 @@ import ( "bytes" "encoding/binary" "errors" - "flag" "fmt" - "go.etcd.io/bbolt/internal/btesting" "hash/fnv" "log" "math/rand" "os" "path/filepath" - "regexp" "sync" "testing" "time" "unsafe" + "github.com/stretchr/testify/require" + bolt "go.etcd.io/bbolt" + "go.etcd.io/bbolt/internal/btesting" ) -var statsFlag = flag.Bool("stats", false, "show performance stats") - // pageSize is the size of one page in the data file. const pageSize = 4096 @@ -148,12 +146,11 @@ func TestOpen_ErrVersionMismatch(t *testing.T) { } // Create empty database. - db := MustOpenDB() + db := btesting.MustCreateDB(t) path := db.Path() - defer db.MustClose() // Close database. - if err := db.DB.Close(); err != nil { + if err := db.Close(); err != nil { t.Fatal(err) } @@ -185,12 +182,11 @@ func TestOpen_ErrChecksum(t *testing.T) { } // Create empty database. - db := MustOpenDB() + db := btesting.MustCreateDB(t) path := db.Path() - defer db.MustClose() // Close database. - if err := db.DB.Close(); err != nil { + if err := db.Close(); err != nil { t.Fatal(err) } @@ -272,9 +268,8 @@ func TestOpen_Size_Large(t *testing.T) { } // Open a data file. - db := MustOpenDB() + db := btesting.MustCreateDB(t) path := db.Path() - defer db.MustClose() pagesize := db.Info().PageSize @@ -454,8 +449,7 @@ func TestDB_Open_InitialMmapSize(t *testing.T) { // TestDB_Open_ReadOnly checks a database in read only mode can read but not write. func TestDB_Open_ReadOnly(t *testing.T) { // Create a writable db, write k-v and close it. - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -473,7 +467,7 @@ func TestDB_Open_ReadOnly(t *testing.T) { t.Fatal(err) } - f := db.f + f := db.Path() o := &bolt.Options{ReadOnly: true} readOnlyDB, err := bolt.Open(f, 0666, o) if err != nil { @@ -510,13 +504,11 @@ func TestDB_Open_ReadOnly(t *testing.T) { func TestOpen_BigPage(t *testing.T) { pageSize := os.Getpagesize() - db1 := MustOpenWithOption(&bolt.Options{PageSize: pageSize * 2}) - defer db1.MustClose() + db1 := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize * 2}) - db2 := MustOpenWithOption(&bolt.Options{PageSize: pageSize * 4}) - defer db2.MustClose() + db2 := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize * 4}) - if db1sz, db2sz := fileSize(db1.f), fileSize(db2.f); db1sz >= db2sz { + if db1sz, db2sz := fileSize(db1.Path()), fileSize(db2.Path()); db1sz >= db2sz { t.Errorf("expected %d < %d", db1sz, db2sz) } } @@ -525,8 +517,7 @@ func TestOpen_BigPage(t *testing.T) { // write-out after no free list sync will recover the free list // and write it out. func TestOpen_RecoverFreeList(t *testing.T) { - db := MustOpenWithOption(&bolt.Options{NoFreelistSync: true}) - defer db.MustClose() + db := btesting.MustCreateDBWithOption(t, &bolt.Options{NoFreelistSync: true}) // Write some pages. tx, err := db.Begin(true) @@ -568,6 +559,7 @@ func TestOpen_RecoverFreeList(t *testing.T) { if err := db.DB.Close(); err != nil { t.Fatal(err) } + db.MustClose() // Record freelist count from opening with NoFreelistSync. db.MustReopen() @@ -578,9 +570,10 @@ func TestOpen_RecoverFreeList(t *testing.T) { if err := db.DB.Close(); err != nil { t.Fatal(err) } + db.MustClose() // Check free page count is reconstructed when opened with freelist sync. - db.o = &bolt.Options{} + db.SetOptions(&bolt.Options{}) db.MustReopen() // One less free page for syncing the free list on open. freepages-- @@ -599,33 +592,22 @@ func TestDB_Begin_ErrDatabaseNotOpen(t *testing.T) { // Ensure that a read-write transaction can be retrieved. func TestDB_BeginRW(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } else if tx == nil { - t.Fatal("expected tx") - } - - if tx.DB() != db.DB { - t.Fatal("unexpected tx database") - } else if !tx.Writable() { - t.Fatal("expected writable tx") - } + require.NoError(t, err) + require.NotNil(t, tx, "expected tx") + defer func() { require.NoError(t, tx.Commit()) }() - if err := tx.Commit(); err != nil { - t.Fatal(err) - } + require.True(t, tx.Writable(), "expected writable tx") + require.Same(t, db.DB, tx.DB()) } // TestDB_Concurrent_WriteTo checks that issuing WriteTo operations concurrently // with commits does not produce corrupted db files. func TestDB_Concurrent_WriteTo(t *testing.T) { o := &bolt.Options{NoFreelistSync: false} - db := MustOpenWithOption(o) - defer db.MustClose() + db := btesting.MustCreateDBWithOption(t, o) var wg sync.WaitGroup wtxs, rtxs := 5, 5 @@ -646,8 +628,7 @@ func TestDB_Concurrent_WriteTo(t *testing.T) { panic(err) } f.Close() - snap := &DB{nil, f.Name(), o} - snap.MustReopen() + snap := btesting.MustOpenDBWithOption(t, f.Name(), o) defer snap.MustClose() snap.MustCheck() } @@ -698,7 +679,7 @@ func TestDB_Close_PendingTx_RO(t *testing.T) { testDB_Close_PendingTx(t, false) // Ensure that a database cannot close while transactions are open. func testDB_Close_PendingTx(t *testing.T, writable bool) { - db := MustOpenDB() + db := btesting.MustCreateDB(t) // Start transaction. tx, err := db.Begin(writable) @@ -748,8 +729,7 @@ func testDB_Close_PendingTx(t *testing.T, writable bool) { // Ensure a database can provide a transactional block. func TestDB_Update(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -797,8 +777,7 @@ func TestDB_Update_Closed(t *testing.T) { // Ensure a panic occurs while trying to commit a managed transaction. func TestDB_Update_ManualCommit(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) var panicked bool if err := db.Update(func(tx *bolt.Tx) error { @@ -823,8 +802,7 @@ func TestDB_Update_ManualCommit(t *testing.T) { // Ensure a panic occurs while trying to rollback a managed transaction. func TestDB_Update_ManualRollback(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) var panicked bool if err := db.Update(func(tx *bolt.Tx) error { @@ -849,8 +827,7 @@ func TestDB_Update_ManualRollback(t *testing.T) { // Ensure a panic occurs while trying to commit a managed transaction. func TestDB_View_ManualCommit(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) var panicked bool if err := db.View(func(tx *bolt.Tx) error { @@ -875,8 +852,7 @@ func TestDB_View_ManualCommit(t *testing.T) { // Ensure a panic occurs while trying to rollback a managed transaction. func TestDB_View_ManualRollback(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) var panicked bool if err := db.View(func(tx *bolt.Tx) error { @@ -901,8 +877,7 @@ func TestDB_View_ManualRollback(t *testing.T) { // Ensure a write transaction that panics does not hold open locks. func TestDB_Update_Panic(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) // Panic during update but recover. func() { @@ -945,8 +920,7 @@ func TestDB_Update_Panic(t *testing.T) { // Ensure a database can return an error through a read-only transactional block. func TestDB_View_Error(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.View(func(tx *bolt.Tx) error { return errors.New("xxx") @@ -957,8 +931,7 @@ func TestDB_View_Error(t *testing.T) { // Ensure a read transaction that panics does not hold open locks. func TestDB_View_Panic(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucket([]byte("widgets")); err != nil { @@ -1000,8 +973,7 @@ func TestDB_View_Panic(t *testing.T) { // Ensure that DB stats can be returned. func TestDB_Stats(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("widgets")) return err @@ -1021,8 +993,7 @@ func TestDB_Stats(t *testing.T) { // Ensure that database pages are in expected order and type. func TestDB_Consistency(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("widgets")) return err @@ -1107,8 +1078,7 @@ func TestDBStats_Sub(t *testing.T) { // Ensure two functions can perform updates in a single batch. func TestDB_Batch(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucket([]byte("widgets")); err != nil { @@ -1152,8 +1122,7 @@ func TestDB_Batch(t *testing.T) { } func TestDB_Batch_Panic(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) var sentinel int var bork = &sentinel @@ -1183,8 +1152,7 @@ func TestDB_Batch_Panic(t *testing.T) { } func TestDB_BatchFull(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("widgets")) return err @@ -1242,8 +1210,7 @@ func TestDB_BatchFull(t *testing.T) { } func TestDB_BatchTime(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("widgets")) return err @@ -1431,8 +1398,8 @@ func ExampleDB_Begin() { } func BenchmarkDBBatchAutomatic(b *testing.B) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(b) + if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("bench")) return err @@ -1476,8 +1443,7 @@ func BenchmarkDBBatchAutomatic(b *testing.B) { } func BenchmarkDBBatchSingle(b *testing.B) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(b) if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("bench")) return err @@ -1520,8 +1486,7 @@ func BenchmarkDBBatchSingle(b *testing.B) { } func BenchmarkDBBatchManual10x100(b *testing.B) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(b) if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("bench")) return err @@ -1574,7 +1539,7 @@ func BenchmarkDBBatchManual10x100(b *testing.B) { validateBatchBench(b, db) } -func validateBatchBench(b *testing.B, db *DB) { +func validateBatchBench(b *testing.B, db *btesting.DB) { var rollback = errors.New("sentinel error to cause rollback") validate := func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte("bench")) @@ -1609,149 +1574,6 @@ func validateBatchBench(b *testing.B, db *DB) { } } -// DB is a test wrapper for bolt.DB. -type DB struct { - *bolt.DB - f string - o *bolt.Options -} - -// MustOpenDB returns a new, open DB at a temporary location. -// Deprecated: Please use btesting.MustCreateDB(...). -func MustOpenDB() *DB { - return MustOpenWithOption(nil) -} - -// MustOpenDBWithOption returns a new, open DB at a temporary location with given options. -// Deprecated: Please use btesting.MustCreateWithOption(...). -func MustOpenWithOption(o *bolt.Options) *DB { - f := tempfile() - if o == nil { - o = bolt.DefaultOptions - } - - freelistType := bolt.FreelistArrayType - if env := os.Getenv(bolt.TestFreelistType); env == string(bolt.FreelistMapType) { - freelistType = bolt.FreelistMapType - } - o.FreelistType = freelistType - - db, err := bolt.Open(f, 0666, o) - if err != nil { - panic(err) - } - return &DB{ - DB: db, - f: f, - o: o, - } -} - -// Closes the DB without removing the file. -// Allows for Reopen(). -func (db *DB) CloseTemporarily() error { - if err := db.DB.Close(); err != nil { - return err - } - db.DB = nil - return nil -} - -// Close closes the database and deletes the underlying file. -func (db *DB) Close() error { - // Log statistics. - if *statsFlag { - db.PrintStats() - } - - // Check database consistency after every test. - db.MustCheck() - - // Close database and remove file. - defer os.Remove(db.Path()) - return db.DB.Close() -} - -// MustClose closes the database and deletes the underlying file. Panic on error. -func (db *DB) MustClose() { - if err := db.Close(); err != nil { - panic(err) - } -} - -// MustReopen reopen the database. Panic on error. -func (db *DB) MustReopen() { - indb, err := bolt.Open(db.f, 0666, db.o) - if err != nil { - panic(err) - } - db.DB = indb -} - -// PrintStats prints the database stats -func (db *DB) PrintStats() { - var stats = db.Stats() - fmt.Printf("[db] %-20s %-20s %-20s\n", - fmt.Sprintf("pg(%d/%d)", stats.TxStats.PageCount, stats.TxStats.PageAlloc), - fmt.Sprintf("cur(%d)", stats.TxStats.CursorCount), - fmt.Sprintf("node(%d/%d)", stats.TxStats.NodeCount, stats.TxStats.NodeDeref), - ) - fmt.Printf(" %-20s %-20s %-20s\n", - fmt.Sprintf("rebal(%d/%v)", stats.TxStats.Rebalance, truncDuration(stats.TxStats.RebalanceTime)), - fmt.Sprintf("spill(%d/%v)", stats.TxStats.Spill, truncDuration(stats.TxStats.SpillTime)), - fmt.Sprintf("w(%d/%v)", stats.TxStats.Write, truncDuration(stats.TxStats.WriteTime)), - ) -} - -// MustCheck runs a consistency check on the database and panics if any errors are found. -func (db *DB) MustCheck() { - if err := db.Update(func(tx *bolt.Tx) error { - // Collect all the errors. - var errors []error - for err := range tx.Check() { - errors = append(errors, err) - if len(errors) > 10 { - break - } - } - - // If errors occurred, copy the DB and print the errors. - if len(errors) > 0 { - var path = tempfile() - if err := tx.CopyFile(path, 0600); err != nil { - panic(err) - } - - // Print errors. - fmt.Print("\n\n") - fmt.Printf("consistency check failed (%d errors)\n", len(errors)) - for _, err := range errors { - fmt.Println(err) - } - fmt.Println("") - fmt.Println("db saved to:") - fmt.Println(path) - fmt.Print("\n\n") - os.Exit(-1) - } - - return nil - }); err != nil && err != bolt.ErrDatabaseNotOpen { - panic(err) - } -} - -// CopyTempFile copies a database to a temporary file. -func (db *DB) CopyTempFile() { - path := tempfile() - if err := db.View(func(tx *bolt.Tx) error { - return tx.CopyFile(path, 0600) - }); err != nil { - panic(err) - } - fmt.Println("db copied to: ", path) -} - // tempfile returns a temporary file path. func tempfile() string { f, err := os.CreateTemp("", "bolt-") @@ -1774,10 +1596,6 @@ func trunc(b []byte, length int) []byte { return b } -func truncDuration(d time.Duration) string { - return regexp.MustCompile(`^(\d+)(\.\d+)`).ReplaceAllString(d.String(), "$1") -} - func fileSize(path string) int64 { fi, err := os.Stat(path) if err != nil { diff --git a/internal/btesting/btesting.go b/internal/btesting/btesting.go index fba561f59..9c65352f1 100644 --- a/internal/btesting/btesting.go +++ b/internal/btesting/btesting.go @@ -3,6 +3,7 @@ package btesting import ( "flag" "fmt" + "github.com/stretchr/testify/require" bolt "go.etcd.io/bbolt" "os" "path/filepath" @@ -36,15 +37,15 @@ func MustCreateDBWithOption(t testing.TB, o *bolt.Options) *DB { } func MustOpenDBWithOption(t testing.TB, f string, o *bolt.Options) *DB { + t.Logf("Opening bbolt DB at: %s", f) if o == nil { o = bolt.DefaultOptions } - freelistType := bolt.FreelistMapType - if env := os.Getenv(TestFreelistType); env == string(bolt.FreelistArrayType) { - freelistType = bolt.FreelistArrayType + freelistType := bolt.FreelistArrayType + if env := os.Getenv(TestFreelistType); env == string(bolt.FreelistMapType) { + freelistType = bolt.FreelistMapType } - o.FreelistType = freelistType o.FreelistType = freelistType @@ -77,7 +78,7 @@ func (db *DB) Close() error { if *statsFlag { db.PrintStats() } - + db.t.Logf("Closing bbolt DB at: %s", db.f) err := db.DB.Close() if err != nil { return err @@ -87,13 +88,18 @@ func (db *DB) Close() error { return nil } -// MustClose closes the database and deletes the underlying file. Panic on error. +// MustClose closes the database but does NOT delete the underlying file. func (db *DB) MustClose() { if err := db.Close(); err != nil { panic(err) } } +func (db *DB) MustDeleteFile() { + err := os.Remove(db.Path()) + require.NoError(db.t, err) +} + func (db *DB) SetOptions(o *bolt.Options) { db.o = o } @@ -103,6 +109,7 @@ func (db *DB) MustReopen() { if db.DB != nil { panic("Please call Close() before MustReopen()") } + db.t.Logf("Reopening bbolt DB at: %s", db.f) indb, err := bolt.Open(db.Path(), 0666, db.o) if err != nil { panic(err) diff --git a/simulation_test.go b/simulation_test.go index a96a24161..3166aa398 100644 --- a/simulation_test.go +++ b/simulation_test.go @@ -3,6 +3,7 @@ package bbolt_test import ( "bytes" "fmt" + "go.etcd.io/bbolt/internal/btesting" "math/rand" "sync" "sync/atomic" @@ -43,8 +44,7 @@ func testSimulate(t *testing.T, openOption *bolt.Options, round, threadCount, pa var versions = make(map[int]*QuickDB) versions[1] = NewQuickDB() - db := MustOpenWithOption(openOption) - defer db.MustClose() + db := btesting.MustCreateDBWithOption(t, openOption) var mutex sync.Mutex @@ -146,6 +146,9 @@ func testSimulate(t *testing.T, openOption *bolt.Options, round, threadCount, pa } db.MustClose() + // I have doubts the DB drop is indented here (as 'versions' is not being reset). + // But I'm preserving for now the original behavior. + db.MustDeleteFile() db.MustReopen() } diff --git a/tx_test.go b/tx_test.go index 4fe99db2c..d4f0d7212 100644 --- a/tx_test.go +++ b/tx_test.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "go.etcd.io/bbolt/internal/btesting" "log" "os" "testing" @@ -13,7 +14,7 @@ import ( // TestTx_Check_ReadOnly tests consistency checking on a ReadOnly database. func TestTx_Check_ReadOnly(t *testing.T) { - db := MustOpenDB() + db := btesting.MustCreateDB(t) defer db.Close() if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -27,11 +28,11 @@ func TestTx_Check_ReadOnly(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := db.DB.Close(); err != nil { + if err := db.Close(); err != nil { t.Fatal(err) } - readOnlyDB, err := bolt.Open(db.f, 0666, &bolt.Options{ReadOnly: true}) + readOnlyDB, err := bolt.Open(db.Path(), 0666, &bolt.Options{ReadOnly: true}) if err != nil { t.Fatal(err) } @@ -65,8 +66,7 @@ func TestTx_Check_ReadOnly(t *testing.T) { // Ensure that committing a closed transaction returns an error. func TestTx_Commit_ErrTxClosed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) tx, err := db.Begin(true) if err != nil { t.Fatal(err) @@ -87,8 +87,7 @@ func TestTx_Commit_ErrTxClosed(t *testing.T) { // Ensure that rolling back a closed transaction returns an error. func TestTx_Rollback_ErrTxClosed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) tx, err := db.Begin(true) if err != nil { @@ -105,8 +104,7 @@ func TestTx_Rollback_ErrTxClosed(t *testing.T) { // Ensure that committing a read-only transaction returns an error. func TestTx_Commit_ErrTxNotWritable(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) tx, err := db.Begin(false) if err != nil { t.Fatal(err) @@ -123,8 +121,7 @@ func TestTx_Commit_ErrTxNotWritable(t *testing.T) { // Ensure that a transaction can retrieve a cursor on the root bucket. func TestTx_Cursor(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucket([]byte("widgets")); err != nil { t.Fatal(err) @@ -161,8 +158,7 @@ func TestTx_Cursor(t *testing.T) { // Ensure that creating a bucket with a read-only transaction returns an error. func TestTx_CreateBucket_ErrTxNotWritable(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.View(func(tx *bolt.Tx) error { _, err := tx.CreateBucket([]byte("foo")) if err != bolt.ErrTxNotWritable { @@ -176,8 +172,7 @@ func TestTx_CreateBucket_ErrTxNotWritable(t *testing.T) { // Ensure that creating a bucket on a closed transaction returns an error. func TestTx_CreateBucket_ErrTxClosed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) tx, err := db.Begin(true) if err != nil { t.Fatal(err) @@ -193,8 +188,7 @@ func TestTx_CreateBucket_ErrTxClosed(t *testing.T) { // Ensure that a Tx can retrieve a bucket. func TestTx_Bucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucket([]byte("widgets")); err != nil { t.Fatal(err) @@ -210,8 +204,7 @@ func TestTx_Bucket(t *testing.T) { // Ensure that a Tx retrieving a non-existent key returns nil. func TestTx_Get_NotFound(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -232,8 +225,7 @@ func TestTx_Get_NotFound(t *testing.T) { // Ensure that a bucket can be created and retrieved. func TestTx_CreateBucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) // Create a bucket. if err := db.Update(func(tx *bolt.Tx) error { @@ -261,8 +253,7 @@ func TestTx_CreateBucket(t *testing.T) { // Ensure that a bucket can be created if it doesn't already exist. func TestTx_CreateBucketIfNotExists(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { // Create bucket. if b, err := tx.CreateBucketIfNotExists([]byte("widgets")); err != nil { @@ -296,8 +287,7 @@ func TestTx_CreateBucketIfNotExists(t *testing.T) { // Ensure transaction returns an error if creating an unnamed bucket. func TestTx_CreateBucketIfNotExists_ErrBucketNameRequired(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucketIfNotExists([]byte{}); err != bolt.ErrBucketNameRequired { t.Fatalf("unexpected error: %s", err) @@ -315,8 +305,7 @@ func TestTx_CreateBucketIfNotExists_ErrBucketNameRequired(t *testing.T) { // Ensure that a bucket cannot be created twice. func TestTx_CreateBucket_ErrBucketExists(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) // Create a bucket. if err := db.Update(func(tx *bolt.Tx) error { @@ -341,8 +330,7 @@ func TestTx_CreateBucket_ErrBucketExists(t *testing.T) { // Ensure that a bucket is created with a non-blank name. func TestTx_CreateBucket_ErrBucketNameRequired(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if _, err := tx.CreateBucket(nil); err != bolt.ErrBucketNameRequired { t.Fatalf("unexpected error: %s", err) @@ -355,8 +343,7 @@ func TestTx_CreateBucket_ErrBucketNameRequired(t *testing.T) { // Ensure that a bucket can be deleted. func TestTx_DeleteBucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) // Create a bucket and add a value. if err := db.Update(func(tx *bolt.Tx) error { @@ -402,8 +389,7 @@ func TestTx_DeleteBucket(t *testing.T) { // Ensure that deleting a bucket on a closed transaction returns an error. func TestTx_DeleteBucket_ErrTxClosed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) tx, err := db.Begin(true) if err != nil { t.Fatal(err) @@ -418,8 +404,7 @@ func TestTx_DeleteBucket_ErrTxClosed(t *testing.T) { // Ensure that deleting a bucket with a read-only transaction returns an error. func TestTx_DeleteBucket_ReadOnly(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.View(func(tx *bolt.Tx) error { if err := tx.DeleteBucket([]byte("foo")); err != bolt.ErrTxNotWritable { t.Fatalf("unexpected error: %s", err) @@ -432,8 +417,7 @@ func TestTx_DeleteBucket_ReadOnly(t *testing.T) { // Ensure that nothing happens when deleting a bucket that doesn't exist. func TestTx_DeleteBucket_NotFound(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { if err := tx.DeleteBucket([]byte("widgets")); err != bolt.ErrBucketNotFound { t.Fatalf("unexpected error: %s", err) @@ -447,8 +431,7 @@ func TestTx_DeleteBucket_NotFound(t *testing.T) { // Ensure that no error is returned when a tx.ForEach function does not return // an error. func TestTx_ForEach_NoError(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -471,8 +454,7 @@ func TestTx_ForEach_NoError(t *testing.T) { // Ensure that an error is returned when a tx.ForEach function returns an error. func TestTx_ForEach_WithError(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -496,8 +478,7 @@ func TestTx_ForEach_WithError(t *testing.T) { // Ensure that Tx commit handlers are called after a transaction successfully commits. func TestTx_OnCommit(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) var x int if err := db.Update(func(tx *bolt.Tx) error { @@ -516,8 +497,7 @@ func TestTx_OnCommit(t *testing.T) { // Ensure that Tx commit handlers are NOT called after a transaction rolls back. func TestTx_OnCommit_Rollback(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) var x int if err := db.Update(func(tx *bolt.Tx) error { @@ -536,8 +516,7 @@ func TestTx_OnCommit_Rollback(t *testing.T) { // Ensure that the database can be copied to a file path. func TestTx_CopyFile(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) path := tempfile() if err := db.Update(func(tx *bolt.Tx) error { @@ -607,8 +586,7 @@ func (f *failWriter) Write(p []byte) (n int, err error) { // Ensure that Copy handles write errors right. func TestTx_CopyFile_Error_Meta(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -634,8 +612,7 @@ func TestTx_CopyFile_Error_Meta(t *testing.T) { // Ensure that Copy handles write errors right. func TestTx_CopyFile_Error_Normal(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { @@ -719,8 +696,7 @@ func TestTx_releaseRange(t *testing.T) { // Set initial mmap size well beyond the limit we will hit in this // test, since we are testing with long running read transactions // and will deadlock if db.grow is triggered. - db := MustOpenWithOption(&bolt.Options{InitialMmapSize: os.Getpagesize() * 100}) - defer db.MustClose() + db := btesting.MustCreateDBWithOption(t, &bolt.Options{InitialMmapSize: os.Getpagesize() * 100}) bucket := "bucket" diff --git a/unix_test.go b/unix_test.go index d39dfd820..927aa7a66 100644 --- a/unix_test.go +++ b/unix_test.go @@ -5,6 +5,7 @@ package bbolt_test import ( "fmt" + "go.etcd.io/bbolt/internal/btesting" "testing" bolt "go.etcd.io/bbolt" @@ -15,8 +16,7 @@ func TestMlock_DbOpen(t *testing.T) { // 32KB skipOnMemlockLimitBelow(t, 32*1024) - db := MustOpenWithOption(&bolt.Options{Mlock: true}) - defer db.MustClose() + btesting.MustCreateDBWithOption(t, &bolt.Options{Mlock: true}) } // Test change between "empty" (16KB) and "non-empty" db @@ -24,8 +24,7 @@ func TestMlock_DbCanGrow_Small(t *testing.T) { // 32KB skipOnMemlockLimitBelow(t, 32*1024) - db := MustOpenWithOption(&bolt.Options{Mlock: true}) - defer db.MustClose() + db := btesting.MustCreateDBWithOption(t, &bolt.Options{Mlock: true}) if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucketIfNotExists([]byte("bucket")) @@ -58,25 +57,24 @@ func TestMlock_DbCanGrow_Big(t *testing.T) { chunksBefore := 64 chunksAfter := 64 - db := MustOpenWithOption(&bolt.Options{Mlock: true}) - defer db.MustClose() + db := btesting.MustCreateDBWithOption(t, &bolt.Options{Mlock: true}) for chunk := 0; chunk < chunksBefore; chunk++ { insertChunk(t, db, chunk) } - dbSize := fileSize(db.f) + dbSize := fileSize(db.Path()) for chunk := 0; chunk < chunksAfter; chunk++ { insertChunk(t, db, chunksBefore+chunk) } - newDbSize := fileSize(db.f) + newDbSize := fileSize(db.Path()) if newDbSize <= dbSize { t.Errorf("db didn't grow: %v <= %v", newDbSize, dbSize) } } -func insertChunk(t *testing.T, db *DB, chunkId int) { +func insertChunk(t *testing.T, db *btesting.DB, chunkId int) { chunkSize := 1024 if err := db.Update(func(tx *bolt.Tx) error { From 93380c59e855e430f29d209c3f13d5e660e37e28 Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Wed, 28 Dec 2022 08:46:29 +0100 Subject: [PATCH 4/6] testing: group (reorder) imports. Signed-off-by: Piotr Tabor --- cursor_test.go | 2 +- go.mod | 2 ++ go.sum | 4 ++++ internal/btesting/btesting.go | 6 ++++-- simulation_test.go | 2 +- tx_test.go | 2 +- unix_test.go | 5 +++-- 7 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cursor_test.go b/cursor_test.go index d58a97152..8e112c14e 100644 --- a/cursor_test.go +++ b/cursor_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/binary" "fmt" - "go.etcd.io/bbolt/internal/btesting" "log" "os" "reflect" @@ -13,6 +12,7 @@ import ( "testing/quick" bolt "go.etcd.io/bbolt" + "go.etcd.io/bbolt/internal/btesting" ) // Ensure that a cursor can return a reference to the bucket that created it. diff --git a/go.mod b/go.mod index a58befa15..dd9fd703b 100644 --- a/go.mod +++ b/go.mod @@ -10,5 +10,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/mod v0.7.0 // indirect + golang.org/x/tools v0.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3d9dedd3e..1d1d2b208 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/btesting/btesting.go b/internal/btesting/btesting.go index 9c65352f1..110cc3486 100644 --- a/internal/btesting/btesting.go +++ b/internal/btesting/btesting.go @@ -3,13 +3,15 @@ package btesting import ( "flag" "fmt" - "github.com/stretchr/testify/require" - bolt "go.etcd.io/bbolt" "os" "path/filepath" "regexp" "testing" "time" + + "github.com/stretchr/testify/require" + + bolt "go.etcd.io/bbolt" ) var statsFlag = flag.Bool("stats", false, "show performance stats") diff --git a/simulation_test.go b/simulation_test.go index 3166aa398..037b7183c 100644 --- a/simulation_test.go +++ b/simulation_test.go @@ -3,13 +3,13 @@ package bbolt_test import ( "bytes" "fmt" - "go.etcd.io/bbolt/internal/btesting" "math/rand" "sync" "sync/atomic" "testing" bolt "go.etcd.io/bbolt" + "go.etcd.io/bbolt/internal/btesting" ) func TestSimulate_1op_1p(t *testing.T) { testSimulate(t, nil, 1, 1, 1) } diff --git a/tx_test.go b/tx_test.go index d4f0d7212..9ea16b0f2 100644 --- a/tx_test.go +++ b/tx_test.go @@ -4,12 +4,12 @@ import ( "bytes" "errors" "fmt" - "go.etcd.io/bbolt/internal/btesting" "log" "os" "testing" bolt "go.etcd.io/bbolt" + "go.etcd.io/bbolt/internal/btesting" ) // TestTx_Check_ReadOnly tests consistency checking on a ReadOnly database. diff --git a/unix_test.go b/unix_test.go index 927aa7a66..8924abf79 100644 --- a/unix_test.go +++ b/unix_test.go @@ -5,11 +5,12 @@ package bbolt_test import ( "fmt" - "go.etcd.io/bbolt/internal/btesting" "testing" - bolt "go.etcd.io/bbolt" "golang.org/x/sys/unix" + + bolt "go.etcd.io/bbolt" + "go.etcd.io/bbolt/internal/btesting" ) func TestMlock_DbOpen(t *testing.T) { From 37d72cc1cd9f9173a2471229e38286423f4a0f05 Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Wed, 28 Dec 2022 13:46:46 +0100 Subject: [PATCH 5/6] Use require.NoError in btesting.go Signed-off-by: Piotr Tabor --- db_test.go | 10 ++-------- internal/btesting/btesting.go | 32 ++++++++++++-------------------- tx_test.go | 1 - 3 files changed, 14 insertions(+), 29 deletions(-) diff --git a/db_test.go b/db_test.go index d36d280e9..2b50e10ec 100644 --- a/db_test.go +++ b/db_test.go @@ -291,7 +291,7 @@ func TestOpen_Size_Large(t *testing.T) { } // Close database and grab the size. - if err := db.DB.Close(); err != nil { + if err := db.Close(); err != nil { t.Fatal(err) } sz := fileSize(path) @@ -463,7 +463,7 @@ func TestDB_Open_ReadOnly(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := db.DB.Close(); err != nil { + if err := db.Close(); err != nil { t.Fatal(err) } @@ -556,9 +556,6 @@ func TestOpen_RecoverFreeList(t *testing.T) { if err := tx.Commit(); err != nil { t.Fatal(err) } - if err := db.DB.Close(); err != nil { - t.Fatal(err) - } db.MustClose() // Record freelist count from opening with NoFreelistSync. @@ -567,9 +564,6 @@ func TestOpen_RecoverFreeList(t *testing.T) { if freepages == 0 { t.Fatalf("no free pages on NoFreelistSync reopen") } - if err := db.DB.Close(); err != nil { - t.Fatal(err) - } db.MustClose() // Check free page count is reconstructed when opened with freelist sync. diff --git a/internal/btesting/btesting.go b/internal/btesting/btesting.go index 110cc3486..e070ec7e5 100644 --- a/internal/btesting/btesting.go +++ b/internal/btesting/btesting.go @@ -52,9 +52,7 @@ func MustOpenDBWithOption(t testing.TB, f string, o *bolt.Options) *DB { o.FreelistType = freelistType db, err := bolt.Open(f, 0666, o) - if err != nil { - panic(err) - } + require.NoError(t, err) resDB := &DB{ DB: db, f: f, @@ -92,9 +90,8 @@ func (db *DB) Close() error { // MustClose closes the database but does NOT delete the underlying file. func (db *DB) MustClose() { - if err := db.Close(); err != nil { - panic(err) - } + err := db.Close() + require.NoError(db.t, err) } func (db *DB) MustDeleteFile() { @@ -113,15 +110,13 @@ func (db *DB) MustReopen() { } db.t.Logf("Reopening bbolt DB at: %s", db.f) indb, err := bolt.Open(db.Path(), 0666, db.o) - if err != nil { - panic(err) - } + require.NoError(db.t, err) db.DB = indb } // MustCheck runs a consistency check on the database and panics if any errors are found. func (db *DB) MustCheck() { - if err := db.Update(func(tx *bolt.Tx) error { + err := db.Update(func(tx *bolt.Tx) error { // Collect all the errors. var errors []error for err := range tx.Check() { @@ -134,9 +129,8 @@ func (db *DB) MustCheck() { // If errors occurred, copy the DB and print the errors. if len(errors) > 0 { var path = filepath.Join(db.t.TempDir(), "db.backup") - if err := tx.CopyFile(path, 0600); err != nil { - panic(err) - } + err := tx.CopyFile(path, 0600) + require.NoError(db.t, err) // Print errors. fmt.Print("\n\n") @@ -152,9 +146,8 @@ func (db *DB) MustCheck() { } return nil - }); err != nil && err != bolt.ErrDatabaseNotOpen { - panic(err) - } + }) + require.NoError(db.t, err) } // Fill - fills the DB using numTx transactions and numKeysPerTx. @@ -185,11 +178,10 @@ func (db *DB) Path() string { // CopyTempFile copies a database to a temporary file. func (db *DB) CopyTempFile() { path := filepath.Join(db.t.TempDir(), "db.copy") - if err := db.View(func(tx *bolt.Tx) error { + err := db.View(func(tx *bolt.Tx) error { return tx.CopyFile(path, 0600) - }); err != nil { - panic(err) - } + }) + require.NoError(db.t, err) fmt.Println("db copied to: ", path) } diff --git a/tx_test.go b/tx_test.go index 9ea16b0f2..7cfcef4f5 100644 --- a/tx_test.go +++ b/tx_test.go @@ -15,7 +15,6 @@ import ( // TestTx_Check_ReadOnly tests consistency checking on a ReadOnly database. func TestTx_Check_ReadOnly(t *testing.T) { db := btesting.MustCreateDB(t) - defer db.Close() if err := db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucket([]byte("widgets")) if err != nil { From 1f46d6c7e731b5c777ea512fd74425c80cef44ff Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Wed, 28 Dec 2022 13:55:09 +0100 Subject: [PATCH 6/6] cmd/main_test.go is using btesting as well. Signed-off-by: Piotr Tabor --- cmd/bbolt/main_test.go | 100 +++++++++++++---------------------------- 1 file changed, 32 insertions(+), 68 deletions(-) diff --git a/cmd/bbolt/main_test.go b/cmd/bbolt/main_test.go index f65698023..fb0c8f750 100644 --- a/cmd/bbolt/main_test.go +++ b/cmd/bbolt/main_test.go @@ -5,6 +5,7 @@ import ( crypto "crypto/rand" "encoding/binary" "fmt" + "go.etcd.io/bbolt/internal/btesting" "io" "math/rand" "os" @@ -18,15 +19,14 @@ import ( // Ensure the "info" command can print information about a database. func TestInfoCommand_Run(t *testing.T) { - db := MustOpen(0666, nil) - db.DB.Close() - defer db.Close() + db := btesting.MustCreateDB(t) + db.Close() - defer requireDBNoChange(t, dbData(t, db.Path), db.Path) + defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) // Run the info command. m := NewMain() - if err := m.Run("info", db.Path); err != nil { + if err := m.Run("info", db.Path()); err != nil { t.Fatal(err) } } @@ -38,11 +38,10 @@ func TestStatsCommand_Run_EmptyDatabase(t *testing.T) { t.Skip("system does not use 4KB page size") } - db := MustOpen(0666, nil) - defer db.Close() - db.DB.Close() + db := btesting.MustCreateDB(t) + db.Close() - defer requireDBNoChange(t, dbData(t, db.Path), db.Path) + defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) // Generate expected result. exp := "Aggregate statistics for 0 buckets\n\n" + @@ -66,7 +65,7 @@ func TestStatsCommand_Run_EmptyDatabase(t *testing.T) { // Run the command. m := NewMain() - if err := m.Run("stats", db.Path); err != nil { + if err := m.Run("stats", db.Path()); err != nil { t.Fatal(err) } else if m.Stdout.String() != exp { t.Fatalf("unexpected stdout:\n\n%s", m.Stdout.String()) @@ -80,8 +79,7 @@ func TestStatsCommand_Run(t *testing.T) { t.Skip("system does not use 4KB page size") } - db := MustOpen(0666, nil) - defer db.Close() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { // Create "foo" bucket. @@ -119,9 +117,9 @@ func TestStatsCommand_Run(t *testing.T) { }); err != nil { t.Fatal(err) } - db.DB.Close() + db.Close() - defer requireDBNoChange(t, dbData(t, db.Path), db.Path) + defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) // Generate expected result. exp := "Aggregate statistics for 3 buckets\n\n" + @@ -145,7 +143,7 @@ func TestStatsCommand_Run(t *testing.T) { // Run the command. m := NewMain() - if err := m.Run("stats", db.Path); err != nil { + if err := m.Run("stats", db.Path()); err != nil { t.Fatal(err) } else if m.Stdout.String() != exp { t.Fatalf("unexpected stdout:\n\n%s", m.Stdout.String()) @@ -154,8 +152,7 @@ func TestStatsCommand_Run(t *testing.T) { // Ensure the "buckets" command can print a list of buckets. func TestBucketsCommand_Run(t *testing.T) { - db := MustOpen(0666, nil) - defer db.Close() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { for _, name := range []string{"foo", "bar", "baz"} { @@ -168,15 +165,15 @@ func TestBucketsCommand_Run(t *testing.T) { }); err != nil { t.Fatal(err) } - db.DB.Close() + db.Close() - defer requireDBNoChange(t, dbData(t, db.Path), db.Path) + defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) expected := "bar\nbaz\nfoo\n" // Run the command. m := NewMain() - if err := m.Run("buckets", db.Path); err != nil { + if err := m.Run("buckets", db.Path()); err != nil { t.Fatal(err) } else if actual := m.Stdout.String(); actual != expected { t.Fatalf("unexpected stdout:\n\n%s", actual) @@ -185,8 +182,7 @@ func TestBucketsCommand_Run(t *testing.T) { // Ensure the "keys" command can print a list of keys for a bucket. func TestKeysCommand_Run(t *testing.T) { - db := MustOpen(0666, nil) - defer db.Close() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { for _, name := range []string{"foo", "bar"} { @@ -205,15 +201,15 @@ func TestKeysCommand_Run(t *testing.T) { }); err != nil { t.Fatal(err) } - db.DB.Close() + db.Close() - defer requireDBNoChange(t, dbData(t, db.Path), db.Path) + defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) expected := "foo-0\nfoo-1\nfoo-2\n" // Run the command. m := NewMain() - if err := m.Run("keys", db.Path, "foo"); err != nil { + if err := m.Run("keys", db.Path(), "foo"); err != nil { t.Fatal(err) } else if actual := m.Stdout.String(); actual != expected { t.Fatalf("unexpected stdout:\n\n%s", actual) @@ -222,8 +218,7 @@ func TestKeysCommand_Run(t *testing.T) { // Ensure the "get" command can print the value of a key in a bucket. func TestGetCommand_Run(t *testing.T) { - db := MustOpen(0666, nil) - defer db.Close() + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { for _, name := range []string{"foo", "bar"} { @@ -243,15 +238,15 @@ func TestGetCommand_Run(t *testing.T) { }); err != nil { t.Fatal(err) } - db.DB.Close() + db.Close() - defer requireDBNoChange(t, dbData(t, db.Path), db.Path) + defer requireDBNoChange(t, dbData(t, db.Path()), db.Path()) expected := "val-foo-1\n" // Run the command. m := NewMain() - if err := m.Run("get", db.Path, "foo", "foo-1"); err != nil { + if err := m.Run("get", db.Path(), "foo", "foo-1"); err != nil { t.Fatal(err) } else if actual := m.Stdout.String(); actual != expected { t.Fatalf("unexpected stdout:\n\n%s", actual) @@ -275,32 +270,6 @@ func NewMain() *Main { return m } -// MustOpen creates a Bolt database in a temporary location. -func MustOpen(mode os.FileMode, options *bolt.Options) *DB { - // Create temporary path. - f, _ := os.CreateTemp("", "bolt-") - f.Close() - os.Remove(f.Name()) - - db, err := bolt.Open(f.Name(), mode, options) - if err != nil { - panic(err.Error()) - } - return &DB{DB: db, Path: f.Name()} -} - -// DB is a test wrapper for bolt.DB. -type DB struct { - *bolt.DB - Path string -} - -// Close closes and removes the database. -func (db *DB) Close() error { - defer os.Remove(db.Path) - return db.DB.Close() -} - func TestCompactCommand_Run(t *testing.T) { var s int64 if err := binary.Read(crypto.Reader, binary.BigEndian, &s); err != nil { @@ -308,11 +277,11 @@ func TestCompactCommand_Run(t *testing.T) { } rand.Seed(s) - dstdb := MustOpen(0666, nil) + dstdb := btesting.MustCreateDB(t) dstdb.Close() // fill the db - db := MustOpen(0666, nil) + db := btesting.MustCreateDB(t) if err := db.Update(func(tx *bolt.Tx) error { n := 2 + rand.Intn(5) for i := 0; i < n; i++ { @@ -330,7 +299,6 @@ func TestCompactCommand_Run(t *testing.T) { } return nil }); err != nil { - db.Close() t.Fatal(err) } @@ -353,7 +321,6 @@ func TestCompactCommand_Run(t *testing.T) { } return nil }); err != nil { - db.Close() t.Fatal(err) } if err := db.Update(func(tx *bolt.Tx) error { @@ -365,29 +332,26 @@ func TestCompactCommand_Run(t *testing.T) { } return tx.DeleteBucket([]byte("large_vals")) }); err != nil { - db.Close() t.Fatal(err) } - db.DB.Close() - defer db.Close() - defer dstdb.Close() + db.Close() - dbChk, err := chkdb(db.Path) + dbChk, err := chkdb(db.Path()) if err != nil { t.Fatal(err) } m := NewMain() - if err := m.Run("compact", "-o", dstdb.Path, db.Path); err != nil { + if err := m.Run("compact", "-o", dstdb.Path(), db.Path()); err != nil { t.Fatal(err) } - dbChkAfterCompact, err := chkdb(db.Path) + dbChkAfterCompact, err := chkdb(db.Path()) if err != nil { t.Fatal(err) } - dstdbChk, err := chkdb(dstdb.Path) + dstdbChk, err := chkdb(dstdb.Path()) if err != nil { t.Fatal(err) }