From 386b851495d42c4e02908838373a06d0a533e170 Mon Sep 17 00:00:00 2001 From: Anthony Romano Date: Thu, 16 Nov 2017 07:57:56 -0800 Subject: [PATCH 1/2] freelist: set alloc tx for freelist to prior txn Was causing freelist corruption on tx.WriteTo --- freelist.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freelist.go b/freelist.go index 318cc3bd2..266f15429 100644 --- a/freelist.go +++ b/freelist.go @@ -132,9 +132,9 @@ func (f *freelist) free(txid txid, p *page) { allocTxid, ok := f.allocs[p.id] if ok { delete(f.allocs, p.id) - } else if (p.flags & (freelistPageFlag | metaPageFlag)) != 0 { - // Safe to claim txid as allocating since these types are private to txid. - allocTxid = txid + } else if (p.flags & freelistPageFlag) != 0 { + // Freelist is always allocated by prior tx. + allocTxid = txid - 1 } for id := p.id; id <= p.id+pgid(p.overflow); id++ { From 41fefe7322263b61e5669a9bdd136570c15c0c69 Mon Sep 17 00:00:00 2001 From: Anthony Romano Date: Thu, 16 Nov 2017 07:58:34 -0800 Subject: [PATCH 2/2] test: check concurrent WriteTo operations aren't corrupted Reliably triggers consistency check failures on ramdisk without freelist free fix. --- db_test.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/db_test.go b/db_test.go index 629150c9e..e3a58c3ca 100644 --- a/db_test.go +++ b/db_test.go @@ -9,6 +9,7 @@ import ( "hash/fnv" "io/ioutil" "log" + "math/rand" "os" "path/filepath" "regexp" @@ -588,6 +589,65 @@ func TestDB_BeginRW(t *testing.T) { } } +// TestDB_Concurrent_WriteTo checks that issuing WriteTo operations concurrently +// with commits does not produce corrupted db files. +func TestDB_Concurrent_WriteTo(t *testing.T) { + o := &bolt.Options{NoFreelistSync: false} + db := MustOpenWithOption(o) + defer db.MustClose() + + var wg sync.WaitGroup + wtxs, rtxs := 5, 5 + wg.Add(wtxs * rtxs) + f := func(tx *bolt.Tx) { + defer wg.Done() + f, err := ioutil.TempFile("", "bolt-") + if err != nil { + panic(err) + } + time.Sleep(time.Duration(rand.Intn(20)+1) * time.Millisecond) + tx.WriteTo(f) + tx.Rollback() + f.Close() + snap := &DB{nil, f.Name(), o} + snap.MustReopen() + defer snap.MustClose() + snap.MustCheck() + } + + tx1, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + if _, err := tx1.CreateBucket([]byte("abc")); err != nil { + t.Fatal(err) + } + if err := tx1.Commit(); err != nil { + t.Fatal(err) + } + + for i := 0; i < wtxs; i++ { + tx, err := db.Begin(true) + if err != nil { + t.Fatal(err) + } + if err := tx.Bucket([]byte("abc")).Put([]byte{0}, []byte{0}); err != nil { + t.Fatal(err) + } + for j := 0; j < rtxs; j++ { + rtx, rerr := db.Begin(false) + if rerr != nil { + t.Fatal(rerr) + } + go f(rtx) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + } + wg.Wait() +} + // Ensure that opening a transaction while the DB is closed returns an error. func TestDB_BeginRW_Closed(t *testing.T) { var db bolt.DB