diff --git a/mvcc/backend/backend.go b/mvcc/backend/backend.go index a240c8e6280..b1379edeb34 100644 --- a/mvcc/backend/backend.go +++ b/mvcc/backend/backend.go @@ -307,21 +307,36 @@ func (b *backend) defrag() error { b.batchTx.unsafeCommit(true) b.batchTx.tx = nil - tmpdb, err := bolt.Open(b.db.Path()+".tmp", 0600, boltOpenOptions) + // Create a temporary file to ensure we start with a clean slate. + // Snapshotter.cleanupSnapdir cleans up any of these that are found during startup. + dir := filepath.Dir(b.db.Path()) + temp, err := ioutil.TempFile(dir, "db.tmp.*") + if err != nil { + return err + } + // bbolt v1.3.1-coreos.5 does not provide options.OpenFile so we + // instead close the temp file and then let bolt reopen it. + tdbp := temp.Name() + if err := temp.Close(); err != nil { + return err + } + tmpdb, err := bolt.Open(tdbp, 0600, boltOpenOptions) if err != nil { return err } + // gofail: var defragBeforeCopy struct{} err = defragdb(b.db, tmpdb, defragLimit) if err != nil { tmpdb.Close() - os.RemoveAll(tmpdb.Path()) + if rmErr := os.RemoveAll(tmpdb.Path()); rmErr != nil { + plog.Fatalf("failed to remove db.tmp after defragmentation completed: %v", rmErr) + } return err } dbp := b.db.Path() - tdbp := tmpdb.Path() err = b.db.Close() if err != nil { @@ -331,6 +346,7 @@ func (b *backend) defrag() error { if err != nil { plog.Fatalf("cannot close database (%s)", err) } + // gofail: var defragBeforeRename struct{} err = os.Rename(tdbp, dbp) if err != nil { plog.Fatalf("cannot rename database (%s)", err) diff --git a/snap/snapshotter.go b/snap/snapshotter.go index 00755592129..5d79b175915 100644 --- a/snap/snapshotter.go +++ b/snap/snapshotter.go @@ -172,6 +172,9 @@ func (s *Snapshotter) snapNames() ([]string, error) { if err != nil { return nil, err } + if err = s.cleanupSnapdir(names); err != nil { + return nil, err + } snaps := checkSuffix(names) if len(snaps) == 0 { return nil, ErrNoSnapshot @@ -202,3 +205,17 @@ func renameBroken(path string) { plog.Warningf("cannot rename broken snapshot file %v to %v: %v", path, brokenPath, err) } } + +// cleanupSnapdir removes any files that should not be in the snapshot directory: +// - db.tmp prefixed files that can be orphaned by defragmentation +func (s *Snapshotter) cleanupSnapdir(filenames []string) error { + for _, filename := range filenames { + if strings.HasPrefix(filename, "db.tmp") { + plog.Infof("found orphaned defragmentation file; deleting: %s", filename) + if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) { + return fmt.Errorf("failed to remove orphaned defragmentation file %s: %v", filename, rmErr) + } + } + } + return nil +}