add fixing code using checkpoint files

siva/checkpoint_test.go

@@ -0,0 +1,71 @@
+package siva
+
+import (
+	"io/ioutil"
+	"os"
+	"strconv"
+	"testing"
+
+	borges "github.com/src-d/go-borges"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/src-d/go-billy.v4/memfs"
+	"gopkg.in/src-d/go-billy.v4/util"
+)
+
+func TestCheckpoint(t *testing.T) {
+	require := require.New(t)
+
+	sivaData, err := ioutil.ReadFile("../_testdata/siva/foo-bar.siva")
+	require.NoError(err)
+
+	fs := memfs.New()
+	lib := NewLibrary("test", fs)
+
+	var l borges.Location
+
+	// correct file
+
+	err = util.WriteFile(fs, "correct.siva", sivaData, 0666)
+	require.NoError(err)
+	l, err = lib.Location("correct")
+	require.NoError(err)
+	_, err = l.Get("github.com/foo/bar", borges.ReadOnlyMode)
+	require.NoError(err)
+
+	// broken file with correct checkpoint file
+
+	size := strconv.Itoa(len(sivaData))
+	err = util.WriteFile(fs, "broken.siva.checkpoint", []byte(size), 0666)
+	require.NoError(err)
+	brokenData := append(sivaData[:], []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0}...)
+	err = util.WriteFile(fs, "broken.siva", brokenData, 0666)
+	require.NoError(err)
+	l, err = lib.Location("broken")
+	require.NoError(err)
+	_, err = l.Get("github.com/foo/bar", borges.ReadOnlyMode)
+	require.NoError(err)
+	_, err = fs.Stat("broken.siva.checkpoint")
+	require.True(os.IsNotExist(err))
+
+	// dangling checkpoint file
+
+	size = strconv.Itoa(len(sivaData))
+	err = util.WriteFile(fs, "dangling.siva.checkpoint", []byte(size), 0666)
+	require.NoError(err)
+	l, err = lib.Location("dangling")
+	require.True(borges.ErrLocationNotExists.Is(err))
+	_, err = fs.Stat("dangling.siva.checkpoint")
+	require.True(os.IsNotExist(err))
+
+	// broken siva file without checkpoint
+
+	// TODO: there's a bug in memfs and it crashes with this test. This will be
+	// enabled after it is fixed (check negative offsets in ReadAt/WriteAt).
+
+	// err = util.WriteFile(fs, "really_broken.siva", brokenData, 0666)
+	// require.NoError(err)
+	// l, err = lib.Location("really_broken")
+	// require.NoError(err)
+	// _, err = l.Get("github.com/foo/bar", borges.ReadOnlyMode)
+	// require.True(borges.ErrLocationNotExists.Is(err))
+}

siva/library.go

@@ -3,7 +3,6 @@ package siva
 import (
 	"fmt"
 	"io"
-	"os"
 	"strings"
 
 	borges "github.com/src-d/go-borges"
@@ -113,16 +112,7 @@ func (l *Library) Repositories(mode borges.Mode) (borges.RepositoryIterator, err
 
 // Location implements borges.Library interface.
 func (l *Library) Location(id borges.LocationID) (borges.Location, error) {
-	return l.generateLocation(id)
-}
-
-func (l *Library) generateLocation(id borges.LocationID) (*Location, error) {
 	path := fmt.Sprintf("%s.siva", id)
-	_, err := l.fs.Stat(path)
-	if os.IsNotExist(err) {
-		return nil, borges.ErrLocationNotExists.New(id)
-	}
-
 	return NewLocation(id, l, path)
 }
 
@@ -145,7 +135,7 @@ func (l *Library) locations() ([]borges.Location, error) {
 	}
 
 	for _, s := range sivas {
-		loc, err := l.generateLocation(toLocID(s))
+		loc, err := l.Location(toLocID(s))
 		if err != nil {
 			continue
 		}

siva/location.go

@@ -1,45 +1,165 @@
 package siva
 
 import (
+	"fmt"
 	"io"
 	"os"
+	"strconv"
 
 	borges "github.com/src-d/go-borges"
 	sivafs "gopkg.in/src-d/go-billy-siva.v4"
 	billy "gopkg.in/src-d/go-billy.v4"
 	"gopkg.in/src-d/go-billy.v4/memfs"
+	"gopkg.in/src-d/go-errors.v1"
 	"gopkg.in/src-d/go-git.v4/config"
 )
 
+var (
+	// ErrCannotUseCheckpointFile is returned on checkpoint problems.
+	ErrCannotUseCheckpointFile = errors.NewKind("cannot use checkpoint file: %s")
+	// ErrCannotUseSivaFile is returned on siva problems.
+	ErrCannotUseSivaFile = errors.NewKind("cannot use siva file: %s")
+	// ErrMalformedData when checkpoint data is invalid.
+	ErrMalformedData = errors.NewKind("malformed data")
+)
 var _ borges.Location = new(Location)
 
 func NewLocation(
 	id borges.LocationID,
 	l *Library,
 	path string,
 ) (*Location, error) {
-	_, err := l.fs.Stat(path)
+	err := fixSiva(l.fs, path)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = l.fs.Stat(path)
 	if os.IsNotExist(err) {
 		return nil, borges.ErrLocationNotExists.New(id)
 	}
 
-	sfs, err := sivafs.NewFilesystem(l.fs, path, memfs.New())
+	location := &Location{
+		id:      id,
+		path:    path,
+		library: l,
+	}
+
+	_, err = location.FS()
 	if err != nil {
 		return nil, err
 	}
 
-	return &Location{id: id, fs: sfs, library: l}, nil
+	return location, nil
 }
 
 type Location struct {
 	id            borges.LocationID
-	fs            billy.Filesystem
+	path          string
+	cachedFS      billy.Filesystem
 	transactional bool
 	library       *Library
 }
 
 var _ borges.Location = (*Location)(nil)
 
+// fixSiva searches for a file named path.checkpoint. If it's found it truncates
+// the siva file to the size written in it.
+func fixSiva(fs billy.Filesystem, path string) error {
+	checkpointPath := fmt.Sprintf("%s.checkpoint", path)
+
+	checkErr := func(err error) error {
+		return ErrCannotUseCheckpointFile.Wrap(err, checkpointPath)
+	}
+	sivaErr := func(err error) error {
+		return ErrCannotUseSivaFile.Wrap(err, path)
+	}
+
+	cf, err := fs.Open(checkpointPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return checkErr(err)
+	}
+	defer cf.Close()
+
+	// there's a checkpoint file we can use to fix the siva file
+
+	sf, err := fs.Open(path)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return sivaErr(err)
+		}
+
+		// there's checkpoint but not siva file, delete checkpoint
+		err = cf.Close()
+		if err != nil {
+			return checkErr(err)
+		}
+
+		err = fs.Remove(checkpointPath)
+		if err != nil {
+			return checkErr(err)
+		}
+
+		return nil
+	}
+	defer sf.Close()
+
+	// the biggest 64 bit number in decimal ASCII is 19 characters
+	data := make([]byte, 32)
+	n, err := cf.Read(data)
+	if err != nil {
+		return checkErr(err)
+	}
+
+	size, err := strconv.ParseInt(string(data[:n]), 10, 64)
+	if err != nil {
+		return checkErr(err)
+	}
+	if size < 0 {
+		return checkErr(ErrMalformedData.New())
+	}
+
+	err = sf.Truncate(size)
+	if err != nil {
+		return sivaErr(err)
+	}
+
+	err = cf.Close()
+	if err != nil {
+		return checkErr(err)
+	}
+
+	err = fs.Remove(checkpointPath)
+	if err != nil {
+		return checkErr(err)
+	}
+
+	return nil
+}
+
+// FS returns a filesystem for the location's siva file.
+func (l *Location) FS() (billy.Filesystem, error) {
+	if l.cachedFS != nil {
+		return l.cachedFS, nil
+	}
+
+	err := fixSiva(l.library.fs, l.path)
+	if err != nil {
+		return nil, err
+	}
+
+	sfs, err := sivafs.NewFilesystem(l.library.fs, l.path, memfs.New())
+	if err != nil {
+		return nil, err
+	}
+
+	l.cachedFS = sfs
+	return sfs, nil
+}
+
 func (l *Location) ID() borges.LocationID {
 	return l.id
 }
@@ -53,7 +173,12 @@ func (l *Location) Init(id borges.RepositoryID) (borges.Repository, error) {
 		return nil, borges.ErrRepositoryExists.New(id)
 	}
 
-	repo, err := NewRepository(id, l.fs, borges.RWMode, l)
+	fs, err := l.FS()
+	if err != nil {
+		return nil, err
+	}
+
+	repo, err := NewRepository(id, fs, borges.RWMode, l)
 	if err != nil {
 		return nil, err
 	}
@@ -147,7 +272,12 @@ func (l *Location) repository(
 	id borges.RepositoryID,
 	mode borges.Mode,
 ) (borges.Repository, error) {
-	return NewRepository(id, l.fs, mode, l)
+	fs, err := l.FS()
+	if err != nil {
+		return nil, err
+	}
+
+	return NewRepository(id, fs, mode, l)
 }
 
 type repositoryIterator struct {
@@ -170,8 +300,13 @@ func (i *repositoryIterator) Next() (borges.Repository, error) {
 			continue
 		}
 
+		fs, err := i.l.FS()
+		if err != nil {
+			return nil, err
+		}
+
 		id := toRepoID(r.URLs[0])
-		return NewRepository(id, i.l.fs, i.mode, i.l)
+		return NewRepository(id, fs, i.mode, i.l)
 	}
 }
 

siva/repository.go

@@ -2,9 +2,10 @@ package siva
 
 import (
 	borges "github.com/src-d/go-borges"
-	"github.com/src-d/go-borges/util"
 	billy "gopkg.in/src-d/go-billy.v4"
 	git "gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem"
 )
 
 type Repository struct {
@@ -24,14 +25,10 @@ func NewRepository(
 	m borges.Mode,
 	l *Location,
 ) (*Repository, error) {
-	sto, _, err := util.RepositoryStorer(fs, l.library.fs, m, l.transactional)
-	if err != nil {
-		return nil, err
-	}
-
+	sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault())
 	repo, err := git.Open(sto, nil)
 	if err != nil {
-		return nil, err
+		return nil, borges.ErrLocationNotExists.New(id)
 	}
 
 	return &Repository{