From f37005225062725e3e9dacfc6186183828fb7468 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 14 Feb 2019 14:22:19 +0100 Subject: [PATCH] add fixing code using checkpoint files If a *.siva.checkpoint is found it tries to fix the siva file truncating it to the size written in the checkpoint. Signed-off-by: Javi Fontan --- siva/checkpoint_test.go | 71 +++++++++++++++++++ siva/library.go | 12 +--- siva/location.go | 149 ++++++++++++++++++++++++++++++++++++++-- siva/repository.go | 11 ++- 4 files changed, 218 insertions(+), 25 deletions(-) create mode 100644 siva/checkpoint_test.go diff --git a/siva/checkpoint_test.go b/siva/checkpoint_test.go new file mode 100644 index 0000000..9671965 --- /dev/null +++ b/siva/checkpoint_test.go @@ -0,0 +1,71 @@ +package siva + +import ( + "io/ioutil" + "os" + "strconv" + "testing" + + borges "github.com/src-d/go-borges" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-billy.v4/util" +) + +func TestCheckpoint(t *testing.T) { + require := require.New(t) + + sivaData, err := ioutil.ReadFile("../_testdata/siva/foo-bar.siva") + require.NoError(err) + + fs := memfs.New() + lib := NewLibrary("test", fs) + + var l borges.Location + + // correct file + + err = util.WriteFile(fs, "correct.siva", sivaData, 0666) + require.NoError(err) + l, err = lib.Location("correct") + require.NoError(err) + _, err = l.Get("github.com/foo/bar", borges.ReadOnlyMode) + require.NoError(err) + + // broken file with correct checkpoint file + + size := strconv.Itoa(len(sivaData)) + err = util.WriteFile(fs, "broken.siva.checkpoint", []byte(size), 0666) + require.NoError(err) + brokenData := append(sivaData[:], []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0}...) + err = util.WriteFile(fs, "broken.siva", brokenData, 0666) + require.NoError(err) + l, err = lib.Location("broken") + require.NoError(err) + _, err = l.Get("github.com/foo/bar", borges.ReadOnlyMode) + require.NoError(err) + _, err = fs.Stat("broken.siva.checkpoint") + require.True(os.IsNotExist(err)) + + // dangling checkpoint file + + size = strconv.Itoa(len(sivaData)) + err = util.WriteFile(fs, "dangling.siva.checkpoint", []byte(size), 0666) + require.NoError(err) + l, err = lib.Location("dangling") + require.True(borges.ErrLocationNotExists.Is(err)) + _, err = fs.Stat("dangling.siva.checkpoint") + require.True(os.IsNotExist(err)) + + // broken siva file without checkpoint + + // TODO: there's a bug in memfs and it crashes with this test. This will be + // enabled after it is fixed (check negative offsets in ReadAt/WriteAt). + + // err = util.WriteFile(fs, "really_broken.siva", brokenData, 0666) + // require.NoError(err) + // l, err = lib.Location("really_broken") + // require.NoError(err) + // _, err = l.Get("github.com/foo/bar", borges.ReadOnlyMode) + // require.True(borges.ErrLocationNotExists.Is(err)) +} diff --git a/siva/library.go b/siva/library.go index 68785b6..7352406 100644 --- a/siva/library.go +++ b/siva/library.go @@ -3,7 +3,6 @@ package siva import ( "fmt" "io" - "os" "strings" borges "github.com/src-d/go-borges" @@ -113,16 +112,7 @@ func (l *Library) Repositories(mode borges.Mode) (borges.RepositoryIterator, err // Location implements borges.Library interface. func (l *Library) Location(id borges.LocationID) (borges.Location, error) { - return l.generateLocation(id) -} - -func (l *Library) generateLocation(id borges.LocationID) (*Location, error) { path := fmt.Sprintf("%s.siva", id) - _, err := l.fs.Stat(path) - if os.IsNotExist(err) { - return nil, borges.ErrLocationNotExists.New(id) - } - return NewLocation(id, l, path) } @@ -145,7 +135,7 @@ func (l *Library) locations() ([]borges.Location, error) { } for _, s := range sivas { - loc, err := l.generateLocation(toLocID(s)) + loc, err := l.Location(toLocID(s)) if err != nil { continue } diff --git a/siva/location.go b/siva/location.go index 785317e..a245885 100644 --- a/siva/location.go +++ b/siva/location.go @@ -1,16 +1,27 @@ package siva import ( + "fmt" "io" "os" + "strconv" borges "github.com/src-d/go-borges" sivafs "gopkg.in/src-d/go-billy-siva.v4" billy "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-errors.v1" "gopkg.in/src-d/go-git.v4/config" ) +var ( + // ErrCannotUseCheckpointFile is returned on checkpoint problems. + ErrCannotUseCheckpointFile = errors.NewKind("cannot use checkpoint file: %s") + // ErrCannotUseSivaFile is returned on siva problems. + ErrCannotUseSivaFile = errors.NewKind("cannot use siva file: %s") + // ErrMalformedData when checkpoint data is invalid. + ErrMalformedData = errors.NewKind("malformed data") +) var _ borges.Location = new(Location) func NewLocation( @@ -18,28 +29,137 @@ func NewLocation( l *Library, path string, ) (*Location, error) { - _, err := l.fs.Stat(path) + err := fixSiva(l.fs, path) + if err != nil { + return nil, err + } + + _, err = l.fs.Stat(path) if os.IsNotExist(err) { return nil, borges.ErrLocationNotExists.New(id) } - sfs, err := sivafs.NewFilesystem(l.fs, path, memfs.New()) + location := &Location{ + id: id, + path: path, + library: l, + } + + _, err = location.FS() if err != nil { return nil, err } - return &Location{id: id, fs: sfs, library: l}, nil + return location, nil } type Location struct { id borges.LocationID - fs billy.Filesystem + path string + cachedFS billy.Filesystem transactional bool library *Library } var _ borges.Location = (*Location)(nil) +// fixSiva searches for a file named path.checkpoint. If it's found it truncates +// the siva file to the size written in it. +func fixSiva(fs billy.Filesystem, path string) error { + checkpointPath := fmt.Sprintf("%s.checkpoint", path) + + checkErr := func(err error) error { + return ErrCannotUseCheckpointFile.Wrap(err, checkpointPath) + } + sivaErr := func(err error) error { + return ErrCannotUseSivaFile.Wrap(err, path) + } + + cf, err := fs.Open(checkpointPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return checkErr(err) + } + defer cf.Close() + + // there's a checkpoint file we can use to fix the siva file + + sf, err := fs.Open(path) + if err != nil { + if !os.IsNotExist(err) { + return sivaErr(err) + } + + // there's checkpoint but not siva file, delete checkpoint + err = cf.Close() + if err != nil { + return checkErr(err) + } + + err = fs.Remove(checkpointPath) + if err != nil { + return checkErr(err) + } + + return nil + } + defer sf.Close() + + // the biggest 64 bit number in decimal ASCII is 19 characters + data := make([]byte, 32) + n, err := cf.Read(data) + if err != nil { + return checkErr(err) + } + + size, err := strconv.ParseInt(string(data[:n]), 10, 64) + if err != nil { + return checkErr(err) + } + if size < 0 { + return checkErr(ErrMalformedData.New()) + } + + err = sf.Truncate(size) + if err != nil { + return sivaErr(err) + } + + err = cf.Close() + if err != nil { + return checkErr(err) + } + + err = fs.Remove(checkpointPath) + if err != nil { + return checkErr(err) + } + + return nil +} + +// FS returns a filesystem for the location's siva file. +func (l *Location) FS() (billy.Filesystem, error) { + if l.cachedFS != nil { + return l.cachedFS, nil + } + + err := fixSiva(l.library.fs, l.path) + if err != nil { + return nil, err + } + + sfs, err := sivafs.NewFilesystem(l.library.fs, l.path, memfs.New()) + if err != nil { + return nil, err + } + + l.cachedFS = sfs + return sfs, nil +} + func (l *Location) ID() borges.LocationID { return l.id } @@ -53,7 +173,12 @@ func (l *Location) Init(id borges.RepositoryID) (borges.Repository, error) { return nil, borges.ErrRepositoryExists.New(id) } - repo, err := NewRepository(id, l.fs, borges.RWMode, l) + fs, err := l.FS() + if err != nil { + return nil, err + } + + repo, err := NewRepository(id, fs, borges.RWMode, l) if err != nil { return nil, err } @@ -147,7 +272,12 @@ func (l *Location) repository( id borges.RepositoryID, mode borges.Mode, ) (borges.Repository, error) { - return NewRepository(id, l.fs, mode, l) + fs, err := l.FS() + if err != nil { + return nil, err + } + + return NewRepository(id, fs, mode, l) } type repositoryIterator struct { @@ -170,8 +300,13 @@ func (i *repositoryIterator) Next() (borges.Repository, error) { continue } + fs, err := i.l.FS() + if err != nil { + return nil, err + } + id := toRepoID(r.URLs[0]) - return NewRepository(id, i.l.fs, i.mode, i.l) + return NewRepository(id, fs, i.mode, i.l) } } diff --git a/siva/repository.go b/siva/repository.go index 6b091f0..5a2eb17 100644 --- a/siva/repository.go +++ b/siva/repository.go @@ -2,9 +2,10 @@ package siva import ( borges "github.com/src-d/go-borges" - "github.com/src-d/go-borges/util" billy "gopkg.in/src-d/go-billy.v4" git "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/storage/filesystem" ) type Repository struct { @@ -24,14 +25,10 @@ func NewRepository( m borges.Mode, l *Location, ) (*Repository, error) { - sto, _, err := util.RepositoryStorer(fs, l.library.fs, m, l.transactional) - if err != nil { - return nil, err - } - + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) repo, err := git.Open(sto, nil) if err != nil { - return nil, err + return nil, borges.ErrLocationNotExists.New(id) } return &Repository{