Add --repair option for command that work with cache (#200)

cache.go

@@ -32,7 +32,7 @@ func (c Cache) GetChunk(id ChunkID) (*Chunk, error) {
 	default:
 		return chunk, err
 	}
-	// At this point we failed to find it in the local cache. Ask the remote
+	// At this point we failed to find chunk in the local cache. Ask the remote
 	chunk, err = c.s.GetChunk(id)
 	if err != nil {
 		return chunk, err
@@ -61,3 +61,40 @@ func (c Cache) Close() error {
 	c.l.Close()
 	return c.s.Close()
 }
+
+// New cache which GetChunk() function will return ChunkMissing error instead of ChunkInvalid
+// so caller can redownload invalid chunk from store
+type RepairableCache struct {
+	l WriteStore
+}
+
+// Create new RepairableCache that wraps WriteStore and modify its GetChunk() so ChunkInvalid error
+// will be replaced by ChunkMissing error
+func NewRepairableCache(l WriteStore) RepairableCache {
+	return RepairableCache{l: l}
+}
+
+func (r RepairableCache) GetChunk(id ChunkID) (*Chunk, error) {
+	chunk, err := r.l.GetChunk(id)
+	var chunkInvalidErr ChunkInvalid
+	if err != nil && errors.As(err, &chunkInvalidErr) {
+		return chunk, ChunkMissing{ID: chunkInvalidErr.ID}
+	}
+	return chunk, err
+}
+
+func (r RepairableCache) HasChunk(id ChunkID) (bool, error) {
+	return r.l.HasChunk(id)
+}
+
+func (r RepairableCache) Close() error {
+	return r.l.Close()
+}
+
+func (r RepairableCache) String() string {
+	return r.l.String()
+}
+
+func (r RepairableCache) StoreChunk(c *Chunk) error {
+	return r.l.StoreChunk(c)
+}

cmd/desync/options.go

@@ -16,6 +16,7 @@ type cmdStoreOptions struct {
 	caCert        string
 	skipVerify    bool
 	trustInsecure bool
+	cacheRepair   bool
 }
 
 // MergeWith takes store options as read from the config, and applies command-line
@@ -55,6 +56,7 @@ func addStoreOptions(o *cmdStoreOptions, f *pflag.FlagSet) {
 	f.StringVar(&o.clientKey, "client-key", "", "path to client key for TLS authentication")
 	f.StringVar(&o.caCert, "ca-cert", "", "trust authorities in this file, instead of OS trust store")
 	f.BoolVarP(&o.trustInsecure, "trust-insecure", "t", false, "trust invalid certificates")
+	f.BoolVarP(&o.cacheRepair, "cache-repair", "r", true, "replace invalid chunks in the cache from source")
 }
 
 // cmdServerOptions hold command line options used in HTTP servers.

cmd/desync/store.go

@@ -37,6 +37,9 @@ func MultiStoreWithCache(cmdOpt cmdStoreOptions, cacheLocation string, storeLoca
 		if ls, ok := cache.(desync.LocalStore); ok {
 			ls.UpdateTimes = true
 		}
+		if cmdOpt.cacheRepair {
+			cache = desync.NewRepairableCache(cache)
+		}
 		store = desync.NewCache(store, cache)
 	}
 	return store, nil

cmd/desync/untar_test.go

@@ -4,8 +4,10 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -36,3 +38,35 @@ func TestUntarCommandIndex(t *testing.T) {
 	_, err = cmd.ExecuteC()
 	require.NoError(t, err)
 }
+
+// Check that we repair broken chunks in chache
+func TestUntarCommandRepair(t *testing.T) {
+	// Create an output dir to extract into
+	out, err := ioutil.TempDir("", "")
+	require.NoError(t, err)
+	defer os.RemoveAll(out)
+
+	// Create cache with invalid chunk
+	cache, err := ioutil.TempDir("", "brokencache")
+	require.NoError(t, err)
+	defer os.RemoveAll(cache)
+
+	chunkId := "0589328ff916d08f5fe59a9aa0731571448e91341f37ca5484a85b9f0af14de3"
+	badChunkHash := "c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a"
+	err = os.Mkdir(path.Join(cache, chunkId[:4]), os.ModePerm)
+	require.NoError(t, err)
+	err = ioutil.WriteFile(path.Join(cache, chunkId[:4], chunkId+".cacnk"), []byte("42"), os.ModePerm)
+	require.NoError(t, err)
+
+	// Run "untar" with "--repair=false" -> get error
+	cmd := newUntarCommand(context.Background())
+	cmd.SetArgs([]string{"-s", "testdata/tree.store", "-c", cache, "--cache-repair=false", "-i", "--no-same-owner", "--no-same-permissions", "testdata/tree.caidx", out})
+	_, err = cmd.ExecuteC()
+	require.EqualError(t, err, fmt.Sprintf("chunk id %s does not match its hash %s", chunkId, badChunkHash))
+
+	// Now run "untar" with "--repair=true" -> no error
+	cmd = newUntarCommand(context.Background())
+	cmd.SetArgs([]string{"-s", "testdata/tree.store", "-c", cache, "--cache-repair=true", "-i", "--no-same-owner", "--no-same-permissions", "testdata/tree.caidx", out})
+	_, err = cmd.ExecuteC()
+	require.NoError(t, err)
+}

store.go

@@ -23,7 +23,7 @@ type WriteStore interface {
 	StoreChunk(c *Chunk) error
 }
 
-// PruneStore is a store that supports pruning of chunks
+// PruneStore is a store that supports read, write and pruning of chunks
 type PruneStore interface {
 	WriteStore
 	Prune(ctx context.Context, ids map[ChunkID]struct{}) error