Skip to content

Commit

Permalink
swarm/storage: Get all chunk references for a given file (ethereum#19002
Browse files Browse the repository at this point in the history
)

(cherry picked from commit 3eff652)
  • Loading branch information
holisticode authored and dshulyak committed Mar 14, 2019
1 parent 8386a90 commit 32fd6a8
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 0 deletions.
40 changes: 40 additions & 0 deletions swarm/storage/filestore.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package storage
import (
"context"
"io"
"sort"
)

/*
Expand Down Expand Up @@ -96,3 +97,42 @@ func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEnc
func (f *FileStore) HashSize() int {
return f.hashFunc().Size()
}

// Public API. This endpoint returns all chunk hashes (only) for a given file
func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) {
// create a special kind of putter, which only will store the references
putter := &HashExplorer{
hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt),
References: make([]Reference, 0),
}
// do the actual splitting anyway, no way around it
_, _, err = PyramidSplit(ctx, data, putter, putter)
if err != nil {
return nil, err
}
// collect all references
addrs = NewAddressCollection(0)
for _, ref := range putter.References {
addrs = append(addrs, Address(ref))
}
sort.Sort(addrs)
return addrs, nil
}

// HashExplorer is a special kind of putter which will only store chunk references
type HashExplorer struct {
*hasherStore
References []Reference
}

// HashExplorer's Put will add just the chunk hashes to its `References`
func (he *HashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) {
// Need to do the actual Put, which returns the references
ref, err := he.hasherStore.Put(ctx, chunkData)
if err != nil {
return nil, err
}
// internally store the reference
he.References = append(he.References, ref)
return ref, nil
}
36 changes: 36 additions & 0 deletions swarm/storage/filestore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,3 +173,39 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) {
t.Fatalf("Comparison error after clearing memStore.")
}
}

// TestGetAllReferences only tests that GetAllReferences returns an expected
// number of references for a given file
func TestGetAllReferences(t *testing.T) {
tdb, cleanup, err := newTestDbStore(false, false)
defer cleanup()
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
db := tdb.LDBStore
memStore := NewMemStore(NewDefaultStoreParams(), db)
localStore := &LocalStore{
memStore: memStore,
DbStore: db,
}
fileStore := NewFileStore(localStore, NewFileStoreParams())

checkRefs := func(dataSize int, expectedLen int) {
slice := testutil.RandomBytes(1, dataSize)

addrs, err := fileStore.GetAllReferences(context.Background(), bytes.NewReader(slice), false)
if err != nil {
t.Fatal(err)
}
if len(addrs) != expectedLen {
t.Fatalf("Expected reference array length to be %d, but is %d", expectedLen, len(addrs))
}
}

// testRuns[i] and expectedLen[i] are dataSize and expected length respectively
testRuns := []int{1024, 8192, 16000, 30000, 1000000}
expectedLens := []int{1, 3, 5, 9, 248}
for i, r := range testRuns {
checkRefs(r, expectedLens[i])
}
}

0 comments on commit 32fd6a8

Please sign in to comment.