From 9c6c1ccb03498c8647d29739b9d42c59f87ea75a Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 24 Jan 2022 16:15:10 +0100 Subject: [PATCH] feat: CAR offset writer --- v2/car_offset_writer.go | 168 ++++++++++++++++++++++++++++ v2/car_offset_writer_test.go | 207 +++++++++++++++++++++++++++++++++++ v2/car_reader_seeker.go | 107 ++++++++++++++++++ v2/car_reader_seeker_test.go | 176 +++++++++++++++++++++++++++++ v2/go.mod | 9 +- v2/go.sum | 26 ++++- 6 files changed, 689 insertions(+), 4 deletions(-) create mode 100644 v2/car_offset_writer.go create mode 100644 v2/car_offset_writer_test.go create mode 100644 v2/car_reader_seeker.go create mode 100644 v2/car_reader_seeker_test.go diff --git a/v2/car_offset_writer.go b/v2/car_offset_writer.go new file mode 100644 index 00000000..224e25da --- /dev/null +++ b/v2/car_offset_writer.go @@ -0,0 +1,168 @@ +package car + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + offline "github.com/ipfs/go-ipfs-exchange-offline" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + "github.com/ipld/go-car/v2/internal/carv1" + "github.com/ipld/go-car/v2/internal/carv1/util" +) + +type blockInfo struct { + offset uint64 + // Note: size is the size of the block and metadata + size uint64 + links []*format.Link +} + +// CarOffsetWriter turns a blockstore and a root CID into a CAR file stream, +// starting from an offset +type CarOffsetWriter struct { + payloadCid cid.Cid + nodeGetter format.NodeGetter + blockInfos map[cid.Cid]*blockInfo + header carv1.CarHeader +} + +func NewCarOffsetWriter(payloadCid cid.Cid, bstore blockstore.Blockstore) *CarOffsetWriter { + ng := merkledag.NewDAGService(blockservice.New(bstore, offline.Exchange(bstore))) + return &CarOffsetWriter{ + payloadCid: payloadCid, + nodeGetter: ng, + blockInfos: make(map[cid.Cid]*blockInfo), + header: carHeader(payloadCid), + } +} + +func carHeader(payloadCid cid.Cid) carv1.CarHeader { + return carv1.CarHeader{ + Roots: []cid.Cid{payloadCid}, + Version: 1, + } +} + +func (s *CarOffsetWriter) Write(ctx context.Context, w io.Writer, offset uint64) error { + headerSize, err := s.writeHeader(w, offset) + if err != nil { + return err + } + + return s.writeBlocks(ctx, w, headerSize, offset) +} + +func (s *CarOffsetWriter) writeHeader(w io.Writer, offset uint64) (uint64, error) { + headerSize, err := carv1.HeaderSize(&s.header) + if err != nil { + return 0, fmt.Errorf("failed to size car header: %w", err) + } + + // Check if the offset from which to start writing is after the header + if offset >= headerSize { + return headerSize, nil + } + + // Write out the header, starting at the offset + _, err = skipWrite(w, offset, func(sw io.Writer) (int, error) { + return 0, carv1.WriteHeader(&s.header, sw) + }) + if err != nil { + return 0, fmt.Errorf("failed to write car header: %w", err) + } + + return headerSize, nil +} + +func (s *CarOffsetWriter) writeBlocks(ctx context.Context, w io.Writer, headerSize uint64, writeOffset uint64) error { + // The first block's offset is the size of the header + offset := headerSize + + // This function gets called for each CID during the merkle DAG walk + nextCid := func(ctx context.Context, c cid.Cid) ([]*format.Link, error) { + // There will be an item in the cache if writeBlocks has already been + // called before, and the DAG traversal reached this CID + cached, ok := s.blockInfos[c] + if ok { + // Check if the offset from which to start writing is after this + // block + nextBlockOffset := cached.offset + cached.size + if writeOffset >= nextBlockOffset { + // The offset from which to start writing is after this block + // so don't write anything, just skip over this block + offset = nextBlockOffset + return cached.links, nil + } + } + + // Get the block from the blockstore + nd, err := s.nodeGetter.Get(ctx, c) + if err != nil { + return nil, fmt.Errorf("getting block %s: %w", c, err) + } + + // Get the size of the block and metadata + ldsize := util.LdSize(nd.Cid().Bytes(), nd.RawData()) + + // Check if the offset from which to start writing is in or before this + // block + nextBlockOffset := offset + ldsize + if writeOffset < nextBlockOffset { + // Check if the offset from which to start writing is in this block + var blockWriteOffset uint64 + if writeOffset >= offset { + blockWriteOffset = writeOffset - offset + } + + // Write the block data to the writer, starting at the block offset + _, err = skipWrite(w, blockWriteOffset, func(sw io.Writer) (int, error) { + return 0, util.LdWrite(sw, nd.Cid().Bytes(), nd.RawData()) + }) + if err != nil { + return nil, fmt.Errorf("writing CAR block %s: %w", c, err) + } + } + + // Add the block to the cache + s.blockInfos[nd.Cid()] = &blockInfo{ + offset: offset, + size: ldsize, + links: nd.Links(), + } + + offset = nextBlockOffset + + // Return any links from this block to other DAG blocks + return nd.Links(), nil + } + + seen := cid.NewSet() + return merkledag.Walk(ctx, nextCid, s.payloadCid, seen.Visit) +} + +// Write data to the writer, skipping the first skip bytes +func skipWrite(w io.Writer, skip uint64, write func(sw io.Writer) (int, error)) (int, error) { + // If there's nothing to skip, just do a normal write + if skip == 0 { + return write(w) + } + + // Write to a buffer + var buff bytes.Buffer + if count, err := write(&buff); err != nil { + return count, err + } + + // Write the buffer to the writer, skipping the first skip bytes + bz := buff.Bytes() + if skip >= uint64(len(bz)) { + return 0, nil + } + return w.Write(bz[skip:]) +} diff --git a/v2/car_offset_writer_test.go b/v2/car_offset_writer_test.go new file mode 100644 index 00000000..0791f50e --- /dev/null +++ b/v2/car_offset_writer_test.go @@ -0,0 +1,207 @@ +package car + +import ( + "bytes" + "context" + "io" + "math/rand" + "testing" + + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cidutil" + "github.com/ipfs/go-datastore" + dss "github.com/ipfs/go-datastore/sync" + bstore "github.com/ipfs/go-ipfs-blockstore" + chunk "github.com/ipfs/go-ipfs-chunker" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-unixfs/importer/balanced" + "github.com/ipfs/go-unixfs/importer/helpers" + "github.com/ipld/go-car/v2/internal/carv1" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +func TestCarOffsetWriter(t *testing.T) { + ds := dss.MutexWrap(datastore.NewMapDatastore()) + bs := bstore.NewBlockstore(ds) + bserv := blockservice.New(bs, nil) + dserv := merkledag.NewDAGService(bserv) + + rseed := 5 + size := 2 * 1024 * 1024 + source := io.LimitReader(rand.New(rand.NewSource(int64(rseed))), int64(size)) + nd, err := DAGImport(dserv, source) + require.NoError(t, err) + + // Write the CAR to a buffer from offset 0 so the buffer can be used for + // comparison + payloadCid := nd.Cid() + fullCarCow := NewCarOffsetWriter(payloadCid, bs) + var fullBuff bytes.Buffer + err = fullCarCow.Write(context.Background(), &fullBuff, 0) + require.NoError(t, err) + + fullCar := fullBuff.Bytes() + header := carHeader(nd.Cid()) + headerSize, err := carv1.HeaderSize(&header) + + testCases := []struct { + name string + offset uint64 + }{{ + name: "1 byte offset", + offset: 1, + }, { + name: "offset < header size", + offset: headerSize - 1, + }, { + name: "offset == header size", + offset: headerSize, + }, { + name: "offset > header size", + offset: headerSize + 1, + }, { + name: "offset > header + one block size", + offset: headerSize + 1024*1024 + 512*1024, + }} + + runTestCases := func(name string, runTCWithCow func() *CarOffsetWriter) { + for _, tc := range testCases { + t.Run(name+" - "+tc.name, func(t *testing.T) { + cow := runTCWithCow() + var buff bytes.Buffer + err = cow.Write(context.Background(), &buff, tc.offset) + require.NoError(t, err) + require.Equal(t, len(fullCar)-int(tc.offset), len(buff.Bytes())) + require.Equal(t, fullCar[tc.offset:], buff.Bytes()) + }) + } + } + + // Run tests with a new CarOffsetWriter + runTestCases("new car offset writer", func() *CarOffsetWriter { + return NewCarOffsetWriter(payloadCid, bs) + }) + + // Run tests with a CarOffsetWriter that has already been used to write + // a CAR starting at offset 0 + runTestCases("fully written car offset writer", func() *CarOffsetWriter { + fullCarCow := NewCarOffsetWriter(payloadCid, bs) + var buff bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff, 0) + require.NoError(t, err) + return fullCarCow + }) + + // Run tests with a CarOffsetWriter that has already been used to write + // a CAR starting at offset 1 + runTestCases("car offset writer written from offset 1", func() *CarOffsetWriter { + fullCarCow := NewCarOffsetWriter(payloadCid, bs) + var buff bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff, 1) + require.NoError(t, err) + return fullCarCow + }) + + // Run tests with a CarOffsetWriter that has already been used to write + // a CAR starting part way through the second block + runTestCases("car offset writer written from offset 1.5 blocks", func() *CarOffsetWriter { + fullCarCow := NewCarOffsetWriter(payloadCid, bs) + var buff bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff, 1024*1024+512*1024) + require.NoError(t, err) + return fullCarCow + }) + + // Run tests with a CarOffsetWriter that has already been used to write + // a CAR repeatedly + runTestCases("car offset writer written from offset repeatedly", func() *CarOffsetWriter { + fullCarCow := NewCarOffsetWriter(payloadCid, bs) + var buff bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff, 1024) + require.NoError(t, err) + fullCarCow = NewCarOffsetWriter(payloadCid, bs) + var buff2 bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff2, 10) + require.NoError(t, err) + fullCarCow = NewCarOffsetWriter(payloadCid, bs) + var buff3 bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff3, 1024*1024+512*1024) + require.NoError(t, err) + return fullCarCow + }) +} + +func TestSkipWriter(t *testing.T) { + testCases := []struct { + name string + size int + skip int + expected int + }{{ + name: "no skip", + size: 1024, + skip: 0, + expected: 1024, + }, { + name: "skip 1", + size: 1024, + skip: 1, + expected: 1023, + }, { + name: "skip all", + size: 1024, + skip: 1024, + expected: 0, + }, { + name: "skip overflow", + size: 1024, + skip: 1025, + expected: 0, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var buff bytes.Buffer + write := func(sw io.Writer) (int, error) { + bz := make([]byte, tc.size) + return sw.Write(bz) + } + count, err := skipWrite(&buff, uint64(tc.skip), write) + require.NoError(t, err) + require.Equal(t, tc.expected, count) + require.Equal(t, tc.expected, len(buff.Bytes())) + }) + } +} + +var DefaultHashFunction = uint64(mh.SHA2_256) + +func DAGImport(dserv format.DAGService, fi io.Reader) (format.Node, error) { + prefix, err := merkledag.PrefixForCidVersion(1) + if err != nil { + return nil, err + } + prefix.MhType = DefaultHashFunction + + spl := chunk.NewSizeSplitter(fi, 1024*1024) + dbp := helpers.DagBuilderParams{ + Maxlinks: 1024, + RawLeaves: true, + + CidBuilder: cidutil.InlineBuilder{ + Builder: prefix, + Limit: 32, + }, + + Dagserv: dserv, + } + + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + + return balanced.Layout(db) +} diff --git a/v2/car_reader_seeker.go b/v2/car_reader_seeker.go new file mode 100644 index 00000000..7d00b90a --- /dev/null +++ b/v2/car_reader_seeker.go @@ -0,0 +1,107 @@ +package car + +import ( + "context" + "fmt" + "io" + + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "golang.org/x/xerrors" +) + +// CarReaderSeeker wraps CarOffsetWriter with a ReadSeeker implementation. +// Note that Read and Seek are not thread-safe, they must not be called +// concurrently. +type CarReaderSeeker struct { + parentCtx context.Context + size uint64 + offset int64 + + cow *CarOffsetWriter // 🐮 + reader *io.PipeReader + writeCancel context.CancelFunc + writeComplete chan struct{} +} + +var _ io.ReadSeeker = (*CarReaderSeeker)(nil) + +func NewCarReaderSeeker(ctx context.Context, payloadCid cid.Cid, bstore blockstore.Blockstore, size uint64) *CarReaderSeeker { + cow := NewCarOffsetWriter(payloadCid, bstore) + + return &CarReaderSeeker{ + parentCtx: ctx, + size: size, + cow: cow, + writeComplete: make(chan struct{}, 1), + } +} + +// Note: not threadsafe +func (c *CarReaderSeeker) Read(p []byte) (n int, err error) { + if uint64(c.offset) >= c.size { + return 0, fmt.Errorf("cannot read from offset %d >= file size %d", c.offset, c.size) + } + + // Check if there's already a write in progress + if c.reader == nil { + // No write in progress, start a new write from the current offset + // in a go routine + writeCtx, writeCancel := context.WithCancel(c.parentCtx) + c.writeCancel = writeCancel + pr, pw := io.Pipe() + c.reader = pr + + go func() { + err := c.cow.Write(writeCtx, pw, uint64(c.offset)) + if err != nil && !xerrors.Is(err, context.Canceled) { + pw.CloseWithError(err) + } else { + pw.Close() + } + c.writeComplete <- struct{}{} + }() + } + + return c.reader.Read(p) +} + +// Note: not threadsafe +func (c *CarReaderSeeker) Seek(offset int64, whence int) (int64, error) { + // Update the offset + switch whence { + case io.SeekStart: + if offset < 0 { + return 0, fmt.Errorf("invalid offset %d from start: must be positive", offset) + } + c.offset = offset + case io.SeekCurrent: + if c.offset+offset < 0 { + return 0, fmt.Errorf("invalid offset %d from current %d: resulting offset is negative", offset, c.offset) + } + c.offset += offset + case io.SeekEnd: + if int64(c.size)+offset < 0 { + return 0, fmt.Errorf("invalid offset %d from end: larger than total size %d", offset, c.size) + } + c.offset = int64(c.size) + offset + } + + // Cancel any ongoing write and wait for it to complete + if c.reader != nil { + c.writeCancel() + + // Seek and Read should not be called concurrently so this is safe + c.reader.Close() + + select { + case <-c.parentCtx.Done(): + return 0, c.parentCtx.Err() + case <-c.writeComplete: + } + + c.reader = nil + } + + return c.offset, nil +} diff --git a/v2/car_reader_seeker_test.go b/v2/car_reader_seeker_test.go new file mode 100644 index 00000000..f4042b72 --- /dev/null +++ b/v2/car_reader_seeker_test.go @@ -0,0 +1,176 @@ +package car + +import ( + "bytes" + "context" + "io" + "math/rand" + "testing" + + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-datastore" + dss "github.com/ipfs/go-datastore/sync" + bstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-merkledag" + "github.com/stretchr/testify/require" +) + +func TestCarReaderSeeker(t *testing.T) { + ctx := context.Background() + + ds := dss.MutexWrap(datastore.NewMapDatastore()) + bs := bstore.NewBlockstore(ds) + bserv := blockservice.New(bs, nil) + dserv := merkledag.NewDAGService(bserv) + + rseed := 5 + size := 2 * 1024 * 1024 + source := io.LimitReader(rand.New(rand.NewSource(int64(rseed))), int64(size)) + nd, err := DAGImport(dserv, source) + require.NoError(t, err) + + // Write the CAR to a buffer so it can be used for comparisons + fullCarCow := NewCarOffsetWriter(nd.Cid(), bs) + var fullBuff bytes.Buffer + err = fullCarCow.Write(context.Background(), &fullBuff, 0) + require.NoError(t, err) + carSize := fullBuff.Len() + + readTestCases := []struct { + name string + offset int64 + }{{ + name: "read all from start", + offset: 0, + }, { + name: "read all from byte 1", + offset: 1, + }, { + name: "read all from middle", + offset: int64(carSize / 2), + }, { + name: "read all from end - 1", + offset: int64(carSize - 1), + }} + + for _, tc := range readTestCases { + t.Run(tc.name, func(t *testing.T) { + crs := NewCarReaderSeeker(ctx, nd.Cid(), bs, uint64(carSize)) + if tc.offset > 0 { + _, err := crs.Seek(tc.offset, io.SeekStart) + require.NoError(t, err) + } + buff, err := io.ReadAll(crs) + require.NoError(t, err) + require.Equal(t, carSize-int(tc.offset), len(buff)) + require.Equal(t, fullBuff.Bytes()[tc.offset:], buff) + }) + } + + seekTestCases := []struct { + name string + whence int + offset int64 + expectSeekErr bool + expectReadErr bool + expectOffset int64 + }{{ + name: "start +0", + whence: io.SeekStart, + offset: 0, + expectOffset: 0, + }, { + name: "start -1", + whence: io.SeekStart, + offset: -1, + expectSeekErr: true, + }, { + name: "start +full size", + whence: io.SeekStart, + offset: int64(carSize), + expectOffset: int64(carSize), + expectReadErr: true, + }, { + name: "current +0", + whence: io.SeekCurrent, + offset: 0, + expectOffset: 0, + }, { + name: "current +10", + whence: io.SeekCurrent, + offset: 10, + expectOffset: 10, + }, { + name: "current -1", + whence: io.SeekCurrent, + offset: -1, + expectSeekErr: true, + }, { + name: "current +full size", + whence: io.SeekCurrent, + offset: int64(carSize), + expectOffset: int64(carSize), + expectReadErr: true, + }, { + name: "end +0", + whence: io.SeekEnd, + offset: 0, + expectOffset: int64(carSize), + expectReadErr: true, + }, { + name: "end -10", + whence: io.SeekEnd, + offset: -10, + expectOffset: int64(carSize) - 10, + }, { + name: "end +1", + whence: io.SeekEnd, + offset: 1, + expectOffset: int64(carSize) + 1, + expectReadErr: true, + }, { + name: "end -(full size+1)", + whence: io.SeekEnd, + offset: int64(-(carSize + 1)), + expectSeekErr: true, + }} + + for _, tc := range seekTestCases { + t.Run(tc.name, func(t *testing.T) { + crs := NewCarReaderSeeker(ctx, nd.Cid(), bs, uint64(carSize)) + newOffset, err := crs.Seek(tc.offset, tc.whence) + if tc.expectSeekErr { + require.Error(t, err) + return + } else { + require.NoError(t, err) + } + require.Equal(t, tc.expectOffset, newOffset) + + buff, err := io.ReadAll(crs) + if tc.expectReadErr { + require.Error(t, err) + return + } else { + require.NoError(t, err) + } + + require.Equal(t, carSize-int(tc.expectOffset), len(buff)) + require.Equal(t, fullBuff.Bytes()[tc.expectOffset:], buff) + }) + } + + t.Run("double seek", func(t *testing.T) { + crs := NewCarReaderSeeker(ctx, nd.Cid(), bs, uint64(carSize)) + _, err := crs.Seek(10, io.SeekStart) + newOffset, err := crs.Seek(-5, io.SeekCurrent) + require.NoError(t, err) + require.EqualValues(t, 5, newOffset) + + buff, err := io.ReadAll(crs) + require.NoError(t, err) + + require.Equal(t, carSize-5, len(buff)) + require.Equal(t, fullBuff.Bytes()[5:], buff) + }) +} diff --git a/v2/go.mod b/v2/go.mod index e3b2043e..65876e96 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -4,13 +4,19 @@ go 1.16 require ( github.com/ipfs/go-block-format v0.0.3 + github.com/ipfs/go-blockservice v0.2.1 github.com/ipfs/go-cid v0.1.0 + github.com/ipfs/go-cidutil v0.0.2 + github.com/ipfs/go-datastore v0.5.1 github.com/ipfs/go-ipfs-blockstore v1.1.2 + github.com/ipfs/go-ipfs-chunker v0.0.5 + github.com/ipfs/go-ipfs-exchange-offline v0.1.1 github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-merkledag v0.5.1 + github.com/ipfs/go-unixfs v0.3.1 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.14.0 + github.com/ipld/go-ipld-prime v0.14.3-0.20211207234443-319145880958 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 github.com/multiformats/go-multihash v0.1.0 @@ -21,4 +27,5 @@ require ( golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect golang.org/x/exp v0.0.0-20210615023648-acb5c1269671 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 ) diff --git a/v2/go.sum b/v2/go.sum index 9f120d9a..7ba7944a 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -26,6 +26,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= +github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -246,6 +248,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ= +github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= github.com/ipfs/go-bitswap v0.5.1 h1:721YAEDBnLIrvcIMkCHCdqp34hA8jwL9yKMkyJpSpco= github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -262,14 +266,17 @@ github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqg github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= +github.com/ipfs/go-cidutil v0.0.2 h1:CNOboQf1t7Qp0nuNh8QMmhJs0+Q//bRL1axtCnIB1Yo= +github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= -github.com/ipfs/go-datastore v0.5.0 h1:rQicVCEacWyk4JZ6G5bD9TKR7lZEG1MWcG7UdWYrFAU= github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= +github.com/ipfs/go-datastore v0.5.1 h1:WkRhLuISI+XPD0uk3OskB0fYFSyqK8Ob5ZYew9Qa1nQ= +github.com/ipfs/go-datastore v0.5.1/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -284,6 +291,9 @@ github.com/ipfs/go-ipfs-blockstore v1.1.2 h1:WCXoZcMYnvOTmlpX+RSSnhVN0uCmbWTeepT github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= +github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= +github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= +github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -294,6 +304,10 @@ github.com/ipfs/go-ipfs-exchange-interface v0.1.0 h1:TiMekCrOGQuWYtZO3mf4YJXDIdN github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI= github.com/ipfs/go-ipfs-exchange-offline v0.1.1 h1:mEiXWdbMN6C7vtDG21Fphx8TGCbZPpQnz/496w/PL4g= github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= +github.com/ipfs/go-ipfs-files v0.0.3 h1:ME+QnC3uOyla1ciRPezDW0ynQYK2ikOh9OCKAEg4uUA= +github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= +github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= +github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FFiW0BY= @@ -327,14 +341,16 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.7.0 h1:VyO6G4sbzX80K58N60cCaHsSsypbUNs1GjO5seGNsQ0= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= +github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8= +github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.14.0 h1:2FnBqUjmmgxgZD6/zB3eygWlmIsHNGrZ57L99x3xD6Q= -github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= +github.com/ipld/go-ipld-prime v0.14.3-0.20211207234443-319145880958 h1:olscE5Sv+ts+N9YLQsIL9k6eS6y6CXMGRl5RCr2Cn/E= +github.com/ipld/go-ipld-prime v0.14.3-0.20211207234443-319145880958/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= @@ -858,6 +874,8 @@ github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIf github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2USAXB5oMDwRl3piOux2XMV9TANaYxXHdkoE= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= @@ -1007,6 +1025,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1022,6 +1041,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=