From 3d969443294c2416fc58e02510937148a1592eca Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 24 Jan 2022 16:15:10 +0100 Subject: [PATCH] feat: CAR offset writer --- v2/car_offset_writer.go | 168 ++++++++++++++++++++++++++++ v2/car_offset_writer_test.go | 207 +++++++++++++++++++++++++++++++++++ v2/car_reader_seeker.go | 107 ++++++++++++++++++ v2/car_reader_seeker_test.go | 177 ++++++++++++++++++++++++++++++ v2/go.mod | 16 ++- v2/go.sum | 9 +- 6 files changed, 677 insertions(+), 7 deletions(-) create mode 100644 v2/car_offset_writer.go create mode 100644 v2/car_offset_writer_test.go create mode 100644 v2/car_reader_seeker.go create mode 100644 v2/car_reader_seeker_test.go diff --git a/v2/car_offset_writer.go b/v2/car_offset_writer.go new file mode 100644 index 00000000..224e25da --- /dev/null +++ b/v2/car_offset_writer.go @@ -0,0 +1,168 @@ +package car + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + offline "github.com/ipfs/go-ipfs-exchange-offline" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + "github.com/ipld/go-car/v2/internal/carv1" + "github.com/ipld/go-car/v2/internal/carv1/util" +) + +type blockInfo struct { + offset uint64 + // Note: size is the size of the block and metadata + size uint64 + links []*format.Link +} + +// CarOffsetWriter turns a blockstore and a root CID into a CAR file stream, +// starting from an offset +type CarOffsetWriter struct { + payloadCid cid.Cid + nodeGetter format.NodeGetter + blockInfos map[cid.Cid]*blockInfo + header carv1.CarHeader +} + +func NewCarOffsetWriter(payloadCid cid.Cid, bstore blockstore.Blockstore) *CarOffsetWriter { + ng := merkledag.NewDAGService(blockservice.New(bstore, offline.Exchange(bstore))) + return &CarOffsetWriter{ + payloadCid: payloadCid, + nodeGetter: ng, + blockInfos: make(map[cid.Cid]*blockInfo), + header: carHeader(payloadCid), + } +} + +func carHeader(payloadCid cid.Cid) carv1.CarHeader { + return carv1.CarHeader{ + Roots: []cid.Cid{payloadCid}, + Version: 1, + } +} + +func (s *CarOffsetWriter) Write(ctx context.Context, w io.Writer, offset uint64) error { + headerSize, err := s.writeHeader(w, offset) + if err != nil { + return err + } + + return s.writeBlocks(ctx, w, headerSize, offset) +} + +func (s *CarOffsetWriter) writeHeader(w io.Writer, offset uint64) (uint64, error) { + headerSize, err := carv1.HeaderSize(&s.header) + if err != nil { + return 0, fmt.Errorf("failed to size car header: %w", err) + } + + // Check if the offset from which to start writing is after the header + if offset >= headerSize { + return headerSize, nil + } + + // Write out the header, starting at the offset + _, err = skipWrite(w, offset, func(sw io.Writer) (int, error) { + return 0, carv1.WriteHeader(&s.header, sw) + }) + if err != nil { + return 0, fmt.Errorf("failed to write car header: %w", err) + } + + return headerSize, nil +} + +func (s *CarOffsetWriter) writeBlocks(ctx context.Context, w io.Writer, headerSize uint64, writeOffset uint64) error { + // The first block's offset is the size of the header + offset := headerSize + + // This function gets called for each CID during the merkle DAG walk + nextCid := func(ctx context.Context, c cid.Cid) ([]*format.Link, error) { + // There will be an item in the cache if writeBlocks has already been + // called before, and the DAG traversal reached this CID + cached, ok := s.blockInfos[c] + if ok { + // Check if the offset from which to start writing is after this + // block + nextBlockOffset := cached.offset + cached.size + if writeOffset >= nextBlockOffset { + // The offset from which to start writing is after this block + // so don't write anything, just skip over this block + offset = nextBlockOffset + return cached.links, nil + } + } + + // Get the block from the blockstore + nd, err := s.nodeGetter.Get(ctx, c) + if err != nil { + return nil, fmt.Errorf("getting block %s: %w", c, err) + } + + // Get the size of the block and metadata + ldsize := util.LdSize(nd.Cid().Bytes(), nd.RawData()) + + // Check if the offset from which to start writing is in or before this + // block + nextBlockOffset := offset + ldsize + if writeOffset < nextBlockOffset { + // Check if the offset from which to start writing is in this block + var blockWriteOffset uint64 + if writeOffset >= offset { + blockWriteOffset = writeOffset - offset + } + + // Write the block data to the writer, starting at the block offset + _, err = skipWrite(w, blockWriteOffset, func(sw io.Writer) (int, error) { + return 0, util.LdWrite(sw, nd.Cid().Bytes(), nd.RawData()) + }) + if err != nil { + return nil, fmt.Errorf("writing CAR block %s: %w", c, err) + } + } + + // Add the block to the cache + s.blockInfos[nd.Cid()] = &blockInfo{ + offset: offset, + size: ldsize, + links: nd.Links(), + } + + offset = nextBlockOffset + + // Return any links from this block to other DAG blocks + return nd.Links(), nil + } + + seen := cid.NewSet() + return merkledag.Walk(ctx, nextCid, s.payloadCid, seen.Visit) +} + +// Write data to the writer, skipping the first skip bytes +func skipWrite(w io.Writer, skip uint64, write func(sw io.Writer) (int, error)) (int, error) { + // If there's nothing to skip, just do a normal write + if skip == 0 { + return write(w) + } + + // Write to a buffer + var buff bytes.Buffer + if count, err := write(&buff); err != nil { + return count, err + } + + // Write the buffer to the writer, skipping the first skip bytes + bz := buff.Bytes() + if skip >= uint64(len(bz)) { + return 0, nil + } + return w.Write(bz[skip:]) +} diff --git a/v2/car_offset_writer_test.go b/v2/car_offset_writer_test.go new file mode 100644 index 00000000..0791f50e --- /dev/null +++ b/v2/car_offset_writer_test.go @@ -0,0 +1,207 @@ +package car + +import ( + "bytes" + "context" + "io" + "math/rand" + "testing" + + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cidutil" + "github.com/ipfs/go-datastore" + dss "github.com/ipfs/go-datastore/sync" + bstore "github.com/ipfs/go-ipfs-blockstore" + chunk "github.com/ipfs/go-ipfs-chunker" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-unixfs/importer/balanced" + "github.com/ipfs/go-unixfs/importer/helpers" + "github.com/ipld/go-car/v2/internal/carv1" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +func TestCarOffsetWriter(t *testing.T) { + ds := dss.MutexWrap(datastore.NewMapDatastore()) + bs := bstore.NewBlockstore(ds) + bserv := blockservice.New(bs, nil) + dserv := merkledag.NewDAGService(bserv) + + rseed := 5 + size := 2 * 1024 * 1024 + source := io.LimitReader(rand.New(rand.NewSource(int64(rseed))), int64(size)) + nd, err := DAGImport(dserv, source) + require.NoError(t, err) + + // Write the CAR to a buffer from offset 0 so the buffer can be used for + // comparison + payloadCid := nd.Cid() + fullCarCow := NewCarOffsetWriter(payloadCid, bs) + var fullBuff bytes.Buffer + err = fullCarCow.Write(context.Background(), &fullBuff, 0) + require.NoError(t, err) + + fullCar := fullBuff.Bytes() + header := carHeader(nd.Cid()) + headerSize, err := carv1.HeaderSize(&header) + + testCases := []struct { + name string + offset uint64 + }{{ + name: "1 byte offset", + offset: 1, + }, { + name: "offset < header size", + offset: headerSize - 1, + }, { + name: "offset == header size", + offset: headerSize, + }, { + name: "offset > header size", + offset: headerSize + 1, + }, { + name: "offset > header + one block size", + offset: headerSize + 1024*1024 + 512*1024, + }} + + runTestCases := func(name string, runTCWithCow func() *CarOffsetWriter) { + for _, tc := range testCases { + t.Run(name+" - "+tc.name, func(t *testing.T) { + cow := runTCWithCow() + var buff bytes.Buffer + err = cow.Write(context.Background(), &buff, tc.offset) + require.NoError(t, err) + require.Equal(t, len(fullCar)-int(tc.offset), len(buff.Bytes())) + require.Equal(t, fullCar[tc.offset:], buff.Bytes()) + }) + } + } + + // Run tests with a new CarOffsetWriter + runTestCases("new car offset writer", func() *CarOffsetWriter { + return NewCarOffsetWriter(payloadCid, bs) + }) + + // Run tests with a CarOffsetWriter that has already been used to write + // a CAR starting at offset 0 + runTestCases("fully written car offset writer", func() *CarOffsetWriter { + fullCarCow := NewCarOffsetWriter(payloadCid, bs) + var buff bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff, 0) + require.NoError(t, err) + return fullCarCow + }) + + // Run tests with a CarOffsetWriter that has already been used to write + // a CAR starting at offset 1 + runTestCases("car offset writer written from offset 1", func() *CarOffsetWriter { + fullCarCow := NewCarOffsetWriter(payloadCid, bs) + var buff bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff, 1) + require.NoError(t, err) + return fullCarCow + }) + + // Run tests with a CarOffsetWriter that has already been used to write + // a CAR starting part way through the second block + runTestCases("car offset writer written from offset 1.5 blocks", func() *CarOffsetWriter { + fullCarCow := NewCarOffsetWriter(payloadCid, bs) + var buff bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff, 1024*1024+512*1024) + require.NoError(t, err) + return fullCarCow + }) + + // Run tests with a CarOffsetWriter that has already been used to write + // a CAR repeatedly + runTestCases("car offset writer written from offset repeatedly", func() *CarOffsetWriter { + fullCarCow := NewCarOffsetWriter(payloadCid, bs) + var buff bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff, 1024) + require.NoError(t, err) + fullCarCow = NewCarOffsetWriter(payloadCid, bs) + var buff2 bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff2, 10) + require.NoError(t, err) + fullCarCow = NewCarOffsetWriter(payloadCid, bs) + var buff3 bytes.Buffer + err = fullCarCow.Write(context.Background(), &buff3, 1024*1024+512*1024) + require.NoError(t, err) + return fullCarCow + }) +} + +func TestSkipWriter(t *testing.T) { + testCases := []struct { + name string + size int + skip int + expected int + }{{ + name: "no skip", + size: 1024, + skip: 0, + expected: 1024, + }, { + name: "skip 1", + size: 1024, + skip: 1, + expected: 1023, + }, { + name: "skip all", + size: 1024, + skip: 1024, + expected: 0, + }, { + name: "skip overflow", + size: 1024, + skip: 1025, + expected: 0, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var buff bytes.Buffer + write := func(sw io.Writer) (int, error) { + bz := make([]byte, tc.size) + return sw.Write(bz) + } + count, err := skipWrite(&buff, uint64(tc.skip), write) + require.NoError(t, err) + require.Equal(t, tc.expected, count) + require.Equal(t, tc.expected, len(buff.Bytes())) + }) + } +} + +var DefaultHashFunction = uint64(mh.SHA2_256) + +func DAGImport(dserv format.DAGService, fi io.Reader) (format.Node, error) { + prefix, err := merkledag.PrefixForCidVersion(1) + if err != nil { + return nil, err + } + prefix.MhType = DefaultHashFunction + + spl := chunk.NewSizeSplitter(fi, 1024*1024) + dbp := helpers.DagBuilderParams{ + Maxlinks: 1024, + RawLeaves: true, + + CidBuilder: cidutil.InlineBuilder{ + Builder: prefix, + Limit: 32, + }, + + Dagserv: dserv, + } + + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + + return balanced.Layout(db) +} diff --git a/v2/car_reader_seeker.go b/v2/car_reader_seeker.go new file mode 100644 index 00000000..7d00b90a --- /dev/null +++ b/v2/car_reader_seeker.go @@ -0,0 +1,107 @@ +package car + +import ( + "context" + "fmt" + "io" + + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "golang.org/x/xerrors" +) + +// CarReaderSeeker wraps CarOffsetWriter with a ReadSeeker implementation. +// Note that Read and Seek are not thread-safe, they must not be called +// concurrently. +type CarReaderSeeker struct { + parentCtx context.Context + size uint64 + offset int64 + + cow *CarOffsetWriter // 🐮 + reader *io.PipeReader + writeCancel context.CancelFunc + writeComplete chan struct{} +} + +var _ io.ReadSeeker = (*CarReaderSeeker)(nil) + +func NewCarReaderSeeker(ctx context.Context, payloadCid cid.Cid, bstore blockstore.Blockstore, size uint64) *CarReaderSeeker { + cow := NewCarOffsetWriter(payloadCid, bstore) + + return &CarReaderSeeker{ + parentCtx: ctx, + size: size, + cow: cow, + writeComplete: make(chan struct{}, 1), + } +} + +// Note: not threadsafe +func (c *CarReaderSeeker) Read(p []byte) (n int, err error) { + if uint64(c.offset) >= c.size { + return 0, fmt.Errorf("cannot read from offset %d >= file size %d", c.offset, c.size) + } + + // Check if there's already a write in progress + if c.reader == nil { + // No write in progress, start a new write from the current offset + // in a go routine + writeCtx, writeCancel := context.WithCancel(c.parentCtx) + c.writeCancel = writeCancel + pr, pw := io.Pipe() + c.reader = pr + + go func() { + err := c.cow.Write(writeCtx, pw, uint64(c.offset)) + if err != nil && !xerrors.Is(err, context.Canceled) { + pw.CloseWithError(err) + } else { + pw.Close() + } + c.writeComplete <- struct{}{} + }() + } + + return c.reader.Read(p) +} + +// Note: not threadsafe +func (c *CarReaderSeeker) Seek(offset int64, whence int) (int64, error) { + // Update the offset + switch whence { + case io.SeekStart: + if offset < 0 { + return 0, fmt.Errorf("invalid offset %d from start: must be positive", offset) + } + c.offset = offset + case io.SeekCurrent: + if c.offset+offset < 0 { + return 0, fmt.Errorf("invalid offset %d from current %d: resulting offset is negative", offset, c.offset) + } + c.offset += offset + case io.SeekEnd: + if int64(c.size)+offset < 0 { + return 0, fmt.Errorf("invalid offset %d from end: larger than total size %d", offset, c.size) + } + c.offset = int64(c.size) + offset + } + + // Cancel any ongoing write and wait for it to complete + if c.reader != nil { + c.writeCancel() + + // Seek and Read should not be called concurrently so this is safe + c.reader.Close() + + select { + case <-c.parentCtx.Done(): + return 0, c.parentCtx.Err() + case <-c.writeComplete: + } + + c.reader = nil + } + + return c.offset, nil +} diff --git a/v2/car_reader_seeker_test.go b/v2/car_reader_seeker_test.go new file mode 100644 index 00000000..694a3795 --- /dev/null +++ b/v2/car_reader_seeker_test.go @@ -0,0 +1,177 @@ +package car + +import ( + "bytes" + "context" + "io" + "math/rand" + "testing" + + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-datastore" + dss "github.com/ipfs/go-datastore/sync" + bstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-merkledag" + "github.com/stretchr/testify/require" +) + +func TestCarReaderSeeker(t *testing.T) { + ctx := context.Background() + + ds := dss.MutexWrap(datastore.NewMapDatastore()) + bs := bstore.NewBlockstore(ds) + bserv := blockservice.New(bs, nil) + dserv := merkledag.NewDAGService(bserv) + + rseed := 5 + size := 2 * 1024 * 1024 + source := io.LimitReader(rand.New(rand.NewSource(int64(rseed))), int64(size)) + nd, err := DAGImport(dserv, source) + require.NoError(t, err) + + // Write the CAR to a buffer so it can be used for comparisons + fullCarCow := NewCarOffsetWriter(nd.Cid(), bs) + var fullBuff bytes.Buffer + err = fullCarCow.Write(context.Background(), &fullBuff, 0) + require.NoError(t, err) + carSize := fullBuff.Len() + + readTestCases := []struct { + name string + offset int64 + }{{ + name: "read all from start", + offset: 0, + }, { + name: "read all from byte 1", + offset: 1, + }, { + name: "read all from middle", + offset: int64(carSize / 2), + }, { + name: "read all from end - 1", + offset: int64(carSize - 1), + }} + + for _, tc := range readTestCases { + t.Run(tc.name, func(t *testing.T) { + crs := NewCarReaderSeeker(ctx, nd.Cid(), bs, uint64(carSize)) + if tc.offset > 0 { + _, err := crs.Seek(tc.offset, io.SeekStart) + require.NoError(t, err) + } + buff, err := io.ReadAll(crs) + require.NoError(t, err) + require.Equal(t, carSize-int(tc.offset), len(buff)) + require.Equal(t, fullBuff.Bytes()[tc.offset:], buff) + }) + } + + seekTestCases := []struct { + name string + whence int + offset int64 + expectSeekErr bool + expectReadErr bool + expectOffset int64 + }{{ + name: "start +0", + whence: io.SeekStart, + offset: 0, + expectOffset: 0, + }, { + name: "start -1", + whence: io.SeekStart, + offset: -1, + expectSeekErr: true, + }, { + name: "start +full size", + whence: io.SeekStart, + offset: int64(carSize), + expectOffset: int64(carSize), + expectReadErr: true, + }, { + name: "current +0", + whence: io.SeekCurrent, + offset: 0, + expectOffset: 0, + }, { + name: "current +10", + whence: io.SeekCurrent, + offset: 10, + expectOffset: 10, + }, { + name: "current -1", + whence: io.SeekCurrent, + offset: -1, + expectSeekErr: true, + }, { + name: "current +full size", + whence: io.SeekCurrent, + offset: int64(carSize), + expectOffset: int64(carSize), + expectReadErr: true, + }, { + name: "end +0", + whence: io.SeekEnd, + offset: 0, + expectOffset: int64(carSize), + expectReadErr: true, + }, { + name: "end -10", + whence: io.SeekEnd, + offset: -10, + expectOffset: int64(carSize) - 10, + }, { + name: "end +1", + whence: io.SeekEnd, + offset: 1, + expectOffset: int64(carSize) + 1, + expectReadErr: true, + }, { + name: "end -(full size+1)", + whence: io.SeekEnd, + offset: int64(-(carSize + 1)), + expectSeekErr: true, + }} + + for _, tc := range seekTestCases { + t.Run(tc.name, func(t *testing.T) { + crs := NewCarReaderSeeker(ctx, nd.Cid(), bs, uint64(carSize)) + newOffset, err := crs.Seek(tc.offset, tc.whence) + if tc.expectSeekErr { + require.Error(t, err) + return + } else { + require.NoError(t, err) + } + require.Equal(t, tc.expectOffset, newOffset) + + buff, err := io.ReadAll(crs) + if tc.expectReadErr { + require.Error(t, err) + return + } else { + require.NoError(t, err) + } + + require.Equal(t, carSize-int(tc.expectOffset), len(buff)) + require.Equal(t, fullBuff.Bytes()[tc.expectOffset:], buff) + }) + } + + t.Run("double seek", func(t *testing.T) { + crs := NewCarReaderSeeker(ctx, nd.Cid(), bs, uint64(carSize)) + _, err := crs.Seek(10, io.SeekStart) + require.NoError(t, err) + newOffset, err := crs.Seek(-5, io.SeekCurrent) + require.NoError(t, err) + require.EqualValues(t, 5, newOffset) + + buff, err := io.ReadAll(crs) + require.NoError(t, err) + + require.Equal(t, carSize-5, len(buff)) + require.Equal(t, fullBuff.Bytes()[5:], buff) + }) +} diff --git a/v2/go.mod b/v2/go.mod index b2686a7c..36d08b37 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -4,11 +4,17 @@ go 1.17 require ( github.com/ipfs/go-block-format v0.0.3 - github.com/ipfs/go-cid v0.1.0 + github.com/ipfs/go-blockservice v0.3.0 + github.com/ipfs/go-cid v0.2.0 + github.com/ipfs/go-cidutil v0.1.0 + github.com/ipfs/go-datastore v0.5.0 github.com/ipfs/go-ipfs-blockstore v1.2.0 + github.com/ipfs/go-ipfs-chunker v0.0.1 + github.com/ipfs/go-ipfs-exchange-offline v0.2.0 github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-merkledag v0.6.0 + github.com/ipfs/go-unixfs v0.3.1 github.com/ipfs/go-unixfsnode v1.4.0 github.com/ipld/go-codec-dagpb v1.3.1 github.com/ipld/go-ipld-prime v0.16.0 @@ -20,6 +26,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 golang.org/x/exp v0.0.0-20210615023648-acb5c1269671 + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 ) require ( @@ -30,12 +37,10 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.0.0 // indirect - github.com/ipfs/go-blockservice v0.3.0 // indirect - github.com/ipfs/go-datastore v0.5.0 // indirect - github.com/ipfs/go-ipfs-chunker v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.1.0 // indirect - github.com/ipfs/go-ipfs-exchange-offline v0.2.0 // indirect + github.com/ipfs/go-ipfs-files v0.0.3 // indirect + github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-legacy v0.1.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect @@ -63,7 +68,6 @@ require ( go.uber.org/zap v1.16.0 // indirect golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect lukechampine.com/blake3 v1.1.6 // indirect diff --git a/v2/go.sum b/v2/go.sum index 1b7eb9f9..1eb4227f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -28,6 +28,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -269,8 +270,11 @@ github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= +github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0= +github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= +github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= @@ -307,7 +311,9 @@ github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48 github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= github.com/ipfs/go-ipfs-exchange-offline v0.2.0 h1:2PF4o4A7W656rC0RxuhUace997FTcDTcIQ6NoEtyjAI= github.com/ipfs/go-ipfs-exchange-offline v0.2.0/go.mod h1:HjwBeW0dvZvfOMwDP0TSKXIHf2s+ksdP4E3MLDRtLKY= +github.com/ipfs/go-ipfs-files v0.0.3 h1:ME+QnC3uOyla1ciRPezDW0ynQYK2ikOh9OCKAEg4uUA= github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= +github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= @@ -683,6 +689,7 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multicodec v0.5.0 h1:EgU6cBe/D7WRwQb1KmnBvU7lrcFGMggZVTPtOW9dDHs= github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=