diff --git a/cmd/lassie/fetch.go b/cmd/lassie/fetch.go index e14f3d29..b094f43b 100644 --- a/cmd/lassie/fetch.go +++ b/cmd/lassie/fetch.go @@ -13,6 +13,8 @@ import ( "github.com/filecoin-project/lassie/pkg/storage" "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/storage/deferred" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" trustlessutils "github.com/ipld/go-trustless-utils" @@ -253,14 +255,14 @@ func defaultFetchRun( lassie.RegisterSubscriber(pp.subscriber) } - var carWriter *storage.DeferredCarWriter + var carWriter *deferred.DeferredCarWriter if outfile == stdoutFileString { // we need the onlyWriter because stdout is presented as an os.File, and // therefore pretend to support seeks, so feature-checking in go-car // will make bad assumptions about capabilities unless we hide it - carWriter = storage.NewDeferredCarWriterForStream(rootCid, &onlyWriter{dataWriter}) + carWriter = deferred.NewDeferredCarWriterForStream(&onlyWriter{dataWriter}, []cid.Cid{rootCid}) } else { - carWriter = storage.NewDeferredCarWriterForPath(rootCid, outfile) + carWriter = deferred.NewDeferredCarWriterForPath(outfile, []cid.Cid{rootCid}, car.WriteAsCarV1(true)) } tempStore := storage.NewDeferredStorageCar(tempDir, rootCid) diff --git a/go.mod b/go.mod index ea71e415..7c7118e2 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-unixfsnode v1.8.0 - github.com/ipld/go-car/v2 v2.11.0 + github.com/ipld/go-car/v2 v2.13.0 github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.21.0 github.com/ipld/go-trustless-utils v0.0.0 @@ -90,7 +90,7 @@ require ( github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect - github.com/ipfs/go-ipld-cbor v0.0.6 // indirect + github.com/ipfs/go-ipld-cbor v0.1.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect @@ -149,7 +149,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/warpfork/go-testmark v0.12.1 // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect - github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect + github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.opentelemetry.io/otel/sdk v1.14.0 // indirect diff --git a/go.sum b/go.sum index 93ab667c..55f0333d 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,7 @@ github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6Uh github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -139,6 +140,7 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -176,6 +178,7 @@ github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaW github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -279,6 +282,7 @@ github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSO github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= +github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= @@ -288,8 +292,9 @@ github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.6-0.20211211231443-5d9b9e1f6fa8/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= -github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= +github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs= +github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U= @@ -317,8 +322,10 @@ github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= github.com/ipfs/go-unixfsnode v1.8.0 h1:yCkakzuE365glu+YkgzZt6p38CSVEBPgngL9ZkfnyQU= github.com/ipfs/go-unixfsnode v1.8.0/go.mod h1:HxRu9HYHOjK6HUqFBAi++7DVoWAHn0o4v/nZ/VA+0g8= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= -github.com/ipld/go-car/v2 v2.11.0 h1:lkAPwbbTFqbdfawgm+bfmFc8PjGC7D12VcaLXPCLNfM= -github.com/ipld/go-car/v2 v2.11.0/go.mod h1:aDszqev0zjtU8l96g4lwXHaU9bzArj56Y7eEN0q/xqA= +github.com/ipld/go-car/v2 v2.12.1-0.20230904004620-0621cff7a758 h1:VIPmhpq7uRkM4loVAfF3p3lyIBxUMC6AkW/jTf45axM= +github.com/ipld/go-car/v2 v2.12.1-0.20230904004620-0621cff7a758/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo= +github.com/ipld/go-car/v2 v2.13.0 h1:ogsB0rm6cclI2/HxP2XM+z+KehNj2ovpz0YfiF1hJwE= +github.com/ipld/go-car/v2 v2.13.0/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= @@ -347,11 +354,13 @@ github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -371,6 +380,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= @@ -432,6 +442,7 @@ github.com/mitchellh/go-server-timing v1.0.1 h1:f00/aIe8T3MrnLhQHu3tSWvnwc5GV/p5 github.com/mitchellh/go-server-timing v1.0.1/go.mod h1:Mo6GKi9FSLwWFAMn3bqVPWe20y5ri5QGQuO9D9MCOxk= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -473,6 +484,7 @@ github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= @@ -556,6 +568,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -566,10 +579,12 @@ github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hg github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -577,6 +592,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -608,10 +624,11 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:f github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= -github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 h1:yVYDLoN2gmB3OdBXFW8e1UwgVbmCvNlnAKhvHPaNARI= +github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I= github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk= @@ -805,6 +822,7 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -911,6 +929,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/server/http/ipfs.go b/pkg/server/http/ipfs.go index f61ba434..34e77b0b 100644 --- a/pkg/server/http/ipfs.go +++ b/pkg/server/http/ipfs.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-cid" "github.com/ipfs/go-unixfsnode" + "github.com/ipld/go-car/v2/storage/deferred" cidlink "github.com/ipld/go-ipld-prime/linking/cid" trustlessutils "github.com/ipld/go-trustless-utils" trustlesshttp "github.com/ipld/go-trustless-utils/http" @@ -50,9 +51,9 @@ func IpfsHandler(fetcher types.Fetcher, cfg HttpServerConfig) func(http.Response tempStore := storage.NewDeferredStorageCar(cfg.TempDir, request.Root) var carWriter storage.DeferredWriter if request.Duplicates { - carWriter = storage.NewDuplicateAdderCarForStream(req.Context(), request.Root, request.Path, request.Scope, request.Bytes, tempStore, res) + carWriter = storage.NewDuplicateAdderCarForStream(req.Context(), res, request.Root, request.Path, request.Scope, request.Bytes, tempStore) } else { - carWriter = storage.NewDeferredCarWriterForStream(request.Root, res) + carWriter = deferred.NewDeferredCarWriterForStream(res, []cid.Cid{request.Root}) } carStore := storage.NewCachingTempStore(carWriter.BlockWriteOpener(), tempStore) defer func() { diff --git a/pkg/storage/cachingtempstore_test.go b/pkg/storage/cachingtempstore_test.go index f3bb7fa3..993c4221 100644 --- a/pkg/storage/cachingtempstore_test.go +++ b/pkg/storage/cachingtempstore_test.go @@ -4,14 +4,21 @@ import ( "bytes" "context" "io" + "math/rand" + "sync" "testing" "time" "github.com/ipfs/go-cid" carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/storage/deferred" + mh "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" ) +var rng = rand.New(rand.NewSource(3333)) +var rngLk sync.Mutex + func TestDeferredCarWriterWritesCARv1(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -48,7 +55,7 @@ func TestDeferredCarWriterWritesCARv1(t *testing.T) { testCid2, testData2 := randBlock() var buf bytes.Buffer - cw := NewDeferredCarWriterForStream(testCid1, &buf) + cw := deferred.NewDeferredCarWriterForStream(&buf, []cid.Cid{testCid1}) ss := NewCachingTempStore(cw.BlockWriteOpener(), NewDeferredStorageCar("", testCid1)) t.Cleanup(func() { ss.Close() }) @@ -155,3 +162,20 @@ func TestDeferredCarWriterWritesCARv1(t *testing.T) { }) } } + +func randBlock() (cid.Cid, []byte) { + data := make([]byte, 1024) + rngLk.Lock() + rng.Read(data) + rngLk.Unlock() + h, err := mh.Sum(data, mh.SHA2_512, -1) + if err != nil { + panic(err) + } + return cid.NewCidV1(cid.Raw, h), data +} + +func randCid() cid.Cid { + c, _ := randBlock() + return c +} diff --git a/pkg/storage/deferredcarwriter.go b/pkg/storage/deferredcarwriter.go deleted file mode 100644 index 276b3ef1..00000000 --- a/pkg/storage/deferredcarwriter.go +++ /dev/null @@ -1,160 +0,0 @@ -package storage - -import ( - "context" - "io" - "os" - "sync" - - "github.com/ipfs/go-cid" - carv2 "github.com/ipld/go-car/v2" - carstorage "github.com/ipld/go-car/v2/storage" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/linking" - ipldstorage "github.com/ipld/go-ipld-prime/storage" -) - -type putCb struct { - cb func(int) - once bool -} - -var _ ipldstorage.WritableStorage = (*DeferredCarWriter)(nil) -var _ io.Closer = (*DeferredCarWriter)(nil) - -type DeferredWriter interface { - ipldstorage.WritableStorage - io.Closer - BlockWriteOpener() linking.BlockWriteOpener - OnPut(cb func(int), once bool) -} - -// DeferredCarWriter creates a write-only CARv1 either to an existing stream or -// to a file designated by a supplied path. CARv1 content (including header) -// only begins when the first Put() operation is performed. If the output is a -// file, it will be created when the first Put() operation is performed. -// DeferredCarWriter is threadsafe, and can be used concurrently. -// Closing the writer will close, but not delete, the underlying file. This -// writer is intended for constructing the final output CARv1 for the user. -type DeferredCarWriter struct { - root cid.Cid - outPath string - outStream io.Writer - - lk sync.Mutex - f *os.File - w carstorage.WritableCar - putCb []putCb - opts []carv2.Option -} - -// NewDeferredCarWriterForPath creates a DeferredCarWriter that will write to a -// file designated by the supplied path. The file will only be created on the -// first Put() operation. -func NewDeferredCarWriterForPath(root cid.Cid, outPath string, opts ...carv2.Option) *DeferredCarWriter { - return &DeferredCarWriter{root: root, outPath: outPath, opts: opts} -} - -// NewDeferredCarWriterForStream creates a DeferredCarWriter that will write to -// the supplied stream. The stream will only be written to on the first Put() -// operation. -func NewDeferredCarWriterForStream(root cid.Cid, outStream io.Writer, opts ...carv2.Option) *DeferredCarWriter { - return &DeferredCarWriter{root: root, outStream: outStream, opts: opts} -} - -// OnPut will call a callback when each Put() operation is started. The argument -// to the callback is the number of bytes being written. If once is true, the -// callback will be removed after the first call. -func (dcw *DeferredCarWriter) OnPut(cb func(int), once bool) { - if dcw.putCb == nil { - dcw.putCb = make([]putCb, 0) - } - dcw.putCb = append(dcw.putCb, putCb{cb: cb, once: once}) -} - -// Has returns false if the key was not already written to the CARv1 output. -func (dcw *DeferredCarWriter) Has(ctx context.Context, key string) (bool, error) { - dcw.lk.Lock() - defer dcw.lk.Unlock() - - if dcw.w == nil { // shortcut, haven't written anything, don't even initialise - return false, nil - } - - writer, err := dcw.writer() - if err != nil { - return false, err - } - - return writer.Has(ctx, key) -} - -// Put writes the given content to the CARv1 output stream, creating it if it -// doesn't exist yet. -func (dcw *DeferredCarWriter) Put(ctx context.Context, key string, content []byte) error { - dcw.lk.Lock() - defer dcw.lk.Unlock() - - if dcw.putCb != nil { - // call all callbacks, remove those that were only needed once - for i := 0; i < len(dcw.putCb); i++ { - cb := dcw.putCb[i] - cb.cb(len(content)) - if cb.once { - dcw.putCb = append(dcw.putCb[:i], dcw.putCb[i+1:]...) - i-- - } - } - } - - // first Put() call, initialise writer, which will write a CARv1 header - writer, err := dcw.writer() - if err != nil { - return err - } - - return writer.Put(ctx, key, content) -} - -// writer() -func (dcw *DeferredCarWriter) writer() (carstorage.WritableCar, error) { - if dcw.w == nil { - outStream := dcw.outStream - if outStream == nil { - openedFile, err := os.OpenFile(dcw.outPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) - if err != nil { - return nil, err - } - dcw.f = openedFile - outStream = openedFile - } - w, err := carstorage.NewWritable(outStream, []cid.Cid{dcw.root}, append([]carv2.Option{carv2.WriteAsCarV1(true)}, dcw.opts...)...) - if err != nil { - return nil, err - } - dcw.w = w - } - return dcw.w, nil -} - -// Close closes the underlying file, if one was created. -func (dcw *DeferredCarWriter) Close() error { - dcw.lk.Lock() - defer dcw.lk.Unlock() - - if dcw.f != nil { - defer func() { dcw.f = nil }() - return dcw.f.Close() - } - return nil -} - -// BlockWriteOpener returns a BlockWriteOpener that operates on this storage. -func (dcw *DeferredCarWriter) BlockWriteOpener() linking.BlockWriteOpener { - return func(lctx linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { - wr, wrcommit, err := ipldstorage.PutStream(lctx.Ctx, dcw) - return wr, func(lnk ipld.Link) error { - return wrcommit(lnk.Binary()) - }, err - } -} diff --git a/pkg/storage/deferredcarwriter_test.go b/pkg/storage/deferredcarwriter_test.go deleted file mode 100644 index 486177c6..00000000 --- a/pkg/storage/deferredcarwriter_test.go +++ /dev/null @@ -1,224 +0,0 @@ -package storage - -import ( - "bytes" - "context" - "io" - "math/rand" - "os" - "sync" - "testing" - - "github.com/ipfs/go-cid" - carv2 "github.com/ipld/go-car/v2" - mh "github.com/multiformats/go-multihash" - "github.com/stretchr/testify/require" -) - -var rng = rand.New(rand.NewSource(3333)) -var rngLk sync.Mutex - -func TestDeferredCarWriterForPath(t *testing.T) { - ctx := context.Background() - testCid1, testData1 := randBlock() - testCid2, testData2 := randBlock() - - tmpFile := t.TempDir() + "/test.car" - - cw := NewDeferredCarWriterForPath(testCid1, tmpFile) - - _, err := os.Stat(tmpFile) - require.True(t, os.IsNotExist(err)) - - require.NoError(t, cw.Put(ctx, testCid1.KeyString(), testData1)) - require.NoError(t, cw.Put(ctx, testCid2.KeyString(), testData2)) - - stat, err := os.Stat(tmpFile) - require.NoError(t, err) - require.True(t, stat.Size() > int64(len(testData1)+len(testData2))) - - require.NoError(t, cw.Close()) - - // shouldn't be deleted - _, err = os.Stat(tmpFile) - require.NoError(t, err) - - r, err := os.Open(tmpFile) - require.NoError(t, err) - t.Cleanup(func() { r.Close() }) - carv2, err := carv2.NewBlockReader(r) - require.NoError(t, err) - - // compare CAR contents to what we wrote - require.Equal(t, carv2.Roots, []cid.Cid{testCid1}) - require.Equal(t, carv2.Version, uint64(1)) - - blk, err := carv2.Next() - require.NoError(t, err) - require.Equal(t, blk.Cid(), testCid1) - require.Equal(t, blk.RawData(), testData1) - - blk, err = carv2.Next() - require.NoError(t, err) - require.Equal(t, blk.Cid(), testCid2) - require.Equal(t, blk.RawData(), testData2) - - _, err = carv2.Next() - require.ErrorIs(t, err, io.EOF) -} - -func TestDeferredCarWriterForStream(t *testing.T) { - for _, tc := range []string{"path", "stream"} { - tc := tc - t.Run(tc, func(t *testing.T) { - t.Parallel() - ctx := context.Background() - testCid1, testData1 := randBlock() - testCid2, testData2 := randBlock() - testCid3, _ := randBlock() - - var cw *DeferredCarWriter - var buf bytes.Buffer - tmpFile := t.TempDir() + "/test.car" - - if tc == "path" { - cw = NewDeferredCarWriterForPath(testCid1, tmpFile) - _, err := os.Stat(tmpFile) - require.True(t, os.IsNotExist(err)) - } else { - cw = NewDeferredCarWriterForStream(testCid1, &buf) - require.Equal(t, buf.Len(), 0) - } - - has, err := cw.Has(ctx, testCid3.KeyString()) - require.NoError(t, err) - require.False(t, has) - - require.NoError(t, cw.Put(ctx, testCid1.KeyString(), testData1)) - has, err = cw.Has(ctx, testCid1.KeyString()) - require.NoError(t, err) - require.True(t, has) - require.NoError(t, cw.Put(ctx, testCid2.KeyString(), testData2)) - has, err = cw.Has(ctx, testCid1.KeyString()) - require.NoError(t, err) - require.True(t, has) - has, err = cw.Has(ctx, testCid2.KeyString()) - require.NoError(t, err) - require.True(t, has) - has, err = cw.Has(ctx, testCid3.KeyString()) - require.NoError(t, err) - require.False(t, has) - - if tc == "path" { - stat, err := os.Stat(tmpFile) - require.NoError(t, err) - require.True(t, stat.Size() > int64(len(testData1)+len(testData2))) - } else { - require.True(t, buf.Len() > len(testData1)+len(testData2)) - } - - require.NoError(t, cw.Close()) - - var rdr *carv2.BlockReader - if tc == "path" { - r, err := os.Open(tmpFile) - require.NoError(t, err) - rdr, err = carv2.NewBlockReader(r) - require.NoError(t, err) - t.Cleanup(func() { r.Close() }) - } else { - rdr, err = carv2.NewBlockReader(&buf) - require.NoError(t, err) - } - - // compare CAR contents to what we wrote - require.Equal(t, rdr.Roots, []cid.Cid{testCid1}) - require.Equal(t, rdr.Version, uint64(1)) - - blk, err := rdr.Next() - require.NoError(t, err) - require.Equal(t, blk.Cid(), testCid1) - require.Equal(t, blk.RawData(), testData1) - - blk, err = rdr.Next() - require.NoError(t, err) - require.Equal(t, blk.Cid(), testCid2) - require.Equal(t, blk.RawData(), testData2) - - _, err = rdr.Next() - require.ErrorIs(t, err, io.EOF) - }) - } -} - -func TestDeferredCarWriterPutCb(t *testing.T) { - ctx := context.Background() - testCid1, testData1 := randBlock() - testCid2, testData2 := randBlock() - - var buf bytes.Buffer - cw := NewDeferredCarWriterForStream(testCid1, &buf) - - var pc1 int - cw.OnPut(func(ii int) { - switch pc1 { - case 0: - require.Equal(t, buf.Len(), 0) // called before first write - require.Equal(t, len(testData1), ii) - case 1: - require.Equal(t, len(testData2), ii) - default: - require.Fail(t, "unexpected put callback") - } - pc1++ - }, false) - var pc2 int - cw.OnPut(func(ii int) { - switch pc2 { - case 0: - require.Equal(t, buf.Len(), 0) // called before first write - require.Equal(t, len(testData1), ii) - case 1: - require.Equal(t, len(testData2), ii) - default: - require.Fail(t, "unexpected put callback") - } - pc2++ - }, false) - var pc3 int - cw.OnPut(func(ii int) { - switch pc3 { - case 0: - require.Equal(t, buf.Len(), 0) // called before first write - require.Equal(t, len(testData1), ii) - default: - require.Fail(t, "unexpected put callback") - } - pc3++ - }, true) - - require.NoError(t, cw.Put(ctx, testCid1.KeyString(), testData1)) - require.NoError(t, cw.Put(ctx, testCid2.KeyString(), testData2)) - require.NoError(t, cw.Close()) - - require.Equal(t, 2, pc1) - require.Equal(t, 2, pc2) - require.Equal(t, 1, pc3) -} - -func randBlock() (cid.Cid, []byte) { - data := make([]byte, 1024) - rngLk.Lock() - rng.Read(data) - rngLk.Unlock() - h, err := mh.Sum(data, mh.SHA2_512, -1) - if err != nil { - panic(err) - } - return cid.NewCidV1(cid.Raw, h), data -} - -func randCid() cid.Cid { - c, _ := randBlock() - return c -} diff --git a/pkg/storage/duplicateaddercar.go b/pkg/storage/duplicateaddercar.go index 90bd46c3..07b6f3ae 100644 --- a/pkg/storage/duplicateaddercar.go +++ b/pkg/storage/duplicateaddercar.go @@ -11,15 +11,26 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/storage/deferred" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + ipldstorage "github.com/ipld/go-ipld-prime/storage" trustlessutils "github.com/ipld/go-trustless-utils" "github.com/ipld/go-trustless-utils/traversal" ) +type DeferredWriter interface { + ipldstorage.WritableStorage + io.Closer + BlockWriteOpener() linking.BlockWriteOpener + OnPut(cb func(int), once bool) +} + +var _ DeferredWriter = (*DuplicateAdderCar)(nil) + type DuplicateAdderCar struct { - *DeferredCarWriter + *deferred.DeferredCarWriter ctx context.Context root cid.Cid path string @@ -33,12 +44,12 @@ type DuplicateAdderCar struct { func NewDuplicateAdderCarForStream( ctx context.Context, + outStream io.Writer, root cid.Cid, path string, scope trustlessutils.DagScope, bytes *trustlessutils.ByteRange, store *DeferredStorageCar, - outStream io.Writer, ) *DuplicateAdderCar { blockStream := &blockStream{ctx: ctx, seen: make(map[cid.Cid]struct{})} @@ -46,7 +57,7 @@ func NewDuplicateAdderCarForStream( blockStream.cond = sync.NewCond(&blockStream.mu) // create the car writer for the final stream - outgoing := NewDeferredCarWriterForStream(root, outStream, carv2.AllowDuplicatePuts(true)) + outgoing := deferred.NewDeferredCarWriterForStream(outStream, []cid.Cid{root}, carv2.AllowDuplicatePuts(true)) return &DuplicateAdderCar{ DeferredCarWriter: outgoing, ctx: ctx, diff --git a/pkg/storage/duplicateaddercar_test.go b/pkg/storage/duplicateaddercar_test.go index f7f8ea06..33c05197 100644 --- a/pkg/storage/duplicateaddercar_test.go +++ b/pkg/storage/duplicateaddercar_test.go @@ -33,7 +33,7 @@ func TestDuplicateAdderCar(t *testing.T) { store := storage.NewDeferredStorageCar("", unixfsFileWithDups.Root) ctx := context.Background() - carWriter := storage.NewDuplicateAdderCarForStream(ctx, unixfsFileWithDups.Root, "", trustlessutils.DagScopeAll, nil, store, buf) + carWriter := storage.NewDuplicateAdderCarForStream(ctx, buf, unixfsFileWithDups.Root, "", trustlessutils.DagScopeAll, nil, store) cachingTempStore := storage.NewCachingTempStore(carWriter.BlockWriteOpener(), store) // write the root block, containing sharding metadata