From 7d4502e472dd47662f8da6ab4e811b618661183d Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 22 Nov 2021 21:14:57 +0100 Subject: [PATCH 01/10] Propose traversal-based car creation --- v2/go.mod | 7 +- v2/go.sum | 32 +++++-- v2/selective.go | 221 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 249 insertions(+), 11 deletions(-) create mode 100644 v2/selective.go diff --git a/v2/go.mod b/v2/go.mod index 0befc4ba..6c346160 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -9,16 +9,13 @@ require ( github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-merkledag v0.3.2 - github.com/klauspost/cpuid/v2 v2.0.8 // indirect - github.com/kr/pretty v0.2.1 // indirect + github.com/ipld/go-ipld-prime v0.14.0 github.com/mattn/go-colorable v0.1.8 // indirect github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 - github.com/multiformats/go-multihash v0.0.15 + github.com/multiformats/go-multihash v0.1.0 github.com/multiformats/go-varint v0.0.6 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 - github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect github.com/stretchr/testify v1.7.0 - github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect golang.org/x/exp v0.0.0-20210615023648-acb5c1269671 diff --git a/v2/go.sum b/v2/go.sum index a21583a7..75414c69 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -55,6 +55,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -76,6 +77,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= @@ -111,6 +114,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -171,6 +176,7 @@ github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7s github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.8-0.20210716091050-de6c03deae1c h1:uVMZWk8sJN1l/47TtXRXDz0M9/6v0yw7neDn5WfeLwg= @@ -225,6 +231,8 @@ github.com/ipfs/go-peertaskqueue v0.1.0 h1:bpRbgv76eT4avutNPDFZuCPOQus6qTgurEYxf github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= +github.com/ipld/go-ipld-prime v0.14.0 h1:2FnBqUjmmgxgZD6/zB3eygWlmIsHNGrZ57L99x3xD6Q= +github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -251,18 +259,19 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.8 h1:bhR2mgIlno/Sfk4oUbH4sPlc83z1yGrN9bvqiq3C33I= -github.com/klauspost/cpuid/v2 v2.0.8/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muMD06dzLiahtGM1eouRNOzVV7tdQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= @@ -375,6 +384,7 @@ github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 h1:ZrUuMKNgJ52qHPoQ+bx0h0uBfcWmN7Px+4uKSZeesiI= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= @@ -382,8 +392,9 @@ github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcM github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= +github.com/multiformats/go-multihash v0.1.0 h1:CgAgwqk3//SVEw3T+6DqI4mWMyRuDwZtOWcJT0q9+EA= +github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -452,6 +463,8 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -473,6 +486,7 @@ github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a h1:/eS3yfGjQ github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -490,6 +504,8 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA= +github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= @@ -692,6 +708,8 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -699,5 +717,7 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= +lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/v2/selective.go b/v2/selective.go new file mode 100644 index 00000000..f1ff7e18 --- /dev/null +++ b/v2/selective.go @@ -0,0 +1,221 @@ +package car + +import ( + "bytes" + "context" + "io" + + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2/internal/carv1" + ipld "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" + "github.com/multiformats/go-varint" +) + +// PrepareTraversal walks through the proposed dag traversal to learn it's total size in order to be able to +// stream out a car to a writer in the expected traversal order in one go. +func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, opts ...Option) (Traversal, error) { + cls, cntr := countingLinkSystem(*ls) + + c1h := carv1.CarHeader{Roots: []cid.Cid{root}, Version: 1} + headSize, err := carv1.HeaderSize(&c1h) + if err != nil { + return nil, err + } + if err := traverse(ctx, &cls, root, selector, opts...); err != nil { + return nil, err + } + tc := traversalCar{ + size: headSize + cntr.totalRead, + ctx: ctx, + root: root, + selector: selector, + ls: ls, + opts: opts, + } + return &tc, nil +} + +// Traversal is a allows writing a car with the data specified by a selector. +type Traversal interface { + io.WriterTo +} + +type traversalCar struct { + size uint64 + ctx context.Context + root cid.Cid + selector ipld.Node + ls *ipld.LinkSystem + opts []Option +} + +func (tc *traversalCar) WriteTo(w io.Writer) (int64, error) { + n, err := w.Write(Pragma) + if err != nil { + return int64(n), err + } + h, err := tc.WriteHeader(w) + if err != nil { + return int64(n) + h, err + } + h += int64(n) + + // write the v1 header + c1h := carv1.CarHeader{Roots: []cid.Cid{tc.root}, Version: 1} + if err := carv1.WriteHeader(&c1h, w); err != nil { + return h, err + } + hn, err := carv1.HeaderSize(&c1h) + h += int64(hn) + if err != nil { + return h, err + } + + // write the block. + wls, writer := teeingLinkSystem(*tc.ls, w) + err = traverse(tc.ctx, &wls, tc.root, tc.selector, tc.opts...) + h += int64(writer.size) + if err != nil { + return h, err + } + + return h, nil +} + +func (tc *traversalCar) WriteHeader(w io.Writer) (int64, error) { + h := NewHeader(tc.size) + // TODO: support calculation / inclusion of the index. + h.IndexOffset = 0 + return h.WriteTo(w) +} + +type counter struct { + totalRead uint64 +} + +type countingReader struct { + r io.Reader + c *counter + read uint64 + cid string +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + if err == io.EOF { + // add in the overall length of the block. + n += len(c.cid) + uv := varint.ToUvarint(uint64(n)) + n += len(uv) + } + c.c.totalRead += uint64(n) + c.read += uint64(n) + return n, err +} + +func countingLinkSystem(ls ipld.LinkSystem) (ipld.LinkSystem, *counter) { + c := counter{} + return linking.LinkSystem{ + EncoderChooser: ls.EncoderChooser, + DecoderChooser: ls.DecoderChooser, + HasherChooser: ls.HasherChooser, + StorageWriteOpener: ls.StorageWriteOpener, + StorageReadOpener: func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { + r, err := ls.StorageReadOpener(lc, l) + if err != nil { + return nil, err + } + return &countingReader{r, &c, 0, l.Binary()}, nil + }, + TrustedStorage: ls.TrustedStorage, + NodeReifier: ls.NodeReifier, + }, &c +} + +func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Node, opts ...Option) error { + sel, err := selector.CompileSelector(s) + if err != nil { + return err + } + + progress := traversal.Progress{ + Cfg: &traversal.Config{ + Ctx: ctx, + LinkSystem: *ls, + LinkTargetNodePrototypeChooser: func(_ ipld.Link, _ linking.LinkContext) (ipld.NodePrototype, error) { + return basicnode.Prototype.Any, nil + }, + LinkVisitOnlyOnce: true, // TODO: from opts, + }, + } + + lnk := cidlink.Link{Cid: root} + rootNode, err := ls.Load(ipld.LinkContext{}, lnk, basicnode.Prototype.Any) + if err != nil { + return err + } + return progress.WalkMatching(rootNode, sel, func(_ traversal.Progress, _ ipld.Node) error { + return nil + }) +} + +type writerOutput struct { + w io.Writer + size uint64 +} + +type writingReader struct { + r io.Reader + len int64 + cid string + wo *writerOutput +} + +func (w *writingReader) Read(p []byte) (int, error) { + if w.wo != nil { + // write the cid + size := varint.ToUvarint(uint64(w.len)) + if _, err := w.wo.w.Write(size); err != nil { + return 0, err + } + if _, err := w.wo.w.Write([]byte(w.cid)); err != nil { + return 0, err + } + cpy := bytes.NewBuffer(w.r.(*bytes.Buffer).Bytes()) + if _, err := cpy.WriteTo(w.wo.w); err != nil { + return 0, err + } + w.wo = nil + } + + return w.r.Read(p) +} + +func teeingLinkSystem(ls ipld.LinkSystem, w io.Writer) (ipld.LinkSystem, *writerOutput) { + wo := writerOutput{w, 0} + return linking.LinkSystem{ + EncoderChooser: ls.EncoderChooser, + DecoderChooser: ls.DecoderChooser, + HasherChooser: ls.HasherChooser, + StorageWriteOpener: ls.StorageWriteOpener, + StorageReadOpener: func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { + r, err := ls.StorageReadOpener(lc, l) + if err != nil { + return nil, err + } + buf := bytes.NewBuffer(nil) + n, err := buf.ReadFrom(r) + if err != nil { + return nil, err + } + return &writingReader{buf, n, l.Binary(), &wo}, nil + }, + TrustedStorage: ls.TrustedStorage, + NodeReifier: ls.NodeReifier, + }, &wo +} From 44245ff39c46c2abd1ac83b753b0241c84ba80c2 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 23 Nov 2021 10:43:26 +0100 Subject: [PATCH 02/10] file mode - traverses once and fixes up carv2 header at the end --- v2/selective.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/v2/selective.go b/v2/selective.go index f1ff7e18..57fd3e1e 100644 --- a/v2/selective.go +++ b/v2/selective.go @@ -3,7 +3,9 @@ package car import ( "bytes" "context" + "fmt" "io" + "os" "github.com/ipfs/go-cid" "github.com/ipld/go-car/v2/internal/carv1" @@ -16,6 +18,8 @@ import ( "github.com/multiformats/go-varint" ) +const ErrSizeMismatch = "car-error-sizemismatch" + // PrepareTraversal walks through the proposed dag traversal to learn it's total size in order to be able to // stream out a car to a writer in the expected traversal order in one go. func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, opts ...Option) (Traversal, error) { @@ -40,6 +44,40 @@ func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, se return &tc, nil } +func FileTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, destination string, opts ...Option) error { + tc := traversalCar{ + size: 0, + ctx: ctx, + root: root, + selector: selector, + ls: ls, + opts: opts, + } + + fp, err := os.Create(destination) + if err != nil { + return err + } + defer fp.Close() + + _, err = tc.WriteTo(fp) + if err != nil { + return err + } + + // fix header size. + if _, err = fp.Seek(0, 0); err != nil { + return err + } + + tc.size = uint64(tc.size) + if _, err = tc.WriteHeader(fp); err != nil { + return err + } + + return nil +} + // Traversal is a allows writing a car with the data specified by a selector. type Traversal interface { io.WriterTo @@ -75,14 +113,20 @@ func (tc *traversalCar) WriteTo(w io.Writer) (int64, error) { if err != nil { return h, err } + v1Size := hn // write the block. wls, writer := teeingLinkSystem(*tc.ls, w) err = traverse(tc.ctx, &wls, tc.root, tc.selector, tc.opts...) h += int64(writer.size) + v1Size += writer.size if err != nil { return h, err } + if tc.size != 0 && tc.size != v1Size { + return h, fmt.Errorf(ErrSizeMismatch) + } + tc.size = v1Size return h, nil } From 642b02b00f484339da110565b782ee753e9166cb Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 25 Nov 2021 12:20:13 +0100 Subject: [PATCH 03/10] add traversal budget option and respect set options on creation --- v2/options.go | 8 +++++- v2/selective.go | 72 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/v2/options.go b/v2/options.go index 5a3b1930..484d28b1 100644 --- a/v2/options.go +++ b/v2/options.go @@ -1,6 +1,10 @@ package car -import "github.com/multiformats/go-multicodec" +import ( + "math" + + "github.com/multiformats/go-multicodec" +) // DefaultMaxIndexCidSize specifies the maximum size in byptes accepted as a section CID by CARv2 index. const DefaultMaxIndexCidSize = 2 << 10 // 2 KiB @@ -33,6 +37,7 @@ type Options struct { BlockstoreAllowDuplicatePuts bool BlockstoreUseWholeCIDs bool + MaxTraversalLinks uint64 } // ApplyOptions applies given opts and returns the resulting Options. @@ -40,6 +45,7 @@ type Options struct { // side effect of Option. func ApplyOptions(opt ...Option) Options { var opts Options + opts.MaxTraversalLinks = math.MaxInt64 //default: traverse all for _, o := range opt { o(&opts) } diff --git a/v2/selective.go b/v2/selective.go index 57fd3e1e..80a4f0f8 100644 --- a/v2/selective.go +++ b/v2/selective.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io" + "math" "os" "github.com/ipfs/go-cid" @@ -18,11 +19,24 @@ import ( "github.com/multiformats/go-varint" ) +// ErrSizeMismatch is returned when a written traversal realizes the written header size does not +// match the actual number of car bytes written. const ErrSizeMismatch = "car-error-sizemismatch" +// MaxTraversalLinks changes the allowed number of links a selector traversal +// can execute before failing. +// +// Note that setting this option may cause an error to be returned from selector +// execution when building a SelectiveCar. +func MaxTraversalLinks(MaxTraversalLinks uint64) Option { + return func(sco *Options) { + sco.MaxTraversalLinks = MaxTraversalLinks + } +} + // PrepareTraversal walks through the proposed dag traversal to learn it's total size in order to be able to // stream out a car to a writer in the expected traversal order in one go. -func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, opts ...Option) (Traversal, error) { +func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, opts ...Option) (Writer, error) { cls, cntr := countingLinkSystem(*ls) c1h := carv1.CarHeader{Roots: []cid.Cid{root}, Version: 1} @@ -30,7 +44,7 @@ func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, se if err != nil { return nil, err } - if err := traverse(ctx, &cls, root, selector, opts...); err != nil { + if err := traverse(ctx, &cls, root, selector, ApplyOptions(opts...)); err != nil { return nil, err } tc := traversalCar{ @@ -44,6 +58,7 @@ func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, se return &tc, nil } +// FileTraversal writes a carv2 matching a given root and selector to a file path. func FileTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, destination string, opts ...Option) error { tc := traversalCar{ size: 0, @@ -78,8 +93,8 @@ func FileTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selec return nil } -// Traversal is a allows writing a car with the data specified by a selector. -type Traversal interface { +// Writer is an interface allowing writing a car with the data specified by a selector. +type Writer interface { io.WriterTo } @@ -102,40 +117,43 @@ func (tc *traversalCar) WriteTo(w io.Writer) (int64, error) { return int64(n) + h, err } h += int64(n) + v1s, err := tc.WriteV1(w) + h += int64(v1s) + + return h, err +} + +func (tc *traversalCar) WriteHeader(w io.Writer) (int64, error) { + h := NewHeader(tc.size) + // TODO: support calculation / inclusion of the index. + h.IndexOffset = 0 + return h.WriteTo(w) +} +func (tc *traversalCar) WriteV1(w io.Writer) (uint64, error) { // write the v1 header c1h := carv1.CarHeader{Roots: []cid.Cid{tc.root}, Version: 1} if err := carv1.WriteHeader(&c1h, w); err != nil { - return h, err + return 0, err } - hn, err := carv1.HeaderSize(&c1h) - h += int64(hn) + v1Size, err := carv1.HeaderSize(&c1h) if err != nil { - return h, err + return v1Size, err } - v1Size := hn // write the block. + opts := ApplyOptions(tc.opts...) wls, writer := teeingLinkSystem(*tc.ls, w) - err = traverse(tc.ctx, &wls, tc.root, tc.selector, tc.opts...) - h += int64(writer.size) + err = traverse(tc.ctx, &wls, tc.root, tc.selector, opts) v1Size += writer.size if err != nil { - return h, err + return v1Size, err } if tc.size != 0 && tc.size != v1Size { - return h, fmt.Errorf(ErrSizeMismatch) + return v1Size, fmt.Errorf(ErrSizeMismatch) } tc.size = v1Size - - return h, nil -} - -func (tc *traversalCar) WriteHeader(w io.Writer) (int64, error) { - h := NewHeader(tc.size) - // TODO: support calculation / inclusion of the index. - h.IndexOffset = 0 - return h.WriteTo(w) + return v1Size, nil } type counter struct { @@ -181,7 +199,7 @@ func countingLinkSystem(ls ipld.LinkSystem) (ipld.LinkSystem, *counter) { }, &c } -func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Node, opts ...Option) error { +func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Node, opts Options) error { sel, err := selector.CompileSelector(s) if err != nil { return err @@ -194,9 +212,15 @@ func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Nod LinkTargetNodePrototypeChooser: func(_ ipld.Link, _ linking.LinkContext) (ipld.NodePrototype, error) { return basicnode.Prototype.Any, nil }, - LinkVisitOnlyOnce: true, // TODO: from opts, + LinkVisitOnlyOnce: !opts.BlockstoreAllowDuplicatePuts, }, } + if opts.MaxTraversalLinks < math.MaxInt64 { + progress.Budget = &traversal.Budget{ + NodeBudget: math.MaxInt64, + LinkBudget: int64(opts.MaxTraversalLinks), + } + } lnk := cidlink.Link{Cid: root} rootNode, err := ls.Load(ipld.LinkContext{}, lnk, basicnode.Prototype.Any) From df2a186fdc5588c6778d999cf8ded551eb359385 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 25 Nov 2021 11:26:36 -0800 Subject: [PATCH 04/10] * Handle the various possible car writer options. * Provide a method for writing a carv1 to a writer stream. * Return count of bytes written across various interfaces. --- v2/blockstore/insertionindex.go | 9 +- v2/blockstore/readwrite.go | 2 +- v2/index/example_test.go | 2 +- v2/index/index.go | 16 +- v2/index/index_test.go | 6 +- v2/index/indexsorted.go | 28 ++-- v2/index/mhindexsorted.go | 20 ++- v2/index/mhindexsorted_test.go | 2 +- v2/internal/loader/counting_loader.go | 68 ++++++++ v2/internal/loader/writing_loader.go | 119 ++++++++++++++ v2/options.go | 8 + v2/selective.go | 218 ++++++++++++-------------- v2/writer.go | 5 +- 13 files changed, 347 insertions(+), 156 deletions(-) create mode 100644 v2/internal/loader/counting_loader.go create mode 100644 v2/internal/loader/writing_loader.go diff --git a/v2/blockstore/insertionindex.go b/v2/blockstore/insertionindex.go index 7cca61b3..e8575ee1 100644 --- a/v2/blockstore/insertionindex.go +++ b/v2/blockstore/insertionindex.go @@ -103,10 +103,13 @@ func (ii *insertionIndex) GetAll(c cid.Cid, fn func(uint64) bool) error { return nil } -func (ii *insertionIndex) Marshal(w io.Writer) error { +func (ii *insertionIndex) Marshal(w io.Writer) (uint64, error) { + l := uint64(0) if err := binary.Write(w, binary.LittleEndian, int64(ii.items.Len())); err != nil { - return err + return l, err } + l += 8 + var err error iter := func(i llrb.Item) bool { if err = cbor.Encode(w, i.(recordDigest).Record); err != nil { @@ -115,7 +118,7 @@ func (ii *insertionIndex) Marshal(w io.Writer) error { return true } ii.items.AscendGreaterOrEqual(ii.items.Min(), iter) - return err + return l, err } func (ii *insertionIndex) Unmarshal(r io.Reader) error { diff --git a/v2/blockstore/readwrite.go b/v2/blockstore/readwrite.go index 0dda5a0b..4f288fa0 100644 --- a/v2/blockstore/readwrite.go +++ b/v2/blockstore/readwrite.go @@ -376,7 +376,7 @@ func (b *ReadWrite) Finalize() error { if err != nil { return err } - if err := index.WriteTo(fi, internalio.NewOffsetWriter(b.f, int64(b.header.IndexOffset))); err != nil { + if _, err := index.WriteTo(fi, internalio.NewOffsetWriter(b.f, int64(b.header.IndexOffset))); err != nil { return err } if _, err := b.header.WriteTo(internalio.NewOffsetWriter(b.f, carv2.PragmaSize)); err != nil { diff --git a/v2/index/example_test.go b/v2/index/example_test.go index ca0ac73a..3e484afb 100644 --- a/v2/index/example_test.go +++ b/v2/index/example_test.go @@ -77,7 +77,7 @@ func ExampleWriteTo() { panic(err) } }() - err = index.WriteTo(idx, f) + _, err = index.WriteTo(idx, f) if err != nil { panic(err) } diff --git a/v2/index/index.go b/v2/index/index.go index 998a17a0..c8eccefc 100644 --- a/v2/index/index.go +++ b/v2/index/index.go @@ -15,6 +15,9 @@ import ( "github.com/ipfs/go-cid" ) +// CarIndexNoIndex is a sentinal value used as a multicodec code for the index indicating no index. +const CarIndexNoIndex = 0x300000 + type ( // Record is a pre-processed record of a car item and location. Record struct { @@ -40,7 +43,7 @@ type ( Codec() multicodec.Code // Marshal encodes the index in serial form. - Marshal(w io.Writer) error + Marshal(w io.Writer) (uint64, error) // Unmarshal decodes the index from its serial form. Unmarshal(r io.Reader) error @@ -118,13 +121,16 @@ func New(codec multicodec.Code) (Index, error) { // WriteTo writes the given idx into w. // The written bytes include the index encoding. // This can then be read back using index.ReadFrom -func WriteTo(idx Index, w io.Writer) error { +func WriteTo(idx Index, w io.Writer) (uint64, error) { buf := make([]byte, binary.MaxVarintLen64) b := varint.PutUvarint(buf, uint64(idx.Codec())) - if _, err := w.Write(buf[:b]); err != nil { - return err + n, err := w.Write(buf[:b]) + if err != nil { + return uint64(n), err } - return idx.Marshal(w) + + l, err := idx.Marshal(w) + return uint64(n) + l, err } // ReadFrom reads index from r. diff --git a/v2/index/index_test.go b/v2/index/index_test.go index bb369cfe..267beb03 100644 --- a/v2/index/index_test.go +++ b/v2/index/index_test.go @@ -100,7 +100,8 @@ func TestWriteTo(t *testing.T) { destF, err := os.Create(dest) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, destF.Close()) }) - require.NoError(t, WriteTo(wantIdx, destF)) + _, err = WriteTo(wantIdx, destF) + require.NoError(t, err) // Seek to the beginning of the written out file. _, err = destF.Seek(0, io.SeekStart) @@ -126,6 +127,7 @@ func TestMarshalledIndexStartsWithCodec(t *testing.T) { // Assert the first two bytes are the corresponding multicodec code. buf := new(bytes.Buffer) - require.NoError(t, WriteTo(wantIdx, buf)) + _, err = WriteTo(wantIdx, buf) + require.NoError(t, err) require.Equal(t, varint.ToUvarint(uint64(multicodec.CarIndexSorted)), buf.Bytes()[:2]) } diff --git a/v2/index/indexsorted.go b/v2/index/indexsorted.go index 86994dd8..2c05a922 100644 --- a/v2/index/indexsorted.go +++ b/v2/index/indexsorted.go @@ -46,16 +46,18 @@ func (r recordSet) Swap(i, j int) { r[i], r[j] = r[j], r[i] } -func (s *singleWidthIndex) Marshal(w io.Writer) error { +func (s *singleWidthIndex) Marshal(w io.Writer) (uint64, error) { + l := uint64(0) if err := binary.Write(w, binary.LittleEndian, s.width); err != nil { - return err + return 0, err } + l += 4 if err := binary.Write(w, binary.LittleEndian, int64(len(s.index))); err != nil { - return err + return l, err } - // TODO: we could just w.Write(s.index) here and avoid overhead - _, err := io.Copy(w, bytes.NewBuffer(s.index)) - return err + l += 8 + n, err := w.Write(s.index) + return l + uint64(n), err } func (s *singleWidthIndex) Unmarshal(r io.Reader) error { @@ -158,10 +160,12 @@ func (m *multiWidthIndex) Codec() multicodec.Code { return multicodec.CarIndexSorted } -func (m *multiWidthIndex) Marshal(w io.Writer) error { +func (m *multiWidthIndex) Marshal(w io.Writer) (uint64, error) { + l := uint64(0) if err := binary.Write(w, binary.LittleEndian, int32(len(*m))); err != nil { - return err + return l, err } + l += 4 // The widths are unique, but ranging over a map isn't deterministic. // As per the CARv2 spec, we must order buckets by digest length. @@ -176,11 +180,13 @@ func (m *multiWidthIndex) Marshal(w io.Writer) error { for _, width := range widths { bucket := (*m)[width] - if err := bucket.Marshal(w); err != nil { - return err + n, err := bucket.Marshal(w) + l += n + if err != nil { + return l, err } } - return nil + return l, nil } func (m *multiWidthIndex) Unmarshal(r io.Reader) error { diff --git a/v2/index/mhindexsorted.go b/v2/index/mhindexsorted.go index e3cae3d0..55975b8e 100644 --- a/v2/index/mhindexsorted.go +++ b/v2/index/mhindexsorted.go @@ -31,11 +31,12 @@ func newMultiWidthCodedIndex() *multiWidthCodedIndex { } } -func (m *multiWidthCodedIndex) Marshal(w io.Writer) error { +func (m *multiWidthCodedIndex) Marshal(w io.Writer) (uint64, error) { if err := binary.Write(w, binary.LittleEndian, m.code); err != nil { - return err + return 8, err } - return m.multiWidthIndex.Marshal(w) + n, err := m.multiWidthIndex.Marshal(w) + return 8 + n, err } func (m *multiWidthCodedIndex) Unmarshal(r io.Reader) error { @@ -59,22 +60,25 @@ func (m *MultihashIndexSorted) Codec() multicodec.Code { return multicodec.CarMultihashIndexSorted } -func (m *MultihashIndexSorted) Marshal(w io.Writer) error { +func (m *MultihashIndexSorted) Marshal(w io.Writer) (uint64, error) { if err := binary.Write(w, binary.LittleEndian, int32(len(*m))); err != nil { - return err + return 4, err } // The codes are unique, but ranging over a map isn't deterministic. // As per the CARv2 spec, we must order buckets by digest length. // TODO update CARv2 spec to reflect this for the new index type. codes := m.sortedMultihashCodes() + l := uint64(4) for _, code := range codes { mwci := (*m)[code] - if err := mwci.Marshal(w); err != nil { - return err + n, err := mwci.Marshal(w) + l += n + if err != nil { + return l, err } } - return nil + return l, nil } func (m *MultihashIndexSorted) sortedMultihashCodes() []uint64 { diff --git a/v2/index/mhindexsorted_test.go b/v2/index/mhindexsorted_test.go index e02ba059..79fc9c5f 100644 --- a/v2/index/mhindexsorted_test.go +++ b/v2/index/mhindexsorted_test.go @@ -32,7 +32,7 @@ func TestMultiWidthCodedIndex_MarshalUnmarshal(t *testing.T) { // Marshal the index. buf := new(bytes.Buffer) - err = subject.Marshal(buf) + _, err = subject.Marshal(buf) require.NoError(t, err) // Unmarshal it back to another instance of mh sorted index. diff --git a/v2/internal/loader/counting_loader.go b/v2/internal/loader/counting_loader.go new file mode 100644 index 00000000..84086b1d --- /dev/null +++ b/v2/internal/loader/counting_loader.go @@ -0,0 +1,68 @@ +package loader + +import ( + "io" + + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/linking" + "github.com/multiformats/go-varint" +) + +// counter tracks how much data has been read. +type counter struct { + totalRead uint64 +} + +func (c *counter) Size() uint64 { + return c.totalRead +} + +// ReadCounter provides an externally consumable interface to the +// additional data tracked about the linksystem. +type ReadCounter interface { + Size() uint64 +} + +type countingReader struct { + r io.Reader + c *counter + read uint64 + cid string +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + if err == io.EOF { + // add in the overall length of the block. + n += len(c.cid) + uv := varint.ToUvarint(uint64(n)) + n += len(uv) + } + c.c.totalRead += uint64(n) + c.read += uint64(n) + return n, err +} + +// CountingLinkSystem wraps an ipld linksystem with to track the size of +// data loaded in a `counter` object. Each time nodes are loaded from the +// link system which trigger block reads, the size of the block as it would +// appear in a CAR file is added to the counter (included the size of the +// CID and the varint length for the block data). +func CountingLinkSystem(ls ipld.LinkSystem) (ipld.LinkSystem, ReadCounter) { + c := counter{} + return linking.LinkSystem{ + EncoderChooser: ls.EncoderChooser, + DecoderChooser: ls.DecoderChooser, + HasherChooser: ls.HasherChooser, + StorageWriteOpener: ls.StorageWriteOpener, + StorageReadOpener: func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { + r, err := ls.StorageReadOpener(lc, l) + if err != nil { + return nil, err + } + return &countingReader{r, &c, 0, l.Binary()}, nil + }, + TrustedStorage: ls.TrustedStorage, + NodeReifier: ls.NodeReifier, + }, &c +} diff --git a/v2/internal/loader/writing_loader.go b/v2/internal/loader/writing_loader.go new file mode 100644 index 00000000..6b43ae90 --- /dev/null +++ b/v2/internal/loader/writing_loader.go @@ -0,0 +1,119 @@ +package loader + +import ( + "bytes" + "io" + + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2/index" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/linking" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-varint" +) + +type writerOutput struct { + w io.Writer + size uint64 + code multicodec.Code + rcrds []index.Record +} + +func (w *writerOutput) Size() uint64 { + return w.size +} + +func (w *writerOutput) Index() (index.Index, error) { + idx, err := index.New(w.code) + if err != nil { + return nil, err + } + if err := idx.Load(w.rcrds); err != nil { + return nil, err + } + + return idx, nil +} + +// An IndexTracker tracks the records loaded/written, calculate an +// index based on them. +type IndexTracker interface { + ReadCounter + Index() (index.Index, error) +} + +type writingReader struct { + r io.Reader + len int64 + cid string + wo *writerOutput +} + +func (w *writingReader) Read(p []byte) (int, error) { + if w.wo != nil { + // write the cid + size := varint.ToUvarint(uint64(w.len) + uint64(len(w.cid))) + if _, err := w.wo.w.Write(size); err != nil { + return 0, err + } + if _, err := w.wo.w.Write([]byte(w.cid)); err != nil { + return 0, err + } + cpy := bytes.NewBuffer(w.r.(*bytes.Buffer).Bytes()) + if _, err := cpy.WriteTo(w.wo.w); err != nil { + return 0, err + } + + // maybe write the index. + if w.wo.code != index.CarIndexNoIndex { + _, c, err := cid.CidFromBytes([]byte(w.cid)) + if err != nil { + return 0, err + } + w.wo.rcrds = append(w.wo.rcrds, index.Record{ + Cid: c, + Offset: w.wo.size, + }) + } + w.wo.size += uint64(w.len) + uint64(len(size)+len(w.cid)) + + w.wo = nil + } + + return w.r.Read(p) +} + +// TeeingLinkSystem wraps an IPLD.LinkSystem so that each time a block is loaded from it, +// that block is also written as a CAR block to the provided io.Writer. Metadata +// (the size of data written) is provided in the second return value. +// The `initialOffset` is used to calculate the offsets recorded for the index, and will be +// included in the `.Size()` of the IndexTracker. +// An indexCodec of `index.CarIndexNoIndex` can be used to not track these offsets. +func TeeingLinkSystem(ls ipld.LinkSystem, w io.Writer, initialOffset uint64, indexCodec multicodec.Code) (ipld.LinkSystem, IndexTracker) { + wo := writerOutput{ + w: w, + size: initialOffset, + code: indexCodec, + rcrds: make([]index.Record, 0), + } + return linking.LinkSystem{ + EncoderChooser: ls.EncoderChooser, + DecoderChooser: ls.DecoderChooser, + HasherChooser: ls.HasherChooser, + StorageWriteOpener: ls.StorageWriteOpener, + StorageReadOpener: func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { + r, err := ls.StorageReadOpener(lc, l) + if err != nil { + return nil, err + } + buf := bytes.NewBuffer(nil) + n, err := buf.ReadFrom(r) + if err != nil { + return nil, err + } + return &writingReader{buf, n, l.Binary(), &wo}, nil + }, + TrustedStorage: ls.TrustedStorage, + NodeReifier: ls.NodeReifier, + }, &wo +} diff --git a/v2/options.go b/v2/options.go index 484d28b1..61f6d542 100644 --- a/v2/options.go +++ b/v2/options.go @@ -3,6 +3,7 @@ package car import ( "math" + "github.com/ipld/go-car/v2/index" "github.com/multiformats/go-multicodec" ) @@ -90,6 +91,13 @@ func UseIndexCodec(c multicodec.Code) Option { } } +// WithoutIndex flags that no index should be included in generation. +func WithoutIndex() Option { + return func(o *Options) { + o.IndexCodec = index.CarIndexNoIndex + } +} + // StoreIdentityCIDs sets whether to persist sections that are referenced by // CIDs with multihash.IDENTITY digest. // When writing CAR files with this option, diff --git a/v2/selective.go b/v2/selective.go index 80a4f0f8..b736cdae 100644 --- a/v2/selective.go +++ b/v2/selective.go @@ -1,7 +1,6 @@ package car import ( - "bytes" "context" "fmt" "io" @@ -9,19 +8,24 @@ import ( "os" "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2/index" "github.com/ipld/go-car/v2/internal/carv1" + "github.com/ipld/go-car/v2/internal/loader" ipld "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" - "github.com/multiformats/go-varint" ) // ErrSizeMismatch is returned when a written traversal realizes the written header size does not // match the actual number of car bytes written. -const ErrSizeMismatch = "car-error-sizemismatch" +var ErrSizeMismatch = fmt.Errorf("car-error-sizemismatch") + +// ErrOffsetImpossible is returned when specified paddings or offsets of either a wrapped carv1 +// or index cannot be satisfied based on the data being written. +var ErrOffsetImpossible = fmt.Errorf("car-error-offsetimpossible") // MaxTraversalLinks changes the allowed number of links a selector traversal // can execute before failing. @@ -37,7 +41,7 @@ func MaxTraversalLinks(MaxTraversalLinks uint64) Option { // PrepareTraversal walks through the proposed dag traversal to learn it's total size in order to be able to // stream out a car to a writer in the expected traversal order in one go. func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, opts ...Option) (Writer, error) { - cls, cntr := countingLinkSystem(*ls) + cls, cntr := loader.CountingLinkSystem(*ls) c1h := carv1.CarHeader{Roots: []cid.Cid{root}, Version: 1} headSize, err := carv1.HeaderSize(&c1h) @@ -48,7 +52,7 @@ func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, se return nil, err } tc := traversalCar{ - size: headSize + cntr.totalRead, + size: headSize + cntr.Size(), ctx: ctx, root: root, selector: selector, @@ -58,7 +62,8 @@ func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, se return &tc, nil } -// FileTraversal writes a carv2 matching a given root and selector to a file path. +// FileTraversal writes a car file matching a given root and selector to the +// path at `destination` using one read of each block. func FileTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, destination string, opts ...Option) error { tc := traversalCar{ size: 0, @@ -86,13 +91,29 @@ func FileTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selec } tc.size = uint64(tc.size) - if _, err = tc.WriteHeader(fp); err != nil { + if _, err = tc.WriteV2Header(fp); err != nil { return err } return nil } +// V1Traversal walks through the proposed dag traversal and writes a carv1 to the provided io.Writer +func V1Traversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, writer io.Writer, opts ...Option) (uint64, error) { + opts = append(opts, WithoutIndex()) + tc := traversalCar{ + size: 0, + ctx: ctx, + root: root, + selector: selector, + ls: ls, + opts: opts, + } + + len, _, err := tc.WriteV1(writer) + return len, err +} + // Writer is an interface allowing writing a car with the data specified by a selector. type Writer interface { io.WriterTo @@ -108,95 +129,104 @@ type traversalCar struct { } func (tc *traversalCar) WriteTo(w io.Writer) (int64, error) { - n, err := w.Write(Pragma) + n, err := tc.WriteV2Header(w) if err != nil { - return int64(n), err + return n, err } - h, err := tc.WriteHeader(w) + v1s, idx, err := tc.WriteV1(w) + n += int64(v1s) + if err != nil { - return int64(n) + h, err + return n, err } - h += int64(n) - v1s, err := tc.WriteV1(w) - h += int64(v1s) - return h, err + // index padding, then index + opts := ApplyOptions(tc.opts...) + if opts.IndexCodec != index.CarIndexNoIndex { + if opts.IndexPadding > 0 { + buf := make([]byte, opts.IndexPadding) + pn, err := w.Write(buf) + n += int64(pn) + if err != nil { + return n, err + } + } + in, err := index.WriteTo(idx, w) + n += int64(in) + if err != nil { + return n, err + } + } + + return n, err } -func (tc *traversalCar) WriteHeader(w io.Writer) (int64, error) { +func (tc *traversalCar) WriteV2Header(w io.Writer) (int64, error) { + n, err := w.Write(Pragma) + if err != nil { + return int64(n), err + } + h := NewHeader(tc.size) + conf := ApplyOptions(tc.opts...) + if p := conf.DataPadding; p > 0 { + h = h.WithDataPadding(p) + } + if p := conf.IndexPadding; p > 0 { + h = h.WithIndexPadding(p) + } // TODO: support calculation / inclusion of the index. h.IndexOffset = 0 - return h.WriteTo(w) + hn, err := h.WriteTo(w) + if err != nil { + return int64(n) + hn, err + } + hn += int64(n) + + // We include the initial data padding after the carv2 header + if h.DataOffset > uint64(hn) { + buf := make([]byte, h.DataOffset-uint64(hn)) + n, err = w.Write(buf) + hn += int64(n) + if err != nil { + return hn, err + } + } else if h.DataOffset < uint64(hn) { + return hn, ErrOffsetImpossible + } + + return hn, nil } -func (tc *traversalCar) WriteV1(w io.Writer) (uint64, error) { +func (tc *traversalCar) WriteV1(w io.Writer) (uint64, index.Index, error) { // write the v1 header c1h := carv1.CarHeader{Roots: []cid.Cid{tc.root}, Version: 1} if err := carv1.WriteHeader(&c1h, w); err != nil { - return 0, err + return 0, nil, err } v1Size, err := carv1.HeaderSize(&c1h) if err != nil { - return v1Size, err + return v1Size, nil, err } // write the block. opts := ApplyOptions(tc.opts...) - wls, writer := teeingLinkSystem(*tc.ls, w) + wls, writer := loader.TeeingLinkSystem(*tc.ls, w, v1Size, opts.IndexCodec) err = traverse(tc.ctx, &wls, tc.root, tc.selector, opts) - v1Size += writer.size + v1Size = writer.Size() if err != nil { - return v1Size, err + return v1Size, nil, err } if tc.size != 0 && tc.size != v1Size { - return v1Size, fmt.Errorf(ErrSizeMismatch) + return v1Size, nil, ErrSizeMismatch } tc.size = v1Size - return v1Size, nil -} - -type counter struct { - totalRead uint64 -} - -type countingReader struct { - r io.Reader - c *counter - read uint64 - cid string -} -func (c *countingReader) Read(p []byte) (int, error) { - n, err := c.r.Read(p) - if err == io.EOF { - // add in the overall length of the block. - n += len(c.cid) - uv := varint.ToUvarint(uint64(n)) - n += len(uv) + if opts.IndexCodec == index.CarIndexNoIndex { + return v1Size, nil, nil } - c.c.totalRead += uint64(n) - c.read += uint64(n) - return n, err -} - -func countingLinkSystem(ls ipld.LinkSystem) (ipld.LinkSystem, *counter) { - c := counter{} - return linking.LinkSystem{ - EncoderChooser: ls.EncoderChooser, - DecoderChooser: ls.DecoderChooser, - HasherChooser: ls.HasherChooser, - StorageWriteOpener: ls.StorageWriteOpener, - StorageReadOpener: func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { - r, err := ls.StorageReadOpener(lc, l) - if err != nil { - return nil, err - } - return &countingReader{r, &c, 0, l.Binary()}, nil - }, - TrustedStorage: ls.TrustedStorage, - NodeReifier: ls.NodeReifier, - }, &c + idx, err := writer.Index() + return v1Size, idx, err } func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Node, opts Options) error { @@ -231,59 +261,3 @@ func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Nod return nil }) } - -type writerOutput struct { - w io.Writer - size uint64 -} - -type writingReader struct { - r io.Reader - len int64 - cid string - wo *writerOutput -} - -func (w *writingReader) Read(p []byte) (int, error) { - if w.wo != nil { - // write the cid - size := varint.ToUvarint(uint64(w.len)) - if _, err := w.wo.w.Write(size); err != nil { - return 0, err - } - if _, err := w.wo.w.Write([]byte(w.cid)); err != nil { - return 0, err - } - cpy := bytes.NewBuffer(w.r.(*bytes.Buffer).Bytes()) - if _, err := cpy.WriteTo(w.wo.w); err != nil { - return 0, err - } - w.wo = nil - } - - return w.r.Read(p) -} - -func teeingLinkSystem(ls ipld.LinkSystem, w io.Writer) (ipld.LinkSystem, *writerOutput) { - wo := writerOutput{w, 0} - return linking.LinkSystem{ - EncoderChooser: ls.EncoderChooser, - DecoderChooser: ls.DecoderChooser, - HasherChooser: ls.HasherChooser, - StorageWriteOpener: ls.StorageWriteOpener, - StorageReadOpener: func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { - r, err := ls.StorageReadOpener(lc, l) - if err != nil { - return nil, err - } - buf := bytes.NewBuffer(nil) - n, err := buf.ReadFrom(r) - if err != nil { - return nil, err - } - return &writingReader{buf, n, l.Binary(), &wo}, nil - }, - TrustedStorage: ls.TrustedStorage, - NodeReifier: ls.NodeReifier, - }, &wo -} diff --git a/v2/writer.go b/v2/writer.go index 72675cb2..e82c4b3b 100644 --- a/v2/writer.go +++ b/v2/writer.go @@ -84,7 +84,7 @@ func WrapV1(src io.ReadSeeker, dst io.Writer, opts ...Option) error { if _, err := io.Copy(dst, src); err != nil { return err } - if err := index.WriteTo(idx, dst); err != nil { + if _, err := index.WriteTo(idx, dst); err != nil { return err } @@ -211,7 +211,8 @@ func AttachIndex(path string, idx index.Index, offset uint64) error { } defer out.Close() indexWriter := internalio.NewOffsetWriter(out, int64(offset)) - return index.WriteTo(idx, indexWriter) + _, err = index.WriteTo(idx, indexWriter) + return err } // ReplaceRootsInFile replaces the root CIDs in CAR file at given path with the given roots. From 6d41ba2f820481bb9d6eff17e9afa3e28f4d77be Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 25 Nov 2021 11:34:01 -0800 Subject: [PATCH 05/10] [fixup] options test --- v2/options_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/v2/options_test.go b/v2/options_test.go index 307568a4..7e060acf 100644 --- a/v2/options_test.go +++ b/v2/options_test.go @@ -1,6 +1,7 @@ package car_test import ( + "math" "testing" carv2 "github.com/ipld/go-car/v2" @@ -11,8 +12,9 @@ import ( func TestApplyOptions_SetsExpectedDefaults(t *testing.T) { require.Equal(t, carv2.Options{ - IndexCodec: multicodec.CarMultihashIndexSorted, - MaxIndexCidSize: carv2.DefaultMaxIndexCidSize, + IndexCodec: multicodec.CarMultihashIndexSorted, + MaxIndexCidSize: carv2.DefaultMaxIndexCidSize, + MaxTraversalLinks: math.MaxInt64, }, carv2.ApplyOptions()) } @@ -27,6 +29,7 @@ func TestApplyOptions_AppliesOptions(t *testing.T) { StoreIdentityCIDs: true, BlockstoreAllowDuplicatePuts: true, BlockstoreUseWholeCIDs: true, + MaxTraversalLinks: math.MaxInt64, }, carv2.ApplyOptions( carv2.UseDataPadding(123), From ac5d450b91f36f51e39b1eee8252de55ae4e008a Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 26 Nov 2021 18:01:07 -0800 Subject: [PATCH 06/10] Add round-trip test --- v2/go.mod | 5 +- v2/go.sum | 74 ++++++++++++++++++++------ v2/internal/loader/counting_loader.go | 23 ++++---- v2/selective.go | 9 +++- v2/selective_test.go | 64 ++++++++++++++++++++++ v2/testdata/sample-unixfs-v2.car | Bin 0 -> 485 bytes 6 files changed, 142 insertions(+), 33 deletions(-) create mode 100644 v2/selective_test.go create mode 100644 v2/testdata/sample-unixfs-v2.car diff --git a/v2/go.mod b/v2/go.mod index 6c346160..ac512109 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -4,13 +4,14 @@ go 1.16 require ( github.com/ipfs/go-block-format v0.0.3 - github.com/ipfs/go-cid v0.0.8-0.20210716091050-de6c03deae1c + github.com/ipfs/go-cid v0.1.0 github.com/ipfs/go-ipfs-blockstore v1.0.3 github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-merkledag v0.3.2 + github.com/ipld/go-codec-dagpb v1.3.0 github.com/ipld/go-ipld-prime v0.14.0 - github.com/mattn/go-colorable v0.1.8 // indirect + github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211119015904-892d2f92ba8b github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 github.com/multiformats/go-multihash v0.1.0 github.com/multiformats/go-varint v0.0.6 diff --git a/v2/go.sum b/v2/go.sum index 75414c69..cea8376a 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -77,6 +78,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -96,8 +98,9 @@ github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFG github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -106,6 +109,7 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -113,14 +117,16 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= @@ -179,20 +185,23 @@ github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.0.8-0.20210716091050-de6c03deae1c h1:uVMZWk8sJN1l/47TtXRXDz0M9/6v0yw7neDn5WfeLwg= -github.com/ipfs/go-cid v0.0.8-0.20210716091050-de6c03deae1c/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= +github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0= +github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= -github.com/ipfs/go-datastore v0.4.2 h1:h8/n7WPzhp239kkLws+epN3Ic7YtcBPgcaXfEfdVDWM= github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg= +github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= +github.com/ipfs/go-ipfs-blockstore v0.1.6/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blockstore v1.0.3 h1:RDhK6fdg5YsonkpMuMpdvk/pRtOQlrIRIybuQfkvB2M= github.com/ipfs/go-ipfs-blockstore v1.0.3/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= @@ -201,6 +210,7 @@ github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1Y github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= +github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs= github.com/ipfs/go-ipfs-ds-help v1.0.0 h1:bEQ8hMGs80h0sR8O4tfDgV6B01aaF9qeTrujrTLYV3g= github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE= github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6rY30GUu7G6LUfpMvM= @@ -221,8 +231,11 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= -github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= +github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.1.3 h1:1iS3IU7aXRlbgUpN8yTTpJ53NXYjAe37vcI5+5nYrzk= +github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-merkledag v0.3.2 h1:MRqj40QkrWkvPswXs4EfSslhZ4RVPRbxwX11js0t1xY= github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= @@ -231,8 +244,13 @@ github.com/ipfs/go-peertaskqueue v0.1.0 h1:bpRbgv76eT4avutNPDFZuCPOQus6qTgurEYxf github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= +github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8= +github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= +github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0 h1:2FnBqUjmmgxgZD6/zB3eygWlmIsHNGrZ57L99x3xD6Q= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= +github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211119015904-892d2f92ba8b h1:uxXI0WOuhYE9Yhs/IC8lRSKuYmo8qJytr4di1RxgttA= +github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211119015904-892d2f92ba8b/go.mod h1:UIhTqo70axxJ2GXIIiLmk0+uNtZqJInbiw0DGzCgtbk= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -241,8 +259,9 @@ github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4 github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= -github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -256,9 +275,11 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -266,6 +287,8 @@ github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muM github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -330,14 +353,10 @@ github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0Q github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -420,8 +439,9 @@ github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWEr github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= @@ -436,6 +456,7 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -514,7 +535,6 @@ github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/S github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2USAXB5oMDwRl3piOux2XMV9TANaYxXHdkoE= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= -github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f h1:M/lL30eFZTKnomXY6huvM6G0+gVquFNf6mxghaWlFUg= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= @@ -523,6 +543,7 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -543,9 +564,12 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -562,6 +586,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -575,13 +600,17 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -602,6 +631,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -634,12 +664,11 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -668,6 +697,10 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -693,11 +726,15 @@ google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -708,14 +745,17 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= diff --git a/v2/internal/loader/counting_loader.go b/v2/internal/loader/counting_loader.go index 84086b1d..ee7f1e2e 100644 --- a/v2/internal/loader/counting_loader.go +++ b/v2/internal/loader/counting_loader.go @@ -1,6 +1,7 @@ package loader import ( + "bytes" "io" "github.com/ipld/go-ipld-prime" @@ -24,22 +25,13 @@ type ReadCounter interface { } type countingReader struct { - r io.Reader - c *counter - read uint64 - cid string + r io.Reader + c *counter } func (c *countingReader) Read(p []byte) (int, error) { n, err := c.r.Read(p) - if err == io.EOF { - // add in the overall length of the block. - n += len(c.cid) - uv := varint.ToUvarint(uint64(n)) - n += len(uv) - } c.c.totalRead += uint64(n) - c.read += uint64(n) return n, err } @@ -60,7 +52,14 @@ func CountingLinkSystem(ls ipld.LinkSystem) (ipld.LinkSystem, ReadCounter) { if err != nil { return nil, err } - return &countingReader{r, &c, 0, l.Binary()}, nil + buf := bytes.NewBuffer(nil) + n, err := buf.ReadFrom(r) + if err != nil { + return nil, err + } + size := varint.ToUvarint(uint64(n) + uint64(len(l.Binary()))) + c.totalRead += uint64(len(size)) + uint64(len(l.Binary())) + return &countingReader{buf, &c}, nil }, TrustedStorage: ls.TrustedStorage, NodeReifier: ls.NodeReifier, diff --git a/v2/selective.go b/v2/selective.go index b736cdae..07bcc1b3 100644 --- a/v2/selective.go +++ b/v2/selective.go @@ -253,11 +253,16 @@ func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Nod } lnk := cidlink.Link{Cid: root} + ls.TrustedStorage = true rootNode, err := ls.Load(ipld.LinkContext{}, lnk, basicnode.Prototype.Any) if err != nil { - return err + return fmt.Errorf("root blk load failed: %s", err) } - return progress.WalkMatching(rootNode, sel, func(_ traversal.Progress, _ ipld.Node) error { + err = progress.WalkMatching(rootNode, sel, func(_ traversal.Progress, _ ipld.Node) error { return nil }) + if err != nil { + return fmt.Errorf("walk failed: %s", err) + } + return nil } diff --git a/v2/selective_test.go b/v2/selective_test.go new file mode 100644 index 00000000..908bb29e --- /dev/null +++ b/v2/selective_test.go @@ -0,0 +1,64 @@ +package car_test + +import ( + "bytes" + "context" + "io/ioutil" + "os" + "path" + "testing" + + "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/blockstore" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/storage/bsadapter" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" + "github.com/stretchr/testify/require" + + _ "github.com/ipld/go-codec-dagpb" + _ "github.com/ipld/go-ipld-prime/codec/dagcbor" + _ "github.com/ipld/go-ipld-prime/codec/raw" +) + +func TestPrepareTraversal(t *testing.T) { + from, err := blockstore.OpenReadOnly("testdata/sample-unixfs-v2.car") + require.NoError(t, err) + ls := cidlink.DefaultLinkSystem() + bsa := bsadapter.Adapter{Wrapped: from} + ls.SetReadStorage(&bsa) + + rts, _ := from.Roots() + writer, err := car.PrepareTraversal(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively) + require.NoError(t, err) + + buf := bytes.Buffer{} + n, err := writer.WriteTo(&buf) + require.NoError(t, err) + require.Equal(t, int64(len(buf.Bytes())), n) + + fi, _ := os.Stat("testdata/sample-unixfs-v2.car") + require.Equal(t, fi.Size(), n) +} + +func TestFileTraversal(t *testing.T) { + from, err := blockstore.OpenReadOnly("testdata/sample-unixfs-v2.car") + require.NoError(t, err) + ls := cidlink.DefaultLinkSystem() + bsa := bsadapter.Adapter{Wrapped: from} + ls.SetReadStorage(&bsa) + + rts, _ := from.Roots() + outDir, err := ioutil.TempDir("", "car-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(outDir) + err = car.FileTraversal(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively, path.Join(outDir, "out.car")) + require.NoError(t, err) + + require.FileExists(t, path.Join(outDir, "out.car")) + + fa, _ := os.Stat("testdata/sample-unixfs-v2.car") + fb, _ := os.Stat(path.Join(outDir, "out.car")) + require.Equal(t, fa.Size(), fb.Size()) +} diff --git a/v2/testdata/sample-unixfs-v2.car b/v2/testdata/sample-unixfs-v2.car new file mode 100644 index 0000000000000000000000000000000000000000..0d8d8fd6451f778cc3ed8d936714ebe286e659eb GIT binary patch literal 485 zcmd;Dm|m7zRGgWg$HagJjG=rPMhL?nN?R>TEy~X?DQ>)>6`{(&SRkapW#UnI@7eJX zh5E>xKU3FfURkh6u0VNGRBWQELhhASTVPf&8Zd?mDXjFE;IGf`KD)nhdZxF5gg39= z#e-+H7jKM;zNqQv}B@#SbOdO1%#5i0?8)z~yE?`WQU;vrg$N>r@ gA!x{f*cw1w0p$}4r(;lcghI#~s*X^= Date: Sat, 27 Nov 2021 07:22:03 -0800 Subject: [PATCH 07/10] cleaner loaders --- v2/internal/loader/counting_loader.go | 38 +++++++++++---------------- v2/internal/loader/writing_loader.go | 35 +++++++++++------------- 2 files changed, 31 insertions(+), 42 deletions(-) diff --git a/v2/internal/loader/counting_loader.go b/v2/internal/loader/counting_loader.go index ee7f1e2e..e428993e 100644 --- a/v2/internal/loader/counting_loader.go +++ b/v2/internal/loader/counting_loader.go @@ -42,26 +42,20 @@ func (c *countingReader) Read(p []byte) (int, error) { // CID and the varint length for the block data). func CountingLinkSystem(ls ipld.LinkSystem) (ipld.LinkSystem, ReadCounter) { c := counter{} - return linking.LinkSystem{ - EncoderChooser: ls.EncoderChooser, - DecoderChooser: ls.DecoderChooser, - HasherChooser: ls.HasherChooser, - StorageWriteOpener: ls.StorageWriteOpener, - StorageReadOpener: func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { - r, err := ls.StorageReadOpener(lc, l) - if err != nil { - return nil, err - } - buf := bytes.NewBuffer(nil) - n, err := buf.ReadFrom(r) - if err != nil { - return nil, err - } - size := varint.ToUvarint(uint64(n) + uint64(len(l.Binary()))) - c.totalRead += uint64(len(size)) + uint64(len(l.Binary())) - return &countingReader{buf, &c}, nil - }, - TrustedStorage: ls.TrustedStorage, - NodeReifier: ls.NodeReifier, - }, &c + clc := ls + clc.StorageReadOpener = func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { + r, err := ls.StorageReadOpener(lc, l) + if err != nil { + return nil, err + } + buf := bytes.NewBuffer(nil) + n, err := buf.ReadFrom(r) + if err != nil { + return nil, err + } + size := varint.ToUvarint(uint64(n) + uint64(len(l.Binary()))) + c.totalRead += uint64(len(size)) + uint64(len(l.Binary())) + return &countingReader{buf, &c}, nil + } + return clc, &c } diff --git a/v2/internal/loader/writing_loader.go b/v2/internal/loader/writing_loader.go index 6b43ae90..2a73fa71 100644 --- a/v2/internal/loader/writing_loader.go +++ b/v2/internal/loader/writing_loader.go @@ -96,24 +96,19 @@ func TeeingLinkSystem(ls ipld.LinkSystem, w io.Writer, initialOffset uint64, ind code: indexCodec, rcrds: make([]index.Record, 0), } - return linking.LinkSystem{ - EncoderChooser: ls.EncoderChooser, - DecoderChooser: ls.DecoderChooser, - HasherChooser: ls.HasherChooser, - StorageWriteOpener: ls.StorageWriteOpener, - StorageReadOpener: func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { - r, err := ls.StorageReadOpener(lc, l) - if err != nil { - return nil, err - } - buf := bytes.NewBuffer(nil) - n, err := buf.ReadFrom(r) - if err != nil { - return nil, err - } - return &writingReader{buf, n, l.Binary(), &wo}, nil - }, - TrustedStorage: ls.TrustedStorage, - NodeReifier: ls.NodeReifier, - }, &wo + + tls := ls + tls.StorageReadOpener = func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) { + r, err := ls.StorageReadOpener(lc, l) + if err != nil { + return nil, err + } + buf := bytes.NewBuffer(nil) + n, err := buf.ReadFrom(r) + if err != nil { + return nil, err + } + return &writingReader{buf, n, l.Binary(), &wo}, nil + } + return tls, &wo } From 4076b7a281d6fff17704c4f5eae28018e5a2e84f Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 29 Nov 2021 11:45:06 -0800 Subject: [PATCH 08/10] renames / comments per review --- v2/index/index.go | 4 +-- v2/internal/loader/writing_loader.go | 2 +- v2/options.go | 2 +- v2/selective.go | 49 ++++++++++++++-------------- v2/selective_test.go | 12 +++---- 5 files changed, 33 insertions(+), 36 deletions(-) diff --git a/v2/index/index.go b/v2/index/index.go index c8eccefc..10195b43 100644 --- a/v2/index/index.go +++ b/v2/index/index.go @@ -15,8 +15,8 @@ import ( "github.com/ipfs/go-cid" ) -// CarIndexNoIndex is a sentinal value used as a multicodec code for the index indicating no index. -const CarIndexNoIndex = 0x300000 +// CarIndexNone is a sentinal value used as a multicodec code for the index indicating no index. +const CarIndexNone = 0x300000 type ( // Record is a pre-processed record of a car item and location. diff --git a/v2/internal/loader/writing_loader.go b/v2/internal/loader/writing_loader.go index 2a73fa71..13236f1c 100644 --- a/v2/internal/loader/writing_loader.go +++ b/v2/internal/loader/writing_loader.go @@ -65,7 +65,7 @@ func (w *writingReader) Read(p []byte) (int, error) { } // maybe write the index. - if w.wo.code != index.CarIndexNoIndex { + if w.wo.code != index.CarIndexNone { _, c, err := cid.CidFromBytes([]byte(w.cid)) if err != nil { return 0, err diff --git a/v2/options.go b/v2/options.go index 61f6d542..2fb1d1fc 100644 --- a/v2/options.go +++ b/v2/options.go @@ -94,7 +94,7 @@ func UseIndexCodec(c multicodec.Code) Option { // WithoutIndex flags that no index should be included in generation. func WithoutIndex() Option { return func(o *Options) { - o.IndexCodec = index.CarIndexNoIndex + o.IndexCodec = index.CarIndexNone } } diff --git a/v2/selective.go b/v2/selective.go index 07bcc1b3..22351e71 100644 --- a/v2/selective.go +++ b/v2/selective.go @@ -38,9 +38,9 @@ func MaxTraversalLinks(MaxTraversalLinks uint64) Option { } } -// PrepareTraversal walks through the proposed dag traversal to learn it's total size in order to be able to +// NewSelectiveWriter walks through the proposed dag traversal to learn its total size in order to be able to // stream out a car to a writer in the expected traversal order in one go. -func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, opts ...Option) (Writer, error) { +func NewSelectiveWriter(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, opts ...Option) (Writer, error) { cls, cntr := loader.CountingLinkSystem(*ls) c1h := carv1.CarHeader{Roots: []cid.Cid{root}, Version: 1} @@ -57,21 +57,21 @@ func PrepareTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, se root: root, selector: selector, ls: ls, - opts: opts, + opts: ApplyOptions(opts...), } return &tc, nil } -// FileTraversal writes a car file matching a given root and selector to the +// TraverseToFile writes a car file matching a given root and selector to the // path at `destination` using one read of each block. -func FileTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, destination string, opts ...Option) error { +func TraverseToFile(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, destination string, opts ...Option) error { tc := traversalCar{ size: 0, ctx: ctx, root: root, selector: selector, ls: ls, - opts: opts, + opts: ApplyOptions(opts...), } fp, err := os.Create(destination) @@ -98,8 +98,8 @@ func FileTraversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selec return nil } -// V1Traversal walks through the proposed dag traversal and writes a carv1 to the provided io.Writer -func V1Traversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, writer io.Writer, opts ...Option) (uint64, error) { +// TraverseV1 walks through the proposed dag traversal and writes a carv1 to the provided io.Writer +func TraverseV1(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, writer io.Writer, opts ...Option) (uint64, error) { opts = append(opts, WithoutIndex()) tc := traversalCar{ size: 0, @@ -107,25 +107,27 @@ func V1Traversal(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selecto root: root, selector: selector, ls: ls, - opts: opts, + opts: ApplyOptions(opts...), } len, _, err := tc.WriteV1(writer) return len, err } -// Writer is an interface allowing writing a car with the data specified by a selector. +// Writer is an interface allowing writing a car prepared by PrepareTraversal type Writer interface { io.WriterTo } +var _ Writer = (*traversalCar)(nil) + type traversalCar struct { size uint64 ctx context.Context root cid.Cid selector ipld.Node ls *ipld.LinkSystem - opts []Option + opts Options } func (tc *traversalCar) WriteTo(w io.Writer) (int64, error) { @@ -141,10 +143,9 @@ func (tc *traversalCar) WriteTo(w io.Writer) (int64, error) { } // index padding, then index - opts := ApplyOptions(tc.opts...) - if opts.IndexCodec != index.CarIndexNoIndex { - if opts.IndexPadding > 0 { - buf := make([]byte, opts.IndexPadding) + if tc.opts.IndexCodec != index.CarIndexNone { + if tc.opts.IndexPadding > 0 { + buf := make([]byte, tc.opts.IndexPadding) pn, err := w.Write(buf) n += int64(pn) if err != nil { @@ -168,15 +169,15 @@ func (tc *traversalCar) WriteV2Header(w io.Writer) (int64, error) { } h := NewHeader(tc.size) - conf := ApplyOptions(tc.opts...) - if p := conf.DataPadding; p > 0 { + if p := tc.opts.DataPadding; p > 0 { h = h.WithDataPadding(p) } - if p := conf.IndexPadding; p > 0 { + if p := tc.opts.IndexPadding; p > 0 { h = h.WithIndexPadding(p) } - // TODO: support calculation / inclusion of the index. - h.IndexOffset = 0 + if tc.opts.IndexCodec == index.CarIndexNone { + h.IndexOffset = 0 + } hn, err := h.WriteTo(w) if err != nil { return int64(n) + hn, err @@ -185,6 +186,7 @@ func (tc *traversalCar) WriteV2Header(w io.Writer) (int64, error) { // We include the initial data padding after the carv2 header if h.DataOffset > uint64(hn) { + // TODO: buffer writes if this needs to be big. buf := make([]byte, h.DataOffset-uint64(hn)) n, err = w.Write(buf) hn += int64(n) @@ -210,9 +212,8 @@ func (tc *traversalCar) WriteV1(w io.Writer) (uint64, index.Index, error) { } // write the block. - opts := ApplyOptions(tc.opts...) - wls, writer := loader.TeeingLinkSystem(*tc.ls, w, v1Size, opts.IndexCodec) - err = traverse(tc.ctx, &wls, tc.root, tc.selector, opts) + wls, writer := loader.TeeingLinkSystem(*tc.ls, w, v1Size, tc.opts.IndexCodec) + err = traverse(tc.ctx, &wls, tc.root, tc.selector, tc.opts) v1Size = writer.Size() if err != nil { return v1Size, nil, err @@ -222,7 +223,7 @@ func (tc *traversalCar) WriteV1(w io.Writer) (uint64, index.Index, error) { } tc.size = v1Size - if opts.IndexCodec == index.CarIndexNoIndex { + if tc.opts.IndexCodec == index.CarIndexNone { return v1Size, nil, nil } idx, err := writer.Index() diff --git a/v2/selective_test.go b/v2/selective_test.go index 908bb29e..82a01dcd 100644 --- a/v2/selective_test.go +++ b/v2/selective_test.go @@ -3,9 +3,9 @@ package car_test import ( "bytes" "context" - "io/ioutil" "os" "path" + "path/filepath" "testing" "github.com/ipld/go-car/v2" @@ -28,7 +28,7 @@ func TestPrepareTraversal(t *testing.T) { ls.SetReadStorage(&bsa) rts, _ := from.Roots() - writer, err := car.PrepareTraversal(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively) + writer, err := car.NewSelectiveWriter(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) buf := bytes.Buffer{} @@ -48,12 +48,8 @@ func TestFileTraversal(t *testing.T) { ls.SetReadStorage(&bsa) rts, _ := from.Roots() - outDir, err := ioutil.TempDir("", "car-*") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(outDir) - err = car.FileTraversal(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively, path.Join(outDir, "out.car")) + outDir := filepath.Join(t.TempDir(), "car-file-traversal.car") + err = car.TraverseToFile(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively, path.Join(outDir, "out.car")) require.NoError(t, err) require.FileExists(t, path.Join(outDir, "out.car")) From 21907ee6128aab4ad04255263f47eb2f71e98e8d Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 29 Nov 2021 11:57:17 -0800 Subject: [PATCH 09/10] fix test --- v2/selective_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/v2/selective_test.go b/v2/selective_test.go index 82a01dcd..75600166 100644 --- a/v2/selective_test.go +++ b/v2/selective_test.go @@ -5,7 +5,6 @@ import ( "context" "os" "path" - "path/filepath" "testing" "github.com/ipld/go-car/v2" @@ -48,7 +47,7 @@ func TestFileTraversal(t *testing.T) { ls.SetReadStorage(&bsa) rts, _ := from.Roots() - outDir := filepath.Join(t.TempDir(), "car-file-traversal.car") + outDir := t.TempDir() err = car.TraverseToFile(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively, path.Join(outDir, "out.car")) require.NoError(t, err) From 0b392bc59b3c88f32b6a57a9f9aed28a33f58859 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 29 Nov 2021 15:19:34 -0800 Subject: [PATCH 10/10] additional testing as requested --- v2/selective_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/v2/selective_test.go b/v2/selective_test.go index 75600166..f2bba6f8 100644 --- a/v2/selective_test.go +++ b/v2/selective_test.go @@ -37,6 +37,13 @@ func TestPrepareTraversal(t *testing.T) { fi, _ := os.Stat("testdata/sample-unixfs-v2.car") require.Equal(t, fi.Size(), n) + + // Headers should be equal + h1, _ := car.OpenReader("testdata/sample-unixfs-v2.car") + h1h := bytes.Buffer{} + h1h.Write(car.Pragma) + h1.Header.WriteTo(&h1h) + require.Equal(t, buf.Bytes()[:h1h.Len()], h1h.Bytes()) } func TestFileTraversal(t *testing.T) { @@ -57,3 +64,20 @@ func TestFileTraversal(t *testing.T) { fb, _ := os.Stat(path.Join(outDir, "out.car")) require.Equal(t, fa.Size(), fb.Size()) } + +func TestV1Traversal(t *testing.T) { + from, err := blockstore.OpenReadOnly("testdata/sample-v1.car") + require.NoError(t, err) + ls := cidlink.DefaultLinkSystem() + bsa := bsadapter.Adapter{Wrapped: from} + ls.SetReadStorage(&bsa) + + rts, _ := from.Roots() + w := bytes.NewBuffer(nil) + n, err := car.TraverseV1(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively, w) + require.NoError(t, err) + require.Equal(t, int64(len(w.Bytes())), int64(n)) + + fa, _ := os.Stat("testdata/sample-v1.car") + require.Equal(t, fa.Size(), int64(n)) +}