From 1a8799065da2d3a97475602b95d917be445d0eb4 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Thu, 13 Apr 2023 06:07:10 -0700 Subject: [PATCH] merge: main to lid (#1370) * add free check (#1315) * chore: bump version to 1.6.1 (#1317) * fix legacy deal verified status (#1324) * fix: update go-unixfsnode enough to make sure unixfs-preload is available (#1323) * release v1.6.2-rc1 (#1328) * use full path (#1330) * fix bug (#1332) * use forks of graphsync, go-data-transfer and go-fil-markets (#1333) * refactor: use forks of graphsync, go-data-transfer and go-fil-markets * refactor: convert from data transfer v1 to v2 voucher type * fix: index provider validation voucher type * fix: pass index provider engine link system through to graphsync's transport configurer * feat: use tagged version of boost-gfm * fix: retrieval client imports * feat: tagged version of lotus * feat: require go 1.19 * lint: fix lint errors * fix: itests * fix: cbor-gen, docsgen * fix: update CI lint version * fix: lint * fix: docgen * fix: go mod tidy * fix: protocol proxy TestOutboundForwarding * fix: docsgen * fix: update filecoin-ffi submodule * fix: prometheus duplicate register panic * fix: cleanup imports * fix: legs voucher processing * chore: release v1.6.2-rc2 (#1340) * release v1.6.2-rc2 * fix test * fix: flaky TestLibp2pCarServerNewTransferCancelsPreviousTransfer (#1350) * fix: flaky TestDealCompletionOnProcessResumption (#1351) * fix: occasional panic on shutdown (#1353) * feat: query UI (#1352) * log insert * fix display error * refactor code * shorten status strings * remove comment * apply suggestion * feat: add download block link to inspect page (#1312) * fix(devnet): update golang and lotus default versions (#1354) * fix(devnet): bump golang to 1.19 * chore(devnet): bump lotus default version * chore(devnet): remove unused stable env * booster-http: implement IPFS HTTP gateway (#1225) * feat: implement http api gateway * feat: use go-libipfs lib (instead of copying to extern) * feat: bump booster-bitswap info minor version * feat: http gateway metrics * fix: TestHttpInfo * feat: by default only serve blocks and CARs, with option to serve original files (jpg, mov etc) * fix: correct link for download root block (#1355) * feat: option to cleanup data for offline deals after add piece (#1341) * chore: add support for multiple node.js versions in makefile (#1356) * chore: release v.1.7.0-rc1 (#1357) * release v.1.7.0-rc1 * fix version * fix: dagstore initialize-all parameter (#1363) * fix: show verifying commp state for offline deals (#1364) * fix: boost run missing staging-area dir (#1368) * merge(wip): main to lid TODO: remoteblockstore needs to handle nil metrics * fix: flaky TestNewHttpServer (#1372) * feat: group agent version by binary name (#1369) * fix: wrap stats in nil checks for now we should probably revisit how stats are handled now that we have all 3 transports being tracked * test(fix): incorrect test urls --------- Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Co-authored-by: Rod Vagg Co-authored-by: dirkmc --- Makefile | 4 +- api/api.go | 30 +- build/openrpc/boost.json.gz | Bin 5351 -> 5368 bytes build/version.go | 2 +- cmd/boostd/import_data.go | 3 + cmd/booster-bitswap/run.go | 20 +- cmd/booster-http/blocks.go | 237 +++++++ cmd/booster-http/gateway_handler.go | 10 +- cmd/booster-http/http_test.go | 29 +- cmd/booster-http/mocks/mock_booster_http.go | 21 +- cmd/booster-http/run.go | 51 +- cmd/booster-http/server.go | 55 +- cmd/boostx/stats_cmd.go | 2 + cmd/lib/filters/filteredblockstore.go | 4 +- cmd/lib/filters/filteredblockstore_test.go | 2 +- .../deals_announce_to_ipni_test.go | 8 +- .../dashboards/provider-retrievals.json | 655 +++++++++++++++++- go.mod | 40 +- go.sum | 70 +- gql/serve_download.go | 2 +- gql/server.go | 20 +- metrics/metrics.go | 22 +- node/builder.go | 6 +- node/impl/boost.go | 4 +- node/modules/graphsync.go | 12 +- node/modules/legacy_markets.go | 2 + node/modules/storageminer.go | 6 +- react/src/Inspect.css | 8 + react/src/LID.js | 14 +- storagemarket/provider_test.go | 14 - 30 files changed, 1134 insertions(+), 219 deletions(-) create mode 100644 cmd/booster-http/blocks.go diff --git a/Makefile b/Makefile index 92508770a..606e28345 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,7 @@ update-react: validate-node-version npm run --prefix react build .PHONY: react -build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid +build-go: boost devnet .PHONY: build-go build: react build-go @@ -217,7 +217,7 @@ docsgen-openrpc-boost: docsgen-openrpc-bin ## DOCKER IMAGES docker_user?=filecoin -lotus_version?=v1.23.2 +lotus_version?=v1.21.0-rc2 ffi_from_source?=0 build_lotus?=0 ifeq ($(build_lotus),1) diff --git a/api/api.go b/api/api.go index 7086848af..5877f52be 100644 --- a/api/api.go +++ b/api/api.go @@ -33,21 +33,21 @@ type Boost interface { Net // MethodGroup: Boost - BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin - BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error) //perm:admin - BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string) (*ProviderDealRejectionInfo, error) //perm:admin - BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error) //perm:admin - BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error) //perm:admin - BoostDummyDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:admin - BoostDagstoreRegisterShard(ctx context.Context, key string) error //perm:admin - BoostDagstoreDestroyShard(ctx context.Context, key string) error //perm:admin - BoostDagstoreInitializeShard(ctx context.Context, key string) error //perm:admin - BoostDagstoreInitializeAll(ctx context.Context, params DagstoreInitializeAllParams) (<-chan DagstoreInitializeAllEvent, error) //perm:admin - BoostDagstoreRecoverShard(ctx context.Context, key string) error //perm:admin - BoostDagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin - BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) //perm:read - BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:admin - BoostMakeDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:write + BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin + BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error) //perm:admin + BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*ProviderDealRejectionInfo, error) //perm:admin + BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error) //perm:admin + BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error) //perm:admin + BoostDummyDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:admin + BoostDagstoreRegisterShard(ctx context.Context, key string) error //perm:admin + BoostDagstoreDestroyShard(ctx context.Context, key string) error //perm:admin + BoostDagstoreInitializeShard(ctx context.Context, key string) error //perm:admin + BoostDagstoreInitializeAll(ctx context.Context, params DagstoreInitializeAllParams) (<-chan DagstoreInitializeAllEvent, error) //perm:admin + BoostDagstoreRecoverShard(ctx context.Context, key string) error //perm:admin + BoostDagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin + BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) //perm:read + BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:admin + BoostMakeDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:write // MethodGroup: Blockstore BlockstoreGet(ctx context.Context, c cid.Cid) ([]byte, error) //perm:read diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index b76f31dca840c6d1be8c631e36af6fb398192486..be25749c3d23b04eb7188ac24d2d5423ca630d04 100644 GIT binary patch delta 5261 zcmV;86msk5DflUn7XdJl7$Se9#04NhigFy4Z6=W@;P3!AI}Q#Wb;pI+b4~qB@3gwD zfo|vsPaXa2QKyL7P(RbJ9qJS9W^}2YkA}LT-$Rce2lF1st@bO=?C2{)UqCu{ETW%% ze$+8ou%2&_4NV8(*11U?Zw&e0=m*pd-GapQkV^wVU%!5xIbZP9$EJVwf@m)<9EZ>` z^?Z{aXsOOG+_q*qn8F9zx(xm`G`FJx%km&0t@GgD1MLNvpw_0Jp+V1cyaVm=3;sgU zR3oh+rqIK{PUENj_M3*09kl#HJxF~IYcDbUo`QK`XspC8V7VEKMe}$do zFL~j%8Quq9@ahZx`t^USZs^2cEP%Jv&-9OT=s^v5P{R(@mXK-yFE!IyEFh*tn>wCG z=ZI**q_=cKcYy~M!Ib$PoBYZ0mFp11XE^*hu{drgvka*zOkl*mm~*%Qj5Oc_ShzN1 z@bqcB)9v;3TZ47yuLJhTb6n_A#9B;&O`xIEr3+beXYw~RseXUEGIS4q^N|NF{fv4( zWXQN4Gy!E#c(2thYeamTvX_QAQf+X8XY3ULwOlzSVQ40?dW-1XpJ*OjI239&!gso%`<|_z>2pe5 za&~w)<4szovp9d8hV1vs^&B?ohZDwDhZEbG94-K%&^rV!I&?kfVflYEh+8w~f3CZO zp1xWcSsuQy9rKP*$AfPnEyux$t}^~Lk=43H)`9lo^X+6wbNMv^Fo#_3B;TQvrKzsTsXfuC7cowV^rjcxgmWI~QTH25@ z{+K%60$SQ+sR8ZJKs6W|&@LSdn%ax+b7amnge}DAoV{b=+Z37X*HQfGs@+hFO%-pUYD~QnlV!Qp<_+oD2f6d|l z{`Q+*6~AXHQjYryBBi`xkm+75S`ennaz$@N75sNvGS} zGrQ#Es7Q#s1Ei_|c?U?b@R&G`ttLI*0V0+h!EAq4k{$tyjpX>JNe)JJnVAhA7TXr= zO)+d(KM3%zEeTEs5=p>1Ux68Cmn%r9=Pbu_;8`W5f((pILIg{E5J(20V30u+3yBia zAqUl+ONx-Pl`Pz#t#%J=A6B+*Mcnbhx3k<4K8(q%wH~zXUz+xN?PCSne|xzF#lO82 z&^dqi%A*#X!fyCViV=(#ANBzOz$eQ{;yxzAd=}#8G8dwiKgl5(^-O$Ir zX+p9Iz`bS*rJw0v{C2x%@@3;6p}xAkHFSSOVaV~hVLUhX=rMvBF<8XTD z^>2|%TCD^@!kDN4o(Gn{HuH8)6qt925kLOB_fb?U@Gl$6>KPaS2Iwt=m#+Sm)KE4IXFoX7W!W+tB|$(*KDp_koiF1lh?@Aa@9C8Fl+UvRa=$4X+Ng z7ca?zO(bcJJm(%+&|@9BTT*P;LST!j9V5HEy&53Aa|&M%UX94x$HD)v1b=@;|L?|7 zU4VyS0JJkO^aaAPZ*>{5^3~7ao!hi{A|G|u{*#X^{Y-6A4gJ!CYyjt!b!oRdulS#j z?e-b}^A~GkBZxWeFz?(Uz-I4Fd}jX78#W4bq)Pb=rL>9;;b6Z ze}I^N7UjMgdNgnGXV+i??O=b_#uRw;wd259>awH5FnCzyo#QW(F}utnI83zoT|Epeg!Dd&$2mZtCgV-$sg0HjWFDd z6awtKL7({GM-X~!-t|uRs`tC&{QK>XhabQH_1}B=?myIhJ2)rar+^~P~%eXBRb6UWCE6S5;f=WCr8>M=wA=mAWo(A&&N zxM!$e-AV%sHVljb`XUhBEtpYUqDV zrv%c7;I1K_(RuXkat?pZJJ&&s+&b~Ed5rVg^RnpQK+2gI_|(x2{lmqQLUIF5i0+vj zIW_b*fKb*7!TSxZpJVL!*n}UQ;poFKc)BV|_D+V*(jc;1KdD(0$l0(*_F*MwOTx0jrki9U-=W7UF*&CI|y>`5K#!#fHF` ziogM9j?CZ!1}L7qP2VtHxia*vMqAh^^SZ2{D+^&GZ&j!zl-*@t64yt$VVcZZ#Pp5| zej9yL8byIFDGMX6&Q!1!tG2RlO={wt{o{2X5#c7)mmq8(GfY&aL|U1ZTl$Ud^cY#1 zX-7qDR~F8Dv6X)r8kwt5X9%Vvr4&RmFr#$^U}Bbq2t=gxRH!Mfxm5Z@DprZNV%oaw zT82tiv>UY6_$oNW?Ud5uRYKU2{Zdg|s0*jHUXkp3W%;w@v9M?A>YvIolp`1TV#)M8 zv?Apx+F(%g{soYV^Ov^_2MFF-Z-hWRs3zn2pO*351TKG?z(o_dXaW~a;Gzj!G=YmI zaM1)Vn!rU9xM%_wP2gfTfs1}S_dun;SS+h1Dnxyyxj3Qw#_bTk%m>*n z2k1qtb8g#g1*@F@l|T|60?t1NAc0iNNdP&PPhDtnvdPrH?sxO5GlWZZjR;+Dq{1yI zGX+ScD5~JRWuZSTep~LLRMC;x(oVB4ZD7lr#B!~ko*&q0d!j_!A9E;eW`HmHrEdykbqa zft2m^1RFZiL~kY7+h%;L4q)e5W!*4P);7_{>?t;dko5p@=Ukukw0NLXF-S8(x2QG2BWEaWbzvVcq& zRvCy~lK8Xt2xm$qyax~2X9y>&B78&(NGMpin@R`|E@DVUbr9gU4^DWjzib7AlE-R- zp_b?op{jByH8~qfJrZ-O*rVF%aAX#L*C5Cq2oF|9=|t{kl~tOjD6AY}qVwOsSfz#i zI;l12OL8#QQggjuR9#KPK}ht47c~hbqN$c?+7Jvot1HBfR1`#1-{E3O zBV`a2A&9Go5m+uDrhEqYT?hSc=HU>V9t0#P;Zja=XTU2}(~!MhtrSOm)MX`qXJUdX zO1J$G4M&pvlYm8r3&u-P@s28zAqk;-F{>J4&h8K+dTX0Oht3{E8JlpDP|#HPd46RnaBBLyQfIg&wR9#CZM z)y$66J8}&V$%`r)AKB4=?a2I)pe-;#W=n zK!cm8quVYwPonLQigww0K`QnSuAL8pZ;SxKi@aO5Lw&MfxE)RjUAUcYk6V}&Sb&23 zR%08Gd2)$`mt9NkRX%GskU)=LIm#FO22S~6hMGiNCA!|K)KSKN(R%UX9MbI@`VIKb zvyozDrqAyIvcbfLZyYZ(;H$(v<=hlP4$ik~Q!dMp(`U?Hth(?wkho4@R8|GS-J$hm zt3e(DzuG*!qW1FqCR^67s&$=Urm_~cWL4x>j0;V&U$jJblFrK8-0HW)E8Je3V(qfn z#3@U#LV2!GQg(?|8bn*A;-cKreoYDVrUd%4FM*yhaZ0X9%x)=vOZT)SYitznj%g>U zz?FJbDi@!$S`^BVrgZr)z1d)^?xPP*SFHr zzvwyx=U!k9qP{O2K#C1^IEr4khCv+tvV&+; z$Mtb9w;IlP`+9!|{7`(qA`d?Qx>fN8qw$+YGm`6&H?4t4#Z$$v$rE07&r(M3a|-V zUvM3nKdFCli!*<+jTEx+^&o;mbn!(2WkP(JfZl$k!ktP=X#_c>e`V_zVX>Cl<%)W|ko}g$az<7h?gJfRP4#087V$ z44yvgbbI}Qe%oT*`Rjl+c5Mf`6tNaFU=i5T>B@nuxjp?G8dQJ3UAJ@>e)EtEP5q3z z9%RTkE;ImTPk68OEo($Pi?WvncqdmNJBIYnk|BiDjlJa*x*0GZ_+t$b4QNr5*s%M9 z{?X}i|D-=Wx!nvdXsI?h!87&>pIWYLgD^CcSiMDb;Y~FcE^P`m3*kGxrF*WWpXm!q z9ddSfIOk2;hP{70oCWOn%5iNr>4#IsR)`+u%` z!@j;=x3WBZVcEtVp|%U(LRyZ46J2HeYap|IiOd7-#pm1Uistfb0$>ie`jAR-X&Tq( z#-qVB6%#g}TTHc)2{A=8Bp~ujWNI#>LBcTx8iCgAmEV76ityZDCrm@x3QY|~(AwIF zGX9v^?h=~Xbfp39PhT~(G@xDDCN#7c-xtVOXb78#(K&m^(z7Ts*sre~VBA47dJE&C z0}WtP1KNAmY5~Zio#eYm{gBNYn5}x#AHaa1|DBkN;i?b%Lptq~JGgZIwjaJZ-pXD& ze@~Y)xSD_U2Q$?7RzuH&gWml>U&mOMP#59(7V^lavCP~?j}=7df3em1YjUwL@V^%D ze}DT;uZrI@6)DF91(8zTFvxT-7A*)5zl!&Lu@a*-93!-&VT^mIo_aw<7NN;8|Jj2p+~{R-^|l>zAhee&<+$_TOG^LGf=d z1$2MTz4EXHr?4Bo61nOy2{qSE7l1D3wP!}KHR+3eJprNu*H7H4m`^@G>c7JkS3q+x zgkV4*_y&z7{c()IUB7RV8CjrV?-)#f4A5XWJThnA54ZnqMh4R*Fo)(tAECMVO>gNF z&oCg_1mIq?h0@RTFJ7n9H~6ygk5FIV-nM^qL}9@3xnVpf_UJK$88TSN&g+&x48ijV ztLV2-C9T(jAYn{Y0M`YpUz>TmCko6v!iXYZp&xKt-Ian=kNQdhtyP0%5pI;&J}R+6 zM}ZHn5uuz6$~tvoAWBMYglD$0O54CzS8ePC)fHP}G*6-67`jF3Eok$^%8dfb@a2CP z@Gs~009jyaLH*gNIv>Z(FZ-|Oj=Bo`Jx0)ggs~z(7~y&3k#E87QkXK`w~WF7QBy7z zf+&cxEReo9S2Tp7rOE;!Z@fEKor1Yj)1w#P{Vk3SA65x=W@4Kac}Oe;UwW??NI~Z2f(-^^axg zBuPq`BL-l5;*Y_pi~3_AYbO6BvJL(3BmJMqavwM;K#-jb1#*YLl2NzkA+!DI)9C6z zd-0Mi*+i1|*tPGG30>BayCubjEd-XB+A*@q+p9jpJE!ny@M=WfIrjg5CHQ|U{C_uw z>JmJRe4yQ7OJ5=!`&O3`D_{K#-nmVSC-PBe?LT?Q)X&r=wWVLWkPYCRvM!xY_Z9#1 zvC}!@fBs@kECey99p;_e1X%37iO9+LO*i1f{P}>D_IChZ%$IuOa z)H^Zgm$qdA3SEW?|G`Hj^>`gVpP)GgJ~KV)A30U^TKW~BKtIdooUPX_eMJ74%`Ajr za-?GjT{g*jw|CY5-M0Vz_Q%7I-~amWJ$&~c>bxDE6Zg}N-aAPuSO z8sa%!gx@X~(71DK#Atu38~>WeC9hpKi~bFyoMVAUZM~&`xL8q0ZlD3tJrf$IE&UB3 zl(j%i0nR3YSsioHRut1UneJ~T>Eq0gSoJP z&7lb>d_~a`ZhCD3c4B!`V(bF8E799$%tw6*O~gM;5eDw+H8y{2lVQkshQJ}G{S59t zeHYKorf(2?T(|VCMqAh^^SZ3iDhpvFJ5`7wki%tP64x!cA(zZr#B_)XejD9U8byI# zC<`O5npChAtEsYYO={Jgz2K;ih;WnINf3gM85ycVAg!y)E&aw0dJL_`w9_HB5ew(N z*kTL}#&w_)1k-bHp>cT0~E0R5|EPs|f;`L13_ETB5ap=xoteBpMW~e+xn+0l~ zyZ};hF7lSHz2Keodi&x*HQCGmwCv?32+;%~njl0IglK<)5KR!G2|_eMh$aZp1RsQN}0z#EpAJpH$-hG#=>Fd7!&-6`#Npyn##m9)7|E0t=d#wzCS?YI27h z<_m?MQ6+!3s_{)4-=y(Pej(qa20!AHzra7wk0>zoMIuksE-L^Dkd-g6-uZ0eqmRmn^#w2giZJW-H>(8EsL#S zm2%dvdwN{~$k{q>-iS2bB7SgLD8XtXH`x1h`vAeCaJf^(3C z{;>FMxraqXM^#HZ&C*deJM;gvop~We3C_D#J?0yp^G%$e*!_{;y*fEM=%41uzW3k` zRtx}!U7src7rw8)@Ly~Mla2})e~JDz(Z4co#!+5DzJLMQvi%#;1=qvnJq`hs!V(58 z&FMwuI<|2?1TU}86A_P$jfCOf)H~Qky@L&{I=$yv=cJL0dm$Nzl2V3zL*UZJgsod3 z0ivwN3cDl4@Wcg7i|(W-hFjA`6-ooxNzKycf`l~D9!u)YiPk7^SZX5Ke>k-I2sRFN zhvb??5U)pn_!9R60R9wwGr#i zMXLz)=a5Cw&~=Kgk2;c)2Z{CkgztmcWKFa!23P}qivjO~#-&4O1oD#*ZdY%qlLOsFHe*%bgF@|ysbPLW9 zqu;Q`-mu1T-nMIo5^7W!Dv{ReY_U^yAws=jiM?Tole|sc7#-9Wp)G`26Kj+S)K+H? z=@1$g*&7x)&D%v4?s{xH@x4&~I2{Usz+<$=kG^Kw3xCSp-JL1e+jn zF()VnpUacK2-}_0e|!^Qe$D~rMpj%8yOKnJ_rSe_^b%kL@^eMeahX!0LIy=9DNH~C z4cfHB#=+SD2ST^aKlh8&s_eDf%95Pnghv1vdOH;#wNNIlKPHs=w}LGHt<&~{ z#Z=4NuT%uSYJPtV?S5m!{4}}LljM2$Ech1Eb8=Tj7WqBJe_P})iEk1wdCRtcsYzfa zpOd@1jq^D0T233Ku-(bVc)@rDmv$7jS31B#?)<6-$biAA{?H|fKYNdGrZmBO@Q{6m zaIy-)N3?{5f~B*mJn-NkhE!O00Dk*m2gm5kRtzY4ti~T|i5?-UDu+Omv!T=@F{g?> zs-4D0M)8^df9wHle`Sft{%oM<$%W{0=i?!`&7J; zF`NRio2e&SB^yQxMrLv(gT_3d$l9x!9jSNZ8Xl4tRWv@bquY`B ze<49zV1me7Dw!co9pN3-5#E%Iz4R~QtY=2gP4kV1O~Qft7dwZyOKzTo+aDF}vS>jn z_V=%B_k(YY0D^n5+m=l|vShd&&InyP-Cmztm=u_Rg8Wt!3y?)}iG_DZOYK!YYd4TU zmtXP87yJg!_+kc{L|m1*-YU>x#!<9*e{l}!4O;pQ_|CJDVr8Ar?*X#F)PiqpH#6X? z#I@ht6hb!6w`x-^%ZSrw%wDXz@Hdb+wr^Bc1;O2+=yFp(4}sql9$X1~d47{EYgg5} zPB2qh3tO@(!kfmUO3)c7zi3J9B%PJFxz#USSGX-W#oA@DiBpzfh4NgXr0fzkR0y|9 z#YM2C{hE^FP08_RUvfNS;*?yInB7wTR_s|x*4QZC9n(%yfh+Z>RGvL)wJ4PRPAgR< z(u$c~YNQCLYPoWe5mI77RTYRR4i+b*$YiNlIg{@YApsDR5fK=FtSlRt_3m{rSBObE zi_1C&_y&BJdFYWn5jbPnB$Ki!C8D$xrk^2u3;Q_~cURBP=i<}BtB&s^805=~@K&si z&+2O>_WiV_|1p~pNW*Z^HN27 zaKfeMx*58-hI3#Ftj4ZQZNs+0JC>8O5hs5r0Lp|Re9o@6NmeEp5Xu9UwxJ|ZGFu}N zR4p8m-hr#S1Y#d+n;gK6loajN#3Ru8F?O3y-OMkL(bF6+5@Cs&rY7{%!85E(KnQ=)Y)%$-+ z!-VT*V{kVYjNV3}uQ7CyZE~#Tn>C5H`q{7({Z}%5eJd^fi|$i2ar7pR{_NxEN)}~O z3Mfs!K*&7mGuvGP>Yo{huP9o=95Zw;PJbMCC8bY%wS5B#o11Ox83@2UNkId_d+RwS zA|6mv(x546(3CW&Pd4>(ORY>Qmo$Hv$VwV;3B=})v@g(ngeez0sRw-xOi}xC0TAX| zu%$Tz{%8zL6Pl3~(9)bE(epy7HOVQ8P*zYyLygNOhB3o#r!p!&Yl;`J>P0qtHN~4J zgPtRnS&@?@xSff4?giE$>IcFBq}X7Gqv-c)7{t*pJBUVgTp#yytKm$xuXleiDXe$E zm5Jz7zls7VE_hHu6QmH{c`?7Hq`|(IG-yQ4_3=P*UzJ~S`~nzvo>NZ3#CysFOJR+K zIFM~6ND3jj;MiS?PM(};4(a~{vOkhk6dIcs9tw(dCF`VMmrB+V zjX^A$vS|6xh(;GsBb&!`0Vub2AO=&5y&YlSz(JT$fDPFGg6qiqNsWJ7ocWV&q>zoT z`w - -

Booster HTTP Server

- Endpoints: - - - - - - - -
- Download a raw piece by its piece CID - - /piece/ -
- - -` - func (s *HttpServer) handleIndex(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(s.idxPage)) //nolint:errcheck @@ -306,7 +295,7 @@ func (s *HttpServer) getPieceContent(ctx context.Context, pieceCid cid.Cid) (io. } func (s *HttpServer) unsealedDeal(ctx context.Context, pieceCid cid.Cid, pieceDeals []model.DealInfo) (*model.DealInfo, error) { - // There should always been deals in the PieceInfo, but check just in case + // There should always be deals in the PieceInfo, but check just in case if len(pieceDeals) == 0 { return nil, fmt.Errorf("there are no deals containing piece %s: %w", pieceCid, ErrNotFound) } diff --git a/cmd/boostx/stats_cmd.go b/cmd/boostx/stats_cmd.go index 1b8775955..42bcc1a96 100644 --- a/cmd/boostx/stats_cmd.go +++ b/cmd/boostx/stats_cmd.go @@ -2,6 +2,8 @@ package main import ( "fmt" + "github.com/filecoin-project/boost/retrievalmarket/lp2pimpl" + transports_types "github.com/filecoin-project/boost/retrievalmarket/types" "regexp" "sort" "strings" diff --git a/cmd/lib/filters/filteredblockstore.go b/cmd/lib/filters/filteredblockstore.go index 5a01a1f7a..bff2d7efe 100644 --- a/cmd/lib/filters/filteredblockstore.go +++ b/cmd/lib/filters/filteredblockstore.go @@ -3,10 +3,10 @@ package filters import ( "context" "fmt" - blockstore "github.com/ipfs/boxo/blockstore" - blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/blocks" ) type FilteredBlockstore struct { diff --git a/cmd/lib/filters/filteredblockstore_test.go b/cmd/lib/filters/filteredblockstore_test.go index b6342e2c0..154a355c1 100644 --- a/cmd/lib/filters/filteredblockstore_test.go +++ b/cmd/lib/filters/filteredblockstore_test.go @@ -43,7 +43,7 @@ func TestFilteredBlockstore(t *testing.T) { filter.EXPECT().FulfillRequest(gomock.Any(), blk.Cid()).Return(true, nil) gotBlk, err := fbs.Get(ctx, blk.Cid()) require.NoError(t, err) - require.Equal(t, blk.RawData(), gotBlk.RawData()) + require.Equal(t, blk, gotBlk) // Filter rejects cid filter.EXPECT().FulfillRequest(gomock.Any(), blk.Cid()).Return(false, nil) diff --git a/db/migrations_tests/deals_announce_to_ipni_test.go b/db/migrations_tests/deals_announce_to_ipni_test.go index 696dc2e63..2a766e828 100644 --- a/db/migrations_tests/deals_announce_to_ipni_test.go +++ b/db/migrations_tests/deals_announce_to_ipni_test.go @@ -29,15 +29,13 @@ func TestDealAnnounceToIPNI(t *testing.T) { // Insert the deal into the DB deal := deals[0] - deal.AnnounceToIPNI = false - _, err = sqldb.Exec(`INSERT INTO Deals ("ID", "CreatedAt", "DealProposalSignature", "PieceCID", "PieceSize", "VerifiedDeal", "IsOffline", "ClientAddress", "ProviderAddress","Label", "StartEpoch", "EndEpoch", "StoragePricePerEpoch", "ProviderCollateral", "ClientCollateral", "ClientPeerID", "DealDataRoot", "InboundFilePath", "TransferType", "TransferParams", "TransferSize", "ChainDealID", "PublishCID", "SectorID", "Offset", "Length", "Checkpoint", "CheckpointAt", "Error", "Retry", "SignedProposalCID", - "FastRetrieval","AnnounceToIPNI") - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + "FastRetrieval") + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, deal.DealUuid, deal.CreatedAt, []byte("test"), deal.ClientDealProposal.Proposal.PieceCID.String(), deal.ClientDealProposal.Proposal.PieceSize, deal.ClientDealProposal.Proposal.VerifiedDeal, deal.IsOffline, deal.ClientDealProposal.Proposal.Client.String(), deal.ClientDealProposal.Proposal.Provider.String(), "test", @@ -45,7 +43,7 @@ func TestDealAnnounceToIPNI(t *testing.T) { deal.ClientDealProposal.Proposal.ProviderCollateral.Int64(), deal.ClientDealProposal.Proposal.ClientCollateral.Uint64(), deal.ClientPeerID.String(), deal.DealDataRoot.String(), deal.InboundFilePath, deal.Transfer.Type, deal.Transfer.Params, deal.Transfer.Size, deal.ChainDealID, deal.PublishCID.String(), deal.SectorID, deal.Offset, deal.Length, deal.Checkpoint.String(), deal.CheckpointAt, deal.Err, deal.Retry, []byte("test"), - deal.FastRetrieval, deal.AnnounceToIPNI) + deal.FastRetrieval) req.NoError(err) diff --git a/docker/monitoring/grafana/dashboards/provider-retrievals.json b/docker/monitoring/grafana/dashboards/provider-retrievals.json index d6bdab783..e8acde859 100644 --- a/docker/monitoring/grafana/dashboards/provider-retrievals.json +++ b/docker/monitoring/grafana/dashboards/provider-retrievals.json @@ -24,7 +24,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 3, "links": [], "liveNow": false, "panels": [ @@ -1614,6 +1614,424 @@ "title": "booster-http: ipfs gateway", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "exemplar": true, + "expr": "irate(booster_http_http_rbls_bytes_sent_count{}[$__rate_interval]) * 15", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Http Gateway Bytes Served", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 73 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "irate(booster_http_http_rbls_get_request_count{}[$__rate_interval]) * 15", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Http Gateway Block Get Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 73 + }, + "id": 78, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "irate(booster_http_http_rbls_get_success_response_count{}[$__rate_interval]) * 15", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "irate(booster_http_http_rbls_get_fail_response_count{}[$__rate_interval]) * 15", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Http Gateway Block Get Response Counts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 73 + }, + "id": 80, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "(irate(booster_http_http_rbls_get_success_response_count{}[$__rate_interval]) * 15) / (irate(booster_http_http_rbls_get_request_count{}[$__rate_interval]) * 15)", + "legendFormat": "Success", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "(irate(booster_http_http_rbls_get_fail_response_count{}[$__rate_interval]) * 15) / (irate(booster_http_http_rbls_get_request_count{}[$__rate_interval]) * 15)", + "hide": false, + "legendFormat": "Fail", + "range": true, + "refId": "B" + } + ], + "title": "Http Block Get Success / Failure Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 30, + "panels": [], + "title": "booster-http: ipfs gateway", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -1677,7 +2095,7 @@ "h": 7, "w": 24, "x": 0, - "y": 59 + "y": 82 }, "id": 74, "options": { @@ -1767,6 +2185,101 @@ } ] } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0-77684pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "irate(booster_http_http_piece_by_cid_request_count{host=~\"$host\"}[$__rate_interval]) * 15", + "legendFormat": "Requests", + "range": true, + "refId": "A" + } + ], + "title": "Piece by CID Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, "overrides": [] }, @@ -1774,7 +2287,7 @@ "h": 8, "w": 8, "x": 0, - "y": 66 + "y": 89 }, "id": 76, "options": { @@ -2193,10 +2706,10 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 83 + "h": 8, + "w": 12, + "x": 12, + "y": 89 }, "id": 37, "options": { @@ -2351,9 +2864,9 @@ }, "gridPos": { "h": 7, - "w": 8, - "x": 8, - "y": 83 + "w": 12, + "x": 0, + "y": 97 }, "id": 3, "options": { @@ -2444,9 +2957,9 @@ }, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 83 + "w": 12, + "x": 12, + "y": 97 }, "id": 6, "options": { @@ -2502,13 +3015,109 @@ "title": "Piece by CID Request Durations", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "go_goroutines{job=~\"booster-http\", host=~\"$host\"}", + "legendFormat": "Goroutines", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 90 + "y": 112 }, "id": 82, "panels": [], @@ -2578,7 +3187,7 @@ "h": 8, "w": 12, "x": 0, - "y": 91 + "y": 113 }, "id": 49, "options": { @@ -2676,7 +3285,7 @@ "h": 8, "w": 12, "x": 12, - "y": 91 + "y": 113 }, "id": 50, "options": { @@ -2773,7 +3382,7 @@ "h": 8, "w": 12, "x": 0, - "y": 99 + "y": 121 }, "id": 51, "options": { @@ -2870,7 +3479,7 @@ "h": 8, "w": 12, "x": 12, - "y": 99 + "y": 121 }, "id": 52, "options": { @@ -2966,7 +3575,7 @@ "h": 8, "w": 12, "x": 0, - "y": 107 + "y": 129 }, "id": 54, "options": { @@ -3062,7 +3671,7 @@ "h": 8, "w": 12, "x": 12, - "y": 107 + "y": 129 }, "id": 57, "options": { @@ -3159,7 +3768,7 @@ "h": 8, "w": 12, "x": 0, - "y": 115 + "y": 137 }, "id": 56, "options": { @@ -3256,7 +3865,7 @@ "h": 8, "w": 12, "x": 12, - "y": 115 + "y": 137 }, "id": 55, "options": { @@ -3333,6 +3942,6 @@ "timezone": "", "title": "Provider Retrievals", "uid": "VG2IXa4Vk", - "version": 5, + "version": 16, "weekStart": "" } \ No newline at end of file diff --git a/go.mod b/go.mod index f8d33b279..edbed72f3 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.4 github.com/filecoin-project/go-state-types v0.11.1 github.com/filecoin-project/go-statestore v0.2.0 - github.com/filecoin-project/lotus v1.21.0-rc1 + github.com/filecoin-project/lotus v1.21.0-rc2 github.com/filecoin-project/specs-actors v0.9.15 github.com/gbrlsnchs/jwt/v3 v3.0.1 github.com/golang/mock v1.6.0 @@ -55,9 +55,9 @@ require ( github.com/graph-gophers/graphql-transport-ws v0.0.2 github.com/hashicorp/go-multierror v1.1.1 github.com/hnlq715/golang-lru v0.3.0 - github.com/ipfs/go-block-format v0.1.2 - github.com/ipfs/go-blockservice v0.5.1 // indirect - github.com/ipfs/go-cid v0.4.1 + github.com/ipfs/go-block-format v0.1.1 // indirect + github.com/ipfs/go-blockservice v0.5.0 + github.com/ipfs/go-cid v0.4.0 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-graphsync v0.14.3 @@ -71,7 +71,7 @@ require ( github.com/ipfs/go-ipfs-routing v0.3.0 github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-ipld-legacy v0.1.1 - github.com/ipfs/go-libipfs v0.5.0 + github.com/ipfs/go-libipfs v0.7.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.9.0 github.com/ipfs/go-metrics-interface v0.0.1 @@ -122,7 +122,7 @@ require ( golang.org/x/text v0.7.0 golang.org/x/tools v0.3.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 - google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect + google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 ) @@ -234,10 +234,11 @@ require ( github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipfs/go-path v0.3.1 github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-unixfsnode v1.5.2 github.com/ipfs/go-verifcid v0.0.2 // indirect - github.com/ipfs/interface-go-ipfs-core v0.11.1 // indirect + github.com/ipfs/interface-go-ipfs-core v0.11.1 github.com/ipld/go-codec-dagpb v1.5.0 github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect @@ -273,7 +274,7 @@ require ( github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect @@ -326,10 +327,10 @@ require ( go4.org v0.0.0-20200411211856-f5505b9728dd // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 + golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect - google.golang.org/grpc v1.47.0 // indirect + google.golang.org/grpc v1.52.3 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -342,8 +343,17 @@ require ( github.com/filecoin-project/boostd-data v0.0.0-00010101000000-000000000000 github.com/gabriel-vasile/mimetype v1.4.1 // indirect github.com/ipfs/go-ds-flatfs v0.5.1 + github.com/ipfs/go-fetcher v1.6.1 + github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect + github.com/ipfs/go-namesys v0.7.0 +) + +require ( + github.com/filecoin-project/boost-gfm v1.26.5 + github.com/filecoin-project/boost-graphsync v0.13.6 + github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc4 github.com/ipfs/kubo v0.18.1 - github.com/schollz/progressbar/v3 v3.11.0 + github.com/schollz/progressbar/v3 v3.13.1 ) require ( @@ -354,9 +364,6 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/docker v20.10.21+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect - github.com/filecoin-project/boost-gfm v1.26.5 - github.com/filecoin-project/boost-graphsync v0.13.6 - github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc4 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect @@ -364,6 +371,7 @@ require ( require ( github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect + github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/onsi/ginkgo/v2 v2.5.1 // indirect github.com/quic-go/qpack v0.4.0 // indirect @@ -371,6 +379,8 @@ require ( github.com/quic-go/qtls-go1-20 v0.1.1 // indirect github.com/quic-go/quic-go v0.33.0 // indirect github.com/quic-go/webtransport-go v0.5.2 // indirect + github.com/tidwall/gjson v1.14.0 // indirect + github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/zyedidia/generic v1.2.1 // indirect nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index ae4721ee1..49e7b4d6d 100644 --- a/go.sum +++ b/go.sum @@ -204,11 +204,6 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -338,8 +333,6 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakRE22OTI12k+2LkyY= github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= @@ -426,8 +419,8 @@ github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNd github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5zQkWJM/CpAZa/uOuus= github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8= -github.com/filecoin-project/lotus v1.21.0-rc1 h1:LcU1T4tyGbBYDHA6Hl/lVBcR80wUsLN9xiIB42XuUqM= -github.com/filecoin-project/lotus v1.21.0-rc1/go.mod h1:EQk2GAEwWeLCZNRU1MaF8Kj6cGVMK1F6Tt30OMXnL/M= +github.com/filecoin-project/lotus v1.21.0-rc2 h1:EAADt5PpVei/sANsa//oQK8wm3bxxeFn/lbtiEtT4sI= +github.com/filecoin-project/lotus v1.21.0-rc2/go.mod h1:a4qjLilt0mDENhD3RmV5N8hj8d0VxEcRn53266V0KNY= github.com/filecoin-project/pubsub v1.0.0 h1:ZTmT27U07e54qV1mMiQo4HDr0buo8I1LDHBYLXlsNXM= github.com/filecoin-project/pubsub v1.0.0/go.mod h1:GkpB33CcUtUNrLPhJgfdy4FDx4OMNR9k+46DHx/Lqrg= github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= @@ -463,6 +456,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= +github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/gbrlsnchs/jwt/v3 v3.0.1 h1:lbUmgAKpxnClrKloyIwpxm4OuWeDl5wLk52G91ODPw4= github.com/gbrlsnchs/jwt/v3 v3.0.1/go.mod h1:AncDcjXz18xetI3A6STfXq2w+LuTx8pQ8bGEwRN8zVM= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= @@ -675,7 +670,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= @@ -788,8 +782,8 @@ github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqg github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= -github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= -github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA= +github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -824,6 +818,10 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ= github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE= +github.com/ipfs/go-fetcher v1.6.1 h1:UFuRVYX5AIllTiRhi5uK/iZkfhSpBCGX7L70nSZEmK8= +github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo= +github.com/ipfs/go-filestore v1.2.0 h1:O2wg7wdibwxkEDcl7xkuQsPvJFRBVgVSsOJ/GP6z3yU= +github.com/ipfs/go-filestore v1.2.0/go.mod h1:HLJrCxRXquTeEEpde4lTLMaE/MYJZD7WHLkp9z6+FF8= github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U= github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc= @@ -900,8 +898,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= -github.com/ipfs/go-libipfs v0.5.0 h1:gtvzeoTdUHPUN4B5izzyBS3Cxtpvi+l7hd2mmjN+teM= -github.com/ipfs/go-libipfs v0.5.0/go.mod h1:iZ9QyhzNr3AkxRXrbQYb//rv7iLyvZJX0GNuc3lJDiQ= +github.com/ipfs/go-libipfs v0.7.0 h1:Mi54WJTODaOL2/ZSm5loi3SwI3jI2OuFWUrQIkJ5cpM= +github.com/ipfs/go-libipfs v0.7.0/go.mod h1:KsIf/03CqhICzyRGyGo68tooiBE2iFbI/rXW7FhAYr0= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= @@ -932,6 +930,10 @@ github.com/ipfs/go-merkledag v0.9.0 h1:DFC8qZ96Dz1hMT7dtIpcY524eFFDiEWAF8hNJHWW2 github.com/ipfs/go-merkledag v0.9.0/go.mod h1:bPHqkHt5OZ0p1n3iqPeDiw2jIBkjAytRjS3WSBwjq90= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/ipfs/go-namesys v0.7.0 h1:xqosk71GIVRkFDtF2UNRcXn4LdNeo7tzuy8feHD6NbU= +github.com/ipfs/go-namesys v0.7.0/go.mod h1:KYSZBVZG3VJC34EfqqJPG7T48aWgxseoMPAPA5gLyyQ= +github.com/ipfs/go-path v0.3.1 h1:wkeaCWE/NTuuPGlEkLTsED5UkzfKYZpxaFFPgk8ZVLE= +github.com/ipfs/go-path v0.3.1/go.mod h1:eNLsxJEEMxn/CDzUJ6wuNl+6No6tEUhOZcPKsZsYX0E= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= github.com/ipfs/go-peertaskqueue v0.8.0/go.mod h1:cz8hEnnARq4Du5TGqiWKgMr/BOSQ5XOgMOh1K5YYKKM= @@ -941,7 +943,6 @@ github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o= github.com/ipfs/go-unixfs v0.4.3 h1:EdDc1sNZNFDUlo4UrVAvvAofVI5EwTnKu8Nv8mgXkWQ= github.com/ipfs/go-unixfs v0.4.3/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= -github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s= github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo= github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew= github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks= @@ -1756,8 +1757,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/schollz/progressbar/v3 v3.11.0 h1:3nIBUF1Zw/pGUaRHP7PZWmARP7ZQbWQ6vL6hwoQiIvU= -github.com/schollz/progressbar/v3 v3.11.0/go.mod h1:R2djRgv58sn00AGysc4fN0ip4piOGd3z88K+zVBjczs= +github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= +github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sercand/kuberesolver v2.4.0+incompatible h1:WE2OlRf6wjLxHwNkkFLQGaZcVLEXjMjBPjjEU5vksH8= @@ -1859,13 +1860,15 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= +github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= @@ -1879,6 +1882,8 @@ github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= +github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= +github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= @@ -2016,8 +2021,8 @@ go.opentelemetry.io/otel/sdk v1.13.0 h1:BHib5g8MvdqS65yo2vV1s6Le42Hm6rrw08qU6yz5 go.opentelemetry.io/otel/sdk v1.13.0/go.mod h1:YLKPx5+6Vx/o1TCUYYs+bpymtkmazOMT6zoRrC7AQ7I= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY= +go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -2360,22 +2365,19 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= @@ -2525,8 +2527,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2549,10 +2551,8 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -2566,7 +2566,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/gql/serve_download.go b/gql/serve_download.go index af03446a0..eb6191936 100644 --- a/gql/serve_download.go +++ b/gql/serve_download.go @@ -3,8 +3,8 @@ package gql import ( "context" "fmt" - blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" "net/http" ) diff --git a/gql/server.go b/gql/server.go index b0d2360e4..1404beb53 100644 --- a/gql/server.go +++ b/gql/server.go @@ -23,22 +23,14 @@ import ( var log = logging.Logger("gql") type Server struct { - resolver *resolver - bstore BlockGetter - cfgHandler http.Handler - srv *http.Server - wg sync.WaitGroup + resolver *resolver + bstore BlockGetter + srv *http.Server + wg sync.WaitGroup } -func NewServer(cfg *config.Boost, resolver *resolver, bstore BlockGetter) *Server { - webCfg := &corsHandler{sub: &webConfigServer{ - cfg: webConfig{ - Ipni: webConfigIpni{ - IndexerHost: cfg.IndexProvider.WebHost, - }, - }, - }} - return &Server{resolver: resolver, bstore: bstore, cfgHandler: webCfg} +func NewServer(resolver *resolver, bstore BlockGetter) *Server { + return &Server{resolver: resolver, bstore: bstore} } //go:embed schema.graphql diff --git a/metrics/metrics.go b/metrics/metrics.go index 15228872e..553d04e1d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -29,12 +29,12 @@ var ( APIRequestDuration = stats.Float64("api/request_duration_ms", "Duration of API requests", stats.UnitMilliseconds) // http - HttpPayloadByCidRequestCount = stats.Int64("http/payload_by_cid_request_count", "Counter of /payload/ requests", stats.UnitDimensionless) + HttpPayloadByCidRequestCount = stats.Int64("http/payload_by_cid_request_count", "Counter of /ipfs/ requests", stats.UnitDimensionless) HttpPayloadByCidRequestDuration = stats.Float64("http/payload_by_cid_request_duration_ms", "Time spent retrieving a payload by cid", stats.UnitMilliseconds) - HttpPayloadByCid200ResponseCount = stats.Int64("http/payload_by_cid_200_response_count", "Counter of /payload/ 200 responses", stats.UnitDimensionless) - HttpPayloadByCid400ResponseCount = stats.Int64("http/payload_by_cid_400_response_count", "Counter of /payload/ 400 responses", stats.UnitDimensionless) - HttpPayloadByCid404ResponseCount = stats.Int64("http/payload_by_cid_404_response_count", "Counter of /payload/ 404 responses", stats.UnitDimensionless) - HttpPayloadByCid500ResponseCount = stats.Int64("http/payload_by_cid_500_response_count", "Counter of /payload/ 500 responses", stats.UnitDimensionless) + HttpPayloadByCid200ResponseCount = stats.Int64("http/payload_by_cid_200_response_count", "Counter of /ipfs/ 200 responses", stats.UnitDimensionless) + HttpPayloadByCid400ResponseCount = stats.Int64("http/payload_by_cid_400_response_count", "Counter of /ipfs/ 400 responses", stats.UnitDimensionless) + HttpPayloadByCid404ResponseCount = stats.Int64("http/payload_by_cid_404_response_count", "Counter of /ipfs/ 404 responses", stats.UnitDimensionless) + HttpPayloadByCid500ResponseCount = stats.Int64("http/payload_by_cid_500_response_count", "Counter of /ipfs/ 500 responses", stats.UnitDimensionless) HttpPieceByCidRequestCount = stats.Int64("http/piece_by_cid_request_count", "Counter of /piece/ requests", stats.UnitDimensionless) HttpPieceByCidRequestDuration = stats.Float64("http/piece_by_cid_request_duration_ms", "Time spent retrieving a piece by cid", stats.UnitMilliseconds) HttpPieceByCid200ResponseCount = stats.Int64("http/piece_by_cid_200_response_count", "Counter of /piece/ 200 responses", stats.UnitDimensionless) @@ -48,6 +48,18 @@ var ( HttpBlockByCid404ResponseCount = stats.Int64("http/block_by_cid_404_response_count", "Counter of /block/ 404 responses", stats.UnitDimensionless) HttpBlockByCid500ResponseCount = stats.Int64("http/block_by_cid_500_response_count", "Counter of /block/ 500 responses", stats.UnitDimensionless) + // http remote blockstore + HttpRblsGetRequestCount = stats.Int64("http/rbls_get_request_count", "Counter of RemoteBlockstore Get requests", stats.UnitDimensionless) + HttpRblsGetSuccessResponseCount = stats.Int64("http/rbls_get_success_response_count", "Counter of successful RemoteBlockstore Get responses", stats.UnitDimensionless) + HttpRblsGetFailResponseCount = stats.Int64("http/rbls_get_fail_response_count", "Counter of failed RemoteBlockstore Get responses", stats.UnitDimensionless) + HttpRblsGetSizeRequestCount = stats.Int64("http/rbls_getsize_request_count", "Counter of RemoteBlockstore GetSize requests", stats.UnitDimensionless) + HttpRblsGetSizeSuccessResponseCount = stats.Int64("http/rbls_getsize_success_response_count", "Counter of successful RemoteBlockstore GetSize responses", stats.UnitDimensionless) + HttpRblsGetSizeFailResponseCount = stats.Int64("http/rbls_getsize_fail_response_count", "Counter of failed RemoteBlockstore GetSize responses", stats.UnitDimensionless) + HttpRblsHasRequestCount = stats.Int64("http/rbls_has_request_count", "Counter of RemoteBlockstore Has requests", stats.UnitDimensionless) + HttpRblsHasSuccessResponseCount = stats.Int64("http/rbls_has_success_response_count", "Counter of successful RemoteBlockstore Has responses", stats.UnitDimensionless) + HttpRblsHasFailResponseCount = stats.Int64("http/rbls_has_fail_response_count", "Counter of failed RemoteBlockstore Has responses", stats.UnitDimensionless) + HttpRblsBytesSentCount = stats.Int64("http/rbls_bytes_sent_count", "Counter of the number of bytes sent by bitswap since startup", stats.UnitBytes) + // bitswap BitswapRblsGetRequestCount = stats.Int64("bitswap/rbls_get_request_count", "Counter of RemoteBlockstore Get requests", stats.UnitDimensionless) BitswapRblsGetSuccessResponseCount = stats.Int64("bitswap/rbls_get_success_response_count", "Counter of successful RemoteBlockstore Get responses", stats.UnitDimensionless) diff --git a/node/builder.go b/node/builder.go index 34db6318a..7b8bb7369 100644 --- a/node/builder.go +++ b/node/builder.go @@ -44,10 +44,6 @@ import ( "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/filecoin-project/dagstore" "github.com/filecoin-project/go-address" - //"github.com/filecoin-project/go-fil-markets/retrievalmarket" - //rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - //lotus_storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" - //"github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" lotus_gfm_storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" lotus_api "github.com/filecoin-project/lotus/api" @@ -525,7 +521,7 @@ func ConfigBoost(cfg *config.Boost) Option { Override(new(*mpoolmonitor.MpoolMonitor), modules.NewMpoolMonitor(cfg)), // GraphQL server - Override(new(gql.BlockGetter), modules.NewBlockGetter), + Override(new(gql.BlockGetter), From(new(dtypes.IndexBackedBlockstore))), Override(new(*gql.Server), modules.NewGraphqlServer(cfg)), // Tracing diff --git a/node/impl/boost.go b/node/impl/boost.go index d9ad41870..5b2886f31 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -172,8 +172,8 @@ func (sm *BoostAPI) BoostIndexerListMultihashes(ctx context.Context, proposalCid } } -func (sm *BoostAPI) BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string) (*api.ProviderDealRejectionInfo, error) { - res, err := sm.StorageProvider.ImportOfflineDealData(ctx, dealUuid, filePath) +func (sm *BoostAPI) BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*api.ProviderDealRejectionInfo, error) { + res, err := sm.StorageProvider.ImportOfflineDealData(ctx, dealUuid, filePath, delAfterImport) return res, err } diff --git a/node/modules/graphsync.go b/node/modules/graphsync.go index 5f442fad2..e2c3dcf2b 100644 --- a/node/modules/graphsync.go +++ b/node/modules/graphsync.go @@ -3,12 +3,15 @@ package modules import ( "context" + "sync" + "time" + "github.com/filecoin-project/boost-gfm/retrievalmarket" retrievalimpl "github.com/filecoin-project/boost-gfm/retrievalmarket/impl" graphsync "github.com/filecoin-project/boost-graphsync/impl" gsnet "github.com/filecoin-project/boost-graphsync/network" "github.com/filecoin-project/boost-graphsync/storeutil" - "github.com/filecoin-project/boost/cmd/booster-bitswap/remoteblockstore" + "github.com/filecoin-project/boost/cmd/lib/remoteblockstore" "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/boost/node/modules/dtypes" "github.com/filecoin-project/boost/piecedirectory" @@ -20,7 +23,6 @@ import ( "github.com/libp2p/go-libp2p/core/host" "go.opencensus.io/stats" "go.uber.org/fx" - "time" ) var _ server.AskGetter = (*ProxyAskGetter)(nil) @@ -55,7 +57,8 @@ func SetAskGetter(proxy *ProxyAskGetter, rp retrievalmarket.RetrievalProvider) { // RetrievalGraphsync creates a graphsync instance used to serve retrievals. func RetrievalGraphsync(parallelTransfersForStorage uint64, parallelTransfersForStoragePerPeer uint64, parallelTransfersForRetrieval uint64) func(mctx lotus_helpers.MetricsCtx, lc fx.Lifecycle, pid *piecedirectory.PieceDirectory, h host.Host, net dtypes.ProviderTransferNetwork, dealDecider dtypes.RetrievalDealFilter, pstore dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, askGetter server.AskGetter) (*server.GraphsyncUnpaidRetrieval, error) { return func(mctx lotus_helpers.MetricsCtx, lc fx.Lifecycle, pid *piecedirectory.PieceDirectory, h host.Host, net dtypes.ProviderTransferNetwork, dealDecider dtypes.RetrievalDealFilter, pstore dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, askGetter server.AskGetter) (*server.GraphsyncUnpaidRetrieval, error) { - rb := remoteblockstore.NewRemoteBlockstore(pid) + // Graphsync tracks metrics separately, pass nothing to the remote blockstore + rb := remoteblockstore.NewRemoteBlockstore(pid, nil) // Create a Graphsync instance mkgs := Graphsync(parallelTransfersForStorage, parallelTransfersForStoragePerPeer, parallelTransfersForRetrieval) @@ -108,6 +111,7 @@ func Graphsync(parallelTransfersForStorage uint64, parallelTransfersForStoragePe } func graphsyncStats(mctx helpers.MetricsCtx, lc fx.Lifecycle, gs dtypes.Graphsync) { + var closeOnce sync.Once stopStats := make(chan struct{}) lc.Append(fx.Hook{ OnStart: func(context.Context) error { @@ -140,7 +144,7 @@ func graphsyncStats(mctx helpers.MetricsCtx, lc fx.Lifecycle, gs dtypes.Graphsyn return nil }, OnStop: func(ctx context.Context) error { - close(stopStats) + closeOnce.Do(func() { close(stopStats) }) return nil }, }) diff --git a/node/modules/legacy_markets.go b/node/modules/legacy_markets.go index 382e7ddca..09c3d99c7 100644 --- a/node/modules/legacy_markets.go +++ b/node/modules/legacy_markets.go @@ -2,6 +2,8 @@ package modules import ( "context" + "fmt" + "os" "path/filepath" piecefilestore "github.com/filecoin-project/boost-gfm/filestore" diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 8e9e209fe..aa6ea06ee 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -629,15 +629,15 @@ func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func( } } -func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer, ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode) *gql.Server { +func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer, ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode, bg gql.BlockGetter) *gql.Server { return func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer, - ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode) *gql.Server { + ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode, bg gql.BlockGetter) *gql.Server { resolverCtx, cancel := context.WithCancel(context.Background()) resolver := gql.NewResolver(resolverCtx, cfg, r, h, dealsDB, logsDB, retDB, plDB, fundsDB, fundMgr, storageMgr, spApi, prov, legacyProv, legacyDT, ps, sa, piecedirectory, publisher, fullNode) - server := gql.NewServer(resolver) + server := gql.NewServer(resolver, bg) lc.Append(fx.Hook{ OnStart: server.Start, diff --git a/react/src/Inspect.css b/react/src/Inspect.css index 6bf974f48..ad98e519a 100644 --- a/react/src/Inspect.css +++ b/react/src/Inspect.css @@ -104,3 +104,11 @@ display: inline-block; margin: 1em 0; } + +.inspect a.download { + background-image: url("./bootstrap-icons/icons/download.svg"); + background-position: left; + background-repeat: no-repeat; + padding-left: 1.25em; + margin-left: 1em; +} diff --git a/react/src/LID.js b/react/src/LID.js index 037326668..24150f5e1 100644 --- a/react/src/LID.js +++ b/react/src/LID.js @@ -615,16 +615,26 @@ function PieceStatus({pieceCid, pieceStatus, searchQuery}) { {searchIsAnyCid ? ( - Searched CID + Searched CID (non-root) {searchQuery} -   Download block ) : null} + {rootCid ? ( + + Data Root CID + + { searchIsRootCid ? {rootCid} : rootCid } + + Download block + + + + ) : null} Piece CID {searchIsPieceCid ? ( diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index f0a870e80..2d39f9eb3 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -2429,17 +2429,3 @@ func copyFile(source string, dest string) error { return nil } - -func mockTipset(minerAddr address.Address, height abi.ChainEpoch) (*chaintypes.TipSet, error) { - dummyCid, _ := cid.Parse("bafkqaaa") - return chaintypes.NewTipSet([]*chaintypes.BlockHeader{{ - Miner: minerAddr, - Height: height, - ParentStateRoot: dummyCid, - Messages: dummyCid, - ParentMessageReceipts: dummyCid, - BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, - BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, - Timestamp: 1, - }}) -}