From 23dafa122b1ba64251755fbf7377983b57edb748 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 19 Dec 2023 11:32:56 -0800 Subject: [PATCH 1/2] Retry fetch request when libp2p stream is reset The libp2p stream may be reset for a number of reasons, such as network error, rate limiting, etc. Often a subsequent request will succeed. This change allows each fetch request to retry once upon receiving stream reset error. --- dagsync/ipnisync/sync.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 3bc497c..8b4feed 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -24,6 +24,7 @@ import ( headschema "github.com/ipni/go-libipni/dagsync/ipnisync/head" "github.com/ipni/go-libipni/maurl" "github.com/ipni/go-libipni/mautil" + "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/multiformats/go-multihash" @@ -190,7 +191,7 @@ func (s *Syncer) GetHead(ctx context.Context) (cid.Cid, error) { if s.peerID == "" { log.Warn("Cannot verify publisher signature without peer ID") } else if signerID != s.peerID { - return cid.Undef, errors.New("found head signed by an unexpected peer") + return cid.Undef, fmt.Errorf("found head signed by an unexpected peer, peerID: %s, signed-by: %s", s.peerID.String(), signerID.String()) } // TODO: Check that the returned topic, if any, matches the expected topic. @@ -284,6 +285,8 @@ func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Se func (s *Syncer) fetch(ctx context.Context, rsrc string, cb func(io.Reader) error) error { nextURL: fetchURL := s.rootURL.JoinPath(rsrc) + var doneRetry bool +retry: req, err := http.NewRequestWithContext(ctx, "GET", fetchURL.String(), nil) if err != nil { return err @@ -300,6 +303,12 @@ nextURL: } goto nextURL } + if !doneRetry && errors.Is(err, network.ErrReset) { + log.Errorw("stream reset err, retrying", "publisher", s.peerID, "url", fetchURL.String()) + // Only retry the same fetch once. + doneRetry = true + goto retry + } return fmt.Errorf("fetch request failed: %w", err) } defer resp.Body.Close() From 540466e6fb165453715bf377aff5eb3c9b2f0e1a Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 19 Dec 2023 11:36:20 -0800 Subject: [PATCH 2/2] Update libp2p to get security fix --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 51cc49c..0b3ce2c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.21.0 - github.com/libp2p/go-libp2p v0.32.1 + github.com/libp2p/go-libp2p v0.32.2 github.com/libp2p/go-libp2p-pubsub v0.10.0 github.com/libp2p/go-msgio v0.3.0 github.com/mr-tron/base58 v1.2.0 @@ -112,7 +112,7 @@ require ( github.com/prometheus/procfs v0.9.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.3.4 // indirect - github.com/quic-go/quic-go v0.39.3 // indirect + github.com/quic-go/quic-go v0.39.4 // indirect github.com/quic-go/webtransport-go v0.6.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index 48d391f..f9b9d54 100644 --- a/go.sum +++ b/go.sum @@ -300,8 +300,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.32.1 h1:wy1J4kZIZxOaej6NveTWCZmHiJ/kY7GoAqXgqNCnPps= -github.com/libp2p/go-libp2p v0.32.1/go.mod h1:hXXC3kXPlBZ1eu8Q2hptGrMB4mZ3048JUoS4EKaHW5c= +github.com/libp2p/go-libp2p v0.32.2 h1:s8GYN4YJzgUoyeYNPdW7JZeZ5Ee31iNaIBfGYMAY4FQ= +github.com/libp2p/go-libp2p v0.32.2/go.mod h1:E0LKe+diV/ZVJVnOJby8VC5xzHF0660osg71skcxJvk= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-pubsub v0.10.0 h1:wS0S5FlISavMaAbxyQn3dxMOe2eegMfswM471RuHJwA= @@ -423,8 +423,8 @@ github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNVm7Cg= github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= -github.com/quic-go/quic-go v0.39.3 h1:o3YB6t2SR+HU/pgwF29kJ6g4jJIJEwEZ8CKia1h1TKg= -github.com/quic-go/quic-go v0.39.3/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q= +github.com/quic-go/quic-go v0.39.4 h1:PelfiuG7wXEffUT2yceiqz5V6Pc0TA5ruOd1LcmFc1s= +github.com/quic-go/quic-go v0.39.4/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q= github.com/quic-go/webtransport-go v0.6.0 h1:CvNsKqc4W2HljHJnoT+rMmbRJybShZ0YPFDD3NxaZLY= github.com/quic-go/webtransport-go v0.6.0/go.mod h1:9KjU4AEBqEQidGHNDkZrb8CAa1abRaosM2yGOyiikEc= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk=