Merge branch 'main' into ipip-379

.github/workflows/gateway-conformance.yml

@@ -61,3 +61,9 @@ jobs:
         with:
           name: gateway-conformance.html
           path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v3
+        with:
+          name: gateway-conformance.json
+          path: output.json

CHANGELOG.md

@@ -16,14 +16,32 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
-* The `routing/http` client and server now support Delegated IPNS as per [IPIP-379](https://specs.ipfs.tech/ipips/ipip-0379/).
+* The `routing/http` client and server now support Delegated IPNS at `/routing/v1`
+  as per [IPIP-379](https://specs.ipfs.tech/ipips/ipip-0379/).
+* The `verifycid` package has been updated with the new Allowlist interface as part of
+  reducing globals efforts. Still, existing global accessor funcs are kept for
+  backwards-compatibility.
+* The `blockservice` and `provider` packages has been updated to accommodate for 
+  changes in `verifycid`.
 
 ### Changed
 
+* 🛠 `blockservice.New` now accepts a variadic of func options following the [Functional
+  Options pattern](https://www.sohamkamani.com/golang/options-pattern/).
+
 ### Removed
 
 ### Fixed
 
+- HTTP Gateway API: Not having a block will result in a 5xx error rather than 404
+- HTTP Gateway API: CAR requests will return 200s and a CAR file proving a requested path does not exist rather than returning an error
+- 🛠 `MultiFileReader` has been updated with a new header with the encoded file name instead of the plain filename, due to a regression found in  [`net/textproto`](https://github.com/golang/go/issues/60674). This only affects files with binary characters in their name. By keeping the old header, we maximize backwards compatibility.
+  |            | New Client | Old Client  |
+  |------------|------------|-------------|
+  | New Server | ✅         | 🟡*         |
+  | Old Server | ✅         | ✅          |
+   *Old clients can only send Unicode file paths to the server.
+
 ### Security
 
 ## [v0.11.0]

bitswap/benchmarks_test.go

@@ -114,7 +114,7 @@ func BenchmarkFixedDelay(b *testing.B) {
 	}
 
 	out, _ := json.MarshalIndent(benchmarkLog, "", "  ")
-	_ = os.WriteFile("tmp/benchmark.json", out, 0666)
+	_ = os.WriteFile("tmp/benchmark.json", out, 0o666)
 	printResults(benchmarkLog)
 }
 
@@ -182,28 +182,30 @@ func BenchmarkFetchFromOldBitswap(b *testing.B) {
 	}
 
 	out, _ := json.MarshalIndent(benchmarkLog, "", "  ")
-	_ = os.WriteFile("tmp/benchmark.json", out, 0666)
+	_ = os.WriteFile("tmp/benchmark.json", out, 0o666)
 	printResults(benchmarkLog)
 }
 
-const datacenterSpeed = 5 * time.Millisecond
-const fastSpeed = 60 * time.Millisecond
-const mediumSpeed = 200 * time.Millisecond
-const slowSpeed = 800 * time.Millisecond
-const superSlowSpeed = 4000 * time.Millisecond
-const datacenterDistribution = 3 * time.Millisecond
-const distribution = 20 * time.Millisecond
-const datacenterBandwidth = 125000000.0
-const datacenterBandwidthDeviation = 3000000.0
-const fastBandwidth = 1250000.0
-const fastBandwidthDeviation = 300000.0
-const mediumBandwidth = 500000.0
-const mediumBandwidthDeviation = 80000.0
-const slowBandwidth = 100000.0
-const slowBandwidthDeviation = 16500.0
-const rootBlockSize = 800
-const stdBlockSize = 8000
-const largeBlockSize = int64(256 * 1024)
+const (
+	datacenterSpeed              = 5 * time.Millisecond
+	fastSpeed                    = 60 * time.Millisecond
+	mediumSpeed                  = 200 * time.Millisecond
+	slowSpeed                    = 800 * time.Millisecond
+	superSlowSpeed               = 4000 * time.Millisecond
+	datacenterDistribution       = 3 * time.Millisecond
+	distribution                 = 20 * time.Millisecond
+	datacenterBandwidth          = 125000000.0
+	datacenterBandwidthDeviation = 3000000.0
+	fastBandwidth                = 1250000.0
+	fastBandwidthDeviation       = 300000.0
+	mediumBandwidth              = 500000.0
+	mediumBandwidthDeviation     = 80000.0
+	slowBandwidth                = 100000.0
+	slowBandwidthDeviation       = 16500.0
+	rootBlockSize                = 800
+	stdBlockSize                 = 8000
+	largeBlockSize               = int64(256 * 1024)
+)
 
 func BenchmarkRealWorld(b *testing.B) {
 	benchmarkLog = nil
@@ -240,7 +242,7 @@ func BenchmarkRealWorld(b *testing.B) {
 		subtestDistributeAndFetchRateLimited(b, 300, 200, slowNetworkDelay, slowBandwidthGenerator, stdBlockSize, bstoreLatency, allToAll, batchFetchAll)
 	})
 	out, _ := json.MarshalIndent(benchmarkLog, "", "  ")
-	_ = os.WriteFile("tmp/rw-benchmark.json", out, 0666)
+	_ = os.WriteFile("tmp/rw-benchmark.json", out, 0o666)
 	printResults(benchmarkLog)
 }
 
@@ -263,7 +265,7 @@ func BenchmarkDatacenter(b *testing.B) {
 		subtestDistributeAndFetchRateLimited(b, 3, 100, datacenterNetworkDelay, datacenterBandwidthGenerator, largeBlockSize, bstoreLatency, allToAll, unixfsFileFetch)
 	})
 	out, _ := json.MarshalIndent(benchmarkLog, "", "  ")
-	_ = os.WriteFile("tmp/rb-benchmark.json", out, 0666)
+	_ = os.WriteFile("tmp/rb-benchmark.json", out, 0o666)
 	printResults(benchmarkLog)
 }
 
@@ -304,7 +306,7 @@ func BenchmarkDatacenterMultiLeechMultiSeed(b *testing.B) {
 	})
 
 	out, _ := json.MarshalIndent(benchmarkLog, "", "  ")
-	_ = os.WriteFile("tmp/rb-benchmark.json", out, 0666)
+	_ = os.WriteFile("tmp/rb-benchmark.json", out, 0o666)
 	printResults(benchmarkLog)
 }
 

bitswap/bitswap.go

@@ -44,9 +44,11 @@ type bitswap interface {
 	WantlistForPeer(p peer.ID) []cid.Cid
 }
 
-var _ exchange.SessionExchange = (*Bitswap)(nil)
-var _ bitswap = (*Bitswap)(nil)
-var HasBlockBufferSize = defaults.HasBlockBufferSize
+var (
+	_                  exchange.SessionExchange = (*Bitswap)(nil)
+	_                  bitswap                  = (*Bitswap)(nil)
+	HasBlockBufferSize                          = defaults.HasBlockBufferSize
+)
 
 type Bitswap struct {
 	*client.Client

bitswap/bitswap_test.go

@@ -390,7 +390,6 @@ func TestSendToWantingPeer(t *testing.T) {
 	if !blkrecvd.Cid().Equals(alpha.Cid()) {
 		t.Fatal("Wrong block!")
 	}
-
 }
 
 func TestEmptyKey(t *testing.T) {
@@ -828,6 +827,7 @@ func (tsl *testingScoreLedger) Start(scorePeer server.ScorePeerFunc) {
 	tsl.scorePeer = scorePeer
 	close(tsl.started)
 }
+
 func (tsl *testingScoreLedger) Stop() {
 	close(tsl.closed)
 }

bitswap/client/client.go

@@ -132,7 +132,8 @@ func New(parent context.Context, network bsnet.BitSwapNetwork, bstore blockstore
 		notif notifications.PubSub,
 		provSearchDelay time.Duration,
 		rebroadcastDelay delay.D,
-		self peer.ID) bssm.Session {
+		self peer.ID,
+	) bssm.Session {
 		return bssession.New(sessctx, sessmgr, id, spm, pqm, sim, pm, bpm, notif, provSearchDelay, rebroadcastDelay, self)
 	}
 	sessionPeerManagerFactory := func(ctx context.Context, id uint64) bssession.SessionPeerManager {

bitswap/client/internal/getter/getter.go

@@ -68,7 +68,8 @@ type WantFunc func(context.Context, []cid.Cid)
 // blocks, a want function, and a close function, and returns a channel of
 // incoming blocks.
 func AsyncGetBlocks(ctx context.Context, sessctx context.Context, keys []cid.Cid, notif notifications.PubSub,
-	want WantFunc, cwants func([]cid.Cid)) (<-chan blocks.Block, error) {
+	want WantFunc, cwants func([]cid.Cid),
+) (<-chan blocks.Block, error) {
 	ctx, span := internal.StartSpan(ctx, "Getter.AsyncGetBlocks")
 	defer span.End()
 
@@ -99,8 +100,8 @@ func AsyncGetBlocks(ctx context.Context, sessctx context.Context, keys []cid.Cid
 // If the context is cancelled or the incoming channel closes, calls cfun with
 // any keys corresponding to blocks that were never received.
 func handleIncoming(ctx context.Context, sessctx context.Context, remaining *cid.Set,
-	in <-chan blocks.Block, out chan blocks.Block, cfun func([]cid.Cid)) {
-
+	in <-chan blocks.Block, out chan blocks.Block, cfun func([]cid.Cid),
+) {
 	ctx, cancel := context.WithCancel(ctx)
 
 	// Clean up before exiting this function, and call the cancel function on

bitswap/client/internal/messagequeue/donthavetimeoutmgr.go

@@ -107,8 +107,8 @@ func newDontHaveTimeoutMgrWithParams(
 	messageLatencyMultiplier int,
 	maxExpectedWantProcessTime time.Duration,
 	clock clock.Clock,
-	timeoutsTriggered chan struct{}) *dontHaveTimeoutMgr {
-
+	timeoutsTriggered chan struct{},
+) *dontHaveTimeoutMgr {
 	ctx, shutdown := context.WithCancel(context.Background())
 	mqp := &dontHaveTimeoutMgr{
 		clock:                      clock,
@@ -222,7 +222,6 @@ func (dhtm *dontHaveTimeoutMgr) measurePingLatency() {
 // checkForTimeouts checks pending wants to see if any are over the timeout.
 // Note: this function should only be called within the lock.
 func (dhtm *dontHaveTimeoutMgr) checkForTimeouts() {
-
 	if len(dhtm.wantQueue) == 0 {
 		return
 	}

bitswap/client/internal/messagequeue/messagequeue.go

@@ -18,8 +18,10 @@ import (
 	"go.uber.org/zap"
 )
 
-var log = logging.Logger("bitswap")
-var sflog = log.Desugar()
+var (
+	log   = logging.Logger("bitswap")
+	sflog = log.Desugar()
+)
 
 const (
 	defaultRebroadcastInterval = 30 * time.Second
@@ -240,8 +242,8 @@ func newMessageQueue(
 	maxValidLatency time.Duration,
 	dhTimeoutMgr DontHaveTimeoutManager,
 	clock clock.Clock,
-	events chan messageEvent) *MessageQueue {
-
+	events chan messageEvent,
+) *MessageQueue {
 	ctx, cancel := context.WithCancel(ctx)
 	return &MessageQueue{
 		ctx:                 ctx,

bitswap/client/internal/messagequeue/messagequeue_test.go

@@ -61,6 +61,7 @@ func (fp *fakeDontHaveTimeoutMgr) AddPending(ks []cid.Cid) {
 	}
 	fp.ks = s.Keys()
 }
+
 func (fp *fakeDontHaveTimeoutMgr) CancelPending(ks []cid.Cid) {
 	fp.lk.Lock()
 	defer fp.lk.Unlock()
@@ -74,18 +75,21 @@ func (fp *fakeDontHaveTimeoutMgr) CancelPending(ks []cid.Cid) {
 	}
 	fp.ks = s.Keys()
 }
+
 func (fp *fakeDontHaveTimeoutMgr) UpdateMessageLatency(elapsed time.Duration) {
 	fp.lk.Lock()
 	defer fp.lk.Unlock()
 
 	fp.latencyUpds = append(fp.latencyUpds, elapsed)
 }
+
 func (fp *fakeDontHaveTimeoutMgr) latencyUpdates() []time.Duration {
 	fp.lk.Lock()
 	defer fp.lk.Unlock()
 
 	return fp.latencyUpds
 }
+
 func (fp *fakeDontHaveTimeoutMgr) pendingCount() int {
 	fp.lk.Lock()
 	defer fp.lk.Unlock()
@@ -101,8 +105,8 @@ type fakeMessageSender struct {
 }
 
 func newFakeMessageSender(reset chan<- struct{},
-	messagesSent chan<- []bsmsg.Entry, supportsHave bool) *fakeMessageSender {
-
+	messagesSent chan<- []bsmsg.Entry, supportsHave bool,
+) *fakeMessageSender {
 	return &fakeMessageSender{
 		reset:        reset,
 		messagesSent: messagesSent,
@@ -126,7 +130,8 @@ func mockTimeoutCb(peer.ID, []cid.Cid) {}
 func collectMessages(ctx context.Context,
 	t *testing.T,
 	messagesSent <-chan []bsmsg.Entry,
-	timeout time.Duration) [][]bsmsg.Entry {
+	timeout time.Duration,
+) [][]bsmsg.Entry {
 	var messagesReceived [][]bsmsg.Entry
 	timeoutctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()

bitswap/client/internal/notifications/notifications.go

@@ -68,7 +68,6 @@ func (ps *impl) Shutdown() {
 // is closed if the |ctx| times out or is cancelled, or after receiving the blocks
 // corresponding to |keys|.
 func (ps *impl) Subscribe(ctx context.Context, keys ...cid.Cid) <-chan blocks.Block {
-
 	blocksCh := make(chan blocks.Block, len(keys))
 	valuesCh := make(chan interface{}, len(keys)) // provide our own channel to control buffer, prevent blocking
 	if len(keys) == 0 {

bitswap/client/internal/notifications/notifications_test.go

@@ -58,7 +58,6 @@ func TestPublishSubscribe(t *testing.T) {
 	}
 
 	assertBlocksEqual(t, blockRecvd, blockSent)
-
 }
 
 func TestSubscribeMany(t *testing.T) {

bitswap/client/internal/peermanager/peermanager_test.go

@@ -30,12 +30,15 @@ func (fp *mockPeerQueue) Shutdown() {}
 func (fp *mockPeerQueue) AddBroadcastWantHaves(whs []cid.Cid) {
 	fp.msgs <- msg{fp.p, nil, whs, nil}
 }
+
 func (fp *mockPeerQueue) AddWants(wbs []cid.Cid, whs []cid.Cid) {
 	fp.msgs <- msg{fp.p, wbs, whs, nil}
 }
+
 func (fp *mockPeerQueue) AddCancels(cs []cid.Cid) {
 	fp.msgs <- msg{fp.p, nil, nil, cs}
 }
+
 func (fp *mockPeerQueue) ResponseReceived(ks []cid.Cid) {
 }
 
@@ -271,6 +274,7 @@ func TestSendCancels(t *testing.T) {
 func (s *sess) ID() uint64 {
 	return s.id
 }
+
 func (s *sess) SignalAvailability(p peer.ID, isAvailable bool) {
 	s.available[p] = isAvailable
 }
@@ -332,8 +336,7 @@ func TestSessionRegistration(t *testing.T) {
 	}
 }
 
-type benchPeerQueue struct {
-}
+type benchPeerQueue struct{}
 
 func (*benchPeerQueue) Startup()  {}
 func (*benchPeerQueue) Shutdown() {}

bitswap/client/internal/peermanager/peerwantmanager_test.go

@@ -16,6 +16,7 @@ type gauge struct {
 func (g *gauge) Inc() {
 	g.count++
 }
+
 func (g *gauge) Dec() {
 	g.count--
 }
@@ -40,13 +41,16 @@ func (mpq *mockPQ) Shutdown() {}
 func (mpq *mockPQ) AddBroadcastWantHaves(whs []cid.Cid) {
 	mpq.bcst = append(mpq.bcst, whs...)
 }
+
 func (mpq *mockPQ) AddWants(wbs []cid.Cid, whs []cid.Cid) {
 	mpq.wbs = append(mpq.wbs, wbs...)
 	mpq.whs = append(mpq.whs, whs...)
 }
+
 func (mpq *mockPQ) AddCancels(cs []cid.Cid) {
 	mpq.cancels = append(mpq.cancels, cs...)
 }
+
 func (mpq *mockPQ) ResponseReceived(ks []cid.Cid) {
 }
 

bitswap/client/internal/providerquerymanager/providerquerymanager_test.go

@@ -95,7 +95,6 @@ func TestNormalSimultaneousFetch(t *testing.T) {
 	if fpn.queriesMade != 2 {
 		t.Fatal("Did not dedup provider requests running simultaneously")
 	}
-
 }
 
 func TestDedupingProviderRequests(t *testing.T) {
@@ -256,7 +255,6 @@ func TestPeersWithConnectionErrorsNotAddedToPeerList(t *testing.T) {
 	if len(firstPeersReceived) != 0 || len(secondPeersReceived) != 0 {
 		t.Fatal("Did not filter out peers with connection issues")
 	}
-
 }
 
 func TestRateLimitingRequests(t *testing.T) {