From 06155c535c1a44624988687396d2ee91c11a914e Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Thu, 15 Aug 2024 12:25:44 +0200
Subject: [PATCH 01/14] feat: add check using only a CID

Fixes #6
---
 daemon.go | 109 ++++++++++++++++++++++++++++++++++++++++--------------
 main.go   |  19 +++++++++-
 2 files changed, 100 insertions(+), 28 deletions(-)

diff --git a/daemon.go b/daemon.go
index c922595..2a5a525 100644
--- a/daemon.go
+++ b/daemon.go
@@ -2,10 +2,8 @@ package main
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"log"
-	"net/url"
 	"sync"
 	"time"
 
@@ -38,6 +36,10 @@ type daemon struct {
 	createTestHost func() (host.Host, error)
 }
 
+// number of providers at which to stop looking for providers in the DHT
+// When doing a check only with a CID
+var MaxProvidersCount = 3
+
 func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) {
 	rm, err := NewResourceManager()
 	if err != nil {
@@ -108,18 +110,79 @@ func (d *daemon) mustStart() {
 
 }
 
-func (d *daemon) runCheck(query url.Values) (*output, error) {
-	maStr := query.Get("multiaddr")
-	cidStr := query.Get("cid")
+type providerOutput struct {
+	ID                 string
+	Addrs              []string
+	ConnectionMaddrs   []string
+	BitswapCheckOutput BitswapCheckOutput
+}
 
-	if maStr == "" {
-		return nil, errors.New("missing 'multiaddr' argument")
+func (d *daemon) runCidCheck(cidStr string) (*[]providerOutput, error) {
+	cid, err := cid.Decode(cidStr)
+	if err != nil {
+		return nil, err
 	}
 
-	if cidStr == "" {
-		return nil, errors.New("missing 'cid' argument")
+	ctx := context.Background()
+	out := make([]providerOutput, 0, 3)
+
+	queryCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	provsCh := d.dht.FindProvidersAsync(queryCtx, cid, MaxProvidersCount)
+
+	for provider := range provsCh {
+		addrs := make([]string, len(provider.Addrs))
+		for i, addr := range provider.Addrs {
+			addrs[i] = addr.String()
+		}
+
+		provOutput := providerOutput{
+			ID:                 provider.ID.String(),
+			Addrs:              addrs,
+			BitswapCheckOutput: BitswapCheckOutput{},
+		}
+
+		testHost, err := d.createTestHost()
+		if err != nil {
+			return nil, fmt.Errorf("server error: %w", err)
+		}
+		defer testHost.Close()
+
+		// Test Is the target connectable
+		dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15)
+
+		// we call NewStream to force NAT hole punching
+		// See https://github.com/libp2p/go-libp2p/issues/2714
+		testHost.Connect(dialCtx, provider)
+		_, connErr := testHost.NewStream(dialCtx, provider.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap")
+		dialCancel()
+
+		if connErr != nil {
+			provOutput.BitswapCheckOutput.Error = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
+		} else {
+			// TODO: Modify checkBitswapCID and vole to accept `AddrInfo` so that it can test any of the connections
+			provOutput.BitswapCheckOutput = checkBitswapCID(ctx, testHost, cid, provider.Addrs[0])
+
+			for _, c := range testHost.Network().ConnsToPeer(provider.ID) {
+				provOutput.ConnectionMaddrs = append(provOutput.ConnectionMaddrs, c.RemoteMultiaddr().String())
+			}
+		}
+
+		out = append(out, provOutput)
 	}
 
+	return &out, nil
+}
+
+type peerCheckOutput struct {
+	ConnectionError          string
+	PeerFoundInDHT           map[string]int
+	CidInDHT                 bool
+	ConnectionMaddrs         []string
+	DataAvailableOverBitswap BitswapCheckOutput
+}
+
+func (d *daemon) runPeerCheck(maStr, cidStr string) (*peerCheckOutput, error) {
 	ma, err := multiaddr.NewMultiaddr(maStr)
 	if err != nil {
 		return nil, err
@@ -139,11 +202,11 @@ func (d *daemon) runCheck(query url.Values) (*output, error) {
 	}
 
 	ctx := context.Background()
-	out := &output{}
+	out := &peerCheckOutput{}
 
 	connectionFailed := false
 
-	out.CidInDHT = providerRecordInDHT(ctx, d.dht, c, ai.ID)
+	out.CidInDHT = providerRecordForPeerInDHT(ctx, d.dht, c, ai.ID)
 
 	addrMap, peerAddrDHTErr := peerAddrsInDHT(ctx, d.dht, d.dhtMessenger, ai.ID)
 	out.PeerFoundInDHT = addrMap
@@ -202,6 +265,13 @@ func (d *daemon) runCheck(query url.Values) (*output, error) {
 	return out, nil
 }
 
+type BitswapCheckOutput struct {
+	Duration  time.Duration
+	Found     bool
+	Responded bool
+	Error     string
+}
+
 func checkBitswapCID(ctx context.Context, host host.Host, c cid.Cid, ma multiaddr.Multiaddr) BitswapCheckOutput {
 	log.Printf("Start of Bitswap check for cid %s by attempting to connect to ma: %v with the temporary peer: %s", c, ma, host.ID())
 	out := BitswapCheckOutput{}
@@ -223,21 +293,6 @@ func checkBitswapCID(ctx context.Context, host host.Host, c cid.Cid, ma multiadd
 	return out
 }
 
-type BitswapCheckOutput struct {
-	Duration  time.Duration
-	Found     bool
-	Responded bool
-	Error     string
-}
-
-type output struct {
-	ConnectionError          string
-	PeerFoundInDHT           map[string]int
-	CidInDHT                 bool
-	ConnectionMaddrs         []string
-	DataAvailableOverBitswap BitswapCheckOutput
-}
-
 func peerAddrsInDHT(ctx context.Context, d kademlia, messenger *dhtpb.ProtocolMessenger, p peer.ID) (map[string]int, error) {
 	closestPeers, err := d.GetClosestPeers(ctx, string(p))
 	if err != nil {
@@ -281,7 +336,7 @@ func peerAddrsInDHT(ctx context.Context, d kademlia, messenger *dhtpb.ProtocolMe
 	return addrMap, nil
 }
 
-func providerRecordInDHT(ctx context.Context, d kademlia, c cid.Cid, p peer.ID) bool {
+func providerRecordForPeerInDHT(ctx context.Context, d kademlia, c cid.Cid, p peer.ID) bool {
 	queryCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	provsCh := d.FindProvidersAsync(queryCtx, c, 0)
diff --git a/main.go b/main.go
index c330cb0..1f85045 100644
--- a/main.go
+++ b/main.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/subtle"
 	"encoding/json"
+	"errors"
 	"log"
 	"net"
 	"net/http"
@@ -77,7 +78,23 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m
 
 	checkHandler := func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Add("Access-Control-Allow-Origin", "*")
-		data, err := d.runCheck(r.URL.Query())
+
+		maStr := r.URL.Query().Get("multiaddr")
+		cidStr := r.URL.Query().Get("cid")
+
+		if cidStr == "" {
+			err = errors.New("missing 'cid' argument")
+		}
+
+		var err error
+		var data interface{}
+
+		if maStr == "" {
+			data, err = d.runCidCheck(cidStr)
+		} else {
+			data, err = d.runPeerCheck(maStr, cidStr)
+		}
+
 		if err == nil {
 			w.Header().Add("Content-Type", "application/json")
 			_ = json.NewEncoder(w).Encode(data)

From cdb4edc9d2bb25e01acb3c767bb11991d75e96b9 Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Mon, 26 Aug 2024 16:47:42 +0200
Subject: [PATCH 02/14] feat: run checks concurrently

---
 daemon.go | 74 ++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 43 insertions(+), 31 deletions(-)

diff --git a/daemon.go b/daemon.go
index 2a5a525..a0c6758 100644
--- a/daemon.go
+++ b/daemon.go
@@ -124,53 +124,65 @@ func (d *daemon) runCidCheck(cidStr string) (*[]providerOutput, error) {
 	}
 
 	ctx := context.Background()
-	out := make([]providerOutput, 0, 3)
+	out := make([]providerOutput, 0, MaxProvidersCount)
 
 	queryCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	provsCh := d.dht.FindProvidersAsync(queryCtx, cid, MaxProvidersCount)
 
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+
 	for provider := range provsCh {
-		addrs := make([]string, len(provider.Addrs))
-		for i, addr := range provider.Addrs {
-			addrs[i] = addr.String()
-		}
+		wg.Add(1)
+		go func(provider peer.AddrInfo) {
+			defer wg.Done()
 
-		provOutput := providerOutput{
-			ID:                 provider.ID.String(),
-			Addrs:              addrs,
-			BitswapCheckOutput: BitswapCheckOutput{},
-		}
+			addrs := make([]string, len(provider.Addrs))
+			for i, addr := range provider.Addrs {
+				addrs[i] = addr.String()
+			}
 
-		testHost, err := d.createTestHost()
-		if err != nil {
-			return nil, fmt.Errorf("server error: %w", err)
-		}
-		defer testHost.Close()
+			provOutput := providerOutput{
+				ID:                 provider.ID.String(),
+				Addrs:              addrs,
+				BitswapCheckOutput: BitswapCheckOutput{},
+			}
 
-		// Test Is the target connectable
-		dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15)
+			testHost, err := d.createTestHost()
+			if err != nil {
+				log.Printf("Error creating test host: %v", err)
+				return
+			}
+			defer testHost.Close()
 
-		// we call NewStream to force NAT hole punching
-		// See https://github.com/libp2p/go-libp2p/issues/2714
-		testHost.Connect(dialCtx, provider)
-		_, connErr := testHost.NewStream(dialCtx, provider.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap")
-		dialCancel()
+			// Test Is the target connectable
+			dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15)
+			defer dialCancel()
 
-		if connErr != nil {
-			provOutput.BitswapCheckOutput.Error = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
-		} else {
-			// TODO: Modify checkBitswapCID and vole to accept `AddrInfo` so that it can test any of the connections
-			provOutput.BitswapCheckOutput = checkBitswapCID(ctx, testHost, cid, provider.Addrs[0])
+			// we call NewStream to force NAT hole punching
+			// See https://github.com/libp2p/go-libp2p/issues/2714
+			testHost.Connect(dialCtx, provider)
+			_, connErr := testHost.NewStream(dialCtx, provider.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap")
 
-			for _, c := range testHost.Network().ConnsToPeer(provider.ID) {
-				provOutput.ConnectionMaddrs = append(provOutput.ConnectionMaddrs, c.RemoteMultiaddr().String())
+			if connErr != nil {
+				provOutput.BitswapCheckOutput.Error = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
+			} else {
+				// TODO: Modify checkBitswapCID and vole to accept `AddrInfo` so that it can test any of the connections
+				provOutput.BitswapCheckOutput = checkBitswapCID(ctx, testHost, cid, provider.Addrs[0])
+
+				for _, c := range testHost.Network().ConnsToPeer(provider.ID) {
+					provOutput.ConnectionMaddrs = append(provOutput.ConnectionMaddrs, c.RemoteMultiaddr().String())
+				}
 			}
-		}
 
-		out = append(out, provOutput)
+			mu.Lock()
+			out = append(out, provOutput)
+			mu.Unlock()
+		}(provider)
 	}
 
+	wg.Wait()
 	return &out, nil
 }
 

From 99c8ffd4b3ea364a8a2f65db4608d8dafad3363b Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Mon, 26 Aug 2024 21:44:43 +0200
Subject: [PATCH 03/14] fix: use idiomatic approach with ctx termination

---
 daemon.go | 89 ++++++++++++++++++++++++++++++-------------------------
 1 file changed, 48 insertions(+), 41 deletions(-)

diff --git a/daemon.go b/daemon.go
index a0c6758..8c3785e 100644
--- a/daemon.go
+++ b/daemon.go
@@ -133,57 +133,64 @@ func (d *daemon) runCidCheck(cidStr string) (*[]providerOutput, error) {
 	var wg sync.WaitGroup
 	var mu sync.Mutex
 
-	for provider := range provsCh {
-		wg.Add(1)
-		go func(provider peer.AddrInfo) {
-			defer wg.Done()
-
-			addrs := make([]string, len(provider.Addrs))
-			for i, addr := range provider.Addrs {
-				addrs[i] = addr.String()
+	for {
+		select {
+		case provider, ok := <-provsCh:
+			if !ok {
+				// Channel closed, all providers processed
+				return &out, nil
 			}
+			wg.Add(1)
+			go func(provider peer.AddrInfo) {
+				defer wg.Done()
 
-			provOutput := providerOutput{
-				ID:                 provider.ID.String(),
-				Addrs:              addrs,
-				BitswapCheckOutput: BitswapCheckOutput{},
-			}
+				addrs := make([]string, len(provider.Addrs))
+				for i, addr := range provider.Addrs {
+					addrs[i] = addr.String()
+				}
 
-			testHost, err := d.createTestHost()
-			if err != nil {
-				log.Printf("Error creating test host: %v", err)
-				return
-			}
-			defer testHost.Close()
+				provOutput := providerOutput{
+					ID:                 provider.ID.String(),
+					Addrs:              addrs,
+					BitswapCheckOutput: BitswapCheckOutput{},
+				}
 
-			// Test Is the target connectable
-			dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15)
-			defer dialCancel()
+				testHost, err := d.createTestHost()
+				if err != nil {
+					log.Printf("Error creating test host: %v", err)
+					return
+				}
+				defer testHost.Close()
 
-			// we call NewStream to force NAT hole punching
-			// See https://github.com/libp2p/go-libp2p/issues/2714
-			testHost.Connect(dialCtx, provider)
-			_, connErr := testHost.NewStream(dialCtx, provider.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap")
+				// Test Is the target connectable
+				dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15)
+				defer dialCancel()
 
-			if connErr != nil {
-				provOutput.BitswapCheckOutput.Error = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
-			} else {
-				// TODO: Modify checkBitswapCID and vole to accept `AddrInfo` so that it can test any of the connections
-				provOutput.BitswapCheckOutput = checkBitswapCID(ctx, testHost, cid, provider.Addrs[0])
+				// we call NewStream to force NAT hole punching
+				// See https://github.com/libp2p/go-libp2p/issues/2714
+				testHost.Connect(dialCtx, provider)
+				_, connErr := testHost.NewStream(dialCtx, provider.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap")
 
-				for _, c := range testHost.Network().ConnsToPeer(provider.ID) {
-					provOutput.ConnectionMaddrs = append(provOutput.ConnectionMaddrs, c.RemoteMultiaddr().String())
+				if connErr != nil {
+					provOutput.BitswapCheckOutput.Error = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
+				} else {
+					// TODO: Modify checkBitswapCID and vole to accept `AddrInfo` so that it can test any of the connections
+					provOutput.BitswapCheckOutput = checkBitswapCID(ctx, testHost, cid, provider.Addrs[0])
+
+					for _, c := range testHost.Network().ConnsToPeer(provider.ID) {
+						provOutput.ConnectionMaddrs = append(provOutput.ConnectionMaddrs, c.RemoteMultiaddr().String())
+					}
 				}
-			}
 
-			mu.Lock()
-			out = append(out, provOutput)
-			mu.Unlock()
-		}(provider)
+				mu.Lock()
+				out = append(out, provOutput)
+				mu.Unlock()
+			}(provider)
+		case <-ctx.Done():
+			// Context cancelled
+			return &out, ctx.Err()
+		}
 	}
-
-	wg.Wait()
-	return &out, nil
 }
 
 type peerCheckOutput struct {

From ac2e96d12c7264d99337435fddcb2d3ea1f7d0eb Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Mon, 26 Aug 2024 22:21:16 +0200
Subject: [PATCH 04/14] fix: rename output field

and simplify code
fixes #43
---
 README.md           | 14 +++++++-------
 daemon.go           | 31 +++++++++++++------------------
 integration_test.go |  6 +++---
 test/e2e_test.go    | 10 +++++-----
 test/tools.go       |  2 +-
 web/index.html      |  2 +-
 6 files changed, 30 insertions(+), 35 deletions(-)

diff --git a/README.md b/README.md
index 6337cdb..fe2e1be 100644
--- a/README.md
+++ b/README.md
@@ -75,11 +75,11 @@ The server performs several checks given a CID. The results of the check are exp
 
 ```go
 type output struct {
-	ConnectionError          string
-	PeerFoundInDHT           map[string]int
-	CidInDHT                 bool
-	ConnectionMaddrs         string[]
-	DataAvailableOverBitswap BitswapCheckOutput
+	ConnectionError             string
+	PeerFoundInDHT              map[string]int
+	ProviderRecordFromPeerInDHT bool
+	ConnectionMaddrs            []string
+	DataAvailableOverBitswap    BitswapCheckOutput
 }
 
 type BitswapCheckOutput struct {
@@ -90,9 +90,9 @@ type BitswapCheckOutput struct {
 }
 ```
 
-1. Is the CID (really multihash) advertised in the DHT (or later IPNI)?
+1. Is the CID (really multihash) advertised in the DHT by the Passed PeerID (or later IPNI)?
 
-- `CidInDHT`
+- `ProviderRecordFromPeerInDHT`
 
 2. Are the peer's addresses discoverable (particularly useful if the announcements are DHT based, but also independently useful)
 
diff --git a/daemon.go b/daemon.go
index e16ab36..4d3819a 100644
--- a/daemon.go
+++ b/daemon.go
@@ -144,7 +144,7 @@ func (d *daemon) runCheck(query url.Values) (*output, error) {
 
 	connectionFailed := false
 
-	out.CidInDHT = providerRecordInDHT(ctx, d.dht, c, ai.ID)
+	out.ProviderRecordFromPeerInDHT = ProviderRecordFromPeerInDHT(ctx, d.dht, c, ai.ID)
 
 	addrMap, peerAddrDHTErr := peerAddrsInDHT(ctx, d.dht, d.dhtMessenger, ai.ID)
 	out.PeerFoundInDHT = addrMap
@@ -183,21 +183,16 @@ func (d *daemon) runCheck(query url.Values) (*output, error) {
 		dialCancel()
 		if connErr != nil {
 			out.ConnectionError = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
-			connectionFailed = true
+			return out, nil
 		}
 	}
 
-	if connectionFailed {
-		out.DataAvailableOverBitswap.Error = "could not connect to peer"
-	} else {
-		// If so is the data available over Bitswap?
-		out.DataAvailableOverBitswap = checkBitswapCID(ctx, testHost, c, ma)
+	// If so is the data available over Bitswap?
+	out.DataAvailableOverBitswap = checkBitswapCID(ctx, testHost, c, ma)
 
-		// Get the direct connection in case it was hole punched and we have both a limited connection
-		// directMaddr := getDirectMaddr()
-		for _, c := range testHost.Network().ConnsToPeer(ai.ID) {
-			out.ConnectionMaddrs = append(out.ConnectionMaddrs, c.RemoteMultiaddr().String())
-		}
+	// Get all connection maddrs to the peer (in case we hole punched, there will usually be two: limited relay and direct)
+	for _, c := range testHost.Network().ConnsToPeer(ai.ID) {
+		out.ConnectionMaddrs = append(out.ConnectionMaddrs, c.RemoteMultiaddr().String())
 	}
 
 	return out, nil
@@ -232,11 +227,11 @@ type BitswapCheckOutput struct {
 }
 
 type output struct {
-	ConnectionError          string
-	PeerFoundInDHT           map[string]int
-	CidInDHT                 bool
-	ConnectionMaddrs         []string
-	DataAvailableOverBitswap BitswapCheckOutput
+	ConnectionError             string
+	PeerFoundInDHT              map[string]int
+	ProviderRecordFromPeerInDHT bool
+	ConnectionMaddrs            []string
+	DataAvailableOverBitswap    BitswapCheckOutput
 }
 
 func peerAddrsInDHT(ctx context.Context, d kademlia, messenger *dhtpb.ProtocolMessenger, p peer.ID) (map[string]int, error) {
@@ -282,7 +277,7 @@ func peerAddrsInDHT(ctx context.Context, d kademlia, messenger *dhtpb.ProtocolMe
 	return addrMap, nil
 }
 
-func providerRecordInDHT(ctx context.Context, d kademlia, c cid.Cid, p peer.ID) bool {
+func ProviderRecordFromPeerInDHT(ctx context.Context, d kademlia, c cid.Cid, p peer.ID) bool {
 	queryCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	provsCh := d.FindProvidersAsync(queryCtx, c, 0)
diff --git a/integration_test.go b/integration_test.go
index 6dd2da0..fbb341a 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -112,7 +112,7 @@ func TestBasicIntegration(t *testing.T) {
 
 		obj := test.Query(t, "http://localhost:1234", testCid.String(), hostAddr.String())
 
-		obj.Value("CidInDHT").Boolean().IsTrue()
+		obj.Value("ProviderRecordFromPeerInDHT").Boolean().IsTrue()
 		obj.Value("ConnectionError").String().IsEmpty()
 		obj.Value("ConnectionMaddrs").Array().ContainsAll(h.Addrs()[0])
 		obj.Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty()
@@ -132,7 +132,7 @@ func TestBasicIntegration(t *testing.T) {
 
 		obj := test.Query(t, "http://localhost:1234", testCid.String(), hostAddr.String())
 
-		obj.Value("CidInDHT").Boolean().IsFalse()
+		obj.Value("ProviderRecordFromPeerInDHT").Boolean().IsFalse()
 		obj.Value("ConnectionError").String().IsEmpty()
 		obj.Value("ConnectionMaddrs").Array().ContainsAll(h.Addrs()[0])
 		obj.Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty()
@@ -150,7 +150,7 @@ func TestBasicIntegration(t *testing.T) {
 
 		obj := test.Query(t, "http://localhost:1234", testCid.String(), hostAddr.String())
 
-		obj.Value("CidInDHT").Boolean().IsTrue()
+		obj.Value("ProviderRecordFromPeerInDHT").Boolean().IsTrue()
 		obj.Value("ConnectionError").String().IsEmpty()
 		obj.Value("ConnectionMaddrs").Array().ContainsAll(h.Addrs()[0])
 		obj.Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty()
diff --git a/test/e2e_test.go b/test/e2e_test.go
index a05a5cd..1fdcd0d 100644
--- a/test/e2e_test.go
+++ b/test/e2e_test.go
@@ -39,7 +39,7 @@ func TestEmptyDirOnBoostrapPeer(t *testing.T) {
 	}
 	obj := Q(t, EMPTY_DIR_CID, BOOTSTRAP_PEER_ADDR)
 
-	obj.Value("CidInDHT").Boolean().IsTrue()
+	obj.Value("ProviderRecordFromPeerInDHT").Boolean().IsTrue()
 	obj.Value("ConnectionError").String().IsEmpty()
 	obj.Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty()
 	obj.Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsTrue()
@@ -51,7 +51,7 @@ func TestWikipediaOnSomeProviderPeer(t *testing.T) {
 		t.Skip("Skipping e2e tests")
 	}
 	obj := Q(t, WIKIPEDIA_CID, WIKIPEDIA_PEER_ADDR)
-	obj.Value("CidInDHT").Boolean().IsTrue()
+	obj.Value("ProviderRecordFromPeerInDHT").Boolean().IsTrue()
 	// It seems that most peers do not provide over bitswap:
 	// obj.Value("ConnectionError").String().IsEmpty()
 	// obj.Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty()
@@ -72,7 +72,7 @@ func TestRandomFileOnBootstrapPeer(t *testing.T) {
 			time.Sleep(60 * time.Second)
 			obj := Q(t, randomFileCid, BOOTSTRAP_PEER_ADDR)
 
-			obj.Value("CidInDHT").Boolean().IsTrue()
+			obj.Value("ProviderRecordFromPeerInDHT").Boolean().IsTrue()
 			obj.Value("ConnectionError").String().IsEmpty()
 			obj.Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty()
 			obj.Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsTrue()
@@ -97,7 +97,7 @@ func TestRandomFileOnLocalPeer(t *testing.T) {
 			time.Sleep(25 * time.Second)
 			obj := Q(t, randomFileCid, localAddr)
 
-			obj.Value("CidInDHT").Boolean().IsTrue()
+			obj.Value("ProviderRecordFromPeerInDHT").Boolean().IsTrue()
 			obj.Value("ConnectionError").String().IsEmpty()
 			obj.Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty()
 			obj.Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsTrue()
@@ -115,7 +115,7 @@ func TestRandomFileNeverUploadedOnBootstrapPeer(t *testing.T) {
 
 	obj := Q(t, randomFileCid, BOOTSTRAP_PEER_ADDR)
 
-	obj.Value("CidInDHT").Boolean().IsFalse()
+	obj.Value("ProviderRecordFromPeerInDHT").Boolean().IsFalse()
 	obj.Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsFalse()
 	obj.Value("DataAvailableOverBitswap").Object().Value("Responded").Boolean().IsTrue()
 }
diff --git a/test/tools.go b/test/tools.go
index 39cc275..dd4b4bf 100644
--- a/test/tools.go
+++ b/test/tools.go
@@ -62,7 +62,7 @@ func GetEnv(key string, fallback string) string {
 Example of outputs:
 ```json
 {
-	"CidInDHT": true,
+	"ProviderRecordFromPeerInDHT": true,
 	"ConnectionError": "no addresses",
 	"DataAvailableOverBitswap": {
 		"Duration": 0,
diff --git a/web/index.html b/web/index.html
index a5e92fe..c921c24 100644
--- a/web/index.html
+++ b/web/index.html
@@ -226,7 +226,7 @@ <h2 class="f4">What does it mean if I get an error?</h2>
             }
         }
         
-        if (respObj.CidInDHT === true) {
+        if (respObj.ProviderRecordFromPeerInDHT === true) {
             outText += "✅ Found multihash advertised in the dht\n"
         } else {
             outText += "❌ Could not find the multihash in the dht\n"

From 23bef225a5cc067e454e16df4b025464a9f5c365 Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Tue, 27 Aug 2024 16:40:11 +0200
Subject: [PATCH 05/14] feat: run cid check and add test

---
 daemon.go           | 97 +++++++++++++++++++++------------------------
 integration_test.go | 28 +++++++++++++
 main.go             |  2 +-
 test/tools.go       | 20 ++++++++++
 web/index.html      |  9 ++++-
 5 files changed, 103 insertions(+), 53 deletions(-)

diff --git a/daemon.go b/daemon.go
index 8c3785e..7c9470c 100644
--- a/daemon.go
+++ b/daemon.go
@@ -38,7 +38,7 @@ type daemon struct {
 
 // number of providers at which to stop looking for providers in the DHT
 // When doing a check only with a CID
-var MaxProvidersCount = 3
+var MaxProvidersCount = 10
 
 func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) {
 	rm, err := NewResourceManager()
@@ -112,18 +112,18 @@ func (d *daemon) mustStart() {
 
 type providerOutput struct {
 	ID                 string
+	ConnectionError    string
 	Addrs              []string
 	ConnectionMaddrs   []string
 	BitswapCheckOutput BitswapCheckOutput
 }
 
-func (d *daemon) runCidCheck(cidStr string) (*[]providerOutput, error) {
+func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (*[]providerOutput, error) {
 	cid, err := cid.Decode(cidStr)
 	if err != nil {
 		return nil, err
 	}
 
-	ctx := context.Background()
 	out := make([]providerOutput, 0, MaxProvidersCount)
 
 	queryCtx, cancel := context.WithCancel(ctx)
@@ -133,64 +133,59 @@ func (d *daemon) runCidCheck(cidStr string) (*[]providerOutput, error) {
 	var wg sync.WaitGroup
 	var mu sync.Mutex
 
-	for {
-		select {
-		case provider, ok := <-provsCh:
-			if !ok {
-				// Channel closed, all providers processed
-				return &out, nil
+	for provider := range provsCh {
+		wg.Add(1)
+		go func(provider peer.AddrInfo) {
+			defer wg.Done()
+			addrs := make([]string, len(provider.Addrs))
+			for i, addr := range provider.Addrs {
+				addrs[i] = addr.String()
 			}
-			wg.Add(1)
-			go func(provider peer.AddrInfo) {
-				defer wg.Done()
 
-				addrs := make([]string, len(provider.Addrs))
-				for i, addr := range provider.Addrs {
-					addrs[i] = addr.String()
-				}
-
-				provOutput := providerOutput{
-					ID:                 provider.ID.String(),
-					Addrs:              addrs,
-					BitswapCheckOutput: BitswapCheckOutput{},
-				}
+			provOutput := providerOutput{
+				ID:                 provider.ID.String(),
+				Addrs:              addrs,
+				BitswapCheckOutput: BitswapCheckOutput{},
+			}
+			log.Printf("provider output: %v\n", provOutput)
 
-				testHost, err := d.createTestHost()
-				if err != nil {
-					log.Printf("Error creating test host: %v", err)
-					return
-				}
-				defer testHost.Close()
+			testHost, err := d.createTestHost()
+			if err != nil {
+				log.Printf("Error creating test host: %v\n", err)
+				return
+			}
+			defer testHost.Close()
 
-				// Test Is the target connectable
-				dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15)
-				defer dialCancel()
+			// Test Is the target connectable
+			dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15)
+			defer dialCancel()
 
-				// we call NewStream to force NAT hole punching
-				// See https://github.com/libp2p/go-libp2p/issues/2714
-				testHost.Connect(dialCtx, provider)
-				_, connErr := testHost.NewStream(dialCtx, provider.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap")
+			// Call NewStream to force NAT hole punching. see https://github.com/libp2p/go-libp2p/issues/2714
+			testHost.Connect(dialCtx, provider)
+			_, connErr := testHost.NewStream(dialCtx, provider.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap")
 
-				if connErr != nil {
-					provOutput.BitswapCheckOutput.Error = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
-				} else {
-					// TODO: Modify checkBitswapCID and vole to accept `AddrInfo` so that it can test any of the connections
-					provOutput.BitswapCheckOutput = checkBitswapCID(ctx, testHost, cid, provider.Addrs[0])
+			if connErr != nil {
+				provOutput.ConnectionError = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
+			} else {
+				// since we pass a libp2p host that's already connected to the peer the actual connection maddr we pass in doesn't matter
+				p2pAddr, _ := multiaddr.NewMultiaddr("/p2p/" + provider.ID.String())
+				provOutput.BitswapCheckOutput = checkBitswapCID(ctx, testHost, cid, p2pAddr)
 
-					for _, c := range testHost.Network().ConnsToPeer(provider.ID) {
-						provOutput.ConnectionMaddrs = append(provOutput.ConnectionMaddrs, c.RemoteMultiaddr().String())
-					}
+				for _, c := range testHost.Network().ConnsToPeer(provider.ID) {
+					provOutput.ConnectionMaddrs = append(provOutput.ConnectionMaddrs, c.RemoteMultiaddr().String())
 				}
+			}
 
-				mu.Lock()
-				out = append(out, provOutput)
-				mu.Unlock()
-			}(provider)
-		case <-ctx.Done():
-			// Context cancelled
-			return &out, ctx.Err()
-		}
+			mu.Lock()
+			out = append(out, provOutput)
+			mu.Unlock()
+		}(provider)
 	}
+
+	// Wait for all goroutines to finish
+	wg.Wait()
+
+	return &out, nil
 }
 
 type peerCheckOutput struct {
diff --git a/integration_test.go b/integration_test.go
index 6dd2da0..ae93617 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -157,4 +157,32 @@ func TestBasicIntegration(t *testing.T) {
 		obj.Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsFalse()
 		obj.Value("DataAvailableOverBitswap").Object().Value("Responded").Boolean().IsTrue()
 	})
+
+	t.Run("Data found on reachable peer with just cid", func(t *testing.T) {
+		testData := []byte(t.Name())
+		mh, err := multihash.Sum(testData, multihash.SHA2_256, -1)
+		require.NoError(t, err)
+		testCid := cid.NewCidV1(cid.Raw, mh)
+		testBlock, err := blocks.NewBlockWithCid(testData, testCid)
+		require.NoError(t, err)
+		err = bstore.Put(ctx, testBlock)
+		require.NoError(t, err)
+		err = dhtClient.Provide(ctx, testCid, true)
+		require.NoError(t, err)
+
+		res := test.Query(t, "http://localhost:1234", testCid.String())
+
+		res.Length().IsEqual(1)
+		res.Value(0).Object().Value("ID").String().IsEqual(h.ID().String())
+		res.Value(0).Object().Value("ConnectionError").String().IsEmpty()
+		testHostAddrs := h.Addrs()
+		for _, addr := range testHostAddrs {
+			res.Value(0).Object().Value("Addrs").Array().ContainsAny(addr.String())
+		}
+
+		res.Value(0).Object().Value("ConnectionMaddrs").Array()
+		res.Value(0).Object().Value("BitswapCheckOutput").Object().Value("Error").String().IsEmpty()
+		res.Value(0).Object().Value("BitswapCheckOutput").Object().Value("Found").Boolean().IsTrue()
+		res.Value(0).Object().Value("BitswapCheckOutput").Object().Value("Responded").Boolean().IsTrue()
+	})
 }
diff --git a/main.go b/main.go
index 1f85045..fa0761f 100644
--- a/main.go
+++ b/main.go
@@ -90,7 +90,7 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m
 		var data interface{}
 
 		if maStr == "" {
-			data, err = d.runCidCheck(cidStr)
+			data, err = d.runCidCheck(r.Context(), cidStr)
 		} else {
 			data, err = d.runPeerCheck(maStr, cidStr)
 		}
diff --git a/test/tools.go b/test/tools.go
index 39cc275..8ab1e9b 100644
--- a/test/tools.go
+++ b/test/tools.go
@@ -51,6 +51,26 @@ func Query(
 		JSON(opts).Object()
 }
 
+func QueryCid(
+	t *testing.T,
+	url string,
+	cid string,
+) *httpexpect.Array {
+	expectedContentType := "application/json"
+
+	opts := httpexpect.ContentOpts{
+		MediaType: expectedContentType,
+	}
+
+	e := httpexpect.Default(t, url)
+
+	return e.GET("/check").
+		WithQuery("cid", cid).
+		Expect().
+		Status(http.StatusOK).
+		JSON(opts).Array()
+}
+
 func GetEnv(key string, fallback string) string {
 	if value, ok := os.LookupEnv(key); ok {
 		return value
diff --git a/web/index.html b/web/index.html
index e7621e4..1bdcfeb 100644
--- a/web/index.html
+++ b/web/index.html
@@ -28,7 +28,7 @@ <h1 class="dib pa3 ma0 lh-tight">
             <label class="db mt3 f6 fw6" for="cid">CID</label>
             <input class="db w-100 pa2" type="text" id="cid" name="cid" required>
             <label class="db mt3 f6 fw6" for="ma">Multiaddr</label>
-            <input class="db w-100 pa2" type="text" id="multiaddr" name="multiaddr" placeholder="/p2p/12D3Koo..." required />
+            <input class="db w-100 pa2" type="text" id="multiaddr" name="multiaddr" placeholder="/p2p/12D3Koo..." />
             <details class="mt3">
                 <summary class="f6 fw6">Optional fields</summary>
                 <label class="db mt3 f6 fw6" for="backendURL">Backend URL</label>
@@ -47,6 +47,10 @@ <h1 class="dib pa3 ma0 lh-tight">
                 </button>
             </div>
             <div id="output" style="white-space:pre;" class="lh-copy fw6"></div>
+            <details class="mt3">
+              <summary class="f6 fw6">Output</summary>
+              <pre><code id="raw-output" style="white-space:pre;" class="lh-copy fw6 language-json"></code></pre>
+            </details>
         </form>
     </section>
     <section class="mw8 center lh-copy dark-gray pv4 ph2 ph4-ns">
@@ -133,6 +137,9 @@ <h2 class="f4">What does it mean if I get an error?</h2>
 
               if (res.ok) {
                   const respObj = await res.json()
+                  const rawOutput = document.getElementById('raw-output')
+                  rawOutput.textContent = JSON.stringify(respObj, null, 2)
+
                   const output = formatOutput(formData, respObj)
                   showOutput(output)
               } else {

From 8cd701c2c9623bdd6280312979840a547c733560 Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Tue, 27 Aug 2024 18:03:01 +0200
Subject: [PATCH 06/14] feat: format cid only output in frontend

---
 daemon.go           | 13 +++++++----
 integration_test.go |  2 +-
 web/index.html      | 57 +++++++++++++++++++++++++++++++++++++++------
 3 files changed, 60 insertions(+), 12 deletions(-)

diff --git a/daemon.go b/daemon.go
index 9b10529..e168f83 100644
--- a/daemon.go
+++ b/daemon.go
@@ -138,9 +138,15 @@ func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (*[]providerOut
 		wg.Add(1)
 		go func(provider peer.AddrInfo) {
 			defer wg.Done()
-			addrs := make([]string, len(provider.Addrs))
-			for i, addr := range provider.Addrs {
-				addrs[i] = addr.String()
+
+			var addrs []string
+			if len(provider.Addrs) == 0 {
+				addrs = make([]string, len(provider.Addrs))
+				for i, addr := range provider.Addrs {
+					addrs[i] = addr.String()
+				}
+			} else {
+				addrs = nil
 			}
 
 			provOutput := providerOutput{
@@ -148,7 +154,6 @@ func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (*[]providerOut
 				Addrs:              addrs,
 				BitswapCheckOutput: BitswapCheckOutput{},
 			}
-			log.Printf("provider output: %v\n", provOutput)
 
 			testHost, err := d.createTestHost()
 			if err != nil {
diff --git a/integration_test.go b/integration_test.go
index ae93617..f22a312 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -170,7 +170,7 @@ func TestBasicIntegration(t *testing.T) {
 		err = dhtClient.Provide(ctx, testCid, true)
 		require.NoError(t, err)
 
-		res := test.Query(t, "http://localhost:1234", testCid.String())
+		res := test.QueryCid(t, "http://localhost:1234", testCid.String())
 
 		res.Length().IsEqual(1)
 		res.Value(0).Object().Value("ID").String().IsEqual(h.ID().String())
diff --git a/web/index.html b/web/index.html
index f155276..966ca8e 100644
--- a/web/index.html
+++ b/web/index.html
@@ -27,7 +27,7 @@ <h1 class="dib pa3 ma0 lh-tight">
         <form id="queryForm" class="mw8 center lh-copy dark-gray br2 pv4 ph2 ph4-ns">
             <label class="db mt3 f6 fw6" for="cid">CID</label>
             <input class="db w-100 pa2" type="text" id="cid" name="cid" required>
-            <label class="db mt3 f6 fw6" for="ma">Multiaddr</label>
+            <label class="db mt3 f6 fw6" for="ma">Multiaddr (optional)</label>
             <input class="db w-100 pa2" type="text" id="multiaddr" name="multiaddr" placeholder="/p2p/12D3Koo..." />
             <details class="mt3">
                 <summary class="f6 fw6">Optional fields</summary>
@@ -48,8 +48,8 @@ <h1 class="dib pa3 ma0 lh-tight">
             </div>
             <div id="output" style="white-space:pre; overflow-x: scroll;" class="lh-copy fw6"></div>
             <details class="mt3">
-              <summary class="f6 fw6">Output</summary>
-              <pre><code id="raw-output" style="white-space:pre;" class="lh-copy fw6 language-json"></code></pre>
+              <summary class="f6 fw6">Raw Output</summary>
+              <pre style="white-space:pre;" class="lh-copy fw6 language-json"><code id="raw-output"></code></pre>
             </details>
         </form>
     </section>
@@ -139,14 +139,19 @@ <h2 class="f4">What does it mean if I get an error?</h2>
                   const respObj = await res.json()
                   const rawOutput = document.getElementById('raw-output')
                   rawOutput.textContent = JSON.stringify(respObj, null, 2)
-
-                  const output = formatOutput(formData, respObj)
-                  showOutput(output)
+                  if(formData.get('multiaddr') == '') {
+                    const output = formatJustCidOutput(formData, respObj)
+                    showOutput(output)
+                  } else {
+                    const output = formatMaddrOutput(formData, respObj)
+                    showOutput(output)
+                  }
               } else {
                   const resText = await res.text()
                   showOutput(`⚠️ backend returned an error: ${res.status} ${resText}`)
               }
             } catch (e) {
+              console.log(e)
               showOutput(`⚠️ backend error: ${e}`)
             } finally {
               toggleSubmitButton()
@@ -191,7 +196,7 @@ <h2 class="f4">What does it mean if I get an error?</h2>
         spinner.classList.toggle('dn')
     }
 
-    function formatOutput (formData, respObj) {
+    function formatMaddrOutput (formData, respObj) {
         const ma = formData.get('multiaddr')
         const peerIDStartIndex = ma.lastIndexOf("/p2p/")
         const peerID = ma.slice(peerIDStartIndex + 5);
@@ -250,6 +255,44 @@ <h2 class="f4">What does it mean if I get an error?</h2>
         }
         return outText
     }
+
+    function formatJustCidOutput (formData, respObj) {
+        let outText = ""
+        if (respObj.length === 0) {
+            outText += "❌ No providers found for the given CID\n"
+            return outText
+        }
+
+        const successfulProviders = respObj.reduce((acc, provider) => {
+            if(provider.ConnectionError === '' && provider.BitswapCheckOutput.Found === true) {
+                acc++
+            }
+            return acc
+        }, 0)
+
+        const failedProviders = respObj.length - successfulProviders
+
+        respObj.sort((a, b) => {
+            if (a.ConnectionError === '' && b.ConnectionError !== '') {
+                return -1;
+            } else if (a.ConnectionError !== '' && b.ConnectionError === '') {
+                return 1;
+            } else {
+                return 0;
+            }
+
+        });
+        outText += `${successfulProviders > 0 ? '✅' : '❌'} Found ${successfulProviders} providers (out of ${respObj.length} providers in the DHT) that could be connected to and had the CID available over Bitswap:\n`
+        for (const provider of respObj) {
+            const couldConnect = provider.ConnectionError === ''
+
+            outText += `\n\t${provider.ID}\n\t\tConnected: ${couldConnect ? "✅" : `❌ ${provider.ConnectionError.replaceAll('\n', '\n\t\t')}` }`
+            outText += couldConnect ? provider.ConnectionMaddrs && `\n\t\tConnection Multiaddrs:${provider.ConnectionMaddrs?.join('\n\t') || ''}` : ''
+            outText += couldConnect ? `\n\t\tBitswap Check: ${provider.BitswapCheckOutput.Found ? `✅` : "❌"} ${provider.BitswapCheckOutput.Error || ''}` : ''
+        }
+
+        return outText
+    }
 </script>
 </body>
 </html>

From 4dece841fee7da29538b013ba796ae38bc08276d Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Tue, 27 Aug 2024 18:03:37 +0200
Subject: [PATCH 07/14] chore: reorder for cleaner look

---
 web/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/index.html b/web/index.html
index 966ca8e..10bbb6a 100644
--- a/web/index.html
+++ b/web/index.html
@@ -287,8 +287,8 @@ <h2 class="f4">What does it mean if I get an error?</h2>
             const couldConnect = provider.ConnectionError === ''
 
             outText += `\n\t${provider.ID}\n\t\tConnected: ${couldConnect ? "✅" : `❌ ${provider.ConnectionError.replaceAll('\n', '\n\t\t')}` }`
-            outText += couldConnect ? provider.ConnectionMaddrs && `\n\t\tConnection Multiaddrs:${provider.ConnectionMaddrs?.join('\n\t') || ''}` : ''
             outText += couldConnect ? `\n\t\tBitswap Check: ${provider.BitswapCheckOutput.Found ? `✅` : "❌"} ${provider.BitswapCheckOutput.Error || ''}` : ''
+            outText += couldConnect ? provider.ConnectionMaddrs && `\n\t\tConnection Multiaddrs:${provider.ConnectionMaddrs?.join('\n\t') || ''}` : ''
         }
 
         return outText

From 8a9d84cbb0129e158615f802af99c540bddbf3d4 Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Wed, 28 Aug 2024 15:29:36 +0200
Subject: [PATCH 08/14] feat: improve output rendering

---
 daemon.go      | 31 +++++++++++++-------------
 main.go        |  2 +-
 web/index.html | 60 +++++++++++++++++++++++++++++++-------------------
 3 files changed, 53 insertions(+), 40 deletions(-)

diff --git a/daemon.go b/daemon.go
index e168f83..9b32e09 100644
--- a/daemon.go
+++ b/daemon.go
@@ -18,10 +18,10 @@ import (
 	record "github.com/libp2p/go-libp2p-record"
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/libp2p/go-libp2p/core/peerstore"
 	"github.com/libp2p/go-libp2p/core/routing"
 	"github.com/libp2p/go-libp2p/p2p/net/connmgr"
 	"github.com/multiformats/go-multiaddr"
+	manet "github.com/multiformats/go-multiaddr/net"
 )
 
 type kademlia interface {
@@ -119,6 +119,7 @@ type providerOutput struct {
 	BitswapCheckOutput BitswapCheckOutput
 }
 
+// runCidCheck looks up the DHT for providers of a given CID and then checks their connectivity and Bitswap availability
 func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (*[]providerOutput, error) {
 	cid, err := cid.Decode(cidStr)
 	if err != nil {
@@ -139,14 +140,13 @@ func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (*[]providerOut
 		go func(provider peer.AddrInfo) {
 			defer wg.Done()
 
-			var addrs []string
-			if len(provider.Addrs) == 0 {
-				addrs = make([]string, len(provider.Addrs))
-				for i, addr := range provider.Addrs {
-					addrs[i] = addr.String()
+			addrs := []string{}
+			if len(provider.Addrs) > 0 {
+				for _, addr := range provider.Addrs {
+					if manet.IsPublicAddr(addr) { // only return public addrs
+						addrs = append(addrs, addr.String())
+					}
 				}
-			} else {
-				addrs = nil
 			}
 
 			provOutput := providerOutput{
@@ -166,12 +166,12 @@ func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (*[]providerOut
 			dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15)
 			defer dialCancel()
 
-			// Call NewStream to force NAT hole punching. see https://github.com/libp2p/go-libp2p/issues/2714
 			testHost.Connect(dialCtx, provider)
+			// Call NewStream to force NAT hole punching. see https://github.com/libp2p/go-libp2p/issues/2714
 			_, connErr := testHost.NewStream(dialCtx, provider.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap")
 
 			if connErr != nil {
-				provOutput.ConnectionError = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
+				provOutput.ConnectionError = connErr.Error()
 			} else {
 				// since we pass a libp2p host that's already connected to the peer the actual connection maddr we pass in doesn't matter
 				p2pAddr, _ := multiaddr.NewMultiaddr("/p2p/" + provider.ID.String())
@@ -202,7 +202,8 @@ type peerCheckOutput struct {
 	DataAvailableOverBitswap BitswapCheckOutput
 }
 
-func (d *daemon) runPeerCheck(maStr, cidStr string) (*peerCheckOutput, error) {
+// runPeerCheck checks the connectivity and Bitswap availability of a CID from a given peer (either with just peer ID or specific multiaddr)
+func (d *daemon) runPeerCheck(ctx context.Context, maStr, cidStr string) (*peerCheckOutput, error) {
 	ma, err := multiaddr.NewMultiaddr(maStr)
 	if err != nil {
 		return nil, err
@@ -221,7 +222,6 @@ func (d *daemon) runPeerCheck(maStr, cidStr string) (*peerCheckOutput, error) {
 		return nil, err
 	}
 
-	ctx := context.Background()
 	out := &peerCheckOutput{}
 
 	connectionFailed := false
@@ -258,13 +258,12 @@ func (d *daemon) runPeerCheck(maStr, cidStr string) (*peerCheckOutput, error) {
 		// Test Is the target connectable
 		dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15)
 
-		// we call NewStream instead of Connect to force NAT hole punching
-		// See https://github.com/libp2p/go-libp2p/issues/2714
-		testHost.Peerstore().AddAddrs(ai.ID, ai.Addrs, peerstore.RecentlyConnectedAddrTTL)
+		testHost.Connect(dialCtx, *ai)
+		// Call NewStream to force NAT hole punching. see https://github.com/libp2p/go-libp2p/issues/2714
 		_, connErr := testHost.NewStream(dialCtx, ai.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap")
 		dialCancel()
 		if connErr != nil {
-			out.ConnectionError = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
+			out.ConnectionError = connErr.Error()
 			connectionFailed = true
 		}
 	}
diff --git a/main.go b/main.go
index 8675691..0f753ac 100644
--- a/main.go
+++ b/main.go
@@ -93,7 +93,7 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m
 		if maStr == "" {
 			data, err = d.runCidCheck(r.Context(), cidStr)
 		} else {
-			data, err = d.runPeerCheck(maStr, cidStr)
+			data, err = d.runPeerCheck(r.Context(), maStr, cidStr)
 		}
 
 		if err == nil {
diff --git a/web/index.html b/web/index.html
index 10bbb6a..2b9d480 100644
--- a/web/index.html
+++ b/web/index.html
@@ -17,10 +17,10 @@ <h1 class="dib pa3 ma0 lh-tight">
 <main>
     <section class="mw7 center lh-copy dark-gray ph1 pb4">
         <p class="ma0 pv0 ph2 f4 fw6">
-            Check the retrievability of CID from an IPFS peer
+            Check the retrievability of CID
         </p>
         <p class="ma0 pv0 mt2 ph2 f5 fw4">
-            Paste in a Content ID and the multiaddr of a host to check if it is expected to be retrievable
+            Paste in a Content ID and the multiaddr (optional) of a host to check if it is expected to be retrievable
         </p>
     </section>
     <section class="bg-near-white">
@@ -30,7 +30,7 @@ <h1 class="dib pa3 ma0 lh-tight">
             <label class="db mt3 f6 fw6" for="ma">Multiaddr (optional)</label>
             <input class="db w-100 pa2" type="text" id="multiaddr" name="multiaddr" placeholder="/p2p/12D3Koo..." />
             <details class="mt3">
-                <summary class="f6 fw6">Optional fields</summary>
+                <summary class="f6 fw6">Backend Config</summary>
                 <label class="db mt3 f6 fw6" for="backendURL">Backend URL</label>
                 <input class="db w-100 pa2" type="url" id="backendURL" name="backendURL" value="https://ipfs-check-backend.ipfs.io" placeholder="https://ipfs-check-backend.ipfs.io" list="defaultBackendURLs" required>
                 <datalist id="defaultBackendURLs">
@@ -126,6 +126,7 @@ <h2 class="f4">What does it mean if I get an error?</h2>
             e.preventDefault() // dont do a browser form post
 
             showOutput('') // clear out previous results
+            showRawOutput('') // clear out previous results
 
             const formData = new FormData(document.getElementById('queryForm'))
             const backendURL = getBackendUrl(formData)
@@ -137,13 +138,13 @@ <h2 class="f4">What does it mean if I get an error?</h2>
 
               if (res.ok) {
                   const respObj = await res.json()
-                  const rawOutput = document.getElementById('raw-output')
-                  rawOutput.textContent = JSON.stringify(respObj, null, 2)
+                  showRawOutput(JSON.stringify(respObj, null, 2))
+
                   if(formData.get('multiaddr') == '') {
-                    const output = formatJustCidOutput(formData, respObj)
+                    const output = formatJustCidOutput(respObj)
                     showOutput(output)
                   } else {
-                    const output = formatMaddrOutput(formData, respObj)
+                    const output = formatMaddrOutput(formData.get('multiaddr'), respObj)
                     showOutput(output)
                   }
               } else {
@@ -189,6 +190,11 @@ <h2 class="f4">What does it mean if I get an error?</h2>
         outObj.textContent = output
     }
 
+    function showRawOutput (output) {
+        const outObj = document.getElementById('raw-output')
+        outObj.textContent = output
+    }
+
     function toggleSubmitButton() {
         const button = document.getElementById('submit')
         button.toggleAttribute('disabled')
@@ -196,11 +202,10 @@ <h2 class="f4">What does it mean if I get an error?</h2>
         spinner.classList.toggle('dn')
     }
 
-    function formatMaddrOutput (formData, respObj) {
-        const ma = formData.get('multiaddr')
-        const peerIDStartIndex = ma.lastIndexOf("/p2p/")
-        const peerID = ma.slice(peerIDStartIndex + 5);
-        const addrPart = ma.slice(0, peerIDStartIndex);
+    function formatMaddrOutput (multiaddr, respObj) {
+        const peerIDStartIndex = multiaddr.lastIndexOf("/p2p/")
+        const peerID = multiaddr.slice(peerIDStartIndex + 5);
+        const addrPart = multiaddr.slice(0, peerIDStartIndex);
         let outText = ""
 
         if (respObj.ConnectionError !== "") {
@@ -210,7 +215,7 @@ <h2 class="f4">What does it mean if I get an error?</h2>
             outText += `✅ Successfully connected to multiaddr${madrs?.length > 1 ? 's' : '' }: \n\t${madrs.join('\n\t')}\n`
         }
 
-        if (ma.indexOf("/p2p/") === 0 && ma.lastIndexOf("/") === 4) {
+        if (multiaddr.indexOf("/p2p/") === 0 && multiaddr.lastIndexOf("/") === 4) {
             // only peer id passed with /p2p/PeerID
             if (Object.keys(respObj.PeerFoundInDHT).length === 0) {
                 outText += "❌ Could not find any multiaddrs in the dht\n"
@@ -256,39 +261,48 @@ <h2 class="f4">What does it mean if I get an error?</h2>
         return outText
     }
 
-    function formatJustCidOutput (formData, respObj) {
+    function formatJustCidOutput (resp) {
         let outText = ""
-        if (respObj.length === 0) {
+        if (resp.length === 0) {
             outText += "❌ No providers found for the given CID\n"
             return outText
         }
 
-        const successfulProviders = respObj.reduce((acc, provider) => {
+        const successfulProviders = resp.reduce((acc, provider) => {
             if(provider.ConnectionError === '' && provider.BitswapCheckOutput.Found === true) {
                 acc++
             }
             return acc
         }, 0)
 
-        const failedProviders = respObj.length - successfulProviders
+        const failedProviders = resp.length - successfulProviders
 
-        respObj.sort((a, b) => {
+        // Show providers without connection errors first
+        resp.sort((a, b) => {
             if (a.ConnectionError === '' && b.ConnectionError !== '') {
                 return -1;
             } else if (a.ConnectionError !== '' && b.ConnectionError === '') {
                 return 1;
+            }
+
+            // If both have a connection error, list the one with addresses first
+            if(a.Addrs.length > 0 && b.Addrs.length === 0) {
+                return -1
+            } else if(a.Addrs.length === 0 && b.Addrs.length > 0) {
+                return 1
             } else {
-                return 0;
+                return 0
             }
+        })
 
-        });
-        outText += `${successfulProviders > 0 ? '✅' : '❌'} Found ${successfulProviders} providers (out of ${respObj.length} providers in the DHT) that could be connected to and had the CID available over Bitswap:\n`
-        for (const provider of respObj) {
+        outText += `${successfulProviders > 0 ? '✅' : '❌'} Found ${successfulProviders} providers (out of ${resp.length} providers in the DHT) that could be connected to and had the CID available over Bitswap:\n`
+        for (const provider of resp) {
             const couldConnect = provider.ConnectionError === ''
 
             outText += `\n\t${provider.ID}\n\t\tConnected: ${couldConnect ? "✅" : `❌ ${provider.ConnectionError.replaceAll('\n', '\n\t\t')}` }`
             outText += couldConnect ? `\n\t\tBitswap Check: ${provider.BitswapCheckOutput.Found ? `✅` : "❌"} ${provider.BitswapCheckOutput.Error || ''}` : ''
-            outText += couldConnect ? provider.ConnectionMaddrs && `\n\t\tConnection Multiaddrs:${provider.ConnectionMaddrs?.join('\n\t') || ''}` : ''
+            outText += couldConnect ? provider.ConnectionMaddrs && `\n\t\tSuccessful Connection Multiaddr${provider.ConnectionMaddrs.length > 1 ? 's' : ''}:${provider.ConnectionMaddrs?.join('\n\t\t\t') || ''}` : ''
+            outText += provider.Addrs.length > 0 ? `\n\t\tPeer Multiaddrs:\n\t\t\t${provider.Addrs.join('\n\t\t\t')}` : ''
         }
 
         return outText

From 2adced12d792bf78adfc0c952445ac49957da696 Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Wed, 28 Aug 2024 16:05:07 +0200
Subject: [PATCH 09/14] fix: tests and ui

---
 daemon.go           | 18 +++++++++---------
 integration_test.go | 11 +++++++----
 web/index.html      | 12 ++++++------
 3 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/daemon.go b/daemon.go
index 9b32e09..faf7887 100644
--- a/daemon.go
+++ b/daemon.go
@@ -112,11 +112,11 @@ func (d *daemon) mustStart() {
 }
 
 type providerOutput struct {
-	ID                 string
-	ConnectionError    string
-	Addrs              []string
-	ConnectionMaddrs   []string
-	BitswapCheckOutput BitswapCheckOutput
+	ID                       string
+	ConnectionError          string
+	Addrs                    []string
+	ConnectionMaddrs         []string
+	DataAvailableOverBitswap BitswapCheckOutput
 }
 
 // runCidCheck looks up the DHT for providers of a given CID and then checks their connectivity and Bitswap availability
@@ -150,9 +150,9 @@ func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (*[]providerOut
 			}
 
 			provOutput := providerOutput{
-				ID:                 provider.ID.String(),
-				Addrs:              addrs,
-				BitswapCheckOutput: BitswapCheckOutput{},
+				ID:                       provider.ID.String(),
+				Addrs:                    addrs,
+				DataAvailableOverBitswap: BitswapCheckOutput{},
 			}
 
 			testHost, err := d.createTestHost()
@@ -175,7 +175,7 @@ func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (*[]providerOut
 			} else {
 				// since we pass a libp2p host that's already connected to the peer the actual connection maddr we pass in doesn't matter
 				p2pAddr, _ := multiaddr.NewMultiaddr("/p2p/" + provider.ID.String())
-				provOutput.BitswapCheckOutput = checkBitswapCID(ctx, testHost, cid, p2pAddr)
+				provOutput.DataAvailableOverBitswap = checkBitswapCID(ctx, testHost, cid, p2pAddr)
 
 				for _, c := range testHost.Network().ConnsToPeer(provider.ID) {
 					provOutput.ConnectionMaddrs = append(provOutput.ConnectionMaddrs, c.RemoteMultiaddr().String())
diff --git a/integration_test.go b/integration_test.go
index f22a312..375f3f8 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -21,6 +21,7 @@ import (
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/libp2p/go-libp2p/core/protocol"
 	"github.com/libp2p/go-libp2p/p2p/net/connmgr"
+	manet "github.com/multiformats/go-multiaddr/net"
 	"github.com/multiformats/go-multihash"
 	"github.com/stretchr/testify/require"
 )
@@ -177,12 +178,14 @@ func TestBasicIntegration(t *testing.T) {
 		res.Value(0).Object().Value("ConnectionError").String().IsEmpty()
 		testHostAddrs := h.Addrs()
 		for _, addr := range testHostAddrs {
-			res.Value(0).Object().Value("Addrs").Array().ContainsAny(addr.String())
+			if manet.IsPublicAddr(addr) {
+				res.Value(0).Object().Value("Addrs").Array().ContainsAny(addr.String())
+			}
 		}
 
 		res.Value(0).Object().Value("ConnectionMaddrs").Array()
-		res.Value(0).Object().Value("BitswapCheckOutput").Object().Value("Error").String().IsEmpty()
-		res.Value(0).Object().Value("BitswapCheckOutput").Object().Value("Found").Boolean().IsTrue()
-		res.Value(0).Object().Value("BitswapCheckOutput").Object().Value("Responded").Boolean().IsTrue()
+		res.Value(0).Object().Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty()
+		res.Value(0).Object().Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsTrue()
+		res.Value(0).Object().Value("DataAvailableOverBitswap").Object().Value("Responded").Boolean().IsTrue()
 	})
 }
diff --git a/web/index.html b/web/index.html
index 2b9d480..215acb3 100644
--- a/web/index.html
+++ b/web/index.html
@@ -46,10 +46,10 @@ <h1 class="dib pa3 ma0 lh-tight">
                   Run Test
                 </button>
             </div>
-            <div id="output" style="white-space:pre; overflow-x: scroll;" class="lh-copy fw6"></div>
+            <div id="output" style="white-space:pre; overflow-x: scroll;" class="lh-copy fw5"></div>
             <details class="mt3">
               <summary class="f6 fw6">Raw Output</summary>
-              <pre style="white-space:pre;" class="lh-copy fw6 language-json"><code id="raw-output"></code></pre>
+              <pre style="white-space:pre;" class="lh-copy fw5 language-json"><code id="raw-output"></code></pre>
             </details>
         </form>
     </section>
@@ -264,12 +264,12 @@ <h2 class="f4">What does it mean if I get an error?</h2>
     function formatJustCidOutput (resp) {
         let outText = ""
         if (resp.length === 0) {
-            outText += "❌ No providers found for the given CID\n"
+            outText += "❌ No providers found for the given CID"
             return outText
         }
 
         const successfulProviders = resp.reduce((acc, provider) => {
-            if(provider.ConnectionError === '' && provider.BitswapCheckOutput.Found === true) {
+            if(provider.ConnectionError === '' && provider.DataAvailableOverBitswap?.Found === true) {
                 acc++
             }
             return acc
@@ -295,12 +295,12 @@ <h2 class="f4">What does it mean if I get an error?</h2>
             }
         })
 
-        outText += `${successfulProviders > 0 ? '✅' : '❌'} Found ${successfulProviders} providers (out of ${resp.length} providers in the DHT) that could be connected to and had the CID available over Bitswap:\n`
+        outText += `${successfulProviders > 0 ? '✅' : '❌'} Found ${successfulProviders} providers (out of ${resp.length} providers in the DHT) that could be connected to and had the CID available over Bitswap:`
         for (const provider of resp) {
             const couldConnect = provider.ConnectionError === ''
 
             outText += `\n\t${provider.ID}\n\t\tConnected: ${couldConnect ? "✅" : `❌ ${provider.ConnectionError.replaceAll('\n', '\n\t\t')}` }`
-            outText += couldConnect ? `\n\t\tBitswap Check: ${provider.BitswapCheckOutput.Found ? `✅` : "❌"} ${provider.BitswapCheckOutput.Error || ''}` : ''
+            outText += couldConnect ? `\n\t\tBitswap Check: ${provider.DataAvailableOverBitswap.Found ? `✅` : "❌"} ${provider.DataAvailableOverBitswap.Error || ''}` : ''
             outText += couldConnect ? provider.ConnectionMaddrs && `\n\t\tSuccessful Connection Multiaddr${provider.ConnectionMaddrs.length > 1 ? 's' : ''}:${provider.ConnectionMaddrs?.join('\n\t\t\t') || ''}` : ''
             outText += provider.Addrs.length > 0 ? `\n\t\tPeer Multiaddrs:\n\t\t\t${provider.Addrs.join('\n\t\t\t')}` : ''
         }

From ef4ddd0a2cb62ed752c3960b6f14df079f9a77dc Mon Sep 17 00:00:00 2001
From: Daniel Norman <1992255+2color@users.noreply.github.com>
Date: Fri, 30 Aug 2024 11:25:40 +0200
Subject: [PATCH 10/14] Apply suggestions from code review

Co-authored-by: Marcin Rataj <lidel@lidel.org>
---
 web/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/index.html b/web/index.html
index 215acb3..2e9b6a8 100644
--- a/web/index.html
+++ b/web/index.html
@@ -295,7 +295,7 @@ <h2 class="f4">What does it mean if I get an error?</h2>
             }
         })
 
-        outText += `${successfulProviders > 0 ? '✅' : '❌'} Found ${successfulProviders} providers (out of ${resp.length} providers in the DHT) that could be connected to and had the CID available over Bitswap:`
+        outText += `${successfulProviders > 0 ? '✅' : '❌'} Found ${successfulProviders} working providers (out of ${resp.length} provider records sampled from Amino DHT) that could be connected to and had the CID available over Bitswap:`
         for (const provider of resp) {
             const couldConnect = provider.ConnectionError === ''
 

From fd5cf04464d4f2d57170ef16ea2f8f24785d335e Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Fri, 30 Aug 2024 11:46:51 +0200
Subject: [PATCH 11/14] docs: update readme with test results

---
 README.md | 32 ++++++++++++++++++++++++--
 daemon.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++---------
 2 files changed, 88 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 6337cdb..817f564 100644
--- a/README.md
+++ b/README.md
@@ -71,10 +71,38 @@ Note that the `multiaddr` can be:
 
 ### Check results
 
-The server performs several checks given a CID. The results of the check are expressed by the `output` type:
+The server performs several checks depending on whether you also pass a **multiaddr** or just a **cid**.
+
+#### Results when only a `cid` is passed
+
+The results of the check are expressed by the `cidCheckOutput` type:
+
+```go
+type cidCheckOutput *[]providerOutput
+
+type providerOutput struct {
+	ID                       string
+	ConnectionError          string
+	Addrs                    []string
+	ConnectionMaddrs         []string
+	DataAvailableOverBitswap BitswapCheckOutput
+}
+```
+
+The `providerOutput` type contains the following fields:
+
+- `ID`: The peer ID of the provider.
+- `ConnectionError`: An error message if the connection to the provider failed.
+- `Addrs`: The multiaddrs of the provider from the DHT.
+- `ConnectionMaddrs`: The multiaddrs that were used to connect to the provider.
+- `DataAvailableOverBitswap`: The result of the Bitswap check.
+
+#### Results when a `multiaddr` and a `cid` are passed
+
+The results of the check are expressed by the `peerCheckOutput` type:
 
 ```go
-type output struct {
+type peerCheckOutput struct {
 	ConnectionError          string
 	PeerFoundInDHT           map[string]int
 	CidInDHT                 bool
diff --git a/daemon.go b/daemon.go
index faf7887..8f2fed3 100644
--- a/daemon.go
+++ b/daemon.go
@@ -20,6 +20,8 @@ import (
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/libp2p/go-libp2p/core/routing"
 	"github.com/libp2p/go-libp2p/p2p/net/connmgr"
+	"github.com/libp2p/go-libp2p/p2p/protocol/identify"
+	libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"
 	"github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
 )
@@ -90,15 +92,44 @@ func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) {
 		return nil, err
 	}
 
-	return &daemon{h: h, dht: d, dhtMessenger: pm, createTestHost: func() (host.Host, error) {
-		return libp2p.New(
-			libp2p.ConnectionGater(&privateAddrFilterConnectionGater{}),
-			libp2p.DefaultMuxers,
-			libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport),
-			libp2p.EnableHolePunching(),
-			libp2p.UserAgent(userAgent),
-		)
-	}}, nil
+	return &daemon{h: h, dht: d, dhtMessenger: pm,
+		createTestHost: func() (host.Host, error) {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+			defer cancel()
+
+			testHost, err := libp2p.New(
+				libp2p.ConnectionGater(&privateAddrFilterConnectionGater{}),
+				libp2p.DefaultMuxers,
+				libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport),
+				libp2p.EnableHolePunching(),
+				// libp2p.ResourceManager(rm),
+				// libp2p.ConnectionManager(c),
+				libp2p.Transport(libp2pquic.NewTransport),
+				libp2p.ListenAddrStrings("/ip4/0.0.0.0/udp/0/quic-v1"),
+				libp2p.UserAgent(userAgent),
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			testHostDht, err := dht.New(ctx, testHost, dht.Mode(dht.ModeClient), dht.BootstrapPeers(dht.GetDefaultBootstrapPeerAddrInfos()...))
+			if err != nil {
+				return nil, err
+			}
+
+			err = testHostDht.Bootstrap(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			log.Printf("Created host %s with listen addrs %v", testHost.ID(), testHost.Network().ListenAddresses())
+			ids, ok := testHost.(interface{ IDService() identify.IDService })
+			if ok {
+				log.Printf("Own observed addrs: %v", ids.IDService().OwnObservedAddrs())
+			}
+
+			return testHost, nil
+		}}, nil
 }
 
 func (d *daemon) mustStart() {
@@ -111,6 +142,8 @@ func (d *daemon) mustStart() {
 
 }
 
+type cidCheckOutput *[]providerOutput
+
 type providerOutput struct {
 	ID                       string
 	ConnectionError          string
@@ -120,7 +153,7 @@ type providerOutput struct {
 }
 
 // runCidCheck looks up the DHT for providers of a given CID and then checks their connectivity and Bitswap availability
-func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (*[]providerOutput, error) {
+func (d *daemon) runCidCheck(ctx context.Context, cidStr string) (cidCheckOutput, error) {
 	cid, err := cid.Decode(cidStr)
 	if err != nil {
 		return nil, err
@@ -256,13 +289,27 @@ func (d *daemon) runPeerCheck(ctx context.Context, maStr, cidStr string) (*peerC
 
 	if !connectionFailed {
 		// Test Is the target connectable
-		dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*15)
+		dialCtx, dialCancel := context.WithTimeout(ctx, time.Second*120)
 
 		testHost.Connect(dialCtx, *ai)
 		// Call NewStream to force NAT hole punching. see https://github.com/libp2p/go-libp2p/issues/2714
 		_, connErr := testHost.NewStream(dialCtx, ai.ID, "/ipfs/bitswap/1.2.0", "/ipfs/bitswap/1.1.0", "/ipfs/bitswap/1.0.0", "/ipfs/bitswap")
 		dialCancel()
 		if connErr != nil {
+			log.Printf("Error connecting to peer %s: %v", ai.ID, connErr)
+			ids, ok := testHost.(interface{ IDService() identify.IDService })
+			if ok {
+				log.Printf("Own observed addrs: %v", ids.IDService().OwnObservedAddrs())
+			}
+
+			// Log all open connections
+			for _, conn := range testHost.Network().Conns() {
+				log.Printf("Open connection: Peer ID: %s, Remote Addr: %s, Local Addr: %s",
+					conn.RemotePeer(),
+					conn.RemoteMultiaddr(),
+					conn.LocalMultiaddr(),
+				)
+			}
 			out.ConnectionError = connErr.Error()
 			connectionFailed = true
 		}

From 1b08c2b2908a74156f3a25fb9b0afe68d837392b Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Fri, 30 Aug 2024 12:30:27 +0200
Subject: [PATCH 12/14] chore: consolidate metrics into single endpoint

---
 daemon.go | 45 +++++++++++++--------------------------------
 main.go   | 35 ++++++++++++++++++-----------------
 2 files changed, 31 insertions(+), 49 deletions(-)

diff --git a/daemon.go b/daemon.go
index 8f2fed3..fb2f385 100644
--- a/daemon.go
+++ b/daemon.go
@@ -21,9 +21,9 @@ import (
 	"github.com/libp2p/go-libp2p/core/routing"
 	"github.com/libp2p/go-libp2p/p2p/net/connmgr"
 	"github.com/libp2p/go-libp2p/p2p/protocol/identify"
-	libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"
 	"github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 type kademlia interface {
@@ -36,6 +36,7 @@ type daemon struct {
 	dht            kademlia
 	dhtMessenger   *dhtpb.ProtocolMessenger
 	createTestHost func() (host.Host, error)
+	promRegistry   *prometheus.Registry
 }
 
 // number of providers at which to stop looking for providers in the DHT
@@ -53,6 +54,9 @@ func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) {
 		return nil, err
 	}
 
+	// Create a custom registry for all prometheus metrics
+	promRegistry := prometheus.NewRegistry()
+
 	h, err := libp2p.New(
 		libp2p.DefaultMuxers,
 		libp2p.Muxer(mplex.ID, mplex.DefaultTransport),
@@ -60,6 +64,7 @@ func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) {
 		libp2p.ConnectionGater(&privateAddrFilterConnectionGater{}),
 		libp2p.ResourceManager(rm),
 		libp2p.EnableHolePunching(),
+		libp2p.PrometheusRegisterer(promRegistry),
 		libp2p.UserAgent(userAgent),
 	)
 	if err != nil {
@@ -92,43 +97,19 @@ func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) {
 		return nil, err
 	}
 
-	return &daemon{h: h, dht: d, dhtMessenger: pm,
+	return &daemon{
+		h:            h,
+		dht:          d,
+		dhtMessenger: pm,
+		promRegistry: promRegistry,
 		createTestHost: func() (host.Host, error) {
-			ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
-			defer cancel()
-
-			testHost, err := libp2p.New(
+			return libp2p.New(
 				libp2p.ConnectionGater(&privateAddrFilterConnectionGater{}),
 				libp2p.DefaultMuxers,
 				libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport),
 				libp2p.EnableHolePunching(),
-				// libp2p.ResourceManager(rm),
-				// libp2p.ConnectionManager(c),
-				libp2p.Transport(libp2pquic.NewTransport),
-				libp2p.ListenAddrStrings("/ip4/0.0.0.0/udp/0/quic-v1"),
 				libp2p.UserAgent(userAgent),
 			)
-			if err != nil {
-				return nil, err
-			}
-
-			testHostDht, err := dht.New(ctx, testHost, dht.Mode(dht.ModeClient), dht.BootstrapPeers(dht.GetDefaultBootstrapPeerAddrInfos()...))
-			if err != nil {
-				return nil, err
-			}
-
-			err = testHostDht.Bootstrap(ctx)
-			if err != nil {
-				return nil, err
-			}
-
-			log.Printf("Created host %s with listen addrs %v", testHost.ID(), testHost.Network().ListenAddresses())
-			ids, ok := testHost.(interface{ IDService() identify.IDService })
-			if ok {
-				log.Printf("Own observed addrs: %v", ids.IDService().OwnObservedAddrs())
-			}
-
-			return testHost, nil
 		}}, nil
 }
 
@@ -339,7 +320,7 @@ type BitswapCheckOutput struct {
 }
 
 func checkBitswapCID(ctx context.Context, host host.Host, c cid.Cid, ma multiaddr.Multiaddr) BitswapCheckOutput {
-	log.Printf("Start of Bitswap check for cid %s by attempting to connect to ma: %v with the temporary peer: %s", c, ma, host.ID())
+	log.Printf("Start of Bitswap check for cid %s by attempting to connect to ma: %v with the peer: %s", c, ma, host.ID())
 	out := BitswapCheckOutput{}
 	start := time.Now()
 
diff --git a/main.go b/main.go
index 0f753ac..889a921 100644
--- a/main.go
+++ b/main.go
@@ -11,6 +11,7 @@ import (
 	"os"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"github.com/urfave/cli/v2"
@@ -48,6 +49,7 @@ func main() {
 	}
 	app.Action = func(cctx *cli.Context) error {
 		ctx := cctx.Context
+
 		d, err := newDaemon(ctx, cctx.Bool("accelerated-dht"))
 		if err != nil {
 			return err
@@ -68,7 +70,6 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m
 		return err
 	}
 
-	log.Printf("listening on %v\n", l.Addr())
 	log.Printf("Libp2p host peer id %s\n", d.h.ID())
 	log.Printf("Libp2p host listening on %v\n", d.h.Addrs())
 	log.Printf("listening on %v\n", l.Addr())
@@ -105,13 +106,16 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m
 		}
 	}
 
-	// Create a custom registry
-	reg := prometheus.NewRegistry()
+	// Register the default Go collector
+	d.promRegistry.MustRegister(collectors.NewGoCollector())
+
+	// Register the process collector
+	d.promRegistry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
 
 	requestsTotal := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "http_requests_total",
-			Help: "Total number of slow requests",
+			Help: "Total number of HTTP requests",
 		},
 		[]string{"code"},
 	)
@@ -119,22 +123,23 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m
 	requestDuration := prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name:    "http_request_duration_seconds",
-			Help:    "Duration of slow requests",
+			Help:    "Duration of HTTP requests",
 			Buckets: prometheus.DefBuckets,
 		},
 		[]string{"code"},
 	)
 
 	requestsInFlight := prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "slow_requests_in_flight",
-		Help: "Number of slow requests currently being served",
+		Name: "http_requests_in_flight",
+		Help: "Number of HTTP requests currently being served",
 	})
 
 	// Register metrics with our custom registry
-	reg.MustRegister(requestsTotal)
-	reg.MustRegister(requestDuration)
-	reg.MustRegister(requestsInFlight)
-	// Instrument the slowHandler
+	d.promRegistry.MustRegister(requestsTotal)
+	d.promRegistry.MustRegister(requestDuration)
+	d.promRegistry.MustRegister(requestsInFlight)
+
+	// Instrument the checkHandler
 	instrumentedHandler := promhttp.InstrumentHandlerCounter(
 		requestsTotal,
 		promhttp.InstrumentHandlerDuration(
@@ -146,14 +151,10 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m
 		),
 	)
 
-	// 1. Is the peer findable in the DHT?
-	// 2. Does the multiaddr work? If not, what's the error?
-	// 3. Is the CID in the DHT?
-	// 4. Does the peer respond that it has the given data over Bitswap?
 	http.Handle("/check", instrumentedHandler)
 
-	http.Handle("/metrics/libp2p", BasicAuth(promhttp.Handler(), metricsUsername, metricPassword))
-	http.Handle("/metrics/http", BasicAuth(promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), metricsUsername, metricPassword))
+	// Use a single metrics endpoint for all Prometheus metrics
+	http.Handle("/metrics", BasicAuth(promhttp.HandlerFor(d.promRegistry, promhttp.HandlerOpts{}), metricsUsername, metricPassword))
 
 	done := make(chan error, 1)
 	go func() {

From 8419a336d6397c99bd3c66a6cfa5a1970c424483 Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Fri, 30 Aug 2024 12:36:09 +0200
Subject: [PATCH 13/14] fix: improve output rendering in frontend

---
 web/index.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/web/index.html b/web/index.html
index 2e9b6a8..f295e78 100644
--- a/web/index.html
+++ b/web/index.html
@@ -301,8 +301,8 @@ <h2 class="f4">What does it mean if I get an error?</h2>
 
             outText += `\n\t${provider.ID}\n\t\tConnected: ${couldConnect ? "✅" : `❌ ${provider.ConnectionError.replaceAll('\n', '\n\t\t')}` }`
             outText += couldConnect ? `\n\t\tBitswap Check: ${provider.DataAvailableOverBitswap.Found ? `✅` : "❌"} ${provider.DataAvailableOverBitswap.Error || ''}` : ''
-            outText += couldConnect ? provider.ConnectionMaddrs && `\n\t\tSuccessful Connection Multiaddr${provider.ConnectionMaddrs.length > 1 ? 's' : ''}:${provider.ConnectionMaddrs?.join('\n\t\t\t') || ''}` : ''
-            outText += provider.Addrs.length > 0 ? `\n\t\tPeer Multiaddrs:\n\t\t\t${provider.Addrs.join('\n\t\t\t')}` : ''
+            outText += (couldConnect && provider.ConnectionMaddrs) ? `\n\t\tSuccessful Connection Multiaddr${provider.ConnectionMaddrs.length > 1 ? 's' : ''}:\n\t\t\t${provider.ConnectionMaddrs?.join('\n\t\t\t') || ''}` : ''
+            outText += (provider.Addrs.length > 0) ? `\n\t\tPeer Multiaddrs:\n\t\t\t${provider.Addrs.join('\n\t\t\t')}` : ''
         }
 
         return outText

From 233eab6cf95f4df07d61023fd14aba8fe203e435 Mon Sep 17 00:00:00 2001
From: Daniel N <2color@users.noreply.github.com>
Date: Fri, 30 Aug 2024 12:38:52 +0200
Subject: [PATCH 14/14] test: fix tests

---
 integration_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/integration_test.go b/integration_test.go
index 375f3f8..f2ed060 100644
--- a/integration_test.go
+++ b/integration_test.go
@@ -23,6 +23,7 @@ import (
 	"github.com/libp2p/go-libp2p/p2p/net/connmgr"
 	manet "github.com/multiformats/go-multiaddr/net"
 	"github.com/multiformats/go-multihash"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 )
 
@@ -62,6 +63,7 @@ func TestBasicIntegration(t *testing.T) {
 		require.NoError(t, err)
 
 		d := &daemon{
+			promRegistry: prometheus.NewRegistry(),
 			h:            queryHost,
 			dht:          queryDHT,
 			dhtMessenger: pm,