refactor: hamt dir and range test as go test

test/cli/fixtures/README.md

@@ -0,0 +1,72 @@
+# Dataset Description / Sources
+
+TestGatewayHAMTDirectory.car generated with:
+
+```bash
+ipfs version
+# ipfs version 0.19.0
+
+export HAMT_DIR=bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm
+export IPFS_PATH=$(mktemp -d)
+
+# Init and start daemon, ensure we have an empty repository.
+ipfs init --empty-repo
+ipfs daemon &> /dev/null &
+export IPFS_PID=$!
+
+# Retrieve the directory listing, forcing the daemon to download all required DAGs. Kill daemon.
+curl -o dir.html http://127.0.0.1:8080/ipfs/$HAMT_DIR/
+kill $IPFS_PID
+
+# Get the list with all the downloaded refs and sanity check.
+ipfs refs local > required_refs
+cat required_refs | wc -l
+# 962
+
+# Get the list of all the files CIDs inside the directory and sanity check.
+cat dir.html| pup '#content tbody .ipfs-hash attr{href}' | sed 's/\/ipfs\///g;s/\?filename=.*//g' > files_refs
+cat files_refs | wc -l
+# 10100
+
+# Make and export our fixture.
+ipfs files mkdir --cid-version 1 /fixtures
+cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{}
+cat files_refs | ipfs files write --create /fixtures/files_refs
+export FIXTURE_CID=$(ipfs files stat --hash /fixtures/)
+echo $FIXTURE_CID
+# bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i
+ipfs dag export $FIXTURE_CID > TestGatewayHAMTDirectory.car
+```
+
+TestGatewayMultiRange.car generated with:
+
+
+```sh
+ipfs version
+# ipfs version 0.19.0
+
+export FILE_CID=bafybeiae5abzv6j3ucqbzlpnx3pcqbr2otbnpot7d2k5pckmpymin4guau
+export IPFS_PATH=$(mktemp -d)
+
+# Init and start daemon, ensure we have an empty repository.
+ipfs init --empty-repo
+ipfs daemon &> /dev/null &
+export IPFS_PID=$!
+
+# Get a specific byte range from the file. 
+curl http://127.0.0.1:8080/ipfs/$FILE_CID -i -H "Range: bytes=1276-1279, 29839070-29839080"
+kill $IPFS_PID
+
+# Get the list with all the downloaded refs and sanity check.
+ipfs refs local > required_refs
+cat required_refs | wc -l
+# 19
+
+# Make and export our fixture.
+ipfs files mkdir --cid-version 1 /fixtures
+cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{}
+export FIXTURE_CID=$(ipfs files stat --hash /fixtures/)
+echo $FIXTURE_CID
+# bafybeicgsg3lwyn3yl75lw7sn4zhyj5dxtb7wfxwscpq6yzippetmr2w3y
+ipfs dag export $FIXTURE_CID > TestGatewayMultiRange.car
+```

test/cli/gateway_range_test.go

@@ -0,0 +1,75 @@
+package cli
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"testing"
+
+	"github.com/ipfs/kubo/test/cli/harness"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGatewayHAMTDirectory(t *testing.T) {
+	t.Parallel()
+
+	const (
+		// The CID of the HAMT-sharded directory that has 10k items
+		hamtCid = "bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm"
+
+		// fixtureCid is the CID of root of the DAG that is a subset of hamtCid DAG
+		// representing the minimal set of blocks necessary for directory listing.
+		// It also includes a "files_refs" file with the list of the references
+		// we do NOT needs to fetch (files inside the directory)
+		fixtureCid = "bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i"
+	)
+
+	// Start node
+	h := harness.NewT(t)
+	node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline")
+	client := node.GatewayClient()
+
+	// Import fixtures
+	r, err := os.Open("./fixtures/TestGatewayHAMTDirectory.car")
+	assert.Nil(t, err)
+	defer r.Close()
+	err = node.IPFSDagImport(r, fixtureCid)
+	assert.Nil(t, err)
+
+	// Fetch HAMT directory succeeds with minimal refs
+	resp := client.Get(fmt.Sprintf("/ipfs/%s/", hamtCid))
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+}
+
+func TestGatewayMultiRange(t *testing.T) {
+	t.Parallel()
+
+	const (
+		// fileCid is the CID of the large HAMT-sharded file.
+		fileCid = "bafybeiae5abzv6j3ucqbzlpnx3pcqbr2otbnpot7d2k5pckmpymin4guau"
+
+		// fixtureCid is the CID of root of the DAG that is a subset of fileCid DAG
+		// representing the minimal set of blocks necessary for a simple byte range request.
+		fixtureCid = "bafybeicgsg3lwyn3yl75lw7sn4zhyj5dxtb7wfxwscpq6yzippetmr2w3y"
+	)
+
+	// Start node
+	h := harness.NewT(t)
+	node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline")
+	client := node.GatewayClient()
+
+	// Import fixtures
+	r, err := os.Open("./fixtures/TestGatewayMultiRange.car")
+	assert.Nil(t, err)
+	defer r.Close()
+	err = node.IPFSDagImport(r, fixtureCid)
+	assert.Nil(t, err)
+
+	// Succeeds fetching a range of blocks we have
+	resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) {
+		r.Header.Set("Range", "bytes=1276-1279, 29839070-29839080")
+	})
+	assert.Equal(t, http.StatusPartialContent, resp.StatusCode)
+	assert.Contains(t, resp.Body, "Content-Range: bytes 1276-1279/109266405\r\nContent-Type: text/plain; charset=utf-8\r\n\r\niana\r\n")
+	assert.Contains(t, resp.Body, "Content-Range: bytes 29839070-29839080/109266405\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nEXAMPLE.COM\r\n")
+}

test/cli/harness/ipfs.go

@@ -78,3 +78,22 @@ func (n *Node) IPFSAdd(content io.Reader, args ...string) string {
 	log.Debugf("add result: %q", out)
 	return out
 }
+
+func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) error {
+	log.Debugf("node %d dag import with args: %v", n.ID, args)
+	fullArgs := []string{"dag", "import", "--pin-roots=false"}
+	fullArgs = append(fullArgs, args...)
+	res := n.Runner.MustRun(RunRequest{
+		Path:    n.IPFSBin,
+		Args:    fullArgs,
+		CmdOpts: []CmdOpt{RunWithStdin(content)},
+	})
+	if res.Err != nil {
+		return res.Err
+	}
+	res = n.Runner.MustRun(RunRequest{
+		Path: n.IPFSBin,
+		Args: []string{"block", "stat", "--offline", cid},
+	})
+	return res.Err
+}

test/sharness/t0115-gateway-dir-listing.sh

@@ -8,61 +8,21 @@ test_description="Test directory listing (dir-index-html) on the HTTP gateway"
 . lib/test-lib.sh
 
 ## ============================================================================
-## HAMT-sharded directory test. We must execute this test first as we want to
-## start from an empty repository and count the exact number of refs we want.
+## Start IPFS Node and prepare test CIDs
 ## ============================================================================
 
 test_expect_success "ipfs init" '
-  export IPFS_PATH="$(pwd)/.ipfs-for-hamt" &&
-  ipfs init --empty-repo --profile=test > /dev/null
+  export IPFS_PATH="$(pwd)/.ipfs" &&
+  ipfs init --profile=test > /dev/null
 '
 
 test_launch_ipfs_daemon_without_network
 
-# HAMT_CID is a hamt-sharded-directory that has 10k of items
-HAMT_CID=bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm
-
-# HAMT_REFS_CID is root of a DAG that is a subset of HAMT_CID DAG
-# representing minimal set of block necessary for directory listing
-# without fetching ANY root blocks of child items.
-HAMT_REFS_CID=bafybeicv4utmj46mgbpamvw2k6zg4qjs5yvspfnlxz2y6iuey5smjfcn4a
-HAMT_REFS_COUNT=963
-
-test_expect_success "hamt: import fixture with necessary refs" '
-  ipfs dag import ../t0115-gateway-dir-listing/hamt-refs.car &&
-  ipfs refs local | wc -l | tr -d " " > refs_count_actual &&
-  echo $HAMT_REFS_COUNT > refs_count_expected &&
-  test_cmp refs_count_expected refs_count_actual
-'
-
-# we want to confirm that code responsible for directory listing works
-# with minimal set of blocks, and does not fetch entire thing (regression test)
-test_expect_success "hamt: fetch directory from gateway in offline mode" '
-  curl --max-time 30 -sD - http://127.0.0.1:$GWAY_PORT/ipfs/$HAMT_CID/ > hamt_list_response &&
-  ipfs refs local | wc -l | tr -d " " > refs_count_actual &&
-  echo $HAMT_REFS_COUNT > refs_count_expected &&
-  test_cmp refs_count_expected refs_count_actual
-'
-
-test_kill_ipfs_daemon
-
-## ============================================================================
-## Start IPFS Node and prepare test CIDs
-## ============================================================================
-
-test_expect_success "ipfs init" '
-  export IPFS_PATH="$(pwd)/.ipfs" &&
-  ipfs init --empty-repo --profile=test > /dev/null
-'
-#
 # Import test case
 # See the static fixtures in ./t0115-gateway-dir-listing/
-test_expect_success "add remaining test fixtures" '
+test_expect_success "Add the test directory" '
   ipfs dag import ../t0115-gateway-dir-listing/fixtures.car
 '
-
-test_launch_ipfs_daemon_without_network
-
 DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/
 FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
 FILE_SIZE=34

test/sharness/t0115-gateway-dir-listing/README.md

@@ -2,12 +2,8 @@
 
 - fixtures.car
   - raw CARv1
-- hamt-refs.car
-  - raw CARv1 containing all the necessary blocks to present a directory listing
-    for `bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm`, which is
-    a directory containing over 10k items.
 
-fixtures.car generated with:
+generated with:
 
 ```sh
 # using ipfs version 0.18.1
@@ -34,20 +30,3 @@ ipfs dag export ${DIR_CID} > ./fixtures.car
 # FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
 # FILE_SIZE=34
 ```
-
-hamt-refs.car generated with:
-
-```sh
-# using ipfs version 0.18.1
-export IPFS_PATH=$(mktemp -d)
-ipfs init --empty-repo
-ipfs daemon &
-curl http://127.0.0.1:8080/ipfs/bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm/
-killall ipfs
-ipfs refs local >> refs
-ipfs files mkdir --cid-version 1 /Test
-cat refs | xargs -I {} ipfs files cp /ipfs/{} /Test/{}
-ipfs files stat /Test
-# Grab CID: bafybeicv4utmj46mgbpamvw2k6zg4qjs5yvspfnlxz2y6iuey5smjfcn4a
-ipfs dag export bafybeicv4utmj46mgbpamvw2k6zg4qjs5yvspfnlxz2y6iuey5smjfcn4a > ./refs.car
-```