feat: CAR order and dups parameters from IPIP-412

Makefile

@@ -15,7 +15,7 @@ provision-cargateway: ./fixtures.car
 	car -c ./fixtures.car &
 
 provision-kubo:
-	find ./fixtures -name '*.car' -exec ipfs dag import {} \;
+	find ./fixtures -name '*.car' -exec ipfs dag import --stats --pin-roots=false {} \;
 	find ./fixtures -name '*.ipns-record' -exec sh -c 'ipfs routing put --allow-offline /ipns/$$(basename -s .ipns-record "{}") "{}"' \;
 
 # tools

fixtures/trustless_gateway_car/README.md

@@ -16,4 +16,33 @@ ipfs dag export $CID > file-3k-and-3-blocks.car
 REMOVE_BLOCK=$(ipfs dag get $CID | jq '.Links[1].Hash["/"]' -r)
 echo $REMOVE_BLOCK | car filter --version 1 --inverse ./file-3k-and-3-blocks.car ./file-3k-and-3-blocks-missing-block.car
 ipfs pin rm $CID; ipfs repo gc
+# First and third outputted CIDs are used in the missing blocks tests.
+ipfs dag get $CID  | jq .Links | jq -r '.[].Hash."/"'
+```
+
+### [dir-with-duplicate-files.car](./dir-with-duplicate-files.car)
+
+```sh
+ipfs version
+# ipfs version 0.21.0
+TEXT=$(cat <<-EOF 
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc non imperdiet nunc. Proin ac quam ut nibh eleifend aliquet. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Sed ligula dolor, imperdiet sagittis arcu et, semper tincidunt urna. Donec et tempor augue, quis sollicitudin metus. Curabitur semper ullamcorper aliquet. Mauris hendrerit sodales lectus eget fermentum. Proin sollicitudin vestibulum commodo. Vivamus nec lectus eu augue aliquet dignissim nec condimentum justo. In hac habitasse platea dictumst. Mauris vel sem neque.
+
+Vivamus finibus, enim at lacinia semper, arcu erat gravida lacus, sit amet gravida magna orci sit amet est. Sed non leo lacus. Nullam viverra ipsum a tincidunt dapibus. Nulla pulvinar ligula sit amet ante ultrices tempus. Proin purus urna, semper sed lobortis quis, gravida vitae ipsum. Aliquam mi urna, pulvinar eu bibendum quis, convallis ac dolor. In gravida justo sed risus ullamcorper, vitae luctus massa hendrerit. Pellentesque habitant amet.
+EOF
+)
+
+ASCII_CID=$(echo "hello application/vnd.ipld.car" | ipfs add --cid-version=1 -q)
+HELLO_CID=$(echo "hello world" | ipfs add --cid-version=1 -q)
+MULTIBLOCK_CID=$(echo -n $TEXT | ipfs add --cid-version=1 --chunker=size-256 -q)
+# Print the Multiblock CIDs (required for some tests)
+ipfs dag get $MULTIBLOCK_CID  | jq .Links | jq -r '.[].Hash."/"'
+ipfs files mkdir -p --cid-version 1 /dir-with-duplicate-files
+ipfs files cp /ipfs/$ASCII_CID /dir-with-duplicate-files/ascii-copy.txt
+ipfs files cp /ipfs/$ASCII_CID /dir-with-duplicate-files/ascii.txt
+ipfs files cp /ipfs/$HELLO_CID /dir-with-duplicate-files/hello.txt
+ipfs files cp /ipfs/$MULTIBLOCK_CID /dir-with-duplicate-files/multiblock.txt
+ipfs files ls -l
+# Manually CID of "dir-with-duplicate-files" and then...
+ipfs dag export $CID
 ```

go.mod

@@ -3,11 +3,11 @@ module github.com/ipfs/gateway-conformance
 go 1.20
 
 require (
-	github.com/ipfs/boxo v0.10.0
+	github.com/ipfs/boxo v0.10.3-0.20230724084731-f6b448b4263a
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-unixfsnode v1.7.1
 	github.com/ipld/go-car v0.6.1
-	github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d
+	github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33
 	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/ipld/go-ipld-prime v0.20.0
 	github.com/libp2p/go-libp2p v0.26.3
@@ -22,6 +22,7 @@ require (
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
 	github.com/ipfs/go-blockservice v0.5.0 // indirect
 	github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
 	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
@@ -32,7 +33,6 @@ require (
 	github.com/libp2p/go-libp2p-record v0.2.0 // indirect
 	github.com/multiformats/go-multiaddr v0.8.0 // indirect
 	github.com/multiformats/go-multistream v0.4.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect

go.sum

@@ -43,11 +43,13 @@ github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfm
 github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
+github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY=
-github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
+github.com/ipfs/boxo v0.10.3-0.20230724084731-f6b448b4263a h1:eDO++KVTMF7wNqHHhNCZnW/Ocf2K6zCSizAbWoIMREo=
+github.com/ipfs/boxo v0.10.3-0.20230724084731-f6b448b4263a/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
@@ -107,8 +109,8 @@ github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvT
 github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
 github.com/ipld/go-car v0.6.1 h1:blWbEHf1j62JMWFIqWE//YR0m7k5ZMw0AuUOU5hjrH8=
 github.com/ipld/go-car v0.6.1/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8=
-github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d h1:22g+x1tgWSXK34i25qjs+afr7basaneEkHaglBshd2g=
-github.com/ipld/go-car/v2 v2.9.1-0.20230325062757-fff0e4397a3d/go.mod h1:SH2pi/NgfGBsV/CGBAQPxMfghIgwzbh5lQ2N+6dNRI8=
+github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 h1:0OZwzSYWIuiKEOXd/2vm5cMcEmmGLFn+1h6lHELCm3s=
+github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg=
 github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
 github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
@@ -193,7 +195,6 @@ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+
 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=

tests/trustless_gateway_car_test.go

@@ -398,7 +398,9 @@ func TestTrustlessCarEntityBytes(t *testing.T) {
 						IgnoreRoots().
 						HasBlocks(
 							missingBlockFixture.MustGetCid(),
-							missingBlockFixture.MustGetChildrenCids()[0],
+                                                        // This CID is defined at the SPEC level
+                                                        // See the recipe for `file-3k-and-3-blocks-missing-block.car`
+							"QmPKt7ptM2ZYSGPUc8PmPT2VBkLDK3iqpG9TBJY7PCE9rF",
 						).
 						Exactly(),
 				),
@@ -423,7 +425,9 @@ func TestTrustlessCarEntityBytes(t *testing.T) {
 						IgnoreRoots().
 						HasBlocks(
 							missingBlockFixture.MustGetCid(),
-							missingBlockFixture.MustGetChildrenCids()[2],
+                                                        // This CID is defined at the SPEC level
+                                                        // See the recipe for `file-3k-and-3-blocks-missing-block.car`
+							"QmWXY482zQdwecnfBsj78poUUuPXvyw2JAFAEMw4tzTavV",
 						).
 						Exactly(),
 				),
@@ -629,6 +633,170 @@ func TestTrustlessCarEntityBytes(t *testing.T) {
 	RunWithSpecs(t, helpers.StandardCARTestTransforms(t, tests), specs.TrustlessGatewayCAR)
 }
 
+func TestTrustlessCarOrderAndDuplicates(t *testing.T) {
+	dirWithDuplicateFiles := car.MustOpenUnixfsCar("trustless_gateway_car/dir-with-duplicate-files.car")
+        // This array is defined at the SPEC level and should not depend on library behavior
+        // See the recipe for `dir-with-duplicate-files.car`
+	multiblockCIDs := []string{
+		"bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm",
+		"bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq",
+		"bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue",
+		"bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe",
+		"bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm",
+	}
+
+	tests := SugarTests{
+		{
+			Name: "GET CAR with order=dfs and dups=y of UnixFS Directory With Duplicate Files",
+			Hint: `
+				The response MUST contain all the blocks found during traversal even if they
+				are duplicate. In this test, a directory that contains duplicate files is
+				requested. The blocks corresponding to the duplicate files must be returned.
+			`,
+			Request: Request().
+				Path("/ipfs/{{cid}}", dirWithDuplicateFiles.MustGetCid()).
+				Header("Accept", "application/vnd.ipld.car; version=1; order=dfs; dups=y"),
+			Response: Expect().
+				Status(200).
+				Headers(
+					Header("Content-Type").Contains("application/vnd.ipld.car"),
+					Header("Content-Type").Contains("order=dfs"),
+					Header("Content-Type").Contains("dups=y"),
+				).
+				Body(
+					IsCar().
+						IgnoreRoots().
+						HasBlock(dirWithDuplicateFiles.MustGetCid()).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("ascii.txt")). // ascii.txt = ascii-copy.txt
+						HasBlock(dirWithDuplicateFiles.MustGetCid("ascii-copy.txt")).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("hello.txt")).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("multiblock.txt")).
+						HasBlocks(multiblockCIDs...).
+						Exactly().
+						InThatOrder(),
+				),
+		},
+		{
+			Name: "GET CAR with order=dfs and dups=n of UnixFS Directory With Duplicate Files",
+			Hint: `
+				The response MUST NOT contain duplicate blocks. Tested
+				directory contains duplicate files. The blocks corresponding to
+				the duplicate files must be returned only ONCE.
+			`,
+			Request: Request().
+				Path("/ipfs/{{cid}}", dirWithDuplicateFiles.MustGetCid()).
+				Header("Accept", "application/vnd.ipld.car; version=1; order=dfs; dups=n"),
+			Response: Expect().
+				Status(200).
+				Headers(
+					Header("Content-Type").Contains("application/vnd.ipld.car"),
+					Header("Content-Type").Contains("order=dfs"),
+					Header("Content-Type").Contains("dups=n"),
+				).
+				Body(
+					IsCar().
+						IgnoreRoots().
+						HasBlock(dirWithDuplicateFiles.MustGetCid()).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("ascii.txt")). // ascii.txt = ascii-copy.txt
+						HasBlock(dirWithDuplicateFiles.MustGetCid("hello.txt")).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("multiblock.txt")).
+						HasBlocks(multiblockCIDs...).
+						Exactly().
+						InThatOrder(),
+				),
+		},
+		{
+			Name: "GET CAR smoke-test with order=unk of UnixFS Directory",
+			Hint: `
+				The order=unk is usually used by gateway to explicitly indicate
+				it does not guarantee any block order. In this case, we use it
+				for basic smoke-test to confirm support of IPIP-412. The
+				response for request with explicit order=unk MUST include an
+				explicit order in returned Content-Type and contain all the
+				blocks required to construct the requested CID. However, the
+				gateway is free to return default ordering of own choosing,
+				which means the returned blocks can be in any order and
+				duplicates MAY occur.
+			`,
+			Request: Request().
+				Path("/ipfs/{{cid}}", dirWithDuplicateFiles.MustGetCid()).
+				Header("Accept", "application/vnd.ipld.car; version=1; order=unk"),
+			Response: Expect().
+				Status(200).
+				Headers(
+					Header("Content-Type").Contains("application/vnd.ipld.car"),
+					Header("Content-Type").Contains("order="),
+				).
+				Body(
+					IsCar().
+						IgnoreRoots().
+						HasBlock(dirWithDuplicateFiles.MustGetCid()).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("ascii.txt")).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("ascii-copy.txt")).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("hello.txt")).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("multiblock.txt")).
+						HasBlocks(multiblockCIDs...),
+				),
+		},
+		{
+			Name: "GET CAR with order=dfs and dups=y of identity CID",
+			Hint: `
+				Identity hashes MUST never be manifested as read blocks.
+				These are virtual ones and even when dups=y is set, they never
+				should be returned in CAR response body.
+			`,
+			Request: Request().
+				Path("/ipfs/{{cid}}", "bafkqaf3imvwgy3zaneqgc3janfxgy2lomvscay3jmqfa").
+				Header("Accept", "application/vnd.ipld.car; dups=y"),
+			Response: Expect().
+				Status(200).
+				Headers(
+					Header("Content-Type").Contains("application/vnd.ipld.car"),
+					Header("Content-Type").Contains("dups=y"),
+				).
+				Body(
+					IsCar().
+						IgnoreRoots().
+						Exactly().
+						InThatOrder(),
+				),
+		},
+		{
+			Name: "GET CAR with Accept and ?format, specific Accept header is prioritized",
+			Hint: `
+				The response MUST contain all the blocks found during traversal even if they
+				are duplicate. In this test, a directory that contains duplicate files is
+				requested. The blocks corresponding to the duplicate files must be returned.
+			`,
+			Request: Request().
+				Path("/ipfs/{{cid}}", dirWithDuplicateFiles.MustGetCid()).
+				Query("format", "car").
+				Header("Accept", "application/vnd.ipld.car; version=1; order=dfs; dups=y"),
+			Response: Expect().
+				Status(200).
+				Headers(
+					Header("Content-Type").Contains("application/vnd.ipld.car"),
+					Header("Content-Type").Contains("order=dfs"),
+					Header("Content-Type").Contains("dups=y"),
+				).
+				Body(
+					IsCar().
+						IgnoreRoots().
+						HasBlock(dirWithDuplicateFiles.MustGetCid()).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("ascii.txt")). // ascii.txt = ascii-copy.txt
+						HasBlock(dirWithDuplicateFiles.MustGetCid("ascii-copy.txt")).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("hello.txt")).
+						HasBlock(dirWithDuplicateFiles.MustGetCid("multiblock.txt")).
+						HasBlocks(multiblockCIDs...).
+						Exactly().
+						InThatOrder(),
+				),
+		},
+	}
+
+	RunWithSpecs(t, tests, specs.TrustlessGatewayCAROptional)
+}
+
 func flattenStrings(t *testing.T, values ...interface{}) []string {
 	var res []string
 	for _, v := range values {

tooling/ipns/record.go

@@ -5,27 +5,32 @@ import (
 	"time"
 
 	"github.com/ipfs/boxo/ipns"
-	ipns_pb "github.com/ipfs/boxo/ipns/pb"
 	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p/core/peer"
 	mbase "github.com/multiformats/go-multibase"
 	"github.com/multiformats/go-multicodec"
 )
 
 type IpnsRecord struct {
-	pb       *ipns_pb.IpnsEntry
+	rec      *ipns.Record
 	key      string
-	id       peer.ID
+	value    string
+	name     ipns.Name
 	validity time.Time
 }
 
 func UnmarshalIpnsRecord(data []byte, pubKey string) (*IpnsRecord, error) {
-	pb, err := ipns.UnmarshalIpnsEntry(data)
+	pb, err := ipns.UnmarshalRecord(data)
 	if err != nil {
 		return nil, err
 	}
 
-	validity, err := ipns.GetEOL(pb)
+	validity, err := pb.Validity()
+	if err != nil {
+		return nil, err
+	}
+
+	value, err := pb.Value()
 	if err != nil {
 		return nil, err
 	}
@@ -35,11 +40,17 @@ func UnmarshalIpnsRecord(data []byte, pubKey string) (*IpnsRecord, error) {
 		return nil, err
 	}
 
-	return &IpnsRecord{pb: pb, key: pubKey, id: id, validity: validity}, nil
+	return &IpnsRecord{
+		rec:      pb,
+		key:      pubKey,
+		name:     ipns.NameFromPeer(id),
+		validity: validity,
+		value:    value.String(),
+	}, nil
 }
 
 func (i *IpnsRecord) Value() string {
-	return string(i.pb.Value)
+	return i.value
 }
 
 func (i *IpnsRecord) Key() string {
@@ -51,11 +62,11 @@ func (i *IpnsRecord) Validity() time.Time {
 }
 
 func (i *IpnsRecord) Valid() error {
-	return ipns.ValidateWithPeerID(i.id, i.pb)
+	return ipns.ValidateWithName(i.rec, i.name)
 }
 
 func (i *IpnsRecord) idV1(codec multicodec.Code, base mbase.Encoding) (string, error) {
-	c := peer.ToCid(i.id)
+	c := i.name.Cid()
 	c = cid.NewCidV1(uint64(codec), c.Hash())
 	s, err := c.StringOfBase(base)
 	if err != nil {
@@ -85,5 +96,5 @@ func (i *IpnsRecord) IdV1() string {
 }
 
 func (i *IpnsRecord) B58MH() string {
-	return i.id.String()
+	return i.name.Peer().String()
 }