fix: support directory listings even if a 404 page is present

fixes #4233 (comment)

Basically, there's a trade-off here:

1. We can support directory listings while supporting 404 pages (this PR).
2. If a 404 page is present, directory listings don't work.

Given that option 1 is more flexible and users shouldn't be _too_ confused if
they land on a directory with no index.html page, I've gone with that option.

core/corehttp/gateway_handler.go

@@ -307,10 +307,6 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	if i.servePretty404IfPresent(w, r, parsedPath) {
-		return
-	}
-
 	// storage for directory listing
 	var dirListing []directoryItem
 	dirit := dir.Entries()

core/corehttp/gateway_test.go

@@ -267,8 +267,8 @@ func TestPretty404(t *testing.T) {
 		{"/nope", "*/*", http.StatusNotFound, "Custom 404"},
 		{"/nope", "application/json", http.StatusNotFound, "ipfs resolve -r /ipns/example.net/nope: no link named \"nope\" under QmcmnF7XG5G34RdqYErYDwCKNFQ6jb8oKVR21WAJgubiaj\n"},
 		{"/deeper/nope", "text/html", http.StatusNotFound, "Deep custom 404"},
-		{"/deeper/", "text/html", http.StatusNotFound, "Deep custom 404"},
-		{"/deeper", "text/html", http.StatusNotFound, "Deep custom 404"},
+		{"/deeper/", "text/html", http.StatusOK, ""},
+		{"/deeper", "text/html", http.StatusOK, ""},
 		{"/nope/nope", "text/html", http.StatusNotFound, "Custom 404"},
 	} {
 		var c http.Client
@@ -293,7 +293,7 @@ func TestPretty404(t *testing.T) {
 			t.Fatalf("error reading response from %s: %s", test.path, err)
 		}
 
-		if string(body) != test.text {
+		if test.text != "" && string(body) != test.text {
 			t.Fatalf("unexpected response body from %s: got %q, expected %q", test.path, body, test.text)
 		}
 	}