Gateway renders pretty 404 pages if available

In the same way that an `index.html` file is rendered, if one is present, when the
requested path is a directory, now an `ipfs-404.html` file is rendered if
the requested file is not present within the specified IPFS object.

`ipfs-404.html` files are looked for in the directory of the requested path and each
parent until one is found, falling back on the well-known 404 error message.

License: MIT
Signed-off-by: JP Hastings-Spital <jphastings@gmail.com>

core/corehttp/gateway_handler.go

@@ -11,6 +11,7 @@ import (
 	gopath "path"
 	"regexp"
 	"runtime/debug"
+	"strconv"
 	"strings"
 	"time"
 
@@ -203,6 +204,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusServiceUnavailable)
 		return
 	default:
+		if i.servePretty404IfPresent(w, r, parsedPath) {
+			return
+		}
+
 		webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusNotFound)
 		return
 	}
@@ -290,6 +295,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
+	if i.servePretty404IfPresent(w, r, parsedPath) {
+		return
+	}
+
 	// storage for directory listing
 	var dirListing []directoryItem
 	dirit := dir.Entries()
@@ -406,6 +415,41 @@ func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, nam
 	http.ServeContent(w, req, name, modtime, content)
 }
 
+func (i *gatewayHandler) servePretty404IfPresent(w http.ResponseWriter, r *http.Request, parsedPath ipath.Path) bool {
+	resolved404Path, ctype, err := i.searchUpTreeFor404(r, parsedPath)
+	if err != nil {
+		return false
+	}
+
+	dr, err := i.api.Unixfs().Get(r.Context(), resolved404Path)
+	if err != nil {
+		return false
+	}
+	defer dr.Close()
+
+	f, ok := dr.(files.File)
+	if !ok {
+		return false
+	}
+
+	size, err := f.Size()
+	if err != nil {
+		return false
+	}
+
+	content := &lazySeeker{
+		size:   size,
+		reader: f,
+	}
+
+	log.Debugf("using pretty 404 file for %s", parsedPath.String())
+	w.Header().Set("Content-Type", ctype)
+	w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
+	w.WriteHeader(http.StatusNotFound)
+	_, err = io.CopyN(w, content, size)
+	return err == nil
+}
+
 func (i *gatewayHandler) postHandler(w http.ResponseWriter, r *http.Request) {
 	p, err := i.api.Unixfs().Add(r.Context(), files.NewReaderFile(r.Body))
 	if err != nil {
@@ -619,3 +663,45 @@ func getFilename(s string) string {
 	}
 	return gopath.Base(s)
 }
+
+func (i *gatewayHandler) searchUpTreeFor404(r *http.Request, parsedPath ipath.Path) (ipath.Resolved, string, error) {
+	filename404, ctype, err := preferred404Filename(r.Header.Values("Accept"))
+	if err != nil {
+		return nil, "", err
+	}
+
+	pathComponents := strings.Split(parsedPath.String(), "/")
+
+	for idx := len(pathComponents); idx >= 3; idx-- {
+		pretty404 := gopath.Join(append(pathComponents[0:idx], filename404)...)
+		parsed404Path := ipath.New("/" + pretty404)
+		if parsed404Path.IsValid() != nil {
+			break
+		}
+		resolvedPath, err := i.api.ResolvePath(r.Context(), parsed404Path)
+		if err != nil {
+			continue
+		}
+		return resolvedPath, ctype, nil
+	}
+
+	return nil, "", fmt.Errorf("no pretty 404 in any parent folder")
+}
+
+func preferred404Filename(acceptHeaders []string) (string, string, error) {
+	// If we ever want to offer a 404 file for a different content type
+	// then this function will need to parse q weightings, but for now
+	// the presence of anything matching HTML is enough.
+	for _, acceptHeader := range acceptHeaders {
+		accepted := strings.Split(acceptHeader, ",")
+		for _, spec := range accepted {
+			contentType := strings.SplitN(spec, ";", 1)[0]
+			switch contentType {
+			case "*/*", "text/*", "text/html":
+				return "ipfs-404.html", "text/html", nil
+			}
+		}
+	}
+
+	return "", "", fmt.Errorf("there is no 404 file for the requested content types")
+}

core/corehttp/gateway_test.go

@@ -235,6 +235,70 @@ func TestGatewayGet(t *testing.T) {
 	}
 }
 
+func TestPretty404(t *testing.T) {
+	ns := mockNamesys{}
+	ts, api, ctx := newTestServerAndNode(t, ns)
+	defer ts.Close()
+
+	f1 := files.NewMapDirectory(map[string]files.Node{
+		"ipfs-404.html": files.NewBytesFile([]byte("Custom 404")),
+		"deeper": files.NewMapDirectory(map[string]files.Node{
+			"ipfs-404.html": files.NewBytesFile([]byte("Deep custom 404")),
+		}),
+	})
+
+	k, err := api.Unixfs().Add(ctx, f1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	host := "example.net"
+	ns["/ipns/"+host] = path.FromString(k.String())
+
+	for _, test := range []struct {
+		path   string
+		accept string
+		status int
+		text   string
+	}{
+		{"/ipfs-404.html", "text/html", http.StatusOK, "Custom 404"},
+		{"/nope", "text/html", http.StatusNotFound, "Custom 404"},
+		{"/nope", "text/*", http.StatusNotFound, "Custom 404"},
+		{"/nope", "*/*", http.StatusNotFound, "Custom 404"},
+		{"/nope", "application/json", http.StatusNotFound, "ipfs resolve -r /ipns/example.net/nope: no link named \"nope\" under QmcmnF7XG5G34RdqYErYDwCKNFQ6jb8oKVR21WAJgubiaj\n"},
+		{"/deeper/nope", "text/html", http.StatusNotFound, "Deep custom 404"},
+		{"/deeper/", "text/html", http.StatusNotFound, "Deep custom 404"},
+		{"/deeper", "text/html", http.StatusNotFound, "Deep custom 404"},
+		{"/nope/nope", "text/html", http.StatusNotFound, "Custom 404"},
+	} {
+		var c http.Client
+		req, err := http.NewRequest("GET", ts.URL+test.path, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		req.Header.Add("Accept", test.accept)
+		req.Host = host
+		resp, err := c.Do(req)
+
+		if err != nil {
+			t.Fatalf("error requesting %s: %s", test.path, err)
+		}
+
+		defer resp.Body.Close()
+		if resp.StatusCode != test.status {
+			t.Fatalf("got %d, expected %d, from %s", resp.StatusCode, test.status, test.path)
+		}
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			t.Fatalf("error reading response from %s: %s", test.path, err)
+		}
+
+		if string(body) != test.text {
+			t.Fatalf("unexpected response body from %s: got %q, expected %q", test.path, body, test.text)
+		}
+	}
+}
+
 func TestIPNSHostnameRedirect(t *testing.T) {
 	ns := mockNamesys{}
 	ts, api, ctx := newTestServerAndNode(t, ns)