From b0af543f1f89a09bcc02ac9916a81f7cf14af300 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2020 16:15:34 +0200 Subject: [PATCH] feat: support ED25519 libp2p-key in subdomains This: - adds subdomain gateway support for ED25519 CIDs in a way that fits in a single DNS label to enable TLS for every IPNS website. - cleans up subdomain redirect logic and adds more explicit error handling. TL;DR on router logic: When CID is longer than 63 characters, router at /ipfs/* and /ipns/* converts to Base36, and if that does not help, returns a human readable 400 Bad Request error. Addressing code review: https://github.com/ipfs/go-ipfs/pull/7441#pullrequestreview-440043209 refactor: use b36 for all libp2p-keys in subdomains Consensus reached in https://github.com/ipfs/go-ipfs/pull/7441#discussion_r452372828 https://github.com/ipfs/go-ipfs/pull/7441#discussion_r451477890 https://github.com/ipfs/go-ipfs/pull/7441#discussion_r452500272 --- core/corehttp/hostname.go | 137 ++++++++++++++++----- core/corehttp/hostname_test.go | 54 ++++++-- test/sharness/t0114-gateway-subdomains.sh | 67 +++++++++- test/sharness/t0184-http-proxy-over-p2p.sh | 2 +- 4 files changed, 212 insertions(+), 48 deletions(-) diff --git a/core/corehttp/hostname.go b/core/corehttp/hostname.go index edcd8b71897..f29b4d11f63 100644 --- a/core/corehttp/hostname.go +++ b/core/corehttp/hostname.go @@ -41,12 +41,15 @@ var defaultKnownGateways = map[string]config.GatewaySpec{ "dweb.link": subdomainGatewaySpec, } +// Label's max length in DNS (https://tools.ietf.org/html/rfc1034#page-7) +const dnsLabelMaxLength int = 63 + // HostnameOption rewrites an incoming request based on the Host header. func HostnameOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { childMux := http.NewServeMux() - coreApi, err := coreapi.NewCoreAPI(n) + coreAPI, err := coreapi.NewCoreAPI(n) if err != nil { return nil, err } @@ -101,7 +104,12 @@ func HostnameOption() ServeOption { if gw.UseSubdomains { // Yes, redirect if applicable // Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link - if newURL, ok := toSubdomainURL(host, r.URL.Path, r); ok { + newURL, err := toSubdomainURL(host, r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { // Just to be sure single Origin can't be abused in // web browsers that ignored the redirect for some // reason, Clear-Site-Data header clears browsing @@ -131,7 +139,7 @@ func HostnameOption() ServeOption { // Not a whitelisted path // Try DNSLink, if it was not explicitly disabled for the hostname - if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, host) { + if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreAPI, host) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path childMux.ServeHTTP(w, r) @@ -158,16 +166,44 @@ func HostnameOption() ServeOption { return } - // Do we need to fix multicodec in PeerID represented as CIDv1? - if isPeerIDNamespace(ns) { - keyCid, err := cid.Decode(rootID) - if err == nil && keyCid.Type() != cid.Libp2pKey { - if newURL, ok := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r); ok { - // Redirect to CID fixed inside of toSubdomainURL() + // Check if rootID is a valid CID + if rootCID, err := cid.Decode(rootID); err == nil { + // Do we need to redirect root CID to a canonical DNS representation? + dnsCID, err := toDNSPrefix(rootID, rootCID) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if !strings.HasPrefix(r.Host, dnsCID) { + dnsPrefix := "/" + ns + "/" + dnsCID + newURL, err := toSubdomainURL(hostname, dnsPrefix+r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to deterministic CID to ensure CID + // always gets the same Origin on the web http.Redirect(w, r, newURL, http.StatusMovedPermanently) return } } + + // Do we need to fix multicodec in PeerID represented as CIDv1? + if isPeerIDNamespace(ns) { + if rootCID.Type() != cid.Libp2pKey { + newURL, err := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to CID fixed inside of toSubdomainURL() + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + } } // Rewrite the path to not use subdomains @@ -183,7 +219,7 @@ func HostnameOption() ServeOption { // 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)? // 2. does Host header include a fully qualified domain name (FQDN)? // 3. does DNSLink record exist in DNS? - if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, host) { + if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreAPI, host) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path childMux.ServeHTTP(w, r) @@ -273,18 +309,38 @@ func isPeerIDNamespace(ns string) bool { } } +// Converts an identifier to DNS-safe representation that fits in 63 characters +func toDNSPrefix(rootID string, rootCID cid.Cid) (prefix string, err error) { + // Return as-is if things fit + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Convert to Base36 and see if that helped + rootID, err = cid.NewCidV1(rootCID.Type(), rootCID.Hash()).StringOfBase(mbase.Base36) + if err != nil { + return "", err + } + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Can't win with DNS at this point, return error + return "", fmt.Errorf("CID incompatible with DNS label length limit of 63: %s", rootID) +} + // Converts a hostname/path to a subdomain-based URL, if applicable. -func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok bool) { +func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, err error) { var scheme, ns, rootID, rest string query := r.URL.RawQuery parts := strings.SplitN(path, "/", 4) - safeRedirectURL := func(in string) (out string, ok bool) { + safeRedirectURL := func(in string) (out string, err error) { safeURI, err := url.ParseRequestURI(in) if err != nil { - return "", false + return "", err } - return safeURI.String(), true + return safeURI.String(), nil } // Support X-Forwarded-Proto if added by a reverse proxy @@ -304,11 +360,11 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok ns = parts[1] rootID = parts[2] default: - return "", false + return "", nil } if !isSubdomainNamespace(ns) { - return "", false + return "", nil } // add prefix if query is present @@ -327,25 +383,42 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok } // If rootID is a CID, ensure it uses DNS-friendly text representation - if rootCid, err := cid.Decode(rootID); err == nil { - multicodec := rootCid.Type() - - // PeerIDs represented as CIDv1 are expected to have libp2p-key - // multicodec (https://github.com/libp2p/specs/pull/209). - // We ease the transition by fixing multicodec on the fly: - // https://github.com/ipfs/go-ipfs/issues/5287#issuecomment-492163929 - if isPeerIDNamespace(ns) && multicodec != cid.Libp2pKey { - multicodec = cid.Libp2pKey + if rootCID, err := cid.Decode(rootID); err == nil { + multicodec := rootCID.Type() + var base mbase.Encoding = mbase.Base32 + + // Normalizations specific to /ipns/{libp2p-key} + if isPeerIDNamespace(ns) { + // Using Base36 for /ipns/ for consistency + // Context: https://github.com/ipfs/go-ipfs/pull/7441#discussion_r452372828 + base = mbase.Base36 + + // PeerIDs represented as CIDv1 are expected to have libp2p-key + // multicodec (https://github.com/libp2p/specs/pull/209). + // We ease the transition by fixing multicodec on the fly: + // https://github.com/ipfs/go-ipfs/issues/5287#issuecomment-492163929 + if multicodec != cid.Libp2pKey { + multicodec = cid.Libp2pKey + } } - // if object turns out to be a valid CID, - // ensure text representation used in subdomain is CIDv1 in Base32 - // https://github.com/ipfs/in-web-browsers/issues/89 - rootID, err = cid.NewCidV1(multicodec, rootCid.Hash()).StringOfBase(mbase.Base32) + // Ensure CID text representation used in subdomain is compatible + // with the way DNS and URIs are implemented in user agents. + // + // 1. Switch to CIDv1 and enable case-insensitive Base encoding + // to avoid issues when user agent force-lowercases the hostname + // before making the request + // (https://github.com/ipfs/in-web-browsers/issues/89) + rootCID = cid.NewCidV1(multicodec, rootCID.Hash()) + rootID, err = rootCID.StringOfBase(base) + if err != nil { + return "", err + } + // 2. Make sure CID fits in a DNS label, adjust encoding if needed + // (https://github.com/ipfs/go-ipfs/issues/7318) + rootID, err = toDNSPrefix(rootID, rootCID) if err != nil { - // should not error, but if it does, its clealy not possible to - // produce a subdomain URL - return "", false + return "", err } } diff --git a/core/corehttp/hostname_test.go b/core/corehttp/hostname_test.go index 9a297464891..cf00e827121 100644 --- a/core/corehttp/hostname_test.go +++ b/core/corehttp/hostname_test.go @@ -1,9 +1,11 @@ package corehttp import ( + "errors" "net/http/httptest" "testing" + cid "github.com/ipfs/go-cid" config "github.com/ipfs/go-ipfs-config" ) @@ -15,23 +17,25 @@ func TestToSubdomainURL(t *testing.T) { path string // out: url string - ok bool + err error }{ // DNSLink - {"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", true}, + {"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", nil}, // Hostname with port - {"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", true}, + {"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", nil}, // CIDv0 → CIDv1base32 - {"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", true}, + {"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", nil}, + // CIDv1 with long sha512 + {"localhost", "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, // PeerID as CIDv1 needs to have libp2p-key multicodec - {"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://bafzbeieqhtl2l3mrszjnhv6hf2iloiitsx7mexiolcnywnbcrzkqxwslja.ipns.localhost/", true}, - {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm.ipns.localhost/", true}, - // PeerID: ed25519+identity multihash - {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://bafzaajaiaejcat4yhiwnr2qz73mtu6vrnj2krxlpfoa3wo2pllfi37quorgwh2jw.ipns.localhost/", true}, + {"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://k2k4r8n0flx3ra0y5dr8fmyvwbzy3eiztmtq6th694k5a3rznayp3e4o.ipns.localhost/", nil}, + {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://k2k4r8l9ja7hkzynavdqup76ou46tnvuaqegbd04a4o1mpbsey0meucb.ipns.localhost/", nil}, + // PeerID: ed25519+identity multihash → CIDv1Base36 + {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil}, } { - url, ok := toSubdomainURL(test.hostname, test.path, r) - if ok != test.ok || url != test.url { - t.Errorf("(%s, %s) returned (%s, %t), expected (%s, %t)", test.hostname, test.path, url, ok, test.url, ok) + url, err := toSubdomainURL(test.hostname, test.path, r) + if url != test.url || !equalError(err, test.err) { + t.Errorf("(%s, %s) returned (%s, %v), expected (%s, %v)", test.hostname, test.path, url, err, test.url, test.err) } } } @@ -75,6 +79,30 @@ func TestPortStripping(t *testing.T) { } +func TestDNSPrefix(t *testing.T) { + for _, test := range []struct { + in string + out string + err error + }{ + // <= 63 + {"QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", nil}, + {"bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", nil}, + // > 63 + // PeerID: ed25519+identity multihash → CIDv1Base36 + {"bafzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk", "k51qzi5uqu5dj16qyiq0tajolkojyl9qdkr254920wxv7ghtuwcz593tp69z9m", nil}, + // CIDv1 with long sha512 → error + {"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, + } { + inCID, _ := cid.Decode(test.in) + out, err := toDNSPrefix(test.in, inCID) + if out != test.out || !equalError(err, test.err) { + t.Errorf("(%s): returned (%s, %v) expected (%s, %v)", test.in, out, err, test.out, test.err) + } + } + +} + func TestKnownSubdomainDetails(t *testing.T) { gwSpec := config.GatewaySpec{ UseSubdomains: true, @@ -150,3 +178,7 @@ func TestKnownSubdomainDetails(t *testing.T) { } } + +func equalError(a, b error) bool { + return (a == nil && b == nil) || (a != nil && b != nil && a.Error() == b.Error()) +} diff --git a/test/sharness/t0114-gateway-subdomains.sh b/test/sharness/t0114-gateway-subdomains.sh index e40822671b6..f38a6db39ce 100755 --- a/test/sharness/t0114-gateway-subdomains.sh +++ b/test/sharness/t0114-gateway-subdomains.sh @@ -110,8 +110,8 @@ test_expect_success "Add the test directory" ' test_expect_success "Publish test text file to IPNS" ' PEERID=$(ipfs id --format="") IPNS_IDv0=$(echo "$PEERID" | ipfs cid format -v 0) - IPNS_IDv1=$(echo "$PEERID" | ipfs cid format -v 1 --codec libp2p-key -b base32) - IPNS_IDv1_DAGPB=$(echo "$IPNS_IDv0" | ipfs cid format -v 1 -b base32) + IPNS_IDv1=$(echo "$PEERID" | ipfs cid format -v 1 --codec libp2p-key -b base36) + IPNS_IDv1_DAGPB=$(echo "$IPNS_IDv0" | ipfs cid format -v 1 -b base36) test_check_peerid "${PEERID}" && ipfs name publish --allow-offline -Q "/ipfs/$CIDv1" > name_publish_out && ipfs name resolve "$PEERID" > output && @@ -119,7 +119,6 @@ test_expect_success "Publish test text file to IPNS" ' test_cmp expected2 output ' - # ensure we start with empty Gateway.PublicGateways test_expect_success 'start daemon with empty config for Gateway.PublicGateways' ' test_kill_ipfs_daemon && @@ -262,6 +261,7 @@ test_expect_success "request for deep path resource at {cid}.ipfs.localhost/sub/ test_should_contain "subdir2-bar" list_response ' + # *.ipns.localhost # .ipns.localhost @@ -480,6 +480,66 @@ test_hostname_gateway_response_should_contain \ "http://127.0.0.1:$GWAY_PORT" \ "$CID_VAL" +## Test subdomain handling of CIDs that do not fit in a single DNS Label (>63chars) +## https://github.com/ipfs/go-ipfs/issues/7318 +## ============================================================================ + +# TODO: replace with cidv1 +# ed25519 fits under 63 char limit when represented in base36 +CIDv1_ED25519_RAW="12D3KooWP3ggTJV8LGckDHc4bVyXGhEWuBskoFyE6Rn2BJBqJtpa" +CIDv1_ED25519_DNSSAFE="k51qzi5uqu5dl2yn0d6xu8q5aqa61jh8zeyixz9tsju80n15ssiyew48912c63" +# sha512 will be over 63char limit, even when represented in Base36 +CIDv1_TOO_LONG=$(echo $CID_VAL | ipfs add --cid-version 1 --hash sha2-512 -Q) + +# local: *.localhost +test_localhost_gateway_response_should_contain \ + "request for a ED25519 CID at localhost/ipfs/{CIDv1} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \ + "http://localhost:$GWAY_PORT/ipns/$CIDv1_ED25519_RAW" \ + "Location: http://${CIDv1_ED25519_DNSSAFE}.ipns.localhost:$GWAY_PORT/" + +# router should not redirect to hostnames that could fail due to DNS limits +test_localhost_gateway_response_should_contain \ + "request for a too long CID at localhost/ipfs/{CIDv1} returns human readable error" \ + "http://localhost:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \ + "CID incompatible with DNS label length limit of 63" + +test_localhost_gateway_response_should_contain \ + "request for a too long CID at localhost/ipfs/{CIDv1} returns HTTP Error 400 Bad Request" \ + "http://localhost:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \ + "400 Bad Request" + +# direct request should also fail (provides the same UX as router and avoids confusion) +test_localhost_gateway_response_should_contain \ + "request for a too long CID at {CIDv1}.ipfs.localhost returns expected payload" \ + "http://$CIDv1_TOO_LONG.ipfs.localhost:$GWAY_PORT" \ + "400 Bad Request" + +# public subdomain gateway: *.example.com + +test_hostname_gateway_response_should_contain \ + "request for a ED25519 CID at example.com/ipfs/{CIDv1} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipns/$CIDv1_ED25519_RAW" \ + "Location: http://${CIDv1_ED25519_DNSSAFE}.ipns.example.com" + +test_hostname_gateway_response_should_contain \ + "request for a too long CID at example.com/ipfs/{CIDv1} returns human readable error" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \ + "CID incompatible with DNS label length limit of 63" + +test_hostname_gateway_response_should_contain \ + "request for a too long CID at example.com/ipfs/{CIDv1} returns HTTP Error 400 Bad Request" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \ + "400 Bad Request" + +test_hostname_gateway_response_should_contain \ + "request for a too long CID at {CIDv1}.ipfs.example.com returns HTTP Error 400 Bad Request" \ + "$CIDv1_TOO_LONG.ipfs.example.com" \ + "http://127.0.0.1:$GWAY_PORT/" \ + "400 Bad Request" + # Disable selected Paths for the subdomain gateway hostname # ============================================================================= @@ -501,7 +561,6 @@ test_hostname_gateway_response_should_contain \ "http://127.0.0.1:$GWAY_PORT" \ "404 Not Found" - ## ============================================================================ ## Test path-based requests with a custom hostname config ## ============================================================================ diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 06a3b9ccb18..ca9e856bc92 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -216,7 +216,7 @@ test_expect_success 'handle multipart/form-data http request' ' ' # subdomain gateway at *.p2p.example.com requires PeerdID in base32 -RECEIVER_ID_CIDv1=$( ipfs cid format -v 1 -b b --codec libp2p-key -- $RECEIVER_ID) +RECEIVER_ID_CIDv1=$( ipfs cid format -v 1 --codec libp2p-key -b base36 -- $RECEIVER_ID) # OK: $peerid.p2p.example.com/http/index.txt test_expect_success "handle http request to a subdomain gateway" '