feat: context actions on DNSLink sites

This adds context actions such as "Copy IPFS path", "Copy CID", "Pin"
to DNSLink websites without redirect.

It includes refactoring of ipfsPathValidator to expose high level
resolvers:

- resolveToPublicUrl: always return a meaningful, publicly accessible
URL that can be accessed without the need of IPFS client.
- resolveToIpfsPath: return a valid IPFS path that can be accessed with
IPFS client.
- resolveToImmutableIpfsPath: same as resolveToIpfsPath, but the path is
always immutable /ipfs/
- resolveToCid: returnis direct CID without anything else

add-on/src/lib/copier.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const { safeIpfsPath, trimHashAndSearch } = require('./ipfs-path')
 const { findValueForContext } = require('./context-menus')
 
 async function copyTextToClipboard (text, notify) {
@@ -32,21 +31,19 @@ async function copyTextToClipboard (text, notify) {
   }
 }
 
-function createCopier (getState, getIpfs, notify) {
+function createCopier (notify, ipfsPathValidator) {
   return {
     async copyCanonicalAddress (context, contextType) {
       const url = await findValueForContext(context, contextType)
-      const rawIpfsAddress = safeIpfsPath(url)
-      await copyTextToClipboard(rawIpfsAddress, notify)
+      const ipfsPath = ipfsPathValidator.resolveToIpfsPath(url)
+      await copyTextToClipboard(ipfsPath, notify)
     },
 
     async copyRawCid (context, contextType) {
       try {
-        const ipfs = getIpfs()
         const url = await findValueForContext(context, contextType)
-        const rawIpfsAddress = trimHashAndSearch(safeIpfsPath(url))
-        const directCid = (await ipfs.resolve(rawIpfsAddress, { recursive: true, dhtt: '5s', dhtrc: 1 })).split('/')[2]
-        await copyTextToClipboard(directCid, notify)
+        const cid = await ipfsPathValidator.resolveToCid(url)
+        await copyTextToClipboard(cid, notify)
       } catch (error) {
         console.error('Unable to resolve/copy direct CID:', error.message)
         if (notify) {
@@ -65,9 +62,8 @@ function createCopier (getState, getIpfs, notify) {
 
     async copyAddressAtPublicGw (context, contextType) {
       const url = await findValueForContext(context, contextType)
-      const state = getState()
-      const urlAtPubGw = url.replace(state.gwURLString, state.pubGwURLString)
-      await copyTextToClipboard(urlAtPubGw, notify)
+      const publicUrl = ipfsPathValidator.resolveToPublicUrl(url)
+      await copyTextToClipboard(publicUrl, notify)
     }
   }
 }

add-on/src/lib/dnslink.js

@@ -7,6 +7,8 @@ const PQueue = require('p-queue')
 const { offlinePeerCount } = require('./state')
 const { pathAtHttpGateway } = require('./ipfs-path')
 
+// TODO: add Preferences toggle to disable redirect of DNSLink  websites (while keeping async dnslink lookup)
+
 module.exports = function createDnslinkResolver (getState) {
   // DNSLink lookup result cache
   const cacheOptions = { max: 1000, maxAge: 1000 * 60 * 60 * 12 }

add-on/src/lib/ipfs-companion.js

@@ -5,7 +5,7 @@ const browser = require('webextension-polyfill')
 const toMultiaddr = require('uri-to-multiaddr')
 const { optionDefaults, storeMissingOptions, migrateOptions } = require('./options')
 const { initState, offlinePeerCount } = require('./state')
-const { createIpfsPathValidator, pathAtHttpGateway } = require('./ipfs-path')
+const { createIpfsPathValidator } = require('./ipfs-path')
 const createDnslinkResolver = require('./dnslink')
 const { createRequestModifier, redirectOptOutHint } = require('./ipfs-request')
 const { initIpfsClient, destroyIpfsClient } = require('./ipfs-client')
@@ -56,9 +56,9 @@ module.exports = async function init () {
       }
     }
 
-    copier = createCopier(getState, getIpfs, notify)
     dnslinkResolver = createDnslinkResolver(getState)
-    ipfsPathValidator = createIpfsPathValidator(getState, dnslinkResolver)
+    ipfsPathValidator = createIpfsPathValidator(getState, getIpfs, dnslinkResolver)
+    copier = createCopier(notify, ipfsPathValidator)
     contextMenus = createContextMenus(getState, runtime, ipfsPathValidator, {
       onAddFromContext,
       onCopyCanonicalAddress: copier.copyCanonicalAddress,
@@ -174,7 +174,7 @@ module.exports = async function init () {
     // console.log((sender.tab ? 'Message from a content script:' + sender.tab.url : 'Message from the extension'), request)
     if (request.pubGwUrlForIpfsOrIpnsPath) {
       const path = request.pubGwUrlForIpfsOrIpnsPath
-      const result = ipfsPathValidator.validIpfsOrIpnsPath(path) ? pathAtHttpGateway(path, state.pubGwURLString) : null
+      const result = ipfsPathValidator.validIpfsOrIpnsPath(path) ? ipfsPathValidator.resolveToPublicUrl(path, state.pubGwURLString) : null
       return Promise.resolve({ pubGwUrlForIpfsOrIpnsPath: result })
     }
   }
@@ -257,7 +257,7 @@ module.exports = async function init () {
     return new Promise((resolve, reject) => {
       const http = new XMLHttpRequest()
       // Make sure preload request is excluded from global redirect
-      const preloadUrl = pathAtHttpGateway(`${path}#${redirectOptOutHint}`, state.pubGwURLString)
+      const preloadUrl = ipfsPathValidator.resolveToPublicUrl(`${path}#${redirectOptOutHint}`, state.pubGwURLString)
       http.open('HEAD', preloadUrl)
       http.onreadystatechange = function () {
         if (this.readyState === this.DONE) {
@@ -699,6 +699,10 @@ module.exports = async function init () {
       return dnslinkResolver
     },
 
+    get ipfsPathValidator () {
+      return ipfsPathValidator
+    },
+
     get notify () {
       return notify
     },

add-on/src/lib/ipfs-path.js

@@ -3,23 +3,32 @@
 
 const IsIpfs = require('is-ipfs')
 
-function safeIpfsPath (urlOrPath) {
+function normalizedIpfsPath (urlOrPath) {
+  let result = urlOrPath
+  // Convert CID-in-subdomain URL to /ipns/<fqdn>/ path
   if (IsIpfs.subdomain(urlOrPath)) {
-    urlOrPath = subdomainToIpfsPath(urlOrPath)
+    result = subdomainToIpfsPath(urlOrPath)
   }
-  // better safe than sorry: https://github.com/ipfs/ipfs-companion/issues/303
-  return decodeURIComponent(urlOrPath.replace(/^.*(\/ip(f|n)s\/.+)$/, '$1'))
+  // Drop everything before the IPFS path
+  result = result.replace(/^.*(\/ip(f|n)s\/.+)$/, '$1')
+  // Remove Unescape special characters
+  // https://github.com/ipfs/ipfs-companion/issues/303
+  result = decodeURIComponent(result)
+  // Return a valid IPFS path or null otherwise
+  return IsIpfs.path(result) ? result : null
 }
-exports.safeIpfsPath = safeIpfsPath
+exports.normalizedIpfsPath = normalizedIpfsPath
 
 function subdomainToIpfsPath (url) {
   if (typeof url === 'string') {
     url = new URL(url)
   }
   const fqdn = url.hostname.split('.')
+  // TODO: support CID split with commas
   const cid = fqdn[0]
+  // TODO: support .ip(f|n)s. being at deeper levels
   const protocol = fqdn[1]
-  return `/${protocol}/${cid}${url.pathname}`
+  return `/${protocol}/${cid}${url.pathname}${url.search}${url.hash}`
 }
 
 function pathAtHttpGateway (path, gatewayUrl) {
@@ -39,34 +48,38 @@ function trimHashAndSearch (urlString) {
 }
 exports.trimHashAndSearch = trimHashAndSearch
 
-function createIpfsPathValidator (getState, dnsLink) {
+function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) {
   const ipfsPathValidator = {
     // Test if URL is a Public IPFS resource
     // (pass validIpfsOrIpnsUrl(url) and not at the local gateway or API)
     publicIpfsOrIpnsResource (url) {
       // exclude custom gateway and api, otherwise we have infinite loops
       if (!url.startsWith(getState().gwURLString) && !url.startsWith(getState().apiURLString)) {
-        return validIpfsOrIpnsUrl(url, dnsLink)
+        return validIpfsOrIpnsUrl(url, dnslinkResolver)
       }
       return false
     },
 
     // Test if URL is a valid IPFS or IPNS
     // (IPFS needs to be a CID, IPNS can be PeerId or have dnslink entry)
     validIpfsOrIpnsUrl (url) {
-      return validIpfsOrIpnsUrl(url, dnsLink)
+      return validIpfsOrIpnsUrl(url, dnslinkResolver)
     },
 
     // Same as validIpfsOrIpnsUrl (url) but for paths
     // (we have separate methods to avoid 'new URL' where possible)
     validIpfsOrIpnsPath (path) {
-      return validIpfsOrIpnsPath(path, dnsLink)
+      return validIpfsOrIpnsPath(path, dnslinkResolver)
     },
 
     // Test if actions such as 'copy URL', 'pin/unpin' should be enabled for the URL
-    // TODO: include hostname check for DNSLink and display option to copy CID even if no redirect
     isIpfsPageActionsContext (url) {
-      return (IsIpfs.url(url) && !url.startsWith(getState().apiURLString)) || IsIpfs.subdomain(url)
+      console.log(url)
+      return Boolean(url && !url.startsWith(getState().apiURLString) && (
+        IsIpfs.url(url) ||
+        IsIpfs.subdomain(url) ||
+        dnslinkResolver.cachedDnslink(new URL(url).hostname)
+      ))
     },
 
     // Test if actions such as 'per site redirect toggle' should be enabled for the URL
@@ -77,7 +90,89 @@ function createIpfsPathValidator (getState, dnsLink) {
         (url.startsWith('http') && // hide on non-HTTP pages
          !url.startsWith(state.gwURLString) && // hide on /ipfs/*
           !url.startsWith(state.apiURLString))) // hide on api port
+    },
+
+    // Resolve URL or path to HTTP URL:
+    // - IPFS paths are attached to HTTP Gateway root
+    // - URL of DNSLinked websites are returned as-is
+    // The purpose of this resolver is to always return a meaningful, publicly
+    // accessible URL that can be accessed without the need of IPFS client.
+    resolveToPublicUrl (urlOrPath, optionalGatewayUrl) {
+      const input = urlOrPath
+      // CID-in-subdomain is good as-is
+      if (IsIpfs.subdomain(input)) return input
+      // IPFS Paths should be attached to the public gateway
+      const ipfsPath = normalizedIpfsPath(input)
+      const gateway = optionalGatewayUrl || getState().pubGwURLString
+      if (ipfsPath) return pathAtHttpGateway(ipfsPath, gateway)
+      // Return original URL (eg. DNSLink domains) or null if not an URL
+      return input.startsWith('http') ? input : null
+    },
+
+    // Resolve URL or path to IPFS Path:
+    // - The path can be /ipfs/ or /ipns/
+    // - Keeps pathname + ?search + #hash from original URL
+    // - Returns null if no valid path can be produced
+    // The purpose of this resolver is to return a valid IPFS path
+    // that can be accessed with IPFS client.
+    resolveToIpfsPath (urlOrPath) {
+      const input = urlOrPath
+      // Try to normalize to IPFS path (gateway path or CID-in-subdomain)
+      const ipfsPath = normalizedIpfsPath(input)
+      if (ipfsPath) return ipfsPath
+      // Check URL for DNSLink
+      if (!input.startsWith('http')) return null
+      const { hostname } = new URL(input)
+      const dnslink = dnslinkResolver.cachedDnslink(hostname)
+      if (dnslink) {
+        // Return full IPNS path (keeps pathname + ?search + #hash)
+        return dnslinkResolver.convertToIpnsPath(input)
+      }
+      // No IPFS path by this point
+      return null
+    },
+
+    // Resolve URL or path to Immutable IPFS Path:
+    // - Same as resolveToIpfsPath, but the path is always immutable /ipfs/
+    // - Keeps pathname + ?search + #hash from original URL
+    // - Returns null if no valid path can be produced
+    // The purpose of this resolver is to return immutable /ipfs/ address
+    // even if /ipns/ is present in its input.
+    async resolveToImmutableIpfsPath (urlOrPath) {
+      const path = ipfsPathValidator.resolveToIpfsPath(urlOrPath)
+      // Fail fast if no IPFS Path
+      if (!path) return null
+      // Resolve /ipns/ → /ipfs/
+      if (IsIpfs.ipnsPath(path)) {
+        const labels = path.split('/')
+        // We resolve /ipns/<fqdn> as value in DNSLink cache may be out of date
+        const ipnsRoot = `/ipns/${labels[2]}`
+        const result = await getIpfs().name.resolve(ipnsRoot, { recursive: true, nocache: false })
+        // Old API returned object, latest one returns string ¯\_(ツ)_/¯
+        const ipfsRoot = result.Path ? result.Path : result
+        // Return original path with swapped root (keeps pathname + ?search + #hash)
+        return path.replace(ipnsRoot, ipfsRoot)
+      }
+      // Return /ipfs/ path
+      return path
+    },
+
+    // TODO: add description and tests
+    // Resolve URL or path to a raw CID:
+    // - Result is the direct CID
+    // - Ignores ?search and #hash from original URL
+    // - Returns null if no CID can be produced
+    async resolveToCid (urlOrPath) {
+      const path = ipfsPathValidator.resolveToIpfsPath(urlOrPath)
+      // Fail fast if no IPFS Path
+      if (!path) return null
+      // Resolve to raw CID
+      const rawPath = trimHashAndSearch(path)
+      const result = await getIpfs().resolve(rawPath, { recursive: true, dhtt: '5s', dhtrc: 1 })
+      const directCid = result.split('/')[2]
+      return directCid
     }
+
   }
 
   return ipfsPathValidator
@@ -122,6 +217,7 @@ function validIpnsPath (path, dnsLink) {
       return true
     }
     // then see if there is an DNSLink entry for 'ipnsRoot' hostname
+    // TODO: use dnslink cache only
     if (dnsLink.readAndCacheDnslink(ipnsRoot)) {
       // console.log('==> IPNS for FQDN with valid dnslink: ', ipnsRoot)
       return true

add-on/src/lib/ipfs-request.js

@@ -3,7 +3,7 @@
 
 const LRU = require('lru-cache')
 const IsIpfs = require('is-ipfs')
-const { safeIpfsPath, pathAtHttpGateway } = require('./ipfs-path')
+const { pathAtHttpGateway } = require('./ipfs-path')
 const redirectOptOutHint = 'x-ipfs-companion-no-redirect'
 const recoverableErrors = new Set([
   // Firefox
@@ -127,7 +127,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
         }
         // Detect valid /ipfs/ and /ipns/ on any site
         if (ipfsPathValidator.publicIpfsOrIpnsResource(request.url) && isSafeToRedirect(request, runtime)) {
-          return redirectToGateway(request.url, state, dnslinkResolver)
+          return redirectToGateway(request.url, state, ipfsPathValidator)
         }
         // Detect dnslink using heuristics enabled in Preferences
         if (state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) {
@@ -321,7 +321,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
               return dnslinkRedirect
             }
           }
-          return redirectToGateway(request.url, state, dnslinkResolver)
+          return redirectToGateway(request.url, state, ipfsPathValidator)
         }
 
         // Detect X-Ipfs-Path Header and upgrade transport to IPFS:
@@ -368,7 +368,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
                 // redirect only if anything changed
                 if (newUrl !== request.url) {
                   console.log(`[ipfs-companion] onHeadersReceived: normalized ${request.url} to  ${newUrl}`)
-                  return redirectToGateway(newUrl, state, dnslinkResolver)
+                  return redirectToGateway(newUrl, state, ipfsPathValidator)
                 }
               }
             }
@@ -426,11 +426,11 @@ exports.redirectOptOutHint = redirectOptOutHint
 exports.createRequestModifier = createRequestModifier
 exports.onHeadersReceivedRedirect = onHeadersReceivedRedirect
 
-function redirectToGateway (requestUrl, state, dnslinkResolver) {
+function redirectToGateway (requestUrl, state, ipfsPathValidator) {
   // TODO: redirect to `ipfs://` if hasNativeProtocolHandler === true
   const gateway = state.ipfsNodeType === 'embedded' ? state.pubGwURLString : state.gwURLString
-  const path = safeIpfsPath(requestUrl)
-  return { redirectUrl: pathAtHttpGateway(path, gateway) }
+  const redirectUrl = ipfsPathValidator.resolveToPublicUrl(requestUrl, gateway)
+  return { redirectUrl }
 }
 
 function isSafeToRedirect (request, runtime) {

add-on/src/popup/browser-action/store.js

@@ -3,7 +3,7 @@
 
 const browser = require('webextension-polyfill')
 const IsIpfs = require('is-ipfs')
-const { safeIpfsPath, trimHashAndSearch } = require('../../lib/ipfs-path')
+const { trimHashAndSearch } = require('../../lib/ipfs-path')
 const { contextMenuCopyAddressAtPublicGw, contextMenuCopyRawCid, contextMenuCopyCanonicalAddress } = require('../../lib/context-menus')
 
 // The store contains and mutates the state for the app
@@ -312,15 +312,16 @@ async function getIpfsApi () {
   return (bg && bg.ipfsCompanion) ? bg.ipfsCompanion.ipfs : null
 }
 
+async function getIpfsPathValidator () {
+  const bg = await getBackgroundPage()
+  return (bg && bg.ipfsCompanion) ? bg.ipfsCompanion.ipfsPathValidator : null
+}
+
 async function resolveToPinPath (ipfs, url) {
+  // Prior issues:
   // https://github.com/ipfs-shipyard/ipfs-companion/issues/567
-  url = trimHashAndSearch(url)
   // https://github.com/ipfs/ipfs-companion/issues/303
-  let path = safeIpfsPath(url)
-  if (/^\/ipns/.test(path)) {
-    const response = await ipfs.name.resolve(path, { recursive: true, nocache: false })
-    // old API returned object, latest one returns string ¯\_(ツ)_/¯
-    return response.Path ? response.Path : response
-  }
-  return path
+  const pathValidator = await getIpfsPathValidator()
+  const pinPath = trimHashAndSearch(pathValidator.resolveToImmutableIpfsPath(url))
+  return pinPath
 }

package.json

@@ -93,7 +93,7 @@
     "request-progress": "3.0.0",
     "shx": "0.3.2",
     "simple-progress-webpack-plugin": "1.1.2",
-    "sinon": "7.2.3",
+    "sinon": "7.2.7",
     "sinon-chrome": "2.3.2",
     "standard": "12.0.1",
     "tar": "4.4.8",