feat: poc dnslink policy with x-ipfs-header

add-on/_locales/en/messages.json

@@ -297,9 +297,33 @@
     "message": "Turn plaintext /ipfs/ paths into clickable links",
     "description": "An option description on the Preferences screen (option_linkify_description)"
   },
-  "option_dnslink_title": {
+  "option_dnslinkPolicy_title": {
     "message": "DNSLINK Support",
-    "description": "An option title on the Preferences screen (option_dnslink_title)"
+    "description": "An option title on the Preferences screen (option_dnslinkPolicy_title)"
+  },
+  "option_dnslinkPolicy_description": {
+    "message": "Select DNS TXT lookup policy",
+    "description": "An option description on the Preferences screen (option_dnslinkPolicy_description)"
+  },
+  "option_dnslinkPolicy_disabled": {
+    "message": "Disabled",
+    "description": "A select field option description on the Preferences screen (option_dnslinkPolicy_disabled)"
+  },
+  "option_dnslinkPolicy_detectIpfsPathHeader": {
+    "message": "Lookup if x-ipfs-path found",
+    "description": "A select field option description on the Preferences screen (option_dnslinkPolicy_detectIpfsPathHeader)"
+  },
+  "option_dnslinkPolicy_eagerDnsTxtLookup": {
+    "message": "Lookup every FQDN",
+    "description": "A select field option description on the Preferences screen (option_dnslinkPolicy_)"
+  },
+  "option_detectIpfsPathHeader_title": {
+    "message": "Detect x-ipfs-path Header",
+    "description": "An option title on the Preferences screen (option_detectIpfsPathHeader_title)"
+  },
+  "option_detectIpfsPathHeader_description": {
+    "message": "Stop reading data and redirect to IPFS when the header is found in HTTP response",
+    "description": "An option description on the Preferences screen (option_detectIpfsPathHeader_description)"
   },
   "option_ipfsProxy_title": {
     "message": "window.ipfs",
@@ -321,10 +345,6 @@
     "message": "Enables automatic preload of uploaded assets via asynchronous HTTP HEAD request to a Public Gateway",
     "description": "An option description on the Preferences screen (option_preloadAtPublicGateway_description)"
   },
-  "option_dnslink_description": {
-    "message": "Perform DNS lookup for every visited website and use Custom Gateway if DNSLINK is present in its DNS TXT record (known to slow down the browser)",
-    "description": "An option description on the Preferences screen (option_dnslink_description)"
-  },
   "option_resetAllOptions_title": {
     "message": "Reset Everything",
     "description": "An option title and button label on the Preferences screen (option_resetAllOptions_title)"

add-on/src/lib/dns-link.js → add-on/src/lib/dnslink.js

@@ -2,11 +2,12 @@
 /* eslint-env browser */
 
 const IsIpfs = require('is-ipfs')
-const { LRUMap } = require('lru_map')
+const LRU = require('lru-cache')
 
-module.exports = function createDnsLink (getState) {
-  // TODO: expire keys after 12h-24h
-  const cache = new LRUMap(1000)
+module.exports = function createDnslinkResolver (getState) {
+  // DNSLink lookup result cache
+  const cacheOptions = {max: 1000, maxAge: 1000 * 60 * 60}
+  const cache = new LRU(cacheOptions)
 
   const dnslinkResolver = {
     isDnslookupPossible () {
@@ -16,12 +17,13 @@ module.exports = function createDnsLink (getState) {
 
     isDnslookupSafeForURL (requestUrl) {
       // skip URLs that could produce infinite recursion or weird loops
-      return getState().dnslink &&
+      const state = getState()
+      return state.dnslinkPolicy &&
         dnslinkResolver.isDnslookupPossible() &&
         requestUrl.startsWith('http') &&
         !IsIpfs.url(requestUrl) &&
-        !requestUrl.startsWith(getState().apiURLString) &&
-        !requestUrl.startsWith(getState().gwURLString)
+        !requestUrl.startsWith(state.apiURLString) &&
+        !requestUrl.startsWith(state.gwURLString)
     },
 
     dnslinkRedirect (requestUrl, dnslink) {
@@ -34,6 +36,14 @@ module.exports = function createDnsLink (getState) {
       }
     },
 
+    setDnslink (fqdn, value) {
+      cache.set(fqdn, value)
+    },
+
+    clearCache() {
+      cache.reset()
+    },
+
     cachedDnslink (fqdn) {
       return cache.get(fqdn)
     },
@@ -45,14 +55,14 @@ module.exports = function createDnsLink (getState) {
           console.info(`[ipfs-companion] dnslink cache miss for '${fqdn}', running DNS TXT lookup`)
           dnslink = dnslinkResolver.readDnslinkFromTxtRecord(fqdn)
           if (dnslink) {
-            cache.set(fqdn, dnslink)
+            dnslinkResolver.setDnslink(fqdn, dnslink)
             console.info(`[ipfs-companion] found dnslink: '${fqdn}' -> '${dnslink}'`)
           } else {
-            cache.set(fqdn, false)
+            dnslinkResolver.setDnslink(fqdn, false)
             console.info(`[ipfs-companion] found NO dnslink for '${fqdn}'`)
           }
         } catch (error) {
-          console.error(`[ipfs-companion] Error in dnslinkRedirect for '${fqdn}'`)
+          console.error(`[ipfs-companion] Error in readAndCacheDnslink for '${fqdn}'`)
           console.error(error)
         }
       } else {
@@ -101,10 +111,13 @@ module.exports = function createDnsLink (getState) {
       const httpGatewayPath = path.startsWith('/ipfs/') || path.startsWith('/ipns/') || path.startsWith('/api/v')
       if (!httpGatewayPath) {
         const fqdn = url.hostname
-        // If dnslinkEagerDnsTxtLookup is enabled lookups will be executed for every unique hostname on every visited website
+        // If dnslink policy is 'eagerDnsTxtLookup' then lookups will be executed for every unique hostname on every visited website
         // Until we get efficient  DNS TXT Lookup API it will come with overhead, so it is opt-in for now,
         // and we do lookup to populate dnslink cache only when X-Ipfs-Path header is found in initial response.
-        const foundDnslink = dnslink || (getState().dnslinkEagerDnsTxtLookup ? dnslinkResolver.readAndCacheDnslink(fqdn) : dnslinkResolver.cachedDnslink(fqdn))
+        const foundDnslink = dnslink ||
+          (getState().dnslinkPolicy === 'eagerDnsTxtLookup'
+            ? dnslinkResolver.readAndCacheDnslink(fqdn)
+            : dnslinkResolver.cachedDnslink(fqdn))
         if (foundDnslink) {
           return true
         }

add-on/src/lib/ipfs-companion.js

@@ -2,10 +2,10 @@
 /* eslint-env browser, webextensions */
 
 const browser = require('webextension-polyfill')
-const { optionDefaults, storeMissingOptions } = require('./options')
+const { optionDefaults, storeMissingOptions, migrateOptions } = require('./options')
 const { initState, offlinePeerCount } = require('./state')
 const { createIpfsPathValidator, urlAtPublicGw } = require('./ipfs-path')
-const createDnsLink = require('./dns-link')
+const createDnslinkResolver = require('./dnslink')
 const { createRequestModifier, redirectOptOutHint } = require('./ipfs-request')
 const { initIpfsClient, destroyIpfsClient } = require('./ipfs-client')
 const { createIpfsUrlProtocolHandler } = require('./ipfs-protocol')
@@ -35,6 +35,7 @@ module.exports = async function init () {
   const browserActionPortName = 'browser-action-port'
 
   try {
+    await migrateOptions(browser.storage.local)
     const options = await browser.storage.local.get(optionDefaults)
     runtime = await createRuntimeChecks(browser)
     state = initState(options)
@@ -54,7 +55,7 @@ module.exports = async function init () {
     }
 
     copier = createCopier(getState, notify)
-    dnslinkResolver = createDnsLink(getState)
+    dnslinkResolver = createDnslinkResolver(getState)
     ipfsPathValidator = createIpfsPathValidator(getState, dnslinkResolver)
     contextMenus = createContextMenus(getState, runtime, ipfsPathValidator, {
       onAddToIpfsRawCid: addFromURL,
@@ -585,12 +586,19 @@ module.exports = async function init () {
           state[key] = change.newValue
           ipfsProxyContentScript = await registerIpfsProxyContentScript()
           break
+        case 'dnslinkPolicy':
+          state.dnslinkPolicy = String(change.newValue) === 'false' ? false : change.newValue
+          if (state.dnslinkPolicy === 'detectIpfsPathHeader' && !state.detectIpfsPathHeader) {
+            await browser.storage.local.set({ detectIpfsPathHeader: true })
+          }
+          break
         case 'linkify':
         case 'catchUnhandledProtocols':
         case 'displayNotifications':
         case 'automaticMode':
-        case 'dnslink':
+        case 'detectIpfsPathHeader':
         case 'preloadAtPublicGateway':
+          console.log(`state[${key}]=${change.newValue}`)
           state[key] = change.newValue
           break
       }

add-on/src/lib/ipfs-request.js

@@ -11,6 +11,7 @@ const onHeadersReceivedRedirect = new Set()
 function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, runtime) {
   // Request modifier provides event listeners for the various stages of making an HTTP request
   // API Details: https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/webRequest
+  const browser = runtime.browser
   return {
     onBeforeRequest (request) {
       // This event is triggered when a request is about to be made, and before headers are available.
@@ -46,7 +47,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
           return redirectToGateway(request.url, state, dnslinkResolver)
         }
         // Detect dnslink using heuristics enabled in Preferences
-        if (state.dnslink && dnslinkResolver.isDnslookupSafeForURL(request.url)) {
+        if (dnslinkResolver.isDnslookupSafeForURL(request.url)) {
           const dnslinkRedirect = dnslinkResolver.dnslinkRedirect(request.url)
           if (dnslinkRedirect && isSafeToRedirect(request, runtime)) {
             return dnslinkRedirect
@@ -110,18 +111,18 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
         // and upgrade transport to IPFS using path from header value
         // TODO: hide this behind a flag
         // https://github.com/ipfs-shipyard/ipfs-companion/issues/548#issuecomment-410891930
-        if (request.responseHeaders && !request.url.startsWith(state.gwURLString) && !request.url.startsWith(state.apiURLString)) {
+        if (state.detectIpfsPathHeader && request.responseHeaders && !request.url.startsWith(state.gwURLString) && !request.url.startsWith(state.apiURLString)) {
           console.log('onHeadersReceived.request', request)
           for (let header of request.responseHeaders) {
             if (header.name.toLowerCase() === 'x-ipfs-path' && isSafeToRedirect(request, runtime)) {
               console.log(`[ipfs-companion] detected x-ipfs-path for ${request.url}: ${header.value}`)
               // First: Check if dnslink heuristic yields any results
               // Note: this depends on which dnslink lookup policy is selecten in Preferences
               // TODO: add tests
-              if (state.dnslink && dnslinkResolver.isDnslookupSafeForURL(request.url)) {
+              if (dnslinkResolver.isDnslookupSafeForURL(request.url)) {
                 // x-ipfs-path is a strong indicator of IPFS support
                 // so we force dnslink lookup to pre-populate dnslink cache
-                // in a way that works even when state.dnslinkEagerDnsTxtLookup is disabled
+                // in a way that works even when state.dnslinkPolicy !== 'eagerDnsTxtLookup'
                 // All the following requests will be upgraded to IPNS
                 const cachedDnslink = dnslinkResolver.readAndCacheDnslink(new URL(request.url).hostname)
                 const dnslinkRedirect = dnslinkResolver.dnslinkRedirect(request.url, cachedDnslink)
@@ -148,16 +149,39 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
       }
     },
 
-    onErrorOccurred (request) {
+    async onErrorOccurred (request) {
       // Fired when a request could not be processed due to an error:
       // for example, a lack of Internet connectivity.
+      const state = getState()
 
-      // Cleanup after https://github.com/ipfs-shipyard/ipfs-companion/issues/436
-      if (onHeadersReceivedRedirect.has(request.requestId)) {
-        onHeadersReceivedRedirect.delete(request.requestId)
-      }
+      if (state.active) {
+        // Identify first request
+        const mainRequest = request.type === 'main_frame'
+
+        // Try to recover via DNSLink
+        if (mainRequest && dnslinkResolver.isDnslookupSafeForURL(request.url)) {
+          // Explicit call to ignore global DNSLink policy and force DNS TXT lookup
+          const cachedDnslink = dnslinkResolver.readAndCacheDnslink(new URL(request.url).hostname)
+          const dnslinkRedirect = dnslinkResolver.dnslinkRedirect(request.url, cachedDnslink)
+          // We can't redirect in onErrorOccurred, so if DNSLink is present
+          // recover by opening IPNS version in a new tab
+          // TODO: add tests and demo
+          if (dnslinkRedirect) {
+            console.log(`[ipfs-companion] onErrorOccurred: recovering using dnslink for ${request.url}`, dnslinkRedirect)
+            const currentTabId = await browser.tabs.query({active: true, currentWindow: true}).then(tabs => tabs[0].id)
+            await browser.tabs.create({
+              active: true,
+              openerTabId: currentTabId,
+              url: dnslinkRedirect.redirectUrl
+            })
+          }
+        }
 
-      // TODO: run dnslookup for request.url and if request was for main frame open it from IPFS in a new tab
+        // Cleanup after https://github.com/ipfs-shipyard/ipfs-companion/issues/436
+        if (onHeadersReceivedRedirect.has(request.requestId)) {
+          onHeadersReceivedRedirect.delete(request.requestId)
+        }
+      }
     }
 
   }
@@ -201,7 +225,7 @@ function postNormalizationSkip (state, request) {
 function redirectToGateway (requestUrl, state, dnslinkResolver) {
   // TODO: redirect to `ipfs://` if hasNativeProtocolHandler === true
   const url = new URL(requestUrl)
-  if (state.dnslink && dnslinkResolver.canRedirectToIpns(url)) {
+  if (state.dnslinkPolicy && dnslinkResolver.canRedirectToIpns(url)) {
     // late dnslink in onHeadersReceived
     return dnslinkResolver.redirectToIpnsPath(url)
   }

add-on/src/lib/options.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const optionDefaults = Object.freeze({
+exports.optionDefaults = Object.freeze({
   active: true, // global ON/OFF switch, overrides everything else
   ipfsNodeType: 'external', // or 'embedded'
   ipfsNodeConfig: JSON.stringify({
@@ -14,8 +14,7 @@ const optionDefaults = Object.freeze({
   useCustomGateway: true,
   automaticMode: true,
   linkify: false,
-  dnslink: true,
-  dnslinkEagerDnsTxtLookup: false,
+  dnslinkPolicy: 'detectIpfsPathHeader',
   detectIpfsPathHeader: true,
   preloadAtPublicGateway: true,
   catchUnhandledProtocols: true,
@@ -26,10 +25,8 @@ const optionDefaults = Object.freeze({
   ipfsProxy: true
 })
 
-exports.optionDefaults = optionDefaults
-
 // `storage` should be a browser.storage.local or similar
-function storeMissingOptions (read, defaults, storage) {
+exports.storeMissingOptions = (read, defaults, storage) => {
   const requiredKeys = Object.keys(defaults)
   const changes = new Set()
   requiredKeys.map(key => {
@@ -53,12 +50,24 @@ function storeMissingOptions (read, defaults, storage) {
   return Promise.all(changes)
 }
 
-exports.storeMissingOptions = storeMissingOptions
-
 function normalizeGatewayURL (url) {
   // https://github.com/ipfs/ipfs-companion/issues/328
   return url
     .replace('/localhost:', '/127.0.0.1:')
 }
 
 exports.normalizeGatewayURL = normalizeGatewayURL
+
+exports.migrateOptions = async (storage) => {
+  // <= v2.4.4
+  // DNSLINK: convert old on/off 'dnslink' flag to text-based 'dnslinkPolicy'
+  const { dnslink } = await storage.get('dnslink')
+  if (dnslink) {
+    console.log(`[ipfs-companion] migrating old dnslink policy '${dnslink}' to 'detectIpfsPathHeader'`)
+    await storage.set({
+      dnslinkPolicy: 'detectIpfsPathHeader',
+      detectIpfsPathHeader: true
+    })
+    await storage.remove('dnslink')
+  }
+}

add-on/src/lib/runtime-checks.js

@@ -26,6 +26,7 @@ async function createRuntimeChecks (browser) {
   const runtimeIsAndroid = platformInfo ? platformInfo.os === 'android' : false
   //
   return Object.freeze({
+    browser,
     isFirefox: runtimeIsFirefox,
     isAndroid: runtimeIsAndroid,
     hasNativeProtocolHandler: runtimeHasNativeProtocol

add-on/src/lib/state.js

@@ -4,27 +4,18 @@
 const offlinePeerCount = -1
 
 function initState (options) {
-  const state = {}
-  // we store the most used values in optimized form
-  // to minimize performance impact on overall browsing experience
-  state.active = options.active
+  // we store options and some pregenerated values to avoid async storage
+  // reads and minimize performance impact on overall browsing experience
+  const state = Object.assign({}, options)
+  // generate some additional values
   state.peerCount = offlinePeerCount
-  state.ipfsNodeType = options.ipfsNodeType
-  state.ipfsNodeConfig = options.ipfsNodeConfig
   state.pubGwURL = new URL(options.publicGatewayUrl)
   state.pubGwURLString = state.pubGwURL.toString()
   state.redirect = options.useCustomGateway
   state.apiURL = new URL(options.ipfsApiUrl)
   state.apiURLString = state.apiURL.toString()
   state.gwURL = new URL(options.customGatewayUrl)
   state.gwURLString = state.gwURL.toString()
-  state.automaticMode = options.automaticMode
-  state.linkify = options.linkify
-  state.dnslink = options.dnslink
-  state.preloadAtPublicGateway = options.preloadAtPublicGateway
-  state.catchUnhandledProtocols = options.catchUnhandledProtocols
-  state.displayNotifications = options.displayNotifications
-  state.ipfsProxy = options.ipfsProxy
   return state
 }