refactor: remove webui TAR from Chromium bundle

We were bundling TAR inside of Chromium package to make it work instantly when opened in Brave. Unfortunately it added 10MB to the package, which meant one-click install was slow in Brave, but also on every other Chromium browser. This replaces bundled TAR with prefetching it from a public gateway on the first run. The idea is that if user had access to Chrome Web Store to install extension, they have access to the gateway, so no need for bundling. We also precache regular webui in Chromium and Firefox via ipfs.refs
ipfs · Oct 5, 2020 · ed319fa · ed319fa
1 parent 7ae89e6
commit ed319fa
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 52 deletions.
diff --git a/add-on/src/lib/ipfs-client/index.js b/add-on/src/lib/ipfs-client/index.js
@@ -34,7 +34,7 @@ async function initIpfsClient (opts) {
 
   const instance = await client.init(opts)
   easeApiChanges(instance)
-  _reloadIpfsClientDependents(instance) // async (API is present)
+  _reloadIpfsClientDependents(instance, opts) // async (API is present)
   return instance
 }
 
@@ -71,9 +71,9 @@ async function _reloadIpfsClientDependents (instance, opts) {
     }
   }
   // online only
-  if (client && instance) {
+  if (client && instance && opts) {
     // add important data to local ipfs repo for instant load
-    setTimeout(() => precache(instance), 10000)
+    setTimeout(() => precache(instance, opts), 5000)
   }
 }
 

diff --git a/add-on/src/lib/ipfs-companion.js b/add-on/src/lib/ipfs-companion.js
@@ -311,8 +311,6 @@ module.exports = async function init () {
     let results
     try {
       const dataSrc = await findValueForContext(context, contextType)
-      log('onAddFromContext.context', context) // TODO
-      log('onAddFromContext.dataSrc', dataSrc) // TODO
       if (contextType === 'selection') {
         // TODO: persist full pageUrl somewhere (eg. append at the end of the content but add toggle to disable it)
         data = {

diff --git a/add-on/src/lib/precache.js b/add-on/src/lib/precache.js
@@ -11,48 +11,86 @@ const debug = require('debug')
 const log = debug('ipfs-companion:precache')
 log.error = debug('ipfs-companion:precache:error')
 
-// Web UI release that should be precached
-// WARNING: do not remove this constant, as its used in package.json
-const webuiCid = 'bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4' // v2.9.0
-module.exports.precachedWebuiCid = webuiCid
-
-const PRECACHE_ARCHIVES = [
-  { tarPath: '/dist/precache/webui.tar', cid: webuiCid }
-]
+// TODO: can be removed when embedded:chromesockets is gone
+module.exports.braveJsIpfsWebuiCid = 'bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4' //  v2.9.0 for js-ipfs in Brave (import in newer ones does not work)
 
 /**
  * Adds important assets such as Web UI to the local js-ipfs-repo.
  * This ensures they load instantly, even in offline environments.
  */
-module.exports.precache = async (ipfs) => {
-  for (const { cid, tarPath } of PRECACHE_ARCHIVES) {
-    if (!await inRepo(ipfs, cid)) {
-      try {
-        const { body } = await fetch(tarPath)
-        log(`importing ${tarPath} to js-ipfs-repo`)
-        await importTar(ipfs, body.getReader(), cid)
-        log(`${tarPath} successfully cached under CID ${cid}`)
-      } catch (err) {
-        if (err.message === 'Error in body stream') {
-          // This error means .tar is missing from the extension bundle:
-          // It is the case in Firefox, due to https://github.com/ipfs-shipyard/ipfs-webui/issues/959
-          log(`unable to find/read ${tarPath}, skipping`)
+module.exports.precache = async (ipfs, state) => {
+  // simplified prefetch over HTTP when in Brave
+  if (state.ipfsNodeType === 'embedded:chromesockets') {
+    return preloadOverHTTP(log, ipfs, state, module.exports.braveJsIpfsWebuiCid)
+  }
+
+  const roots = []
+  // find out the content path of webui, and add it to precache list
+  try {
+    let cid, name
+    if (state.useLatestWebUI) { // resolve DNSLink
+      cid = await ipfs.dns('webui.ipfs.io', { recursive: true })
+      name = 'latest webui from DNSLink at webui.ipfs.io'
+    } else { // find out safelisted path behind <api-port>/webui
+      cid = new URL((await fetch(`${state.apiURLString}webui`)).url).pathname
+      name = `stable webui hardcoded at ${state.apiURLString}webui`
+    }
+    roots.push({
+      nodeType: 'external',
+      name,
+      cid
+    })
+  } catch (e) {
+    log.error('unable to find webui content path for precache', e)
+  }
+
+  // precache each root
+  for (const { name, cid, nodeType } of roots) {
+    if (state.ipfsNodeType !== nodeType) continue
+    if (await inRepo(ipfs, cid)) {
+      log(`${name} (${cid}) already in local repo, skipping import`)
+      continue
+    }
+    log(`importing ${name} (${cid}) to local ipfs repo`)
+
+    // prefetch over IPFS
+    try {
+      for await (const ref of ipfs.refs(cid, { recursive: true })) {
+        if (ref.err) {
+          log.error(`error while preloading ${name} (${cid})`, ref.err)
           continue
         }
-        log.error(`error while processing ${tarPath}`, err)
       }
-    } else {
-      log(`${cid} already in local repo, skipping import`)
+      log(`${name} successfully cached under CID ${cid}`)
+    } catch (err) {
+      log.error(`error while processing ${name}`, err)
     }
   }
 }
 
 async function inRepo (ipfs, cid) {
-  for await (const ref of ipfs.refs.local()) {
-    if (ref.err) return false
-    if (ref.ref === cid) return true
+  // dag.get in offline mode will throw block is not present in local repo
+  // (we also have timeout as a failsafe)
+  try {
+    await ipfs.dag.get(cid, { offline: true, timeout: 5000 })
+    return true
+  } catch (_) {
+    return false
+  }
+}
+
+// Downloads CID from a public gateway
+// (alternative to ipfs.refs -r)
+async function preloadOverHTTP (log, ipfs, state, cid) {
+  const url = `${state.pubGwURLString}api/v0/get?arg=${cid}&archive=true`
+  try {
+    log(`importing ${url} (${cid}) to local ipfs repo`)
+    const { body } = await fetch(url)
+    await importTar(ipfs, body.getReader(), cid)
+    log(`successfully fetched TAR from ${url} and cached under CID ${cid}`)
+  } catch (err) {
+    log.error(`error while processing ${url}`, err)
   }
-  return false
 }
 
 async function importTar (ipfs, tarReader, expectedCid) {
@@ -84,7 +122,7 @@ async function importTar (ipfs, tarReader, expectedCid) {
 
   const root = results.find(e => e.cid.toString(multibaseName) === expectedCid)
   if (!root) {
-    throw new Error('imported CID does not match expected one')
+    throw new Error(`imported CID (${root}) does not match expected one: ${expectedCid}`)
   }
 }
 

diff --git a/add-on/src/lib/state.js b/add-on/src/lib/state.js
@@ -2,7 +2,7 @@
 /* eslint-env browser, webextensions */
 
 const { safeURL } = require('./options')
-const { precachedWebuiCid } = require('./precache')
+const { braveJsIpfsWebuiCid } = require('./precache')
 const offlinePeerCount = -1
 
 function initState (options, overrides) {
@@ -48,12 +48,11 @@ function initState (options, overrides) {
       // Did user opt-in for rolling release published on DNSLink?
       if (state.useLatestWebUI) return `${state.gwURLString}ipns/webui.ipfs.io/`
 
-      // Below is needed to make webui work for embedded js-ipfs
-      // TODO: revisit if below is still needed after upgrading to js-ipfs >= 44
-      const webuiUrl = state.ipfsNodeType === 'embedded:chromesockets'
-        ? `${state.gwURLString}ipfs/${precachedWebuiCid}/`
-        : `${state.apiURLString}webui`
-      return webuiUrl
+      // Below is needed to make webui work for embedded js-ipfs in Brave
+      if (state.ipfsNodeType === 'embedded:chromesockets' && braveJsIpfsWebuiCid) {
+        return `${state.gwURLString}ipfs/${braveJsIpfsWebuiCid}/`
+      }
+      return `${state.apiURLString}webui`
     }
   })
   // apply optional overrides

diff --git a/package.json b/package.json
@@ -23,22 +23,20 @@
     "build:bundle-all": "cross-env RELEASE_CHANNEL=${RELEASE_CHANNEL:=dev} run-s bundle:chromium bundle:brave:$RELEASE_CHANNEL bundle:firefox:$RELEASE_CHANNEL",
     "build:rename-artifacts": "./scripts/rename-artifacts.js",
     "precache:clean": "shx rm -rf add-on/dist/precache",
-    "precache:webui:cid": "shx grep 'const webuiCid' add-on/src/lib/precache.js | shx sed \"s/^const webuiCid = '//\" | shx sed \"s/'.*$//\"",
-    "precache:webui": "shx mkdir -p add-on/dist/precache && ipfs-or-gateway -c $(npm run -s precache:webui:cid) -p add-on/dist/precache/webui.tar --archive",
     "bundle": "run-s bundle:*",
-    "bundle:chromium": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/chromium && run-s build:rename-artifacts",
-    "bundle:firefox": "run-s precache:clean && shx cat add-on/manifest.common.json add-on/manifest.firefox.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/  && run-s build:rename-artifacts",
+    "bundle:chromium": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/chromium && run-s build:rename-artifacts",
+    "bundle:firefox": "shx cat add-on/manifest.common.json add-on/manifest.firefox.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/  && run-s build:rename-artifacts",
     "bundle:firefox:dev": "npm run bundle:firefox",
     "bundle:firefox:stable": "npm run bundle:firefox",
-    "bundle:firefox:beta": "run-s precache:clean && shx cat add-on/manifest.common.json add-on/manifest.firefox.json add-on/manifest.firefox-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/  && run-s build:rename-artifacts",
+    "bundle:firefox:beta": "shx cat add-on/manifest.common.json add-on/manifest.firefox.json add-on/manifest.firefox-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/  && run-s build:rename-artifacts",
     "bundle:fennec": "npm run bundle:firefox",
     "bundle:fennec:dev": "npm run bundle:firefox:dev",
     "bundle:fennec:stable": "npm run bundle:firefox:stable",
     "bundle:fennec:beta": "npm run bundle:firefox:beta",
-    "bundle:brave": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/  && run-s build:rename-artifacts",
+    "bundle:brave": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/  && run-s build:rename-artifacts",
     "bundle:brave:dev": "npm run bundle:brave",
     "bundle:brave:stable": "npm run bundle:brave",
-    "bundle:brave:beta": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json add-on/manifest.brave-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/  && run-s build:rename-artifacts",
+    "bundle:brave:beta": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json add-on/manifest.brave-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/  && run-s build:rename-artifacts",
     "watch": "npm-run-all build:copy --parallel watch:*",
     "watch:js": "run-p watch:js:*",
     "watch:js:webpack": "webpack --watch --progress -d --devtool inline-source-map  --config ./webpack.config.js",
@@ -144,12 +142,10 @@
     "is-fqdn": "1.0.1",
     "is-ipfs": "1.0.3",
     "it-all": "1.0.4",
-    "it-buffer": "0.1.2",
     "it-concat": "1.0.1",
     "it-tar": "1.2.2",
     "lru-cache": "5.1.1",
     "merge-options": "2.0.0",
-    "mortice": "2.0.0",
     "multiaddr": "7.4.3",
     "multiaddr-to-uri": "5.0.0",
     "p-memoize": "4.0.0",

diff --git a/yarn.lock b/yarn.lock
@@ -8436,7 +8436,7 @@ it-batch@^1.0.3, it-batch@^1.0.4:
   resolved "https://registry.yarnpkg.com/it-batch/-/it-batch-1.0.4.tgz#2d68d6e57899e159644aa16c0b4b1c93eb0d5c65"
   integrity sha512-hZ+gaj5MaECauRd+Ahvo9iAxg90YGVBg7AZ32wOeXJ08IRjfQRMSnZ9oA0JjNeJeSGuVjWf91UUD5y2SYmKlwQ==
 
-it-buffer@0.1.2, it-buffer@^0.1.1, it-buffer@^0.1.2:
+it-buffer@^0.1.1, it-buffer@^0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/it-buffer/-/it-buffer-0.1.2.tgz#2b37e2c66bbbb94479c2e47c1904bd729f04fc39"
   integrity sha512-NOJ3ogSNq3Y2c75ZDcPs9qlgitWyCkUQdmgqqMw+/LMmHZqwWQw7OBDodonz250nJ4EEBXkRQ+pIwz1sL9Zuyg==
@@ -10677,7 +10677,7 @@ moment@^2.19.3:
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.28.0.tgz#cdfe73ce01327cee6537b0fafac2e0f21a237d75"
   integrity sha512-Z5KOjYmnHyd/ukynmFd/WwyXHd7L4J9vTI/nn5Ap9AVUgaAE15VvQ9MOGmJJygEUklupqIrFnor/tjTwRU+tQw==
 
-mortice@2.0.0, mortice@^2.0.0:
+mortice@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/mortice/-/mortice-2.0.0.tgz#7be171409c2115561ba3fc035e4527f9082eefde"
   integrity sha512-rXcjRgv2MRhpwGHErxKcDcp5IoA9CPvPFLXmmseQYIuQ2fSVu8tsMKi/eYUXzp/HH1s6y3IID/GwRqlSglDdRA==