From ed319fa492ae380f21d2b4788dde9fad008e703c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 30 Sep 2020 00:46:29 +0200 Subject: [PATCH] refactor: remove webui TAR from Chromium bundle We were bundling TAR inside of Chromium package to make it work instantly when opened in Brave. Unfortunately it added 10MB to the package, which meant one-click install was slow in Brave, but also on every other Chromium browser. This replaces bundled TAR with prefetching it from a public gateway on the first run. The idea is that if user had access to Chrome Web Store to install extension, they have access to the gateway, so no need for bundling. We also precache regular webui in Chromium and Firefox via ipfs.refs --- add-on/src/lib/ipfs-client/index.js | 6 +- add-on/src/lib/ipfs-companion.js | 2 - add-on/src/lib/precache.js | 96 ++++++++++++++++++++--------- add-on/src/lib/state.js | 13 ++-- package.json | 14 ++--- yarn.lock | 4 +- 6 files changed, 83 insertions(+), 52 deletions(-) diff --git a/add-on/src/lib/ipfs-client/index.js b/add-on/src/lib/ipfs-client/index.js index f8be70403..64426ad59 100644 --- a/add-on/src/lib/ipfs-client/index.js +++ b/add-on/src/lib/ipfs-client/index.js @@ -34,7 +34,7 @@ async function initIpfsClient (opts) { const instance = await client.init(opts) easeApiChanges(instance) - _reloadIpfsClientDependents(instance) // async (API is present) + _reloadIpfsClientDependents(instance, opts) // async (API is present) return instance } @@ -71,9 +71,9 @@ async function _reloadIpfsClientDependents (instance, opts) { } } // online only - if (client && instance) { + if (client && instance && opts) { // add important data to local ipfs repo for instant load - setTimeout(() => precache(instance), 10000) + setTimeout(() => precache(instance, opts), 5000) } } diff --git a/add-on/src/lib/ipfs-companion.js b/add-on/src/lib/ipfs-companion.js index f3722fcd4..d5560e7f4 100644 --- a/add-on/src/lib/ipfs-companion.js +++ b/add-on/src/lib/ipfs-companion.js @@ -311,8 +311,6 @@ module.exports = async function init () { let results try { const dataSrc = await findValueForContext(context, contextType) - log('onAddFromContext.context', context) // TODO - log('onAddFromContext.dataSrc', dataSrc) // TODO if (contextType === 'selection') { // TODO: persist full pageUrl somewhere (eg. append at the end of the content but add toggle to disable it) data = { diff --git a/add-on/src/lib/precache.js b/add-on/src/lib/precache.js index 84e1b5862..24ccf15a0 100644 --- a/add-on/src/lib/precache.js +++ b/add-on/src/lib/precache.js @@ -11,48 +11,86 @@ const debug = require('debug') const log = debug('ipfs-companion:precache') log.error = debug('ipfs-companion:precache:error') -// Web UI release that should be precached -// WARNING: do not remove this constant, as its used in package.json -const webuiCid = 'bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4' // v2.9.0 -module.exports.precachedWebuiCid = webuiCid - -const PRECACHE_ARCHIVES = [ - { tarPath: '/dist/precache/webui.tar', cid: webuiCid } -] +// TODO: can be removed when embedded:chromesockets is gone +module.exports.braveJsIpfsWebuiCid = 'bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4' // v2.9.0 for js-ipfs in Brave (import in newer ones does not work) /** * Adds important assets such as Web UI to the local js-ipfs-repo. * This ensures they load instantly, even in offline environments. */ -module.exports.precache = async (ipfs) => { - for (const { cid, tarPath } of PRECACHE_ARCHIVES) { - if (!await inRepo(ipfs, cid)) { - try { - const { body } = await fetch(tarPath) - log(`importing ${tarPath} to js-ipfs-repo`) - await importTar(ipfs, body.getReader(), cid) - log(`${tarPath} successfully cached under CID ${cid}`) - } catch (err) { - if (err.message === 'Error in body stream') { - // This error means .tar is missing from the extension bundle: - // It is the case in Firefox, due to https://github.com/ipfs-shipyard/ipfs-webui/issues/959 - log(`unable to find/read ${tarPath}, skipping`) +module.exports.precache = async (ipfs, state) => { + // simplified prefetch over HTTP when in Brave + if (state.ipfsNodeType === 'embedded:chromesockets') { + return preloadOverHTTP(log, ipfs, state, module.exports.braveJsIpfsWebuiCid) + } + + const roots = [] + // find out the content path of webui, and add it to precache list + try { + let cid, name + if (state.useLatestWebUI) { // resolve DNSLink + cid = await ipfs.dns('webui.ipfs.io', { recursive: true }) + name = 'latest webui from DNSLink at webui.ipfs.io' + } else { // find out safelisted path behind /webui + cid = new URL((await fetch(`${state.apiURLString}webui`)).url).pathname + name = `stable webui hardcoded at ${state.apiURLString}webui` + } + roots.push({ + nodeType: 'external', + name, + cid + }) + } catch (e) { + log.error('unable to find webui content path for precache', e) + } + + // precache each root + for (const { name, cid, nodeType } of roots) { + if (state.ipfsNodeType !== nodeType) continue + if (await inRepo(ipfs, cid)) { + log(`${name} (${cid}) already in local repo, skipping import`) + continue + } + log(`importing ${name} (${cid}) to local ipfs repo`) + + // prefetch over IPFS + try { + for await (const ref of ipfs.refs(cid, { recursive: true })) { + if (ref.err) { + log.error(`error while preloading ${name} (${cid})`, ref.err) continue } - log.error(`error while processing ${tarPath}`, err) } - } else { - log(`${cid} already in local repo, skipping import`) + log(`${name} successfully cached under CID ${cid}`) + } catch (err) { + log.error(`error while processing ${name}`, err) } } } async function inRepo (ipfs, cid) { - for await (const ref of ipfs.refs.local()) { - if (ref.err) return false - if (ref.ref === cid) return true + // dag.get in offline mode will throw block is not present in local repo + // (we also have timeout as a failsafe) + try { + await ipfs.dag.get(cid, { offline: true, timeout: 5000 }) + return true + } catch (_) { + return false + } +} + +// Downloads CID from a public gateway +// (alternative to ipfs.refs -r) +async function preloadOverHTTP (log, ipfs, state, cid) { + const url = `${state.pubGwURLString}api/v0/get?arg=${cid}&archive=true` + try { + log(`importing ${url} (${cid}) to local ipfs repo`) + const { body } = await fetch(url) + await importTar(ipfs, body.getReader(), cid) + log(`successfully fetched TAR from ${url} and cached under CID ${cid}`) + } catch (err) { + log.error(`error while processing ${url}`, err) } - return false } async function importTar (ipfs, tarReader, expectedCid) { @@ -84,7 +122,7 @@ async function importTar (ipfs, tarReader, expectedCid) { const root = results.find(e => e.cid.toString(multibaseName) === expectedCid) if (!root) { - throw new Error('imported CID does not match expected one') + throw new Error(`imported CID (${root}) does not match expected one: ${expectedCid}`) } } diff --git a/add-on/src/lib/state.js b/add-on/src/lib/state.js index 75024895a..2d1ee0caa 100644 --- a/add-on/src/lib/state.js +++ b/add-on/src/lib/state.js @@ -2,7 +2,7 @@ /* eslint-env browser, webextensions */ const { safeURL } = require('./options') -const { precachedWebuiCid } = require('./precache') +const { braveJsIpfsWebuiCid } = require('./precache') const offlinePeerCount = -1 function initState (options, overrides) { @@ -48,12 +48,11 @@ function initState (options, overrides) { // Did user opt-in for rolling release published on DNSLink? if (state.useLatestWebUI) return `${state.gwURLString}ipns/webui.ipfs.io/` - // Below is needed to make webui work for embedded js-ipfs - // TODO: revisit if below is still needed after upgrading to js-ipfs >= 44 - const webuiUrl = state.ipfsNodeType === 'embedded:chromesockets' - ? `${state.gwURLString}ipfs/${precachedWebuiCid}/` - : `${state.apiURLString}webui` - return webuiUrl + // Below is needed to make webui work for embedded js-ipfs in Brave + if (state.ipfsNodeType === 'embedded:chromesockets' && braveJsIpfsWebuiCid) { + return `${state.gwURLString}ipfs/${braveJsIpfsWebuiCid}/` + } + return `${state.apiURLString}webui` } }) // apply optional overrides diff --git a/package.json b/package.json index 0f2a2e9cc..59d63ea7c 100644 --- a/package.json +++ b/package.json @@ -23,22 +23,20 @@ "build:bundle-all": "cross-env RELEASE_CHANNEL=${RELEASE_CHANNEL:=dev} run-s bundle:chromium bundle:brave:$RELEASE_CHANNEL bundle:firefox:$RELEASE_CHANNEL", "build:rename-artifacts": "./scripts/rename-artifacts.js", "precache:clean": "shx rm -rf add-on/dist/precache", - "precache:webui:cid": "shx grep 'const webuiCid' add-on/src/lib/precache.js | shx sed \"s/^const webuiCid = '//\" | shx sed \"s/'.*$//\"", - "precache:webui": "shx mkdir -p add-on/dist/precache && ipfs-or-gateway -c $(npm run -s precache:webui:cid) -p add-on/dist/precache/webui.tar --archive", "bundle": "run-s bundle:*", - "bundle:chromium": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/chromium && run-s build:rename-artifacts", - "bundle:firefox": "run-s precache:clean && shx cat add-on/manifest.common.json add-on/manifest.firefox.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/ && run-s build:rename-artifacts", + "bundle:chromium": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/chromium && run-s build:rename-artifacts", + "bundle:firefox": "shx cat add-on/manifest.common.json add-on/manifest.firefox.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/ && run-s build:rename-artifacts", "bundle:firefox:dev": "npm run bundle:firefox", "bundle:firefox:stable": "npm run bundle:firefox", - "bundle:firefox:beta": "run-s precache:clean && shx cat add-on/manifest.common.json add-on/manifest.firefox.json add-on/manifest.firefox-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/ && run-s build:rename-artifacts", + "bundle:firefox:beta": "shx cat add-on/manifest.common.json add-on/manifest.firefox.json add-on/manifest.firefox-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/ && run-s build:rename-artifacts", "bundle:fennec": "npm run bundle:firefox", "bundle:fennec:dev": "npm run bundle:firefox:dev", "bundle:fennec:stable": "npm run bundle:firefox:stable", "bundle:fennec:beta": "npm run bundle:firefox:beta", - "bundle:brave": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/ && run-s build:rename-artifacts", + "bundle:brave": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/ && run-s build:rename-artifacts", "bundle:brave:dev": "npm run bundle:brave", "bundle:brave:stable": "npm run bundle:brave", - "bundle:brave:beta": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json add-on/manifest.brave-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/ && run-s build:rename-artifacts", + "bundle:brave:beta": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json add-on/manifest.brave-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/ && run-s build:rename-artifacts", "watch": "npm-run-all build:copy --parallel watch:*", "watch:js": "run-p watch:js:*", "watch:js:webpack": "webpack --watch --progress -d --devtool inline-source-map --config ./webpack.config.js", @@ -144,12 +142,10 @@ "is-fqdn": "1.0.1", "is-ipfs": "1.0.3", "it-all": "1.0.4", - "it-buffer": "0.1.2", "it-concat": "1.0.1", "it-tar": "1.2.2", "lru-cache": "5.1.1", "merge-options": "2.0.0", - "mortice": "2.0.0", "multiaddr": "7.4.3", "multiaddr-to-uri": "5.0.0", "p-memoize": "4.0.0", diff --git a/yarn.lock b/yarn.lock index 52a6c9863..890023347 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8436,7 +8436,7 @@ it-batch@^1.0.3, it-batch@^1.0.4: resolved "https://registry.yarnpkg.com/it-batch/-/it-batch-1.0.4.tgz#2d68d6e57899e159644aa16c0b4b1c93eb0d5c65" integrity sha512-hZ+gaj5MaECauRd+Ahvo9iAxg90YGVBg7AZ32wOeXJ08IRjfQRMSnZ9oA0JjNeJeSGuVjWf91UUD5y2SYmKlwQ== -it-buffer@0.1.2, it-buffer@^0.1.1, it-buffer@^0.1.2: +it-buffer@^0.1.1, it-buffer@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/it-buffer/-/it-buffer-0.1.2.tgz#2b37e2c66bbbb94479c2e47c1904bd729f04fc39" integrity sha512-NOJ3ogSNq3Y2c75ZDcPs9qlgitWyCkUQdmgqqMw+/LMmHZqwWQw7OBDodonz250nJ4EEBXkRQ+pIwz1sL9Zuyg== @@ -10677,7 +10677,7 @@ moment@^2.19.3: resolved "https://registry.yarnpkg.com/moment/-/moment-2.28.0.tgz#cdfe73ce01327cee6537b0fafac2e0f21a237d75" integrity sha512-Z5KOjYmnHyd/ukynmFd/WwyXHd7L4J9vTI/nn5Ap9AVUgaAE15VvQ9MOGmJJygEUklupqIrFnor/tjTwRU+tQw== -mortice@2.0.0, mortice@^2.0.0: +mortice@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/mortice/-/mortice-2.0.0.tgz#7be171409c2115561ba3fc035e4527f9082eefde" integrity sha512-rXcjRgv2MRhpwGHErxKcDcp5IoA9CPvPFLXmmseQYIuQ2fSVu8tsMKi/eYUXzp/HH1s6y3IID/GwRqlSglDdRA==