feat: preload on content fetch requests (#1475)

When JS IPFS requests content not stored locally it needs to ask the peers it knows about to provide it if they have it. The peers are not relays so if they don't have it, they won't find it and provide it to JS IPFS. However, if we issue a preload request prior to these requests we prompt the preload nodes to fetch the content using their DHT and they can then provide it to JS IPFS. License: MIT Signed-off-by: Alan Shaw <alan@tableflip.io> * chore: update dependencies (#1473) * chore: update libp2p and is-ipfs dependencies License: MIT Signed-off-by: Alan Shaw <alan@tableflip.io> * chore: update all the deps * fix: rm non used boostrappers from tests * test: increase mfs preload timeout * test: pin-set timeout increase * test: add preload tests for content retrieval License: MIT Signed-off-by: Alan Shaw <alan@tableflip.io> * fix: GET request not HEAD License: MIT Signed-off-by: Alan Shaw <alan@tableflip.io>
ipfs · Jul 29, 2018 · 649b755 · 649b755
1 parent 90e9f68
commit 649b755
Show file tree

Hide file tree

Showing 15 changed files with 223 additions and 45 deletions.
diff --git a/package.json b/package.json
@@ -58,7 +58,7 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs#readme",
   "devDependencies": {
-    "aegir": "^15.0.0",
+    "aegir": "^15.1.0",
     "buffer-loader": "~0.0.1",
     "chai": "^4.1.2",
     "delay": "^3.0.0",
@@ -70,8 +70,8 @@
     "expose-loader": "~0.7.5",
     "form-data": "^2.3.2",
     "hat": "0.0.3",
-    "interface-ipfs-core": "~0.72.0",
-    "ipfsd-ctl": "~0.37.5",
+    "interface-ipfs-core": "~0.72.1",
+    "ipfsd-ctl": "~0.38.0",
     "mocha": "^5.2.0",
     "ncp": "^2.0.0",
     "nexpect": "~0.5.0",
@@ -93,7 +93,7 @@
     "byteman": "^1.3.5",
     "cids": "~0.5.3",
     "debug": "^3.1.0",
-    "file-type": "^8.0.0",
+    "file-type": "^8.1.0",
     "filesize": "^3.6.1",
     "fnv1a": "^1.0.1",
     "fsm-event": "^2.1.0",
@@ -104,30 +104,30 @@
     "hoek": "^5.0.3",
     "human-to-milliseconds": "^1.0.0",
     "interface-datastore": "~0.4.2",
-    "ipfs-api": "^22.2.1",
-    "ipfs-bitswap": "~0.20.2",
+    "ipfs-api": "^22.2.4",
+    "ipfs-bitswap": "~0.20.3",
     "ipfs-block": "~0.7.1",
     "ipfs-block-service": "~0.14.0",
     "ipfs-http-response": "~0.1.2",
-    "ipfs-mfs": "~0.2.2",
+    "ipfs-mfs": "~0.2.3",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.22.1",
     "ipfs-unixfs": "~0.1.15",
     "ipfs-unixfs-engine": "~0.31.3",
     "ipld": "~0.17.3",
     "ipld-dag-cbor": "~0.12.1",
-    "ipld-dag-pb": "~0.14.5",
-    "is-ipfs": "~0.3.2",
+    "ipld-dag-pb": "~0.14.6",
+    "is-ipfs": "~0.4.2",
     "is-pull-stream": "~0.0.0",
     "is-stream": "^1.1.0",
     "joi": "^13.4.0",
     "joi-browser": "^13.4.0",
     "joi-multiaddr": "^2.0.0",
-    "libp2p": "~0.22.0",
+    "libp2p": "~0.23.0",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-circuit": "~0.2.0",
     "libp2p-floodsub": "~0.15.0",
-    "libp2p-kad-dht": "~0.10.0",
+    "libp2p-kad-dht": "~0.10.1",
     "libp2p-keychain": "~0.3.1",
     "libp2p-mdns": "~0.12.0",
     "libp2p-mplex": "~0.8.0",
@@ -138,7 +138,7 @@
     "libp2p-websockets": "~0.12.0",
     "lodash": "^4.17.10",
     "mafmt": "^6.0.0",
-    "mime-types": "^2.1.18",
+    "mime-types": "^2.1.19",
     "mkdirp": "~0.5.1",
     "multiaddr": "^5.0.0",
     "multiaddr-to-uri": "^4.0.0",

diff --git a/src/core/components/block.js b/src/core/components/block.js
@@ -9,8 +9,20 @@ const promisify = require('promisify-es6')
 
 module.exports = function block (self) {
   return {
-    get: promisify((cid, callback) => {
+    get: promisify((cid, options, callback) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
+      options = options || {}
+
       cid = cleanCid(cid)
+
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._blockService.get(cid, callback)
     }),
     put: promisify((block, options, callback) => {
@@ -65,9 +77,18 @@ module.exports = function block (self) {
       cid = cleanCid(cid)
       self._blockService.delete(cid, callback)
     }),
-    stat: promisify((cid, callback) => {
+    stat: promisify((cid, options, callback) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
       cid = cleanCid(cid)
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._blockService.get(cid, (err, block) => {
         if (err) {
           return callback(err)

diff --git a/src/core/components/dag.js b/src/core/components/dag.js
@@ -66,6 +66,10 @@ module.exports = function dag (self) {
         }
       }
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._ipld.get(cid, path, options, callback)
     }),
 
@@ -100,17 +104,28 @@ module.exports = function dag (self) {
         }
       }
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       pull(
         self._ipld.treeStream(cid, path, options),
         pull.collect(callback)
       )
     }),
 
     // TODO - use IPLD selectors once they are implemented
-    _getRecursive: promisify((multihash, callback) => {
+    _getRecursive: promisify((multihash, options, callback) => {
       // gets flat array of all DAGNodes in tree given by multihash
 
-      self.dag.get(new CID(multihash), (err, res) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
+      options = options || {}
+
+      self.dag.get(new CID(multihash), '', options, (err, res) => {
         if (err) { return callback(err) }
 
         mapAsync(res.value.links, (link, cb) => {

diff --git a/src/core/components/files.js b/src/core/components/files.js
@@ -182,11 +182,17 @@ module.exports = function files (self) {
       throw new Error('You must supply an ipfsPath')
     }
 
+    options = options || {}
+
     ipfsPath = normalizePath(ipfsPath)
     const pathComponents = ipfsPath.split('/')
     const restPath = normalizePath(pathComponents.slice(1).join('/'))
     const filterFile = (file) => (restPath && file.path === restPath) || (file.path === ipfsPath)
 
+    if (options.preload !== false) {
+      self._preload(pathComponents[0])
+    }
+
     const d = deferred.source()
 
     pull(
@@ -213,16 +219,21 @@ module.exports = function files (self) {
   }
 
   function _lsPullStreamImmutable (ipfsPath, options) {
+    options = options || {}
+
     const path = normalizePath(ipfsPath)
-    const recursive = options && options.recursive
-    const pathDepth = path.split('/').length
+    const recursive = options.recursive
+    const pathComponents = path.split('/')
+    const pathDepth = pathComponents.length
     const maxDepth = recursive ? global.Infinity : pathDepth
-    const opts = Object.assign({}, {
-      maxDepth: maxDepth
-    }, options)
+    options.maxDepth = options.maxDepth || maxDepth
+
+    if (options.preload !== false) {
+      self._preload(pathComponents[0])
+    }
 
     return pull(
-      exporter(ipfsPath, self._ipld, opts),
+      exporter(ipfsPath, self._ipld, options),
       pull.filter(node =>
         recursive ? node.depth >= pathDepth : node.depth === pathDepth
       ),
@@ -334,8 +345,11 @@ module.exports = function files (self) {
         options = {}
       }
 
-      if (typeof callback !== 'function') {
-        throw new Error('Please supply a callback to ipfs.files.get')
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
       }
 
       pull(
@@ -359,6 +373,13 @@ module.exports = function files (self) {
     }),
 
     getReadableStream: (ipfsPath, options) => {
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       return toStream.source(
         pull(
           exporter(ipfsPath, self._ipld, options),
@@ -375,6 +396,13 @@ module.exports = function files (self) {
     },
 
     getPullStream: (ipfsPath, options) => {
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       return exporter(ipfsPath, self._ipld, options)
     },
 
@@ -384,6 +412,13 @@ module.exports = function files (self) {
         options = {}
       }
 
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       pull(
         _lsPullStreamImmutable(ipfsPath, options),
         pull.collect((err, values) => {
@@ -397,6 +432,13 @@ module.exports = function files (self) {
     }),
 
     lsReadableStreamImmutable: (ipfsPath, options) => {
+      options = options || {}
+
+      if (options.preload !== false) {
+        const pathComponents = normalizePath(ipfsPath).split('/')
+        self._preload(pathComponents[0])
+      }
+
       return toStream.source(_lsPullStreamImmutable(ipfsPath, options))
     },
 

diff --git a/src/core/components/mfs.js b/src/core/components/mfs.js
@@ -7,7 +7,23 @@ module.exports = self => {
   const mfsSelf = Object.assign({}, self)
 
   // A patched dag API to ensure preload doesn't happen for MFS operations
+  // (MFS is preloaded periodically)
   mfsSelf.dag = Object.assign({}, self.dag, {
+    get: promisify((cid, path, opts, cb) => {
+      if (typeof path === 'function') {
+        cb = path
+        path = undefined
+      }
+
+      if (typeof opts === 'function') {
+        cb = opts
+        opts = {}
+      }
+
+      opts = Object.assign({}, opts, { preload: false })
+
+      return self.dag.get(cid, path, opts, cb)
+    }),
     put: promisify((node, opts, cb) => {
       if (typeof opts === 'function') {
         cb = opts

diff --git a/src/core/components/object.js b/src/core/components/object.js
@@ -1,14 +1,14 @@
 'use strict'
 
 const waterfall = require('async/waterfall')
+const setImmediate = require('async/setImmediate')
 const promisify = require('promisify-es6')
 const dagPB = require('ipld-dag-pb')
 const DAGNode = dagPB.DAGNode
 const DAGLink = dagPB.DAGLink
 const CID = require('cids')
 const mh = require('multihashes')
 const Unixfs = require('ipfs-unixfs')
-const assert = require('assert')
 
 function normalizeMultihash (multihash, enc) {
   if (typeof multihash === 'string') {
@@ -116,7 +116,9 @@ module.exports = function object (self) {
       let data
 
       if (template) {
-        assert(template === 'unixfs-dir', 'unkown template')
+        if (template !== 'unixfs-dir') {
+          return setImmediate(() => callback(new Error('unknown template')))
+        }
         data = (new Unixfs('directory')).marshal()
       } else {
         data = Buffer.alloc(0)
@@ -221,6 +223,10 @@ module.exports = function object (self) {
         cid = cid.toV1()
       }
 
+      if (options.preload !== false) {
+        self._preload(cid)
+      }
+
       self._ipld.get(cid, (err, result) => {
         if (err) {
           return callback(err)

diff --git a/src/core/components/pin-set.js b/src/core/components/pin-set.js
@@ -74,7 +74,7 @@ exports = module.exports = function (dag) {
 
           seen[bs58Link] = true
 
-          dag.get(multihash, (err, res) => {
+          dag.get(multihash, '', { preload: false }, (err, res) => {
             if (err) { return someCb(err) }
             searchChildren(res.value, someCb)
           })
@@ -184,7 +184,7 @@ exports = module.exports = function (dag) {
         return callback(new Error('No link found with name ' + name))
       }
 
-      dag.get(link.multihash, (err, res) => {
+      dag.get(link.multihash, '', { preload: false }, (err, res) => {
         if (err) { return callback(err) }
         const keys = []
         const step = link => keys.push(link.multihash)
@@ -211,7 +211,7 @@ exports = module.exports = function (dag) {
 
           if (!emptyKey.equals(linkHash)) {
             // walk the links of this fanout bin
-            return dag.get(linkHash, (err, res) => {
+            return dag.get(linkHash, '', { preload: false }, (err, res) => {
               if (err) { return eachCb(err) }
               pinSet.walkItems(res.value, step, eachCb)
             })

diff --git a/src/core/components/pin.js b/src/core/components/pin.js
@@ -360,7 +360,7 @@ module.exports = (self) => {
         (_, cb) => repo.datastore.has(pinDataStoreKey, cb),
         (has, cb) => has ? cb() : cb(new Error('No pins to load')),
         (cb) => repo.datastore.get(pinDataStoreKey, cb),
-        (mh, cb) => dag.get(new CID(mh), cb)
+        (mh, cb) => dag.get(new CID(mh), '', { preload: false }, cb)
       ], (err, pinRoot) => {
         if (err) {
           if (err.message === 'No pins to load') {

diff --git a/src/core/runtime/preload-browser.js b/src/core/runtime/preload-browser.js
@@ -11,7 +11,7 @@ module.exports = function preload (url, callback) {
 
   const req = new self.XMLHttpRequest()
 
-  req.open('HEAD', url)
+  req.open('GET', url)
 
   req.onreadystatechange = function () {
     if (this.readyState !== this.DONE) {