feat: pin API

package.json

@@ -88,7 +88,7 @@
   },
   "dependencies": {
     "async": "^2.6.0",
-    "big.js": "^5.0.3",
+    "big.js": "^5.1.2",
     "binary-querystring": "~0.1.2",
     "bl": "^1.2.2",
     "boom": "^7.2.0",
@@ -98,13 +98,15 @@
     "debug": "^3.1.0",
     "file-type": "^7.7.1",
     "filesize": "^3.6.1",
+    "fnv1a": "^1.0.1",
     "fsm-event": "^2.1.0",
     "get-folder-size": "^1.0.1",
     "glob": "^7.1.2",
     "hapi": "^16.6.2",
     "hapi-set-header": "^1.0.2",
     "hoek": "^5.0.3",
     "human-to-milliseconds": "^1.0.0",
+    "interface-datastore": "^0.4.1",
     "ipfs-api": "^22.0.0",
     "ipfs-bitswap": "~0.20.0",
     "ipfs-block": "~0.7.1",
@@ -137,6 +139,7 @@
     "libp2p-websocket-star": "~0.8.0",
     "libp2p-websockets": "~0.12.0",
     "lodash.flatmap": "^4.5.0",
+    "lodash.flattendeep": "^4.4.0",
     "lodash.get": "^4.4.2",
     "lodash.set": "^4.3.2",
     "lodash.sortby": "^4.7.0",
@@ -218,7 +221,7 @@
     "Jade Meskill <jade.meskill@gmail.com>",
     "Johannes Wikner <johannes.wikner@gmail.com>",
     "Jon Schlinkert <dev@sellside.com>",
-    "Jonathan <jkrone@vt.edu>",
+    "Jonathan Krone <jonathankrone@gmail.com>",
     "João Antunes <j.goncalo.antunes@gmail.com>",
     "João Santos <joaosantos15@users.noreply.github.com>",
     "Kevin Wang <kevin@fossa.io>",

src/cli/commands/files/add.js

@@ -173,6 +173,11 @@ module.exports = {
       type: 'boolean',
       default: false,
       describe: 'Write no output'
+    },
+    pin: {
+      type: 'boolean',
+      default: true,
+      describe: 'Pin this object when adding'
     }
   },
 
@@ -188,7 +193,8 @@ module.exports = {
       rawLeaves: argv.rawLeaves,
       onlyHash: argv.onlyHash,
       hashAlg: argv.hash,
-      wrapWithDirectory: argv.wrapWithDirectory
+      wrapWithDirectory: argv.wrapWithDirectory,
+      pin: argv.pin
     }
 
     // Temporary restriction on raw-leaves:

src/cli/commands/pin.js

@@ -0,0 +1,15 @@
+'use strict'
+
+module.exports = {
+  command: 'pin',
+
+  description: 'Pin and unpin objects to local storage.',
+
+  builder (yargs) {
+    return yargs
+      .commandDir('pin')
+  },
+
+  handler (argv) {
+  }
+}

src/cli/commands/pin/add.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'add <ipfsPath...>',
+
+  describe: 'Pins object to local storage.',
+
+  builder: {
+    recursive: {
+      type: 'boolean',
+      alias: 'r',
+      default: true,
+      describe: 'Recursively pin the object linked to by the specified object(s).'
+    }
+  },
+
+  handler (argv) {
+    const recursive = argv.recursive
+    const type = recursive ? 'recursive' : 'direct'
+    argv.ipfs.pin.add(argv.ipfsPath, { recursive: recursive }, (err, results) => {
+      if (err) { throw err }
+      results.forEach((res) => {
+        print(`pinned ${res.hash} ${type}ly`)
+      })
+    })
+  }
+}

src/cli/commands/pin/ls.js

@@ -0,0 +1,43 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  // bracket syntax with '...' tells yargs to optionally accept a list
+  command: 'ls [ipfsPath...]',
+
+  describe: 'List objects pinned to local storage.',
+
+  builder: {
+    type: {
+      type: 'string',
+      alias: 't',
+      default: 'all',
+      choices: ['direct', 'indirect', 'recursive', 'all'],
+      describe: 'The type of pinned keys to list.'
+    },
+    quiet: {
+      type: 'boolean',
+      alias: 'q',
+      default: false,
+      describe: 'Write just hashes of objects.'
+    }
+  },
+
+  handler: (argv) => {
+    const paths = argv.ipfsPath
+    const type = argv.type
+    const quiet = argv.quiet
+
+    argv.ipfs.pin.ls(paths, { type }, (err, results) => {
+      if (err) { throw err }
+      results.forEach((res) => {
+        let line = res.hash
+        if (!quiet) {
+          line += ` ${res.type}`
+        }
+        print(line)
+      })
+    })
+  }
+}

src/cli/commands/pin/rm.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'rm <ipfsPath...>',
+
+  describe: 'Removes the pinned object from local storage.',
+
+  builder: {
+    recursive: {
+      type: 'boolean',
+      alias: 'r',
+      default: true,
+      describe: 'Recursively unpin the objects linked to by the specified object(s).'
+    }
+  },
+
+  handler: (argv) => {
+    const recursive = argv.recursive
+    argv.ipfs.pin.rm(argv.ipfsPath, { recursive: recursive }, (err, results) => {
+      if (err) { throw err }
+      results.forEach((res) => {
+        print(`unpinned ${res.hash}`)
+      })
+    })
+  }
+}

src/core/boot.js

@@ -29,6 +29,7 @@ module.exports = (self) => {
 
     series([
       (cb) => self._repo.open(cb),
+      (cb) => self.pin._load(cb),
       (cb) => self.preStart(cb),
       (cb) => {
         self.log('initialized')

src/core/components/dag.js

@@ -3,6 +3,8 @@
 const promisify = require('promisify-es6')
 const CID = require('cids')
 const pull = require('pull-stream')
+const mapAsync = require('async/map')
+const flattenDeep = require('lodash.flattendeep')
 
 module.exports = function dag (self) {
   return {
@@ -33,6 +35,12 @@ module.exports = function dag (self) {
         } else {
           path = '/'
         }
+      } else if (Buffer.isBuffer(cid)) {
+        try {
+          cid = new CID(cid)
+        } catch (err) {
+          return callback(err)
+        }
       }
 
       self._ipld.get(cid, path, options, callback)
@@ -73,6 +81,23 @@ module.exports = function dag (self) {
         self._ipld.treeStream(cid, path, options),
         pull.collect(callback)
       )
+    }),
+
+    // TODO - use IPLD selectors once they are implemented
+    _getRecursive: promisify((multihash, callback) => {
+      // gets flat array of all DAGNodes in tree given by multihash
+
+      self.dag.get(new CID(multihash), (err, res) => {
+        if (err) { return callback(err) }
+
+        mapAsync(res.value.links, (link, cb) => {
+          self.dag._getRecursive(link.multihash, cb)
+        }, (err, nodes) => {
+          // console.log('nodes:', nodes)
+          if (err) return callback(err)
+          callback(null, flattenDeep([res.value, nodes]))
+        })
+      })
     })
   }
 }

src/core/components/files.js

@@ -32,7 +32,9 @@ function prepareFile (self, opts, file, callback) {
   }
 
   waterfall([
-    (cb) => opts.onlyHash ? cb(null, file) : self.object.get(file.multihash, opts, cb),
+    (cb) => opts.onlyHash
+      ? cb(null, file)
+      : self.object.get(file.multihash, opts, cb),
     (node, cb) => {
       const b58Hash = cid.toBaseEncodedString()
 
@@ -87,6 +89,19 @@ function normalizeContent (opts, content) {
   })
 }
 
+function pinFile (self, opts, file, cb) {
+  // Pin a file if it is the root dir of a recursive add or the single file
+  // of a direct add.
+  const pin = 'pin' in opts ? opts.pin : true
+  const isRootDir = !file.path.includes('/')
+  const shouldPin = pin && isRootDir && !opts.onlyHash && !opts.hashAlg
+  if (shouldPin) {
+    return self.pin.add(file.hash, err => cb(err, file))
+  } else {
+    cb(null, file)
+  }
+}
+
 class AddHelper extends Duplex {
   constructor (pullStream, push, options) {
     super(Object.assign({ objectMode: true }, options))
@@ -130,7 +145,8 @@ module.exports = function files (self) {
     }
 
     let total = 0
-    let prog = opts.progress || (() => {})
+
+    const prog = opts.progress || noop
     const progress = (bytes) => {
       total += bytes
       prog(total)
@@ -141,7 +157,8 @@ module.exports = function files (self) {
       pull.map(normalizeContent.bind(null, opts)),
       pull.flatten(),
       importer(self._ipld, opts),
-      pull.asyncMap(prepareFile.bind(null, self, opts))
+      pull.asyncMap(prepareFile.bind(null, self, opts)),
+      pull.asyncMap(pinFile.bind(null, self, opts))
     )
   }
 

src/core/components/index.js

@@ -18,6 +18,7 @@ exports.swarm = require('./swarm')
 exports.ping = require('./ping')
 exports.pingPullStream = require('./ping-pull-stream')
 exports.pingReadableStream = require('./ping-readable-stream')
+exports.pin = require('./pin')
 exports.files = require('./files')
 exports.bitswap = require('./bitswap')
 exports.pubsub = require('./pubsub')

src/core/components/init-assets.js

@@ -1,9 +1,7 @@
 'use strict'
 
 const path = require('path')
-const fs = require('fs')
 const glob = require('glob')
-const importer = require('ipfs-unixfs-engine').importer
 const pull = require('pull-stream')
 const file = require('pull-file')
 const CID = require('cids')
@@ -15,23 +13,20 @@ module.exports = function addDefaultAssets (self, log, callback) {
 
   pull(
     pull.values([initDocsPath]),
-    pull.asyncMap((val, cb) => glob(path.join(val, '/**/*'), cb)),
+    pull.asyncMap((val, cb) =>
+      glob(path.join(val, '/**/*'), { nodir: true }, cb)
+    ),
     pull.flatten(),
-    pull.map((element) => {
+    pull.map(element => {
       const addPath = element.substring(index + 1)
-
-      if (fs.statSync(element).isDirectory()) { return }
-
       return { path: addPath, content: file(element) }
     }),
-    // Filter out directories, which are undefined from above
-    pull.filter(Boolean),
-    importer(self._ipld),
-    pull.through((el) => {
-      if (el.path === 'init-docs') {
-        const cid = new CID(el.multihash)
+    self.files.addPullStream(),
+    pull.through(file => {
+      if (file.path === 'init-docs') {
+        const cid = new CID(file.hash)
         log('to get started, enter:\n')
-        log(`\t jsipfs files cat /ipfs/${cid.toBaseEncodedString()}/readme\n`)
+        log(`\tjsipfs files cat /ipfs/${cid.toBaseEncodedString()}/readme\n`)
       }
     }),
     pull.collect((err) => {

src/core/components/init.js

@@ -85,16 +85,18 @@ module.exports = function init (self) {
           return cb(null, true)
         }
 
-        self.log('adding assets')
         const tasks = [
           // add empty unixfs dir object (go-ipfs assumes this exists)
           (cb) => self.object.new('unixfs-dir', cb)
         ]
 
         if (typeof addDefaultAssets === 'function') {
+          // addDefaultAssets is undefined on browsers.
+          // See package.json browser config
           tasks.push((cb) => addDefaultAssets(self, opts.log, cb))
         }
 
+        self.log('adding assets')
         parallel(tasks, (err) => {
           if (err) {
             cb(err)