feat: jsipfs pin improvements (ipfs#1249)

* initial sweep through to understand how pin works. Did make some changes but mostly minor.

* refactor pb schema to it's own file

* fix: don't pin files during files.add if opts.pin === false

* feat: add some http qs parsing, http route/resources cleanup, cleanup core/utils.parseIpfsPath

* feat: expand pin tests. \nFirst draft. still needs some further work.

* feat: Add logging for entry/exit of pins: add/rm/flush/load. Clean some documentation.

* feat: add --pin to files.add, fix: improper pin option parsing in core.

* feat: Use ipfs.files.add to add init-docs instead of directly using the unix-fs importer.

* feat(tests): Add tests for cli --pin option. I know this should be more of an integration test. Should be written in /core. Maybe talk with Victor about testing different layers

* feat: use isIPFS to valiate a multihash.

package.json

@@ -108,6 +108,7 @@
     "hoek": "^5.0.3",
     "human-to-milliseconds": "^1.0.0",
     "ipfs-api": "^18.2.0",
+    "interface-datastore": "^0.4.1",
     "ipfs-bitswap": "~0.19.0",
     "ipfs-block": "~0.6.1",
     "ipfs-block-service": "~0.13.0",

src/cli/commands/files/add.js

@@ -178,6 +178,11 @@ module.exports = {
       type: 'boolean',
       default: false,
       describe: 'Write no output'
+    },
+    pin: {
+      type: 'boolean',
+      default: true,
+      describe: 'Pin this object when adding'
     }
   },
 
@@ -192,6 +197,7 @@ module.exports = {
       cidVersion: argv.cidVersion,
       rawLeaves: argv.rawLeaves,
       onlyHash: argv.onlyHash
+      pin: argv.pin
     }
 
     // Temporary restriction on raw-leaves:

src/cli/commands/pin/add.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'add <ipfs-path>',
+
+  describe: 'Pins objects to local storage.',
+
+  builder: {
+    recursive: {
+      type: 'boolean',
+      alias: 'r',
+      default: true,
+      describe: 'Recursively pin the object linked to by the specified object(s).'
+    }
+  },
+
+  handler (argv) {
+    const paths = argv['ipfs-path'].split(' ')
+    const recursive = argv.recursive
+    const type = recursive ? 'recursive' : 'direct'
+    argv.ipfs.pin.add(paths[0], { recursive: recursive }, (err, results) => {
+      if (err) { throw err }
+      results.forEach((res) => {
+        print(`pinned ${res.hash} ${type}ly`)
+      })
+    })
+  }
+}

src/cli/commands/pin/ls.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  // bracket syntax with '...' tells yargs to optionally accept a list
+  command: 'ls [ipfs-path...]',
+
+  describe: 'List objects pinned to local storage.',
+
+  builder: {
+    type: {
+      type: 'string',
+      alias: 't',
+      default: 'all',
+      choices: ['direct', 'indirect', 'recursive', 'all'],
+      describe: 'The type of pinned keys to list.'
+    },
+    quiet: {
+      type: 'boolean',
+      alias: 'q',
+      default: false,
+      describe: 'Write just hashes of objects.'
+    }
+  },
+
+  handler: (argv) => {
+    const paths = argv.ipfsPath || ''
+    const type = argv.type
+    const quiet = argv.quiet
+    argv.ipfs.pin.ls(paths, { type: type }, (err, results) => {
+      if (err) { throw err }
+      results.forEach((res) => {
+        let line = res.hash
+        if (!quiet) {
+          line += ` ${res.type}`
+        }
+        print(line)
+      })
+    })
+  }
+}

src/cli/commands/pin/rm.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'rm <ipfs-path>',
+
+  describe: 'Removes the pinned object from local storage.',
+
+  builder: {
+    recursive: {
+      type: 'boolean',
+      alias: 'r',
+      default: true,
+      describe: 'Recursively unpin the objects linked to by the specified object(s).'
+    }
+  },
+
+  handler: (argv) => {
+    const paths = argv['ipfs-path'].split(' ')
+    const recursive = argv.recursive
+    argv.ipfs.pin.rm(paths, { recursive: recursive }, (err, results) => {
+      if (err) { throw err }
+      results.forEach((res) => {
+        print(`unpinned ${res.hash}`)
+      })
+    })
+  }
+}

src/core/components/files.js

@@ -18,6 +18,7 @@ const CID = require('cids')
 const toB58String = require('multihashes').toB58String
 
 function noop () {}
+function identity (x) { return x }
 
 function prepareFile (self, opts, file, callback) {
   opts = opts || {}
@@ -111,7 +112,8 @@ module.exports = function files (self) {
     }, options)
 
     let total = 0
-    let prog = opts.progress || (() => {})
+    const shouldPin = 'pin' in opts ? opts.pin : true
+    const prog = opts.progress || noop
     const progress = (bytes) => {
       total += bytes
       prog(total)
@@ -122,7 +124,8 @@ module.exports = function files (self) {
       pull.map(normalizeContent),
       pull.flatten(),
       importer(self._ipld, opts),
-      pull.asyncMap(prepareFile.bind(null, self, opts))
+      pull.asyncMap(prepareFile.bind(null, self, opts)),
+      shouldPin ? pull.asyncMap(pinFile.bind(null, self)) : identity
     )
   }
 

src/core/components/init-assets.js

@@ -15,23 +15,20 @@ module.exports = function addDefaultAssets (self, log, callback) {
 
   pull(
     pull.values([initDocsPath]),
-    pull.asyncMap((val, cb) => glob(path.join(val, '/**/*'), cb)),
+    pull.asyncMap((val, cb) =>
+      glob(path.join(val, '/**/*'), { nodir: true }, cb)
+    ),
     pull.flatten(),
-    pull.map((element) => {
+    pull.map(element => {
       const addPath = element.substring(index + 1)
-
-      if (fs.statSync(element).isDirectory()) { return }
-
       return { path: addPath, content: file(element) }
     }),
-    // Filter out directories, which are undefined from above
-    pull.filter(Boolean),
-    importer(self._ipld),
-    pull.through((el) => {
-      if (el.path === 'init-docs') {
-        const cid = new CID(el.multihash)
+    self.files.addPullStream(),
+    pull.through(file => {
+      if (file.path === 'init-docs') {
+        const cid = new CID(file.hash)
         log('to get started, enter:\n')
-        log(`\t jsipfs files cat /ipfs/${cid.toBaseEncodedString()}/readme\n`)
+        log(`\tjsipfs files cat /ipfs/${cid.toBaseEncodedString()}/readme\n`)
       }
     }),
     pull.collect((err) => {

src/core/components/init.js

@@ -85,16 +85,11 @@ module.exports = function init (self) {
         }
 
         self.log('adding assets')
-        const tasks = [
+        parallel([
           // add empty unixfs dir object (go-ipfs assumes this exists)
-          (cb) => self.object.new('unixfs-dir', cb)
-        ]
-
-        if (typeof addDefaultAssets === 'function') {
-          tasks.push((cb) => addDefaultAssets(self, opts.log, cb))
-        }
-
-        parallel(tasks, (err) => {
+          (cb) => self.object.new('unixfs-dir', cb),
+          (cb) => addDefaultAssets(self, opts.log, cb)
+        ], (err) => {
           if (err) {
             cb(err)
           } else {