feat: jsipfs add --only-hash (#1233)

* add --only-hash flag

* skeleton http option passing, checkpoint commit to check some broken tests.

* deep logging for debugging. Checkpoint

* found the levledown problem! line endings!

* add only-hash as a qs option for the send-files-stream. Also added some tests to help me figure that out

* update --only-hash test, increase timeout for the afterAll hook. I think it takes longer because the --only-hash test leaves an unresolved ipfs.ls command. Would love to be able to cancel it :)

* ipfs-exec/ipfs.fail should throw on a non-failing command && use random file for ipfs add --only-hash test.

* fix an ipfs config test.

* lint

* clean: move test/http-api/extra/files.js to test/http-api/files.js

src/cli/commands/files/add.js

@@ -139,6 +139,12 @@ module.exports = {
       type: 'boolean',
       default: false
     },
+    'only-hash': {
+      alias: 'n',
+      type: 'boolean',
+      default: false,
+      describe: 'Only chunk and hash, do not write'
+    },
     'enable-sharding-experiment': {
       type: 'boolean',
       default: false
@@ -182,7 +188,8 @@ module.exports = {
       strategy: argv.trickle ? 'trickle' : 'balanced',
       shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity,
       'cid-version': argv['cid-version'],
-      'raw-leaves': argv['raw-leaves']
+      'raw-leaves': argv['raw-leaves'],
+      onlyHash: argv['only-hash']
     }
 
     // Temporary restriction on raw-leaves:
@@ -230,8 +237,7 @@ module.exports = {
           }
         }
 
-        const thing = (cb) => cb(null, ipfs.files.addPullStream(options))
-        thing(next)
+        next(null, ipfs.files.addPullStream(options))
       }
     ], (err, addStream) => {
       if (err) throw err

src/core/components/files.js

@@ -23,7 +23,7 @@ function prepareFile (self, opts, file, callback) {
   opts = opts || {}
 
   waterfall([
-    (cb) => self.object.get(file.multihash, cb),
+    (cb) => opts.onlyHash ? cb(null, file) : self.object.get(file.multihash, cb),
     (node, cb) => {
       let cid = new CID(node.multihash)
 

src/http/api/resources/files.js

@@ -147,7 +147,8 @@ exports.add = {
           is: 1,
           then: Joi.boolean().valid(false).required(),
           otherwise: Joi.boolean().valid(false)
-        })
+        }),
+        'only-hash': Joi.boolean()
       })
       // TODO: Necessary until validate "recursive", "stream-channels" etc.
       .options({ allowUnknown: true })
@@ -205,7 +206,8 @@ exports.add = {
     const options = {
       'cid-version': request.query['cid-version'],
       'raw-leaves': request.query['raw-leaves'],
-      progress: request.query['progress'] ? progressHandler : null
+      progress: request.query.progress ? progressHandler : null,
+      onlyHash: Boolean(request.query['only-hash'])
     }
 
     const aborter = abortable()

test/cli/config.js

@@ -68,7 +68,8 @@ describe('config', () => runOnAndOff((thing) => {
     })
 
     it('call config with no arguments', () => {
-      return ipfs.fail('config')
+      return ipfs('config')
+        .then(out => expect(out).to.include('bin.js config <key> [value]'))
     })
   })
 

test/cli/files.js

@@ -2,6 +2,7 @@
 'use strict'
 
 const fs = require('fs')
+const os = require('os')
 const expect = require('chai').expect
 const path = require('path')
 const compareDir = require('dir-compare').compareSync
@@ -270,6 +271,35 @@ describe('files', () => runOnAndOff((thing) => {
       })
   })
 
+  it('add --only-hash outputs correct hash', function () {
+    return ipfs('files add --only-hash src/init-files/init-docs/readme')
+      .then(out =>
+        expect(out)
+          .to.eql('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme\n')
+      )
+  })
+
+  it('add --only-hash does not add a file to the datastore', function () {
+    this.timeout(30 * 1000)
+    this.slow(10 * 1000)
+    const content = String(Math.random() + Date.now())
+    const filepath = path.join(os.tmpdir(), `${content}.txt`)
+    fs.writeFileSync(filepath, content)
+
+    return ipfs(`files add --only-hash ${filepath}`)
+      .then(out => {
+        const hash = out.split(' ')[1]
+
+        // 'jsipfs object get <hash>' should timeout with the daemon on
+        // and should fail fast with the daemon off
+        return Promise.race([
+          ipfs.fail(`object get ${hash}`),
+          new Promise((resolve, reject) => setTimeout(resolve, 4000))
+        ])
+          .then(() => fs.unlinkSync(filepath))
+      })
+  })
+
   it('cat', function () {
     this.timeout(30 * 1000)
 

test/http-api/files.js

@@ -0,0 +1,42 @@
+/* eslint-env mocha */
+/* eslint max-nested-callbacks: ["error", 8] */
+'use strict'
+
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+chai.use(dirtyChai)
+
+describe('.files', () => {
+  let ipfs = null
+  let ipfsd = null
+  before(function (done) {
+    this.timeout(20 * 1000)
+    df.spawn({ initOptions: { bits: 512 } }, (err, _ipfsd) => {
+      expect(err).to.not.exist()
+      ipfsd = _ipfsd
+      ipfs = ipfsd.api
+      done()
+    })
+  })
+
+  after((done) => ipfsd.stop(done))
+
+  describe('.add', function () {
+    it('performs a speculative add, --only-hash', () => {
+      const content = String(Math.random())
+
+      return ipfs.add(Buffer.from(content), { onlyHash: true })
+        .then(files => {
+          const getAttempt = ipfs.object.get(files[0].hash)
+            .then(() => {
+              throw new Error('Should not find an object for content added with --only-hash')
+            })
+
+          return Promise.race([
+            getAttempt,
+            new Promise((resolve, reject) => setTimeout(resolve, 4000))
+          ])
+        })
+    })
+  })
+})

test/utils/ipfs-exec.js

@@ -51,15 +51,28 @@ module.exports = (repoPath, opts) => {
     return res
   }
 
+  /**
+   * Expect the command passed as @param arguments to fail.
+   * @return {Promise} Resolves if the command passed as @param arguments fails,
+   *                    rejects if it was successful.
+   */
   ipfs.fail = function ipfsFail () {
     let args = Array.from(arguments)
+    let caught = false
     if (args.length === 1) {
       args = args[0].split(' ')
     }
 
-    return exec(args).catch((err) => {
-      expect(err).to.exist()
-    })
+    return exec(args)
+      .catch(err => {
+        caught = true
+        expect(err).to.exist()
+      })
+      .then(() => {
+        if (!caught) {
+          throw new Error(`jsipfs expected to fail during command: jsipfs ${args.join(' ')}`)
+        }
+      })
   }
 
   return ipfs