From 5e5fc96f7daa7a145dbf55c74d330e3675202815 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Fri, 16 Mar 2018 16:56:37 -0400 Subject: [PATCH] feat: jsipfs add --only-hash (#1233) * add --only-hash flag * skeleton http option passing, checkpoint commit to check some broken tests. * deep logging for debugging. Checkpoint * found the levledown problem! line endings! * add only-hash as a qs option for the send-files-stream. Also added some tests to help me figure that out * update --only-hash test, increase timeout for the afterAll hook. I think it takes longer because the --only-hash test leaves an unresolved ipfs.ls command. Would love to be able to cancel it :) * ipfs-exec/ipfs.fail should throw on a non-failing command && use random file for ipfs add --only-hash test. * fix an ipfs config test. * lint * clean: move test/http-api/extra/files.js to test/http-api/files.js --- src/cli/commands/files/add.js | 12 +++++++--- src/core/components/files.js | 2 +- src/http/api/resources/files.js | 6 +++-- test/cli/config.js | 3 ++- test/cli/files.js | 30 +++++++++++++++++++++++ test/http-api/files.js | 42 +++++++++++++++++++++++++++++++++ test/utils/ipfs-exec.js | 19 ++++++++++++--- 7 files changed, 104 insertions(+), 10 deletions(-) create mode 100644 test/http-api/files.js diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 5886ac2365..0dbdffe2c6 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -139,6 +139,12 @@ module.exports = { type: 'boolean', default: false }, + 'only-hash': { + alias: 'n', + type: 'boolean', + default: false, + describe: 'Only chunk and hash, do not write' + }, 'enable-sharding-experiment': { type: 'boolean', default: false @@ -182,7 +188,8 @@ module.exports = { strategy: argv.trickle ? 'trickle' : 'balanced', shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity, 'cid-version': argv['cid-version'], - 'raw-leaves': argv['raw-leaves'] + 'raw-leaves': argv['raw-leaves'], + onlyHash: argv['only-hash'] } // Temporary restriction on raw-leaves: @@ -230,8 +237,7 @@ module.exports = { } } - const thing = (cb) => cb(null, ipfs.files.addPullStream(options)) - thing(next) + next(null, ipfs.files.addPullStream(options)) } ], (err, addStream) => { if (err) throw err diff --git a/src/core/components/files.js b/src/core/components/files.js index 5906487570..4c66d954e0 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -23,7 +23,7 @@ function prepareFile (self, opts, file, callback) { opts = opts || {} waterfall([ - (cb) => self.object.get(file.multihash, cb), + (cb) => opts.onlyHash ? cb(null, file) : self.object.get(file.multihash, cb), (node, cb) => { let cid = new CID(node.multihash) diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index 55c406d86c..965c636f45 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -147,7 +147,8 @@ exports.add = { is: 1, then: Joi.boolean().valid(false).required(), otherwise: Joi.boolean().valid(false) - }) + }), + 'only-hash': Joi.boolean() }) // TODO: Necessary until validate "recursive", "stream-channels" etc. .options({ allowUnknown: true }) @@ -205,7 +206,8 @@ exports.add = { const options = { 'cid-version': request.query['cid-version'], 'raw-leaves': request.query['raw-leaves'], - progress: request.query['progress'] ? progressHandler : null + progress: request.query.progress ? progressHandler : null, + onlyHash: Boolean(request.query['only-hash']) } const aborter = abortable() diff --git a/test/cli/config.js b/test/cli/config.js index 76d3c77f3d..2150c39e2c 100644 --- a/test/cli/config.js +++ b/test/cli/config.js @@ -68,7 +68,8 @@ describe('config', () => runOnAndOff((thing) => { }) it('call config with no arguments', () => { - return ipfs.fail('config') + return ipfs('config') + .then(out => expect(out).to.include('bin.js config [value]')) }) }) diff --git a/test/cli/files.js b/test/cli/files.js index b2d317abd3..2e9fec39b5 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -2,6 +2,7 @@ 'use strict' const fs = require('fs') +const os = require('os') const expect = require('chai').expect const path = require('path') const compareDir = require('dir-compare').compareSync @@ -270,6 +271,35 @@ describe('files', () => runOnAndOff((thing) => { }) }) + it('add --only-hash outputs correct hash', function () { + return ipfs('files add --only-hash src/init-files/init-docs/readme') + .then(out => + expect(out) + .to.eql('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme\n') + ) + }) + + it('add --only-hash does not add a file to the datastore', function () { + this.timeout(30 * 1000) + this.slow(10 * 1000) + const content = String(Math.random() + Date.now()) + const filepath = path.join(os.tmpdir(), `${content}.txt`) + fs.writeFileSync(filepath, content) + + return ipfs(`files add --only-hash ${filepath}`) + .then(out => { + const hash = out.split(' ')[1] + + // 'jsipfs object get ' should timeout with the daemon on + // and should fail fast with the daemon off + return Promise.race([ + ipfs.fail(`object get ${hash}`), + new Promise((resolve, reject) => setTimeout(resolve, 4000)) + ]) + .then(() => fs.unlinkSync(filepath)) + }) + }) + it('cat', function () { this.timeout(30 * 1000) diff --git a/test/http-api/files.js b/test/http-api/files.js new file mode 100644 index 0000000000..4e2f297d70 --- /dev/null +++ b/test/http-api/files.js @@ -0,0 +1,42 @@ +/* eslint-env mocha */ +/* eslint max-nested-callbacks: ["error", 8] */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +chai.use(dirtyChai) + +describe('.files', () => { + let ipfs = null + let ipfsd = null + before(function (done) { + this.timeout(20 * 1000) + df.spawn({ initOptions: { bits: 512 } }, (err, _ipfsd) => { + expect(err).to.not.exist() + ipfsd = _ipfsd + ipfs = ipfsd.api + done() + }) + }) + + after((done) => ipfsd.stop(done)) + + describe('.add', function () { + it('performs a speculative add, --only-hash', () => { + const content = String(Math.random()) + + return ipfs.add(Buffer.from(content), { onlyHash: true }) + .then(files => { + const getAttempt = ipfs.object.get(files[0].hash) + .then(() => { + throw new Error('Should not find an object for content added with --only-hash') + }) + + return Promise.race([ + getAttempt, + new Promise((resolve, reject) => setTimeout(resolve, 4000)) + ]) + }) + }) + }) +}) diff --git a/test/utils/ipfs-exec.js b/test/utils/ipfs-exec.js index c9bcb53dc3..2437f136a9 100644 --- a/test/utils/ipfs-exec.js +++ b/test/utils/ipfs-exec.js @@ -51,15 +51,28 @@ module.exports = (repoPath, opts) => { return res } + /** + * Expect the command passed as @param arguments to fail. + * @return {Promise} Resolves if the command passed as @param arguments fails, + * rejects if it was successful. + */ ipfs.fail = function ipfsFail () { let args = Array.from(arguments) + let caught = false if (args.length === 1) { args = args[0].split(' ') } - return exec(args).catch((err) => { - expect(err).to.exist() - }) + return exec(args) + .catch(err => { + caught = true + expect(err).to.exist() + }) + .then(() => { + if (!caught) { + throw new Error(`jsipfs expected to fail during command: jsipfs ${args.join(' ')}`) + } + }) } return ipfs