From 7544b7bef42dd4e83b0d348d711d9b0156dc9c47 Mon Sep 17 00:00:00 2001 From: David Dias Date: Sat, 2 Sep 2017 23:32:53 +0100 Subject: [PATCH] feat: Add --cid-version option to ipfs files add + decodeURIComponent for file and directory names --- package.json | 12 ++++---- src/cli/commands/files/add.js | 29 +++++++++++++++++- src/core/components/files.js | 22 +++++++++----- src/http-api/resources/files.js | 33 +++++++++++++++++++- src/http-api/routes/files.js | 3 +- test/cli/files.js | 54 +++++++++++++++++++++++++++++++++ test/http-api/index.js | 2 +- 7 files changed, 138 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index 863f9b0c9e..b06989a2c0 100644 --- a/package.json +++ b/package.json @@ -73,8 +73,8 @@ "expose-loader": "^0.7.3", "form-data": "^2.3.1", "gulp": "^3.9.1", - "interface-ipfs-core": "~0.31.13", - "ipfsd-ctl": "~0.21.0", + "interface-ipfs-core": "~0.31.16", + "ipfsd-ctl": "~0.22.0", "left-pad": "^1.1.3", "lodash": "^4.17.4", "mocha": "^3.5.0", @@ -99,7 +99,7 @@ "hapi": "^16.5.2", "hapi-set-header": "^1.0.2", "hoek": "^4.2.0", - "ipfs-api": "^14.1.2", + "ipfs-api": "^14.2.1", "ipfs-bitswap": "~0.16.1", "ipfs-block": "~0.6.0", "ipfs-block-service": "~0.12.0", @@ -107,7 +107,7 @@ "ipfs-repo": "~0.17.0", "ipfs-unixfs": "~0.1.12", "ipfs-unixfs-engine": "~0.22.0", - "ipld-resolver": "~0.13.0", + "ipld-resolver": "~0.13.1", "is-ipfs": "^0.3.0", "is-stream": "^1.1.0", "joi": "^10.6.0", @@ -128,7 +128,7 @@ "mafmt": "^2.1.8", "mkdirp": "~0.5.1", "multiaddr": "^2.3.0", - "multihashes": "~0.4.8", + "multihashes": "~0.4.9", "once": "^1.4.0", "path-exists": "^3.0.0", "peer-book": "~0.5.0", @@ -208,4 +208,4 @@ "tcme ", "ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ " ] -} \ No newline at end of file +} diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 9e7d192106..9435c95fc5 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -112,6 +112,15 @@ module.exports = { 'shard-split-threshold': { type: 'integer', default: 1000 + }, + 'raw-leaves': { + type: 'boolean', + default: undefined, + describe: 'Use raw blocks for leaf nodes. (experimental)' + }, + 'cid-version': { + type: 'integer', + describe: 'Cid version. Non-zero value will change default of \'raw-leaves\' to true. (experimental)' } }, @@ -120,7 +129,25 @@ module.exports = { const index = inPath.lastIndexOf('/') + 1 const options = { strategy: argv.trickle ? 'trickle' : 'balanced', - shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity + shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity, + 'cid-version': argv['cid-version'], + 'raw-leaves': argv['raw-leaves'] + } + + // Temporary restriction on raw-leaves: + // When cid-version=1 then raw-leaves MUST be present and false. + // + // This is because raw-leaves is not yet implemented in js-ipfs, + // and go-ipfs changes the value of raw-leaves to true when + // cid-version > 0 unless explicitly set to false. + // + // This retains feature parity without having to implement raw-leaves. + if (argv['cid-version'] > 0 && argv['raw-leaves'] !== false) { + throw new Error('Implied argument raw-leaves must be passed and set to false when cid-version is > 0') + } + + if (argv['raw-leaves']) { + throw new Error('Not implemented: raw-leaves') } if (argv.enableShardingExperiment && utils.isDaemonOn()) { diff --git a/src/core/components/files.js b/src/core/components/files.js index 3d650e74bc..6a9f5fbced 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -4,7 +4,6 @@ const unixfsEngine = require('ipfs-unixfs-engine') const importer = unixfsEngine.importer const exporter = unixfsEngine.exporter const promisify = require('promisify-es6') -const multihashes = require('multihashes') const pull = require('pull-stream') const sort = require('pull-sort') const pushable = require('pull-pushable') @@ -13,6 +12,7 @@ const toPull = require('stream-to-pull-stream') const waterfall = require('async/waterfall') const isStream = require('is-stream') const Duplex = require('stream').Duplex +const CID = require('cids') module.exports = function files (self) { const createAddPullStream = (options) => { @@ -24,7 +24,7 @@ module.exports = function files (self) { pull.map(normalizeContent), pull.flatten(), importer(self._ipldResolver, opts), - pull.asyncMap(prepareFile.bind(null, self)) + pull.asyncMap(prepareFile.bind(null, self, opts)) ) } @@ -68,7 +68,7 @@ module.exports = function files (self) { pull( pull.values(normalizeContent(data)), importer(self._ipldResolver, options), - pull.asyncMap(prepareFile.bind(null, self)), + pull.asyncMap(prepareFile.bind(null, self, options)), sort((a, b) => { if (a.path < b.path) return 1 if (a.path > b.path) return -1 @@ -114,15 +114,23 @@ module.exports = function files (self) { } } -function prepareFile (self, file, callback) { - const bs58mh = multihashes.toB58String(file.multihash) +function prepareFile (self, opts, file, callback) { + opts = opts || {} waterfall([ (cb) => self.object.get(file.multihash, cb), (node, cb) => { + let cid = new CID(node.multihash) + + if (opts['cid-version'] === 1) { + cid = cid.toV1() + } + + const b58Hash = cid.toBaseEncodedString() + cb(null, { - path: file.path || bs58mh, - hash: bs58mh, + path: file.path || b58Hash, + hash: b58Hash, size: node.size }) } diff --git a/src/http-api/resources/files.js b/src/http-api/resources/files.js index 68ecc797bd..35c53a32e6 100644 --- a/src/http-api/resources/files.js +++ b/src/http-api/resources/files.js @@ -11,6 +11,7 @@ const toPull = require('stream-to-pull-stream') const pushable = require('pull-pushable') const EOL = require('os').EOL const toStream = require('pull-stream-to-stream') +const Joi = require('joi') exports = module.exports @@ -140,6 +141,28 @@ exports.get = { } exports.add = { + validate: { + query: Joi.object() + .keys({ + 'cid-version': Joi.number().integer().min(0).max(1), + // Temporary restriction on raw-leaves: + // When cid-version=1 then raw-leaves MUST be present and false. + // + // This is because raw-leaves is not yet implemented in js-ipfs, + // and go-ipfs changes the value of raw-leaves to true when + // cid-version > 0 unless explicitly set to false. + // + // This retains feature parity without having to implement raw-leaves. + 'raw-leaves': Joi.any().when('cid-version', { + is: 1, + then: Joi.boolean().valid(false).required(), + otherwise: Joi.boolean().valid(false) + }) + }) + // TODO: Necessary until validate "recursive", "stream-channels" etc. + .options({ allowUnknown: true }) + }, + handler: (request, reply) => { if (!request.payload) { return reply({ @@ -156,6 +179,7 @@ exports.add = { const fileAdder = pushable() parser.on('file', (fileName, fileStream) => { + fileName = decodeURIComponent(fileName) const filePair = { path: fileName, content: toPull(fileStream) @@ -165,6 +189,8 @@ exports.add = { }) parser.on('directory', (directory) => { + directory = decodeURIComponent(directory) + fileAdder.push({ path: directory, content: '' @@ -181,9 +207,14 @@ exports.add = { fileAdder.end() }) + const options = { + 'cid-version': request.query['cid-version'], + 'raw-leaves': request.query['raw-leaves'] + } + pull( fileAdder, - ipfs.files.createAddPullStream(), + ipfs.files.createAddPullStream(options), pull.map((file) => { return { Name: file.path ? file.path : file.hash, diff --git a/src/http-api/routes/files.js b/src/http-api/routes/files.js index da57b3f2f1..e7ce4f456d 100644 --- a/src/http-api/routes/files.js +++ b/src/http-api/routes/files.js @@ -38,7 +38,8 @@ module.exports = (server) => { parse: false, output: 'stream' }, - handler: resources.files.add.handler + handler: resources.files.add.handler, + validate: resources.files.add.validate } }) } diff --git a/test/cli/files.js b/test/cli/files.js index 7dc01599c6..4d5ca1a0a0 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -136,6 +136,60 @@ describe('files', () => runOnAndOff((thing) => { }) }) + it('add with cid-version=0', () => { + return ipfs('add src/init-files/init-docs/readme --cid-version=0').then((out) => { + expect(out) + .to.eql('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme\n') + }) + }) + + // Temporarily expect to fail as raw-leaves not yet implemented. + // + // When cid-version=1 then raw-leaves MUST be present and false. + // + // This is because raw-leaves is not yet implemented in js-ipfs, + // and go-ipfs changes the value of raw-leaves to true when + // cid-version > 0 unless explicitly set to false. + // + // This retains feature parity without having to implement raw-leaves. + it('add with cid-version=1', () => { + return new Promise((resolve, reject) => { + ipfs('add src/init-files/init-docs/readme --cid-version=1') + .then(() => reject(new Error('Raw leaves not expected to be implemented'))) + .catch((err) => { + expect(err).to.exist() + resolve() + }) + }) + }) + + it('add with cid-version=1 and raw-leaves=false', () => { + return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=false').then((out) => { + expect(out) + .to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n') + }) + }) + + // Temporarily expect to fail as raw-leaves not yet implemented + // + // When cid-version=1 then raw-leaves MUST be present and false. + // + // This is because raw-leaves is not yet implemented in js-ipfs, + // and go-ipfs changes the value of raw-leaves to true when + // cid-version > 0 unless explicitly set to false. + // + // This retains feature parity without having to implement raw-leaves. + it('add with cid-version=1 and raw-leaves=true', () => { + return new Promise((resolve, reject) => { + ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=true') + .then(() => reject(new Error('Raw leaves not expected to be implemented'))) + .catch((err) => { + expect(err).to.exist() + resolve() + }) + }) + }) + it('cat', () => { return ipfs('files cat QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') .then((out) => { diff --git a/test/http-api/index.js b/test/http-api/index.js index ad647e92df..52b6c0616e 100644 --- a/test/http-api/index.js +++ b/test/http-api/index.js @@ -41,7 +41,7 @@ describe('HTTP API', () => { .forEach((file) => require('./spec/' + file)(http)) }) - describe('## interface tests', () => { + describe.only('## interface tests', () => { fs.readdirSync(path.join(__dirname, '/interface')) .forEach((file) => require('./interface/' + file)) })