From 8920d8c47451b7ec0688c3285489449de890686f Mon Sep 17 00:00:00 2001 From: jkrone Date: Thu, 22 Feb 2018 13:43:05 -0500 Subject: [PATCH 01/10] Allow only-hash querystring arg in send-files-stream --- src/utils/send-files-stream.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/utils/send-files-stream.js b/src/utils/send-files-stream.js index 997202b4b..b5c749d50 100644 --- a/src/utils/send-files-stream.js +++ b/src/utils/send-files-stream.js @@ -88,12 +88,19 @@ module.exports = (send, path) => { qs['raw-leaves'] = options.rawLeaves } + if ('onlyHash' != null) { + qs['only-hash'] = options.onlyHash + } else if ('only-hash' != null) { + qs['only-hash'] = options['only-hash'] + } + if (options.hash != null) { qs.hash = options.hash } else if (options.hashAlg != null) { qs.hash = options.hashAlg } + const args = { path: path, qs: qs, From da17b1f72a343b50aa4f199e9335f36a6973e714 Mon Sep 17 00:00:00 2001 From: jkrone Date: Thu, 22 Feb 2018 14:19:46 -0500 Subject: [PATCH 02/10] test --only-hash. --- test/files.spec.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/files.spec.js b/test/files.spec.js index cf5eaaff1..7cb889ad5 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -102,6 +102,25 @@ describe('.files (the MFS API part)', function () { }) }) + it('files.add with only-hash=true', function () { + this.slow(4000) + const content = String(Math.random() + Date.now()) + + return ipfs.files.add(Buffer.from(content), { onlyHash: true }) + .then(files => { + expect(files).to.have.length(1) + const findAttempt = ipfs.object.get(files[0].hash) + .then(() => { + throw new Error('Should not find content added with --only-hash') + }) + + return Promise.race([ + findAttempt, + new Promise(res => setTimeout(res, 3000)) + ]) + }) + }) + it('files.add with options', (done) => { ipfs.files.add(testfile, { pin: false }, (err, res) => { expect(err).to.not.exist() From da80a95b285d36ec4e640d75a3f72cda17ac22eb Mon Sep 17 00:00:00 2001 From: jkrone Date: Thu, 22 Feb 2018 16:48:25 -0500 Subject: [PATCH 03/10] lint --- src/utils/send-files-stream.js | 7 +++---- test/files.spec.js | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/utils/send-files-stream.js b/src/utils/send-files-stream.js index b5c749d50..0145ccb09 100644 --- a/src/utils/send-files-stream.js +++ b/src/utils/send-files-stream.js @@ -88,10 +88,10 @@ module.exports = (send, path) => { qs['raw-leaves'] = options.rawLeaves } - if ('onlyHash' != null) { - qs['only-hash'] = options.onlyHash - } else if ('only-hash' != null) { + if (options['only-hash'] != null) { qs['only-hash'] = options['only-hash'] + } else if (options.onlyHash != null) { + qs['only-hash'] = options.onlyHash } if (options.hash != null) { @@ -100,7 +100,6 @@ module.exports = (send, path) => { qs.hash = options.hashAlg } - const args = { path: path, qs: qs, diff --git a/test/files.spec.js b/test/files.spec.js index 7cb889ad5..26a889932 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -103,20 +103,20 @@ describe('.files (the MFS API part)', function () { }) it('files.add with only-hash=true', function () { - this.slow(4000) const content = String(Math.random() + Date.now()) return ipfs.files.add(Buffer.from(content), { onlyHash: true }) .then(files => { expect(files).to.have.length(1) - const findAttempt = ipfs.object.get(files[0].hash) + const getAttempt = ipfs.object.get(files[0].hash) .then(() => { throw new Error('Should not find content added with --only-hash') }) + // 'ipfs.object.get()' should timeout/hang on a non-existant hash return Promise.race([ - findAttempt, - new Promise(res => setTimeout(res, 3000)) + getAttempt, + new Promise((resolve, reject) => setTimeout(resolve, 4000)) ]) }) }) From 1384cee2b6fb9ca5c17e06a4b59ac0a8fdce0287 Mon Sep 17 00:00:00 2001 From: jkrone Date: Fri, 23 Feb 2018 10:40:02 -0500 Subject: [PATCH 04/10] add a little helper --- src/utils/send-files-stream.js | 35 ++++++++++++---------------------- test/files.spec.js | 2 +- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/src/utils/send-files-stream.js b/src/utils/send-files-stream.js index 0145ccb09..839f82bac 100644 --- a/src/utils/send-files-stream.js +++ b/src/utils/send-files-stream.js @@ -76,29 +76,10 @@ module.exports = (send, path) => { const qs = options.qs || {} - if (options['cid-version'] != null) { - qs['cid-version'] = options['cid-version'] - } else if (options.cidVersion != null) { - qs['cid-version'] = options.cidVersion - } - - if (options['raw-leaves'] != null) { - qs['raw-leaves'] = options['raw-leaves'] - } else if (options.rawLeaves != null) { - qs['raw-leaves'] = options.rawLeaves - } - - if (options['only-hash'] != null) { - qs['only-hash'] = options['only-hash'] - } else if (options.onlyHash != null) { - qs['only-hash'] = options.onlyHash - } - - if (options.hash != null) { - qs.hash = options.hash - } else if (options.hashAlg != null) { - qs.hash = options.hashAlg - } + qs['cid-version'] = propOrProp(options, 'cid-version', 'cidVersion') + qs['raw-leaves'] = propOrProp(options, 'raw-leaves', 'rawLeaves') + qs['only-hash'] = propOrProp(options, 'only-hash', 'onlyHash') + qs.hash = propOrProp(options, 'hash', 'hashAlg') const args = { path: path, @@ -153,3 +134,11 @@ module.exports = (send, path) => { return retStream } } + +function propOrProp (source, prop1, prop2) { + if (prop1 in source) { + return source[prop1] + } else if (prop2 in source) { + return source[prop2] + } +} diff --git a/test/files.spec.js b/test/files.spec.js index 26a889932..845aaeac9 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -113,7 +113,7 @@ describe('.files (the MFS API part)', function () { throw new Error('Should not find content added with --only-hash') }) - // 'ipfs.object.get()' should timeout/hang on a non-existant hash + // 'ipfs.object.get()' should timeout because content wasn't actually added return Promise.race([ getAttempt, new Promise((resolve, reject) => setTimeout(resolve, 4000)) From 2e14ab02373095c464b0058a3725d28a86e056f5 Mon Sep 17 00:00:00 2001 From: jkrone Date: Fri, 23 Feb 2018 12:48:36 -0500 Subject: [PATCH 05/10] add --only-hash support to util.addFrom{Fs|URL}. Changed how qs options were passed but tests seem happy with it. --- src/util/fs-add.js | 6 ++-- src/util/url-add.js | 6 ++-- test/files.spec.js | 11 ++----- test/util.spec.js | 61 +++++++++++++++++++++++++----------- test/utils/expect-timeout.js | 14 +++++++++ 5 files changed, 66 insertions(+), 32 deletions(-) create mode 100644 test/utils/expect-timeout.js diff --git a/src/util/fs-add.js b/src/util/fs-add.js index 8a3ea404f..1a06f7ed5 100644 --- a/src/util/fs-add.js +++ b/src/util/fs-add.js @@ -5,8 +5,8 @@ const promisify = require('promisify-es6') const moduleConfig = require('../utils/module-config') const SendOneFile = require('../utils/send-one-file-multiple-results') -module.exports = (arg) => { - const sendOneFile = SendOneFile(moduleConfig(arg), 'add') +module.exports = (send) => { + const sendOneFile = SendOneFile(send, 'add') return promisify((path, opts, callback) => { if (typeof opts === 'function' && @@ -31,6 +31,6 @@ module.exports = (arg) => { return callback(new Error('"path" must be a string')) } - sendOneFile(path, { qs: opts }, callback) + sendOneFile(path, opts, callback) }) } diff --git a/src/util/url-add.js b/src/util/url-add.js index 3caf11cb2..cbfe20f7b 100644 --- a/src/util/url-add.js +++ b/src/util/url-add.js @@ -6,8 +6,8 @@ const request = require('../utils/request') const moduleConfig = require('../utils/module-config') const SendOneFile = require('../utils/send-one-file-multiple-results') -module.exports = (arg) => { - const sendOneFile = SendOneFile(moduleConfig(arg), 'add') +module.exports = (send) => { + const sendOneFile = SendOneFile(send, 'add') return promisify((url, opts, callback) => { if (typeof (opts) === 'function' && @@ -49,7 +49,7 @@ const requestWithRedirect = (url, opts, sendOneFile, callback) => { } requestWithRedirect(redirection, opts, sendOneFile, callback) } else { - sendOneFile(res, { qs: opts }, callback) + sendOneFile(res, opts, callback) } }).end() } diff --git a/test/files.spec.js b/test/files.spec.js index 845aaeac9..66e62b03b 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -13,6 +13,7 @@ const CID = require('cids') const IPFSApi = require('../src') const f = require('./utils/factory') +const expectTimeout = require('./utils/expect-timeout') const testfile = loadFixture('test/fixtures/testfile.txt') @@ -103,21 +104,15 @@ describe('.files (the MFS API part)', function () { }) it('files.add with only-hash=true', function () { + this.slow(10 * 1000) const content = String(Math.random() + Date.now()) return ipfs.files.add(Buffer.from(content), { onlyHash: true }) .then(files => { expect(files).to.have.length(1) - const getAttempt = ipfs.object.get(files[0].hash) - .then(() => { - throw new Error('Should not find content added with --only-hash') - }) // 'ipfs.object.get()' should timeout because content wasn't actually added - return Promise.race([ - getAttempt, - new Promise((resolve, reject) => setTimeout(resolve, 4000)) - ]) + return expectTimeout(ipfs.object.get(files[0].hash), 4000) }) }) diff --git a/test/util.spec.js b/test/util.spec.js index 1573620fd..1ced228ab 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -9,9 +9,11 @@ chai.use(dirtyChai) const isNode = require('detect-node') const path = require('path') const fs = require('fs') +const os = require('os') const IPFSApi = require('../src') const f = require('./utils/factory') +const expectTimeout = require('./utils/expect-timeout') describe('.util', () => { if (!isNode) { return } @@ -92,31 +94,54 @@ describe('.util', () => { done() }) }) + + it('with only-hash=true', function () { + this.slow(10 * 1000) + const content = String(Math.random() + Date.now()) + const filepath = path.join(require('os').tmpdir(), `${content}.txt`) + fs.writeFileSync(filepath, content) + + return ipfs.util.addFromFs(filepath, { 'only-hash': true }) + .then(out => { + fs.unlinkSync(filepath) + return expectTimeout(ipfs.object.get(out[0].hash), 4000) + }) + }) }) - it('.urlAdd http', function (done) { - this.timeout(20 * 1000) + describe('.urlAdd', () => { + it('http', function (done) { + this.timeout(20 * 1000) - ipfs.util.addFromURL('http://example.com/', (err, result) => { - expect(err).to.not.exist() - expect(result.length).to.equal(1) - done() + ipfs.util.addFromURL('http://example.com/', (err, result) => { + expect(err).to.not.exist() + expect(result.length).to.equal(1) + done() + }) }) - }) - it('.urlAdd https', (done) => { - ipfs.util.addFromURL('https://example.com/', (err, result) => { - expect(err).to.not.exist() - expect(result.length).to.equal(1) - done() + it('https', (done) => { + ipfs.util.addFromURL('https://example.com/', (err, result) => { + expect(err).to.not.exist() + expect(result.length).to.equal(1) + done() + }) }) - }) - it('.urlAdd http with redirection', (done) => { - ipfs.util.addFromURL('http://covers.openlibrary.org/book/id/969165.jpg', (err, result) => { - expect(err).to.not.exist() - expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg') - done() + it('http with redirection', (done) => { + ipfs.util.addFromURL('http://covers.openlibrary.org/book/id/969165.jpg', (err, result) => { + expect(err).to.not.exist() + expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg') + done() + }) + }) + + it('with only-hash=true', function () { + this.timeout(10 * 1000) + this.slow(10 * 1000) + + return ipfs.util.addFromURL('http://www.randomtext.me/#/gibberish', { 'only-hash': true }) + .then(out => expectTimeout(ipfs.object.get(out[0].hash), 4000)) }) }) }) diff --git a/test/utils/expect-timeout.js b/test/utils/expect-timeout.js new file mode 100644 index 000000000..ce49e6433 --- /dev/null +++ b/test/utils/expect-timeout.js @@ -0,0 +1,14 @@ +/** + * Expect a Promise to timeout + * @param {Promise} promise promise that you expect to timeout + * @param {Number} ms millis to wait + * @return {Promise} + */ +module.exports = (promise, ms) => { + return Promise.race([ + promise.then((out) => { + throw new Error('Expected Promise to timeout but it was successful.') + }), + new Promise((resolve, reject) => setTimeout(resolve, ms)) + ]) +} From ff1ec92d144b949fe8ff719b2fb1f017bbde85d3 Mon Sep 17 00:00:00 2001 From: jkrone Date: Fri, 23 Feb 2018 12:52:49 -0500 Subject: [PATCH 06/10] remove unused imports. must've missed the lint check. --- src/util/fs-add.js | 1 - src/util/url-add.js | 1 - 2 files changed, 2 deletions(-) diff --git a/src/util/fs-add.js b/src/util/fs-add.js index 1a06f7ed5..58702ed37 100644 --- a/src/util/fs-add.js +++ b/src/util/fs-add.js @@ -2,7 +2,6 @@ const isNode = require('detect-node') const promisify = require('promisify-es6') -const moduleConfig = require('../utils/module-config') const SendOneFile = require('../utils/send-one-file-multiple-results') module.exports = (send) => { diff --git a/src/util/url-add.js b/src/util/url-add.js index cbfe20f7b..768a45b40 100644 --- a/src/util/url-add.js +++ b/src/util/url-add.js @@ -3,7 +3,6 @@ const promisify = require('promisify-es6') const parseUrl = require('url').parse const request = require('../utils/request') -const moduleConfig = require('../utils/module-config') const SendOneFile = require('../utils/send-one-file-multiple-results') module.exports = (send) => { From 4d877307f32a471865c963ea5958b2a3707efd9d Mon Sep 17 00:00:00 2001 From: jkrone Date: Fri, 23 Feb 2018 13:52:44 -0500 Subject: [PATCH 07/10] clean lint --- test/util.spec.js | 2 +- test/utils/expect-timeout.js | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/util.spec.js b/test/util.spec.js index 1ced228ab..08a482338 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -98,7 +98,7 @@ describe('.util', () => { it('with only-hash=true', function () { this.slow(10 * 1000) const content = String(Math.random() + Date.now()) - const filepath = path.join(require('os').tmpdir(), `${content}.txt`) + const filepath = path.join(os.tmpdir(), `${content}.txt`) fs.writeFileSync(filepath, content) return ipfs.util.addFromFs(filepath, { 'only-hash': true }) diff --git a/test/utils/expect-timeout.js b/test/utils/expect-timeout.js index ce49e6433..51c733075 100644 --- a/test/utils/expect-timeout.js +++ b/test/utils/expect-timeout.js @@ -1,6 +1,8 @@ +'use strict' + /** - * Expect a Promise to timeout - * @param {Promise} promise promise that you expect to timeout + * Resolve if @param promise hangs for at least @param ms, throw otherwise + * @param {Promise} promise promise that you expect to hang * @param {Number} ms millis to wait * @return {Promise} */ From f4312ace65c7f9fe923d079f6b7a340a4fa5a99d Mon Sep 17 00:00:00 2001 From: jkrone Date: Mon, 26 Feb 2018 10:13:53 -0500 Subject: [PATCH 08/10] flatten the options passed to send-files-stream. files/* code passes querystring options flatly and util/* code passes it under the 'qs' property, requiring this flattening. files/* should probably send options under 'qs'. --- src/util/fs-add.js | 2 +- src/util/url-add.js | 2 +- src/utils/send-files-stream.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/util/fs-add.js b/src/util/fs-add.js index 58702ed37..219e47b23 100644 --- a/src/util/fs-add.js +++ b/src/util/fs-add.js @@ -30,6 +30,6 @@ module.exports = (send) => { return callback(new Error('"path" must be a string')) } - sendOneFile(path, opts, callback) + sendOneFile(path, { qs: opts }, callback) }) } diff --git a/src/util/url-add.js b/src/util/url-add.js index 768a45b40..0ff51f75d 100644 --- a/src/util/url-add.js +++ b/src/util/url-add.js @@ -48,7 +48,7 @@ const requestWithRedirect = (url, opts, sendOneFile, callback) => { } requestWithRedirect(redirection, opts, sendOneFile, callback) } else { - sendOneFile(res, opts, callback) + sendOneFile(res, { qs: opts }, callback) } }).end() } diff --git a/src/utils/send-files-stream.js b/src/utils/send-files-stream.js index 839f82bac..0dd676f66 100644 --- a/src/utils/send-files-stream.js +++ b/src/utils/send-files-stream.js @@ -32,7 +32,7 @@ module.exports = (send, path) => { let ended = false let writing = false - options = options || {} + options = options ? Object.assign({}, options, options.qs) : {} const multipart = new Multipart() @@ -43,7 +43,7 @@ module.exports = (send, path) => { retStream._write = (file, enc, _next) => { const next = once(_next) try { - const files = prepareFile(file, Object.assign({}, options, options.qs)) + const files = prepareFile(file, options) .map((file) => Object.assign({headers: headers(file)}, file)) writing = true From 2424a5ab2d57499b18d100a0ea49f5d962716cb0 Mon Sep 17 00:00:00 2001 From: jkrone Date: Fri, 2 Mar 2018 11:17:44 -0500 Subject: [PATCH 09/10] send the options for file and pin operations that hit 'send-files-stream' under the 'qs' key. Allows any options passed to ipfs.file|pin.add-ish to be turned into qs params instead of adding all of them individually --- src/files/add-pull-stream.js | 3 ++- src/files/add.js | 2 +- src/files/write.js | 2 +- test/files.spec.js | 36 ++++++++++++++++++++++++++++++++++++ test/util.spec.js | 10 +++++++--- 5 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/files/add-pull-stream.js b/src/files/add-pull-stream.js index daf050de8..cedd659bd 100644 --- a/src/files/add-pull-stream.js +++ b/src/files/add-pull-stream.js @@ -3,4 +3,5 @@ const SendFilesStream = require('../utils/send-files-stream') const toPull = require('stream-to-pull-stream') -module.exports = (send) => (options) => toPull(SendFilesStream(send, 'add')(options)) +module.exports = (send) => (options) => + toPull(SendFilesStream(send, 'add')({ qs: options })) diff --git a/src/files/add.js b/src/files/add.js index f7bb2b690..55f60857f 100644 --- a/src/files/add.js +++ b/src/files/add.js @@ -33,7 +33,7 @@ module.exports = (send) => { const files = [].concat(_files) - const stream = createAddStream(options) + const stream = createAddStream({ qs: options }) const concat = ConcatStream((result) => callback(null, result)) stream.once('error', callback) stream.pipe(concat) diff --git a/src/files/write.js b/src/files/write.js index 5e9efa03b..33f1ec973 100644 --- a/src/files/write.js +++ b/src/files/write.js @@ -31,7 +31,7 @@ module.exports = (send) => { qs: opts } - const stream = sendFilesStream(options) + const stream = sendFilesStream({ qs: options }) const concat = concatStream((result) => callback(null, result)) stream.once('error', callback) stream.pipe(concat) diff --git a/test/files.spec.js b/test/files.spec.js index 66e62b03b..f346b2f5f 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -127,6 +127,42 @@ describe('.files (the MFS API part)', function () { }) }) + it('files.add pins by default', (done) => { + const newContent = Buffer.from(String(Math.random())) + + ipfs.pin.ls((err, pins) => { + expect(err).to.not.exist() + const initialPinCount = pins.length + ipfs.files.add(newContent, (err, res) => { + expect(err).to.not.exist() + + ipfs.pin.ls((err, pins) => { + expect(err).to.not.exist() + expect(pins.length).to.eql(initialPinCount + 1) + done() + }) + }) + }) + }) + + it('files.add with pin=false', (done) => { + const newContent = Buffer.from(String(Math.random())) + + ipfs.pin.ls((err, pins) => { + expect(err).to.not.exist() + const initialPinCount = pins.length + ipfs.files.add(newContent, { pin: false }, (err, res) => { + expect(err).to.not.exist() + + ipfs.pin.ls((err, pins) => { + expect(err).to.not.exist() + expect(pins.length).to.eql(initialPinCount) + done() + }) + }) + }) + }) + HASH_ALGS.forEach((name) => { it(`files.add with hash=${name} and raw-leaves=false`, (done) => { const content = String(Math.random() + Date.now()) diff --git a/test/util.spec.js b/test/util.spec.js index 08a482338..30201ba35 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -101,7 +101,7 @@ describe('.util', () => { const filepath = path.join(os.tmpdir(), `${content}.txt`) fs.writeFileSync(filepath, content) - return ipfs.util.addFromFs(filepath, { 'only-hash': true }) + return ipfs.util.addFromFs(filepath, { onlyHash: true }) .then(out => { fs.unlinkSync(filepath) return expectTimeout(ipfs.object.get(out[0].hash), 4000) @@ -120,7 +120,9 @@ describe('.util', () => { }) }) - it('https', (done) => { + it('https', function (done) { + this.timeout(20 * 1000) + ipfs.util.addFromURL('https://example.com/', (err, result) => { expect(err).to.not.exist() expect(result.length).to.equal(1) @@ -128,7 +130,9 @@ describe('.util', () => { }) }) - it('http with redirection', (done) => { + it('http with redirection', function (done) { + this.timeout(20 * 1000) + ipfs.util.addFromURL('http://covers.openlibrary.org/book/id/969165.jpg', (err, result) => { expect(err).to.not.exist() expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg') From 43f9f38c6bba84b0d6bd1b2deecb6aaaf414dce3 Mon Sep 17 00:00:00 2001 From: jonkrone Date: Fri, 2 Mar 2018 11:26:33 -0500 Subject: [PATCH 10/10] test: git config name change --- test/util.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/util.spec.js b/test/util.spec.js index 30201ba35..a4a7462a4 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -144,7 +144,7 @@ describe('.util', () => { this.timeout(10 * 1000) this.slow(10 * 1000) - return ipfs.util.addFromURL('http://www.randomtext.me/#/gibberish', { 'only-hash': true }) + return ipfs.util.addFromURL('http://www.randomtext.me/#/gibberish', { onlyHash: true }) .then(out => expectTimeout(ipfs.object.get(out[0].hash), 4000)) }) })