From 6b24463431497bd13b579a730ad7063345729ad9 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 23 Jul 2020 10:01:55 +0100 Subject: [PATCH] fix: send blobs when running ipfs-http-client in the browser (#3184) To support streaming of native types with no buffering, normalise add input to blobs and upload using native FormData when the http client is run in the browser. That is, if the user passes a blob to the http client in the browser leave it alone as enumerating blob contents cause the file data to be read. Browser FormData objects do not allow you to specify headers for each multipart part which means we can't pass UnixFS metadata via the headers so we turn the metadata into a querystring and append it to the field name for each multipart part as a workaround. Fixes #3138 BREAKING CHANGES: - Removes the `mode`, `mtime` and `mtime-nsec` headers from multipart requests - Passes `mode`, `mtime` and `mtime-nsec` as querystring parameters appended to the field name of multipart requests --- docs/core-api/FILES.md | 24 +- packages/ipfs-core-utils/package.json | 10 +- .../src/files/normalise-input.js | 298 ------------------ .../files/normalise-input/index.browser.js | 18 ++ .../src/files/normalise-input/index.js | 18 ++ .../normalise-content.browser.js | 65 ++++ .../normalise-input/normalise-content.js | 66 ++++ .../files/normalise-input/normalise-input.js | 100 ++++++ .../src/files/normalise-input/utils.js | 22 ++ .../test/files/normalise-input.spec.js | 89 ++++-- packages/ipfs-http-client/package.json | 5 +- .../src/lib/multipart-request.browser.js | 58 ++++ .../src/lib/multipart-request.js | 20 +- .../src/lib/to-stream.browser.js | 22 -- .../ipfs-http-client/src/lib/to-stream.js | 7 - packages/ipfs-http-client/test/files.spec.js | 38 +++ .../http/utils/multipart-request-parser.js | 30 +- .../ipfs/test/http-api/inject/mfs/write.js | 25 +- 18 files changed, 527 insertions(+), 388 deletions(-) delete mode 100644 packages/ipfs-core-utils/src/files/normalise-input.js create mode 100644 packages/ipfs-core-utils/src/files/normalise-input/index.browser.js create mode 100644 packages/ipfs-core-utils/src/files/normalise-input/index.js create mode 100644 packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js create mode 100644 packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js create mode 100644 packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js create mode 100644 packages/ipfs-core-utils/src/files/normalise-input/utils.js create mode 100644 packages/ipfs-http-client/src/lib/multipart-request.browser.js delete mode 100644 packages/ipfs-http-client/src/lib/to-stream.browser.js delete mode 100644 packages/ipfs-http-client/src/lib/to-stream.js create mode 100644 packages/ipfs-http-client/test/files.spec.js diff --git a/docs/core-api/FILES.md b/docs/core-api/FILES.md index efcc5ed047..6a4bb0a40a 100644 --- a/docs/core-api/FILES.md +++ b/docs/core-api/FILES.md @@ -7,6 +7,9 @@ _Explore the Mutable File System through interactive coding challenges in our [P - [The Regular API](#the-regular-api) - [`ipfs.add(data, [options])`](#ipfsadddata-options) - [Parameters](#parameters) + - [FileStream](#filestream) + - [FileObject](#fileobject) + - [FileContent](#filecontent) - [Options](#options) - [Returns](#returns) - [`ipfs.addAll(source, [options])`](#ipfsaddallsource-options) @@ -108,12 +111,19 @@ The regular, top-level API for add, cat, get and ls Files on IPFS `data` may be: -* `Blob` -* `String` -* `Uint8Array` +* `FileContent` (see below for definition) * `FileObject` (see below for definition) -* `Iterable` -* `AsyncIterable` +* `FileStream` (see below for definition) + +##### FileStream + +`FileStream` is a stream of `FileContent` or `FileObject` entries of the type: + +```js +Iterable | AsyncIterable | ReadableStream +``` + +##### FileObject `FileObject` is a plain JS object of the following form: @@ -136,10 +146,12 @@ If no `content` is passed, then the item is treated as an empty directory. One of `path` or `content` _must_ be passed. +##### FileContent + `FileContent` is one of the following types: ```js -Uint8Array | Blob | String | Iterable | AsyncIterable +Uint8Array | Blob | String | Iterable | AsyncIterable | ReadableStream ``` `UnixTime` is one of the following types: diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 3c791413bd..b08de20343 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -28,17 +28,21 @@ }, "license": "MIT", "dependencies": { + "blob-to-it": "0.0.1", + "browser-readablestream-to-it": "0.0.1", "buffer": "^5.6.0", "cids": "^0.8.3", "err-code": "^2.0.0", - "ipfs-utils": "^2.2.2" + "ipfs-utils": "^2.2.2", + "it-all": "^1.0.1", + "it-map": "^1.0.0", + "it-peekable": "0.0.1" }, "devDependencies": { "aegir": "^23.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "delay": "^4.3.0", - "dirty-chai": "^2.0.1", - "it-all": "^1.0.1" + "dirty-chai": "^2.0.1" } } diff --git a/packages/ipfs-core-utils/src/files/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input.js deleted file mode 100644 index c3f73168c5..0000000000 --- a/packages/ipfs-core-utils/src/files/normalise-input.js +++ /dev/null @@ -1,298 +0,0 @@ -'use strict' - -const errCode = require('err-code') -const { Buffer } = require('buffer') -const globalThis = require('ipfs-utils/src/globalthis') - -/* - * Transform one of: - * - * ``` - * Bytes (Buffer|ArrayBuffer|TypedArray) [single file] - * Bloby (Blob|File) [single file] - * String [single file] - * { path, content: Bytes } [single file] - * { path, content: Bloby } [single file] - * { path, content: String } [single file] - * { path, content: Iterable } [single file] - * { path, content: Iterable } [single file] - * { path, content: AsyncIterable } [single file] - * Iterable [single file] - * Iterable [single file] - * Iterable [multiple files] - * Iterable [multiple files] - * Iterable<{ path, content: Bytes }> [multiple files] - * Iterable<{ path, content: Bloby }> [multiple files] - * Iterable<{ path, content: String }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: AsyncIterable }> [multiple files] - * AsyncIterable [single file] - * AsyncIterable [multiple files] - * AsyncIterable [multiple files] - * AsyncIterable<{ path, content: Bytes }> [multiple files] - * AsyncIterable<{ path, content: Bloby }> [multiple files] - * AsyncIterable<{ path, content: String }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] - * ``` - * Into: - * - * ``` - * AsyncIterable<{ path, content: AsyncIterable }> - * ``` - * - * @param input Object - * @return AsyncInterable<{ path, content: AsyncIterable }> - */ -module.exports = function normaliseInput (input) { - // must give us something - if (input === null || input === undefined) { - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') - } - - // String - if (typeof input === 'string' || input instanceof String) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - // Buffer|ArrayBuffer|TypedArray - // Blob|File - if (isBytes(input) || isBloby(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - // Iterable - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - // Iterable - if (Number.isInteger(first.value) || isBytes(first.value)) { - yield toFileObject((function * () { - yield first.value - yield * iterator - })()) - return - } - - // Iterable - // Iterable - // Iterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // window.ReadableStream - if (typeof input.getReader === 'function') { - return (async function * () { - for await (const obj of browserStreamToIt(input)) { - yield toFileObject(obj) - } - })() - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - const iterator = input[Symbol.asyncIterator]() - const first = await iterator.next() - if (first.done) return iterator - - // AsyncIterable - if (isBytes(first.value)) { - yield toFileObject((async function * () { // eslint-disable-line require-await - yield first.value - yield * iterator - })()) - return - } - - // AsyncIterable - // AsyncIterable - // AsyncIterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for await (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // { path, content: ? } - // Note: Detected _after_ AsyncIterable because Node.js streams have a - // `path` property that passes this check. - if (isFileObject(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') -} - -function toFileObject (input) { - const obj = { - path: input.path || '', - mode: input.mode, - mtime: input.mtime - } - - if (input.content) { - obj.content = toAsyncIterable(input.content) - } else if (!input.path) { // Not already a file object with path or content prop - obj.content = toAsyncIterable(input) - } - - return obj -} - -function toAsyncIterable (input) { - // Bytes | String - if (isBytes(input) || typeof input === 'string') { - return (async function * () { // eslint-disable-line require-await - yield toBuffer(input) - })() - } - - // Bloby - if (isBloby(input)) { - return blobToAsyncGenerator(input) - } - - // Browser stream - if (typeof input.getReader === 'function') { - return browserStreamToIt(input) - } - - // Iterator - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - if (Number.isInteger(first.value)) { - yield toBuffer(Array.from((function * () { - yield first.value - yield * iterator - })())) - return - } - - // Iterable - if (isBytes(first.value)) { - yield toBuffer(first.value) - for (const chunk of iterator) { - yield toBuffer(chunk) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - for await (const chunk of input) { - yield toBuffer(chunk) - } - })() - } - - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') -} - -function toBuffer (chunk) { - return isBytes(chunk) ? chunk : Buffer.from(chunk) -} - -function isBytes (obj) { - return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer -} - -function isBloby (obj) { - return typeof globalThis.Blob !== 'undefined' && obj instanceof globalThis.Blob -} - -// An object with a path or content property -function isFileObject (obj) { - return typeof obj === 'object' && (obj.path || obj.content) -} - -function blobToAsyncGenerator (blob) { - if (typeof blob.stream === 'function') { - // firefox < 69 does not support blob.stream() - return browserStreamToIt(blob.stream()) - } - - return readBlob(blob) -} - -async function * browserStreamToIt (stream) { - const reader = stream.getReader() - - while (true) { - const result = await reader.read() - - if (result.done) { - return - } - - yield result.value - } -} - -async function * readBlob (blob, options) { - options = options || {} - - const reader = new globalThis.FileReader() - const chunkSize = options.chunkSize || 1024 * 1024 - let offset = options.offset || 0 - - const getNextChunk = () => new Promise((resolve, reject) => { - reader.onloadend = e => { - const data = e.target.result - resolve(data.byteLength === 0 ? null : data) - } - reader.onerror = reject - - const end = offset + chunkSize - const slice = blob.slice(offset, end) - reader.readAsArrayBuffer(slice) - offset = end - }) - - while (true) { - const data = await getNextChunk() - - if (data == null) { - return - } - - yield Buffer.from(data) - } -} diff --git a/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js new file mode 100644 index 0000000000..c96260b528 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js @@ -0,0 +1,18 @@ +'use strict' + +const normaliseContent = require('./normalise-content.browser') +const normaliseInput = require('./normalise-input') + +/* + * Transforms any of the `ipfs.add` input types into + * + * ``` + * AsyncIterable<{ path, mode, mtime, content: Blob }> + * ``` + * + * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options + * + * @param input Object + * @return AsyncInterable<{ path, mode, mtime, content: Blob }> + */ +module.exports = (input) => normaliseInput(input, normaliseContent) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/index.js b/packages/ipfs-core-utils/src/files/normalise-input/index.js new file mode 100644 index 0000000000..00dd946bfa --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/index.js @@ -0,0 +1,18 @@ +'use strict' + +const normaliseContent = require('./normalise-content') +const normaliseInput = require('./normalise-input') + +/* + * Transforms any of the `ipfs.add` input types into + * + * ``` + * AsyncIterable<{ path, mode, mtime, content: AsyncIterable }> + * ``` + * + * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options + * + * @param input Object + * @return AsyncInterable<{ path, mode, mtime, content: AsyncIterable }> + */ +module.exports = (input) => normaliseInput(input, normaliseContent) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js new file mode 100644 index 0000000000..fe55cde87c --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js @@ -0,0 +1,65 @@ +'use strict' + +const errCode = require('err-code') +const { Blob } = require('ipfs-utils/src/globalthis') +const itPeekable = require('it-peekable') +const browserStreamToIt = require('browser-readablestream-to-it') + +const { + isBytes, + isBlob +} = require('./utils') + +async function toBlob (input) { + // Bytes | String + if (isBytes(input) || typeof input === 'string' || input instanceof String) { + return new Blob([input]) + } + + // Blob | File + if (isBlob(input)) { + return input + } + + // Browser stream + if (typeof input.getReader === 'function') { + input = browserStreamToIt(input) + } + + // (Async)Iterator + if (input[Symbol.iterator] || input[Symbol.asyncIterator]) { + const peekable = itPeekable(input) + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + return itToBlob(peekable) + } + + peekable.push(value) + + // (Async)Iterable + if (Number.isInteger(value)) { + return itToBlob(peekable) + } + + // (Async)Iterable + if (isBytes(value) || typeof value === 'string' || value instanceof String) { + return itToBlob(peekable) + } + } + + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') +} + +async function itToBlob (stream) { + const parts = [] + + for await (const chunk of stream) { + parts.push(chunk) + } + + return new Blob(parts) +} + +module.exports = toBlob diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js new file mode 100644 index 0000000000..c55f938e0f --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js @@ -0,0 +1,66 @@ +'use strict' + +const errCode = require('err-code') +const { Buffer } = require('buffer') +const browserStreamToIt = require('browser-readablestream-to-it') +const blobToIt = require('blob-to-it') +const itPeekable = require('it-peekable') +const all = require('it-all') +const map = require('it-map') +const { + isBytes, + isBlob +} = require('./utils') + +async function * toAsyncIterable (input) { + // Bytes | String + if (isBytes(input) || typeof input === 'string' || input instanceof String) { + yield toBuffer(input) + return + } + + // Blob + if (isBlob(input)) { + yield * blobToIt(input) + return + } + + // Browser stream + if (typeof input.getReader === 'function') { + input = browserStreamToIt(input) + } + + // (Async)Iterator + if (input[Symbol.iterator] || input[Symbol.asyncIterator]) { + const peekable = itPeekable(input) + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + yield * peekable + return + } + + peekable.push(value) + + // (Async)Iterable + if (Number.isInteger(value)) { + yield toBuffer(await all(peekable)) + return + } + + // (Async)Iterable + if (isBytes(value) || typeof value === 'string' || value instanceof String) { + yield * map(peekable, chunk => toBuffer(chunk)) + return + } + } + + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') +} + +function toBuffer (chunk) { + return isBytes(chunk) ? chunk : Buffer.from(chunk) +} + +module.exports = toAsyncIterable diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js new file mode 100644 index 0000000000..90bb00015f --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js @@ -0,0 +1,100 @@ +'use strict' + +const errCode = require('err-code') +const browserStreamToIt = require('browser-readablestream-to-it') +const itPeekable = require('it-peekable') +const map = require('it-map') +const { + isBytes, + isBlob, + isFileObject +} = require('./utils') + +module.exports = async function * normaliseInput (input, normaliseContent) { + // must give us something + if (input === null || input === undefined) { + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') + } + + // String + if (typeof input === 'string' || input instanceof String) { + yield toFileObject(input, normaliseContent) + return + } + + // Buffer|ArrayBuffer|TypedArray + // Blob|File + if (isBytes(input) || isBlob(input)) { + yield toFileObject(input, normaliseContent) + return + } + + // Browser ReadableStream + if (typeof input.getReader === 'function') { + input = browserStreamToIt(input) + } + + // Iterable + if (input[Symbol.iterator] || input[Symbol.asyncIterator]) { + const peekable = itPeekable(input) + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + yield * peekable + return + } + + peekable.push(value) + + // (Async)Iterable + // (Async)Iterable + if (Number.isInteger(value) || isBytes(value)) { + yield toFileObject(peekable, normaliseContent) + return + } + + // (Async)Iterable + // (Async)Iterable + // (Async)Iterable<{ path, content }> + if (isFileObject(value) || isBlob(value) || typeof value === 'string' || value instanceof String) { + yield * map(peekable, (value) => toFileObject(value, normaliseContent)) + return + } + + // (Async)Iterable<(Async)Iterable> + // (Async)Iterable> + // ReadableStream<(Async)Iterable> + // ReadableStream> + if (value[Symbol.iterator] || value[Symbol.asyncIterator] || typeof value.getReader === 'function') { + yield * map(peekable, (value) => toFileObject(value, normaliseContent)) + return + } + } + + // { path, content: ? } + // Note: Detected _after_ (Async)Iterable because Node.js streams have a + // `path` property that passes this check. + if (isFileObject(input)) { + yield toFileObject(input, normaliseContent) + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') +} + +async function toFileObject (input, normaliseContent) { + const obj = { + path: input.path || '', + mode: input.mode, + mtime: input.mtime + } + + if (input.content) { + obj.content = await normaliseContent(input.content) + } else if (!input.path) { // Not already a file object with path or content prop + obj.content = await normaliseContent(input) + } + + return obj +} diff --git a/packages/ipfs-core-utils/src/files/normalise-input/utils.js b/packages/ipfs-core-utils/src/files/normalise-input/utils.js new file mode 100644 index 0000000000..c09f0c8241 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/utils.js @@ -0,0 +1,22 @@ +'use strict' + +const { Blob } = require('ipfs-utils/src/globalthis') + +function isBytes (obj) { + return ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer +} + +function isBlob (obj) { + return typeof Blob !== 'undefined' && obj instanceof Blob +} + +// An object with a path or content property +function isFileObject (obj) { + return typeof obj === 'object' && (obj.path || obj.content) +} + +module.exports = { + isBytes, + isBlob, + isFileObject +} diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index e8491bd768..7e206c2bd3 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -2,37 +2,44 @@ /* eslint-env mocha */ const { expect } = require('../utils/chai') -const normalise = require('../../src/files/normalise-input') -const { supportsFileReader } = require('ipfs-utils/src/supports') +const blobToIt = require('blob-to-it') const { Buffer } = require('buffer') const all = require('it-all') -const globalThis = require('ipfs-utils/src/globalthis') +const { Blob, ReadableStream } = require('ipfs-utils/src/globalthis') +const { isBrowser, isWebWorker } = require('ipfs-utils/src/env') + +let normalise = require('../../src/files/normalise-input') + +if (isBrowser || isWebWorker) { + normalise = require('../../src/files/normalise-input/index.browser') +} const STRING = () => 'hello world' +const NEWSTRING = () => new String('hello world') // eslint-disable-line no-new-wrappers const BUFFER = () => Buffer.from(STRING()) const ARRAY = () => Array.from(BUFFER()) const TYPEDARRAY = () => Uint8Array.from(ARRAY()) let BLOB -let WINDOW_READABLE_STREAM -if (supportsFileReader) { - BLOB = () => new globalThis.Blob([ +if (Blob) { + BLOB = () => new Blob([ STRING() ]) - - WINDOW_READABLE_STREAM = () => new globalThis.ReadableStream({ - start (controller) { - controller.enqueue(BUFFER()) - controller.close() - } - }) } async function verifyNormalisation (input) { expect(input.length).to.equal(1) - expect(input[0].content[Symbol.asyncIterator] || input[0].content[Symbol.iterator]).to.be.ok('Content should have been an iterable or an async iterable') - expect(await all(input[0].content)).to.deep.equal([BUFFER()]) expect(input[0].path).to.equal('') + + let content = input[0].content + + if (isBrowser || isWebWorker) { + expect(content).to.be.an.instanceOf(Blob) + content = blobToIt(input[0].content) + } + + expect(content[Symbol.asyncIterator] || content[Symbol.iterator]).to.be.ok('Content should have been an iterable or an async iterable') + await expect(all(content)).to.eventually.deep.equal([BUFFER()]) } async function testContent (input) { @@ -51,6 +58,15 @@ function asyncIterableOf (thing) { }()) } +function browserReadableStreamOf (thing) { + return new ReadableStream({ + start (controller) { + controller.enqueue(thing) + controller.close() + } + }) +} + describe('normalise-input', function () { function testInputType (content, name, isBytes) { it(name, async function () { @@ -58,6 +74,12 @@ describe('normalise-input', function () { }) if (isBytes) { + if (ReadableStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + it(`Iterable<${name}>`, async function () { await testContent(iterableOf(content())) }) @@ -72,6 +94,12 @@ describe('normalise-input', function () { }) if (isBytes) { + if (ReadableStream) { + it(`{ path: '', content: ReadableStream<${name}> }`, async function () { + await testContent({ path: '', content: browserReadableStreamOf(content()) }) + }) + } + it(`{ path: '', content: Iterable<${name}> }`, async function () { await testContent({ path: '', content: iterableOf(content()) }) }) @@ -81,6 +109,12 @@ describe('normalise-input', function () { }) } + if (ReadableStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + it(`Iterable<{ path: '', content: ${name} }`, async function () { await testContent(iterableOf({ path: '', content: content() })) }) @@ -90,6 +124,12 @@ describe('normalise-input', function () { }) if (isBytes) { + if (ReadableStream) { + it(`Iterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: browserReadableStreamOf(content()) })) + }) + } + it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { await testContent(iterableOf({ path: '', content: iterableOf(content()) })) }) @@ -98,6 +138,12 @@ describe('normalise-input', function () { await testContent(iterableOf({ path: '', content: asyncIterableOf(content()) })) }) + if (ReadableStream) { + it(`AsyncIterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: browserReadableStreamOf(content()) })) + }) + } + it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () { await testContent(asyncIterableOf({ path: '', content: iterableOf(content()) })) }) @@ -109,7 +155,8 @@ describe('normalise-input', function () { } describe('String', () => { - testInputType(STRING, 'String', false) + testInputType(STRING, 'String', true) + testInputType(NEWSTRING, 'new String()', true) }) describe('Buffer', () => { @@ -117,21 +164,13 @@ describe('normalise-input', function () { }) describe('Blob', () => { - if (!supportsFileReader) { + if (!Blob) { return } testInputType(BLOB, 'Blob', false) }) - describe('window.ReadableStream', () => { - if (!supportsFileReader) { - return - } - - testInputType(WINDOW_READABLE_STREAM, 'window.ReadableStream', false) - }) - describe('Iterable', () => { testInputType(ARRAY, 'Iterable', false) }) diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 6b768851bc..eb6d4ed75b 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -15,9 +15,10 @@ ], "main": "src/index.js", "browser": { - "./src/lib/to-stream.js": "./src/lib/to-stream.browser.js", + "./src/lib/multipart-request.js": "./src/lib/multipart-request.browser.js", "ipfs-utils/src/files/glob-source": false, - "go-ipfs": false + "go-ipfs": false, + "ipfs-core-utils/src/files/normalise-input": "ipfs-core-utils/src/files/normalise-input/index.browser.js" }, "repository": { "type": "git", diff --git a/packages/ipfs-http-client/src/lib/multipart-request.browser.js b/packages/ipfs-http-client/src/lib/multipart-request.browser.js new file mode 100644 index 0000000000..550c85ba30 --- /dev/null +++ b/packages/ipfs-http-client/src/lib/multipart-request.browser.js @@ -0,0 +1,58 @@ +'use strict' + +const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') +const modeToString = require('./mode-to-string') +const mtimeToObject = require('./mtime-to-object') +const { File, FormData } = require('ipfs-utils/src/globalthis') + +async function multipartRequest (source = '', abortController, headers = {}) { + const formData = new FormData() + let index = 0 + + for await (const { content, path, mode, mtime } of normaliseInput(source)) { + let fileSuffix = '' + const type = content ? 'file' : 'dir' + + if (index > 0) { + fileSuffix = `-${index}` + } + + let fieldName = type + fileSuffix + const qs = [] + + if (mode !== null && mode !== undefined) { + qs.push(`mode=${modeToString(mode)}`) + } + + if (mtime != null) { + const { + secs, nsecs + } = mtimeToObject(mtime) + + qs.push(`mtime=${secs}`) + + if (nsecs != null) { + qs.push(`mtime-nsecs=${nsecs}`) + } + } + + if (qs.length) { + fieldName = `${fieldName}?${qs.join('&')}` + } + + if (content) { + formData.set(fieldName, content, encodeURIComponent(path)) + } else { + formData.set(fieldName, new File([''], encodeURIComponent(path), { type: 'application/x-directory' })) + } + + index++ + } + + return { + headers, + body: formData + } +} + +module.exports = multipartRequest diff --git a/packages/ipfs-http-client/src/lib/multipart-request.js b/packages/ipfs-http-client/src/lib/multipart-request.js index eee4e26b1b..ba712e02ef 100644 --- a/packages/ipfs-http-client/src/lib/multipart-request.js +++ b/packages/ipfs-http-client/src/lib/multipart-request.js @@ -1,11 +1,11 @@ 'use strict' const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') -const toStream = require('./to-stream') const { nanoid } = require('nanoid') const modeToString = require('../lib/mode-to-string') const mtimeToObject = require('../lib/mtime-to-object') const merge = require('merge-options').bind({ ignoreUndefined: true }) +const toStream = require('it-to-stream') async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { async function * streamFiles (source) { @@ -22,12 +22,11 @@ async function multipartRequest (source = '', abortController, headers = {}, bou fileSuffix = `-${index}` } - yield `--${boundary}\r\n` - yield `Content-Disposition: form-data; name="${type}${fileSuffix}"; filename="${encodeURIComponent(path)}"\r\n` - yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` + let fieldName = type + fileSuffix + const qs = [] if (mode !== null && mode !== undefined) { - yield `mode: ${modeToString(mode)}\r\n` + qs.push(`mode=${modeToString(mode)}`) } if (mtime != null) { @@ -35,13 +34,20 @@ async function multipartRequest (source = '', abortController, headers = {}, bou secs, nsecs } = mtimeToObject(mtime) - yield `mtime: ${secs}\r\n` + qs.push(`mtime=${secs}`) if (nsecs != null) { - yield `mtime-nsecs: ${nsecs}\r\n` + qs.push(`mtime-nsecs=${nsecs}`) } } + if (qs.length) { + fieldName = `${fieldName}?${qs.join('&')}` + } + + yield `--${boundary}\r\n` + yield `Content-Disposition: form-data; name="${fieldName}"; filename="${encodeURIComponent(path)}"\r\n` + yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` yield '\r\n' if (content) { diff --git a/packages/ipfs-http-client/src/lib/to-stream.browser.js b/packages/ipfs-http-client/src/lib/to-stream.browser.js deleted file mode 100644 index 9f5784fedb..0000000000 --- a/packages/ipfs-http-client/src/lib/to-stream.browser.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict' - -// browsers can't stream. When the 'Send ReadableStream in request body' row -// is green here: https://developer.mozilla.org/en-US/docs/Web/API/Request/Request#Browser_compatibility -// we'll be able to wrap the passed iterator in the it-to-browser-readablestream module -// in the meantime we have to convert the whole thing to a BufferSource of some sort -const toBuffer = require('it-to-buffer') -const { Buffer } = require('buffer') - -module.exports = (it) => { - async function * bufferise (source) { - for await (const chunk of source) { - if (Buffer.isBuffer(chunk)) { - yield chunk - } else { - yield Buffer.from(chunk) - } - } - } - - return toBuffer(bufferise(it)) -} diff --git a/packages/ipfs-http-client/src/lib/to-stream.js b/packages/ipfs-http-client/src/lib/to-stream.js deleted file mode 100644 index f0f59ffc50..0000000000 --- a/packages/ipfs-http-client/src/lib/to-stream.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict' - -const toStream = require('it-to-stream') - -module.exports = (it) => { - return toStream.readable(it) -} diff --git a/packages/ipfs-http-client/test/files.spec.js b/packages/ipfs-http-client/test/files.spec.js new file mode 100644 index 0000000000..9ad1be1ff7 --- /dev/null +++ b/packages/ipfs-http-client/test/files.spec.js @@ -0,0 +1,38 @@ +/* eslint-env mocha */ + +'use strict' + +const { Buffer } = require('buffer') +const { expect } = require('interface-ipfs-core/src/utils/mocha') +const f = require('./utils/factory')() + +describe('.add', function () { + this.timeout(20 * 1000) + + let ipfs + + before(async function () { + ipfs = (await f.spawn()).api + }) + + after(() => f.clean()) + + it('should ignore metadata until https://github.com/ipfs/go-ipfs/issues/6920 is implemented', async () => { + const data = Buffer.from('some data') + const result = await ipfs.add(data, { + mode: 0o600, + mtime: { + secs: 1000, + nsecs: 0 + } + }) + + expect(result).to.not.have.property('mode') + expect(result).to.not.have.property('mtime') + expect(result).to.have.property('cid') + + const { cid } = result + expect(cid).to.have.property('codec', 'dag-pb') + expect(cid.toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS') + }) +}) diff --git a/packages/ipfs/src/http/utils/multipart-request-parser.js b/packages/ipfs/src/http/utils/multipart-request-parser.js index 6ee29d52bc..84ea04be69 100644 --- a/packages/ipfs/src/http/utils/multipart-request-parser.js +++ b/packages/ipfs/src/http/utils/multipart-request-parser.js @@ -3,6 +3,7 @@ const Content = require('@hapi/content') const multipart = require('it-multipart') const { Buffer } = require('buffer') +const qs = require('querystring') const multipartFormdataType = 'multipart/form-data' const applicationDirectory = 'application/x-directory' @@ -69,20 +70,6 @@ async function * parseEntry (stream, options) { const entry = {} - if (part.headers.mtime) { - entry.mtime = { - secs: parseInt(part.headers.mtime, 10) - } - - if (part.headers['mtime-nsecs']) { - entry.mtime.nsecs = parseInt(part.headers['mtime-nsecs'], 10) - } - } - - if (part.headers.mode) { - entry.mode = parseInt(part.headers.mode, 8) - } - if (isDirectory(type.mime)) { entry.type = 'directory' } else if (type.mime === applicationSymlink) { @@ -92,6 +79,21 @@ async function * parseEntry (stream, options) { } const disposition = parseDisposition(part.headers['content-disposition']) + const query = qs.parse(disposition.name.split('?').pop()) + + if (query.mode) { + entry.mode = parseInt(query.mode, 8) + } + + if (query.mtime) { + entry.mtime = { + secs: parseInt(query.mtime, 10) + } + + if (query['mtime-nsecs']) { + entry.mtime.nsecs = parseInt(query['mtime-nsecs'], 10) + } + } entry.name = decodeURIComponent(disposition.filename) entry.body = part.body diff --git a/packages/ipfs/test/http-api/inject/mfs/write.js b/packages/ipfs/test/http-api/inject/mfs/write.js index 5a8eca4d38..4d466a5187 100644 --- a/packages/ipfs/test/http-api/inject/mfs/write.js +++ b/packages/ipfs/test/http-api/inject/mfs/write.js @@ -29,11 +29,28 @@ const defaultOptions = { signal: sinon.match.instanceOf(AbortSignal) } -async function send (text, headers = {}) { +async function send (text, options = {}) { + let fieldName = 'file-0' + const query = [] + + if (options.mode) { + query.push(`mode=${options.mode}`) + } + + if (options.mtime) { + query.push(`mtime=${options.mtime}`) + } + + if (options.mtimeNsecs) { + query.push(`mtime-nsecs=${options.mtimeNsecs}`) + } + + if (query.length) { + fieldName = `${fieldName}?${query.join('&')}` + } + const form = new FormData() - form.append('file-0', Buffer.from(text), { - header: headers - }) + form.append(fieldName, Buffer.from(text)) return { headers: form.getHeaders(),