From 8ca99b0a1840fa59a1974b176779f58f193f4e79 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 27 Sep 2021 17:12:41 +0100 Subject: [PATCH] fix: do not accept single items for ipfs.add The types allow passing single items to `ipfs.addAll` and multiple items to `ipfs.add`. Instead, only accept single items to `ipfs.add` and streams of item to `ipfs.addAll` and fail with a more helpful error message if you do not do this. --- packages/interface-ipfs-core/src/add-all.js | 20 ++ packages/interface-ipfs-core/src/add.js | 14 + packages/ipfs-cli/src/parser.js | 1 - packages/ipfs-core-utils/package.json | 14 +- .../src/files/normalise-candidate-multiple.js | 114 +++++++++ ...alise.js => normalise-candidate-single.js} | 34 +-- .../src/files/normalise-content.browser.js | 2 +- .../src/files/normalise-content.js | 29 +-- ...js => normalise-input-multiple.browser.js} | 11 +- .../src/files/normalise-input-multiple.js | 21 ++ .../files/normalise-input-single.browser.js | 24 ++ ...ise-input.js => normalise-input-single.js} | 7 +- .../src/multipart-request.browser.js | 5 +- .../ipfs-core-utils/src/multipart-request.js | 3 +- .../src/multipart-request.node.js | 7 +- .../files/normalise-input-multiple.spec.js | 240 ++++++++++++++++++ ...spec.js => normalise-input-single.spec.js} | 4 +- .../ipfs-core/src/components/add-all/index.js | 2 +- packages/ipfs-core/src/components/add.js | 3 +- .../ipfs-grpc-client/src/core-api/add-all.js | 2 +- packages/ipfs-http-client/src/add.js | 3 +- packages/ipfs-http-client/src/block/put.js | 2 +- .../ipfs-http-client/src/config/replace.js | 2 +- packages/ipfs-http-client/src/dag/put.js | 2 +- packages/ipfs-http-client/src/dht/put.js | 2 +- packages/ipfs-http-client/src/files/write.js | 4 +- .../src/object/patch/append-data.js | 2 +- .../src/object/patch/set-data.js | 2 +- .../ipfs-http-client/src/pubsub/publish.js | 2 +- 29 files changed, 496 insertions(+), 82 deletions(-) create mode 100644 packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js rename packages/ipfs-core-utils/src/files/{normalise.js => normalise-candidate-single.js} (70%) rename packages/ipfs-core-utils/src/files/{normalise-input.browser.js => normalise-input-multiple.browser.js} (61%) create mode 100644 packages/ipfs-core-utils/src/files/normalise-input-multiple.js create mode 100644 packages/ipfs-core-utils/src/files/normalise-input-single.browser.js rename packages/ipfs-core-utils/src/files/{normalise-input.js => normalise-input-single.js} (65%) create mode 100644 packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js rename packages/ipfs-core-utils/test/files/{normalise-input.spec.js => normalise-input-single.spec.js} (98%) diff --git a/packages/interface-ipfs-core/src/add-all.js b/packages/interface-ipfs-core/src/add-all.js index 09035d2863..62101f303d 100644 --- a/packages/interface-ipfs-core/src/add-all.js +++ b/packages/interface-ipfs-core/src/add-all.js @@ -301,6 +301,26 @@ export function testAddAll (factory, options) { await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejected() }) + it('should fail when passed single file objects', async () => { + const nonValid = { content: 'hello world' } + + // @ts-expect-error nonValid is non valid + await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejectedWith(/single item passed/) + }) + + it('should fail when passed single strings', async () => { + const nonValid = 'hello world' + + await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejectedWith(/single item passed/) + }) + + it('should fail when passed single buffers', async () => { + const nonValid = uint8ArrayFromString('hello world') + + // @ts-expect-error nonValid is non valid + await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejectedWith(/single item passed/) + }) + it('should wrap content in a directory', async () => { const data = { path: 'testfile.txt', content: fixtures.smallFile.data } diff --git a/packages/interface-ipfs-core/src/add.js b/packages/interface-ipfs-core/src/add.js index aa03c4bb2a..104e85075e 100644 --- a/packages/interface-ipfs-core/src/add.js +++ b/packages/interface-ipfs-core/src/add.js @@ -244,6 +244,20 @@ export function testAdd (factory, options) { await expect(ipfs.add(null)).to.eventually.be.rejected() }) + it('should fail when passed multiple file objects', async () => { + const nonValid = [{ content: 'hello' }, { content: 'world' }] + + // @ts-expect-error nonValid is non valid + await expect(ipfs.add(nonValid)).to.eventually.be.rejectedWith(/multiple items passed/) + }) + + it('should fail when passed multiple strings', async () => { + const nonValid = ['hello', 'world'] + + // @ts-expect-error nonValid is non valid + await expect(ipfs.add(nonValid)).to.eventually.be.rejectedWith(/multiple items passed/) + }) + it('should wrap content in a directory', async () => { const data = { path: 'testfile.txt', content: fixtures.smallFile.data } diff --git a/packages/ipfs-cli/src/parser.js b/packages/ipfs-cli/src/parser.js index 274a2944af..df08a9b604 100644 --- a/packages/ipfs-cli/src/parser.js +++ b/packages/ipfs-cli/src/parser.js @@ -1,4 +1,3 @@ - import yargs from 'yargs' import { ipfsPathHelp, disablePrinting } from './utils.js' import { commandList } from './commands/index.js' diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 62b41e08a5..f979cc6963 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -38,11 +38,17 @@ ".": { "import": "./src/index.js" }, - "./files/normalise-input": { - "import": "./src/files/normalise-input.js" + "./files/normalise-input-single": { + "import": "./src/files/normalise-input-single.js" }, - "./files/normalise-input.browser": { - "import": "./src/files/normalise-input.browser.js" + "./files/normalise-input-single.browser": { + "import": "./src/files/normalise-input-single.browser.js" + }, + "./files/normalise-input-multiple": { + "import": "./src/files/normalise-input-multiple.js" + }, + "./files/normalise-input-multiple.browser": { + "import": "./src/files/normalise-input-multiple.browser.js" }, "./files/normalise-content": { "import": "./src/files/normalise-content.js" diff --git a/packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js b/packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js new file mode 100644 index 0000000000..1e254bfd2a --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js @@ -0,0 +1,114 @@ +import errCode from 'err-code' +import browserStreamToIt from 'browser-readablestream-to-it' +import itPeekable from 'it-peekable' +import map from 'it-map' +import { + isBytes, + isBlob, + isReadableStream, + isFileObject +} from './utils.js' +import { + parseMtime, + parseMode +} from 'ipfs-unixfs' + +/** + * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate + * @typedef {import('ipfs-core-types/src/utils').ToContent} ToContent + * @typedef {import('ipfs-unixfs-importer').ImportCandidate} ImporterImportCandidate + * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream + */ + +/** + * @param {ImportCandidateStream} input + * @param {(content:ToContent) => Promise>} normaliseContent + */ +// eslint-disable-next-line complexity +export async function * normaliseCandidateMultiple (input, normaliseContent) { + // String + // Uint8Array|ArrayBuffer|TypedArray + // Blob|File + if (typeof input === 'string' || input instanceof String || isBytes(input) || isBlob(input)) { + throw errCode(new Error('Unexpected input: single item passed'), 'ERR_UNEXPECTED_INPUT') + } + + // Browser ReadableStream + if (isReadableStream(input)) { + input = browserStreamToIt(input) + } + + // Iterable + if (Symbol.iterator in input || Symbol.asyncIterator in input) { + /** @type {any} */ + // @ts-ignore it's (async)interable + const peekable = itPeekable(input) + + /** @type {any} value **/ + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + yield * [] + return + } + + peekable.push(value) + + // (Async)Iterable + // (Async)Iterable + if (Number.isInteger(value) || isBytes(value)) { + throw errCode(new Error('Unexpected input: single item passed'), 'ERR_UNEXPECTED_INPUT') + } + + // (Async)Iterable + if (value._readableState) { + // @ts-ignore Node fs.ReadStreams have a `.path` property so we need to pass it as the content + yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject({ content: value }, normaliseContent)) + return + } + + // (Async)Iterable<(Async)Iterable> + // (Async)Iterable> + // ReadableStream<(Async)Iterable> + // ReadableStream> + if (isFileObject(value) || value[Symbol.iterator] || value[Symbol.asyncIterator] || isReadableStream(value)) { + yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject(value, normaliseContent)) + return + } + } + + // { path, content: ? } + // Note: Detected _after_ (Async)Iterable because Node.js fs.ReadStreams have a + // `path` property that passes this check. + if (isFileObject(input)) { + throw errCode(new Error('Unexpected input: single item passed'), 'ERR_UNEXPECTED_INPUT') + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') +} + +/** + * @param {ImportCandidate} input + * @param {(content:ToContent) => Promise>} normaliseContent + */ +async function toFileObject (input, normaliseContent) { + // @ts-ignore - Those properties don't exist on most input types + const { path, mode, mtime, content } = input + + /** @type {ImporterImportCandidate} */ + const file = { + path: path || '', + mode: parseMode(mode), + mtime: parseMtime(mtime) + } + + if (content) { + file.content = await normaliseContent(content) + } else if (!path) { // Not already a file object with path or content prop + // @ts-ignore - input still can be different ToContent + file.content = await normaliseContent(input) + } + + return file +} diff --git a/packages/ipfs-core-utils/src/files/normalise.js b/packages/ipfs-core-utils/src/files/normalise-candidate-single.js similarity index 70% rename from packages/ipfs-core-utils/src/files/normalise.js rename to packages/ipfs-core-utils/src/files/normalise-candidate-single.js index 19b888943d..760a1df226 100644 --- a/packages/ipfs-core-utils/src/files/normalise.js +++ b/packages/ipfs-core-utils/src/files/normalise-candidate-single.js @@ -21,11 +21,11 @@ import { */ /** - * @param {ImportCandidate | ImportCandidateStream} input + * @param {ImportCandidate} input * @param {(content:ToContent) => Promise>} normaliseContent */ // eslint-disable-next-line complexity -export async function * normalise (input, normaliseContent) { +export async function * normaliseCandidateSingle (input, normaliseContent) { if (input === null || input === undefined) { throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') } @@ -50,8 +50,7 @@ export async function * normalise (input, normaliseContent) { // Iterable if (Symbol.iterator in input || Symbol.asyncIterator in input) { - /** @type {any} */ - // @ts-ignore it's (async)interable + // @ts-ignore it's (async)iterable const peekable = itPeekable(input) /** @type {any} value **/ @@ -59,7 +58,7 @@ export async function * normalise (input, normaliseContent) { if (done) { // make sure empty iterators result in empty files - yield * [] + yield { content: [] } return } @@ -72,40 +71,25 @@ export async function * normalise (input, normaliseContent) { return } - // fs.ReadStream + // (Async)Iterable if (value._readableState) { - // @ts-ignore Node readable streams have a `.path` property so we need to pass it as the content + // @ts-ignore Node fs.ReadStreams have a `.path` property so we need to pass it as the content yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject({ content: value }, normaliseContent)) return } - // (Async)Iterable - // (Async)Iterable - // (Async)Iterable<{ path, content }> - if (isFileObject(value) || isBlob(value) || typeof value === 'string' || value instanceof String) { - yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject(value, normaliseContent)) - return - } - - // (Async)Iterable<(Async)Iterable> - // (Async)Iterable> - // ReadableStream<(Async)Iterable> - // ReadableStream> - if (value[Symbol.iterator] || value[Symbol.asyncIterator] || isReadableStream(value)) { - yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject(value, normaliseContent)) - return - } + throw errCode(new Error('Unexpected input: multiple items passed'), 'ERR_UNEXPECTED_INPUT') } // { path, content: ? } - // Note: Detected _after_ (Async)Iterable because Node.js streams have a + // Note: Detected _after_ (Async)Iterable because Node.js fs.ReadStreams have a // `path` property that passes this check. if (isFileObject(input)) { yield toFileObject(input, normaliseContent) return } - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + throw errCode(new Error('Unexpected input: cannot convert "' + typeof input + '" into ImportCandidate'), 'ERR_UNEXPECTED_INPUT') } /** diff --git a/packages/ipfs-core-utils/src/files/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-content.browser.js index b208eba8c9..09508f0700 100644 --- a/packages/ipfs-core-utils/src/files/normalise-content.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-content.browser.js @@ -9,7 +9,7 @@ import { } from './utils.js' /** - * @param {import('./normalise').ToContent} input + * @param {import('ipfs-core-types/src/utils').ToContent} input */ export async function normaliseContent (input) { // Bytes diff --git a/packages/ipfs-core-utils/src/files/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-content.js index 4d3e4fa9d6..0cdfa0ab70 100644 --- a/packages/ipfs-core-utils/src/files/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-content.js @@ -12,31 +12,29 @@ import { } from './utils.js' /** - * @param {import('./normalise').ToContent} input + * @template T + * @param {T} thing */ -export async function normaliseContent (input) { - return toAsyncGenerator(input) +async function * toAsyncIterable (thing) { + yield thing } /** - * @param {import('./normalise').ToContent} input + * @param {import('ipfs-core-types/src/utils').ToContent} input */ -async function * toAsyncGenerator (input) { +export async function normaliseContent (input) { // Bytes | String if (isBytes(input)) { - yield toBytes(input) - return + return toAsyncIterable(toBytes(input)) } if (typeof input === 'string' || input instanceof String) { - yield toBytes(input.toString()) - return + return toAsyncIterable(toBytes(input.toString())) } // Blob if (isBlob(input)) { - yield * blobToIt(input) - return + return blobToIt(input) } // Browser stream @@ -54,22 +52,19 @@ async function * toAsyncGenerator (input) { if (done) { // make sure empty iterators result in empty files - yield * [] - return + return toAsyncIterable(new Uint8Array(0)) } peekable.push(value) // (Async)Iterable if (Number.isInteger(value)) { - yield Uint8Array.from((await all(peekable))) - return + return toAsyncIterable(Uint8Array.from(await all(peekable))) } // (Async)Iterable if (isBytes(value) || typeof value === 'string' || value instanceof String) { - yield * map(peekable, toBytes) - return + return map(peekable, toBytes) } } diff --git a/packages/ipfs-core-utils/src/files/normalise-input.browser.js b/packages/ipfs-core-utils/src/files/normalise-input-multiple.browser.js similarity index 61% rename from packages/ipfs-core-utils/src/files/normalise-input.browser.js rename to packages/ipfs-core-utils/src/files/normalise-input-multiple.browser.js index 549d4473e3..a8d4595150 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-input-multiple.browser.js @@ -1,14 +1,13 @@ import { normaliseContent } from './normalise-content.browser.js' -import { normalise } from './normalise.js' +import { normaliseCandidateMultiple } from './normalise-candidate-multiple.js' /** * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream - * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate * @typedef {import('ipfs-core-types/src/utils').BrowserImportCandidate} BrowserImportCandidate */ /** - * Transforms any of the `ipfs.add` input types into + * Transforms any of the `ipfs.addAll` input types into * * ``` * AsyncIterable<{ path, mode, mtime, content: Blob }> @@ -16,10 +15,10 @@ import { normalise } from './normalise.js' * * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options * - * @param {ImportCandidate | ImportCandidateStream} input + * @param {ImportCandidateStream} input * @returns {AsyncGenerator} */ export function normaliseInput (input) { - // @ts-ignore normaliseContent returns Blob and not AsyncIterator - return normalise(input, normaliseContent) + // @ts-expect-error browser normaliseContent returns a Blob not an AsyncIterable + return normaliseCandidateMultiple(input, normaliseContent, true) } diff --git a/packages/ipfs-core-utils/src/files/normalise-input-multiple.js b/packages/ipfs-core-utils/src/files/normalise-input-multiple.js new file mode 100644 index 0000000000..b95760edc5 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input-multiple.js @@ -0,0 +1,21 @@ +import { normaliseContent } from './normalise-content.js' +import { normaliseCandidateMultiple } from './normalise-candidate-multiple.js' + +/** + * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream + */ + +/** + * Transforms any of the `ipfs.addAll` input types into + * + * ``` + * AsyncIterable<{ path, mode, mtime, content: AsyncIterable }> + * ``` + * + * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options + * + * @param {ImportCandidateStream} input + */ +export function normaliseInput (input) { + return normaliseCandidateMultiple(input, normaliseContent) +} diff --git a/packages/ipfs-core-utils/src/files/normalise-input-single.browser.js b/packages/ipfs-core-utils/src/files/normalise-input-single.browser.js new file mode 100644 index 0000000000..50af999fc1 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input-single.browser.js @@ -0,0 +1,24 @@ +import { normaliseContent } from './normalise-content.browser.js' +import { normaliseCandidateSingle } from './normalise-candidate-single.js' + +/** + * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate + * @typedef {import('ipfs-core-types/src/utils').BrowserImportCandidate} BrowserImportCandidate + */ + +/** + * Transforms any of the `ipfs.add` input types into + * + * ``` + * AsyncIterable<{ path, mode, mtime, content: Blob }> + * ``` + * + * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options + * + * @param {ImportCandidate} input + * @returns {BrowserImportCandidate} + */ +export function normaliseInput (input) { + // @ts-expect-error browser normaliseContent returns a Blob not an AsyncIterable + return normaliseCandidateSingle(input, normaliseContent) +} diff --git a/packages/ipfs-core-utils/src/files/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input-single.js similarity index 65% rename from packages/ipfs-core-utils/src/files/normalise-input.js rename to packages/ipfs-core-utils/src/files/normalise-input-single.js index c3bba3d1e3..0946d1e77c 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input.js +++ b/packages/ipfs-core-utils/src/files/normalise-input-single.js @@ -1,8 +1,7 @@ import { normaliseContent } from './normalise-content.js' -import { normalise } from './normalise.js' +import { normaliseCandidateSingle } from './normalise-candidate-single.js' /** - * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate */ @@ -15,8 +14,8 @@ import { normalise } from './normalise.js' * * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options * - * @param {ImportCandidate | ImportCandidateStream} input + * @param {ImportCandidate} input */ export function normaliseInput (input) { - return normalise(input, normaliseContent) + return normaliseCandidateSingle(input, normaliseContent) } diff --git a/packages/ipfs-core-utils/src/multipart-request.browser.js b/packages/ipfs-core-utils/src/multipart-request.browser.js index 2f231b3352..be444082c7 100644 --- a/packages/ipfs-core-utils/src/multipart-request.browser.js +++ b/packages/ipfs-core-utils/src/multipart-request.browser.js @@ -1,16 +1,15 @@ // Import browser version otherwise electron-renderer will end up with node // version and fail. -import { normaliseInput } from './files/normalise-input.browser.js' +import { normaliseInput } from './files/normalise-input-multiple.browser.js' import { modeToString } from './mode-to-string.js' /** * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream - * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate */ /** - * @param {ImportCandidateStream|ImportCandidate} source + * @param {ImportCandidateStream} source * @param {AbortController} abortController * @param {Headers|Record} [headers] */ diff --git a/packages/ipfs-core-utils/src/multipart-request.js b/packages/ipfs-core-utils/src/multipart-request.js index e46fdd4780..2e2a503e23 100644 --- a/packages/ipfs-core-utils/src/multipart-request.js +++ b/packages/ipfs-core-utils/src/multipart-request.js @@ -4,12 +4,11 @@ import { multipartRequest as multipartRequestBrowser } from './multipart-request import { nanoid } from 'nanoid' /** - * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream */ /** - * @param {ImportCandidateStream|ImportCandidate} source + * @param {ImportCandidateStream} source * @param {AbortController} abortController * @param {Headers|Record} [headers] * @param {string} [boundary] diff --git a/packages/ipfs-core-utils/src/multipart-request.node.js b/packages/ipfs-core-utils/src/multipart-request.node.js index fda8a74246..d406deb50f 100644 --- a/packages/ipfs-core-utils/src/multipart-request.node.js +++ b/packages/ipfs-core-utils/src/multipart-request.node.js @@ -1,4 +1,4 @@ -import { normaliseInput } from './files/normalise-input.js' +import { normaliseInput } from './files/normalise-input-multiple.js' import { nanoid } from 'nanoid' import { modeToString } from './mode-to-string.js' import mergeOpts from 'merge-options' @@ -11,18 +11,17 @@ const log = debug('ipfs:core-utils:multipart-request') /** * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream - * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate */ /** - * @param {ImportCandidateStream|ImportCandidate} source + * @param {ImportCandidateStream} source * @param {AbortController} abortController * @param {Headers|Record} [headers] * @param {string} [boundary] */ export async function multipartRequest (source, abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { /** - * @param {ImportCandidateStream|ImportCandidate} source + * @param {ImportCandidateStream} source */ async function * streamFiles (source) { try { diff --git a/packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js b/packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js new file mode 100644 index 0000000000..8f49a833f5 --- /dev/null +++ b/packages/ipfs-core-utils/test/files/normalise-input-multiple.spec.js @@ -0,0 +1,240 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/utils/chai.js' +import blobToIt from 'blob-to-it' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import all from 'it-all' +import { File } from '@web-std/file' +import { normaliseInput } from '../../src/files/normalise-input-multiple.js' +import { isNode } from 'ipfs-utils/src/env.js' +import resolve from 'aegir/utils/resolve.js' + +const { Blob, ReadableStream } = globalThis + +const STRING = () => 'hello world' +const NEWSTRING = () => new String('hello world') // eslint-disable-line no-new-wrappers +const BUFFER = () => uint8ArrayFromString(STRING()) +const ARRAY = () => Array.from(BUFFER()) +const TYPEDARRAY = () => Uint8Array.from(ARRAY()) +/** @type {() => Blob} */ +let BLOB + +if (Blob) { + BLOB = () => new Blob([ + STRING() + ]) +} + +/** + * @param {import('ipfs-unixfs-importer').ImportCandidate[]} input + */ +async function verifyNormalisation (input) { + expect(input.length).to.equal(1) + expect(input[0].path).to.equal('') + + let content = input[0].content + + if (Blob && content instanceof Blob) { + content = blobToIt(content) + } + + if (!content || content instanceof Uint8Array) { + throw new Error('Content expected') + } + + await expect(all(content)).to.eventually.deep.equal([BUFFER()]) +} + +/** + * @param {*} input + */ +async function testContent (input) { + const result = await all(normaliseInput(input)) + + await verifyNormalisation(result) +} + +/** + * @template T + * @param {T} thing + * @returns {T[]} + */ +function iterableOf (thing) { + return [thing] +} + +/** + * @template T + * @param {T} thing + * @returns {AsyncIterable} + */ +function asyncIterableOf (thing) { + return (async function * () { // eslint-disable-line require-await + yield thing + }()) +} + +/** + * @param {*} thing + */ +function browserReadableStreamOf (thing) { + return new ReadableStream({ + start (controller) { + controller.enqueue(thing) + controller.close() + } + }) +} + +describe('normalise-input-multiple', function () { + /** + * @param {() => any} content + * @param {string} name + * @param {boolean} isBytes + */ + function testInputType (content, name, isBytes) { + it(name, async function () { + await testContent(content()) + }) + + if (isBytes) { + if (ReadableStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + + it(`Iterable<${name}>`, async function () { + await testContent(iterableOf(content())) + }) + + it(`AsyncIterable<${name}>`, async function () { + await testContent(asyncIterableOf(content())) + }) + } + + it(`{ path: '', content: ${name} }`, async function () { + await testContent({ path: '', content: content() }) + }) + + if (isBytes) { + if (ReadableStream) { + it(`{ path: '', content: ReadableStream<${name}> }`, async function () { + await testContent({ path: '', content: browserReadableStreamOf(content()) }) + }) + } + + it(`{ path: '', content: Iterable<${name}> }`, async function () { + await testContent({ path: '', content: iterableOf(content()) }) + }) + + it(`{ path: '', content: AsyncIterable<${name}> }`, async function () { + await testContent({ path: '', content: asyncIterableOf(content()) }) + }) + } + + if (ReadableStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + + it(`Iterable<{ path: '', content: ${name} }`, async function () { + await testContent(iterableOf({ path: '', content: content() })) + }) + + it(`AsyncIterable<{ path: '', content: ${name} }`, async function () { + await testContent(asyncIterableOf({ path: '', content: content() })) + }) + + if (isBytes) { + if (ReadableStream) { + it(`Iterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: browserReadableStreamOf(content()) })) + }) + } + + it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: iterableOf(content()) })) + }) + + it(`Iterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: asyncIterableOf(content()) })) + }) + + if (ReadableStream) { + it(`AsyncIterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: browserReadableStreamOf(content()) })) + }) + } + + it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: iterableOf(content()) })) + }) + + it(`AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: asyncIterableOf(content()) })) + }) + } + } + + describe('String', () => { + testInputType(STRING, 'String', true) + testInputType(NEWSTRING, 'new String()', true) + }) + + describe('Buffer', () => { + testInputType(BUFFER, 'Buffer', true) + }) + + describe('Blob', () => { + if (!Blob) { + return + } + + testInputType(BLOB, 'Blob', false) + }) + + describe('@web-std/file', () => { + it('normalizes File input', async () => { + const FILE = new File([BUFFER()], 'test-file.txt') + + await testContent(FILE) + }) + }) + + describe('Iterable', () => { + testInputType(ARRAY, 'Iterable', false) + }) + + describe('TypedArray', () => { + testInputType(TYPEDARRAY, 'TypedArray', true) + }) + + if (isNode) { + /** @type {import('fs')} */ + let fs + + before(async () => { + fs = await import('fs') + }) + + describe('Node fs.ReadStream', () => { + const NODEFSREADSTREAM = () => { + const path = resolve('test/fixtures/file.txt', 'ipfs-core-utils') + + return fs.createReadStream(path) + } + + testInputType(NODEFSREADSTREAM, 'Node fs.ReadStream', false) + + it('Iterable', async function () { + await testContent(iterableOf(NODEFSREADSTREAM())) + }) + + it('AsyncIterable', async function () { + await testContent(asyncIterableOf(NODEFSREADSTREAM())) + }) + }) + } +}) diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input-single.spec.js similarity index 98% rename from packages/ipfs-core-utils/test/files/normalise-input.spec.js rename to packages/ipfs-core-utils/test/files/normalise-input-single.spec.js index 7f1c2d8706..f44ba17012 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input-single.spec.js @@ -5,7 +5,7 @@ import blobToIt from 'blob-to-it' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import all from 'it-all' import { File } from '@web-std/file' -import { normaliseInput } from '../../src/files/normalise-input.js' +import { normaliseInput } from '../../src/files/normalise-input-single.js' import { isNode } from 'ipfs-utils/src/env.js' import resolve from 'aegir/utils/resolve.js' @@ -86,7 +86,7 @@ function browserReadableStreamOf (thing) { }) } -describe('normalise-input', function () { +describe('normalise-input-single', function () { /** * @param {() => any} content * @param {string} name diff --git a/packages/ipfs-core/src/components/add-all/index.js b/packages/ipfs-core/src/components/add-all/index.js index a0f765d2dc..f69c6f54e9 100644 --- a/packages/ipfs-core/src/components/add-all/index.js +++ b/packages/ipfs-core/src/components/add-all/index.js @@ -1,5 +1,5 @@ import { importer } from 'ipfs-unixfs-importer' -import { normaliseInput } from 'ipfs-core-utils/files/normalise-input' +import { normaliseInput } from 'ipfs-core-utils/files/normalise-input-multiple' import { parseChunkerString } from './utils.js' import { pipe } from 'it-pipe' import { withTimeoutOption } from 'ipfs-core-utils/with-timeout-option' diff --git a/packages/ipfs-core/src/components/add.js b/packages/ipfs-core/src/components/add.js index 2a2c500eab..b0cd6e9cb8 100644 --- a/packages/ipfs-core/src/components/add.js +++ b/packages/ipfs-core/src/components/add.js @@ -1,4 +1,5 @@ import last from 'it-last' +import { normaliseInput } from 'ipfs-core-utils/files/normalise-input-single' /** * @param {Object} context @@ -10,7 +11,7 @@ export function createAdd ({ addAll }) { */ async function add (entry, options = {}) { // @ts-ignore TODO: https://github.com/ipfs/js-ipfs/issues/3290 - const result = await last(addAll(entry, options)) + const result = await last(addAll(normaliseInput(entry), options)) // Note this should never happen as `addAll` should yield at least one item // but to satisfy type checker we perfom this check and for good measure // throw an error in case it does happen. diff --git a/packages/ipfs-grpc-client/src/core-api/add-all.js b/packages/ipfs-grpc-client/src/core-api/add-all.js index 690d731f2a..5304123f53 100644 --- a/packages/ipfs-grpc-client/src/core-api/add-all.js +++ b/packages/ipfs-grpc-client/src/core-api/add-all.js @@ -1,4 +1,4 @@ -import { normaliseInput } from 'ipfs-core-utils/files/normalise-input' +import { normaliseInput } from 'ipfs-core-utils/files/normalise-input-multiple' import { CID } from 'multiformats/cid' import { bidiToDuplex } from '../utils/bidi-to-duplex.js' import { withTimeoutOption } from 'ipfs-core-utils/with-timeout-option' diff --git a/packages/ipfs-http-client/src/add.js b/packages/ipfs-http-client/src/add.js index d668440b7c..979fc9a35d 100644 --- a/packages/ipfs-http-client/src/add.js +++ b/packages/ipfs-http-client/src/add.js @@ -1,6 +1,7 @@ import { createAddAll } from './add-all.js' import last from 'it-last' import { configure } from './lib/configure.js' +import { normaliseInput } from 'ipfs-core-utils/files/normalise-input-single' /** * @typedef {import('./types').HTTPClientExtraOptions} HTTPClientExtraOptions @@ -18,7 +19,7 @@ export function createAdd (options) { */ async function add (input, options = {}) { // @ts-ignore - last may return undefined if source is empty - return await last(all(input, options)) + return await last(all(normaliseInput(input), options)) } return add })(options) diff --git a/packages/ipfs-http-client/src/block/put.js b/packages/ipfs-http-client/src/block/put.js index 36ad65d1ae..0b1480402d 100644 --- a/packages/ipfs-http-client/src/block/put.js +++ b/packages/ipfs-http-client/src/block/put.js @@ -25,7 +25,7 @@ export const createPut = configure(api => { signal: signal, searchParams: toUrlSearchParams(options), ...( - await multipartRequest(data, controller, options.headers) + await multipartRequest([data], controller, options.headers) ) }) res = await response.json() diff --git a/packages/ipfs-http-client/src/config/replace.js b/packages/ipfs-http-client/src/config/replace.js index 196288a443..47e9268e54 100644 --- a/packages/ipfs-http-client/src/config/replace.js +++ b/packages/ipfs-http-client/src/config/replace.js @@ -23,7 +23,7 @@ export const createReplace = configure(api => { signal, searchParams: toUrlSearchParams(options), ...( - await multipartRequest(uint8ArrayFromString(JSON.stringify(config)), controller, options.headers) + await multipartRequest([uint8ArrayFromString(JSON.stringify(config))], controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/dag/put.js b/packages/ipfs-http-client/src/dag/put.js index ded70c968d..6192e1750a 100644 --- a/packages/ipfs-http-client/src/dag/put.js +++ b/packages/ipfs-http-client/src/dag/put.js @@ -39,7 +39,7 @@ export const createPut = (codecs, options) => { signal, searchParams: toUrlSearchParams(settings), ...( - await multipartRequest(serialized, controller, settings.headers) + await multipartRequest([serialized], controller, settings.headers) ) }) const data = await res.json() diff --git a/packages/ipfs-http-client/src/dht/put.js b/packages/ipfs-http-client/src/dht/put.js index 536e1099d1..259a70884a 100644 --- a/packages/ipfs-http-client/src/dht/put.js +++ b/packages/ipfs-http-client/src/dht/put.js @@ -28,7 +28,7 @@ export const createPut = configure(api => { ...options }), ...( - await multipartRequest(value, controller, options.headers) + await multipartRequest([value], controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/files/write.js b/packages/ipfs-http-client/src/files/write.js index 8fd918589f..13bf3ac3f9 100644 --- a/packages/ipfs-http-client/src/files/write.js +++ b/packages/ipfs-http-client/src/files/write.js @@ -29,12 +29,12 @@ export const createWrite = configure(api => { ...options }), ...( - await multipartRequest({ + await multipartRequest([{ content: input, path: 'arg', mode: modeToString(options.mode), mtime: parseMtime(options.mtime) - }, controller, options.headers) + }], controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/object/patch/append-data.js b/packages/ipfs-http-client/src/object/patch/append-data.js index 2e4f17bd4a..1496b15682 100644 --- a/packages/ipfs-http-client/src/object/patch/append-data.js +++ b/packages/ipfs-http-client/src/object/patch/append-data.js @@ -26,7 +26,7 @@ export const createAppendData = configure(api => { ...options }), ...( - await multipartRequest(data, controller, options.headers) + await multipartRequest([data], controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/object/patch/set-data.js b/packages/ipfs-http-client/src/object/patch/set-data.js index fcc9037711..cb9663f728 100644 --- a/packages/ipfs-http-client/src/object/patch/set-data.js +++ b/packages/ipfs-http-client/src/object/patch/set-data.js @@ -28,7 +28,7 @@ export const createSetData = configure(api => { ...options }), ...( - await multipartRequest(data, controller, options.headers) + await multipartRequest([data], controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/pubsub/publish.js b/packages/ipfs-http-client/src/pubsub/publish.js index 53191f4c69..5c51dca64e 100644 --- a/packages/ipfs-http-client/src/pubsub/publish.js +++ b/packages/ipfs-http-client/src/pubsub/publish.js @@ -27,7 +27,7 @@ export const createPublish = configure(api => { signal, searchParams, ...( - await multipartRequest(data, controller, options.headers) + await multipartRequest([data], controller, options.headers) ) })