From 4c995ef8b7305affcc5ee516326bf94d45a57726 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 23 Sep 2021 22:03:52 +0100 Subject: [PATCH 1/3] fix: handle node readable streams properly Readable streams returned from `fs.createReadStream` have a `.path` property which was throwing off the content normalisation. Fixes #3882 --- .../ipfs-core-utils/src/files/normalise-content.js | 9 ++++++++- packages/ipfs-core-utils/src/files/normalise.js | 12 +++++++++--- .../ipfs-grpc-client/src/core-api/files/write.js | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/ipfs-core-utils/src/files/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-content.js index 05df86fea9..4d3e4fa9d6 100644 --- a/packages/ipfs-core-utils/src/files/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-content.js @@ -14,7 +14,14 @@ import { /** * @param {import('./normalise').ToContent} input */ -export async function * normaliseContent (input) { +export async function normaliseContent (input) { + return toAsyncGenerator(input) +} + +/** + * @param {import('./normalise').ToContent} input + */ +async function * toAsyncGenerator (input) { // Bytes | String if (isBytes(input)) { yield toBytes(input) diff --git a/packages/ipfs-core-utils/src/files/normalise.js b/packages/ipfs-core-utils/src/files/normalise.js index df17c037ab..b925a8d792 100644 --- a/packages/ipfs-core-utils/src/files/normalise.js +++ b/packages/ipfs-core-utils/src/files/normalise.js @@ -22,7 +22,7 @@ import { /** * @param {ImportCandidate | ImportCandidateStream} input - * @param {(content:ToContent) => AsyncIterable} normaliseContent + * @param {(content:ToContent) => Promise>} normaliseContent */ // eslint-disable-next-line complexity export async function * normalise (input, normaliseContent) { @@ -72,6 +72,13 @@ export async function * normalise (input, normaliseContent) { return } + // Node ReadableStream + if (value._readableState) { + // @ts-ignore Node readable streams have a `.path` property so we need to pass it as the content + yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject({ content: value }, normaliseContent)) + return + } + // (Async)Iterable // (Async)Iterable // (Async)Iterable<{ path, content }> @@ -103,7 +110,7 @@ export async function * normalise (input, normaliseContent) { /** * @param {ImportCandidate} input - * @param {(content:ToContent) => AsyncIterable} normaliseContent + * @param {(content:ToContent) => Promise>} normaliseContent */ async function toFileObject (input, normaliseContent) { // @ts-ignore - Those properties don't exist on most input types @@ -117,7 +124,6 @@ async function toFileObject (input, normaliseContent) { } if (content) { - // @ts-ignore TODO vmx 2021-03-30 enable again file.content = await normaliseContent(content) } else if (!path) { // Not already a file object with path or content prop // @ts-ignore - input still can be different ToContent diff --git a/packages/ipfs-grpc-client/src/core-api/files/write.js b/packages/ipfs-grpc-client/src/core-api/files/write.js index 74c851c19a..f463a78fd1 100644 --- a/packages/ipfs-grpc-client/src/core-api/files/write.js +++ b/packages/ipfs-grpc-client/src/core-api/files/write.js @@ -12,7 +12,7 @@ import { * @param {*} content */ async function * stream (path, content) { - for await (const buf of normaliseContent(content)) { + for await (const buf of await normaliseContent(content)) { yield { path, content: buf } } } From 9ad8fb2d79caea57d2e140b093de2b8ceb3abc76 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 23 Sep 2021 23:06:06 +0100 Subject: [PATCH 2/3] chore: add tests --- .../test/files/normalise-input.spec.js | 36 +++++++++++++++++++ .../ipfs-core-utils/test/fixtures/file.txt | 1 + 2 files changed, 37 insertions(+) create mode 100644 packages/ipfs-core-utils/test/fixtures/file.txt diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index 967ad00724..174dbf89a2 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -6,6 +6,8 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import all from 'it-all' import { File } from '@web-std/file' import { normaliseInput } from '../../src/files/normalise-input.js' +import { isNode } from 'ipfs-utils/src/env.js' +import resolve from 'aegir/utils/resolve.js' const { Blob, ReadableStream } = globalThis @@ -84,6 +86,12 @@ function browserReadableStreamOf (thing) { }) } +function nodeReadStreamOf (thing) { + return (async function * () { // eslint-disable-line require-await + yield thing + }()) +} + describe('normalise-input', function () { /** * @param {() => any} content @@ -208,4 +216,32 @@ describe('normalise-input', function () { describe('TypedArray', () => { testInputType(TYPEDARRAY, 'TypedArray', true) }) + + if (isNode) { + /** @type {import('fs')} */ + let fs + + before(async () => { + fs = await import('fs') + }) + + describe('Node fs.ReadStream', () => { + const NODEFSREADSTREAM = () => { + const path = resolve('test/fixtures/file.txt', 'ipfs-core-utils') + + return fs.createReadStream(path) + } + + testInputType(NODEFSREADSTREAM, 'Node fs.ReadStream', false) + + it(`Iterable`, async function () { + await testContent(iterableOf(NODEFSREADSTREAM())) + }) + + it(`AsyncIterable`, async function () { + await testContent(asyncIterableOf(NODEFSREADSTREAM())) + }) + + }) + } }) diff --git a/packages/ipfs-core-utils/test/fixtures/file.txt b/packages/ipfs-core-utils/test/fixtures/file.txt new file mode 100644 index 0000000000..95d09f2b10 --- /dev/null +++ b/packages/ipfs-core-utils/test/fixtures/file.txt @@ -0,0 +1 @@ +hello world \ No newline at end of file From d4e26fc8ff9ac97cf588f43f36ef12527a0674ad Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 23 Sep 2021 23:10:32 +0100 Subject: [PATCH 3/3] chore: linting --- packages/ipfs-core-utils/src/files/normalise.js | 2 +- .../test/files/normalise-input.spec.js | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/ipfs-core-utils/src/files/normalise.js b/packages/ipfs-core-utils/src/files/normalise.js index b925a8d792..19b888943d 100644 --- a/packages/ipfs-core-utils/src/files/normalise.js +++ b/packages/ipfs-core-utils/src/files/normalise.js @@ -72,7 +72,7 @@ export async function * normalise (input, normaliseContent) { return } - // Node ReadableStream + // fs.ReadStream if (value._readableState) { // @ts-ignore Node readable streams have a `.path` property so we need to pass it as the content yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject({ content: value }, normaliseContent)) diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index 174dbf89a2..7f1c2d8706 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -86,12 +86,6 @@ function browserReadableStreamOf (thing) { }) } -function nodeReadStreamOf (thing) { - return (async function * () { // eslint-disable-line require-await - yield thing - }()) -} - describe('normalise-input', function () { /** * @param {() => any} content @@ -234,14 +228,13 @@ describe('normalise-input', function () { testInputType(NODEFSREADSTREAM, 'Node fs.ReadStream', false) - it(`Iterable`, async function () { + it('Iterable', async function () { await testContent(iterableOf(NODEFSREADSTREAM())) }) - it(`AsyncIterable`, async function () { + it('AsyncIterable', async function () { await testContent(asyncIterableOf(NODEFSREADSTREAM())) }) - }) } })