From 251eff0bc7e04096c3a09eee73e36637bab4066b Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 6 Dec 2019 14:27:55 +0000 Subject: [PATCH] feat: convert to async iterators (#15) * Remove support for Node.js streams and pull streams * Adds a `urlSource` utility that will be exported by `js-ipfs-http-client` and `js-ipfs` along side `globSource` BREAKING CHANGE: Support for Node.js streams and Pull Streams has been removed --- package.json | 15 +-- src/files/add-input-validation.js | 31 ------ src/files/glob-source.js | 3 +- src/files/normalise-input.js | 97 ------------------ src/files/url-source.js | 15 +++ src/streams/stream-from-filereader.js | 37 ------- test/files/add-input-validation.spec.js | 57 ----------- test/files/normalise-input.spec.js | 108 -------------------- test/streams/stream-from-filereader.spec.js | 65 ------------ 9 files changed, 21 insertions(+), 407 deletions(-) delete mode 100644 src/files/add-input-validation.js create mode 100644 src/files/url-source.js delete mode 100644 src/streams/stream-from-filereader.js delete mode 100644 test/files/add-input-validation.spec.js delete mode 100644 test/streams/stream-from-filereader.spec.js diff --git a/package.json b/package.json index 775e172..eb9f781 100644 --- a/package.json +++ b/package.json @@ -29,23 +29,18 @@ "buffer": "^5.2.1", "err-code": "^2.0.0", "fs-extra": "^8.1.0", - "is-buffer": "^2.0.3", "is-electron": "^2.2.0", - "is-pull-stream": "0.0.0", - "is-stream": "^2.0.0", - "it-glob": "0.0.4", - "kind-of": "^6.0.2", - "pull-stream-to-async-iterator": "^1.0.2", - "readable-stream": "^3.4.0" + "it-glob": "0.0.6", + "ky": "^0.15.0", + "ky-universal": "^0.3.0", + "stream-to-it": "^0.2.0" }, "devDependencies": { "aegir": "^20.3.0", "async-iterator-all": "^1.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", - "dirty-chai": "^2.0.1", - "pull-stream": "^3.6.13", - "readable-stream-2": "npm:readable-stream@^2.0.0" + "dirty-chai": "^2.0.1" }, "contributors": [ "Alan Shaw ", diff --git a/src/files/add-input-validation.js b/src/files/add-input-validation.js deleted file mode 100644 index cbfb952..0000000 --- a/src/files/add-input-validation.js +++ /dev/null @@ -1,31 +0,0 @@ -'use strict' - -const kindOf = require('kind-of') -const isStream = require('is-stream') -const { isSource } = require('is-pull-stream') -const isBuffer = require('is-buffer') - -const validateAddInput = (input) => { - // Buffer|ReadableStream|PullStream|File - const isPrimitive = obj => isBuffer(obj) || isStream.readable(obj) || isSource(obj) || kindOf(obj) === 'file' - - // An object like { content?, path? }, where content isBufferOrStream and path isString - const isContentObject = obj => { - if (typeof obj !== 'object') return false - // path is optional if content is present - if (obj.content) return isPrimitive(obj.content) - // path must be a non-empty string if no content - return Boolean(obj.path) && typeof obj.path === 'string' - } - - // An input atom: a buffer, stream or content object - const isInput = obj => isPrimitive(obj) || isContentObject(obj) - - if (isInput(input) || (Array.isArray(input) && input.every(isInput))) { - return true - } else { - throw new Error(`Input not supported. Expected Buffer|ReadableStream|PullStream|File|Array got ${kindOf(input)}. Check the documentation for more info https://github.com/ipfs/interface-js-ipfs-core/blob/master/SPEC/FILES.md#add`) - } -} - -module.exports = validateAddInput diff --git a/src/files/glob-source.js b/src/files/glob-source.js index e7e7b58..cc4a6cb 100644 --- a/src/files/glob-source.js +++ b/src/files/glob-source.js @@ -4,7 +4,6 @@ const fs = require('fs-extra') const glob = require('it-glob') const Path = require('path') const errCode = require('err-code') -const kindOf = require('kind-of') /** * Create an async iterator that yields paths that match requested file paths. @@ -20,7 +19,7 @@ const kindOf = require('kind-of') module.exports = async function * globSource (paths, options) { options = options || {} - if (kindOf(paths) === 'string') { + if (typeof paths === 'string') { paths = [paths] } diff --git a/src/files/normalise-input.js b/src/files/normalise-input.js index 531402f..81e323b 100644 --- a/src/files/normalise-input.js +++ b/src/files/normalise-input.js @@ -2,11 +2,7 @@ const errCode = require('err-code') const { Buffer } = require('buffer') -const pullStreamToIterable = require('pull-stream-to-async-iterator') -const { isSource } = require('is-pull-stream') const globalThis = require('../globalthis') -const { Readable } = require('stream') -const Readable3 = require('readable-stream') /* * Transform one of: @@ -21,8 +17,6 @@ const Readable3 = require('readable-stream') * { path, content: Iterable } [single file] * { path, content: Iterable } [single file] * { path, content: AsyncIterable } [single file] - * { path, content: PullStream } [single file] - * { path, content: Readable } [single file] * Iterable [single file] * Iterable [single file] * Iterable [multiple files] @@ -33,8 +27,6 @@ const Readable3 = require('readable-stream') * Iterable<{ path, content: Iterable }> [multiple files] * Iterable<{ path, content: Iterable }> [multiple files] * Iterable<{ path, content: AsyncIterable }> [multiple files] - * Iterable<{ path, content: PullStream }> [multiple files] - * Iterable<{ path, content: Readable }> [multiple files] * AsyncIterable [single file] * AsyncIterable [multiple files] * AsyncIterable [multiple files] @@ -44,30 +36,6 @@ const Readable3 = require('readable-stream') * AsyncIterable<{ path, content: Iterable }> [multiple files] * AsyncIterable<{ path, content: Iterable }> [multiple files] * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] - * AsyncIterable<{ path, content: PullStream }> [multiple files] - * AsyncIterable<{ path, content: Readable }> [multiple files] - * PullStream [single file] - * PullStream [multiple files] - * PullStream [multiple files] - * PullStream<{ path, content: Bytes }> [multiple files] - * PullStream<{ path, content: Bloby }> [multiple files] - * PullStream<{ path, content: String }> [multiple files] - * PullStream<{ path, content: Iterable }> [multiple files] - * PullStream<{ path, content: Iterable }> [multiple files] - * PullStream<{ path, content: AsyncIterable }> [multiple files] - * PullStream<{ path, content: PullStream }> [multiple files] - * PullStream<{ path, content: Readable }> [multiple files] - * Readable [single file] - * Readable [multiple files] - * Readable [multiple files] - * Readable<{ path, content: Bytes }> [multiple files] - * Readable<{ path, content: Bloby }> [multiple files] - * Readable<{ path, content: String }> [multiple files] - * Readable<{ path, content: Iterable }> [multiple files] - * Readable<{ path, content: Iterable }> [multiple files] - * Readable<{ path, content: AsyncIterable }> [multiple files] - * Readable<{ path, content: PullStream }> [multiple files] - * Readable<{ path, content: Readable }> [multiple files] * ``` * Into: * @@ -99,11 +67,6 @@ module.exports = function normaliseInput (input) { })() } - // Readable - if (isOldReadable(input)) { - input = upgradeOldStream(input) - } - // Iterable if (input[Symbol.iterator]) { return (async function * () { // eslint-disable-line require-await @@ -176,37 +139,6 @@ module.exports = function normaliseInput (input) { })() } - // PullStream - if (isSource(input)) { - return (async function * () { - const iterator = pullStreamToIterable(input)[Symbol.asyncIterator]() - const first = await iterator.next() - if (first.done) return iterator - - // PullStream - if (isBytes(first.value)) { - yield toFileObject((async function * () { // eslint-disable-line require-await - yield first.value - yield * iterator - })()) - return - } - - // PullStream - // PullStream - // PullStream<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for await (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') } @@ -235,11 +167,6 @@ function toAsyncIterable (input) { return blobToAsyncGenerator(input) } - // Readable - if (isOldReadable(input)) { - input = upgradeOldStream(input) - } - // Iterator if (input[Symbol.iterator]) { return (async function * () { // eslint-disable-line require-await @@ -278,22 +205,9 @@ function toAsyncIterable (input) { })() } - // PullStream - if (isSource(input)) { - return pullStreamToIterable(input) - } - throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT')) } -function isOldReadable (obj) { - if (obj[Symbol.iterator] || obj[Symbol.asyncIterator]) { - return false - } - - return Boolean(obj.readable) -} - function toBuffer (chunk) { return isBytes(chunk) ? chunk : Buffer.from(chunk) } @@ -311,17 +225,6 @@ function isFileObject (obj) { return typeof obj === 'object' && (obj.path || obj.content) } -function upgradeOldStream (stream) { - if (stream[Symbol.asyncIterator] || stream[Symbol.iterator]) { - return stream - } - - // in the browser the stream.Readable is not an async iterator but readble-stream@3 is... - stream[Symbol.asyncIterator] = Readable.prototype[Symbol.asyncIterator] || Readable3.prototype[Symbol.asyncIterator] - - return stream -} - function blobToAsyncGenerator (blob) { if (typeof blob.stream === 'function') { // firefox < 69 does not support blob.stream() diff --git a/src/files/url-source.js b/src/files/url-source.js new file mode 100644 index 0000000..a7e6322 --- /dev/null +++ b/src/files/url-source.js @@ -0,0 +1,15 @@ +'use strict' + +const { default: ky } = require('ky-universal') +const toIterable = require('stream-to-it/source') + +module.exports = async function * urlSource (url, options) { + options = options || {} + + const { body } = await ky.get(url) + + yield { + path: decodeURIComponent(new URL(url).pathname.split('/').pop() || ''), + content: toIterable(body) + } +} diff --git a/src/streams/stream-from-filereader.js b/src/streams/stream-from-filereader.js deleted file mode 100644 index cb818d4..0000000 --- a/src/streams/stream-from-filereader.js +++ /dev/null @@ -1,37 +0,0 @@ -'use strict' -const { Readable } = require('readable-stream') -const { supportsFileReader } = require('../supports') - -const streamFromFileReader = (file, options) => { - if (!supportsFileReader) { - throw new Error('FileReader DOM API is not supported.') - } - class FileStream extends Readable { - constructor (file, options = {}) { - super(options) - this.file = file - this.offset = options.offset || 0 - this.chunkSize = options.chunkSize || 1024 * 1024 - this.fileReader = new self.FileReader(file) - this.fileReader.onloadend = (event) => { - const data = event.target.result - if (data.byteLength === 0) { - return this.push(null) - } - this.push(new Uint8Array(data)) - } - this.fileReader.onerror = (err) => this.destroy(err) - } - - _read (size) { - const end = this.offset + this.chunkSize - const slice = file.slice(this.offset, end) - this.fileReader.readAsArrayBuffer(slice) - this.offset = end - } - } - - return new FileStream(file, options) -} - -module.exports = streamFromFileReader diff --git a/test/files/add-input-validation.spec.js b/test/files/add-input-validation.spec.js deleted file mode 100644 index f4ba1a7..0000000 --- a/test/files/add-input-validation.spec.js +++ /dev/null @@ -1,57 +0,0 @@ -'use strict' - -/* eslint-env mocha */ -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const validate = require('../../src/files/add-input-validation') -const { supportsFileReader } = require('../../src/supports') -const { Buffer } = require('buffer') -const { Readable } = require('readable-stream') -const empty = require('pull-stream/sources/empty') - -chai.use(dirtyChai) -const expect = chai.expect - -describe('add-input-validation', function () { - it('validates correct primitive input types', function () { - expect(validate(Buffer.from(('test')))).to.be.true() - expect(validate(new Readable())).to.be.true() - expect(validate(empty())).to.be.true() - - if (supportsFileReader) { - const file = new self.File(['test'], 'test.txt', { type: 'text/plain' }) - expect(validate(file)).to.be.true() - } - }) - - it('validates correct array of primitive input types', function () { - expect(validate([Buffer.from('test'), Buffer.from('test')])).to.be.true() - expect(validate([new Readable(), new Readable()])).to.be.true() - expect(validate([empty(), empty()])).to.be.true() - - if (supportsFileReader) { - const file = new self.File(['test'], 'test.txt', { type: 'text/plain' }) - expect(validate([file, file])).to.be.true() - } - }) - - it('validates correct form of object input', function () { - expect(validate({ path: '/path' })).to.be.true() - expect(validate({ path: '/path', content: Buffer.from('test') })).to.be.true() - expect(validate({ content: new Readable() })).to.be.true() - expect(validate({ content: empty() })).to.be.true() - if (supportsFileReader) { - expect(validate({ content: new Readable() })).to.be.true() - } - }) - - it('should throw with bad input', function () { - const regex = /Input not supported/ - expect(() => validate('test')).throw(regex) - expect(() => validate(2)).throw(regex) - expect(() => validate({ path: 3 })).throw(regex) - expect(() => validate({ path: 'path', content: 'test' })).throw(regex) - expect(() => validate({ path: 'path', content: 2 })).throw(regex) - expect(() => validate({})).throw(regex) - }) -}) diff --git a/test/files/normalise-input.spec.js b/test/files/normalise-input.spec.js index 2c11d75..270868a 100644 --- a/test/files/normalise-input.spec.js +++ b/test/files/normalise-input.spec.js @@ -7,10 +7,6 @@ const normalise = require('../../src/files/normalise-input') const { supportsFileReader } = require('../../src/supports') const { Buffer } = require('buffer') const all = require('async-iterator-all') -const pull = require('pull-stream') -const Readable2 = require('readable-stream-2') -const Readable3 = require('readable-stream') -const ReadableNode = require('stream').Readable const globalThis = require('../../src/globalthis') chai.use(dirtyChai) @@ -55,46 +51,6 @@ function asyncIterableOf (thing) { }()) } -function pullStreamOf (thing) { - return pull.values([thing]) -} - -function readable2Of (thing) { - const stream = new Readable2({ - objectMode: true, - read () { - this.push(thing) - this.push(null) - } - }) - - return stream -} - -function readable3Of (thing) { - const stream = new Readable3({ - objectMode: true, - read () { - this.push(thing) - this.push(null) - } - }) - - return stream -} - -function readableNodeOf (thing) { - const stream = new ReadableNode({ - objectMode: true, - read () { - this.push(thing) - this.push(null) - } - }) - - return stream -} - describe('normalise-input', function () { function testInputType (content, name, isBytes) { it(name, async function () { @@ -109,22 +65,6 @@ describe('normalise-input', function () { it(`AsyncIterable<${name}>`, async function () { await testContent(asyncIterableOf(content)) }) - - it(`PullStream<${name}>`, async function () { - await testContent(pullStreamOf(content)) - }) - - it(`Readable2<${name}>`, async function () { - await testContent(readable2Of(content)) - }) - - it(`Readable3<${name}>`, async function () { - await testContent(readable3Of(content)) - }) - - it(`ReadableNode<${name}>`, async function () { - await testContent(readableNodeOf(content)) - }) } it(`{ path: '', content: ${name} }`, async function () { @@ -139,22 +79,6 @@ describe('normalise-input', function () { it(`{ path: '', content: AsyncIterable<${name}> }`, async function () { await testContent({ path: '', content: asyncIterableOf(content) }) }) - - it(`{ path: '', content: PullStream<${name}> }`, async function () { - await testContent({ path: '', content: pullStreamOf(content) }) - }) - - it(`{ path: '', content: Readable2<${name}> }`, async function () { - await testContent({ path: '', content: readable2Of(content) }) - }) - - it(`{ path: '', content: Readable3<${name}> }`, async function () { - await testContent({ path: '', content: readable3Of(content) }) - }) - - it(`{ path: '', content: ReadableNode<${name}> }`, async function () { - await testContent({ path: '', content: readableNodeOf(content) }) - }) } it(`Iterable<{ path: '', content: ${name} }`, async function () { @@ -165,22 +89,6 @@ describe('normalise-input', function () { await testContent(asyncIterableOf({ path: '', content })) }) - it(`PullStream<{ path: '', content: ${name} }`, async function () { - await testContent(pullStreamOf({ path: '', content })) - }) - - it(`Readable2<{ path: '', content: ${name} }`, async function () { - await testContent(readable2Of({ path: '', content })) - }) - - it(`Readable3<{ path: '', content: ${name} }`, async function () { - await testContent(readable3Of({ path: '', content })) - }) - - it(`ReadableNode<{ path: '', content: ${name} }`, async function () { - await testContent(readableNodeOf({ path: '', content })) - }) - if (isBytes) { it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { await testContent(iterableOf({ path: '', content: iterableOf(content) })) @@ -197,22 +105,6 @@ describe('normalise-input', function () { it(`AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { await testContent(asyncIterableOf({ path: '', content: asyncIterableOf(content) })) }) - - it(`PullStream<{ path: '', content: PullStream<${name}> }>`, async function () { - await testContent(pullStreamOf({ path: '', content: pullStreamOf(content) })) - }) - - it(`Readable2<{ path: '', content: Readable2<${name}> }>`, async function () { - await testContent(readable2Of({ path: '', content: readable2Of(content) })) - }) - - it(`Readable3<{ path: '', content: Readable3<${name}> }>`, async function () { - await testContent(readable3Of({ path: '', content: readable3Of(content) })) - }) - - it(`ReadableNode<{ path: '', content: Readable3<${name}> }>`, async function () { - await testContent(readableNodeOf({ path: '', content: readableNodeOf(content) })) - }) } } diff --git a/test/streams/stream-from-filereader.spec.js b/test/streams/stream-from-filereader.spec.js deleted file mode 100644 index 5e8f8fd..0000000 --- a/test/streams/stream-from-filereader.spec.js +++ /dev/null @@ -1,65 +0,0 @@ -'use strict' - -/* eslint-env mocha */ -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const { supportsFileReader } = require('../../src/supports') -const streamFromFilereader = require('../../src/streams/stream-from-filereader') - -chai.use(dirtyChai) -const expect = chai.expect - -describe('stream-from-filereader', function () { - it('should throw in not supported envs', function () { - if (!supportsFileReader) { - expect(() => streamFromFilereader()).throw('FileReader DOM API is not supported.') - } else { - this.skip() - } - }) - - it('should return correct data from the stream', function (done) { - if (supportsFileReader) { - const s = streamFromFilereader(new self.File(['should return correct data from the stream'], 'filename.txt', { type: 'text/plain' })) - s.on('data', d => { - expect(d.toString()).to.eq('should return correct data from the stream') - done() - }) - } else { - this.skip() - } - }) - - it('should return correct ammount of data from the stream', function (done) { - if (supportsFileReader) { - const file = new self.File(['should return correct data from the stream'], 'filename.txt', { type: 'text/plain' }) - const s = streamFromFilereader(file, { chunkSize: 21 }) - let size = 0 - s.on('data', d => { - expect(d.byteLength).to.eq(21) - size += d.byteLength - }) - s.on('end', () => { - expect(size).to.eq(42) - done() - }) - } else { - this.skip() - } - }) - - it('should return correct data with offset', function (done) { - if (supportsFileReader) { - const file = new self.File(['should return correct data from the stream'], 'filename.txt', { type: 'text/plain' }) - const s = streamFromFilereader(file, { offset: 21 }) - s.on('data', d => { - expect(d.toString()).to.eq(' data from the stream') - }) - s.on('end', () => { - done() - }) - } else { - this.skip() - } - }) -})