feat: consolidate ipfs.add input normalisation

Allows input normalisation function to be shared between ipfs and the
http client.

package.json

@@ -27,6 +27,7 @@
   "license": "MIT",
   "dependencies": {
     "buffer": "^5.2.1",
+    "err-code": "^2.0.0",
     "is-buffer": "^2.0.3",
     "is-electron": "^2.2.0",
     "is-pull-stream": "0.0.0",
@@ -36,9 +37,10 @@
   },
   "devDependencies": {
     "aegir": "^20.0.0",
+    "async-iterator-all": "^1.0.0",
     "chai": "^4.2.0",
     "dirty-chai": "^2.0.1",
-    "electron": "^5.0.7",
+    "electron": "^6.0.6",
     "electron-mocha": "^8.0.3",
     "pull-stream": "^3.6.13"
   },

src/files/normalise-input.js

@@ -0,0 +1,233 @@
+'use strict'
+
+const errCode = require('err-code')
+const { Buffer } = require('buffer')
+
+/*
+ * Transform one of:
+ *
+ * ```
+ * Buffer|ArrayBuffer|TypedArray
+ * Blob|File
+ * { path, content: Blob }
+ * { path, content: String }
+ * { path, content: Iterable<Number> }
+ * { path, content: Iterable<Buffer> }
+ * { path, content: Iterable<Iterable<Number>> }
+ * { path, content: AsyncIterable<Iterable<Number>> }
+ * String
+ * Iterable<Number>
+ * Iterable<Buffer>
+ * Iterable<Blob>
+ * Iterable<{ path, content: Buffer }>
+ * Iterable<{ path, content: Blob }>
+ * Iterable<{ path, content: Iterable<Number> }>
+ * Iterable<{ path, content: AsyncIterable<Buffer> }>
+ * AsyncIterable<Buffer>
+ * AsyncIterable<{ path, content: Buffer }>
+ * AsyncIterable<{ path, content: Blob }>
+ * AsyncIterable<{ path, content: Iterable<Buffer> }>
+ * AsyncIterable<{ path, content: AsyncIterable<Buffer> }>
+ * ```
+ * Into:
+ *
+ * ```
+ * AsyncIterable<{ path, content: AsyncIterable<Buffer> }>
+ * ```
+ *
+ * @param input Object
+ * @return AsyncInterable<{ path, content: AsyncIterable<Buffer> }>
+ */
+module.exports = function normaliseInput (input) {
+  // must give us something
+  if (input === null || input === undefined) {
+    throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT'))
+  }
+
+  // { path, content: ? }
+  if (isFileObject(input)) {
+    return (async function * () { // eslint-disable-line require-await
+      yield toFileObject(input)
+    })()
+  }
+
+  // String
+  if (typeof input === 'string' || input instanceof String) {
+    return (async function * () { // eslint-disable-line require-await
+      yield toFileObject(input)
+    })()
+  }
+
+  // Buffer|ArrayBuffer|TypedArray
+  // Blob|File
+  if (isBytes(input) || isBloby(input)) {
+    return (async function * () { // eslint-disable-line require-await
+      yield toFileObject(input)
+    })()
+  }
+
+  // Iterable<?>
+  if (input[Symbol.iterator]) {
+    // Iterable<Number>
+    if (!isNaN(input[0])) {
+      return (async function * () { // eslint-disable-line require-await
+        yield toFileObject([input])
+      })()
+    }
+
+    // Iterable<Buffer>
+    // Iterable<Blob>
+    return (async function * () { // eslint-disable-line require-await
+      for (const chunk of input) {
+        yield toFileObject(chunk)
+      }
+    })()
+  }
+
+  // AsyncIterable<?>
+  if (input[Symbol.asyncIterator]) {
+    return (async function * () { // eslint-disable-line require-await
+      for await (const chunk of input) {
+        yield toFileObject(chunk)
+      }
+    })()
+  }
+
+  throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')
+}
+
+function toFileObject (input) {
+  return {
+    path: input.path || '',
+    content: toAsyncIterable(input.content || input)
+  }
+}
+
+function toAsyncIterable (input) {
+  // Buffer|ArrayBuffer|TypedArray|array of bytes
+  if (isBytes(input)) {
+    return (async function * () { // eslint-disable-line require-await
+      yield toBuffer(input)
+    })()
+  }
+
+  if (typeof input === 'string' || input instanceof String) {
+    return (async function * () { // eslint-disable-line require-await
+      yield toBuffer(input)
+    })()
+  }
+
+  // Blob|File
+  if (isBloby(input)) {
+    return blobToAsyncGenerator(input)
+  }
+
+  // Iterator<?>
+  if (input[Symbol.iterator]) {
+    if (!isNaN(input[0])) {
+      return (async function * () { // eslint-disable-line require-await
+        yield toBuffer(input)
+      })()
+    }
+
+    return (async function * () { // eslint-disable-line require-await
+      for (const chunk of input) {
+        yield toBuffer(chunk)
+      }
+    })()
+  }
+
+  // AsyncIterable<?>
+  if (input[Symbol.asyncIterator]) {
+    return (async function * () {
+      for await (const chunk of input) {
+        yield toBuffer(chunk)
+      }
+    })()
+  }
+
+  throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT'))
+}
+
+function toBuffer (chunk) {
+  if (isBytes(chunk)) {
+    return chunk
+  }
+
+  if (typeof chunk === 'string' || chunk instanceof String) {
+    return Buffer.from(chunk)
+  }
+
+  if (Array.isArray(chunk)) {
+    return Buffer.from(chunk)
+  }
+
+  throw new Error('Unexpected input: ' + typeof chunk)
+}
+
+function isBytes (obj) {
+  return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer
+}
+
+function isBloby (obj) {
+  return typeof Blob !== 'undefined' && obj instanceof global.Blob
+}
+
+// An object with a path or content property
+function isFileObject (obj) {
+  return typeof obj === 'object' && (obj.path || obj.content)
+}
+
+function blobToAsyncGenerator (blob) {
+  if (typeof blob.stream === 'function') {
+    // firefox < 69 does not support blob.stream()
+    return streamBlob(blob)
+  }
+
+  return readBlob(blob)
+}
+
+async function * streamBlob (blob) {
+  const reader = blob.stream().getReader()
+
+  while (true) {
+    const result = await reader.read()
+
+    if (result.done) {
+      return
+    }
+
+    yield result.value
+  }
+}
+
+async function * readBlob (blob, options) {
+  options = options || {}
+
+  const reader = new global.FileReader()
+  const chunkSize = options.chunkSize || 1024 * 1024
+  let offset = options.offset || 0
+
+  const getNextChunk = () => new Promise((resolve, reject) => {
+    reader.onloadend = e => {
+      const data = e.target.result
+      resolve(data.byteLength === 0 ? null : data)
+    }
+    reader.onerror = reject
+
+    const end = offset + chunkSize
+    const slice = blob.slice(offset, end)
+    reader.readAsArrayBuffer(slice)
+    offset = end
+  })
+
+  while (true) {
+    const data = await getNextChunk()
+
+    if (data == null) {
+      return
+    }
+
+    yield Buffer.from(data)
+  }
+}

test/files/normalise-input.spec.js

@@ -0,0 +1,138 @@
+'use strict'
+
+/* eslint-env mocha */
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const normalise = require('../../src/files/normalise-input')
+const { supportsFileReader } = require('../../src/supports')
+const { Buffer } = require('buffer')
+const all = require('async-iterator-all')
+
+chai.use(dirtyChai)
+const expect = chai.expect
+
+const STRING = 'hello world'
+const BUFFER = Buffer.from(STRING)
+const ARRAY = Array.from(BUFFER)
+let BLOB
+
+if (supportsFileReader) {
+  BLOB = new global.Blob([
+    STRING
+  ])
+}
+
+async function verifyNormalisation (input) {
+  expect(input.length).to.equal(1)
+
+  if (!input[0].content[Symbol.asyncIterator] && !input[0].content[Symbol.iterator]) {
+    chai.assert.fail(`Content should have been an iterable or an async iterable`)
+  }
+
+  expect(await all(input[0].content)).to.deep.equal([BUFFER])
+  expect(input[0].path).to.equal('')
+}
+
+async function testContent (input) {
+  const result = await all(normalise(input))
+
+  await verifyNormalisation(result)
+}
+
+function iterableOf (thing) {
+  return [thing]
+}
+
+function asyncIterableOf (thing) {
+  return (async function * () { // eslint-disable-line require-await
+    yield thing
+  }())
+}
+
+describe('normalise-input', function () {
+  function testInputType (content, name) {
+    it(name, async function () {
+      await testContent(content)
+    })
+
+    it(`Iterable<${name}>`, async function () {
+      await testContent(iterableOf(content))
+    })
+
+    it(`AsyncIterable<${name}>`, async function () {
+      await testContent(asyncIterableOf(content))
+    })
+
+    if (name !== 'Blob') {
+      it(`AsyncIterable<Iterable<${name}>>`, async function () {
+        await testContent(asyncIterableOf(iterableOf(content)))
+      })
+
+      it(`AsyncIterable<AsyncIterable<${name}>>`, async function () {
+        await testContent(asyncIterableOf(asyncIterableOf(content)))
+      })
+    }
+
+    it(`{ path: '', content: ${name} }`, async function () {
+      await testContent({ path: '', content })
+    })
+
+    if (name !== 'Blob') {
+      it(`{ path: '', content: Iterable<${name}> }`, async function () {
+        await testContent({ path: '', content: iterableOf(content) })
+      })
+
+      it(`{ path: '', content: AsyncIterable<${name}> }`, async function () {
+        await testContent({ path: '', content: asyncIterableOf(content) })
+      })
+    }
+
+    it(`Iterable<{ path: '', content: ${name} }`, async function () {
+      await testContent(iterableOf({ path: '', content }))
+    })
+
+    if (name !== 'Blob') {
+      it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () {
+        await testContent(iterableOf({ path: '', content: iterableOf(content) }))
+      })
+
+      it(`Iterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () {
+        await testContent(iterableOf({ path: '', content: asyncIterableOf(content) }))
+      })
+    }
+
+    it(`AsyncIterable<{ path: '', content: ${name} }`, async function () {
+      await testContent(asyncIterableOf({ path: '', content }))
+    })
+
+    if (name !== 'Blob') {
+      it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () {
+        await testContent(asyncIterableOf({ path: '', content: iterableOf(content) }))
+      })
+
+      it(`AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () {
+        await testContent(asyncIterableOf({ path: '', content: asyncIterableOf(content) }))
+      })
+    }
+  }
+
+  describe('String', () => {
+    testInputType(STRING, 'String')
+  })
+
+  describe('Buffer', () => {
+    testInputType(BUFFER, 'Buffer')
+  })
+
+  describe('Blob', () => {
+    if (!supportsFileReader) {
+      return
+    }
+
+    testInputType(BLOB, 'Blob')
+  })
+
+  describe('Iterable<Number>', () => {
+    testInputType(ARRAY, 'Iterable<Number>')
+  })
+})