From 7b49f7e4650f7fbd1dd7f85723fd64dfbcfd99b2 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Thu, 16 May 2019 23:36:16 +0200 Subject: [PATCH] feat: add support for File DOM API to files-regular (#986) * feat: add file dom api support to files api * feat: add support for File DOM API to files-regular * chore: fix package declaration cause npm is dumb * chore: fix lint * chore: add ipfs-utils * fix: change the requires to ipfs-utils * chore: increase max bundle size --- .aegir.js | 2 +- examples/upload-file-via-browser/.eslintrc | 11 ---- examples/upload-file-via-browser/.gitignore | 1 + examples/upload-file-via-browser/package.json | 26 +++++--- examples/upload-file-via-browser/server.js | 13 ---- examples/upload-file-via-browser/src/App.js | 27 ++++---- .../upload-file-via-browser/webpack.config.js | 9 +-- package.json | 5 +- src/files-regular/add.js | 26 ++------ src/utils/multipart.js | 7 --- src/utils/prepare-file.js | 61 ++++++++++++++++--- 11 files changed, 96 insertions(+), 92 deletions(-) delete mode 100644 examples/upload-file-via-browser/.eslintrc delete mode 100644 examples/upload-file-via-browser/server.js diff --git a/.aegir.js b/.aegir.js index 6effabad0..fe08c25af 100644 --- a/.aegir.js +++ b/.aegir.js @@ -5,7 +5,7 @@ const createServer = require('ipfsd-ctl').createServer const server = createServer() module.exports = { - bundlesize: { maxSize: '231kB' }, + bundlesize: { maxSize: '232kB' }, webpack: { resolve: { mainFields: ['browser', 'main'] diff --git a/examples/upload-file-via-browser/.eslintrc b/examples/upload-file-via-browser/.eslintrc deleted file mode 100644 index 676feeeb1..000000000 --- a/examples/upload-file-via-browser/.eslintrc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "standard", - "rules": { - "react/jsx-uses-react": 2, - "react/jsx-uses-vars": 2, - "react/react-in-jsx-scope": 2 - }, - "plugins": [ - "react" - ] -} diff --git a/examples/upload-file-via-browser/.gitignore b/examples/upload-file-via-browser/.gitignore index 44b50488c..c089921a3 100644 --- a/examples/upload-file-via-browser/.gitignore +++ b/examples/upload-file-via-browser/.gitignore @@ -2,3 +2,4 @@ node_modules npm-debug.log .DS_Store dist +yarn.lock \ No newline at end of file diff --git a/examples/upload-file-via-browser/package.json b/examples/upload-file-via-browser/package.json index d5e1fad68..8827052ca 100644 --- a/examples/upload-file-via-browser/package.json +++ b/examples/upload-file-via-browser/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "Upload file to IPFS via browser using js-ipfs-http-client with Webpack", "scripts": { - "start": "node server.js" + "start": "webpack-dev-server" }, "author": "Harlan T Wood ", "contributors": [ @@ -12,12 +12,24 @@ "license": "MIT", "devDependencies": { "@babel/core": "^7.4.3", - "ipfs-http-client": "../../", - "pull-file-reader": "~1.0.2", - "react": "~16.8.6", - "react-dom": "~16.8.6", - "react-hot-loader": "~4.8.4", - "webpack": "~4.31.0", + "@babel/preset-env": "^7.3.1", + "@babel/preset-react": "^7.0.0", + "eslint": "^5.16.0", + "eslint-plugin-react": "^7.11.1", + "react": "~16.6.3", + "react-dom": "~16.6.3", + "webpack": "~4.30.0", "webpack-dev-server": "~3.3.1" + }, + "eslintConfig" : { + "extends": "standard", + "rules": { + "react/jsx-uses-react": 2, + "react/jsx-uses-vars": 2, + "react/react-in-jsx-scope": 2 + }, + "plugins": [ + "react" + ] } } diff --git a/examples/upload-file-via-browser/server.js b/examples/upload-file-via-browser/server.js deleted file mode 100644 index 6b476dcb0..000000000 --- a/examples/upload-file-via-browser/server.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict' -let webpack = require('webpack') -let WebpackDevServer = require('webpack-dev-server') -let config = require('./webpack.config') - -new WebpackDevServer(webpack(config), { - publicPath: config.output.publicPath, - hot: true, - historyApiFallback: true -}).listen(3000, 'localhost', function (err) { - if (err) throw new Error(err) - console.log('Listening at localhost:3000') -}) diff --git a/examples/upload-file-via-browser/src/App.js b/examples/upload-file-via-browser/src/App.js index 77f5b43d2..258dc53ed 100644 --- a/examples/upload-file-via-browser/src/App.js +++ b/examples/upload-file-via-browser/src/App.js @@ -1,9 +1,6 @@ 'use strict' const React = require('react') -const ipfsClient = require('ipfs-http-client') - -// create a stream from a file, which enables uploads of big files without allocating memory twice -const fileReaderPullStream = require('pull-file-reader') +const ipfsClient = require('../../../src') class App extends React.Component { constructor () { @@ -11,7 +8,7 @@ class App extends React.Component { this.state = { added_file_hash: null } - this.ipfs = ipfsClient('localhost', '5001') + this.ipfs = ipfsClient('localhost', '58041') // bind methods this.captureFile = this.captureFile.bind(this) @@ -22,25 +19,23 @@ class App extends React.Component { captureFile (event) { event.stopPropagation() event.preventDefault() - const file = event.target.files[0] if (document.getElementById('keepFilename').checked) { - this.saveToIpfsWithFilename(file) + this.saveToIpfsWithFilename(event.target.files) } else { - this.saveToIpfs(file) + this.saveToIpfs(event.target.files) } } // Example #1 // Add file to IPFS and return a CID - saveToIpfs (file) { + saveToIpfs (files) { let ipfsId - const fileStream = fileReaderPullStream(file) - this.ipfs.add(fileStream, { progress: (prog) => console.log(`received: ${prog}`) }) + this.ipfs.add([...files], { progress: (prog) => console.log(`received: ${prog}`) }) .then((response) => { console.log(response) ipfsId = response[0].hash console.log(ipfsId) - this.setState({added_file_hash: ipfsId}) + this.setState({ added_file_hash: ipfsId }) }).catch((err) => { console.error(err) }) @@ -48,12 +43,12 @@ class App extends React.Component { // Example #2 // Add file to IPFS and wrap it in a directory to keep the original filename - saveToIpfsWithFilename (file) { + saveToIpfsWithFilename (files) { + const file = [...files][0] let ipfsId - const fileStream = fileReaderPullStream(file) const fileDetails = { path: file.name, - content: fileStream + content: file } const options = { wrapWithDirectory: true, @@ -65,7 +60,7 @@ class App extends React.Component { // CID of wrapping directory is returned last ipfsId = response[response.length - 1].hash console.log(ipfsId) - this.setState({added_file_hash: ipfsId}) + this.setState({ added_file_hash: ipfsId }) }).catch((err) => { console.error(err) }) diff --git a/examples/upload-file-via-browser/webpack.config.js b/examples/upload-file-via-browser/webpack.config.js index 6dfc86d24..c28ec1168 100644 --- a/examples/upload-file-via-browser/webpack.config.js +++ b/examples/upload-file-via-browser/webpack.config.js @@ -1,14 +1,11 @@ 'use strict' const path = require('path') -const webpack = require('webpack') module.exports = { mode: 'development', devtool: 'eval', entry: [ - 'webpack-dev-server/client?http://localhost:3000', - 'webpack/hot/only-dev-server', './src/index' ], output: { @@ -16,9 +13,6 @@ module.exports = { filename: 'bundle.js', publicPath: '/static/' }, - plugins: [ - new webpack.HotModuleReplacementPlugin() - ], module: { rules: [ { @@ -28,8 +22,7 @@ module.exports = { { loader: 'babel-loader', options: { - presets: ['@babel/preset-env', '@babel/preset-react'], - plugins: ['react-hot-loader/babel'] + presets: ['@babel/preset-env', '@babel/preset-react'] } } ] diff --git a/package.json b/package.json index 43879e244..5296c01ea 100644 --- a/package.json +++ b/package.json @@ -47,8 +47,10 @@ "is-stream": "^2.0.0", "iso-stream-http": "~0.1.2", "iso-url": "~0.4.6", + "ipfs-utils": "~0.0.3", "just-kebab-case": "^1.1.0", "just-map-keys": "^1.1.0", + "kind-of": "^6.0.2", "lru-cache": "^5.1.1", "multiaddr": "^6.0.6", "multibase": "~0.6.0", @@ -83,9 +85,8 @@ "chai": "^4.2.0", "cross-env": "^5.2.0", "dirty-chai": "^2.0.1", - "eslint-plugin-react": "^7.11.1", "go-ipfs-dep": "0.4.19", - "interface-ipfs-core": "~0.101.1", + "interface-ipfs-core": "~0.102.0", "ipfsd-ctl": "~0.42.0", "nock": "^10.0.2", "stream-equal": "^1.1.1" diff --git a/src/files-regular/add.js b/src/files-regular/add.js index c5506b48b..cb5898265 100644 --- a/src/files-regular/add.js +++ b/src/files-regular/add.js @@ -3,10 +3,10 @@ const promisify = require('promisify-es6') const ConcatStream = require('concat-stream') const once = require('once') -const isStream = require('is-stream') -const isSource = require('is-pull-stream').isSource +const { isSource } = require('is-pull-stream') const FileResultStreamConverter = require('../utils/file-result-stream-converter') const SendFilesStream = require('../utils/send-files-stream') +const validateAddInput = require('ipfs-utils/src/files/add-input-validation') module.exports = (send) => { const createAddStream = SendFilesStream(send, 'add') @@ -16,7 +16,6 @@ module.exports = (send) => { _callback = options options = null } - const callback = once(_callback) if (!options) { @@ -24,23 +23,10 @@ module.exports = (send) => { } options.converter = FileResultStreamConverter - // Buffer, pull stream or Node.js stream - const isBufferOrStream = obj => Buffer.isBuffer(obj) || isStream.readable(obj) || isSource(obj) - // An object like { content?, path? }, where content isBufferOrStream and path isString - const isContentObject = obj => { - if (typeof obj !== 'object') return false - // path is optional if content is present - if (obj.content) return isBufferOrStream(obj.content) - // path must be a non-empty string if no content - return Boolean(obj.path) && typeof obj.path === 'string' - } - // An input atom: a buffer, stream or content object - const isInput = obj => isBufferOrStream(obj) || isContentObject(obj) - // All is ok if data isInput or data is an array of isInput - const ok = isInput(_files) || (Array.isArray(_files) && _files.every(isInput)) - - if (!ok) { - return callback(new Error('invalid input: expected buffer, readable stream, pull stream, object or array of objects')) + try { + validateAddInput(_files) + } catch (err) { + return callback(err) } const files = [].concat(_files) diff --git a/src/utils/multipart.js b/src/utils/multipart.js index 22006c435..a6df61fd4 100644 --- a/src/utils/multipart.js +++ b/src/utils/multipart.js @@ -2,8 +2,6 @@ const Transform = require('readable-stream').Transform const isNode = require('detect-node') -const isSource = require('is-pull-stream').isSource -const toStream = require('pull-to-stream') const PADDING = '--' const NEW_LINE = '\r\n' @@ -77,12 +75,7 @@ class Multipart extends Transform { return callback() // early } - if (isSource(content)) { - content = toStream.readable(content) - } - // From now on we assume content is a stream - content.once('error', this.emit.bind(this, 'error')) content.once('end', () => { diff --git a/src/utils/prepare-file.js b/src/utils/prepare-file.js index 738c4a4c0..1ffe3f31c 100644 --- a/src/utils/prepare-file.js +++ b/src/utils/prepare-file.js @@ -2,6 +2,13 @@ const isNode = require('detect-node') const flatmap = require('flatmap') +const { Readable } = require('readable-stream') +const kindOf = require('kind-of') +const { isSource } = require('is-pull-stream') +const isStream = require('is-stream') +const pullToStream = require('pull-to-stream') +const { supportsFileReader } = require('ipfs-utils/src/supports') +const streamFromFileReader = require('ipfs-utils/src/streams/stream-from-filereader') function loadPaths (opts, file) { const path = require('path') @@ -73,10 +80,36 @@ function loadPaths (opts, file) { } } +function contentToStream (content) { + if (supportsFileReader && kindOf(content) === 'file') { + return streamFromFileReader(content) + } + + if (kindOf(content) === 'buffer') { + return new Readable({ + read () { + this.push(content) + this.push(null) + } + }) + } + + if (isSource(content)) { + return pullToStream.readable(content) + } + + if (isStream.readable(content)) { + return content + } + + throw new Error(`Input not supported. Expected Buffer|ReadableStream|PullStream|File got ${kindOf(content)}. Check the documentation for more info https://github.com/ipfs/interface-js-ipfs-core/blob/master/SPEC/FILES.md#add`) +} + function prepareFile (file, opts) { let files = [].concat(file) return flatmap(files, (file) => { + // add from fs with file path if (typeof file === 'string') { if (!isNode) { throw new Error('Can only add file paths in node') @@ -85,20 +118,34 @@ function prepareFile (file, opts) { return loadPaths(opts, file) } - if (file.path && !file.content) { - file.dir = true - return file - } + // add with object syntax { path : , content: