Skip to content
This repository has been archived by the owner on Feb 12, 2024. It is now read-only.

Commit

Permalink
feat: avoid doing multiple RPC requests for files.add, fixes #522 (#595)
Browse files Browse the repository at this point in the history
* chore: test against go-ipfs 0.4.11

* feat: avoid doing multiple RPC requests for files.add, fixes #522

* new bundle sizes, so much space saved!

* use converter

* add progress tracking back
  • Loading branch information
daviddias authored Oct 18, 2017
1 parent e2d894c commit 0ea5f57
Show file tree
Hide file tree
Showing 9 changed files with 64 additions and 135 deletions.
54 changes: 27 additions & 27 deletions examples/sub-module/bundles-size-KBs.csv
Original file line number Diff line number Diff line change
@@ -1,28 +1,28 @@
name, bundled (KBs), minified (KBs)
IPFS, 1685.41, 661.64
add, 968.11, 344.13
bitswap, 684.80, 232.13
block, 724.77, 250.23
bootstrap, 685.64, 232.39
cat, 725.05, 250.33
commands, 683.89, 231.76
config, 686.87, 233.10
dht, 688.51, 233.41
diag, 684.97, 232.18
files, 1120.38, 404.30
get, 907.74, 318.35
id, 684.31, 231.95
key, 684.59, 232.03
log, 685.40, 232.38
ls, 684.00, 231.80
mount, 684.18, 231.86
name, 684.63, 232.06
object, 923.66, 340.57
pin, 685.51, 232.39
ping, 684.59, 231.90
pubsub, 740.40, 249.54
refs, 684.39, 231.94
repo, 684.56, 232.02
swarm, 1324.18, 527.03
update, 684.45, 231.96
version, 684.21, 231.88
IPFS, 1412.20, 573.44
add, 591.17, 198.23
bitswap, 590.14, 197.96
block, 630.50, 216.31
bootstrap, 590.97, 198.22
cat, 630.78, 216.41
commands, 589.22, 197.59
config, 592.21, 198.93
dht, 593.86, 199.24
diag, 590.31, 198.00
files, 669.07, 235.88
get, 661.57, 233.16
id, 589.65, 197.78
key, 589.93, 197.86
log, 590.74, 198.20
ls, 589.35, 197.63
mount, 589.53, 197.69
name, 589.97, 197.88
object, 833.17, 307.73
pin, 590.86, 198.22
ping, 589.94, 197.73
pubsub, 595.31, 199.76
refs, 589.74, 197.77
repo, 589.91, 197.85
swarm, 1239.42, 498.59
update, 589.79, 197.79
version, 589.55, 197.71
2 changes: 1 addition & 1 deletion examples/sub-module/test-modules-size.sh
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ modules=($(ls modules/))
echo "name, bundled (KBs), minified (KBs)"

# Full IPFS module
webpack --display none --config webpack.confg.js complete-module.js complete-bundle.js
webpack --display none --config webpack.config.js complete-module.js complete-bundle.js
babili complete-bundle.js -o complete-bundle-minified.js

ipfsBundleSize=($(wc -c < complete-bundle.js | awk '{b=$1/1024; printf "%.2f\n", b}' | sed 's/,/./g'))
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
"gulp": "^3.9.1",
"interface-ipfs-core": "~0.32.1",
"hapi": "^16.6.2",
"ipfsd-ctl": "~0.23.0",
"ipfsd-ctl": "~0.24.0",
"pre-commit": "^1.2.2",
"socket.io": "^2.0.3",
"socket.io-client": "^2.0.3",
Expand Down
9 changes: 4 additions & 5 deletions src/files/add.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

const isStream = require('is-stream')
const promisify = require('promisify-es6')
const DAGNodeStream = require('../utils/dagnode-stream')
const ProgressStream = require('../utils/progress-stream')
const converter = require('../utils/converter')

module.exports = (send) => {
return promisify((files, opts, callback) => {
Expand Down Expand Up @@ -44,9 +44,8 @@ module.exports = (send) => {

const request = { path: 'add', files: files, qs: qs, progress: opts.progress }

// Transform the response stream to DAGNode values
const transform = (res, callback) => DAGNodeStream
.streamToValue(send, ProgressStream.fromStream(opts.progress, res), callback)
send.andTransform(request, transform, callback)
send.andTransform(request, (response, cb) => {
converter(ProgressStream.fromStream(opts.progress, response), cb)
}, callback)
})
}
13 changes: 3 additions & 10 deletions src/util/fs-add.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

const isNode = require('detect-node')
const promisify = require('promisify-es6')
const DAGNodeStream = require('../utils/dagnode-stream')
const converter = require('../utils/converter')
const moduleConfig = require('../utils/module-config')

module.exports = (arg) => {
Expand Down Expand Up @@ -31,14 +31,7 @@ module.exports = (arg) => {
return callback(new Error('"path" must be a string'))
}

const request = {
path: 'add',
qs: opts,
files: path
}

// Transform the response stream to DAGNode values
const transform = (res, callback) => DAGNodeStream.streamToValue(send, res, callback)
send.andTransform(request, transform, callback)
const request = { path: 'add', files: path, qs: opts }
send.andTransform(request, converter, callback)
})
}
17 changes: 6 additions & 11 deletions src/util/url-add.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ const promisify = require('promisify-es6')
const once = require('once')
const parseUrl = require('url').parse
const request = require('../utils/request')
const DAGNodeStream = require('../utils/dagnode-stream')
const converter = require('../utils/converter')
const moduleConfig = require('../utils/module-config')

module.exports = (arg) => {
Expand All @@ -25,12 +25,12 @@ module.exports = (arg) => {
opts = {}
}

callback = once(callback)

if (!validUrl(url)) {
return callback(new Error('"url" param must be an http(s) url'))
}

callback = once(callback)

requestWithRedirect(url, opts, send, callback)
})
}
Expand All @@ -52,14 +52,9 @@ const requestWithRedirect = (url, opts, send, callback) => {
}
requestWithRedirect(redirection, opts, send, callback)
} else {
const params = {
path: 'add',
qs: opts,
files: res
}
// Transform the response stream to DAGNode values
const transform = (res, callback) => DAGNodeStream.streamToValue(send, res, callback)
send.andTransform(params, transform, callback)
const request = { path: 'add', files: res, qs: opts }

send.andTransform(request, converter, callback)
}
}).end()
}
44 changes: 20 additions & 24 deletions src/utils/dagnode-stream.js → src/utils/converter.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,18 @@
const pump = require('pump')
const TransformStream = require('readable-stream').Transform
const streamToValue = require('./stream-to-value')
const getDagNode = require('./get-dagnode')

/*
Transforms a stream of {Name, Hash} objects to include size
of the DAG object.
Usage: inputStream.pipe(DAGNodeStream({ send: send }))
Usage: inputStream.pipe(new Converter())
Input object format:
{
Name: '/path/to/file/foo.txt',
Hash: 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP'
Size: '20'
}
Output object format:
Expand All @@ -24,38 +24,34 @@ const getDagNode = require('./get-dagnode')
size: 20
}
*/
class DAGNodeStream extends TransformStream {
class ConverterStream extends TransformStream {
constructor (options) {
const opts = Object.assign(options || {}, { objectMode: true })
super(opts)
this._send = opts.send
}

static streamToValue (send, inputStream, callback) {
const outputStream = pump(inputStream, new DAGNodeStream({ send: send }), (err) => {
if (err) {
callback(err)
}
_transform (obj, enc, callback) {
this.push({
path: obj.Name,
hash: obj.Hash,
size: parseInt(obj.Size, 10)
})
streamToValue(outputStream, callback)

callback(null)
}
}

_transform (obj, enc, callback) {
getDagNode(this._send, obj.Hash, (err, node) => {
function converter (inputStream, callback) {
const outputStream = pump(
inputStream,
new ConverterStream(),
(err) => {
if (err) {
return callback(err)
}

const result = {
path: obj.Name,
hash: obj.Hash,
size: node.size
callback(err)
}

this.push(result)
callback(null)
})
}

streamToValue(outputStream, callback)
}

module.exports = DAGNodeStream
module.exports = converter
54 changes: 0 additions & 54 deletions src/utils/get-dagnode.js

This file was deleted.

4 changes: 2 additions & 2 deletions src/utils/stream-to-value.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ const concat = require('concat-stream')
/*
Concatenate a stream to a single value.
*/
function streamToValue (res, callback) {
function streamToValue (response, callback) {
pump(
res,
response,
concat((data) => callback(null, data)),
(err) => {
if (err) {
Expand Down

0 comments on commit 0ea5f57

Please sign in to comment.