From 6be2938f7709682eebc9fdcce607a01c2cba2824 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 8 Jul 2015 15:59:25 -0700 Subject: [PATCH 001/634] Initial commit --- .gitignore | 27 +++++++++++++++++++++++++++ LICENSE | 22 ++++++++++++++++++++++ README.md | 2 ++ 3 files changed, 51 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..123ae94d05 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# Logs +logs +*.log + +# Runtime data +pids +*.pid +*.seed + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directory +# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git +node_modules diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..59a33bab48 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 David Dias + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 0000000000..afbc441a68 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# node-ipfs-swarm +IPFS swarm implementation in Node.js From a74aac6e11e4eff3c893ffd7919968139efa67a8 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 8 Jul 2015 16:22:59 -0700 Subject: [PATCH 002/634] Initial Commit --- .jshintignore | 1 + .jshintrc | 10 +++ examples/network-c.js | 21 ++++++ examples/network-s.js | 7 ++ package.json | 34 +++++++++ src/identify.js | 45 ++++++++++++ src/index.js | 161 ++++++++++++++++++++++++++++++++++++++++++ tests/swarm-test.js | 0 8 files changed, 279 insertions(+) create mode 100644 .jshintignore create mode 100644 .jshintrc create mode 100644 examples/network-c.js create mode 100644 examples/network-s.js create mode 100644 package.json create mode 100644 src/identify.js create mode 100644 src/index.js create mode 100644 tests/swarm-test.js diff --git a/.jshintignore b/.jshintignore new file mode 100644 index 0000000000..3c3629e647 --- /dev/null +++ b/.jshintignore @@ -0,0 +1 @@ +node_modules diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 0000000000..997b3f7d45 --- /dev/null +++ b/.jshintrc @@ -0,0 +1,10 @@ +{ + "node": true, + + "curly": true, + "latedef": true, + "quotmark": true, + "undef": true, + "unused": true, + "trailing": true +} diff --git a/examples/network-c.js b/examples/network-c.js new file mode 100644 index 0000000000..4e411f43f1 --- /dev/null +++ b/examples/network-c.js @@ -0,0 +1,21 @@ +var swarm = require('./network/swarm') +var Peer = require('./routing/routers/dht/peer') +var Id = require('./routing/routers/dht/peer/id') +var multiaddr = require('multiaddr') + +// create Id +// create multiaddr +// create Peer +// openStream + +var peerId = Id.create() +var mhs = [] +mhs.push(multiaddr('/ip4/127.0.0.1/tcp/4001')) +var p = new Peer(peerId, mhs) + +swarm.openStream(p, '/ipfs/sparkles/1.2.3', function (err, stream) { + if (err) { + return console.log('ERR - ', err) + } + console.log('WoHoo, dialed a stream') +}) diff --git a/examples/network-s.js b/examples/network-s.js new file mode 100644 index 0000000000..439f256b54 --- /dev/null +++ b/examples/network-s.js @@ -0,0 +1,7 @@ +var swarm = require('./network/swarm') + +swarm.listen() + +swarm.registerHandle('/ipfs/sparkles/1.2.3', function (stream) { + console.log('woop got a stream') +}) diff --git a/package.json b/package.json new file mode 100644 index 0000000000..7bcae4bac2 --- /dev/null +++ b/package.json @@ -0,0 +1,34 @@ +{ + "name": "ipfs-swarm", + "version": "0.0.0", + "description": "IPFS swarm implementation in Node.js", + "main": "src/index.js", + "scripts": { + "test": "./node_modules/.bin/lab tests/*-test.js", + "coverage": "./node_modules/.bin/lab -t 100 tests/*-test.js", + "codestyle": "./node_modules/.bin/standard --format" + }, + "repository": { + "type": "git", + "url": "https://github.com/diasdavid/node-ipfs-swarm.git" + }, + "keywords": [ + "IPFS" + ], + "author": "David Dias ", + "license": "MIT", + "bugs": { + "url": "https://github.com/diasdavid/node-ipfs-swarm/issues" + }, + "homepage": "https://github.com/diasdavid/node-ipfs-swarm", + "pre-commit": [ + "codestyle", + "test" + ], + "devDependencies": { + "code": "^1.4.1", + "lab": "^5.13.0", + "precommit-hook": "^3.0.0", + "standard": "^4.5.2" + } +} diff --git a/src/identify.js b/src/identify.js new file mode 100644 index 0000000000..34f9792f37 --- /dev/null +++ b/src/identify.js @@ -0,0 +1,45 @@ +/* + * Identify is one of the protocols swarms speaks in order to broadcast and learn about the ip:port + * pairs a specific peer is available through + */ + +var swarm = require('./../swarm') +var Interactive = require('multistream-select').Interactive + +exports = module.exports + +// peer acting as server, asking whom is talking +exports.inquiry = function (spdyConnection, cb) { + spdyConnection.request({method: 'GET', path: '/', headers: {}}, function (stream) { + var msi = new Interactive() + msi.handle(stream) + msi.select('/ipfs/identify/1.0.0', function (ds) { + var peerId = '' + ds.setEncoding('utf8') + + ds.on('data', function (chunk) { + peerId += chunk + }) + ds.on('end', function () { + cb(null, spdyConnection, peerId) + }) + }) + }) + // 0. open a stream + // 1. negotiate /ipfs/identify/1.0.0 + // 2. check other peerId + // 3. reply back with cb(null, connection, peerId) +} + +// peer asking which pairs ip:port does the other peer see +exports.whoAmI = function () { + +} + +exports.start = function (peerSelf) { + swarm.registerHandle('/ipfs/identify/1.0.0', function (ds) { + ds.setDefaultEncoding('utf8') + ds.write(peerSelf.toB58String()) + ds.end() + }) +} diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000000..7524cd875b --- /dev/null +++ b/src/index.js @@ -0,0 +1,161 @@ +var tcp = require('net') +var Select = require('multistream-select').Select +var Interactive = require('multistream-select').Interactive +var spdy = require('spdy-transport') +var identify = require('./identify') +var log = require('ipfs-logger').group('swarm') +var async = require('async') + +exports = module.exports + +var connections = {} +var handles = [] + +// set the listener + +exports.listen = function () { + + tcp.createServer(function (socket) { + var ms = new Select() + ms.handle(socket) + ms.addHandler('/spdy/3.1.0', function (ds) { + log.info('Negotiated spdy with incoming socket') + log.info('Buffer should be clean - ', ds.read()) + var spdyConnection = spdy.connection.create(ds, { + protocol: 'spdy', + isServer: true + }) + + spdyConnection.start(3.1) + + // attach multistream handlers to incoming streams + spdyConnection.on('stream', function (spdyStream) { + registerHandles(spdyStream) + }) + + // learn about the other peer Identity + /* TODO(daviddias) + identify.inquiry(spdyConnection, function (err, spdyConnection, peerIdB58) { + if (err) { + return log.error(err) + } + if (connections[peerIdB58]) { + return log.error('New connection established with a peer(' + peerIdB58 + ') that we already had a connection') + } + spdyConnection.peerId = peerIdB58 + connections[peerIdB58] = spdyConnection + }) + */ + + // close the connection when all the streams close + spdyConnection.on('close', function () { + delete connections[spdyConnection.peerId] + }) + }) + }).listen(process.env.IPFS_PORT || 4001) +} + +// interface + +exports.openStream = function (peer, protocol, cb) { + // if Connection already open, open a new stream, otherwise, create a new Connection + // then negoatite the protocol and return the opened stream + + // If no connection open yet, open it + if (!connections[peer.id.toB58String()]) { + + // Establish a socket with one of the addresses + var gotOne = false + async.eachSeries(peer.multiaddrs, function (multiaddr, callback) { + if (gotOne) { + return callback() + } + var socket = tcp.connect(multiaddr.toOptions(), function connected () { + gotSocket(socket) + }) + + socket.once('error', function (err) { + log.warn('Could not connect using one of the address of peer - ', peer.id.toB58String(), err) + callback() + }) + + }, function done () { + if (!gotOne) { + cb(new Error('Not able to open a scoket with peer - ', peer.id.toB58String())) + } + }) + + } else { + createStream(peer, protocol, cb) + } + + // do the spdy people dance (multistream-select into spdy) + function gotSocket (socket) { + gotOne = true + var msi = new Interactive() + msi.handle(socket, function () { + msi.select('/spdy/3.1.0', function (err, ds) { + var spdyConnection = spdy.connection.create(ds, { + protocol: 'spdy', + isServer: false + }) + spdyConnection.start(3.1) + connections[peer.id.toB58String()] = spdyConnection + + // attach multistream handlers to incoming streams + spdyConnection.on('stream', function (spdyStream) { + registerHandles(spdyStream) + }) + + createStream(peer, protocol, cb) + }) + }) + } + + function createStream (peer, protocol, cb) { + // 1. to pop a new stream on the connection + // 2. negotiate the requested protocol through multistream + // 3. return back the stream when that is negotiated + var conn = connections[peer.id.toB58String()] + conn.request({path: '/', method: 'GET'}, function (err, stream) { + if (err) { + return cb(err) + } + var msi = new Interactive() + msi.handle(stream, function () { + msi.select(protocol, function (err, ds) { + if (err) { + return cb(err) + } + cb(null, ds) // wohoo we finally delivered the stream the user wanted + }) + }) + }) + + conn.on('close', function () { + // TODO(daviddias) remove it from collections + }) + + } + +} + +exports.registerHandle = function (protocol, cb) { + if (handles[protocol]) { + var err = new Error('Handle for protocol already exists', protocol) + log.error(err) + return cb(err) + } + handles.push({ protocol: protocol, func: cb }) + log.info('Registered handler for protocol:', protocol) +} + +function registerHandles (spdyStream) { + log.info('Preparing stream to handle the registered protocols') + var msH = new Select() + msH.handle(spdyStream) + handles.forEach(function (handle) { + msH.addHandler(handle.protocol, handle.func) + }) +} + diff --git a/tests/swarm-test.js b/tests/swarm-test.js new file mode 100644 index 0000000000..e69de29bb2 From b1f7e65f18f674e05eae71f3d04815a30eb0a82e Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 8 Jul 2015 16:23:03 -0700 Subject: [PATCH 003/634] Initial Commit --- src/identify.js | 12 +++++------- src/index.js | 3 --- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/identify.js b/src/identify.js index 34f9792f37..c185ea5e70 100644 --- a/src/identify.js +++ b/src/identify.js @@ -25,16 +25,14 @@ exports.inquiry = function (spdyConnection, cb) { }) }) }) - // 0. open a stream - // 1. negotiate /ipfs/identify/1.0.0 - // 2. check other peerId - // 3. reply back with cb(null, connection, peerId) +// 0. open a stream +// 1. negotiate /ipfs/identify/1.0.0 +// 2. check other peerId +// 3. reply back with cb(null, connection, peerId) } // peer asking which pairs ip:port does the other peer see -exports.whoAmI = function () { - -} +exports.whoAmI = function () {} exports.start = function (peerSelf) { swarm.registerHandle('/ipfs/identify/1.0.0', function (ds) { diff --git a/src/index.js b/src/index.js index 7524cd875b..c12267f2de 100644 --- a/src/index.js +++ b/src/index.js @@ -14,7 +14,6 @@ var handles = [] // set the listener exports.listen = function () { - tcp.createServer(function (socket) { var ms = new Select() ms.handle(socket) @@ -63,7 +62,6 @@ exports.openStream = function (peer, protocol, cb) { // If no connection open yet, open it if (!connections[peer.id.toB58String()]) { - // Establish a socket with one of the addresses var gotOne = false async.eachSeries(peer.multiaddrs, function (multiaddr, callback) { @@ -158,4 +156,3 @@ function registerHandles (spdyStream) { msH.addHandler(handle.protocol, handle.func) }) } - From 3cd6aef2062b082ecf4e8a6e659f47b7e42d9f5c Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 8 Jul 2015 16:33:57 -0700 Subject: [PATCH 004/634] add badges --- README.md | 12 ++++++++++-- examples/network-c.js | 6 +++--- examples/network-s.js | 2 +- src/identify.js | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index afbc441a68..195bfc1e75 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ -# node-ipfs-swarm -IPFS swarm implementation in Node.js +ipfs-swarm Node.js implementation +================================= + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) + +> IPFS swarm implementation in Node.js + +# Description + +# Usage diff --git a/examples/network-c.js b/examples/network-c.js index 4e411f43f1..4e6835d1e8 100644 --- a/examples/network-c.js +++ b/examples/network-c.js @@ -1,6 +1,6 @@ -var swarm = require('./network/swarm') -var Peer = require('./routing/routers/dht/peer') -var Id = require('./routing/routers/dht/peer/id') +var swarm = require('./../src') +var Peer = require('ipfs-peer') +var Id = require('ipfs-peer-id') var multiaddr = require('multiaddr') // create Id diff --git a/examples/network-s.js b/examples/network-s.js index 439f256b54..de2f7b354d 100644 --- a/examples/network-s.js +++ b/examples/network-s.js @@ -1,4 +1,4 @@ -var swarm = require('./network/swarm') +var swarm = require('./../src') swarm.listen() diff --git a/src/identify.js b/src/identify.js index c185ea5e70..7e4fab8ae4 100644 --- a/src/identify.js +++ b/src/identify.js @@ -3,7 +3,7 @@ * pairs a specific peer is available through */ -var swarm = require('./../swarm') +var swarm = require('./') var Interactive = require('multistream-select').Interactive exports = module.exports From ac2f625f717cae46aa567fa0614b1588dc8406a3 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 8 Jul 2015 17:00:30 -0700 Subject: [PATCH 005/634] dials a stream :) --- package.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/package.json b/package.json index 7bcae4bac2..8ff555521a 100644 --- a/package.json +++ b/package.json @@ -30,5 +30,11 @@ "lab": "^5.13.0", "precommit-hook": "^3.0.0", "standard": "^4.5.2" + }, + "dependencies": { + "async": "^1.3.0", + "multiaddr": "^1.0.0", + "multistream-select": "^0.6.1", + "spdy-transport": "indutny/spdy-transport" } } From 9c062ffeeb38bc8b5df38fa656abfae0f45c73fd Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 8 Jul 2015 23:01:36 -0700 Subject: [PATCH 006/634] have at least one test --- examples/network-c.js | 2 +- examples/network-s.js | 2 +- package.json | 3 +- src/identify.js | 2 +- src/index.js | 160 +------------------------------------ src/swarm.js | 178 ++++++++++++++++++++++++++++++++++++++++++ tests/swarm-test.js | 71 +++++++++++++++++ 7 files changed, 257 insertions(+), 161 deletions(-) create mode 100644 src/swarm.js diff --git a/examples/network-c.js b/examples/network-c.js index 4e6835d1e8..6fcad79679 100644 --- a/examples/network-c.js +++ b/examples/network-c.js @@ -1,4 +1,4 @@ -var swarm = require('./../src') +var swarm = require('./../src').singleton var Peer = require('ipfs-peer') var Id = require('ipfs-peer-id') var multiaddr = require('multiaddr') diff --git a/examples/network-s.js b/examples/network-s.js index de2f7b354d..666c5bfd5d 100644 --- a/examples/network-s.js +++ b/examples/network-s.js @@ -1,4 +1,4 @@ -var swarm = require('./../src') +var swarm = require('./../src').singleton swarm.listen() diff --git a/package.json b/package.json index 8ff555521a..825b777075 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,8 @@ "code": "^1.4.1", "lab": "^5.13.0", "precommit-hook": "^3.0.0", - "standard": "^4.5.2" + "standard": "^4.5.2", + "stream-pair": "^1.0.2" }, "dependencies": { "async": "^1.3.0", diff --git a/src/identify.js b/src/identify.js index 7e4fab8ae4..9695af33c4 100644 --- a/src/identify.js +++ b/src/identify.js @@ -3,7 +3,7 @@ * pairs a specific peer is available through */ -var swarm = require('./') +var swarm = require('./').singleton var Interactive = require('multistream-select').Interactive exports = module.exports diff --git a/src/index.js b/src/index.js index c12267f2de..b825469a3a 100644 --- a/src/index.js +++ b/src/index.js @@ -1,158 +1,4 @@ -var tcp = require('net') -var Select = require('multistream-select').Select -var Interactive = require('multistream-select').Interactive -var spdy = require('spdy-transport') -var identify = require('./identify') -var log = require('ipfs-logger').group('swarm') -var async = require('async') +var Swarm = require('./swarm') -exports = module.exports - -var connections = {} -var handles = [] - -// set the listener - -exports.listen = function () { - tcp.createServer(function (socket) { - var ms = new Select() - ms.handle(socket) - ms.addHandler('/spdy/3.1.0', function (ds) { - log.info('Negotiated spdy with incoming socket') - log.info('Buffer should be clean - ', ds.read()) - var spdyConnection = spdy.connection.create(ds, { - protocol: 'spdy', - isServer: true - }) - - spdyConnection.start(3.1) - - // attach multistream handlers to incoming streams - spdyConnection.on('stream', function (spdyStream) { - registerHandles(spdyStream) - }) - - // learn about the other peer Identity - /* TODO(daviddias) - identify.inquiry(spdyConnection, function (err, spdyConnection, peerIdB58) { - if (err) { - return log.error(err) - } - if (connections[peerIdB58]) { - return log.error('New connection established with a peer(' + peerIdB58 + ') that we already had a connection') - } - spdyConnection.peerId = peerIdB58 - connections[peerIdB58] = spdyConnection - }) - */ - - // close the connection when all the streams close - spdyConnection.on('close', function () { - delete connections[spdyConnection.peerId] - }) - }) - }).listen(process.env.IPFS_PORT || 4001) -} - -// interface - -exports.openStream = function (peer, protocol, cb) { - // if Connection already open, open a new stream, otherwise, create a new Connection - // then negoatite the protocol and return the opened stream - - // If no connection open yet, open it - if (!connections[peer.id.toB58String()]) { - // Establish a socket with one of the addresses - var gotOne = false - async.eachSeries(peer.multiaddrs, function (multiaddr, callback) { - if (gotOne) { - return callback() - } - var socket = tcp.connect(multiaddr.toOptions(), function connected () { - gotSocket(socket) - }) - - socket.once('error', function (err) { - log.warn('Could not connect using one of the address of peer - ', peer.id.toB58String(), err) - callback() - }) - - }, function done () { - if (!gotOne) { - cb(new Error('Not able to open a scoket with peer - ', peer.id.toB58String())) - } - }) - - } else { - createStream(peer, protocol, cb) - } - - // do the spdy people dance (multistream-select into spdy) - function gotSocket (socket) { - gotOne = true - var msi = new Interactive() - msi.handle(socket, function () { - msi.select('/spdy/3.1.0', function (err, ds) { - var spdyConnection = spdy.connection.create(ds, { - protocol: 'spdy', - isServer: false - }) - spdyConnection.start(3.1) - connections[peer.id.toB58String()] = spdyConnection - - // attach multistream handlers to incoming streams - spdyConnection.on('stream', function (spdyStream) { - registerHandles(spdyStream) - }) - - createStream(peer, protocol, cb) - }) - }) - } - - function createStream (peer, protocol, cb) { - // 1. to pop a new stream on the connection - // 2. negotiate the requested protocol through multistream - // 3. return back the stream when that is negotiated - var conn = connections[peer.id.toB58String()] - conn.request({path: '/', method: 'GET'}, function (err, stream) { - if (err) { - return cb(err) - } - var msi = new Interactive() - msi.handle(stream, function () { - msi.select(protocol, function (err, ds) { - if (err) { - return cb(err) - } - cb(null, ds) // wohoo we finally delivered the stream the user wanted - }) - }) - }) - - conn.on('close', function () { - // TODO(daviddias) remove it from collections - }) - - } - -} - -exports.registerHandle = function (protocol, cb) { - if (handles[protocol]) { - var err = new Error('Handle for protocol already exists', protocol) - log.error(err) - return cb(err) - } - handles.push({ protocol: protocol, func: cb }) - log.info('Registered handler for protocol:', protocol) -} - -function registerHandles (spdyStream) { - log.info('Preparing stream to handle the registered protocols') - var msH = new Select() - msH.handle(spdyStream) - handles.forEach(function (handle) { - msH.addHandler(handle.protocol, handle.func) - }) -} +exports = module.exports = Swarm +exports.singleton = new Swarm() diff --git a/src/swarm.js b/src/swarm.js new file mode 100644 index 0000000000..98ae85a33c --- /dev/null +++ b/src/swarm.js @@ -0,0 +1,178 @@ +var tcp = require('net') +var Select = require('multistream-select').Select +var Interactive = require('multistream-select').Interactive +var spdy = require('spdy-transport') +// var identify = require('./identify') +var log = require('ipfs-logger').group('swarm') +var async = require('async') + +exports = module.exports = Swarm + +function Swarm () { + var self = this + + if (!(self instanceof Swarm)) { + throw new Error('Swarm must be called with new') + } + + self.port = parseInt(process.env.IPFS_SWARM_PORT, 10) || 4001 + self.connections = {} + self.handles = [] + + // set the listener + + self.listen = function () { + console.log('GOING TO LISTEN ON: ', self.port) + tcp.createServer(function (socket) { + console.log('GOT INCOMING CONNECTION') + + var ms = new Select() + ms.handle(socket) + ms.addHandler('/spdy/3.1.0', function (ds) { + console.log('GOT SPDY HANDLER REQUEST') + log.info('Negotiated spdy with incoming socket') + log.info('Buffer should be clean - ', ds.read()) + var spdyConnection = spdy.connection.create(ds, { + protocol: 'spdy', + isServer: true + }) + + spdyConnection.start(3.1) + + // attach multistream handlers to incoming streams + spdyConnection.on('stream', function (spdyStream) { + registerHandles(spdyStream) + }) + + // learn about the other peer Identity + /* TODO(daviddias) + identify.inquiry(spdyConnection, function (err, spdyConnection, peerIdB58) { + if (err) { + return log.error(err) + } + if (self.connections[peerIdB58]) { + return log.error('New connection established with a peer(' + peerIdB58 + ') that we already had a connection') + } + spdyConnection.peerId = peerIdB58 + self.connections[peerIdB58] = spdyConnection + }) + */ + + // close the connection when all the streams close + spdyConnection.on('close', function () { + delete self.connections[spdyConnection.peerId] + }) + }) + }).listen(self.port) + } + + // interface + + self.openStream = function (peer, protocol, cb) { + // if Connection already open, open a new stream, otherwise, create a new Connection + // then negoatite the protocol and return the opened stream + + // If no connection open yet, open it + if (!self.connections[peer.id.toB58String()]) { + // Establish a socket with one of the addresses + var gotOne = false + async.eachSeries(peer.multiaddrs, function (multiaddr, callback) { + if (gotOne) { + return callback() + } + var socket = tcp.connect(multiaddr.toOptions(), function connected () { + console.log('CONNECTED TO: ', multiaddr.toString()) + gotSocket(socket) + }) + + socket.once('error', function (err) { + log.warn('Could not connect using one of the address of peer - ', peer.id.toB58String(), err) + callback() + }) + + }, function done () { + if (!gotOne) { + cb(new Error('Not able to open a scoket with peer - ', peer.id.toB58String())) + } + }) + + } else { + createStream(peer, protocol, cb) + } + + // do the spdy people dance (multistream-select into spdy) + function gotSocket (socket) { + console.log('GOT SOCKET') + gotOne = true + var msi = new Interactive() + msi.handle(socket, function () { + console.log('GOING TO NEGOTIATE SPDY') + msi.select('/spdy/3.1.0', function (err, ds) { + if (err) { + return console.log('err', err) + } + var spdyConnection = spdy.connection.create(ds, { + protocol: 'spdy', + isServer: false + }) + spdyConnection.start(3.1) + self.connections[peer.id.toB58String()] = spdyConnection + + // attach multistream handlers to incoming streams + spdyConnection.on('stream', function (spdyStream) { + registerHandles(spdyStream) + }) + + createStream(peer, protocol, cb) + }) + }) + } + + function createStream (peer, protocol, cb) { + // 1. to pop a new stream on the connection + // 2. negotiate the requested protocol through multistream + // 3. return back the stream when that is negotiated + var conn = self.connections[peer.id.toB58String()] + conn.request({path: '/', method: 'GET'}, function (err, stream) { + if (err) { + return cb(err) + } + var msi = new Interactive() + msi.handle(stream, function () { + msi.select(protocol, function (err, ds) { + if (err) { + return cb(err) + } + cb(null, ds) // wohoo we finally delivered the stream the user wanted + }) + }) + }) + + conn.on('close', function () { + // TODO(daviddias) remove it from collections + }) + + } + + } + + self.registerHandle = function (protocol, cb) { + if (self.handles[protocol]) { + var err = new Error('Handle for protocol already exists', protocol) + log.error(err) + return cb(err) + } + self.handles.push({ protocol: protocol, func: cb }) + log.info('Registered handler for protocol:', protocol) + } + + function registerHandles (spdyStream) { + log.info('Preparing stream to handle the registered protocols') + var msH = new Select() + msH.handle(spdyStream) + self.handles.forEach(function (handle) { + msH.addHandler(handle.protocol, handle.func) + }) + } + +} diff --git a/tests/swarm-test.js b/tests/swarm-test.js index e69de29bb2..57ecb3572a 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -0,0 +1,71 @@ +var Lab = require('lab') +var Code = require('code') +var lab = exports.lab = Lab.script() + +var experiment = lab.experiment +var test = lab.test +var beforeEach = lab.beforeEach +var afterEach = lab.afterEach +var expect = Code.expect + +var multiaddr = require('multiaddr') +var Id = require('ipfs-peer-id') +var Peer = require('ipfs-peer') +var Swarm = require('../src/index.js') + +experiment(': ', function () { + var a + var b + var peerA + var peerB + + beforeEach(function (done) { + a = new Swarm() + a.port = 4000 + a.listen() + peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + a.port)]) + + b = new Swarm() + b.port = 4001 + b.listen() + peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + b.port)]) + + setTimeout(done, 1000) + }) + + afterEach(function (done) { + // a.close() + // b.close() + done() + }) + + test('Open a stream', {timeout: false}, function (done) { + var protocol = '/sparkles/3.3.3' + var c = new Counter(2, done) + + b.registerHandle(protocol, function (stream) { + console.log('bim') + c.unamas() + }) + + a.openStream(peerB, protocol, function (err, stream) { + console.log('pim') + expect(err).to.not.be.instanceof(Error) + c.unamas() + }) + }) + + function Counter (target, callback) { + var c = 0 + + this.unamas = count + + function count () { + c += 1 + if (c === target) { + callback() + } + } + } + +}) From 572c7e4cfae29109962d5d505ff63cd9f0ed915f Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 9 Jul 2015 13:53:03 -0700 Subject: [PATCH 007/634] identify + test refactor and reorg --- examples/network-c.js | 24 ++++++---- examples/network-s.js | 19 ++++++-- src/identify.js | 108 +++++++++++++++++++++++++++++++----------- src/swarm.js | 67 ++++++++++++++++---------- tests/swarm-test.js | 87 +++++++++++++++++++--------------- 5 files changed, 203 insertions(+), 102 deletions(-) diff --git a/examples/network-c.js b/examples/network-c.js index 6fcad79679..b0509eb11e 100644 --- a/examples/network-c.js +++ b/examples/network-c.js @@ -1,19 +1,23 @@ -var swarm = require('./../src').singleton +var Identify = require('./../src/identify') +var Swarm = require('./../src') var Peer = require('ipfs-peer') var Id = require('ipfs-peer-id') var multiaddr = require('multiaddr') -// create Id -// create multiaddr -// create Peer -// openStream +var a = new Swarm() +a.port = 4000 +// a.listen() +var peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + a.port)]) -var peerId = Id.create() -var mhs = [] -mhs.push(multiaddr('/ip4/127.0.0.1/tcp/4001')) -var p = new Peer(peerId, mhs) +// attention, peerB Id isn't going to match, but whateves +var peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/4001')]) -swarm.openStream(p, '/ipfs/sparkles/1.2.3', function (err, stream) { +var i = new Identify(a, peerA) +i.on('thenews', function (news) { + console.log('such news') +}) + +a.openStream(peerB, '/ipfs/sparkles/1.2.3', function (err, stream) { if (err) { return console.log('ERR - ', err) } diff --git a/examples/network-s.js b/examples/network-s.js index 666c5bfd5d..c4e507669f 100644 --- a/examples/network-s.js +++ b/examples/network-s.js @@ -1,7 +1,20 @@ -var swarm = require('./../src').singleton +var Identify = require('./../src/identify') +var Swarm = require('./../src') +var Peer = require('ipfs-peer') +var Id = require('ipfs-peer-id') +var multiaddr = require('multiaddr') -swarm.listen() +var b = new Swarm() +b.port = 4001 +var peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + b.port)]) -swarm.registerHandle('/ipfs/sparkles/1.2.3', function (stream) { +var i = new Identify(b, peerB) +i.on('thenews', function (news) { + console.log('such news') +}) + +b.listen() + +b.registerHandle('/ipfs/sparkles/1.2.3', function (stream) { console.log('woop got a stream') }) diff --git a/src/identify.js b/src/identify.js index 9695af33c4..cbcf44fa09 100644 --- a/src/identify.js +++ b/src/identify.js @@ -3,41 +3,93 @@ * pairs a specific peer is available through */ -var swarm = require('./').singleton var Interactive = require('multistream-select').Interactive +var EventEmmiter = require('events').EventEmitter +var util = require('util') -exports = module.exports +exports = module.exports = Identify -// peer acting as server, asking whom is talking -exports.inquiry = function (spdyConnection, cb) { - spdyConnection.request({method: 'GET', path: '/', headers: {}}, function (stream) { - var msi = new Interactive() - msi.handle(stream) - msi.select('/ipfs/identify/1.0.0', function (ds) { - var peerId = '' - ds.setEncoding('utf8') +util.inherits(Identify, EventEmmiter) - ds.on('data', function (chunk) { - peerId += chunk - }) - ds.on('end', function () { - cb(null, spdyConnection, peerId) - }) +function Identify (swarm, peerSelf) { + var self = this + + swarm.registerHandle('/ipfs/identify/1.0.0', function (stream) { + var identifyMsg = {} + identifyMsg = {} + identifyMsg.sender = exportPeer(peerSelf) + // TODO (daviddias) populate with the way I see the other peer + // identifyMsg.receiver = + + stream.write(JSON.stringify(identifyMsg)) + + var answer = '' + + stream.on('data', function (chunk) { + answer += chunk.toString() + }) + + stream.on('end', function () { + console.log(JSON.prse(answer)) + self.emit('thenews', answer) }) + + stream.end() + + // receive their info and how they see us + // send back our stuff }) -// 0. open a stream -// 1. negotiate /ipfs/identify/1.0.0 -// 2. check other peerId -// 3. reply back with cb(null, connection, peerId) -} -// peer asking which pairs ip:port does the other peer see -exports.whoAmI = function () {} + swarm.on('connection', function (spdyConnection) { + spdyConnection.request({ + path: '/', + method: 'GET' + }, function (err, stream) { + if (err) { + return console.log(err) + } + var msi = new Interactive() + msi.handle(stream, function () { + msi.select('/ipfs/identify/1.0.0', function (err, ds) { + if (err) { + return console.log('err') + } + + var identifyMsg = {} + identifyMsg = {} + identifyMsg.sender = exportPeer(peerSelf) + // TODO (daviddias) populate with the way I see the other peer + // identifyMsg.receiver = + + stream.write(JSON.stringify(identifyMsg)) + + var answer = '' + + stream.on('data', function (chunk) { + answer += chunk.toString() + }) -exports.start = function (peerSelf) { - swarm.registerHandle('/ipfs/identify/1.0.0', function (ds) { - ds.setDefaultEncoding('utf8') - ds.write(peerSelf.toB58String()) - ds.end() + stream.on('end', function () { + console.log(JSON.parse(answer)) + // TODO (daviddias), push to the connections list on swarm that we have a new known connection + self.emit('thenews', answer) + }) + + stream.end() + }) + }) + }) + // open a spdy stream + // do the multistream handshake + // send them our data }) + + function exportPeer (peer) { + return { + id: peer.id.toB58String(), + multiaddrs: peer.multiaddrs.map(function (mh) { + return mh.toString() + }) + } + } } diff --git a/src/swarm.js b/src/swarm.js index 98ae85a33c..448c11792d 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -2,12 +2,15 @@ var tcp = require('net') var Select = require('multistream-select').Select var Interactive = require('multistream-select').Interactive var spdy = require('spdy-transport') -// var identify = require('./identify') var log = require('ipfs-logger').group('swarm') var async = require('async') +var EventEmitter = require('events').EventEmitter +var util = require('util') exports = module.exports = Swarm +util.inherits(Swarm, EventEmitter) + function Swarm () { var self = this @@ -16,20 +19,26 @@ function Swarm () { } self.port = parseInt(process.env.IPFS_SWARM_PORT, 10) || 4001 + self.connections = {} self.handles = [] // set the listener - self.listen = function () { - console.log('GOING TO LISTEN ON: ', self.port) - tcp.createServer(function (socket) { - console.log('GOT INCOMING CONNECTION') + self.listen = function (port, ready) { + if (!ready) { + ready = function noop () {} + } + if (typeof port === 'function') { + ready = port + } else if (port) { + self.port = port + } + tcp.createServer(function (socket) { var ms = new Select() ms.handle(socket) ms.addHandler('/spdy/3.1.0', function (ds) { - console.log('GOT SPDY HANDLER REQUEST') log.info('Negotiated spdy with incoming socket') log.info('Buffer should be clean - ', ds.read()) var spdyConnection = spdy.connection.create(ds, { @@ -39,31 +48,19 @@ function Swarm () { spdyConnection.start(3.1) + self.emit('connection', spdyConnection) + // attach multistream handlers to incoming streams spdyConnection.on('stream', function (spdyStream) { registerHandles(spdyStream) }) - // learn about the other peer Identity - /* TODO(daviddias) - identify.inquiry(spdyConnection, function (err, spdyConnection, peerIdB58) { - if (err) { - return log.error(err) - } - if (self.connections[peerIdB58]) { - return log.error('New connection established with a peer(' + peerIdB58 + ') that we already had a connection') - } - spdyConnection.peerId = peerIdB58 - self.connections[peerIdB58] = spdyConnection - }) - */ - // close the connection when all the streams close spdyConnection.on('close', function () { delete self.connections[spdyConnection.peerId] }) }) - }).listen(self.port) + }).listen(self.port, ready) } // interface @@ -81,7 +78,6 @@ function Swarm () { return callback() } var socket = tcp.connect(multiaddr.toOptions(), function connected () { - console.log('CONNECTED TO: ', multiaddr.toString()) gotSocket(socket) }) @@ -102,11 +98,9 @@ function Swarm () { // do the spdy people dance (multistream-select into spdy) function gotSocket (socket) { - console.log('GOT SOCKET') gotOne = true var msi = new Interactive() msi.handle(socket, function () { - console.log('GOING TO NEGOTIATE SPDY') msi.select('/spdy/3.1.0', function (err, ds) { if (err) { return console.log('err', err) @@ -116,6 +110,7 @@ function Swarm () { isServer: false }) spdyConnection.start(3.1) + self.connections[peer.id.toB58String()] = spdyConnection // attach multistream handlers to incoming streams @@ -166,6 +161,18 @@ function Swarm () { log.info('Registered handler for protocol:', protocol) } + self.close = function (cb) { + var keys = Object.keys(self.connections) + var number = keys.length + if (number === 0) { cb() } + var c = new Counter(number, cb) + + keys.map(function (key) { + c.hit() + self.connections[key].end() + }) + } + function registerHandles (spdyStream) { log.info('Preparing stream to handle the registered protocols') var msH = new Select() @@ -176,3 +183,15 @@ function Swarm () { } } + +function Counter (target, callback) { + var c = 0 + this.hit = count + + function count () { + c += 1 + if (c === target) { + callback() + } + } +} diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 57ecb3572a..44cbe8d283 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -13,59 +13,72 @@ var Id = require('ipfs-peer-id') var Peer = require('ipfs-peer') var Swarm = require('../src/index.js') -experiment(': ', function () { - var a - var b - var peerA - var peerB - - beforeEach(function (done) { - a = new Swarm() - a.port = 4000 - a.listen() - peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + a.port)]) - - b = new Swarm() - b.port = 4001 - b.listen() - peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + b.port)]) - - setTimeout(done, 1000) +var swarmA +var swarmB +var peerA +var peerB + +beforeEach(function (done) { + swarmA = new Swarm() + swarmB = new Swarm() + var c = new Counter(2, done) + + swarmA.listen(4000, function () { + peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmA.port)]) + c.hit() + }) + + swarmB.listen(4001, function () { + peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmB.port)]) + c.hit() }) +}) - afterEach(function (done) { - // a.close() - // b.close() - done() +afterEach(function (done) { + var c = new Counter(2, done) + swarmA.close(function () { + c.hit() + }) + swarmB.close(function () { + c.hit() }) +}) + +experiment('BASE', function () { test('Open a stream', {timeout: false}, function (done) { var protocol = '/sparkles/3.3.3' var c = new Counter(2, done) - b.registerHandle(protocol, function (stream) { - console.log('bim') - c.unamas() + swarmB.registerHandle(protocol, function (stream) { + c.hit() }) - a.openStream(peerB, protocol, function (err, stream) { - console.log('pim') + swarmA.openStream(peerB, protocol, function (err, stream) { expect(err).to.not.be.instanceof(Error) - c.unamas() + c.hit() }) }) +}) + +experiment('IDENTIFY', function () { + +}) - function Counter (target, callback) { - var c = 0 +experiment('HARDNESS', function () { - this.unamas = count +}) + +function Counter (target, callback) { + var c = 0 + this.hit = count - function count () { - c += 1 - if (c === target) { - callback() - } + function count () { + c += 1 + if (c === target) { + callback() } } +} + -}) From 6c82973315616f50f1563f662ddb902365cc92a9 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 9 Jul 2015 13:53:09 -0700 Subject: [PATCH 008/634] identify + test refactor and reorg --- src/identify.js | 10 +++++----- tests/swarm-test.js | 11 ++--------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/identify.js b/src/identify.js index cbcf44fa09..e1ba95bae7 100644 --- a/src/identify.js +++ b/src/identify.js @@ -36,8 +36,8 @@ function Identify (swarm, peerSelf) { stream.end() - // receive their info and how they see us - // send back our stuff + // receive their info and how they see us + // send back our stuff }) swarm.on('connection', function (spdyConnection) { @@ -79,9 +79,9 @@ function Identify (swarm, peerSelf) { }) }) }) - // open a spdy stream - // do the multistream handshake - // send them our data + // open a spdy stream + // do the multistream handshake + // send them our data }) function exportPeer (peer) { diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 44cbe8d283..807110912c 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -45,7 +45,6 @@ afterEach(function (done) { }) experiment('BASE', function () { - test('Open a stream', {timeout: false}, function (done) { var protocol = '/sparkles/3.3.3' var c = new Counter(2, done) @@ -61,13 +60,9 @@ experiment('BASE', function () { }) }) -experiment('IDENTIFY', function () { - -}) +experiment('IDENTIFY', function () {}) -experiment('HARDNESS', function () { - -}) +experiment('HARDNESS', function () {}) function Counter (target, callback) { var c = 0 @@ -80,5 +75,3 @@ function Counter (target, callback) { } } } - - From 14d11de2011d7c8a09cbb069193bf4e848601f94 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 9 Jul 2015 15:45:03 -0700 Subject: [PATCH 009/634] clean the code a bit --- README.md | 6 +++ src/identify.js | 2 +- src/swarm.js | 103 ++++++++++++++++++------------------------------ 3 files changed, 46 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 195bfc1e75..7647bd4361 100644 --- a/README.md +++ b/README.md @@ -7,4 +7,10 @@ ipfs-swarm Node.js implementation # Description +ipfs-swarm is an abstraction for the network layer on IPFS. It offers an API to open streams between peers on a specific protocol. + +Ref link (still a WiP) - https://github.com/diasdavid/specs/blob/protocol-spec/protocol/layers.md#network-layer + # Usage + + diff --git a/src/identify.js b/src/identify.js index e1ba95bae7..12757b46b9 100644 --- a/src/identify.js +++ b/src/identify.js @@ -30,7 +30,7 @@ function Identify (swarm, peerSelf) { }) stream.on('end', function () { - console.log(JSON.prse(answer)) + console.log(JSON.parse(answer)) self.emit('thenews', answer) }) diff --git a/src/swarm.js b/src/swarm.js index 448c11792d..e6a7ce706d 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -40,83 +40,72 @@ function Swarm () { ms.handle(socket) ms.addHandler('/spdy/3.1.0', function (ds) { log.info('Negotiated spdy with incoming socket') - log.info('Buffer should be clean - ', ds.read()) - var spdyConnection = spdy.connection.create(ds, { + + var conn = spdy.connection.create(ds, { protocol: 'spdy', isServer: true }) - spdyConnection.start(3.1) + conn.start(3.1) - self.emit('connection', spdyConnection) + self.emit('connection', conn) // attach multistream handlers to incoming streams - spdyConnection.on('stream', function (spdyStream) { - registerHandles(spdyStream) - }) + conn.on('stream', registerHandles) - // close the connection when all the streams close - spdyConnection.on('close', function () { - delete self.connections[spdyConnection.peerId] - }) + // IDENTIFY DOES THAT FOR US + // conn.on('close', function () { delete self.connections[conn.peerId] }) }) }).listen(self.port, ready) } // interface + // open stream account for connection reuse self.openStream = function (peer, protocol, cb) { - // if Connection already open, open a new stream, otherwise, create a new Connection - // then negoatite the protocol and return the opened stream // If no connection open yet, open it if (!self.connections[peer.id.toB58String()]) { + // Establish a socket with one of the addresses - var gotOne = false - async.eachSeries(peer.multiaddrs, function (multiaddr, callback) { - if (gotOne) { - return callback() - } - var socket = tcp.connect(multiaddr.toOptions(), function connected () { - gotSocket(socket) + var socket + async.eachSeries(peer.multiaddrs, function (multiaddr, next) { + if (socket) { return next() } + + var tmp = tcp.connect(multiaddr.toOptions(), function () { + socket = tmp + next() }) - socket.once('error', function (err) { - log.warn('Could not connect using one of the address of peer - ', peer.id.toB58String(), err) - callback() + tmp.once('error', function (err) { + log.warn(multiaddr.toString(), 'on', peer.id.toB58String(), 'not available', err) + next() }) }, function done () { - if (!gotOne) { - cb(new Error('Not able to open a scoket with peer - ', peer.id.toB58String())) + if (!socket) { + return cb(new Error('Not able to open a scoket with peer - ', + peer.id.toB58String())) } + gotSocket(socket) }) - } else { createStream(peer, protocol, cb) } // do the spdy people dance (multistream-select into spdy) function gotSocket (socket) { - gotOne = true var msi = new Interactive() msi.handle(socket, function () { msi.select('/spdy/3.1.0', function (err, ds) { - if (err) { - return console.log('err', err) - } - var spdyConnection = spdy.connection.create(ds, { - protocol: 'spdy', - isServer: false - }) - spdyConnection.start(3.1) + if (err) { cb(err) } - self.connections[peer.id.toB58String()] = spdyConnection + var conn = spdy.connection.create(ds, { protocol: 'spdy', isServer: false }) + conn.start(3.1) + conn.on('stream', registerHandles) + self.connections[peer.id.toB58String()] = conn - // attach multistream handlers to incoming streams - spdyConnection.on('stream', function (spdyStream) { - registerHandles(spdyStream) - }) + conn.on('close', function () { delete self.connections[peer.id.toB58String()] }) createStream(peer, protocol, cb) }) @@ -124,40 +113,28 @@ function Swarm () { } function createStream (peer, protocol, cb) { - // 1. to pop a new stream on the connection - // 2. negotiate the requested protocol through multistream - // 3. return back the stream when that is negotiated + // spawn new stream var conn = self.connections[peer.id.toB58String()] conn.request({path: '/', method: 'GET'}, function (err, stream) { - if (err) { - return cb(err) - } + if (err) { return cb(err) } + + // negotiate desired protocol var msi = new Interactive() msi.handle(stream, function () { msi.select(protocol, function (err, ds) { - if (err) { - return cb(err) - } - cb(null, ds) // wohoo we finally delivered the stream the user wanted + if (err) { return cb(err) } + cb(null, ds) // return the stream }) }) }) - - conn.on('close', function () { - // TODO(daviddias) remove it from collections - }) - } - } - self.registerHandle = function (protocol, cb) { + self.registerHandle = function (protocol, handleFunc) { if (self.handles[protocol]) { - var err = new Error('Handle for protocol already exists', protocol) - log.error(err) - return cb(err) + throw new Error('Handle for protocol already exists', protocol) } - self.handles.push({ protocol: protocol, func: cb }) + self.handles.push({ protocol: protocol, func: handleFunc }) log.info('Registered handler for protocol:', protocol) } @@ -190,8 +167,6 @@ function Counter (target, callback) { function count () { c += 1 - if (c === target) { - callback() - } + if (c === target) { callback() } } } From f08d407bf917602222d184f05f68e8d10cd816a5 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 9 Jul 2015 15:45:13 -0700 Subject: [PATCH 010/634] clean the code a bit --- src/swarm.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/swarm.js b/src/swarm.js index e6a7ce706d..3579b8a8bc 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -53,8 +53,8 @@ function Swarm () { // attach multistream handlers to incoming streams conn.on('stream', registerHandles) - // IDENTIFY DOES THAT FOR US - // conn.on('close', function () { delete self.connections[conn.peerId] }) + // IDENTIFY DOES THAT FOR US + // conn.on('close', function () { delete self.connections[conn.peerId] }) }) }).listen(self.port, ready) } @@ -63,10 +63,8 @@ function Swarm () { // open stream account for connection reuse self.openStream = function (peer, protocol, cb) { - // If no connection open yet, open it if (!self.connections[peer.id.toB58String()]) { - // Establish a socket with one of the addresses var socket async.eachSeries(peer.multiaddrs, function (multiaddr, next) { @@ -85,7 +83,7 @@ function Swarm () { }, function done () { if (!socket) { return cb(new Error('Not able to open a scoket with peer - ', - peer.id.toB58String())) + peer.id.toB58String())) } gotSocket(socket) }) From 0aab8ead56b8b789e7301b34b057abae8ba49591 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 9 Jul 2015 20:00:54 -0700 Subject: [PATCH 011/634] for indutny --- src/swarm.js | 21 +++++++++++++++++++-- tests/swarm-test.js | 36 ++++++++++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/swarm.js b/src/swarm.js index 3579b8a8bc..d381e33f63 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -35,7 +35,11 @@ function Swarm () { self.port = port } - tcp.createServer(function (socket) { + self.listener = tcp.createServer(function (socket) { + socket.on('error', function (err) { + // self.emit('error', err) + }) + socket.on('close', function () {}) var ms = new Select() ms.handle(socket) ms.addHandler('/spdy/3.1.0', function (ds) { @@ -52,11 +56,18 @@ function Swarm () { // attach multistream handlers to incoming streams conn.on('stream', registerHandles) + conn.on('error', function (err) { + // self.emit('error', err) + }) + conn.on('close', function () { + }) // IDENTIFY DOES THAT FOR US // conn.on('close', function () { delete self.connections[conn.peerId] }) }) }).listen(self.port, ready) + self.listener.on('error', function (err) { self.emit('error', err) }) + } // interface @@ -72,6 +83,7 @@ function Swarm () { var tmp = tcp.connect(multiaddr.toOptions(), function () { socket = tmp + socket.on('error', function (err) { self.emit('error', err) }) next() }) @@ -104,6 +116,7 @@ function Swarm () { self.connections[peer.id.toB58String()] = conn conn.on('close', function () { delete self.connections[peer.id.toB58String()] }) + conn.on('error', function (err) { self.emit('error', err) }) createStream(peer, protocol, cb) }) @@ -136,7 +149,7 @@ function Swarm () { log.info('Registered handler for protocol:', protocol) } - self.close = function (cb) { + self.closeConns = function (cb) { var keys = Object.keys(self.connections) var number = keys.length if (number === 0) { cb() } @@ -148,6 +161,10 @@ function Swarm () { }) } + self.closeListener = function (cb) { + self.listener.close(cb) + } + function registerHandles (spdyStream) { log.info('Preparing stream to handle the registered protocols') var msH = new Select() diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 807110912c..66cbf655d6 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -23,24 +23,34 @@ beforeEach(function (done) { swarmB = new Swarm() var c = new Counter(2, done) - swarmA.listen(4000, function () { + swarmA.listen(8100, function () { peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmA.port)]) c.hit() }) - swarmB.listen(4001, function () { + swarmB.listen(8101, function () { peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmB.port)]) c.hit() }) }) -afterEach(function (done) { - var c = new Counter(2, done) - swarmA.close(function () { +afterEach({ timeout: 5000 }, function (done) { + var c = new Counter(4, done) + swarmA.closeConns(function () { c.hit() + swarmA.closeListener(function () { + console.log('AAA CLOSE') + c.hit() + }) }) - swarmB.close(function () { + + swarmB.closeConns(function () { + console.log('bb') c.hit() + swarmB.closeListener(function () { + console.log('BBB CLOSE') + c.hit() + }) }) }) @@ -58,6 +68,20 @@ experiment('BASE', function () { c.hit() }) }) + + test('Reuse stream (from dialer)', {timeout: false}, function (done) { + var protocol = '/sparkles/3.3.3' + var c = new Counter(2, done) + + swarmB.registerHandle(protocol, function (stream) { + c.hit() + }) + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + c.hit() + }) + }) }) experiment('IDENTIFY', function () {}) From 44de1054285f14b25182b0637e620e1bb7e8947f Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 9 Jul 2015 20:00:59 -0700 Subject: [PATCH 012/634] for indutny --- src/swarm.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/swarm.js b/src/swarm.js index d381e33f63..ccc65f8873 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -59,8 +59,7 @@ function Swarm () { conn.on('error', function (err) { // self.emit('error', err) }) - conn.on('close', function () { - }) + conn.on('close', function () {}) // IDENTIFY DOES THAT FOR US // conn.on('close', function () { delete self.connections[conn.peerId] }) From a80a5fc32bd769c8820d6303e18f4e52f636e160 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 9 Jul 2015 23:45:58 -0700 Subject: [PATCH 013/634] fixed Uncaught Error exception bug (node: every stream gets the socket error propagated) --- examples/network-s.js | 4 ++++ src/swarm.js | 26 ++++++++++++-------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/examples/network-s.js b/examples/network-s.js index c4e507669f..dfc64adf50 100644 --- a/examples/network-s.js +++ b/examples/network-s.js @@ -13,6 +13,10 @@ i.on('thenews', function (news) { console.log('such news') }) +b.on('error', function (err) { + if (err) return +}) + b.listen() b.registerHandle('/ipfs/sparkles/1.2.3', function (stream) { diff --git a/src/swarm.js b/src/swarm.js index ccc65f8873..5c8474f785 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -36,10 +36,7 @@ function Swarm () { } self.listener = tcp.createServer(function (socket) { - socket.on('error', function (err) { - // self.emit('error', err) - }) - socket.on('close', function () {}) + socket.on('error', errorEmit) var ms = new Select() ms.handle(socket) ms.addHandler('/spdy/3.1.0', function (ds) { @@ -56,16 +53,13 @@ function Swarm () { // attach multistream handlers to incoming streams conn.on('stream', registerHandles) - conn.on('error', function (err) { - // self.emit('error', err) - }) - conn.on('close', function () {}) + conn.on('error', errorEmit) // IDENTIFY DOES THAT FOR US // conn.on('close', function () { delete self.connections[conn.peerId] }) }) }).listen(self.port, ready) - self.listener.on('error', function (err) { self.emit('error', err) }) + self.listener.on('error', errorEmit) } @@ -82,7 +76,7 @@ function Swarm () { var tmp = tcp.connect(multiaddr.toOptions(), function () { socket = tmp - socket.on('error', function (err) { self.emit('error', err) }) + socket.on('error', errorEmit) next() }) @@ -115,7 +109,7 @@ function Swarm () { self.connections[peer.id.toB58String()] = conn conn.on('close', function () { delete self.connections[peer.id.toB58String()] }) - conn.on('error', function (err) { self.emit('error', err) }) + conn.on('error', errorEmit) createStream(peer, protocol, cb) }) @@ -127,6 +121,7 @@ function Swarm () { var conn = self.connections[peer.id.toB58String()] conn.request({path: '/', method: 'GET'}, function (err, stream) { if (err) { return cb(err) } + stream.on('error', errorEmit) // negotiate desired protocol var msi = new Interactive() @@ -164,15 +159,18 @@ function Swarm () { self.listener.close(cb) } - function registerHandles (spdyStream) { - log.info('Preparing stream to handle the registered protocols') + function registerHandles (stream) { + log.info('Registering protocol handlers on new stream') + stream.on('error', errorEmit) var msH = new Select() - msH.handle(spdyStream) + msH.handle(stream) self.handles.forEach(function (handle) { msH.addHandler(handle.protocol, handle.func) }) } + function errorEmit (err) { self.emit('error', err) } + } function Counter (target, callback) { From a5b2524873b62e9da9947e5317b1394d96c7434e Mon Sep 17 00:00:00 2001 From: David Dias Date: Fri, 10 Jul 2015 12:28:40 -0700 Subject: [PATCH 014/634] connection reuse test --- README.md | 13 ++++++++++ examples/{network-c.js => c.js} | 0 examples/{network-s.js => s.js} | 4 +-- src/swarm.js | 40 +++++++++++++--------------- tests/swarm-test.js | 46 +++++++++++++++------------------ 5 files changed, 54 insertions(+), 49 deletions(-) rename examples/{network-c.js => c.js} (100%) rename examples/{network-s.js => s.js} (84%) diff --git a/README.md b/README.md index 7647bd4361..088a297999 100644 --- a/README.md +++ b/README.md @@ -13,4 +13,17 @@ Ref link (still a WiP) - https://github.com/diasdavid/specs/blob/protocol-spec/p # Usage +## API calls +.openStream +.registerHandle + +## Events emmited + +.on('error') + +.on('connection') +.on('connection-unknown') + +.on('stream') +.on('stream-unknown') diff --git a/examples/network-c.js b/examples/c.js similarity index 100% rename from examples/network-c.js rename to examples/c.js diff --git a/examples/network-s.js b/examples/s.js similarity index 84% rename from examples/network-s.js rename to examples/s.js index dfc64adf50..774f04fe97 100644 --- a/examples/network-s.js +++ b/examples/s.js @@ -14,11 +14,11 @@ i.on('thenews', function (news) { }) b.on('error', function (err) { - if (err) return + console.log(err) }) b.listen() -b.registerHandle('/ipfs/sparkles/1.2.3', function (stream) { +b.registerHandle('/ipfs/sparkles/1.2.3', function (err, stream) { console.log('woop got a stream') }) diff --git a/src/swarm.js b/src/swarm.js index 5c8474f785..ff0b1fee2a 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -26,26 +26,19 @@ function Swarm () { // set the listener self.listen = function (port, ready) { - if (!ready) { - ready = function noop () {} - } + if (!ready) { ready = function noop () {} } if (typeof port === 'function') { ready = port - } else if (port) { - self.port = port - } + } else if (port) { self.port = port } self.listener = tcp.createServer(function (socket) { - socket.on('error', errorEmit) + errorUp(self, socket) var ms = new Select() ms.handle(socket) ms.addHandler('/spdy/3.1.0', function (ds) { log.info('Negotiated spdy with incoming socket') - var conn = spdy.connection.create(ds, { - protocol: 'spdy', - isServer: true - }) + var conn = spdy.connection.create(ds, { protocol: 'spdy', isServer: true }) conn.start(3.1) @@ -53,14 +46,13 @@ function Swarm () { // attach multistream handlers to incoming streams conn.on('stream', registerHandles) - conn.on('error', errorEmit) + errorUp(self, conn) // IDENTIFY DOES THAT FOR US // conn.on('close', function () { delete self.connections[conn.peerId] }) }) }).listen(self.port, ready) - self.listener.on('error', errorEmit) - + errorUp(self, self.listener) } // interface @@ -76,7 +68,7 @@ function Swarm () { var tmp = tcp.connect(multiaddr.toOptions(), function () { socket = tmp - socket.on('error', errorEmit) + errorUp(self, socket) next() }) @@ -109,7 +101,7 @@ function Swarm () { self.connections[peer.id.toB58String()] = conn conn.on('close', function () { delete self.connections[peer.id.toB58String()] }) - conn.on('error', errorEmit) + errorUp(self, conn) createStream(peer, protocol, cb) }) @@ -121,7 +113,7 @@ function Swarm () { var conn = self.connections[peer.id.toB58String()] conn.request({path: '/', method: 'GET'}, function (err, stream) { if (err) { return cb(err) } - stream.on('error', errorEmit) + errorUp(self, stream) // negotiate desired protocol var msi = new Interactive() @@ -135,11 +127,11 @@ function Swarm () { } } - self.registerHandle = function (protocol, handleFunc) { + self.registerHandler = function (protocol, handlerFunc) { if (self.handles[protocol]) { - throw new Error('Handle for protocol already exists', protocol) + return handlerFunc(new Error('Handle for protocol already exists', protocol)) } - self.handles.push({ protocol: protocol, func: handleFunc }) + self.handles.push({ protocol: protocol, func: handlerFunc }) log.info('Registered handler for protocol:', protocol) } @@ -161,7 +153,7 @@ function Swarm () { function registerHandles (stream) { log.info('Registering protocol handlers on new stream') - stream.on('error', errorEmit) + errorUp(self, stream) var msH = new Select() msH.handle(stream) self.handles.forEach(function (handle) { @@ -169,8 +161,12 @@ function Swarm () { }) } - function errorEmit (err) { self.emit('error', err) } +} +function errorUp (self, emitter) { + emitter.on('error', function (err) { + self.emit('error', err) + }) } function Counter (target, callback) { diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 66cbf655d6..5e5d22bdd4 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -32,34 +32,21 @@ beforeEach(function (done) { peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmB.port)]) c.hit() }) -}) -afterEach({ timeout: 5000 }, function (done) { - var c = new Counter(4, done) - swarmA.closeConns(function () { - c.hit() - swarmA.closeListener(function () { - console.log('AAA CLOSE') - c.hit() - }) - }) +}) - swarmB.closeConns(function () { - console.log('bb') - c.hit() - swarmB.closeListener(function () { - console.log('BBB CLOSE') - c.hit() - }) - }) +afterEach(function (done) { + swarmA.closeListener() + swarmB.closeListener() + done() }) experiment('BASE', function () { - test('Open a stream', {timeout: false}, function (done) { + test('Open a stream', function (done) { var protocol = '/sparkles/3.3.3' var c = new Counter(2, done) - swarmB.registerHandle(protocol, function (stream) { + swarmB.registerHandler(protocol, function (stream) { c.hit() }) @@ -69,17 +56,21 @@ experiment('BASE', function () { }) }) - test('Reuse stream (from dialer)', {timeout: false}, function (done) { + test('Reuse connection (from dialer)', function (done) { var protocol = '/sparkles/3.3.3' - var c = new Counter(2, done) - swarmB.registerHandle(protocol, function (stream) { - c.hit() + swarmB.registerHandler(protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) }) swarmA.openStream(peerB, protocol, function (err, stream) { expect(err).to.not.be.instanceof(Error) - c.hit() + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + expect(swarmA.connections.length === 1) + done() + }) }) }) }) @@ -99,3 +90,8 @@ function Counter (target, callback) { } } } + +function checkErr (err) { + console.log('err') + expect(err).to.be.instanceof(Error) +} From 0576b18e167830f46dd4adb7f3cce9450ffc0209 Mon Sep 17 00:00:00 2001 From: David Dias Date: Fri, 10 Jul 2015 12:31:20 -0700 Subject: [PATCH 015/634] remove .jshintrc and jshintignore added by precommit-hook, we have standard after all :) --- .jshintignore | 1 - .jshintrc | 10 ---------- 2 files changed, 11 deletions(-) delete mode 100644 .jshintignore delete mode 100644 .jshintrc diff --git a/.jshintignore b/.jshintignore deleted file mode 100644 index 3c3629e647..0000000000 --- a/.jshintignore +++ /dev/null @@ -1 +0,0 @@ -node_modules diff --git a/.jshintrc b/.jshintrc deleted file mode 100644 index 997b3f7d45..0000000000 --- a/.jshintrc +++ /dev/null @@ -1,10 +0,0 @@ -{ - "node": true, - - "curly": true, - "latedef": true, - "quotmark": true, - "undef": true, - "unused": true, - "trailing": true -} From b4cc1d58528e4436087f820ae9f83a1a8919ada9 Mon Sep 17 00:00:00 2001 From: David Dias Date: Fri, 10 Jul 2015 14:06:51 -0700 Subject: [PATCH 016/634] connection reuse with identify --- src/identify.js | 22 +++++++++--------- src/swarm.js | 2 +- tests/swarm-test.js | 55 ++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/src/identify.js b/src/identify.js index 12757b46b9..dadcc9eb10 100644 --- a/src/identify.js +++ b/src/identify.js @@ -14,7 +14,7 @@ util.inherits(Identify, EventEmmiter) function Identify (swarm, peerSelf) { var self = this - swarm.registerHandle('/ipfs/identify/1.0.0', function (stream) { + swarm.registerHandler('/ipfs/identify/1.0.0', function (stream) { var identifyMsg = {} identifyMsg = {} identifyMsg.sender = exportPeer(peerSelf) @@ -30,8 +30,7 @@ function Identify (swarm, peerSelf) { }) stream.on('end', function () { - console.log(JSON.parse(answer)) - self.emit('thenews', answer) + self.emit('peer-update', answer) }) stream.end() @@ -40,7 +39,7 @@ function Identify (swarm, peerSelf) { // send back our stuff }) - swarm.on('connection', function (spdyConnection) { + swarm.on('connection-unknown', function (spdyConnection) { spdyConnection.request({ path: '/', method: 'GET' @@ -51,28 +50,27 @@ function Identify (swarm, peerSelf) { var msi = new Interactive() msi.handle(stream, function () { msi.select('/ipfs/identify/1.0.0', function (err, ds) { - if (err) { - return console.log('err') - } + if (err) { return console.log('err') } var identifyMsg = {} identifyMsg = {} identifyMsg.sender = exportPeer(peerSelf) // TODO (daviddias) populate with the way I see the other peer - // identifyMsg.receiver = stream.write(JSON.stringify(identifyMsg)) var answer = '' stream.on('data', function (chunk) { - answer += chunk.toString() + answer = answer + chunk.toString() }) stream.on('end', function () { - console.log(JSON.parse(answer)) - // TODO (daviddias), push to the connections list on swarm that we have a new known connection - self.emit('thenews', answer) + answer = JSON.parse(answer) + + swarm.connections[answer.sender.id] = spdyConnection + + self.emit('peer-update', answer) }) stream.end() diff --git a/src/swarm.js b/src/swarm.js index ff0b1fee2a..1053c37845 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -42,7 +42,7 @@ function Swarm () { conn.start(3.1) - self.emit('connection', conn) + self.emit('connection-unknown', conn) // attach multistream handlers to incoming streams conn.on('stream', registerHandles) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 5e5d22bdd4..2519a4e789 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -11,7 +11,8 @@ var expect = Code.expect var multiaddr = require('multiaddr') var Id = require('ipfs-peer-id') var Peer = require('ipfs-peer') -var Swarm = require('../src/index.js') +var Swarm = require('../src/') +var Identify = require('../src/identify') var swarmA var swarmB @@ -75,7 +76,49 @@ experiment('BASE', function () { }) }) -experiment('IDENTIFY', function () {}) +experiment('IDENTIFY', function () { + test('Attach Identify, open a stream, see a peer update', function (done) { + var protocol = '/sparkles/3.3.3' + + var identifyA = new Identify(swarmA, peerA) + var identifyB = new Identify(swarmB, peerB) + + swarmB.registerHandler(protocol, function (stream) {}) + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + }) + + identifyB.on('peer-update', function (answer) { + done() + }) + identifyA.on('peer-update', function (answer) {}) + }) + + test('Attach Identify, open a stream, reuse stream', function (done) { + var protocol = '/sparkles/3.3.3' + + var identifyA = new Identify(swarmA, peerA) + var identifyB = new Identify(swarmB, peerB) + + swarmA.registerHandler(protocol, function (stream) {}) + swarmB.registerHandler(protocol, function (stream) {}) + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + }) + + identifyB.on('peer-update', function (answer) { + expect(Object.keys(swarmB.connections).length).to.equal(1) + swarmB.openStream(peerA, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + expect(Object.keys(swarmB.connections).length).to.equal(1) + done() + }) + }) + identifyA.on('peer-update', function (answer) {}) + }) +}) experiment('HARDNESS', function () {}) @@ -91,7 +134,7 @@ function Counter (target, callback) { } } -function checkErr (err) { - console.log('err') - expect(err).to.be.instanceof(Error) -} +// function checkErr (err) { +// console.log('err') +// expect(err).to.be.instanceof(Error) +// } From bd6b61da69cb88d08fa55e4ccb5ec605aef0a6fc Mon Sep 17 00:00:00 2001 From: David Dias Date: Fri, 10 Jul 2015 16:01:30 -0700 Subject: [PATCH 017/634] update the badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 088a297999..285575ecd8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ipfs-swarm Node.js implementation ================================= -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)[![](https://img.shields.io/badge/IPFS-project-blue.svg?style=flat-square)](http://ipfs.io/)[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) > IPFS swarm implementation in Node.js From bca27aa9b0e7239c92bea099c625c70a309dbbbb Mon Sep 17 00:00:00 2001 From: David Dias Date: Fri, 10 Jul 2015 16:02:31 -0700 Subject: [PATCH 018/634] update the badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 285575ecd8..0814e8bfdd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ipfs-swarm Node.js implementation ================================= -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)[![](https://img.shields.io/badge/IPFS-project-blue.svg?style=flat-square)](http://ipfs.io/)[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/IPFS-project-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) > IPFS swarm implementation in Node.js From 3e5655939632e58a2a5173096c312e87acdffea6 Mon Sep 17 00:00:00 2001 From: David Dias Date: Fri, 10 Jul 2015 16:04:18 -0700 Subject: [PATCH 019/634] update the badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0814e8bfdd..f597d48771 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ipfs-swarm Node.js implementation ================================= -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/IPFS-project-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) > IPFS swarm implementation in Node.js From a2a7df870b804f999e4c100126e79d56426e8b59 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 15 Jul 2015 11:34:37 -0700 Subject: [PATCH 020/634] add multistream and muxer tests --- examples/c.js | 14 +-- examples/s.js | 24 +++-- package.json | 5 +- src/identify.js | 29 ++++-- src/stream-muxer.js | 2 + src/swarm.js | 43 ++++++--- tests/multistream-and-muxer-test.js | 139 ++++++++++++++++++++++++++++ tests/swarm-test.js | 44 +++++---- 8 files changed, 241 insertions(+), 59 deletions(-) create mode 100644 src/stream-muxer.js create mode 100644 tests/multistream-and-muxer-test.js diff --git a/examples/c.js b/examples/c.js index b0509eb11e..9bba479311 100644 --- a/examples/c.js +++ b/examples/c.js @@ -1,4 +1,4 @@ -var Identify = require('./../src/identify') +// var Identify = require('./../src/identify') var Swarm = require('./../src') var Peer = require('ipfs-peer') var Id = require('ipfs-peer-id') @@ -7,19 +7,19 @@ var multiaddr = require('multiaddr') var a = new Swarm() a.port = 4000 // a.listen() -var peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + a.port)]) +// var peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + a.port)]) // attention, peerB Id isn't going to match, but whateves var peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/4001')]) -var i = new Identify(a, peerA) -i.on('thenews', function (news) { - console.log('such news') -}) +// var i = new Identify(a, peerA) +// i.on('thenews', function (news) { +// console.log('such news') +// }) a.openStream(peerB, '/ipfs/sparkles/1.2.3', function (err, stream) { if (err) { - return console.log('ERR - ', err) + return console.log(err) } console.log('WoHoo, dialed a stream') }) diff --git a/examples/s.js b/examples/s.js index 774f04fe97..b4ce44979f 100644 --- a/examples/s.js +++ b/examples/s.js @@ -1,17 +1,17 @@ -var Identify = require('./../src/identify') +// var Identify = require('./../src/identify') var Swarm = require('./../src') -var Peer = require('ipfs-peer') -var Id = require('ipfs-peer-id') -var multiaddr = require('multiaddr') +// var Peer = require('ipfs-peer') +// var Id = require('ipfs-peer-id') +// var multiaddr = require('multiaddr') var b = new Swarm() b.port = 4001 -var peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + b.port)]) +// var peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + b.port)]) -var i = new Identify(b, peerB) -i.on('thenews', function (news) { - console.log('such news') -}) +// var i = new Identify(b, peerB) +// i.on('thenews', function (news) { +// console.log('such news') +// }) b.on('error', function (err) { console.log(err) @@ -19,6 +19,10 @@ b.on('error', function (err) { b.listen() -b.registerHandle('/ipfs/sparkles/1.2.3', function (err, stream) { +b.registerHandler('/ipfs/sparkles/1.2.3', function (stream) { +// if (err) { +// return console.log(err) +// } + console.log('woop got a stream') }) diff --git a/package.json b/package.json index 825b777075..461f8f47d1 100644 --- a/package.json +++ b/package.json @@ -30,12 +30,13 @@ "lab": "^5.13.0", "precommit-hook": "^3.0.0", "standard": "^4.5.2", - "stream-pair": "^1.0.2" + "stream-pair": "^1.0.3" }, "dependencies": { "async": "^1.3.0", "multiaddr": "^1.0.0", + "multiplex-stream-muxer": "^0.2.0", "multistream-select": "^0.6.1", - "spdy-transport": "indutny/spdy-transport" + "spdy-stream-muxer": "^0.2.0" } } diff --git a/src/identify.js b/src/identify.js index dadcc9eb10..559644b870 100644 --- a/src/identify.js +++ b/src/identify.js @@ -1,6 +1,6 @@ /* - * Identify is one of the protocols swarms speaks in order to broadcast and learn about the ip:port - * pairs a specific peer is available through + * Identify is one of the protocols swarms speaks in order to broadcast and learn + * about the ip:port pairs a specific peer is available through */ var Interactive = require('multistream-select').Interactive @@ -15,6 +15,8 @@ function Identify (swarm, peerSelf) { var self = this swarm.registerHandler('/ipfs/identify/1.0.0', function (stream) { + console.log('DO I EVER GET CALLED?') + var identifyMsg = {} identifyMsg = {} identifyMsg.sender = exportPeer(peerSelf) @@ -39,19 +41,25 @@ function Identify (swarm, peerSelf) { // send back our stuff }) - swarm.on('connection-unknown', function (spdyConnection) { - spdyConnection.request({ - path: '/', - method: 'GET' - }, function (err, stream) { + swarm.on('connection-unknown', function (conn) { + console.log('IDENTIFY - DIALING STREAM FROM SERVER') + + conn.on('error', function (err) { + console.log('CAPUT-A', err) + }) + conn.dialStream(function (err, stream) { if (err) { return console.log(err) } + stream.on('error', function (err) { + console.log('CAPUT-B', err) + }) + console.log('GOT STREAM') var msi = new Interactive() msi.handle(stream, function () { + console.log('HANDLE GOOD') msi.select('/ipfs/identify/1.0.0', function (err, ds) { - if (err) { return console.log('err') } - + if (err) { return console.log(err) } var identifyMsg = {} identifyMsg = {} identifyMsg.sender = exportPeer(peerSelf) @@ -68,8 +76,9 @@ function Identify (swarm, peerSelf) { stream.on('end', function () { answer = JSON.parse(answer) - swarm.connections[answer.sender.id] = spdyConnection + swarm.connections[answer.sender.id] = conn + console.log('BAM') self.emit('peer-update', answer) }) diff --git a/src/stream-muxer.js b/src/stream-muxer.js new file mode 100644 index 0000000000..346371fb30 --- /dev/null +++ b/src/stream-muxer.js @@ -0,0 +1,2 @@ +exports = module.exports = require('spdy-stream-muxer') +// exports = module.exports = require('multiplex-stream-muxer') diff --git a/src/swarm.js b/src/swarm.js index 1053c37845..c5278b22fd 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -1,7 +1,7 @@ var tcp = require('net') var Select = require('multistream-select').Select var Interactive = require('multistream-select').Interactive -var spdy = require('spdy-transport') +var Muxer = require('./stream-muxer') var log = require('ipfs-logger').group('swarm') var async = require('async') var EventEmitter = require('events').EventEmitter @@ -38,18 +38,25 @@ function Swarm () { ms.addHandler('/spdy/3.1.0', function (ds) { log.info('Negotiated spdy with incoming socket') - var conn = spdy.connection.create(ds, { protocol: 'spdy', isServer: true }) + var conn = new Muxer().attach(ds, false) - conn.start(3.1) + // attach multistream handlers to incoming streams - self.emit('connection-unknown', conn) + conn.on('stream', function () { + console.log('HERE') + }) + conn.on('error', function () { + console.log('error here') + }) - // attach multistream handlers to incoming streams conn.on('stream', registerHandles) errorUp(self, conn) - // IDENTIFY DOES THAT FOR US - // conn.on('close', function () { delete self.connections[conn.peerId] }) + // FOR IDENTIFY + self.emit('connection-unknown', conn) + + // IDENTIFY DOES THIS FOR US + // conn.on('close', function () { delete self.connections[conn.peerId] }) }) }).listen(self.port, ready) errorUp(self, self.listener) @@ -95,11 +102,15 @@ function Swarm () { msi.select('/spdy/3.1.0', function (err, ds) { if (err) { cb(err) } - var conn = spdy.connection.create(ds, { protocol: 'spdy', isServer: false }) - conn.start(3.1) + var conn = new Muxer().attach(ds, false) + conn.on('stream', function () { + console.log('WOOHO NEW STREAM') + }) + conn.on('error', function () { + console.log('BADUM TSS') + }) conn.on('stream', registerHandles) self.connections[peer.id.toB58String()] = conn - conn.on('close', function () { delete self.connections[peer.id.toB58String()] }) errorUp(self, conn) @@ -109,12 +120,14 @@ function Swarm () { } function createStream (peer, protocol, cb) { - // spawn new stream + // spawn new muxed stream var conn = self.connections[peer.id.toB58String()] - conn.request({path: '/', method: 'GET'}, function (err, stream) { + conn.dialStream(function (err, stream) { if (err) { return cb(err) } errorUp(self, stream) - + stream.on('error', function (err) { + console.log('error here - ', err) + }) // negotiate desired protocol var msi = new Interactive() msi.handle(stream, function () { @@ -128,7 +141,9 @@ function Swarm () { } self.registerHandler = function (protocol, handlerFunc) { + console.log('new handler coming in for - ', protocol) if (self.handles[protocol]) { + console.log('here already - ', protocol) return handlerFunc(new Error('Handle for protocol already exists', protocol)) } self.handles.push({ protocol: protocol, func: handlerFunc }) @@ -153,10 +168,12 @@ function Swarm () { function registerHandles (stream) { log.info('Registering protocol handlers on new stream') + console.log('REGISTERING HANDLES') errorUp(self, stream) var msH = new Select() msH.handle(stream) self.handles.forEach(function (handle) { + console.log(' ->', handle.protocol) msH.addHandler(handle.protocol, handle.func) }) } diff --git a/tests/multistream-and-muxer-test.js b/tests/multistream-and-muxer-test.js new file mode 100644 index 0000000000..a7e294773e --- /dev/null +++ b/tests/multistream-and-muxer-test.js @@ -0,0 +1,139 @@ +var Lab = require('lab') +var Code = require('code') +var lab = exports.lab = Lab.script() + +var experiment = lab.experiment +var test = lab.test +var beforeEach = lab.beforeEach +var afterEach = lab.afterEach +var expect = Code.expect + +var Muxer = require('./../src/stream-muxer.js') +var multistream = require('multistream-select') +var Interactive = multistream.Interactive +var Select = multistream.Select +var streamPair = require('stream-pair') + +beforeEach(function (done) { + done() +}) + +afterEach(function (done) { + done() +}) + +experiment('MULTISTREAM AND STREAM MUXER', function () { + test('Open a socket and multistream-select it into spdy', function (done) { + var pair = streamPair.create() + + var msI = new Interactive() + var msS = new Select() + + var dialerMuxer = new Muxer() + var listenerMuxer = new Muxer() + + msS.handle(pair.other) + + msS.addHandler('/spdy/0.3.1', function (stream) { + var listenerConn = listenerMuxer.attach(stream, true) + expect(typeof listenerConn).to.be.equal('object') + done() + }) + + msI.handle(pair, function () { + msI.select('/spdy/0.3.1', function (err, stream) { + expect(err).to.not.be.instanceof(Error) + var dialerConn = dialerMuxer.attach(stream, false) + expect(typeof dialerConn).to.be.equal('object') + }) + }) + }) + + test('socket->ms-select into spdy->stream from dialer->ms-select into other protocol', function (done) { + var pair = streamPair.create() + + var msI = new Interactive() + var msS = new Select() + + var dialerMuxer = new Muxer() + var listenerMuxer = new Muxer() + + msS.handle(pair.other) + + msS.addHandler('/spdy/0.3.1', function (stream) { + var listenerConn = listenerMuxer.attach(stream, true) + listenerConn.on('stream', function (stream) { + stream.on('data', function (chunk) { + expect(chunk.toString()).to.equal('mux all the streams') + done() + }) + }) + }) + + msI.handle(pair, function () { + msI.select('/spdy/0.3.1', function (err, stream) { + expect(err).to.not.be.instanceof(Error) + var dialerConn = dialerMuxer.attach(stream, false) + dialerConn.dialStream(function (err, stream) { + expect(err).to.not.be.instanceof(Error) + stream.write('mux all the streams') + }) + }) + }) + }) + + test('socket->ms-select into spdy->stream from listener->ms-select into another protocol', function (done) { + var pair = streamPair.create() + + var msI = new Interactive() + var msS = new Select() + + var dialerMuxer = new Muxer() + var listenerMuxer = new Muxer() + + msS.handle(pair.other) + + msS.addHandler('/spdy/0.3.1', function (stream) { + var listenerConn = listenerMuxer.attach(stream, true) + listenerConn.on('stream', function (stream) { + stream.on('data', function (chunk) { + expect(chunk.toString()).to.equal('mux all the streams') + + listenerConn.dialStream(function (err, stream) { + expect(err).to.not.be.instanceof(Error) + var msI2 = new Interactive() + msI2.handle(stream, function () { + msI2.select('/other/protocol', function (err, stream) { + expect(err).to.not.be.instanceof(Error) + stream.write('the other protocol') + }) + }) + }) + }) + }) + }) + + msI.handle(pair, function () { + msI.select('/spdy/0.3.1', function (err, stream) { + expect(err).to.not.be.instanceof(Error) + var dialerConn = dialerMuxer.attach(stream, false) + dialerConn.dialStream(function (err, stream) { + expect(err).to.not.be.instanceof(Error) + stream.write('mux all the streams') + }) + + dialerConn.on('stream', function (stream) { + var msS2 = new Select() + msS2.handle(stream) + msS2.addHandler('/other/protocol', function (stream) { + stream.on('data', function (chunk) { + expect(chunk.toString()).to.equal('the other protocol') + done() + }) + }) + }) + }) + }) + + }) +}) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 2519a4e789..bdd8f4cb26 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -41,7 +41,7 @@ afterEach(function (done) { swarmB.closeListener() done() }) - +/* experiment('BASE', function () { test('Open a stream', function (done) { var protocol = '/sparkles/3.3.3' @@ -60,9 +60,7 @@ experiment('BASE', function () { test('Reuse connection (from dialer)', function (done) { var protocol = '/sparkles/3.3.3' - swarmB.registerHandler(protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - }) + swarmB.registerHandler(protocol, function (stream) {}) swarmA.openStream(peerB, protocol, function (err, stream) { expect(err).to.not.be.instanceof(Error) @@ -75,26 +73,37 @@ experiment('BASE', function () { }) }) }) - +*/ experiment('IDENTIFY', function () { test('Attach Identify, open a stream, see a peer update', function (done) { + + swarmA.on('error', function (err) { + console.log('A - ', err) + }) + + swarmB.on('error', function (err) { + console.log('B - ', err) + }) + var protocol = '/sparkles/3.3.3' var identifyA = new Identify(swarmA, peerA) var identifyB = new Identify(swarmB, peerB) + setTimeout(function () { + swarmB.registerHandler(protocol, function (stream) {}) - swarmB.registerHandler(protocol, function (stream) {}) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - }) + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + }) - identifyB.on('peer-update', function (answer) { - done() - }) - identifyA.on('peer-update', function (answer) {}) + identifyB.on('peer-update', function (answer) { + console.log('SUCH PEER-UPDATE') + done() + }) + identifyA.on('peer-update', function (answer) {}) + }, 500) }) - + /* test('Attach Identify, open a stream, reuse stream', function (done) { var protocol = '/sparkles/3.3.3' @@ -104,13 +113,13 @@ experiment('IDENTIFY', function () { swarmA.registerHandler(protocol, function (stream) {}) swarmB.registerHandler(protocol, function (stream) {}) - swarmA.openStream(peerB, protocol, function (err, stream) { + swarmA.openStream(peerB, protocol, function theOTHER (err, stream) { expect(err).to.not.be.instanceof(Error) }) identifyB.on('peer-update', function (answer) { expect(Object.keys(swarmB.connections).length).to.equal(1) - swarmB.openStream(peerA, protocol, function (err, stream) { + swarmB.openStream(peerA, protocol, function theCALLBACK (err, stream) { expect(err).to.not.be.instanceof(Error) expect(Object.keys(swarmB.connections).length).to.equal(1) done() @@ -118,6 +127,7 @@ experiment('IDENTIFY', function () { }) identifyA.on('peer-update', function (answer) {}) }) + */ }) experiment('HARDNESS', function () {}) From 93509debe165db6ec90f92d485d6a289a1c0e2e0 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 15 Jul 2015 11:34:40 -0700 Subject: [PATCH 021/634] add multistream and muxer tests --- examples/s.js | 6 +++--- src/swarm.js | 4 ++-- tests/swarm-test.js | 37 ++++++++++++++++++------------------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/examples/s.js b/examples/s.js index b4ce44979f..856730d060 100644 --- a/examples/s.js +++ b/examples/s.js @@ -20,9 +20,9 @@ b.on('error', function (err) { b.listen() b.registerHandler('/ipfs/sparkles/1.2.3', function (stream) { -// if (err) { -// return console.log(err) -// } + // if (err) { + // return console.log(err) + // } console.log('woop got a stream') }) diff --git a/src/swarm.js b/src/swarm.js index c5278b22fd..cb4dda5e2f 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -55,8 +55,8 @@ function Swarm () { // FOR IDENTIFY self.emit('connection-unknown', conn) - // IDENTIFY DOES THIS FOR US - // conn.on('close', function () { delete self.connections[conn.peerId] }) + // IDENTIFY DOES THIS FOR US + // conn.on('close', function () { delete self.connections[conn.peerId] }) }) }).listen(self.port, ready) errorUp(self, self.listener) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index bdd8f4cb26..dc3e1870f5 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -76,7 +76,6 @@ experiment('BASE', function () { */ experiment('IDENTIFY', function () { test('Attach Identify, open a stream, see a peer update', function (done) { - swarmA.on('error', function (err) { console.log('A - ', err) }) @@ -103,31 +102,31 @@ experiment('IDENTIFY', function () { identifyA.on('peer-update', function (answer) {}) }, 500) }) - /* - test('Attach Identify, open a stream, reuse stream', function (done) { - var protocol = '/sparkles/3.3.3' +/* +test('Attach Identify, open a stream, reuse stream', function (done) { + var protocol = '/sparkles/3.3.3' - var identifyA = new Identify(swarmA, peerA) - var identifyB = new Identify(swarmB, peerB) + var identifyA = new Identify(swarmA, peerA) + var identifyB = new Identify(swarmB, peerB) - swarmA.registerHandler(protocol, function (stream) {}) - swarmB.registerHandler(protocol, function (stream) {}) + swarmA.registerHandler(protocol, function (stream) {}) + swarmB.registerHandler(protocol, function (stream) {}) - swarmA.openStream(peerB, protocol, function theOTHER (err, stream) { - expect(err).to.not.be.instanceof(Error) - }) + swarmA.openStream(peerB, protocol, function theOTHER (err, stream) { + expect(err).to.not.be.instanceof(Error) + }) - identifyB.on('peer-update', function (answer) { + identifyB.on('peer-update', function (answer) { + expect(Object.keys(swarmB.connections).length).to.equal(1) + swarmB.openStream(peerA, protocol, function theCALLBACK (err, stream) { + expect(err).to.not.be.instanceof(Error) expect(Object.keys(swarmB.connections).length).to.equal(1) - swarmB.openStream(peerA, protocol, function theCALLBACK (err, stream) { - expect(err).to.not.be.instanceof(Error) - expect(Object.keys(swarmB.connections).length).to.equal(1) - done() - }) + done() }) - identifyA.on('peer-update', function (answer) {}) }) - */ + identifyA.on('peer-update', function (answer) {}) +}) +*/ }) experiment('HARDNESS', function () {}) From 1ed32f653136e1471c1a1ba8782c480e26555e93 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 15 Jul 2015 12:20:52 -0700 Subject: [PATCH 022/634] fix RST_STREAM bug --- src/swarm.js | 14 ++++++++++---- tests/swarm-test.js | 38 +++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/swarm.js b/src/swarm.js index cb4dda5e2f..06ee8ffb42 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -19,26 +19,32 @@ function Swarm () { } self.port = parseInt(process.env.IPFS_SWARM_PORT, 10) || 4001 - self.connections = {} self.handles = [] // set the listener self.listen = function (port, ready) { - if (!ready) { ready = function noop () {} } + if (!ready) { + ready = function noop () {} + } if (typeof port === 'function') { ready = port } else if (port) { self.port = port } + // + self.listener = tcp.createServer(function (socket) { - errorUp(self, socket) + socket.on('error', function (err) { + console.log('listener socket err - ', err) + }) + var ms = new Select() ms.handle(socket) ms.addHandler('/spdy/3.1.0', function (ds) { log.info('Negotiated spdy with incoming socket') - var conn = new Muxer().attach(ds, false) + var conn = new Muxer().attach(ds, true) // attach multistream handlers to incoming streams diff --git a/tests/swarm-test.js b/tests/swarm-test.js index dc3e1870f5..807ae8f0ed 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -41,7 +41,7 @@ afterEach(function (done) { swarmB.closeListener() done() }) -/* + experiment('BASE', function () { test('Open a stream', function (done) { var protocol = '/sparkles/3.3.3' @@ -73,7 +73,7 @@ experiment('BASE', function () { }) }) }) -*/ + experiment('IDENTIFY', function () { test('Attach Identify, open a stream, see a peer update', function (done) { swarmA.on('error', function (err) { @@ -102,31 +102,31 @@ experiment('IDENTIFY', function () { identifyA.on('peer-update', function (answer) {}) }, 500) }) -/* -test('Attach Identify, open a stream, reuse stream', function (done) { - var protocol = '/sparkles/3.3.3' - var identifyA = new Identify(swarmA, peerA) - var identifyB = new Identify(swarmB, peerB) + test('Attach Identify, open a stream, reuse stream', function (done) { + var protocol = '/sparkles/3.3.3' - swarmA.registerHandler(protocol, function (stream) {}) - swarmB.registerHandler(protocol, function (stream) {}) + var identifyA = new Identify(swarmA, peerA) + var identifyB = new Identify(swarmB, peerB) - swarmA.openStream(peerB, protocol, function theOTHER (err, stream) { - expect(err).to.not.be.instanceof(Error) - }) + swarmA.registerHandler(protocol, function (stream) {}) + swarmB.registerHandler(protocol, function (stream) {}) - identifyB.on('peer-update', function (answer) { - expect(Object.keys(swarmB.connections).length).to.equal(1) - swarmB.openStream(peerA, protocol, function theCALLBACK (err, stream) { + swarmA.openStream(peerB, protocol, function theOTHER (err, stream) { expect(err).to.not.be.instanceof(Error) + }) + + identifyB.on('peer-update', function (answer) { expect(Object.keys(swarmB.connections).length).to.equal(1) - done() + swarmB.openStream(peerA, protocol, function theCALLBACK (err, stream) { + expect(err).to.not.be.instanceof(Error) + expect(Object.keys(swarmB.connections).length).to.equal(1) + done() + }) }) + identifyA.on('peer-update', function (answer) {}) }) - identifyA.on('peer-update', function (answer) {}) -}) -*/ + }) experiment('HARDNESS', function () {}) From 0d82f9deccc16d389afaba879ebd1d78dad229de Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 15 Jul 2015 15:38:01 -0700 Subject: [PATCH 023/634] remove unnecessary logs --- src/identify.js | 13 ------------- src/swarm.js | 25 +------------------------ tests/swarm-test.js | 1 - 3 files changed, 1 insertion(+), 38 deletions(-) diff --git a/src/identify.js b/src/identify.js index 559644b870..5746fa95d4 100644 --- a/src/identify.js +++ b/src/identify.js @@ -15,8 +15,6 @@ function Identify (swarm, peerSelf) { var self = this swarm.registerHandler('/ipfs/identify/1.0.0', function (stream) { - console.log('DO I EVER GET CALLED?') - var identifyMsg = {} identifyMsg = {} identifyMsg.sender = exportPeer(peerSelf) @@ -42,22 +40,12 @@ function Identify (swarm, peerSelf) { }) swarm.on('connection-unknown', function (conn) { - console.log('IDENTIFY - DIALING STREAM FROM SERVER') - - conn.on('error', function (err) { - console.log('CAPUT-A', err) - }) conn.dialStream(function (err, stream) { if (err) { return console.log(err) } - stream.on('error', function (err) { - console.log('CAPUT-B', err) - }) - console.log('GOT STREAM') var msi = new Interactive() msi.handle(stream, function () { - console.log('HANDLE GOOD') msi.select('/ipfs/identify/1.0.0', function (err, ds) { if (err) { return console.log(err) } var identifyMsg = {} @@ -78,7 +66,6 @@ function Identify (swarm, peerSelf) { swarm.connections[answer.sender.id] = conn - console.log('BAM') self.emit('peer-update', answer) }) diff --git a/src/swarm.js b/src/swarm.js index 06ee8ffb42..bf5ae5cee6 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -35,10 +35,7 @@ function Swarm () { // self.listener = tcp.createServer(function (socket) { - socket.on('error', function (err) { - console.log('listener socket err - ', err) - }) - + errorUp(self, socket) var ms = new Select() ms.handle(socket) ms.addHandler('/spdy/3.1.0', function (ds) { @@ -48,13 +45,6 @@ function Swarm () { // attach multistream handlers to incoming streams - conn.on('stream', function () { - console.log('HERE') - }) - conn.on('error', function () { - console.log('error here') - }) - conn.on('stream', registerHandles) errorUp(self, conn) @@ -109,12 +99,6 @@ function Swarm () { if (err) { cb(err) } var conn = new Muxer().attach(ds, false) - conn.on('stream', function () { - console.log('WOOHO NEW STREAM') - }) - conn.on('error', function () { - console.log('BADUM TSS') - }) conn.on('stream', registerHandles) self.connections[peer.id.toB58String()] = conn conn.on('close', function () { delete self.connections[peer.id.toB58String()] }) @@ -131,9 +115,6 @@ function Swarm () { conn.dialStream(function (err, stream) { if (err) { return cb(err) } errorUp(self, stream) - stream.on('error', function (err) { - console.log('error here - ', err) - }) // negotiate desired protocol var msi = new Interactive() msi.handle(stream, function () { @@ -147,9 +128,7 @@ function Swarm () { } self.registerHandler = function (protocol, handlerFunc) { - console.log('new handler coming in for - ', protocol) if (self.handles[protocol]) { - console.log('here already - ', protocol) return handlerFunc(new Error('Handle for protocol already exists', protocol)) } self.handles.push({ protocol: protocol, func: handlerFunc }) @@ -174,12 +153,10 @@ function Swarm () { function registerHandles (stream) { log.info('Registering protocol handlers on new stream') - console.log('REGISTERING HANDLES') errorUp(self, stream) var msH = new Select() msH.handle(stream) self.handles.forEach(function (handle) { - console.log(' ->', handle.protocol) msH.addHandler(handle.protocol, handle.func) }) } diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 807ae8f0ed..b696f94a94 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -96,7 +96,6 @@ experiment('IDENTIFY', function () { }) identifyB.on('peer-update', function (answer) { - console.log('SUCH PEER-UPDATE') done() }) identifyA.on('peer-update', function (answer) {}) From 6f4011a3cf930cb4d802810d9d6967c71319caef Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 15 Jul 2015 21:19:46 -0700 Subject: [PATCH 024/634] push identify up --- src/identify/identify.proto | 25 ++++++++++++ src/identify/index.js | 81 +++++++++++++++++++++++++++++++++++++ src/identify/proto-test.js | 24 +++++++++++ 3 files changed, 130 insertions(+) create mode 100644 src/identify/identify.proto create mode 100644 src/identify/index.js create mode 100644 src/identify/proto-test.js diff --git a/src/identify/identify.proto b/src/identify/identify.proto new file mode 100644 index 0000000000..280bc401d8 --- /dev/null +++ b/src/identify/identify.proto @@ -0,0 +1,25 @@ +message Identify { + + // protocolVersion determines compatibility between peers + optional string protocolVersion = 5; // e.g. ipfs/1.0.0 + + // agentVersion is like a UserAgent string in browsers, or client version in bittorrent + // includes the client name and client. + optional string agentVersion = 6; // e.g. go-ipfs/0.1.0 + + // publicKey is this node's public key (which also gives its node.ID) + // - may not need to be sent, as secure channel implies it has been sent. + // - then again, if we change / disable secure channel, may still want it. + optional bytes publicKey = 1; + + // listenAddrs are the multiaddrs the sender node listens for open connections on + repeated bytes listenAddrs = 2; + + // oservedAddr is the multiaddr of the remote endpoint that the sender node perceives + // this is useful information to convey to the other side, as it helps the remote endpoint + // determine whether its connection to the local peer goes through NAT. + // optional bytes observedAddr = 4; + + // (DEPRECATED) protocols are the services this node is running + // repeated string protocols = 3; +} diff --git a/src/identify/index.js b/src/identify/index.js new file mode 100644 index 0000000000..606c2216fc --- /dev/null +++ b/src/identify/index.js @@ -0,0 +1,81 @@ +/* + * Identify is one of the protocols swarms speaks in order to broadcast and learn + * about the ip:port pairs a specific peer is available through + */ + +var Interactive = require('multistream-select').Interactive +var EventEmmiter = require('events').EventEmitter +var util = require('util') +var protobufs = require('protocol-buffers-stream') +var fs = require('fs') +var schema = fs.readFileSync(__dirname + '/identify.proto') + +exports = module.exports = Identify + +util.inherits(Identify, EventEmmiter) + +function Identify (swarm, peerSelf) { + var self = this + self.createProtoStream = protobufs(schema) + + swarm.registerHandler('/ipfs/identify/1.0.0', function (stream) { + var ps = self.createProtoStream() + + ps.on('identify', function (msg) { + console.log('RECEIVED PROTOBUF - ', msg) + // 1. wrap the msg + // 2. create a Peer obj using the publick key to derive the ID + // 3. populate it with observedAddr + // 4. maybe emit 2 peers update to update the other peer and also ourselfs? + self.emit('peer-update', {}) + }) + + ps.identify({ + protocolVersion: 'na', + agentVersion: 'na', + publicKey: peerSelf.id.pubKey, + listenAddrs: peerSelf.multiaddrs + // observedAddr: new Buffer() + }) + + ps.pipe(stream).pipe(ps) + + // TODO(daviddias) ps.end() based on https://github.com/mafintosh/protocol-buffers-stream/issues/1 + }) + + swarm.on('connection-unknown', function (conn) { + conn.dialStream(function (err, stream) { + if (err) { return console.log(err) } + var msi = new Interactive() + msi.handle(stream, function () { + msi.select('/ipfs/identify/1.0.0', function (err, ds) { + if (err) { return console.log(err) } + + var ps = self.createProtoStream() + + ps.on('identify', function (msg) { + console.log('RECEIVED PROTOBUF - ', msg) + // 1. wrap the msg + // 2. create a Peer obj using the publick key to derive the ID + // 3. populate it with observedAddr + // 4. maybe emit 2 peers update to update the other peer and also ourselfs? + // 5. add the conn to connections list -> swarm.connections[otherPeerId] = conn + self.emit('peer-update', {}) + }) + + ps.identify({ + protocolVersion: 'na', + agentVersion: 'na', + publicKey: peerSelf.id.pubKey, + listenAddrs: peerSelf.multiaddrs + // observedAddr: new Buffer() + }) + + ps.pipe(ds).pipe(ps) + + // TODO(daviddias) ps.end() based on https://github.com/mafintosh/protocol-buffers-stream/issues/1 + }) + }) + }) + }) +} diff --git a/src/identify/proto-test.js b/src/identify/proto-test.js new file mode 100644 index 0000000000..2fd4d88a65 --- /dev/null +++ b/src/identify/proto-test.js @@ -0,0 +1,24 @@ +var protobufs = require('protocol-buffers-stream') +var fs = require('fs') +var schema = fs.readFileSync(__dirname + '/identify.proto') + +var createProtoStream = protobufs(schema) + +var ps = createProtoStream() + +ps.on('identify', function (msg) { + console.log('RECEIVED PROTOBUF - ', msg) +// self.emit('peer-update', {}) +}) + +ps.identify({ + protocolVersion: 'nop', + agentVersion: 'nop' +// publicKey: new Buffer(), +// listenAddrs: new Buffer([buf1, buf2]) +// observedAddr: new Buffer() +}) + +ps.pipe(ps) + +// ps.end() From 5a4d9ee4ede900b0b10d41945a83815cba1a9193 Mon Sep 17 00:00:00 2001 From: David Dias Date: Fri, 17 Jul 2015 12:05:02 -0700 Subject: [PATCH 025/634] Identify working with protobufs and observed addrs --- package.json | 2 + src/identify.js | 89 ------------------------------- src/identify/identify.proto | 2 +- src/identify/index.js | 102 ++++++++++++++++++++++++++---------- src/identify/proto-test.js | 2 +- src/swarm.js | 13 +++-- tests/swarm-test.js | 6 ++- 7 files changed, 90 insertions(+), 126 deletions(-) delete mode 100644 src/identify.js diff --git a/package.json b/package.json index 461f8f47d1..f7b879aa32 100644 --- a/package.json +++ b/package.json @@ -34,9 +34,11 @@ }, "dependencies": { "async": "^1.3.0", + "ip-address": "^4.0.0", "multiaddr": "^1.0.0", "multiplex-stream-muxer": "^0.2.0", "multistream-select": "^0.6.1", + "protocol-buffers-stream": "^1.2.0", "spdy-stream-muxer": "^0.2.0" } } diff --git a/src/identify.js b/src/identify.js deleted file mode 100644 index 5746fa95d4..0000000000 --- a/src/identify.js +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Identify is one of the protocols swarms speaks in order to broadcast and learn - * about the ip:port pairs a specific peer is available through - */ - -var Interactive = require('multistream-select').Interactive -var EventEmmiter = require('events').EventEmitter -var util = require('util') - -exports = module.exports = Identify - -util.inherits(Identify, EventEmmiter) - -function Identify (swarm, peerSelf) { - var self = this - - swarm.registerHandler('/ipfs/identify/1.0.0', function (stream) { - var identifyMsg = {} - identifyMsg = {} - identifyMsg.sender = exportPeer(peerSelf) - // TODO (daviddias) populate with the way I see the other peer - // identifyMsg.receiver = - - stream.write(JSON.stringify(identifyMsg)) - - var answer = '' - - stream.on('data', function (chunk) { - answer += chunk.toString() - }) - - stream.on('end', function () { - self.emit('peer-update', answer) - }) - - stream.end() - - // receive their info and how they see us - // send back our stuff - }) - - swarm.on('connection-unknown', function (conn) { - conn.dialStream(function (err, stream) { - if (err) { - return console.log(err) - } - var msi = new Interactive() - msi.handle(stream, function () { - msi.select('/ipfs/identify/1.0.0', function (err, ds) { - if (err) { return console.log(err) } - var identifyMsg = {} - identifyMsg = {} - identifyMsg.sender = exportPeer(peerSelf) - // TODO (daviddias) populate with the way I see the other peer - - stream.write(JSON.stringify(identifyMsg)) - - var answer = '' - - stream.on('data', function (chunk) { - answer = answer + chunk.toString() - }) - - stream.on('end', function () { - answer = JSON.parse(answer) - - swarm.connections[answer.sender.id] = conn - - self.emit('peer-update', answer) - }) - - stream.end() - }) - }) - }) - // open a spdy stream - // do the multistream handshake - // send them our data - }) - - function exportPeer (peer) { - return { - id: peer.id.toB58String(), - multiaddrs: peer.multiaddrs.map(function (mh) { - return mh.toString() - }) - } - } -} diff --git a/src/identify/identify.proto b/src/identify/identify.proto index 280bc401d8..e4845aaf88 100644 --- a/src/identify/identify.proto +++ b/src/identify/identify.proto @@ -18,7 +18,7 @@ message Identify { // oservedAddr is the multiaddr of the remote endpoint that the sender node perceives // this is useful information to convey to the other side, as it helps the remote endpoint // determine whether its connection to the local peer goes through NAT. - // optional bytes observedAddr = 4; + optional bytes observedAddr = 4; // (DEPRECATED) protocols are the services this node is running // repeated string protocols = 3; diff --git a/src/identify/index.js b/src/identify/index.js index 606c2216fc..4fbca87a8d 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -9,6 +9,9 @@ var util = require('util') var protobufs = require('protocol-buffers-stream') var fs = require('fs') var schema = fs.readFileSync(__dirname + '/identify.proto') +var v6 = require('ip-address').v6 +var Id = require('ipfs-peer-id') +var multiaddr = require('multiaddr') exports = module.exports = Identify @@ -22,28 +25,32 @@ function Identify (swarm, peerSelf) { var ps = self.createProtoStream() ps.on('identify', function (msg) { - console.log('RECEIVED PROTOBUF - ', msg) - // 1. wrap the msg - // 2. create a Peer obj using the publick key to derive the ID - // 3. populate it with observedAddr - // 4. maybe emit 2 peers update to update the other peer and also ourselfs? - self.emit('peer-update', {}) - }) + // console.log('RECEIVED PROTOBUF - ', msg) + updateSelf(peerSelf, msg.observedAddr) - ps.identify({ - protocolVersion: 'na', - agentVersion: 'na', - publicKey: peerSelf.id.pubKey, - listenAddrs: peerSelf.multiaddrs - // observedAddr: new Buffer() - }) + var peerId = Id.createFromPubKey(msg.publicKey) - ps.pipe(stream).pipe(ps) + var socket = swarm.connections[peerId.toB58String()].socket + var mh = getMultiaddr(socket) + ps.identify({ + protocolVersion: 'na', + agentVersion: 'na', + publicKey: peerSelf.id.pubKey, + listenAddrs: peerSelf.multiaddrs.map(function (mh) {return mh.buffer}), + observedAddr: mh.buffer + }) + + self.emit('peer-update', { + peerId: peerId, + listenAddrs: msg.listenAddrs.map(function (mhb) {return multiaddr(mhb)}) + }) - // TODO(daviddias) ps.end() based on https://github.com/mafintosh/protocol-buffers-stream/issues/1 + ps.finalize() + }) + ps.pipe(stream).pipe(ps) }) - swarm.on('connection-unknown', function (conn) { + swarm.on('connection-unknown', function (conn, socket) { conn.dialStream(function (err, stream) { if (err) { return console.log(err) } var msi = new Interactive() @@ -54,28 +61,67 @@ function Identify (swarm, peerSelf) { var ps = self.createProtoStream() ps.on('identify', function (msg) { - console.log('RECEIVED PROTOBUF - ', msg) - // 1. wrap the msg - // 2. create a Peer obj using the publick key to derive the ID - // 3. populate it with observedAddr - // 4. maybe emit 2 peers update to update the other peer and also ourselfs? - // 5. add the conn to connections list -> swarm.connections[otherPeerId] = conn - self.emit('peer-update', {}) + // console.log('RECEIVED PROTOBUF - SIDE ZZ ', msg) + var peerId = Id.createFromPubKey(msg.publicKey) + + updateSelf(peerSelf, msg.observedAddr) + + swarm.connections[peerId.toB58String()] = { + conn: conn, + socket: socket + } + + self.emit('peer-update', { + peerId: peerId, + listenAddrs: msg.listenAddrs.map(function (mhb) {return multiaddr(mhb)}) + }) }) + var mh = getMultiaddr(socket) + ps.identify({ protocolVersion: 'na', agentVersion: 'na', publicKey: peerSelf.id.pubKey, - listenAddrs: peerSelf.multiaddrs - // observedAddr: new Buffer() + listenAddrs: peerSelf.multiaddrs.map(function (mh) {return mh.buffer}), + observedAddr: mh.buffer }) ps.pipe(ds).pipe(ps) - - // TODO(daviddias) ps.end() based on https://github.com/mafintosh/protocol-buffers-stream/issues/1 + ps.finalize() }) }) }) }) } + +function getMultiaddr (socket) { + var mh + if (~socket.remoteAddress.indexOf(':')) { + var addr = new v6.Address(socket.remoteAddress) + if (addr.v4) { + var ip4 = socket.remoteAddress.split(':')[3] + mh = multiaddr('/ip4/' + ip4 + '/tcp/' + socket.remotePort) + } else { + mh = multiaddr('/ip6/' + socket.remoteAddress + '/tcp/' + socket.remotePort) + } + } else { + mh = multiaddr('/ip4/' + socket.remoteAddress + '/tcp/' + socket.remotePort) + } + return mh +} + +function updateSelf (peerSelf, observedAddr) { + var omh = multiaddr(observedAddr) + var isIn = false + peerSelf.multiaddrs.forEach(function (mh) { + if (mh.toString() === omh.toString()) { + isIn = true + } + }) + + if (!isIn) { + peerSelf.multiaddrs.push(omh) + } +} + diff --git a/src/identify/proto-test.js b/src/identify/proto-test.js index 2fd4d88a65..f3c4b2c61d 100644 --- a/src/identify/proto-test.js +++ b/src/identify/proto-test.js @@ -21,4 +21,4 @@ ps.identify({ ps.pipe(ps) -// ps.end() +ps.end() diff --git a/src/swarm.js b/src/swarm.js index bf5ae5cee6..af686386f3 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -19,7 +19,7 @@ function Swarm () { } self.port = parseInt(process.env.IPFS_SWARM_PORT, 10) || 4001 - self.connections = {} + self.connections = {} // {conn: <>, socket: <>} self.handles = [] // set the listener @@ -49,7 +49,7 @@ function Swarm () { errorUp(self, conn) // FOR IDENTIFY - self.emit('connection-unknown', conn) + self.emit('connection-unknown', conn, socket) // IDENTIFY DOES THIS FOR US // conn.on('close', function () { delete self.connections[conn.peerId] }) @@ -100,7 +100,10 @@ function Swarm () { var conn = new Muxer().attach(ds, false) conn.on('stream', registerHandles) - self.connections[peer.id.toB58String()] = conn + self.connections[peer.id.toB58String()] = { + conn: conn, + socket: socket + } conn.on('close', function () { delete self.connections[peer.id.toB58String()] }) errorUp(self, conn) @@ -111,7 +114,7 @@ function Swarm () { function createStream (peer, protocol, cb) { // spawn new muxed stream - var conn = self.connections[peer.id.toB58String()] + var conn = self.connections[peer.id.toB58String()].conn conn.dialStream(function (err, stream) { if (err) { return cb(err) } errorUp(self, stream) @@ -143,7 +146,7 @@ function Swarm () { keys.map(function (key) { c.hit() - self.connections[key].end() + self.connections[key].conn.end() }) } diff --git a/tests/swarm-test.js b/tests/swarm-test.js index b696f94a94..f3f3d489a0 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -103,6 +103,8 @@ experiment('IDENTIFY', function () { }) test('Attach Identify, open a stream, reuse stream', function (done) { + console.log('\n\n\n') + var protocol = '/sparkles/3.3.3' var identifyA = new Identify(swarmA, peerA) @@ -111,13 +113,13 @@ experiment('IDENTIFY', function () { swarmA.registerHandler(protocol, function (stream) {}) swarmB.registerHandler(protocol, function (stream) {}) - swarmA.openStream(peerB, protocol, function theOTHER (err, stream) { + swarmA.openStream(peerB, protocol, function (err, stream) { expect(err).to.not.be.instanceof(Error) }) identifyB.on('peer-update', function (answer) { expect(Object.keys(swarmB.connections).length).to.equal(1) - swarmB.openStream(peerA, protocol, function theCALLBACK (err, stream) { + swarmB.openStream(peerA, protocol, function (err, stream) { expect(err).to.not.be.instanceof(Error) expect(Object.keys(swarmB.connections).length).to.equal(1) done() From 1defb0f4025a5e297d337012644f9c0f108d431c Mon Sep 17 00:00:00 2001 From: David Dias Date: Fri, 17 Jul 2015 12:05:15 -0700 Subject: [PATCH 026/634] Identify working with protobufs and observed addrs --- src/identify/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/identify/index.js b/src/identify/index.js index 4fbca87a8d..fcb25e8393 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -124,4 +124,3 @@ function updateSelf (peerSelf, observedAddr) { peerSelf.multiaddrs.push(omh) } } - From 1a25932696c40a6bae5a6fa62ee43eaecab9da71 Mon Sep 17 00:00:00 2001 From: David Dias Date: Sun, 19 Jul 2015 14:36:24 -0700 Subject: [PATCH 027/634] update deps, remove the need for npm links --- .jshintignore | 1 + .jshintrc | 10 ++++++++++ package.json | 9 +++++++-- 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 .jshintignore create mode 100644 .jshintrc diff --git a/.jshintignore b/.jshintignore new file mode 100644 index 0000000000..3c3629e647 --- /dev/null +++ b/.jshintignore @@ -0,0 +1 @@ +node_modules diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 0000000000..997b3f7d45 --- /dev/null +++ b/.jshintrc @@ -0,0 +1,10 @@ +{ + "node": true, + + "curly": true, + "latedef": true, + "quotmark": true, + "undef": true, + "unused": true, + "trailing": true +} diff --git a/package.json b/package.json index f7b879aa32..900dac357e 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,9 @@ "scripts": { "test": "./node_modules/.bin/lab tests/*-test.js", "coverage": "./node_modules/.bin/lab -t 100 tests/*-test.js", - "codestyle": "./node_modules/.bin/standard --format" + "codestyle": "./node_modules/.bin/standard --format", + "lint": "jshint .", + "validate": "npm ls" }, "repository": { "type": "git", @@ -35,10 +37,13 @@ "dependencies": { "async": "^1.3.0", "ip-address": "^4.0.0", + "ipfs-logger": "^0.1.0", + "ipfs-peer": "^0.3.0", + "ipfs-peer-id": "^0.3.0", "multiaddr": "^1.0.0", "multiplex-stream-muxer": "^0.2.0", "multistream-select": "^0.6.1", "protocol-buffers-stream": "^1.2.0", - "spdy-stream-muxer": "^0.2.0" + "spdy-stream-muxer": "^0.4.0" } } From d574360d4703df5b64ae874cc18b2a4de55020dc Mon Sep 17 00:00:00 2001 From: David Dias Date: Sun, 19 Jul 2015 14:37:16 -0700 Subject: [PATCH 028/634] remove jshint stuff created by precommit hook --- .gitignore | 5 +++++ .jshintignore | 1 - .jshintrc | 10 ---------- 3 files changed, 5 insertions(+), 11 deletions(-) delete mode 100644 .jshintignore delete mode 100644 .jshintrc diff --git a/.gitignore b/.gitignore index 123ae94d05..554436631f 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,8 @@ build/Release # Dependency directory # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git node_modules + + + +.jshintrc +.jshintignore diff --git a/.jshintignore b/.jshintignore deleted file mode 100644 index 3c3629e647..0000000000 --- a/.jshintignore +++ /dev/null @@ -1 +0,0 @@ -node_modules diff --git a/.jshintrc b/.jshintrc deleted file mode 100644 index 997b3f7d45..0000000000 --- a/.jshintrc +++ /dev/null @@ -1,10 +0,0 @@ -{ - "node": true, - - "curly": true, - "latedef": true, - "quotmark": true, - "undef": true, - "unused": true, - "trailing": true -} From fe026707c917d894593b098424a0352123af6d1d Mon Sep 17 00:00:00 2001 From: David Dias Date: Sun, 19 Jul 2015 14:37:35 -0700 Subject: [PATCH 029/634] Release v0.1.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 900dac357e..8517e216e5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-swarm", - "version": "0.0.0", + "version": "0.1.0", "description": "IPFS swarm implementation in Node.js", "main": "src/index.js", "scripts": { From d67b3dff3262a92a0cbf0784c464a68f4c930625 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 20 Jul 2015 11:19:31 -0700 Subject: [PATCH 030/634] rm old proto-test.js example --- src/identify/proto-test.js | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 src/identify/proto-test.js diff --git a/src/identify/proto-test.js b/src/identify/proto-test.js deleted file mode 100644 index f3c4b2c61d..0000000000 --- a/src/identify/proto-test.js +++ /dev/null @@ -1,24 +0,0 @@ -var protobufs = require('protocol-buffers-stream') -var fs = require('fs') -var schema = fs.readFileSync(__dirname + '/identify.proto') - -var createProtoStream = protobufs(schema) - -var ps = createProtoStream() - -ps.on('identify', function (msg) { - console.log('RECEIVED PROTOBUF - ', msg) -// self.emit('peer-update', {}) -}) - -ps.identify({ - protocolVersion: 'nop', - agentVersion: 'nop' -// publicKey: new Buffer(), -// listenAddrs: new Buffer([buf1, buf2]) -// observedAddr: new Buffer() -}) - -ps.pipe(ps) - -ps.end() From a9855a0b4223f8df2a98be07436f8095b014fa26 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 20 Jul 2015 14:42:47 -0700 Subject: [PATCH 031/634] fix multiaddr explosion --- src/identify/index.js | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/identify/index.js b/src/identify/index.js index fcb25e8393..5653f11e34 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -25,7 +25,6 @@ function Identify (swarm, peerSelf) { var ps = self.createProtoStream() ps.on('identify', function (msg) { - // console.log('RECEIVED PROTOBUF - ', msg) updateSelf(peerSelf, msg.observedAddr) var peerId = Id.createFromPubKey(msg.publicKey) @@ -113,14 +112,31 @@ function getMultiaddr (socket) { function updateSelf (peerSelf, observedAddr) { var omh = multiaddr(observedAddr) - var isIn = false - peerSelf.multiaddrs.forEach(function (mh) { - if (mh.toString() === omh.toString()) { - isIn = true + + if (!peerSelf.previousObservedAddrs) { + peerSelf.previousObservedAddrs = [] + } + + for (var i = 0; i < peerSelf.previousObservedAddrs.length; i++) { + if (peerSelf.previousObservedAddrs[i].toString() === omh.toString()) { + peerSelf.previousObserveredAddrs.splice(i, 1) + addToSelf() + return } - }) + } - if (!isIn) { - peerSelf.multiaddrs.push(omh) + peerSelf.previousObservedAddrs.push(observedAddr) + + function addToSelf () { + var isIn = false + peerSelf.multiaddrs.forEach(function (mh) { + if (mh.toString() === omh.toString()) { + isIn = true + } + }) + + if (!isIn) { + peerSelf.multiaddrs.push(omh) + } } } From e1b4a8ce4e7f5f740651024cc5e89904942314be Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 21 Jul 2015 16:00:59 -0700 Subject: [PATCH 032/634] add a readme --- README.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f597d48771..a9e4e673d9 100644 --- a/README.md +++ b/README.md @@ -9,21 +9,65 @@ ipfs-swarm Node.js implementation ipfs-swarm is an abstraction for the network layer on IPFS. It offers an API to open streams between peers on a specific protocol. -Ref link (still a WiP) - https://github.com/diasdavid/specs/blob/protocol-spec/protocol/layers.md#network-layer +Ref spec (still a WiP) - https://github.com/diasdavid/specs/blob/protocol-spec/protocol/layers.md#network-layer # Usage -## API calls +### Create a new Swarm -.openStream -.registerHandle +```javascript +var Swarm = require('ipfs-swarm') -## Events emmited +var s = new Swarm([port]) // `port` defalts to 4001 +``` +### Set the swarm to listen for incoming streams + +```javascript +s.listen([port], [callback]) // `port` defaults to 4001, `callback` gets called when the socket starts listening +``` + +### Close the listener/socket and every open stream that was multiplexed on it + +```javascript +s.closeListener() +``` + +### Register a protocol to be handled by an incoming stream + +```javascript +s.registerHandler('/name/protocol/you/want/version', function (stream) {}) +``` + +### Dial a new stream + +``` +s.openStream(peerInfo, protocol, function (err, stream) {}) +``` + +peerInfo must be a [`ipfs-peer`](https://www.npmjs.com/package/ipfs-peer) object, contaning both peer-id and multiaddrs. + +## Events emitted + +``` .on('error') .on('connection') -.on('connection-unknown') +.on('connection-unknown') // used by Identify to start the Identify protocol from listener to dialer +``` + +## Identify protocol + +The Identify protocol is an integral part to Swarm. It enables peers to share observedAddrs, identities and other possible address available. This enables us to do better NAT traversal. + +To instantiate Identify: + +``` +var Identify = require('ipfs-swarm/identify') + +var i = new Identify(swarmInstance, peerSelf) +``` + +`swarmInstance` must be an Instance of swarm and `peerSelf` must be a instance of `ipfs-peer` that represents the peer that instantiated this Identify -.on('stream') -.on('stream-unknown') +Identify emits a `peer-update` event each time it receives information from another peer. From 7c08e76cece918cdc50f55948b054e5ba078b6b4 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 21 Jul 2015 16:01:13 -0700 Subject: [PATCH 033/634] Release v0.1.1. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8517e216e5..827d6f97fa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-swarm", - "version": "0.1.0", + "version": "0.1.1", "description": "IPFS swarm implementation in Node.js", "main": "src/index.js", "scripts": { From 8911d726487c2a7c049e8c2cf41cd5c2033dcee4 Mon Sep 17 00:00:00 2001 From: David Dias Date: Sat, 25 Jul 2015 18:42:36 -0700 Subject: [PATCH 034/634] add new openConnection option --- README.md | 9 +++++++++ src/swarm.js | 15 ++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a9e4e673d9..2184a79c2f 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,15 @@ s.closeListener() s.registerHandler('/name/protocol/you/want/version', function (stream) {}) ``` +### Open a new connection + +Used when we want to make sure we can connect to a given peer, but do not intend to establish a stream with any of the services offered right away. + +``` +s.openConnection(peerConnection, function (err) {}) +``` + + ### Dial a new stream ``` diff --git a/src/swarm.js b/src/swarm.js index af686386f3..78ab773d3a 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -61,7 +61,7 @@ function Swarm () { // interface // open stream account for connection reuse - self.openStream = function (peer, protocol, cb) { + self.openConnection = function (peer, cb) { // If no connection open yet, open it if (!self.connections[peer.id.toB58String()]) { // Establish a socket with one of the addresses @@ -88,7 +88,7 @@ function Swarm () { gotSocket(socket) }) } else { - createStream(peer, protocol, cb) + cb() } // do the spdy people dance (multistream-select into spdy) @@ -107,12 +107,17 @@ function Swarm () { conn.on('close', function () { delete self.connections[peer.id.toB58String()] }) errorUp(self, conn) - createStream(peer, protocol, cb) + cb() }) }) } + } - function createStream (peer, protocol, cb) { + self.openStream = function (peer, protocol, cb) { + self.openConnection(peer, function (err) { + if (err) { + return cb(err) + } // spawn new muxed stream var conn = self.connections[peer.id.toB58String()].conn conn.dialStream(function (err, stream) { @@ -127,7 +132,7 @@ function Swarm () { }) }) }) - } + }) } self.registerHandler = function (protocol, handlerFunc) { From b08107dc63871bdaf75c5a9211ea4cba00de9642 Mon Sep 17 00:00:00 2001 From: David Dias Date: Sat, 25 Jul 2015 18:42:49 -0700 Subject: [PATCH 035/634] Release v0.2.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 827d6f97fa..ccf533ce5c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-swarm", - "version": "0.1.1", + "version": "0.2.0", "description": "IPFS swarm implementation in Node.js", "main": "src/index.js", "scripts": { From 648eb7c66666c9cdd7a46a446c44c41d44c2e59b Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 30 Jul 2015 14:35:59 -0700 Subject: [PATCH 036/634] add last seen date to the peer --- src/swarm.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swarm.js b/src/swarm.js index 78ab773d3a..3f89f9f0fe 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -19,7 +19,7 @@ function Swarm () { } self.port = parseInt(process.env.IPFS_SWARM_PORT, 10) || 4001 - self.connections = {} // {conn: <>, socket: <>} + self.connections = {} // {peerIdB58: {conn: <>, socket: <>} self.handles = [] // set the listener @@ -128,6 +128,7 @@ function Swarm () { msi.handle(stream, function () { msi.select(protocol, function (err, ds) { if (err) { return cb(err) } + peer.lastSeen = new Date() cb(null, ds) // return the stream }) }) From 02e07993dae2d03261f091b820bb609697749679 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 30 Jul 2015 14:58:32 -0700 Subject: [PATCH 037/634] add/update lastSeen to a peer once a stream is open --- tests/swarm-test.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index f3f3d489a0..646f0545fb 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -72,6 +72,18 @@ experiment('BASE', function () { }) }) }) + test('Check for lastSeen', function (done) { + var protocol = '/sparkles/3.3.3' + + swarmB.registerHandler(protocol, function (stream) {}) + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + expect(peerB.lastSeen).to.be.instanceof(Date) + done() + }) + }) + }) experiment('IDENTIFY', function () { From 7cf6808e9b7e66cc2d3fd42375fd3ec1251d6af8 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 30 Jul 2015 15:01:57 -0700 Subject: [PATCH 038/634] Release v0.3.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ccf533ce5c..b255ff5052 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-swarm", - "version": "0.2.0", + "version": "0.3.0", "description": "IPFS swarm implementation in Node.js", "main": "src/index.js", "scripts": { From 85e312dc66a841637cfa13f2a2841123488eb6db Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 30 Jul 2015 20:44:10 +0200 Subject: [PATCH 039/634] Start adding more swarm tests --- tests/swarm-test.js | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 646f0545fb..7fc7e26505 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -42,6 +42,39 @@ afterEach(function (done) { done() }) +experiment('BASICS', function () { + experiment('Swarm', function () { + test('enforces instantiation with new', function (done) { + expect(function () { + Swarm() + }).to.throw('Swarm must be called with new') + done() + }) + + test('parses $IPFS_SWARM_PORT', function (done) { + process.env.IPFS_SWARM_PORT = 1111 + var swarm = new Swarm() + expect(swarm.port).to.be.equal(1111) + process.env.IPFS_SWARM_PORT = undefined + done() + }) + }) + + experiment('Swarm.listen', function (done) { + test('handles missing port', function (done) { + var swarm = new Swarm() + swarm.listen(done) + }) + + test('handles passed in port', function (done) { + var swarm = new Swarm() + swarm.listen(1234) + expect(swarm.port).to.be.equal(1234) + done() + }) + }) +}) + experiment('BASE', function () { test('Open a stream', function (done) { var protocol = '/sparkles/3.3.3' From 13659ecb40bacb97e2a391fc0d269ede6f96f76c Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Fri, 31 Jul 2015 17:56:36 +0200 Subject: [PATCH 040/634] swarm: Fix self.handles data structure --- src/swarm.js | 8 ++++---- tests/swarm-test.js | 12 ++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/swarm.js b/src/swarm.js index 3f89f9f0fe..b35bef3c10 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -20,7 +20,7 @@ function Swarm () { self.port = parseInt(process.env.IPFS_SWARM_PORT, 10) || 4001 self.connections = {} // {peerIdB58: {conn: <>, socket: <>} - self.handles = [] + self.handles = {} // set the listener @@ -140,7 +140,7 @@ function Swarm () { if (self.handles[protocol]) { return handlerFunc(new Error('Handle for protocol already exists', protocol)) } - self.handles.push({ protocol: protocol, func: handlerFunc }) + self.handles[protocol] = handlerFunc log.info('Registered handler for protocol:', protocol) } @@ -165,8 +165,8 @@ function Swarm () { errorUp(self, stream) var msH = new Select() msH.handle(stream) - self.handles.forEach(function (handle) { - msH.addHandler(handle.protocol, handle.func) + Object.keys(self.handles).forEach(function (protocol) { + msH.addHandler(protocol, self.handles[protocol]) }) } diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 7fc7e26505..28e87d5864 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -73,6 +73,18 @@ experiment('BASICS', function () { done() }) }) + + experiment('Swarm.registerHandler', function () { + test('throws when registering a protcol handler twice', function (done) { + var swarm = new Swarm() + swarm.registerHandler('/sparkles/1.1.1', function () {}) + swarm.registerHandler('/sparkles/1.1.1', function (err) { + expect(err).to.be.an.instanceOf(Error) + expect(err.message).to.be.equal('Handle for protocol already exists') + done() + }) + }) + }) }) experiment('BASE', function () { From a73066b10b0a7c2faa5397434a06de12e3ddb3b5 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Fri, 31 Jul 2015 18:02:38 +0200 Subject: [PATCH 041/634] Setup travis --- .travis.yml | 14 ++++++++++++++ README.md | 2 +- package.json | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..f66fe91996 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +sudo: false +language: node_js +node_js: + - "iojs" + - "0.12" + - "0.10" + +# Make sure we have new NPM. +before_install: + - npm install -g npm + +script: + - npm run lint + - npm test diff --git a/README.md b/README.md index 2184a79c2f..33ac99672f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ipfs-swarm Node.js implementation ================================= -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) [![Build Status](https://img.shields.io/travis/diasdavid/node-ipfs-swarm/master.svg?style=flat-square)](https://travis-ci.org/diasdavid/node-ipfs-swarm) > IPFS swarm implementation in Node.js diff --git a/package.json b/package.json index b255ff5052..07a0fc1d3a 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "test": "./node_modules/.bin/lab tests/*-test.js", "coverage": "./node_modules/.bin/lab -t 100 tests/*-test.js", "codestyle": "./node_modules/.bin/standard --format", - "lint": "jshint .", + "lint": "./node_modules/.bin/standard", "validate": "npm ls" }, "repository": { From f1d796f47b054f9123765f4045cc83b59bed4dfd Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Fri, 31 Jul 2015 23:13:05 +0200 Subject: [PATCH 042/634] Fix swarm.closeConns --- package.json | 1 + src/swarm.js | 4 ++-- tests/swarm-test.js | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 07a0fc1d3a..112c52427c 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "code": "^1.4.1", "lab": "^5.13.0", "precommit-hook": "^3.0.0", + "sinon": "^1.15.4", "standard": "^4.5.2", "stream-pair": "^1.0.3" }, diff --git a/src/swarm.js b/src/swarm.js index b35bef3c10..89beaa1dfb 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -150,9 +150,9 @@ function Swarm () { if (number === 0) { cb() } var c = new Counter(number, cb) - keys.map(function (key) { - c.hit() + keys.forEach(function (key) { self.connections[key].conn.end() + c.hit() }) } diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 28e87d5864..b8a5c9007e 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -1,5 +1,6 @@ var Lab = require('lab') var Code = require('code') +var sinon = require('sinon') var lab = exports.lab = Lab.script() var experiment = lab.experiment @@ -85,6 +86,19 @@ experiment('BASICS', function () { }) }) }) + + experiment('Swarm.closeConns', function () { + test('calls end on all connections', function (done) { + swarmA.openConnection(peerB, function () { + var key = Object.keys(swarmA.connections)[0] + sinon.spy(swarmA.connections[key].conn, 'end') + swarmA.closeConns(function () { + expect(swarmA.connections[key].conn.end.called).to.be.equal(true) + done() + }) + }) + }) + }) }) experiment('BASE', function () { From cfc9ac582ea2d6bd2b685db3c1b6e9c5ceeed77b Mon Sep 17 00:00:00 2001 From: David Dias Date: Sat, 1 Aug 2015 20:19:25 +0200 Subject: [PATCH 043/634] update spdy-stream-muxer --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 112c52427c..91b14f2f06 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,6 @@ "multiplex-stream-muxer": "^0.2.0", "multistream-select": "^0.6.1", "protocol-buffers-stream": "^1.2.0", - "spdy-stream-muxer": "^0.4.0" + "spdy-stream-muxer": "^0.6.0" } } From da498eec95f92bcc709d09f949dfe33508488536 Mon Sep 17 00:00:00 2001 From: David Dias Date: Sat, 1 Aug 2015 20:19:52 +0200 Subject: [PATCH 044/634] Release v0.3.1. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 91b14f2f06..2d531603ff 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-swarm", - "version": "0.3.0", + "version": "0.3.1", "description": "IPFS swarm implementation in Node.js", "main": "src/index.js", "scripts": { From cc8355fa42884fa1c052a3817da47e48ce8ee816 Mon Sep 17 00:00:00 2001 From: David Dias Date: Sun, 2 Aug 2015 13:26:38 +0200 Subject: [PATCH 045/634] make travis-ci badge green :) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 33ac99672f..f585e5773f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ ipfs-swarm Node.js implementation ipfs-swarm is an abstraction for the network layer on IPFS. It offers an API to open streams between peers on a specific protocol. -Ref spec (still a WiP) - https://github.com/diasdavid/specs/blob/protocol-spec/protocol/layers.md#network-layer +Ref spec (WIP) - https://github.com/diasdavid/specs/blob/protocol-spec/protocol/layers.md#network-layer # Usage From cfa37e8d983a1d783ac8bfa2f3bfc0442e461156 Mon Sep 17 00:00:00 2001 From: David Dias Date: Sun, 2 Aug 2015 13:31:03 +0200 Subject: [PATCH 046/634] remove 0.10 from travis list (due to spdy-transport) --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f66fe91996..7b2bb6ed08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ language: node_js node_js: - "iojs" - "0.12" - - "0.10" # Make sure we have new NPM. before_install: From c6ae35026b8f39ee38d5936e3f8a66abcd9640a9 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 3 Aug 2015 19:45:51 +0200 Subject: [PATCH 047/634] fix issue pointed out here - https://github.com/diasdavid/node-ipfs-swarm/pull/7#discussion_r36044035 --- src/identify/index.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/identify/index.js b/src/identify/index.js index 5653f11e34..08a09bb45c 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -60,7 +60,6 @@ function Identify (swarm, peerSelf) { var ps = self.createProtoStream() ps.on('identify', function (msg) { - // console.log('RECEIVED PROTOBUF - SIDE ZZ ', msg) var peerId = Id.createFromPubKey(msg.publicKey) updateSelf(peerSelf, msg.observedAddr) @@ -119,13 +118,13 @@ function updateSelf (peerSelf, observedAddr) { for (var i = 0; i < peerSelf.previousObservedAddrs.length; i++) { if (peerSelf.previousObservedAddrs[i].toString() === omh.toString()) { - peerSelf.previousObserveredAddrs.splice(i, 1) + peerSelf.previousObservedAddrs.splice(i, 1) addToSelf() return } } - peerSelf.previousObservedAddrs.push(observedAddr) + peerSelf.previousObservedAddrs.push(omh) function addToSelf () { var isIn = false From bef901fecd545b7ba4b9b5f39c1d7f712a67e00d Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 3 Aug 2015 19:46:05 +0200 Subject: [PATCH 048/634] Release v0.4.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2d531603ff..ee6c073ad0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-swarm", - "version": "0.3.1", + "version": "0.4.0", "description": "IPFS swarm implementation in Node.js", "main": "src/index.js", "scripts": { From 17f40911dbea98e2d4af8dc5d3868b131c6df8e2 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Sun, 2 Aug 2015 15:02:58 +0200 Subject: [PATCH 049/634] identify: Fix some issues with updateSelf. --- tests/swarm-test.js | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index b8a5c9007e..408b8f7393 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -38,9 +38,16 @@ beforeEach(function (done) { }) afterEach(function (done) { - swarmA.closeListener() - swarmB.closeListener() - done() + // This should be 2, but for some reason + // that will fail in most of the tests + var c = new Counter(1, done) + + swarmA.closeListener(function () { + c.hit() + }) + swarmB.closeListener(function () { + c.hit() + }) }) experiment('BASICS', function () { @@ -174,8 +181,6 @@ experiment('IDENTIFY', function () { }) test('Attach Identify, open a stream, reuse stream', function (done) { - console.log('\n\n\n') - var protocol = '/sparkles/3.3.3' var identifyA = new Identify(swarmA, peerA) @@ -199,6 +204,34 @@ experiment('IDENTIFY', function () { identifyA.on('peer-update', function (answer) {}) }) + test('Attach Identify, reuse peer', function (done) { + var protocol = '/sparkles/3.3.3' + + var identifyA = new Identify(swarmA, peerA) + var identifyB = new Identify(swarmB, peerB) // eslint-disable-line no-unused-vars + + swarmA.registerHandler(protocol, function (stream) {}) + swarmB.registerHandler(protocol, function (stream) {}) + + var restartA = function (cb) { + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + + stream.end(cb) + }) + } + + restartA(function () { + identifyA.once('peer-update', function () { + expect(peerA.previousObservedAddrs.length).to.be.equal(1) + + var c = new Counter(2, done) + + swarmA.closeConns(c.hit.bind(c)) + swarmB.closeConns(c.hit.bind(c)) + }) + }) + }) }) experiment('HARDNESS', function () {}) From 139ac6e85219a27b557330435375c590cdaf6fd0 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 14 Sep 2015 12:06:19 +0100 Subject: [PATCH 050/634] to v4 --- .travis.yml | 3 +-- package.json | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b2bb6ed08..fdd1203c0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,7 @@ sudo: false language: node_js node_js: - - "iojs" - - "0.12" + - "4.0" # Make sure we have new NPM. before_install: diff --git a/package.json b/package.json index ee6c073ad0..6c836e89d1 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "codestyle", "test" ], + "engines" : { "node" : "^4.0.0" }, "devDependencies": { "code": "^1.4.1", "lab": "^5.13.0", From 326f56206cd7f15e68dcf4b184c441855199b041 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 14 Sep 2015 12:06:41 +0100 Subject: [PATCH 051/634] to v4 --- src/swarm.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swarm.js b/src/swarm.js index 89beaa1dfb..9a27bdad41 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -104,7 +104,7 @@ function Swarm () { conn: conn, socket: socket } - conn.on('close', function () { delete self.connections[peer.id.toB58String()] }) + conn.on('close', function () { delete self.connections[peer.id.toB58String()]}) errorUp(self, conn) cb() From 10ec005695d987df51c46771c96f9e2d817009a8 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 14 Sep 2015 12:06:55 +0100 Subject: [PATCH 052/634] Release v0.4.1. --- package.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 6c836e89d1..f59408d538 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-swarm", - "version": "0.4.0", + "version": "0.4.1", "description": "IPFS swarm implementation in Node.js", "main": "src/index.js", "scripts": { @@ -27,7 +27,9 @@ "codestyle", "test" ], - "engines" : { "node" : "^4.0.0" }, + "engines": { + "node": "^4.0.0" + }, "devDependencies": { "code": "^1.4.1", "lab": "^5.13.0", From 544e4a41653ffe7382f4db7d934289854a4f337a Mon Sep 17 00:00:00 2001 From: David Dias Date: Sun, 20 Sep 2015 21:08:28 +0100 Subject: [PATCH 053/634] update README with new candidate interface --- README.md | 79 +++++++++++++++++++++++-------------------------------- 1 file changed, 33 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index f585e5773f..4fcf3e6827 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,69 @@ -ipfs-swarm Node.js implementation +libp2p-swarm Node.js implementation ================================= [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) [![Build Status](https://img.shields.io/travis/diasdavid/node-ipfs-swarm/master.svg?style=flat-square)](https://travis-ci.org/diasdavid/node-ipfs-swarm) -> IPFS swarm implementation in Node.js +> libp2p swarm implementation in Node.js # Description -ipfs-swarm is an abstraction for the network layer on IPFS. It offers an API to open streams between peers on a specific protocol. +libp2p-swarm is connection abstraction that is able to leverage several transports and connection upgrades (such as congestion control, encrypt a channel, multiplex several streams in one connection, and more. It does this by bringing protocol multiplexing to the application level (instead of the traditional Port level) using multicodec and multistream. -Ref spec (WIP) - https://github.com/diasdavid/specs/blob/protocol-spec/protocol/layers.md#network-layer +libp2p-swarm is used by libp2p but it can be also used as a standalone module. # Usage -### Create a new Swarm +### Install and create a Swarm -```javascript -var Swarm = require('ipfs-swarm') +libp2p-swarm is available on npm and so, like any other npm module, just: -var s = new Swarm([port]) // `port` defalts to 4001 +```bash +$ npm install libp2p-swarm --save ``` -### Set the swarm to listen for incoming streams +And use it on your Node.js code as: -```javascript -s.listen([port], [callback]) // `port` defaults to 4001, `callback` gets called when the socket starts listening -``` - -### Close the listener/socket and every open stream that was multiplexed on it +```JavaScript +var Swarm = require('libp2p-swarm') -```javascript -s.closeListener() +var sw = new Swarm(peerInfoSelf) ``` -### Register a protocol to be handled by an incoming stream - -```javascript -s.registerHandler('/name/protocol/you/want/version', function (stream) {}) -``` +peerInfoSelf is a PeerInfo object that represents the peer creating this swarm instance. -### Open a new connection +### Support a transport -Used when we want to make sure we can connect to a given peer, but do not intend to establish a stream with any of the services offered right away. +libp2p-swarm expects transports that implement [abstract-transport](https://github.com/diasdavid/abstract-transport). For example [libp2p-tcp](https://github.com/diasdavid/node-libp2p-tcp), a simple shim on top of the `net` module to make it work with swarm expectations. -``` -s.openConnection(peerConnection, function (err) {}) +```JavaScript +sw.addTransport(transport, [options, dialOptions, listenOptions]) ``` +### Add an connection upgrade -### Dial a new stream +A connection upgrade must be able to receive and return something that implements the [abstract-connection]() interface. +```JavaScript +sw.addUpgrade(connUpgrade, [options]) ``` -s.openStream(peerInfo, protocol, function (err, stream) {}) -``` - -peerInfo must be a [`ipfs-peer`](https://www.npmjs.com/package/ipfs-peer) object, contaning both peer-id and multiaddrs. -## Events emitted +Upgrading a connection to use a Stream Muxer is still considered a upgrade, but a special case since once this connection is applied, the returned obj will implement the [abstract-stream-muxer]() interface. +```JavaScript +sw.addStreamMuxer(streamMuxer, [options]) ``` -.on('error') -.on('connection') -.on('connection-unknown') // used by Identify to start the Identify protocol from listener to dialer -``` - -## Identify protocol - -The Identify protocol is an integral part to Swarm. It enables peers to share observedAddrs, identities and other possible address available. This enables us to do better NAT traversal. - -To instantiate Identify: +### Dial to another peer +```JavaScript +sw.dial(PeerInfo, protocol, options) +sw.dial(PeerInfo, options) ``` -var Identify = require('ipfs-swarm/identify') -var i = new Identify(swarmInstance, peerSelf) -``` +dial uses the best transport (whatever works first, in the future we can have some criteria), and jump starts the connection until the point we have to negotiate the protocol. If a Muxer is available, then drop the muxer onto that connection. Good to warm up connections or to check for connectivity. If we have already a Muxer for that peerInfo, than do nothing. -`swarmInstance` must be an Instance of swarm and `peerSelf` must be a instance of `ipfs-peer` that represents the peer that instantiated this Identify +### Accept requests on a specific protocol -Identify emits a `peer-update` event each time it receives information from another peer. +```JavaScript +sw.handleProtocol(protocol, handlerFunction) +``` From 1833ded0f730686aebcba108c177ee31b74f9419 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 21 Sep 2015 16:46:04 +0100 Subject: [PATCH 054/634] making progress --- README.md | 2 +- examples/{c.js => peerA.js} | 0 examples/{s.js => peerB.js} | 5 +- src/swarm-old.js | 189 +++++++++++++++++++++++++++++++++ src/swarm.js | 201 +++++++++--------------------------- 5 files changed, 241 insertions(+), 156 deletions(-) rename examples/{c.js => peerA.js} (100%) rename examples/{s.js => peerB.js} (87%) create mode 100644 src/swarm-old.js diff --git a/README.md b/README.md index 4fcf3e6827..8910334387 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ sw.addStreamMuxer(streamMuxer, [options]) ### Dial to another peer ```JavaScript -sw.dial(PeerInfo, protocol, options) +sw.dial(PeerInfo, options, protocol) sw.dial(PeerInfo, options) ``` diff --git a/examples/c.js b/examples/peerA.js similarity index 100% rename from examples/c.js rename to examples/peerA.js diff --git a/examples/s.js b/examples/peerB.js similarity index 87% rename from examples/s.js rename to examples/peerB.js index 856730d060..c480983a36 100644 --- a/examples/s.js +++ b/examples/peerB.js @@ -1,6 +1,7 @@ -// var Identify = require('./../src/identify') var Swarm = require('./../src') -// var Peer = require('ipfs-peer') + + +var Peer = require('ipfs-peer') // var Id = require('ipfs-peer-id') // var multiaddr = require('multiaddr') diff --git a/src/swarm-old.js b/src/swarm-old.js new file mode 100644 index 0000000000..9a27bdad41 --- /dev/null +++ b/src/swarm-old.js @@ -0,0 +1,189 @@ +var tcp = require('net') +var Select = require('multistream-select').Select +var Interactive = require('multistream-select').Interactive +var Muxer = require('./stream-muxer') +var log = require('ipfs-logger').group('swarm') +var async = require('async') +var EventEmitter = require('events').EventEmitter +var util = require('util') + +exports = module.exports = Swarm + +util.inherits(Swarm, EventEmitter) + +function Swarm () { + var self = this + + if (!(self instanceof Swarm)) { + throw new Error('Swarm must be called with new') + } + + self.port = parseInt(process.env.IPFS_SWARM_PORT, 10) || 4001 + self.connections = {} // {peerIdB58: {conn: <>, socket: <>} + self.handles = {} + + // set the listener + + self.listen = function (port, ready) { + if (!ready) { + ready = function noop () {} + } + if (typeof port === 'function') { + ready = port + } else if (port) { self.port = port } + + // + + self.listener = tcp.createServer(function (socket) { + errorUp(self, socket) + var ms = new Select() + ms.handle(socket) + ms.addHandler('/spdy/3.1.0', function (ds) { + log.info('Negotiated spdy with incoming socket') + + var conn = new Muxer().attach(ds, true) + + // attach multistream handlers to incoming streams + + conn.on('stream', registerHandles) + errorUp(self, conn) + + // FOR IDENTIFY + self.emit('connection-unknown', conn, socket) + + // IDENTIFY DOES THIS FOR US + // conn.on('close', function () { delete self.connections[conn.peerId] }) + }) + }).listen(self.port, ready) + errorUp(self, self.listener) + } + + // interface + + // open stream account for connection reuse + self.openConnection = function (peer, cb) { + // If no connection open yet, open it + if (!self.connections[peer.id.toB58String()]) { + // Establish a socket with one of the addresses + var socket + async.eachSeries(peer.multiaddrs, function (multiaddr, next) { + if (socket) { return next() } + + var tmp = tcp.connect(multiaddr.toOptions(), function () { + socket = tmp + errorUp(self, socket) + next() + }) + + tmp.once('error', function (err) { + log.warn(multiaddr.toString(), 'on', peer.id.toB58String(), 'not available', err) + next() + }) + + }, function done () { + if (!socket) { + return cb(new Error('Not able to open a scoket with peer - ', + peer.id.toB58String())) + } + gotSocket(socket) + }) + } else { + cb() + } + + // do the spdy people dance (multistream-select into spdy) + function gotSocket (socket) { + var msi = new Interactive() + msi.handle(socket, function () { + msi.select('/spdy/3.1.0', function (err, ds) { + if (err) { cb(err) } + + var conn = new Muxer().attach(ds, false) + conn.on('stream', registerHandles) + self.connections[peer.id.toB58String()] = { + conn: conn, + socket: socket + } + conn.on('close', function () { delete self.connections[peer.id.toB58String()]}) + errorUp(self, conn) + + cb() + }) + }) + } + } + + self.openStream = function (peer, protocol, cb) { + self.openConnection(peer, function (err) { + if (err) { + return cb(err) + } + // spawn new muxed stream + var conn = self.connections[peer.id.toB58String()].conn + conn.dialStream(function (err, stream) { + if (err) { return cb(err) } + errorUp(self, stream) + // negotiate desired protocol + var msi = new Interactive() + msi.handle(stream, function () { + msi.select(protocol, function (err, ds) { + if (err) { return cb(err) } + peer.lastSeen = new Date() + cb(null, ds) // return the stream + }) + }) + }) + }) + } + + self.registerHandler = function (protocol, handlerFunc) { + if (self.handles[protocol]) { + return handlerFunc(new Error('Handle for protocol already exists', protocol)) + } + self.handles[protocol] = handlerFunc + log.info('Registered handler for protocol:', protocol) + } + + self.closeConns = function (cb) { + var keys = Object.keys(self.connections) + var number = keys.length + if (number === 0) { cb() } + var c = new Counter(number, cb) + + keys.forEach(function (key) { + self.connections[key].conn.end() + c.hit() + }) + } + + self.closeListener = function (cb) { + self.listener.close(cb) + } + + function registerHandles (stream) { + log.info('Registering protocol handlers on new stream') + errorUp(self, stream) + var msH = new Select() + msH.handle(stream) + Object.keys(self.handles).forEach(function (protocol) { + msH.addHandler(protocol, self.handles[protocol]) + }) + } + +} + +function errorUp (self, emitter) { + emitter.on('error', function (err) { + self.emit('error', err) + }) +} + +function Counter (target, callback) { + var c = 0 + this.hit = count + + function count () { + c += 1 + if (c === target) { callback() } + } +} diff --git a/src/swarm.js b/src/swarm.js index 9a27bdad41..4d792999a0 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -1,189 +1,84 @@ -var tcp = require('net') -var Select = require('multistream-select').Select -var Interactive = require('multistream-select').Interactive -var Muxer = require('./stream-muxer') -var log = require('ipfs-logger').group('swarm') -var async = require('async') -var EventEmitter = require('events').EventEmitter -var util = require('util') exports = module.exports = Swarm -util.inherits(Swarm, EventEmitter) - -function Swarm () { +function Swarm (peerInfo) { var self = this if (!(self instanceof Swarm)) { throw new Error('Swarm must be called with new') } - self.port = parseInt(process.env.IPFS_SWARM_PORT, 10) || 4001 - self.connections = {} // {peerIdB58: {conn: <>, socket: <>} - self.handles = {} - - // set the listener + self.peerInfo = peerInfo - self.listen = function (port, ready) { - if (!ready) { - ready = function noop () {} - } - if (typeof port === 'function') { - ready = port - } else if (port) { self.port = port } + // peerIdB58: { conn: } + self.conns = {} - // + // peerIdB58: { muxer: } + self.muxedConns = {} - self.listener = tcp.createServer(function (socket) { - errorUp(self, socket) - var ms = new Select() - ms.handle(socket) - ms.addHandler('/spdy/3.1.0', function (ds) { - log.info('Negotiated spdy with incoming socket') + // transportName: { transport: transport, + // dialOptions: dialOptions, + // listenOptions: listenOptions, + // listeners: [] } + self.transports = {} - var conn = new Muxer().attach(ds, true) + self.listeners = {} - // attach multistream handlers to incoming streams + // public interface - conn.on('stream', registerHandles) - errorUp(self, conn) + self.addTransport = function (transport, options, dialOptions, listenOptions, callback) { + // set up the transport and add the list of incoming streams + // add transport to the list of transports - // FOR IDENTIFY - self.emit('connection-unknown', conn, socket) - - // IDENTIFY DOES THIS FOR US - // conn.on('close', function () { delete self.connections[conn.peerId] }) - }) - }).listen(self.port, ready) - errorUp(self, self.listener) - } + var listener = transport.createListener(options, listen) - // interface - - // open stream account for connection reuse - self.openConnection = function (peer, cb) { - // If no connection open yet, open it - if (!self.connections[peer.id.toB58String()]) { - // Establish a socket with one of the addresses - var socket - async.eachSeries(peer.multiaddrs, function (multiaddr, next) { - if (socket) { return next() } - - var tmp = tcp.connect(multiaddr.toOptions(), function () { - socket = tmp - errorUp(self, socket) - next() - }) - - tmp.once('error', function (err) { - log.warn(multiaddr.toString(), 'on', peer.id.toB58String(), 'not available', err) - next() - }) - - }, function done () { - if (!socket) { - return cb(new Error('Not able to open a scoket with peer - ', - peer.id.toB58String())) - } - gotSocket(socket) - }) - } else { - cb() - } - - // do the spdy people dance (multistream-select into spdy) - function gotSocket (socket) { - var msi = new Interactive() - msi.handle(socket, function () { - msi.select('/spdy/3.1.0', function (err, ds) { - if (err) { cb(err) } - - var conn = new Muxer().attach(ds, false) - conn.on('stream', registerHandles) - self.connections[peer.id.toB58String()] = { - conn: conn, - socket: socket - } - conn.on('close', function () { delete self.connections[peer.id.toB58String()]}) - errorUp(self, conn) - - cb() - }) - }) - } - } + listener.listen(listenOptions, function ready () { + self.transports[options.name] = { + transport: transport, + dialOptions: dialOptions, + listenOptions: listenOptions, + listeners: [listener] + } - self.openStream = function (peer, protocol, cb) { - self.openConnection(peer, function (err) { - if (err) { - return cb(err) + // If a known multiaddr is passed, then add to our list of multiaddrs + if (options.multiaddr) { + self.peerInfo.multiaddrs.push(options.multiaddr) } - // spawn new muxed stream - var conn = self.connections[peer.id.toB58String()].conn - conn.dialStream(function (err, stream) { - if (err) { return cb(err) } - errorUp(self, stream) - // negotiate desired protocol - var msi = new Interactive() - msi.handle(stream, function () { - msi.select(protocol, function (err, ds) { - if (err) { return cb(err) } - peer.lastSeen = new Date() - cb(null, ds) // return the stream - }) - }) - }) + + callback() }) } - self.registerHandler = function (protocol, handlerFunc) { - if (self.handles[protocol]) { - return handlerFunc(new Error('Handle for protocol already exists', protocol)) - } - self.handles[protocol] = handlerFunc - log.info('Registered handler for protocol:', protocol) + self.addUpgrade = function (ConnUpgrade, options) { + } - self.closeConns = function (cb) { - var keys = Object.keys(self.connections) - var number = keys.length - if (number === 0) { cb() } - var c = new Counter(number, cb) + self.addStreamMuxer = function (StreamMuxer, options) { - keys.forEach(function (key) { - self.connections[key].conn.end() - c.hit() - }) } - self.closeListener = function (cb) { - self.listener.close(cb) + self.dial = function (peerInfo, options, protocol, callback) { + // 1. check if we have transports we support } - function registerHandles (stream) { - log.info('Registering protocol handlers on new stream') - errorUp(self, stream) - var msH = new Select() - msH.handle(stream) - Object.keys(self.handles).forEach(function (protocol) { - msH.addHandler(protocol, self.handles[protocol]) - }) + self.closeListener = function (transportName, callback) { + // close a specific listener + // remove it from available transports } -} + self.close = function (callback) { + // close everything + } -function errorUp (self, emitter) { - emitter.on('error', function (err) { - self.emit('error', err) - }) -} + self.handleProtocol = function (protocol, handlerFunction) { + + } -function Counter (target, callback) { - var c = 0 - this.hit = count + // internals - function count () { - c += 1 - if (c === target) { callback() } + function listen (conn) { + console.log('Received new connection') + // apply upgrades + // then add it to the multistreamHandler } } From 73a6a4fd4575abcefa396200d11e7656690c0592 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 21 Sep 2015 19:56:42 +0100 Subject: [PATCH 055/634] adding transports works --- examples/peerB.js | 33 +++++++++------------------------ package.json | 5 +++-- src/swarm.js | 21 +++++++-------------- 3 files changed, 19 insertions(+), 40 deletions(-) diff --git a/examples/peerB.js b/examples/peerB.js index c480983a36..95ca3ff620 100644 --- a/examples/peerB.js +++ b/examples/peerB.js @@ -1,29 +1,14 @@ var Swarm = require('./../src') +var Peer = require('peer-info') +var Id = require('peer-id') +var multiaddr = require('multiaddr') +var tcp = require('libp2p-tcp') -var Peer = require('ipfs-peer') -// var Id = require('ipfs-peer-id') -// var multiaddr = require('multiaddr') +var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') +var p = new Peer(Id.create(), []) +var sw = new Swarm(p) -var b = new Swarm() -b.port = 4001 -// var peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + b.port)]) - -// var i = new Identify(b, peerB) -// i.on('thenews', function (news) { -// console.log('such news') -// }) - -b.on('error', function (err) { - console.log(err) -}) - -b.listen() - -b.registerHandler('/ipfs/sparkles/1.2.3', function (stream) { - // if (err) { - // return console.log(err) - // } - - console.log('woop got a stream') +sw.addTransport('tcp', tcp, { multiaddr: mh }, {}, {port: 8010}, function () { + console.log('transport added') }) diff --git a/package.json b/package.json index f59408d538..4f5129bf57 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "devDependencies": { "code": "^1.4.1", "lab": "^5.13.0", + "libp2p-tcp": "^0.1.1", "precommit-hook": "^3.0.0", "sinon": "^1.15.4", "standard": "^4.5.2", @@ -42,11 +43,11 @@ "async": "^1.3.0", "ip-address": "^4.0.0", "ipfs-logger": "^0.1.0", - "ipfs-peer": "^0.3.0", - "ipfs-peer-id": "^0.3.0", "multiaddr": "^1.0.0", "multiplex-stream-muxer": "^0.2.0", "multistream-select": "^0.6.1", + "peer-id": "^0.3.3", + "peer-info": "^0.3.2", "protocol-buffers-stream": "^1.2.0", "spdy-stream-muxer": "^0.6.0" } diff --git a/src/swarm.js b/src/swarm.js index 4d792999a0..6c38ec09e5 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -1,4 +1,3 @@ - exports = module.exports = Swarm function Swarm (peerInfo) { @@ -26,14 +25,14 @@ function Swarm (peerInfo) { // public interface - self.addTransport = function (transport, options, dialOptions, listenOptions, callback) { + self.addTransport = function (name, transport, options, dialOptions, listenOptions, callback) { // set up the transport and add the list of incoming streams // add transport to the list of transports var listener = transport.createListener(options, listen) listener.listen(listenOptions, function ready () { - self.transports[options.name] = { + self.transports[name] = { transport: transport, dialOptions: dialOptions, listenOptions: listenOptions, @@ -49,13 +48,9 @@ function Swarm (peerInfo) { }) } - self.addUpgrade = function (ConnUpgrade, options) { - - } + self.addUpgrade = function (ConnUpgrade, options) {} - self.addStreamMuxer = function (StreamMuxer, options) { - - } + self.addStreamMuxer = function (StreamMuxer, options) {} self.dial = function (peerInfo, options, protocol, callback) { // 1. check if we have transports we support @@ -70,15 +65,13 @@ function Swarm (peerInfo) { // close everything } - self.handleProtocol = function (protocol, handlerFunction) { - - } + self.handleProtocol = function (protocol, handlerFunction) {} // internals function listen (conn) { console.log('Received new connection') - // apply upgrades - // then add it to the multistreamHandler + // apply upgrades + // then add it to the multistreamHandler } } From e1df0b9ecd70c2af78a4f25b2f58a6e7af122527 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 22 Sep 2015 14:31:23 +0100 Subject: [PATCH 056/634] add transport and close listener test --- package.json | 9 +- src/swarm.js | 44 ++- ...r-test.js => multistream-and-muxer-old.js} | 0 tests/swarm-old.js | 254 ++++++++++++++++++ tests/swarm-test.js | 105 +++++--- 5 files changed, 362 insertions(+), 50 deletions(-) rename tests/{multistream-and-muxer-test.js => multistream-and-muxer-old.js} (100%) create mode 100644 tests/swarm-old.js diff --git a/package.json b/package.json index 4f5129bf57..0532386605 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,13 @@ { - "name": "ipfs-swarm", + "name": "libp2p-swarm", "version": "0.4.1", "description": "IPFS swarm implementation in Node.js", "main": "src/index.js", "scripts": { "test": "./node_modules/.bin/lab tests/*-test.js", "coverage": "./node_modules/.bin/lab -t 100 tests/*-test.js", - "codestyle": "./node_modules/.bin/standard --format", - "lint": "./node_modules/.bin/standard", - "validate": "npm ls" + "laf": "./node_modules/.bin/standard --format", + "lint": "./node_modules/.bin/standard" }, "repository": { "type": "git", @@ -24,7 +23,7 @@ }, "homepage": "https://github.com/diasdavid/node-ipfs-swarm", "pre-commit": [ - "codestyle", + "lint", "test" ], "engines": { diff --git a/src/swarm.js b/src/swarm.js index 6c38ec09e5..7637f23bdd 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -1,3 +1,5 @@ +var multistream = require('multistream-select') + exports = module.exports = Swarm function Swarm (peerInfo) { @@ -23,6 +25,8 @@ function Swarm (peerInfo) { self.listeners = {} + self.protocols = {} + // public interface self.addTransport = function (name, transport, options, dialOptions, listenOptions, callback) { @@ -34,9 +38,10 @@ function Swarm (peerInfo) { listener.listen(listenOptions, function ready () { self.transports[name] = { transport: transport, + options: options, dialOptions: dialOptions, listenOptions: listenOptions, - listeners: [listener] + listener: listener } // If a known multiaddr is passed, then add to our list of multiaddrs @@ -48,30 +53,53 @@ function Swarm (peerInfo) { }) } - self.addUpgrade = function (ConnUpgrade, options) {} + self.addUpgrade = function (ConnUpgrade, options) { + + } - self.addStreamMuxer = function (StreamMuxer, options) {} + self.addStreamMuxer = function (StreamMuxer, options) { + + } self.dial = function (peerInfo, options, protocol, callback) { // 1. check if we have transports we support } self.closeListener = function (transportName, callback) { - // close a specific listener - // remove it from available transports + self.transports[transportName].listener.close(closed) + + // only gets called when all the streams on this transport are closed too + function closed () { + delete self.transports[transportName] + callback() + } } self.close = function (callback) { // close everything } - self.handleProtocol = function (protocol, handlerFunction) {} + self.handleProtocol = function (protocol, handlerFunction) { + self.protocols[protocol] = handlerFunction + } // internals function listen (conn) { console.log('Received new connection') - // apply upgrades - // then add it to the multistreamHandler + // TODO apply upgrades + // TODO then add StreamMuxer if available + + // if no stream muxer, then + userProtocolMuxer(conn) + } + + // Handle user given protocols + function userProtocolMuxer (conn) { + var msS = new multistream.Select() + msS.handle(conn) + Object.keys(self.protocols).forEach(function (protocol) { + msS.addHandler(protocol, self.protocols[protocol]) + }) } } diff --git a/tests/multistream-and-muxer-test.js b/tests/multistream-and-muxer-old.js similarity index 100% rename from tests/multistream-and-muxer-test.js rename to tests/multistream-and-muxer-old.js diff --git a/tests/swarm-old.js b/tests/swarm-old.js new file mode 100644 index 0000000000..408b8f7393 --- /dev/null +++ b/tests/swarm-old.js @@ -0,0 +1,254 @@ +var Lab = require('lab') +var Code = require('code') +var sinon = require('sinon') +var lab = exports.lab = Lab.script() + +var experiment = lab.experiment +var test = lab.test +var beforeEach = lab.beforeEach +var afterEach = lab.afterEach +var expect = Code.expect + +var multiaddr = require('multiaddr') +var Id = require('ipfs-peer-id') +var Peer = require('ipfs-peer') +var Swarm = require('../src/') +var Identify = require('../src/identify') + +var swarmA +var swarmB +var peerA +var peerB + +beforeEach(function (done) { + swarmA = new Swarm() + swarmB = new Swarm() + var c = new Counter(2, done) + + swarmA.listen(8100, function () { + peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmA.port)]) + c.hit() + }) + + swarmB.listen(8101, function () { + peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmB.port)]) + c.hit() + }) + +}) + +afterEach(function (done) { + // This should be 2, but for some reason + // that will fail in most of the tests + var c = new Counter(1, done) + + swarmA.closeListener(function () { + c.hit() + }) + swarmB.closeListener(function () { + c.hit() + }) +}) + +experiment('BASICS', function () { + experiment('Swarm', function () { + test('enforces instantiation with new', function (done) { + expect(function () { + Swarm() + }).to.throw('Swarm must be called with new') + done() + }) + + test('parses $IPFS_SWARM_PORT', function (done) { + process.env.IPFS_SWARM_PORT = 1111 + var swarm = new Swarm() + expect(swarm.port).to.be.equal(1111) + process.env.IPFS_SWARM_PORT = undefined + done() + }) + }) + + experiment('Swarm.listen', function (done) { + test('handles missing port', function (done) { + var swarm = new Swarm() + swarm.listen(done) + }) + + test('handles passed in port', function (done) { + var swarm = new Swarm() + swarm.listen(1234) + expect(swarm.port).to.be.equal(1234) + done() + }) + }) + + experiment('Swarm.registerHandler', function () { + test('throws when registering a protcol handler twice', function (done) { + var swarm = new Swarm() + swarm.registerHandler('/sparkles/1.1.1', function () {}) + swarm.registerHandler('/sparkles/1.1.1', function (err) { + expect(err).to.be.an.instanceOf(Error) + expect(err.message).to.be.equal('Handle for protocol already exists') + done() + }) + }) + }) + + experiment('Swarm.closeConns', function () { + test('calls end on all connections', function (done) { + swarmA.openConnection(peerB, function () { + var key = Object.keys(swarmA.connections)[0] + sinon.spy(swarmA.connections[key].conn, 'end') + swarmA.closeConns(function () { + expect(swarmA.connections[key].conn.end.called).to.be.equal(true) + done() + }) + }) + }) + }) +}) + +experiment('BASE', function () { + test('Open a stream', function (done) { + var protocol = '/sparkles/3.3.3' + var c = new Counter(2, done) + + swarmB.registerHandler(protocol, function (stream) { + c.hit() + }) + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + c.hit() + }) + }) + + test('Reuse connection (from dialer)', function (done) { + var protocol = '/sparkles/3.3.3' + + swarmB.registerHandler(protocol, function (stream) {}) + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + expect(swarmA.connections.length === 1) + done() + }) + }) + }) + test('Check for lastSeen', function (done) { + var protocol = '/sparkles/3.3.3' + + swarmB.registerHandler(protocol, function (stream) {}) + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + expect(peerB.lastSeen).to.be.instanceof(Date) + done() + }) + }) + +}) + +experiment('IDENTIFY', function () { + test('Attach Identify, open a stream, see a peer update', function (done) { + swarmA.on('error', function (err) { + console.log('A - ', err) + }) + + swarmB.on('error', function (err) { + console.log('B - ', err) + }) + + var protocol = '/sparkles/3.3.3' + + var identifyA = new Identify(swarmA, peerA) + var identifyB = new Identify(swarmB, peerB) + setTimeout(function () { + swarmB.registerHandler(protocol, function (stream) {}) + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + }) + + identifyB.on('peer-update', function (answer) { + done() + }) + identifyA.on('peer-update', function (answer) {}) + }, 500) + }) + + test('Attach Identify, open a stream, reuse stream', function (done) { + var protocol = '/sparkles/3.3.3' + + var identifyA = new Identify(swarmA, peerA) + var identifyB = new Identify(swarmB, peerB) + + swarmA.registerHandler(protocol, function (stream) {}) + swarmB.registerHandler(protocol, function (stream) {}) + + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + }) + + identifyB.on('peer-update', function (answer) { + expect(Object.keys(swarmB.connections).length).to.equal(1) + swarmB.openStream(peerA, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + expect(Object.keys(swarmB.connections).length).to.equal(1) + done() + }) + }) + identifyA.on('peer-update', function (answer) {}) + }) + + test('Attach Identify, reuse peer', function (done) { + var protocol = '/sparkles/3.3.3' + + var identifyA = new Identify(swarmA, peerA) + var identifyB = new Identify(swarmB, peerB) // eslint-disable-line no-unused-vars + + swarmA.registerHandler(protocol, function (stream) {}) + swarmB.registerHandler(protocol, function (stream) {}) + + var restartA = function (cb) { + swarmA.openStream(peerB, protocol, function (err, stream) { + expect(err).to.not.be.instanceof(Error) + + stream.end(cb) + }) + } + + restartA(function () { + identifyA.once('peer-update', function () { + expect(peerA.previousObservedAddrs.length).to.be.equal(1) + + var c = new Counter(2, done) + + swarmA.closeConns(c.hit.bind(c)) + swarmB.closeConns(c.hit.bind(c)) + }) + }) + }) +}) + +experiment('HARDNESS', function () {}) + +function Counter (target, callback) { + var c = 0 + this.hit = count + + function count () { + c += 1 + if (c === target) { + callback() + } + } +} + +// function checkErr (err) { +// console.log('err') +// expect(err).to.be.instanceof(Error) +// } diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 408b8f7393..6c014e9414 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -1,55 +1,85 @@ var Lab = require('lab') var Code = require('code') -var sinon = require('sinon') var lab = exports.lab = Lab.script() var experiment = lab.experiment var test = lab.test -var beforeEach = lab.beforeEach -var afterEach = lab.afterEach var expect = Code.expect var multiaddr = require('multiaddr') -var Id = require('ipfs-peer-id') -var Peer = require('ipfs-peer') -var Swarm = require('../src/') -var Identify = require('../src/identify') - -var swarmA -var swarmB -var peerA -var peerB - -beforeEach(function (done) { - swarmA = new Swarm() - swarmB = new Swarm() - var c = new Counter(2, done) - - swarmA.listen(8100, function () { - peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmA.port)]) - c.hit() - }) +var Id = require('peer-id') +var Peer = require('peer-info') +var Swarm = require('../src') +var tcp = require('libp2p-tcp') + +/* TODO +experiment('Basics', function () { + test('enforces creation with new', function (done) {done() }) +}) +*/ + +experiment('Without a Stream Muxer', function () { + experiment('tcp', function () { + test('add the transport', function (done) { + var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') + var p = new Peer(Id.create(), []) + var sw = new Swarm(p) + + sw.addTransport('tcp', tcp, + { multiaddr: mh }, {}, {port: 8010}, function () { + expect(sw.transports['tcp'].options).to.deep.equal({ multiaddr: mh }) + expect(sw.transports['tcp'].dialOptions).to.deep.equal({}) + expect(sw.transports['tcp'].listenOptions).to.deep.equal({port: 8010}) + expect(sw.transports['tcp'].transport).to.deep.equal(tcp) + sw.closeListener('tcp', function () { + done() + }) + }) + }) - swarmB.listen(8101, function () { - peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmB.port)]) - c.hit() + test('dial a conn', function (done) { + done() + }) + test('dial a conn on a protocol', function (done) { done() }) + test('add an upgrade', function (done) { done() }) + test('dial a conn on top of a upgrade', function (done) { done() }) + test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) }) + /* TODO + experiment('udp', function () { + test('add the transport', function (done) { done() }) + test('dial a conn', function (done) { done() }) + test('dial a conn on a protocol', function (done) { done() }) + test('add an upgrade', function (done) { done() }) + test('dial a conn on top of a upgrade', function (done) { done() }) + test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + }) */ + + /* TODO + experiment('udt', function () { + test('add the transport', function (done) { done() }) + test('dial a conn', function (done) { done() }) + test('dial a conn on a protocol', function (done) { done() }) + test('add an upgrade', function (done) { done() }) + test('dial a conn on top of a upgrade', function (done) { done() }) + test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + }) */ + + /* TODO + experiment('utp', function () { + test('add the transport', function (done) { done() }) + test('dial a conn', function (done) { done() }) + test('dial a conn on a protocol', function (done) { done() }) + test('add an upgrade', function (done) { done() }) + test('dial a conn on top of a upgrade', function (done) { done() }) + test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + }) */ }) -afterEach(function (done) { - // This should be 2, but for some reason - // that will fail in most of the tests - var c = new Counter(1, done) - - swarmA.closeListener(function () { - c.hit() - }) - swarmB.closeListener(function () { - c.hit() - }) -}) +experiment('With a SPDY Stream Muxer', function () {}) +/* OLD experiment('BASICS', function () { experiment('Swarm', function () { test('enforces instantiation with new', function (done) { @@ -247,6 +277,7 @@ function Counter (target, callback) { } } } +*/ // function checkErr (err) { // console.log('err') From 5e4cca52c0d1144a84e5ff7e9f6ad548a0868d61 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 22 Sep 2015 16:16:21 +0100 Subject: [PATCH 057/634] dial a conn + test --- src/swarm.js | 111 +++++++++++++++++++++++++++++++++++++++++++- tests/swarm-test.js | 25 +++++++++- 2 files changed, 134 insertions(+), 2 deletions(-) diff --git a/src/swarm.js b/src/swarm.js index 7637f23bdd..b2ce002461 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -1,4 +1,5 @@ var multistream = require('multistream-select') +var async = require('async') exports = module.exports = Swarm @@ -27,6 +28,8 @@ function Swarm (peerInfo) { self.protocols = {} + self.muxer + // public interface self.addTransport = function (name, transport, options, dialOptions, listenOptions, callback) { @@ -63,6 +66,112 @@ function Swarm (peerInfo) { self.dial = function (peerInfo, options, protocol, callback) { // 1. check if we have transports we support + // 2. check if we have a conn waiting + // 3. check if we have a stream muxer available + + if (typeof protocol === 'function') { + callback = protocol + protocol = undefined + } + + // check if a conn is waiting + // if it is and protocol was selected, jump into multistreamHandshake + // if it is and no protocol was selected, do nothing and call and empty callback + + if (self.conns[peerInfo.id.toB58String()]) { + if (protocol) { + multistreamHandshake(self.conns[peerInfo.id.toB58String()]) + self.conns[peerInfo.id.toB58String()] = undefined + return + } else { + return callback() + } + } + + // check if a stream muxer for this peer is available + if (self.muxedConns[peerInfo.id.toB58String()]) { + return openMuxedStream() + } + + // Creating a new conn with this peer routine + + // TODO - check if there is a preference in protocol to use on + // options.protocol + var supportedTransports = Object.keys(self.protocols) + var multiaddrs = peerInfo.multiaddrs.filter(function (multiaddr) { + return multiaddr.protoNames().some(function (proto) { + return supportedTransports.indexOf(proto) >= 0 + }) + }) + + var conn + + async.eachSeries(multiaddrs, function (multiaddr, next) { + if (conn) { + return next() + } + + var transportName = getTransportNameForMultiaddr(multiaddr) + var transport = self.transports[transportName] + var dialOptions = clone(transport.dialOptions) + dialOptions.ready = connected + + var connTry = transport.transport.dial(multiaddr, dialOptions) + + connTry.once('error', function (err) { + if (err) { + return // TODO handle error + } + next() // try next multiaddr + }) + + function connected () { + conn = connTry + next() + } + + function getTransportNameForMultiaddr (multiaddr) { + // this works for all those ip + transport + port tripplets + return multiaddr.protoNames()[1] + } + + function clone (obj) { + var target = {} + for (var i in obj) { + if (obj.hasOwnProperty(i)) { + target[i] = obj[i] + } + } + return target + } + }, done) + + function done () { + // TODO apply upgrades + // TODO apply stream muxer + // if no protocol is selected, save it in the pool + // if protocol is selected, multistream that protocol + + if (protocol) { + multistreamHandshake(conn) + } else { + self.conns[peerInfo.id.toB58String()] = conn + callback() + } + } + + function openMuxedStream () { + // 1. create a new stream on this muxedConn and pass that to + // multistreamHanshake + } + + function multistreamHandshake (conn) { + var msS = new multistream.Select() + msS.handle(conn) + self.protocols.forEach(function (protocol) { + msS.addHandler(protocol, self.protocols[protocol]) + }) + } } self.closeListener = function (transportName, callback) { @@ -88,7 +197,7 @@ function Swarm (peerInfo) { function listen (conn) { console.log('Received new connection') // TODO apply upgrades - // TODO then add StreamMuxer if available + // TODO then add StreamMuxer if available (and point streams from muxer to userProtocolMuxer) // if no stream muxer, then userProtocolMuxer(conn) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 6c014e9414..692ad9bee9 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -38,9 +38,32 @@ experiment('Without a Stream Muxer', function () { }) test('dial a conn', function (done) { - done() + var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + var p1 = new Peer(Id.create(), []) + var sw1 = new Swarm(p1) + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) + + var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + var p2 = new Peer(Id.create(), []) + var sw2 = new Swarm(p2) + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) + + var count = 0 + + function ready () { + count++ + if (count < 2) { + return + } + + sw1.dial(p2, {}, function () { + expect(Object.keys(sw1.conns).length).to.equal(1) + done() + }) + } }) test('dial a conn on a protocol', function (done) { done() }) + test('dial a protocol on a previous created conn', function (done) { done() }) test('add an upgrade', function (done) { done() }) test('dial a conn on top of a upgrade', function (done) { done() }) test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) From 8e8d8e9093aad0c4f961d5e589f4d1e9db977e75 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 22 Sep 2015 16:50:42 +0100 Subject: [PATCH 058/634] dial on a protocol + test --- src/swarm.js | 25 ++++++++++++---- tests/swarm-test.js | 71 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 86 insertions(+), 10 deletions(-) diff --git a/src/swarm.js b/src/swarm.js index b2ce002461..763b73b174 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -79,6 +79,7 @@ function Swarm (peerInfo) { // if it is and no protocol was selected, do nothing and call and empty callback if (self.conns[peerInfo.id.toB58String()]) { + console.log('Had conn warmed up') if (protocol) { multistreamHandshake(self.conns[peerInfo.id.toB58String()]) self.conns[peerInfo.id.toB58String()] = undefined @@ -97,7 +98,7 @@ function Swarm (peerInfo) { // TODO - check if there is a preference in protocol to use on // options.protocol - var supportedTransports = Object.keys(self.protocols) + var supportedTransports = Object.keys(self.transports) var multiaddrs = peerInfo.multiaddrs.filter(function (multiaddr) { return multiaddr.protoNames().some(function (proto) { return supportedTransports.indexOf(proto) >= 0 @@ -120,7 +121,7 @@ function Swarm (peerInfo) { connTry.once('error', function (err) { if (err) { - return // TODO handle error + return console.log(err) // TODO handle error } next() // try next multiaddr }) @@ -151,6 +152,9 @@ function Swarm (peerInfo) { // TODO apply stream muxer // if no protocol is selected, save it in the pool // if protocol is selected, multistream that protocol + if (!conn) { + callback(new Error('Unable to open a connection')) + } if (protocol) { multistreamHandshake(conn) @@ -166,10 +170,9 @@ function Swarm (peerInfo) { } function multistreamHandshake (conn) { - var msS = new multistream.Select() - msS.handle(conn) - self.protocols.forEach(function (protocol) { - msS.addHandler(protocol, self.protocols[protocol]) + var msI = new multistream.Interactive() + msI.handle(conn, function () { + msI.select(protocol, callback) }) } } @@ -184,6 +187,16 @@ function Swarm (peerInfo) { } } + self.closeConns = function (callback) { + // close warmed up cons so that the listener can gracefully exit + Object.keys(self.conns).forEach(function (conn) { + self.conns[conn].destroy() + }) + self.conns = {} + + callback() + } + self.close = function (callback) { // close everything } diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 692ad9bee9..658e2a3e47 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -48,6 +48,68 @@ experiment('Without a Stream Muxer', function () { var sw2 = new Swarm(p2) sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) + var readyCounter = 0 + + function ready () { + readyCounter++ + if (readyCounter < 2) { + return + } + + sw1.dial(p2, {}, function (err) { + expect(err).to.equal(undefined) + expect(Object.keys(sw1.conns).length).to.equal(1) + var cleaningCounter = 0 + sw1.closeConns(cleaningUp) + sw2.closeConns(cleaningUp) + + sw1.closeListener('tcp', cleaningUp) + sw2.closeListener('tcp', cleaningUp) + + function cleaningUp () { + cleaningCounter++ + if (cleaningCounter < 4) { + return + } + + done() + } + }) + } + }) + + test('dial a conn on a protocol', function (done) { + var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + var p1 = new Peer(Id.create(), []) + var sw1 = new Swarm(p1) + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) + + var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + var p2 = new Peer(Id.create(), []) + var sw2 = new Swarm(p2) + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) + + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + conn.end() + conn.on('end', function () { + var cleaningCounter = 0 + sw1.closeConns(cleaningUp) + sw2.closeConns(cleaningUp) + + sw1.closeListener('tcp', cleaningUp) + sw2.closeListener('tcp', cleaningUp) + + function cleaningUp () { + cleaningCounter++ + if (cleaningCounter < 4) { + return + } + + done() + } + }) + }) + var count = 0 function ready () { @@ -56,13 +118,14 @@ experiment('Without a Stream Muxer', function () { return } - sw1.dial(p2, {}, function () { - expect(Object.keys(sw1.conns).length).to.equal(1) - done() + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() }) } }) - test('dial a conn on a protocol', function (done) { done() }) + test('dial a protocol on a previous created conn', function (done) { done() }) test('add an upgrade', function (done) { done() }) test('dial a conn on top of a upgrade', function (done) { done() }) From 59b00f68864d6ec3a4b97fd03ad01d1daffef15e Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 22 Sep 2015 17:27:37 +0100 Subject: [PATCH 059/634] use warmed up connection + test --- src/swarm.js | 3 +-- tests/swarm-test.js | 54 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/swarm.js b/src/swarm.js index 763b73b174..2949c11e13 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -79,10 +79,10 @@ function Swarm (peerInfo) { // if it is and no protocol was selected, do nothing and call and empty callback if (self.conns[peerInfo.id.toB58String()]) { - console.log('Had conn warmed up') if (protocol) { multistreamHandshake(self.conns[peerInfo.id.toB58String()]) self.conns[peerInfo.id.toB58String()] = undefined + delete self.conns[peerInfo.id.toB58String()] return } else { return callback() @@ -208,7 +208,6 @@ function Swarm (peerInfo) { // internals function listen (conn) { - console.log('Received new connection') // TODO apply upgrades // TODO then add StreamMuxer if available (and point streams from muxer to userProtocolMuxer) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 658e2a3e47..129a716f94 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -126,7 +126,59 @@ experiment('Without a Stream Muxer', function () { } }) - test('dial a protocol on a previous created conn', function (done) { done() }) + test('dial a protocol on a previous created conn', function (done) { + var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + var p1 = new Peer(Id.create(), []) + var sw1 = new Swarm(p1) + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) + + var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + var p2 = new Peer(Id.create(), []) + var sw2 = new Swarm(p2) + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) + + var readyCounter = 0 + + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + conn.end() + conn.on('end', function () { + var cleaningCounter = 0 + sw1.closeConns(cleaningUp) + sw2.closeConns(cleaningUp) + + sw1.closeListener('tcp', cleaningUp) + sw2.closeListener('tcp', cleaningUp) + + function cleaningUp () { + cleaningCounter++ + if (cleaningCounter < 4) { + return + } + + done() + } + }) + }) + + function ready () { + readyCounter++ + if (readyCounter < 2) { + return + } + + sw1.dial(p2, {}, function (err) { + expect(err).to.equal(undefined) + expect(Object.keys(sw1.conns).length).to.equal(1) + + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) + }) + } + }) + test('add an upgrade', function (done) { done() }) test('dial a conn on top of a upgrade', function (done) { done() }) test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) From 416e107d641b1e4c4112c32b2b690e3e6a545324 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 22 Sep 2015 17:50:41 +0100 Subject: [PATCH 060/634] quick fix for travis --- src/swarm.js | 4 +++- tests/swarm-test.js | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/swarm.js b/src/swarm.js index 2949c11e13..099917b446 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -28,7 +28,9 @@ function Swarm (peerInfo) { self.protocols = {} - self.muxer + // muxerName: { muxer: muxer + // options: options } + self.muxers = {} // public interface diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 129a716f94..a5a13ea056 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -18,6 +18,11 @@ experiment('Basics', function () { }) */ +// because of Travis-CI +process.on('uncaughtException', function (err) { + console.log('Caught exception: ' + err) +}) + experiment('Without a Stream Muxer', function () { experiment('tcp', function () { test('add the transport', function (done) { From 0040be765df3aeee108acbb98fa85c8101c8c48e Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 17:11:32 +0100 Subject: [PATCH 061/634] add spdy + test --- package.json | 1 + src/swarm.js | 78 +++++++++++++++++++++++++++++++++++++-------- tests/swarm-test.js | 77 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 142 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 0532386605..df32523ad4 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "devDependencies": { "code": "^1.4.1", "lab": "^5.13.0", + "libp2p-spdy": "^0.1.0", "libp2p-tcp": "^0.1.1", "precommit-hook": "^3.0.0", "sinon": "^1.15.4", diff --git a/src/swarm.js b/src/swarm.js index 099917b446..0673c39144 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -24,11 +24,13 @@ function Swarm (peerInfo) { // listeners: [] } self.transports = {} + // transportName: listener self.listeners = {} + // protocolName: handlerFunc self.protocols = {} - // muxerName: { muxer: muxer + // muxerName: { Muxer: Muxer // Muxer is a constructor // options: options } self.muxers = {} @@ -62,8 +64,11 @@ function Swarm (peerInfo) { } - self.addStreamMuxer = function (StreamMuxer, options) { - + self.addStreamMuxer = function (name, StreamMuxer, options) { + self.muxers[name] = { + Muxer: StreamMuxer, + options: options + } } self.dial = function (peerInfo, options, protocol, callback) { @@ -82,7 +87,12 @@ function Swarm (peerInfo) { if (self.conns[peerInfo.id.toB58String()]) { if (protocol) { - multistreamHandshake(self.conns[peerInfo.id.toB58String()]) + if (self.muxers['spdy']) { + // TODO upgrade this conn to a muxer + console.log('TODO: upgrade a warm conn to muxer that was added after') + } else { + multistreamHandshake(self.conns[peerInfo.id.toB58String()]) + } self.conns[peerInfo.id.toB58String()] = undefined delete self.conns[peerInfo.id.toB58String()] return @@ -93,7 +103,11 @@ function Swarm (peerInfo) { // check if a stream muxer for this peer is available if (self.muxedConns[peerInfo.id.toB58String()]) { - return openMuxedStream() + if (protocol) { + return openMuxedStream(self.muxedConns[peerInfo.id.toB58String()]) + } else { + return callback() + } } // Creating a new conn with this peer routine @@ -151,24 +165,49 @@ function Swarm (peerInfo) { function done () { // TODO apply upgrades - // TODO apply stream muxer + // apply stream muxer // if no protocol is selected, save it in the pool // if protocol is selected, multistream that protocol if (!conn) { callback(new Error('Unable to open a connection')) } - if (protocol) { - multistreamHandshake(conn) + if (self.muxers['spdy']) { + var spdy = new self.muxers['spdy'].Muxer(self.muxers['spdy'].options) + spdy.attach(conn, false, function (err, muxer) { + if (err) { + return console.log(err) // TODO Treat error + } + + muxer.on('stream', userProtocolMuxer) + + self.muxedConns[peerInfo.id.toB58String()] = muxer + + if (protocol) { + openMuxedStream(muxer) + } else { + callback() + } + }) } else { - self.conns[peerInfo.id.toB58String()] = conn - callback() + if (protocol) { + multistreamHandshake(conn) + } else { + self.conns[peerInfo.id.toB58String()] = conn + callback() + } } } - function openMuxedStream () { + function openMuxedStream (muxer) { // 1. create a new stream on this muxedConn and pass that to // multistreamHanshake + muxer.dialStream(function (err, conn) { + if (err) { + return console.log(err) // TODO Treat error + } + multistreamHandshake(conn) + }) } function multistreamHandshake (conn) { @@ -213,8 +252,21 @@ function Swarm (peerInfo) { // TODO apply upgrades // TODO then add StreamMuxer if available (and point streams from muxer to userProtocolMuxer) - // if no stream muxer, then - userProtocolMuxer(conn) + if (self.muxers['spdy']) { + var spdy = new self.muxers['spdy'].Muxer(self.muxers['spdy'].options) + spdy.attach(conn, true, function (err, muxer) { + if (err) { + return console.log(err) // TODO treat error + } + + // TODO This muxer has to be identified! + + muxer.on('stream', userProtocolMuxer) + }) + } else { + // if no stream muxer, then + userProtocolMuxer(conn) + } } // Handle user given protocols diff --git a/tests/swarm-test.js b/tests/swarm-test.js index a5a13ea056..8fd474a7ac 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -11,6 +11,7 @@ var Id = require('peer-id') var Peer = require('peer-info') var Swarm = require('../src') var tcp = require('libp2p-tcp') +var Spdy = require('libp2p-spdy') /* TODO experiment('Basics', function () { @@ -220,7 +221,81 @@ experiment('Without a Stream Muxer', function () { }) */ }) -experiment('With a SPDY Stream Muxer', function () {}) +experiment('With a SPDY Stream Muxer', function () { + experiment('tcp', function () { + test('add Stream Muxer', function (done) { + // var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') + var p = new Peer(Id.create(), []) + var sw = new Swarm(p) + sw.addStreamMuxer('spdy', Spdy, {}) + + done() + }) + + test('dial a conn on a protocol', function (done) { + var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + var p1 = new Peer(Id.create(), []) + var sw1 = new Swarm(p1) + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) + sw1.addStreamMuxer('spdy', Spdy, {}) + + var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + var p2 = new Peer(Id.create(), []) + var sw2 = new Swarm(p2) + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) + sw2.addStreamMuxer('spdy', Spdy, {}) + + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + // formallity so that the conn starts flowing + conn.on('data', function (chunk) {}) + + conn.end() + conn.on('end', function () { + expect(Object.keys(sw1.muxedConns).length).to.equal(1) + expect(Object.keys(sw2.muxedConns).length).to.equal(0) + var cleaningCounter = 0 + sw1.closeConns(cleaningUp) + sw2.closeConns(cleaningUp) + + sw1.closeListener('tcp', cleaningUp) + sw2.closeListener('tcp', cleaningUp) + + function cleaningUp () { + cleaningCounter++ + // TODO FIX: here should be 4, but because super wrapping of + // streams, it makes it so hard to properly close the muxed + // streams - https://github.com/indutny/spdy-transport/issues/14 + if (cleaningCounter < 3) { + return + } + + done() + } + }) + }) + + var count = 0 + + function ready () { + count++ + if (count < 2) { + return + } + + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) + } + }) + test('dial two conns (transport reuse)', function (done) { + done() + }) + test('identify', function (done) { done() }) + }) +}) /* OLD experiment('BASICS', function () { From 168d01befd6ed8c5efadcf39a953c4800b128ab5 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 17:25:21 +0100 Subject: [PATCH 062/634] stream muxer for connection reuse test --- src/swarm.js | 3 + tests/swarm-test.js | 248 ++++++++++---------------------------------- 2 files changed, 58 insertions(+), 193 deletions(-) diff --git a/src/swarm.js b/src/swarm.js index 0673c39144..873bb8e956 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -34,6 +34,9 @@ function Swarm (peerInfo) { // options: options } self.muxers = {} + // for connection reuse + self.identify = false + // public interface self.addTransport = function (name, transport, options, dialOptions, listenOptions, callback) { diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 8fd474a7ac..6ea7a45b50 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -291,213 +291,75 @@ experiment('With a SPDY Stream Muxer', function () { } }) test('dial two conns (transport reuse)', function (done) { - done() - }) - test('identify', function (done) { done() }) - }) -}) + var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + var p1 = new Peer(Id.create(), []) + var sw1 = new Swarm(p1) + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) + sw1.addStreamMuxer('spdy', Spdy, {}) -/* OLD -experiment('BASICS', function () { - experiment('Swarm', function () { - test('enforces instantiation with new', function (done) { - expect(function () { - Swarm() - }).to.throw('Swarm must be called with new') - done() - }) + var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + var p2 = new Peer(Id.create(), []) + var sw2 = new Swarm(p2) + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) + sw2.addStreamMuxer('spdy', Spdy, {}) - test('parses $IPFS_SWARM_PORT', function (done) { - process.env.IPFS_SWARM_PORT = 1111 - var swarm = new Swarm() - expect(swarm.port).to.be.equal(1111) - process.env.IPFS_SWARM_PORT = undefined - done() - }) - }) + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + // formallity so that the conn starts flowing + conn.on('data', function (chunk) {}) - experiment('Swarm.listen', function (done) { - test('handles missing port', function (done) { - var swarm = new Swarm() - swarm.listen(done) - }) + conn.end() + conn.on('end', function () { + expect(Object.keys(sw1.muxedConns).length).to.equal(1) + expect(Object.keys(sw2.muxedConns).length).to.equal(0) + conn.end() - test('handles passed in port', function (done) { - var swarm = new Swarm() - swarm.listen(1234) - expect(swarm.port).to.be.equal(1234) - done() - }) - }) + var cleaningCounter = 0 + sw1.closeConns(cleaningUp) + sw2.closeConns(cleaningUp) - experiment('Swarm.registerHandler', function () { - test('throws when registering a protcol handler twice', function (done) { - var swarm = new Swarm() - swarm.registerHandler('/sparkles/1.1.1', function () {}) - swarm.registerHandler('/sparkles/1.1.1', function (err) { - expect(err).to.be.an.instanceOf(Error) - expect(err.message).to.be.equal('Handle for protocol already exists') - done() - }) - }) - }) + sw1.closeListener('tcp', cleaningUp) + sw2.closeListener('tcp', cleaningUp) - experiment('Swarm.closeConns', function () { - test('calls end on all connections', function (done) { - swarmA.openConnection(peerB, function () { - var key = Object.keys(swarmA.connections)[0] - sinon.spy(swarmA.connections[key].conn, 'end') - swarmA.closeConns(function () { - expect(swarmA.connections[key].conn.end.called).to.be.equal(true) - done() + function cleaningUp () { + cleaningCounter++ + // TODO FIX: here should be 4, but because super wrapping of + // streams, it makes it so hard to properly close the muxed + // streams - https://github.com/indutny/spdy-transport/issues/14 + if (cleaningCounter < 3) { + return + } + + done() + } }) }) - }) - }) -}) -experiment('BASE', function () { - test('Open a stream', function (done) { - var protocol = '/sparkles/3.3.3' - var c = new Counter(2, done) - - swarmB.registerHandler(protocol, function (stream) { - c.hit() - }) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - c.hit() - }) - }) - - test('Reuse connection (from dialer)', function (done) { - var protocol = '/sparkles/3.3.3' + var count = 0 - swarmB.registerHandler(protocol, function (stream) {}) + function ready () { + count++ + if (count < 2) { + return + } - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + // TODO Improve clarity + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - expect(swarmA.connections.length === 1) - done() - }) + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) + } }) - }) - test('Check for lastSeen', function (done) { - var protocol = '/sparkles/3.3.3' - - swarmB.registerHandler(protocol, function (stream) {}) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - expect(peerB.lastSeen).to.be.instanceof(Date) + test('identify', function (done) { done() }) }) - -}) - -experiment('IDENTIFY', function () { - test('Attach Identify, open a stream, see a peer update', function (done) { - swarmA.on('error', function (err) { - console.log('A - ', err) - }) - - swarmB.on('error', function (err) { - console.log('B - ', err) - }) - - var protocol = '/sparkles/3.3.3' - - var identifyA = new Identify(swarmA, peerA) - var identifyB = new Identify(swarmB, peerB) - setTimeout(function () { - swarmB.registerHandler(protocol, function (stream) {}) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - }) - - identifyB.on('peer-update', function (answer) { - done() - }) - identifyA.on('peer-update', function (answer) {}) - }, 500) - }) - - test('Attach Identify, open a stream, reuse stream', function (done) { - var protocol = '/sparkles/3.3.3' - - var identifyA = new Identify(swarmA, peerA) - var identifyB = new Identify(swarmB, peerB) - - swarmA.registerHandler(protocol, function (stream) {}) - swarmB.registerHandler(protocol, function (stream) {}) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - }) - - identifyB.on('peer-update', function (answer) { - expect(Object.keys(swarmB.connections).length).to.equal(1) - swarmB.openStream(peerA, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - expect(Object.keys(swarmB.connections).length).to.equal(1) - done() - }) - }) - identifyA.on('peer-update', function (answer) {}) - }) - - test('Attach Identify, reuse peer', function (done) { - var protocol = '/sparkles/3.3.3' - - var identifyA = new Identify(swarmA, peerA) - var identifyB = new Identify(swarmB, peerB) // eslint-disable-line no-unused-vars - - swarmA.registerHandler(protocol, function (stream) {}) - swarmB.registerHandler(protocol, function (stream) {}) - - var restartA = function (cb) { - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - - stream.end(cb) - }) - } - - restartA(function () { - identifyA.once('peer-update', function () { - expect(peerA.previousObservedAddrs.length).to.be.equal(1) - - var c = new Counter(2, done) - - swarmA.closeConns(c.hit.bind(c)) - swarmB.closeConns(c.hit.bind(c)) - }) - }) - }) }) - -experiment('HARDNESS', function () {}) - -function Counter (target, callback) { - var c = 0 - this.hit = count - - function count () { - c += 1 - if (c === target) { - callback() - } - } -} -*/ - -// function checkErr (err) { -// console.log('err') -// expect(err).to.be.instanceof(Error) -// } From 0bcbe63005a20e006403c03b991b6a4cf30798fa Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 17:26:26 +0100 Subject: [PATCH 063/634] rm old code --- src/stream-muxer.js | 2 - src/swarm-old.js | 189 -------------------------------------------- 2 files changed, 191 deletions(-) delete mode 100644 src/stream-muxer.js delete mode 100644 src/swarm-old.js diff --git a/src/stream-muxer.js b/src/stream-muxer.js deleted file mode 100644 index 346371fb30..0000000000 --- a/src/stream-muxer.js +++ /dev/null @@ -1,2 +0,0 @@ -exports = module.exports = require('spdy-stream-muxer') -// exports = module.exports = require('multiplex-stream-muxer') diff --git a/src/swarm-old.js b/src/swarm-old.js deleted file mode 100644 index 9a27bdad41..0000000000 --- a/src/swarm-old.js +++ /dev/null @@ -1,189 +0,0 @@ -var tcp = require('net') -var Select = require('multistream-select').Select -var Interactive = require('multistream-select').Interactive -var Muxer = require('./stream-muxer') -var log = require('ipfs-logger').group('swarm') -var async = require('async') -var EventEmitter = require('events').EventEmitter -var util = require('util') - -exports = module.exports = Swarm - -util.inherits(Swarm, EventEmitter) - -function Swarm () { - var self = this - - if (!(self instanceof Swarm)) { - throw new Error('Swarm must be called with new') - } - - self.port = parseInt(process.env.IPFS_SWARM_PORT, 10) || 4001 - self.connections = {} // {peerIdB58: {conn: <>, socket: <>} - self.handles = {} - - // set the listener - - self.listen = function (port, ready) { - if (!ready) { - ready = function noop () {} - } - if (typeof port === 'function') { - ready = port - } else if (port) { self.port = port } - - // - - self.listener = tcp.createServer(function (socket) { - errorUp(self, socket) - var ms = new Select() - ms.handle(socket) - ms.addHandler('/spdy/3.1.0', function (ds) { - log.info('Negotiated spdy with incoming socket') - - var conn = new Muxer().attach(ds, true) - - // attach multistream handlers to incoming streams - - conn.on('stream', registerHandles) - errorUp(self, conn) - - // FOR IDENTIFY - self.emit('connection-unknown', conn, socket) - - // IDENTIFY DOES THIS FOR US - // conn.on('close', function () { delete self.connections[conn.peerId] }) - }) - }).listen(self.port, ready) - errorUp(self, self.listener) - } - - // interface - - // open stream account for connection reuse - self.openConnection = function (peer, cb) { - // If no connection open yet, open it - if (!self.connections[peer.id.toB58String()]) { - // Establish a socket with one of the addresses - var socket - async.eachSeries(peer.multiaddrs, function (multiaddr, next) { - if (socket) { return next() } - - var tmp = tcp.connect(multiaddr.toOptions(), function () { - socket = tmp - errorUp(self, socket) - next() - }) - - tmp.once('error', function (err) { - log.warn(multiaddr.toString(), 'on', peer.id.toB58String(), 'not available', err) - next() - }) - - }, function done () { - if (!socket) { - return cb(new Error('Not able to open a scoket with peer - ', - peer.id.toB58String())) - } - gotSocket(socket) - }) - } else { - cb() - } - - // do the spdy people dance (multistream-select into spdy) - function gotSocket (socket) { - var msi = new Interactive() - msi.handle(socket, function () { - msi.select('/spdy/3.1.0', function (err, ds) { - if (err) { cb(err) } - - var conn = new Muxer().attach(ds, false) - conn.on('stream', registerHandles) - self.connections[peer.id.toB58String()] = { - conn: conn, - socket: socket - } - conn.on('close', function () { delete self.connections[peer.id.toB58String()]}) - errorUp(self, conn) - - cb() - }) - }) - } - } - - self.openStream = function (peer, protocol, cb) { - self.openConnection(peer, function (err) { - if (err) { - return cb(err) - } - // spawn new muxed stream - var conn = self.connections[peer.id.toB58String()].conn - conn.dialStream(function (err, stream) { - if (err) { return cb(err) } - errorUp(self, stream) - // negotiate desired protocol - var msi = new Interactive() - msi.handle(stream, function () { - msi.select(protocol, function (err, ds) { - if (err) { return cb(err) } - peer.lastSeen = new Date() - cb(null, ds) // return the stream - }) - }) - }) - }) - } - - self.registerHandler = function (protocol, handlerFunc) { - if (self.handles[protocol]) { - return handlerFunc(new Error('Handle for protocol already exists', protocol)) - } - self.handles[protocol] = handlerFunc - log.info('Registered handler for protocol:', protocol) - } - - self.closeConns = function (cb) { - var keys = Object.keys(self.connections) - var number = keys.length - if (number === 0) { cb() } - var c = new Counter(number, cb) - - keys.forEach(function (key) { - self.connections[key].conn.end() - c.hit() - }) - } - - self.closeListener = function (cb) { - self.listener.close(cb) - } - - function registerHandles (stream) { - log.info('Registering protocol handlers on new stream') - errorUp(self, stream) - var msH = new Select() - msH.handle(stream) - Object.keys(self.handles).forEach(function (protocol) { - msH.addHandler(protocol, self.handles[protocol]) - }) - } - -} - -function errorUp (self, emitter) { - emitter.on('error', function (err) { - self.emit('error', err) - }) -} - -function Counter (target, callback) { - var c = 0 - this.hit = count - - function count () { - c += 1 - if (c === target) { callback() } - } -} From 2000827273efc7b40794be9ab6f47cc0c3ffe3d7 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 19:14:29 +0100 Subject: [PATCH 064/634] add identify --- src/identify/index.js | 142 ++++++++++++++++++++++++------------------ src/swarm.js | 41 ++++++++++-- tests/swarm-test.js | 63 ++++++++++++++++++- 3 files changed, 179 insertions(+), 67 deletions(-) diff --git a/src/identify/index.js b/src/identify/index.js index 08a09bb45c..a37b4a04a3 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -4,93 +4,111 @@ */ var Interactive = require('multistream-select').Interactive -var EventEmmiter = require('events').EventEmitter -var util = require('util') var protobufs = require('protocol-buffers-stream') var fs = require('fs') var schema = fs.readFileSync(__dirname + '/identify.proto') var v6 = require('ip-address').v6 -var Id = require('ipfs-peer-id') +var Id = require('peer-id') var multiaddr = require('multiaddr') -exports = module.exports = Identify +exports = module.exports = identify -util.inherits(Identify, EventEmmiter) +var protoId = '/ipfs/identify/1.0.0' -function Identify (swarm, peerSelf) { - var self = this - self.createProtoStream = protobufs(schema) +exports.protoId = protoId +var createProtoStream = protobufs(schema) - swarm.registerHandler('/ipfs/identify/1.0.0', function (stream) { - var ps = self.createProtoStream() +function identify (muxedConns, peerInfoSelf, socket, conn, muxer) { + var msi = new Interactive() + msi.handle(conn, function () { + msi.select(protoId, function (err, ds) { + if (err) { + return console.log(err) // TODO Treat error + } - ps.on('identify', function (msg) { - updateSelf(peerSelf, msg.observedAddr) + var ps = createProtoStream() - var peerId = Id.createFromPubKey(msg.publicKey) + ps.on('identify', function (msg) { + var peerId = Id.createFromPubKey(msg.publicKey) + + updateSelf(peerInfoSelf, msg.observedAddr) + + muxedConns[peerId.toB58String()] = { + muxer: muxer, + socket: socket + } + console.log('do I get back') + + // TODO: Pass the new discovered info about the peer that contacted us + // to something like the Kademlia Router, so the peerInfo for this peer + // is fresh + // - before this was exectued through a event emitter + // self.emit('peer-update', { + // peerId: peerId, + // listenAddrs: msg.listenAddrs.map(function (mhb) {return multiaddr(mhb)}) + // }) + }) - var socket = swarm.connections[peerId.toB58String()].socket var mh = getMultiaddr(socket) + ps.identify({ protocolVersion: 'na', agentVersion: 'na', - publicKey: peerSelf.id.pubKey, - listenAddrs: peerSelf.multiaddrs.map(function (mh) {return mh.buffer}), + publicKey: peerInfoSelf.id.pubKey, + listenAddrs: peerInfoSelf.multiaddrs.map(function (mh) { + return mh.buffer + }), observedAddr: mh.buffer }) - self.emit('peer-update', { - peerId: peerId, - listenAddrs: msg.listenAddrs.map(function (mhb) {return multiaddr(mhb)}) - }) - + ps.pipe(ds).pipe(ps) ps.finalize() }) - ps.pipe(stream).pipe(ps) }) +} + +exports.getHandlerFunction = function (peerInfoSelf, muxedConns) { + return function (conn) { + // wait for the other peer to identify itself + // update our multiaddr with observed addr list + // then get the socket from our list of muxedConns and send the reply back + + var ps = createProtoStream() + + ps.on('identify', function (msg) { + updateSelf(peerInfoSelf, msg.observedAddr) - swarm.on('connection-unknown', function (conn, socket) { - conn.dialStream(function (err, stream) { - if (err) { return console.log(err) } - var msi = new Interactive() - msi.handle(stream, function () { - msi.select('/ipfs/identify/1.0.0', function (err, ds) { - if (err) { return console.log(err) } - - var ps = self.createProtoStream() - - ps.on('identify', function (msg) { - var peerId = Id.createFromPubKey(msg.publicKey) - - updateSelf(peerSelf, msg.observedAddr) - - swarm.connections[peerId.toB58String()] = { - conn: conn, - socket: socket - } - - self.emit('peer-update', { - peerId: peerId, - listenAddrs: msg.listenAddrs.map(function (mhb) {return multiaddr(mhb)}) - }) - }) - - var mh = getMultiaddr(socket) - - ps.identify({ - protocolVersion: 'na', - agentVersion: 'na', - publicKey: peerSelf.id.pubKey, - listenAddrs: peerSelf.multiaddrs.map(function (mh) {return mh.buffer}), - observedAddr: mh.buffer - }) - - ps.pipe(ds).pipe(ps) - ps.finalize() - }) + var peerId = Id.createFromPubKey(msg.publicKey) + + var socket = muxedConns[peerId.toB58String()].socket + + var mh = getMultiaddr(socket) + + ps.identify({ + protocolVersion: 'na', + agentVersion: 'na', + publicKey: peerInfoSelf.id.pubKey, + listenAddrs: peerInfoSelf.multiaddrs.map(function (mh) { + return mh.buffer + }), + observedAddr: mh.buffer }) + + // TODO: Pass the new discovered info about the peer that contacted us + // to something like the Kademlia Router, so the peerInfo for this peer + // is fresh + // - before this was exectued through a event emitter + // self.emit('peer-update', { + // peerId: peerId, + // listenAddrs: msg.listenAddrs.map(function (mhb) { + // return multiaddr(mhb) + // }) + // }) + + ps.finalize() }) - }) + ps.pipe(conn).pipe(ps) + } } function getMultiaddr (socket) { diff --git a/src/swarm.js b/src/swarm.js index 873bb8e956..7284738b62 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -1,5 +1,6 @@ var multistream = require('multistream-select') var async = require('async') +var identify = require('./identify') exports = module.exports = Swarm @@ -15,7 +16,10 @@ function Swarm (peerInfo) { // peerIdB58: { conn: } self.conns = {} - // peerIdB58: { muxer: } + // peerIdB58: { + // muxer: , + // socket: socket // so we can extract the info we need for identify + // } self.muxedConns = {} // transportName: { transport: transport, @@ -107,7 +111,7 @@ function Swarm (peerInfo) { // check if a stream muxer for this peer is available if (self.muxedConns[peerInfo.id.toB58String()]) { if (protocol) { - return openMuxedStream(self.muxedConns[peerInfo.id.toB58String()]) + return openMuxedStream(self.muxedConns[peerInfo.id.toB58String()].muxer) } else { return callback() } @@ -184,7 +188,10 @@ function Swarm (peerInfo) { muxer.on('stream', userProtocolMuxer) - self.muxedConns[peerInfo.id.toB58String()] = muxer + self.muxedConns[peerInfo.id.toB58String()] = { + muxer: muxer, + socket: conn + } if (protocol) { openMuxedStream(muxer) @@ -245,6 +252,17 @@ function Swarm (peerInfo) { // close everything } + self.enableIdentify = function () { + // set flag to true + // add identify to the list of handled protocols + self.identify = true + + // we pass muxedConns so that identify can access the socket of the other + // peer + self.handleProtocol(identify.protoId, + identify.getHandlerFunction(self.peerInfo, self.muxedConns)) + } + self.handleProtocol = function (protocol, handlerFunction) { self.protocols[protocol] = handlerFunction } @@ -253,7 +271,7 @@ function Swarm (peerInfo) { function listen (conn) { // TODO apply upgrades - // TODO then add StreamMuxer if available (and point streams from muxer to userProtocolMuxer) + // add StreamMuxer if available (and point streams from muxer to userProtocolMuxer) if (self.muxers['spdy']) { var spdy = new self.muxers['spdy'].Muxer(self.muxers['spdy'].options) @@ -263,6 +281,21 @@ function Swarm (peerInfo) { } // TODO This muxer has to be identified! + // pass to identify a reference of + // our muxedConn list + // ourselves (peerInfo) + // the conn, which is the socket + // and a stream it can send stuff + if (self.identify) { + muxer.dialStream(function (err, stream) { + if (err) { + return console.log(err) // TODO Treat error + } + // conn === socket at this point + console.log('bimbas') + identify(self.muxedConns, self.peerInfo, conn, stream, muxer) + }) + } muxer.on('stream', userProtocolMuxer) }) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 6ea7a45b50..ddf64824ad 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -358,8 +358,69 @@ experiment('With a SPDY Stream Muxer', function () { }) } }) + test('identify', function (done) { - done() + var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + var p1 = new Peer(Id.create(), []) + var sw1 = new Swarm(p1) + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) + sw1.addStreamMuxer('spdy', Spdy, {}) + sw1.enableIdentify() + + var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + var p2 = new Peer(Id.create(), []) + var sw2 = new Swarm(p2) + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) + sw2.addStreamMuxer('spdy', Spdy, {}) + sw2.enableIdentify() + + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + // formallity so that the conn starts flowing + conn.on('data', function (chunk) {}) + + conn.end() + conn.on('end', function () { + expect(Object.keys(sw1.muxedConns).length).to.equal(1) + + var cleaningCounter = 0 + sw1.closeConns(cleaningUp) + sw2.closeConns(cleaningUp) + + sw1.closeListener('tcp', cleaningUp) + sw2.closeListener('tcp', cleaningUp) + + function cleaningUp () { + cleaningCounter++ + // TODO FIX: here should be 4, but because super wrapping of + // streams, it makes it so hard to properly close the muxed + // streams - https://github.com/indutny/spdy-transport/issues/14 + if (cleaningCounter < 3) { + return + } + // give time for identify to finish + setTimeout(function () { + expect(Object.keys(sw2.muxedConns).length).to.equal(1) + done() + }, 500) + } + }) + }) + + var count = 0 + + function ready () { + count++ + if (count < 2) { + return + } + + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) + } }) }) }) From fb37b4dec91e684f5a63c2c89bd0bd1a70061d3f Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 19:57:37 +0100 Subject: [PATCH 065/634] clear unused console logs --- src/identify/index.js | 1 - src/swarm.js | 1 - 2 files changed, 2 deletions(-) diff --git a/src/identify/index.js b/src/identify/index.js index a37b4a04a3..181cd3808a 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -37,7 +37,6 @@ function identify (muxedConns, peerInfoSelf, socket, conn, muxer) { muxer: muxer, socket: socket } - console.log('do I get back') // TODO: Pass the new discovered info about the peer that contacted us // to something like the Kademlia Router, so the peerInfo for this peer diff --git a/src/swarm.js b/src/swarm.js index 7284738b62..1cfff09509 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -292,7 +292,6 @@ function Swarm (peerInfo) { return console.log(err) // TODO Treat error } // conn === socket at this point - console.log('bimbas') identify(self.muxedConns, self.peerInfo, conn, stream, muxer) }) } From e6bcde41fb3aa203f1d57d615b2735285ac4776c Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 19:58:14 +0100 Subject: [PATCH 066/634] change cov --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index df32523ad4..d5d93a269c 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "src/index.js", "scripts": { "test": "./node_modules/.bin/lab tests/*-test.js", - "coverage": "./node_modules/.bin/lab -t 100 tests/*-test.js", + "coverage": "./node_modules/.bin/lab -t 88 tests/*-test.js", "laf": "./node_modules/.bin/standard --format", "lint": "./node_modules/.bin/standard" }, @@ -24,7 +24,8 @@ "homepage": "https://github.com/diasdavid/node-ipfs-swarm", "pre-commit": [ "lint", - "test" + "test", + "coverage" ], "engines": { "node": "^4.0.0" From 92b499df822a39fbcb0b7d88692eb15fa1b9c981 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 20:06:10 +0100 Subject: [PATCH 067/634] fix readme typos and missing links --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8910334387..7a87626e2a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ libp2p-swarm Node.js implementation # Description -libp2p-swarm is connection abstraction that is able to leverage several transports and connection upgrades (such as congestion control, encrypt a channel, multiplex several streams in one connection, and more. It does this by bringing protocol multiplexing to the application level (instead of the traditional Port level) using multicodec and multistream. +libp2p-swarm is a connection abstraction that is able to leverage several transports and connection upgrades, such as congestion control, channel encryption, multiplexing several streams in one connection, and more. It does this by bringing protocol multiplexing to the application level (instead of the traditional Port level) using multicodec and multistream. libp2p-swarm is used by libp2p but it can be also used as a standalone module. @@ -21,7 +21,7 @@ libp2p-swarm is available on npm and so, like any other npm module, just: $ npm install libp2p-swarm --save ``` -And use it on your Node.js code as: +And use it in your Node.js code as: ```JavaScript var Swarm = require('libp2p-swarm') @@ -29,7 +29,7 @@ var Swarm = require('libp2p-swarm') var sw = new Swarm(peerInfoSelf) ``` -peerInfoSelf is a PeerInfo object that represents the peer creating this swarm instance. +peerInfoSelf is a [PeerInfo](https://github.com/diasdavid/node-peer-info) object that represents the peer creating this swarm instance. ### Support a transport @@ -39,15 +39,15 @@ libp2p-swarm expects transports that implement [abstract-transport](https://gith sw.addTransport(transport, [options, dialOptions, listenOptions]) ``` -### Add an connection upgrade +### Add a connection upgrade -A connection upgrade must be able to receive and return something that implements the [abstract-connection]() interface. +A connection upgrade must be able to receive and return something that implements the [abstract-connection](https://github.com/diasdavid/abstract-connection) interface. ```JavaScript sw.addUpgrade(connUpgrade, [options]) ``` -Upgrading a connection to use a Stream Muxer is still considered a upgrade, but a special case since once this connection is applied, the returned obj will implement the [abstract-stream-muxer]() interface. +Upgrading a connection to use a stream muxer is still considered an upgrade, but a special case since once this connection is applied, the returned obj will implement the [abstract-stream-muxer](https://github.com/diasdavid/abstract-stream-muxer) interface. ```JavaScript sw.addStreamMuxer(streamMuxer, [options]) @@ -60,7 +60,7 @@ sw.dial(PeerInfo, options, protocol) sw.dial(PeerInfo, options) ``` -dial uses the best transport (whatever works first, in the future we can have some criteria), and jump starts the connection until the point we have to negotiate the protocol. If a Muxer is available, then drop the muxer onto that connection. Good to warm up connections or to check for connectivity. If we have already a Muxer for that peerInfo, than do nothing. +dial uses the best transport (whatever works first, in the future we can have some criteria), and jump starts the connection until the point we have to negotiate the protocol. If a muxer is available, then drop the muxer onto that connection. Good to warm up connections or to check for connectivity. If we have already a muxer for that peerInfo, than do nothing. ### Accept requests on a specific protocol From 5fe94446d83e48867e4e4387b662bb5f3395a3ef Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 20:07:05 +0100 Subject: [PATCH 068/634] rm old test file --- tests/swarm-old.js | 254 --------------------------------------------- 1 file changed, 254 deletions(-) delete mode 100644 tests/swarm-old.js diff --git a/tests/swarm-old.js b/tests/swarm-old.js deleted file mode 100644 index 408b8f7393..0000000000 --- a/tests/swarm-old.js +++ /dev/null @@ -1,254 +0,0 @@ -var Lab = require('lab') -var Code = require('code') -var sinon = require('sinon') -var lab = exports.lab = Lab.script() - -var experiment = lab.experiment -var test = lab.test -var beforeEach = lab.beforeEach -var afterEach = lab.afterEach -var expect = Code.expect - -var multiaddr = require('multiaddr') -var Id = require('ipfs-peer-id') -var Peer = require('ipfs-peer') -var Swarm = require('../src/') -var Identify = require('../src/identify') - -var swarmA -var swarmB -var peerA -var peerB - -beforeEach(function (done) { - swarmA = new Swarm() - swarmB = new Swarm() - var c = new Counter(2, done) - - swarmA.listen(8100, function () { - peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmA.port)]) - c.hit() - }) - - swarmB.listen(8101, function () { - peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + swarmB.port)]) - c.hit() - }) - -}) - -afterEach(function (done) { - // This should be 2, but for some reason - // that will fail in most of the tests - var c = new Counter(1, done) - - swarmA.closeListener(function () { - c.hit() - }) - swarmB.closeListener(function () { - c.hit() - }) -}) - -experiment('BASICS', function () { - experiment('Swarm', function () { - test('enforces instantiation with new', function (done) { - expect(function () { - Swarm() - }).to.throw('Swarm must be called with new') - done() - }) - - test('parses $IPFS_SWARM_PORT', function (done) { - process.env.IPFS_SWARM_PORT = 1111 - var swarm = new Swarm() - expect(swarm.port).to.be.equal(1111) - process.env.IPFS_SWARM_PORT = undefined - done() - }) - }) - - experiment('Swarm.listen', function (done) { - test('handles missing port', function (done) { - var swarm = new Swarm() - swarm.listen(done) - }) - - test('handles passed in port', function (done) { - var swarm = new Swarm() - swarm.listen(1234) - expect(swarm.port).to.be.equal(1234) - done() - }) - }) - - experiment('Swarm.registerHandler', function () { - test('throws when registering a protcol handler twice', function (done) { - var swarm = new Swarm() - swarm.registerHandler('/sparkles/1.1.1', function () {}) - swarm.registerHandler('/sparkles/1.1.1', function (err) { - expect(err).to.be.an.instanceOf(Error) - expect(err.message).to.be.equal('Handle for protocol already exists') - done() - }) - }) - }) - - experiment('Swarm.closeConns', function () { - test('calls end on all connections', function (done) { - swarmA.openConnection(peerB, function () { - var key = Object.keys(swarmA.connections)[0] - sinon.spy(swarmA.connections[key].conn, 'end') - swarmA.closeConns(function () { - expect(swarmA.connections[key].conn.end.called).to.be.equal(true) - done() - }) - }) - }) - }) -}) - -experiment('BASE', function () { - test('Open a stream', function (done) { - var protocol = '/sparkles/3.3.3' - var c = new Counter(2, done) - - swarmB.registerHandler(protocol, function (stream) { - c.hit() - }) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - c.hit() - }) - }) - - test('Reuse connection (from dialer)', function (done) { - var protocol = '/sparkles/3.3.3' - - swarmB.registerHandler(protocol, function (stream) {}) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - expect(swarmA.connections.length === 1) - done() - }) - }) - }) - test('Check for lastSeen', function (done) { - var protocol = '/sparkles/3.3.3' - - swarmB.registerHandler(protocol, function (stream) {}) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - expect(peerB.lastSeen).to.be.instanceof(Date) - done() - }) - }) - -}) - -experiment('IDENTIFY', function () { - test('Attach Identify, open a stream, see a peer update', function (done) { - swarmA.on('error', function (err) { - console.log('A - ', err) - }) - - swarmB.on('error', function (err) { - console.log('B - ', err) - }) - - var protocol = '/sparkles/3.3.3' - - var identifyA = new Identify(swarmA, peerA) - var identifyB = new Identify(swarmB, peerB) - setTimeout(function () { - swarmB.registerHandler(protocol, function (stream) {}) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - }) - - identifyB.on('peer-update', function (answer) { - done() - }) - identifyA.on('peer-update', function (answer) {}) - }, 500) - }) - - test('Attach Identify, open a stream, reuse stream', function (done) { - var protocol = '/sparkles/3.3.3' - - var identifyA = new Identify(swarmA, peerA) - var identifyB = new Identify(swarmB, peerB) - - swarmA.registerHandler(protocol, function (stream) {}) - swarmB.registerHandler(protocol, function (stream) {}) - - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - }) - - identifyB.on('peer-update', function (answer) { - expect(Object.keys(swarmB.connections).length).to.equal(1) - swarmB.openStream(peerA, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - expect(Object.keys(swarmB.connections).length).to.equal(1) - done() - }) - }) - identifyA.on('peer-update', function (answer) {}) - }) - - test('Attach Identify, reuse peer', function (done) { - var protocol = '/sparkles/3.3.3' - - var identifyA = new Identify(swarmA, peerA) - var identifyB = new Identify(swarmB, peerB) // eslint-disable-line no-unused-vars - - swarmA.registerHandler(protocol, function (stream) {}) - swarmB.registerHandler(protocol, function (stream) {}) - - var restartA = function (cb) { - swarmA.openStream(peerB, protocol, function (err, stream) { - expect(err).to.not.be.instanceof(Error) - - stream.end(cb) - }) - } - - restartA(function () { - identifyA.once('peer-update', function () { - expect(peerA.previousObservedAddrs.length).to.be.equal(1) - - var c = new Counter(2, done) - - swarmA.closeConns(c.hit.bind(c)) - swarmB.closeConns(c.hit.bind(c)) - }) - }) - }) -}) - -experiment('HARDNESS', function () {}) - -function Counter (target, callback) { - var c = 0 - this.hit = count - - function count () { - c += 1 - if (c === target) { - callback() - } - } -} - -// function checkErr (err) { -// console.log('err') -// expect(err).to.be.instanceof(Error) -// } From 5b7a6051ad332e2670f4b9cdb0cb0d6e56b0b599 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 20:07:55 +0100 Subject: [PATCH 069/634] comment undone tests --- tests/swarm-test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index ddf64824ad..7975abf2ae 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -185,9 +185,9 @@ experiment('Without a Stream Muxer', function () { } }) - test('add an upgrade', function (done) { done() }) - test('dial a conn on top of a upgrade', function (done) { done() }) - test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + // test('add an upgrade', function (done) { done() }) + // test('dial a conn on top of a upgrade', function (done) { done() }) + // test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) }) /* TODO From 1ba8e80d4d7ea9acb6344d618141c0bde4472126 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 20:34:31 +0100 Subject: [PATCH 070/634] rm laf --- package.json | 1 - src/identify/index.js | 16 ++++++++-------- src/swarm.js | 8 +++----- tests/swarm-test.js | 40 ++++++++++++++++++++-------------------- 4 files changed, 31 insertions(+), 34 deletions(-) diff --git a/package.json b/package.json index d5d93a269c..a53645ca18 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,6 @@ "scripts": { "test": "./node_modules/.bin/lab tests/*-test.js", "coverage": "./node_modules/.bin/lab -t 88 tests/*-test.js", - "laf": "./node_modules/.bin/standard --format", "lint": "./node_modules/.bin/standard" }, "repository": { diff --git a/src/identify/index.js b/src/identify/index.js index 181cd3808a..e7a857c420 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -38,14 +38,14 @@ function identify (muxedConns, peerInfoSelf, socket, conn, muxer) { socket: socket } - // TODO: Pass the new discovered info about the peer that contacted us - // to something like the Kademlia Router, so the peerInfo for this peer - // is fresh - // - before this was exectued through a event emitter - // self.emit('peer-update', { - // peerId: peerId, - // listenAddrs: msg.listenAddrs.map(function (mhb) {return multiaddr(mhb)}) - // }) + // TODO: Pass the new discovered info about the peer that contacted us + // to something like the Kademlia Router, so the peerInfo for this peer + // is fresh + // - before this was exectued through a event emitter + // self.emit('peer-update', { + // peerId: peerId, + // listenAddrs: msg.listenAddrs.map(function (mhb) {return multiaddr(mhb)}) + // }) }) var mh = getMultiaddr(socket) diff --git a/src/swarm.js b/src/swarm.js index 1cfff09509..2fa4f5d494 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -67,9 +67,7 @@ function Swarm (peerInfo) { }) } - self.addUpgrade = function (ConnUpgrade, options) { - - } + self.addUpgrade = function (ConnUpgrade, options) {} self.addStreamMuxer = function (name, StreamMuxer, options) { self.muxers[name] = { @@ -260,7 +258,7 @@ function Swarm (peerInfo) { // we pass muxedConns so that identify can access the socket of the other // peer self.handleProtocol(identify.protoId, - identify.getHandlerFunction(self.peerInfo, self.muxedConns)) + identify.getHandlerFunction(self.peerInfo, self.muxedConns)) } self.handleProtocol = function (protocol, handlerFunction) { @@ -299,7 +297,7 @@ function Swarm (peerInfo) { muxer.on('stream', userProtocolMuxer) }) } else { - // if no stream muxer, then + // if no stream muxer, then userProtocolMuxer(conn) } } diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 7975abf2ae..bf5baeaa32 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -32,15 +32,15 @@ experiment('Without a Stream Muxer', function () { var sw = new Swarm(p) sw.addTransport('tcp', tcp, - { multiaddr: mh }, {}, {port: 8010}, function () { - expect(sw.transports['tcp'].options).to.deep.equal({ multiaddr: mh }) - expect(sw.transports['tcp'].dialOptions).to.deep.equal({}) - expect(sw.transports['tcp'].listenOptions).to.deep.equal({port: 8010}) - expect(sw.transports['tcp'].transport).to.deep.equal(tcp) - sw.closeListener('tcp', function () { - done() + { multiaddr: mh }, {}, {port: 8010}, function () { + expect(sw.transports['tcp'].options).to.deep.equal({ multiaddr: mh }) + expect(sw.transports['tcp'].dialOptions).to.deep.equal({}) + expect(sw.transports['tcp'].listenOptions).to.deep.equal({port: 8010}) + expect(sw.transports['tcp'].transport).to.deep.equal(tcp) + sw.closeListener('tcp', function () { + done() + }) }) - }) }) test('dial a conn', function (done) { @@ -185,9 +185,9 @@ experiment('Without a Stream Muxer', function () { } }) - // test('add an upgrade', function (done) { done() }) - // test('dial a conn on top of a upgrade', function (done) { done() }) - // test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + // test('add an upgrade', function (done) { done() }) + // test('dial a conn on top of a upgrade', function (done) { done() }) + // test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) }) /* TODO @@ -210,15 +210,15 @@ experiment('Without a Stream Muxer', function () { test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) }) */ - /* TODO - experiment('utp', function () { - test('add the transport', function (done) { done() }) - test('dial a conn', function (done) { done() }) - test('dial a conn on a protocol', function (done) { done() }) - test('add an upgrade', function (done) { done() }) - test('dial a conn on top of a upgrade', function (done) { done() }) - test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) - }) */ +/* TODO +experiment('utp', function () { + test('add the transport', function (done) { done() }) + test('dial a conn', function (done) { done() }) + test('dial a conn on a protocol', function (done) { done() }) + test('add an upgrade', function (done) { done() }) + test('dial a conn on top of a upgrade', function (done) { done() }) + test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) +}) */ }) experiment('With a SPDY Stream Muxer', function () { From 490022cf02da77a6788418d3039dfd791bf0a8b3 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 23:12:35 +0100 Subject: [PATCH 071/634] ready to publish new version --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index a53645ca18..c3df93e7be 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "libp2p-swarm", "version": "0.4.1", - "description": "IPFS swarm implementation in Node.js", + "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { "test": "./node_modules/.bin/lab tests/*-test.js", @@ -10,7 +10,7 @@ }, "repository": { "type": "git", - "url": "https://github.com/diasdavid/node-ipfs-swarm.git" + "url": "https://github.com/diasdavid/node-libp2p-swarm.git" }, "keywords": [ "IPFS" @@ -18,9 +18,9 @@ "author": "David Dias ", "license": "MIT", "bugs": { - "url": "https://github.com/diasdavid/node-ipfs-swarm/issues" + "url": "https://github.com/diasdavid/node-libp2p-swarm/issues" }, - "homepage": "https://github.com/diasdavid/node-ipfs-swarm", + "homepage": "https://github.com/diasdavid/node-libp2p-swarm", "pre-commit": [ "lint", "test", From cd5334444119bd63d219e083b1437265823d2a40 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Sep 2015 23:12:50 +0100 Subject: [PATCH 072/634] Release v0.5.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c3df93e7be..81b9e0a275 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.4.1", + "version": "0.5.0", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { From 0514b0034bbe9472cf7bb06845437899dcdc7c9f Mon Sep 17 00:00:00 2001 From: Pau Ramon Revilla Date: Fri, 25 Sep 2015 09:26:50 +0200 Subject: [PATCH 073/634] Do not allow undefined `peerInfo` # What The code assumes that `peerInfo` exists, the API doesn't. # How to test ```js swarm = new Swarm() swarm.addTransport('tcp', tcp, { multiaddr: mh }, {}, {port: 8095}) ``` This shouldn't explode because of `self.peerInfo.multiaddrs`. --- src/index.js | 1 - src/swarm.js | 4 ++++ tests/swarm-test.js | 10 ++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index b825469a3a..5e3bb3994c 100644 --- a/src/index.js +++ b/src/index.js @@ -1,4 +1,3 @@ var Swarm = require('./swarm') exports = module.exports = Swarm -exports.singleton = new Swarm() diff --git a/src/swarm.js b/src/swarm.js index 2fa4f5d494..5f56208cfc 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -11,6 +11,10 @@ function Swarm (peerInfo) { throw new Error('Swarm must be called with new') } + if (!peerInfo) { + throw new Error('You must provide a value for `peerInfo`') + } + self.peerInfo = peerInfo // peerIdB58: { conn: } diff --git a/tests/swarm-test.js b/tests/swarm-test.js index bf5baeaa32..e9bd63cf38 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -24,6 +24,16 @@ process.on('uncaughtException', function (err) { console.log('Caught exception: ' + err) }) +experiment('Without a peer', function () { + test('it throws an exception', function (done) { + expect(function () { + var sw = new Swarm() + sw.close() + }).to.throw(Error) + done() + }) +}) + experiment('Without a Stream Muxer', function () { experiment('tcp', function () { test('add the transport', function (done) { From 8dc46da80f2e158106238b9ad7e6687f221ee466 Mon Sep 17 00:00:00 2001 From: Pau Ramon Revilla Date: Fri, 25 Sep 2015 09:07:45 +0200 Subject: [PATCH 074/634] Protect from peers without supported transports # What Trying to run compliance tests from the kad router module. I've tried to port the new swarm API but forgot to add a transport. The tests ended up blowing up instead of failing gracefully. # How to test ```js peerOne = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8090')]) peerTwo = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8091')]) swarm = new Swarm(peerZero) swarm.dial(peerTwo, {}, function (err) { console.log(err); }); ``` This just work and display the error. --- README.md | 4 ++-- src/swarm.js | 6 ++++++ tests/swarm-test.js | 30 ++++++++++++++++++++++++------ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7a87626e2a..1c14d3d163 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ sw.addStreamMuxer(streamMuxer, [options]) ### Dial to another peer ```JavaScript -sw.dial(PeerInfo, options, protocol) -sw.dial(PeerInfo, options) +sw.dial(PeerInfo, options, protocol, callback) +sw.dial(PeerInfo, options, callback) ``` dial uses the best transport (whatever works first, in the future we can have some criteria), and jump starts the connection until the point we have to negotiate the protocol. If a muxer is available, then drop the muxer onto that connection. Good to warm up connections or to check for connectivity. If we have already a muxer for that peerInfo, than do nothing. diff --git a/src/swarm.js b/src/swarm.js index 2fa4f5d494..c12c99c096 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -126,6 +126,11 @@ function Swarm (peerInfo) { }) }) + if (!multiaddrs.length) { + callback(new Error("The swarm doesn't support any of the peer transports")) + return + } + var conn async.eachSeries(multiaddrs, function (multiaddr, next) { @@ -175,6 +180,7 @@ function Swarm (peerInfo) { // if protocol is selected, multistream that protocol if (!conn) { callback(new Error('Unable to open a connection')) + return } if (self.muxers['spdy']) { diff --git a/tests/swarm-test.js b/tests/swarm-test.js index bf5baeaa32..8a20bd103b 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -13,17 +13,35 @@ var Swarm = require('../src') var tcp = require('libp2p-tcp') var Spdy = require('libp2p-spdy') -/* TODO -experiment('Basics', function () { - test('enforces creation with new', function (done) {done() }) -}) -*/ - // because of Travis-CI process.on('uncaughtException', function (err) { console.log('Caught exception: ' + err) }) +experiment('Basics', function () { + test('enforces creation with new', function (done) { + expect(function () { + Swarm() + }).to.throw() + done() + }) +}) + +experiment('When dialing', function () { + experiment('if the swarm does add any of the peer transports', function () { + test('it returns an error', function (done) { + var peerOne = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8090')]) + var peerTwo = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8091')]) + var swarm = new Swarm(peerOne) + + swarm.dial(peerTwo, {}, function (err) { + expect(err).to.exist() + done() + }) + }) + }) +}) + experiment('Without a Stream Muxer', function () { experiment('tcp', function () { test('add the transport', function (done) { From 5c53540e92f337a77cfc7027c438e596594e26c4 Mon Sep 17 00:00:00 2001 From: Pau Ramon Revilla Date: Sat, 26 Sep 2015 20:11:06 +0200 Subject: [PATCH 075/634] Implemented `close` and improved the tests with it --- README.md | 24 +++ src/swarm.js | 26 ++- tests/swarm-test.js | 441 ++++++++++++++++++-------------------------- 3 files changed, 222 insertions(+), 269 deletions(-) diff --git a/README.md b/README.md index 7a87626e2a..7c21bc1d89 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,27 @@ dial uses the best transport (whatever works first, in the future we can have so ```JavaScript sw.handleProtocol(protocol, handlerFunction) ``` + +### Cleaning up before exiting + +Each time you add a transport or dial you create connections. Be sure to close +them up before exiting. To do so you can: + +Close a transport listener: + +```js +sw.closeListener(transportName, callback) +sw.closeAllListeners(callback) +``` + +Close all open connections + +```js +sw.closeConns(callback) +``` + +Close everything! + +```js +sw.close(callback) +``` diff --git a/src/swarm.js b/src/swarm.js index 2fa4f5d494..fdeedc2b1c 100644 --- a/src/swarm.js +++ b/src/swarm.js @@ -47,6 +47,12 @@ function Swarm (peerInfo) { // set up the transport and add the list of incoming streams // add transport to the list of transports + var multiaddr = options.multiaddr + if (multiaddr) { + // no need to pass that to the transports + delete options.multiaddr + } + var listener = transport.createListener(options, listen) listener.listen(listenOptions, function ready () { @@ -59,8 +65,8 @@ function Swarm (peerInfo) { } // If a known multiaddr is passed, then add to our list of multiaddrs - if (options.multiaddr) { - self.peerInfo.multiaddrs.push(options.multiaddr) + if (multiaddr) { + self.peerInfo.multiaddrs.push(multiaddr) } callback() @@ -236,6 +242,14 @@ function Swarm (peerInfo) { } } + // Iterates all the listeners closing them + // one by one. It calls back once all are closed. + self.closeAllListeners = function (callback) { + var transportNames = Object.keys(self.transports) + + async.each(transportNames, self.closeListener, callback) + } + self.closeConns = function (callback) { // close warmed up cons so that the listener can gracefully exit Object.keys(self.conns).forEach(function (conn) { @@ -246,8 +260,14 @@ function Swarm (peerInfo) { callback() } + // Closes both transport listeners and + // connections. It calls back once everything + // is closed self.close = function (callback) { - // close everything + async.parallel([ + self.closeAllListeners, + self.closeConns + ], callback) } self.enableIdentify = function () { diff --git a/tests/swarm-test.js b/tests/swarm-test.js index bf5baeaa32..9670846939 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -1,9 +1,12 @@ var Lab = require('lab') var Code = require('code') var lab = exports.lab = Lab.script() +var async = require('async') var experiment = lab.experiment var test = lab.test +var beforeEach = lab.beforeEach +var afterEach = lab.afterEach var expect = Code.expect var multiaddr = require('multiaddr') @@ -25,164 +28,89 @@ process.on('uncaughtException', function (err) { }) experiment('Without a Stream Muxer', function () { - experiment('tcp', function () { + experiment('and one swarm over tcp', function () { test('add the transport', function (done) { var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') var p = new Peer(Id.create(), []) var sw = new Swarm(p) - sw.addTransport('tcp', tcp, - { multiaddr: mh }, {}, {port: 8010}, function () { - expect(sw.transports['tcp'].options).to.deep.equal({ multiaddr: mh }) - expect(sw.transports['tcp'].dialOptions).to.deep.equal({}) - expect(sw.transports['tcp'].listenOptions).to.deep.equal({port: 8010}) - expect(sw.transports['tcp'].transport).to.deep.equal(tcp) - sw.closeListener('tcp', function () { - done() - }) - }) - }) + sw.addTransport('tcp', tcp, { multiaddr: mh }, {}, {port: 8010}, ready) - test('dial a conn', function (done) { - var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - var p1 = new Peer(Id.create(), []) - var sw1 = new Swarm(p1) - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) + function ready () { + expect(sw.transports['tcp'].options).to.deep.equal({}) + expect(sw.transports['tcp'].dialOptions).to.deep.equal({}) + expect(sw.transports['tcp'].listenOptions).to.deep.equal({port: 8010}) + expect(sw.transports['tcp'].transport).to.deep.equal(tcp) - var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - var p2 = new Peer(Id.create(), []) - var sw2 = new Swarm(p2) - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) + sw.close(done) + } + }) + }) - var readyCounter = 0 + experiment('and two swarms over tcp', function () { + var mh1, p1, sw1, mh2, p2, sw2 - function ready () { - readyCounter++ - if (readyCounter < 2) { - return - } + beforeEach(function (done) { + mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + p1 = new Peer(Id.create(), []) + sw1 = new Swarm(p1) - sw1.dial(p2, {}, function (err) { - expect(err).to.equal(undefined) - expect(Object.keys(sw1.conns).length).to.equal(1) - var cleaningCounter = 0 - sw1.closeConns(cleaningUp) - sw2.closeConns(cleaningUp) + mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + p2 = new Peer(Id.create(), []) + sw2 = new Swarm(p2) - sw1.closeListener('tcp', cleaningUp) - sw2.closeListener('tcp', cleaningUp) + async.parallel([ + function (cb) { + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) + }, + function (cb) { + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) + } + ], done) + }) - function cleaningUp () { - cleaningCounter++ - if (cleaningCounter < 4) { - return - } + afterEach(function (done) { + async.parallel([sw1.close, sw2.close], done) + }) - done() - } - }) - } + test('dial a conn', function (done) { + sw1.dial(p2, {}, function (err) { + expect(err).to.equal(undefined) + expect(Object.keys(sw1.conns).length).to.equal(1) + done() + }) }) test('dial a conn on a protocol', function (done) { - var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - var p1 = new Peer(Id.create(), []) - var sw1 = new Swarm(p1) - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) - - var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - var p2 = new Peer(Id.create(), []) - var sw2 = new Swarm(p2) - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { conn.end() - conn.on('end', function () { - var cleaningCounter = 0 - sw1.closeConns(cleaningUp) - sw2.closeConns(cleaningUp) - - sw1.closeListener('tcp', cleaningUp) - sw2.closeListener('tcp', cleaningUp) - - function cleaningUp () { - cleaningCounter++ - if (cleaningCounter < 4) { - return - } - - done() - } - }) + conn.on('end', done) }) - var count = 0 - - function ready () { - count++ - if (count < 2) { - return - } - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) - } + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) }) test('dial a protocol on a previous created conn', function (done) { - var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - var p1 = new Peer(Id.create(), []) - var sw1 = new Swarm(p1) - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) - - var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - var p2 = new Peer(Id.create(), []) - var sw2 = new Swarm(p2) - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) - - var readyCounter = 0 - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { conn.end() - conn.on('end', function () { - var cleaningCounter = 0 - sw1.closeConns(cleaningUp) - sw2.closeConns(cleaningUp) - - sw1.closeListener('tcp', cleaningUp) - sw2.closeListener('tcp', cleaningUp) - - function cleaningUp () { - cleaningCounter++ - if (cleaningCounter < 4) { - return - } - - done() - } - }) + conn.on('end', done) }) - function ready () { - readyCounter++ - if (readyCounter < 2) { - return - } + sw1.dial(p2, {}, function (err) { + expect(err).to.equal(undefined) + expect(Object.keys(sw1.conns).length).to.equal(1) - sw1.dial(p2, {}, function (err) { - expect(err).to.equal(undefined) - expect(Object.keys(sw1.conns).length).to.equal(1) + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) + conn.end() }) - } + }) }) // test('add an upgrade', function (done) { done() }) @@ -222,7 +150,8 @@ experiment('utp', function () { }) experiment('With a SPDY Stream Muxer', function () { - experiment('tcp', function () { + experiment('and one swarm over tcp', function () { + // TODO: What is the test here? test('add Stream Muxer', function (done) { // var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') var p = new Peer(Id.create(), []) @@ -231,77 +160,54 @@ experiment('With a SPDY Stream Muxer', function () { done() }) + }) - test('dial a conn on a protocol', function (done) { - var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - var p1 = new Peer(Id.create(), []) - var sw1 = new Swarm(p1) - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) + experiment('and two swarms over tcp', function () { + var mh1, p1, sw1, mh2, p2, sw2 + + beforeEach(function (done) { + mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + p1 = new Peer(Id.create(), []) + sw1 = new Swarm(p1) sw1.addStreamMuxer('spdy', Spdy, {}) - var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - var p2 = new Peer(Id.create(), []) - var sw2 = new Swarm(p2) - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) + mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + p2 = new Peer(Id.create(), []) + sw2 = new Swarm(p2) sw2.addStreamMuxer('spdy', Spdy, {}) - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - // formallity so that the conn starts flowing - conn.on('data', function (chunk) {}) + async.parallel([ + function (cb) { + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) + }, + function (cb) { + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) + } + ], done) + }) - conn.end() - conn.on('end', function () { - expect(Object.keys(sw1.muxedConns).length).to.equal(1) - expect(Object.keys(sw2.muxedConns).length).to.equal(0) - var cleaningCounter = 0 - sw1.closeConns(cleaningUp) - sw2.closeConns(cleaningUp) - - sw1.closeListener('tcp', cleaningUp) - sw2.closeListener('tcp', cleaningUp) - - function cleaningUp () { - cleaningCounter++ - // TODO FIX: here should be 4, but because super wrapping of - // streams, it makes it so hard to properly close the muxed - // streams - https://github.com/indutny/spdy-transport/issues/14 - if (cleaningCounter < 3) { - return - } - - done() - } - }) - }) + function afterEach (done) { + var cleaningCounter = 0 + sw1.closeConns(cleaningUp) + sw2.closeConns(cleaningUp) - var count = 0 + sw1.closeListener('tcp', cleaningUp) + sw2.closeListener('tcp', cleaningUp) - function ready () { - count++ - if (count < 2) { + function cleaningUp () { + cleaningCounter++ + // TODO FIX: here should be 4, but because super wrapping of + // streams, it makes it so hard to properly close the muxed + // streams - https://github.com/indutny/spdy-transport/issues/14 + if (cleaningCounter < 3) { return } - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) + done() } - }) - test('dial two conns (transport reuse)', function (done) { - var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - var p1 = new Peer(Id.create(), []) - var sw1 = new Swarm(p1) - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) - sw1.addStreamMuxer('spdy', Spdy, {}) + } - var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - var p2 = new Peer(Id.create(), []) - var sw2 = new Swarm(p2) - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) - sw2.addStreamMuxer('spdy', Spdy, {}) + test('dial a conn on a protocol', function (done) { sw2.handleProtocol('/sparkles/1.0.0', function (conn) { // formallity so that the conn starts flowing @@ -311,69 +217,102 @@ experiment('With a SPDY Stream Muxer', function () { conn.on('end', function () { expect(Object.keys(sw1.muxedConns).length).to.equal(1) expect(Object.keys(sw2.muxedConns).length).to.equal(0) - conn.end() + afterEach(done) + }) + }) - var cleaningCounter = 0 - sw1.closeConns(cleaningUp) - sw2.closeConns(cleaningUp) + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) + }) - sw1.closeListener('tcp', cleaningUp) - sw2.closeListener('tcp', cleaningUp) + test('dial two conns (transport reuse)', function (done) { + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + // formality so that the conn starts flowing + conn.on('data', function (chunk) {}) - function cleaningUp () { - cleaningCounter++ - // TODO FIX: here should be 4, but because super wrapping of - // streams, it makes it so hard to properly close the muxed - // streams - https://github.com/indutny/spdy-transport/issues/14 - if (cleaningCounter < 3) { - return - } + conn.end() + conn.on('end', function () { + expect(Object.keys(sw1.muxedConns).length).to.equal(1) + expect(Object.keys(sw2.muxedConns).length).to.equal(0) - done() - } + afterEach(done) }) }) - var count = 0 - - function ready () { - count++ - if (count < 2) { - return - } - + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + // TODO Improve clarity sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - // TODO Improve clarity - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) - conn.on('data', function () {}) expect(err).to.equal(null) expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() }) - } + + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + + conn.end() + }) }) + }) - test('identify', function (done) { - var mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - var p1 = new Peer(Id.create(), []) - var sw1 = new Swarm(p1) - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, ready) + experiment('and two identity enabled swarms over tcp', function () { + var mh1, p1, sw1, mh2, p2, sw2 + + beforeEach(function (done) { + mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + p1 = new Peer(Id.create(), []) + sw1 = new Swarm(p1) sw1.addStreamMuxer('spdy', Spdy, {}) sw1.enableIdentify() - var mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - var p2 = new Peer(Id.create(), []) - var sw2 = new Swarm(p2) - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, ready) + mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + p2 = new Peer(Id.create(), []) + sw2 = new Swarm(p2) sw2.addStreamMuxer('spdy', Spdy, {}) sw2.enableIdentify() + async.parallel([ + function (cb) { + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) + }, + function (cb) { + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) + } + ], done) + }) + + afterEach(function (done) { + var cleaningCounter = 0 + sw1.closeConns(cleaningUp) + sw2.closeConns(cleaningUp) + + sw1.closeListener('tcp', cleaningUp) + sw2.closeListener('tcp', cleaningUp) + + function cleaningUp () { + cleaningCounter++ + // TODO FIX: here should be 4, but because super wrapping of + // streams, it makes it so hard to properly close the muxed + // streams - https://github.com/indutny/spdy-transport/issues/14 + if (cleaningCounter < 3) { + return + } + // give time for identify to finish + setTimeout(function () { + expect(Object.keys(sw2.muxedConns).length).to.equal(1) + done() + }, 500) + } + }) + + test('identify', function (done) { sw2.handleProtocol('/sparkles/1.0.0', function (conn) { // formallity so that the conn starts flowing conn.on('data', function (chunk) {}) @@ -381,46 +320,16 @@ experiment('With a SPDY Stream Muxer', function () { conn.end() conn.on('end', function () { expect(Object.keys(sw1.muxedConns).length).to.equal(1) - - var cleaningCounter = 0 - sw1.closeConns(cleaningUp) - sw2.closeConns(cleaningUp) - - sw1.closeListener('tcp', cleaningUp) - sw2.closeListener('tcp', cleaningUp) - - function cleaningUp () { - cleaningCounter++ - // TODO FIX: here should be 4, but because super wrapping of - // streams, it makes it so hard to properly close the muxed - // streams - https://github.com/indutny/spdy-transport/issues/14 - if (cleaningCounter < 3) { - return - } - // give time for identify to finish - setTimeout(function () { - expect(Object.keys(sw2.muxedConns).length).to.equal(1) - done() - }, 500) - } + done() }) }) - var count = 0 - - function ready () { - count++ - if (count < 2) { - return - } - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) - } + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) }) }) }) From adb5ce19b1967141f828646ee64a7ce4013b3d43 Mon Sep 17 00:00:00 2001 From: David Dias Date: Sat, 26 Sep 2015 21:12:46 +0100 Subject: [PATCH 076/634] Release v0.5.1. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 81b9e0a275..14c7dc67fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.5.0", + "version": "0.5.1", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { From f53124393c1cd9943a70d82ecd834bd2cebe9c28 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 28 Sep 2015 04:01:45 +0100 Subject: [PATCH 077/634] Release v0.5.2. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 14c7dc67fd..7a2df2860b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.5.1", + "version": "0.5.2", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { From f309d4f7b77033dbace1f3e940b950e041dc5eb9 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 28 Sep 2015 16:12:01 +0100 Subject: [PATCH 078/634] Release v0.5.3. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index f889081abe..38f040dc53 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.5.2", + "version": "0.5.3", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { @@ -52,4 +52,4 @@ "protocol-buffers-stream": "^1.2.0", "spdy-stream-muxer": "^0.6.0" } -} \ No newline at end of file +} From 348615b5aa28652c9f07807548a424f3ad263e29 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 29 Oct 2015 00:26:18 +0000 Subject: [PATCH 079/634] update readme and package.json --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3e40b773d5..8d2d1398a9 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -libp2p-swarm Node.js implementation -================================= +libp2p-swarm JavaScript implementation +====================================== -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) [![Build Status](https://img.shields.io/travis/diasdavid/node-ipfs-swarm/master.svg?style=flat-square)](https://travis-ci.org/diasdavid/node-ipfs-swarm) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freejs-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) [![Build Status](https://img.shields.io/travis/diasdavid/js-ipfs-swarm/master.svg?style=flat-square)](https://travis-ci.org/diasdavid/js-ipfs-swarm) -> libp2p swarm implementation in Node.js +> libp2p swarm implementation in JavaScript # Description @@ -29,11 +29,11 @@ var Swarm = require('libp2p-swarm') var sw = new Swarm(peerInfoSelf) ``` -peerInfoSelf is a [PeerInfo](https://github.com/diasdavid/node-peer-info) object that represents the peer creating this swarm instance. +peerInfoSelf is a [PeerInfo](https://github.com/diasdavid/js-peer-info) object that represents the peer creating this swarm instance. ### Support a transport -libp2p-swarm expects transports that implement [abstract-transport](https://github.com/diasdavid/abstract-transport). For example [libp2p-tcp](https://github.com/diasdavid/node-libp2p-tcp), a simple shim on top of the `net` module to make it work with swarm expectations. +libp2p-swarm expects transports that implement [abstract-transport](https://github.com/diasdavid/abstract-transport). For example [libp2p-tcp](https://github.com/diasdavid/js-libp2p-tcp), a simple shim on top of the `net` module to make it work with swarm expectations. ```JavaScript sw.addTransport(transport, [options, dialOptions, listenOptions]) From 555f2199df7ef02757da1e87aed2d7f22acd7616 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 29 Oct 2015 00:27:18 +0000 Subject: [PATCH 080/634] Release v0.5.4. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 38f040dc53..11d01fa5d2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.5.3", + "version": "0.5.4", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { From c827bd847015cc1a626868bf80bc7f267b4eb866 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 29 Oct 2015 00:27:49 +0000 Subject: [PATCH 081/634] package.json --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 11d01fa5d2..953507129e 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ }, "repository": { "type": "git", - "url": "https://github.com/diasdavid/node-libp2p-swarm.git" + "url": "https://github.com/diasdavid/js-libp2p-swarm.git" }, "keywords": [ "IPFS" @@ -19,9 +19,9 @@ "author": "David Dias ", "license": "MIT", "bugs": { - "url": "https://github.com/diasdavid/node-libp2p-swarm/issues" + "url": "https://github.com/diasdavid/js-libp2p-swarm/issues" }, - "homepage": "https://github.com/diasdavid/node-libp2p-swarm", + "homepage": "https://github.com/diasdavid/js-libp2p-swarm", "pre-commit": [ "lint", "test", From 3b9465de92966492652bb633d4428383455ef792 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 29 Oct 2015 00:27:59 +0000 Subject: [PATCH 082/634] Release v0.5.5. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 953507129e..bfec71413f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.5.4", + "version": "0.5.5", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { From c39eb4a83056203040c8431e270e9006b9a891c6 Mon Sep 17 00:00:00 2001 From: Pau Ramon Revilla Date: Sun, 1 Nov 2015 19:50:11 +0100 Subject: [PATCH 083/634] Removed ipv6 to ipv4 hack --- package.json | 2 +- src/identify/index.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index bfec71413f..ab24c4fdf2 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ }, "dependencies": { "async": "^1.3.0", - "ip-address": "^4.0.0", + "ip-address": "^5.0.2", "ipfs-logger": "^0.1.0", "multiaddr": "^1.0.0", "multiplex-stream-muxer": "^0.2.0", diff --git a/src/identify/index.js b/src/identify/index.js index e7a857c420..63684631e1 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -7,7 +7,7 @@ var Interactive = require('multistream-select').Interactive var protobufs = require('protocol-buffers-stream') var fs = require('fs') var schema = fs.readFileSync(__dirname + '/identify.proto') -var v6 = require('ip-address').v6 +var Address6 = require('ip-address').Address6 var Id = require('peer-id') var multiaddr = require('multiaddr') @@ -112,10 +112,10 @@ exports.getHandlerFunction = function (peerInfoSelf, muxedConns) { function getMultiaddr (socket) { var mh - if (~socket.remoteAddress.indexOf(':')) { - var addr = new v6.Address(socket.remoteAddress) + if (socket.remoteFamily === 'IPv6') { + var addr = new Address6(socket.remoteAddress) if (addr.v4) { - var ip4 = socket.remoteAddress.split(':')[3] + var ip4 = addr.to4().correctForm() mh = multiaddr('/ip4/' + ip4 + '/tcp/' + socket.remotePort) } else { mh = multiaddr('/ip6/' + socket.remoteAddress + '/tcp/' + socket.remotePort) From aea0940b2dfd51bfcd713749c6f613c6170a4714 Mon Sep 17 00:00:00 2001 From: Richard Littauer Date: Mon, 28 Dec 2015 17:18:08 -0500 Subject: [PATCH 084/634] Freejs => Freenode See https://github.com/ipfs/community/issues/93 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d2d1398a9..dc4200c125 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ libp2p-swarm JavaScript implementation ====================================== -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freejs-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) [![Build Status](https://img.shields.io/travis/diasdavid/js-ipfs-swarm/master.svg?style=flat-square)](https://travis-ci.org/diasdavid/js-ipfs-swarm) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) [![Build Status](https://img.shields.io/travis/diasdavid/js-ipfs-swarm/master.svg?style=flat-square)](https://travis-ci.org/diasdavid/js-ipfs-swarm) > libp2p swarm implementation in JavaScript From 0e636597ee9b93aa978982239bb5fedbeab39371 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 3 Mar 2016 12:08:46 +0000 Subject: [PATCH 085/634] update swarm --- package.json | 22 +- src/{identify/index.js => identify.js} | 0 src/{identify => }/identify.proto | 0 src/index.js | 342 +++++++++++++++++++++++- src/swarm.js | 343 ------------------------- tests/multistream-and-muxer-old.js | 139 ---------- tests/swarm-test.js | 106 ++++---- 7 files changed, 401 insertions(+), 551 deletions(-) rename src/{identify/index.js => identify.js} (100%) rename src/{identify => }/identify.proto (100%) delete mode 100644 src/swarm.js delete mode 100644 tests/multistream-and-muxer-old.js diff --git a/package.json b/package.json index ab24c4fdf2..2574c65826 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,9 @@ "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { - "test": "./node_modules/.bin/lab tests/*-test.js", - "coverage": "./node_modules/.bin/lab -t 88 tests/*-test.js", - "lint": "./node_modules/.bin/standard", - "validate": "npm ls" + "test": "mocha tests/*-test.js", + "coverage": "istanbul cover --print both -- _mocha tests/*-test.js", + "lint": "standard" }, "repository": { "type": "git", @@ -24,20 +23,20 @@ "homepage": "https://github.com/diasdavid/js-libp2p-swarm", "pre-commit": [ "lint", - "test", - "coverage" + "test" ], "engines": { "node": "^4.0.0" }, "devDependencies": { - "code": "^1.4.1", - "lab": "^5.13.0", + "chai": "^3.5.0", + "istanbul": "^0.4.2", "libp2p-spdy": "^0.1.0", "libp2p-tcp": "^0.1.1", - "precommit-hook": "^3.0.0", + "mocha": "^2.4.5", + "pre-commit": "^1.1.2", "sinon": "^1.15.4", - "standard": "^4.5.2", + "standard": "^6.0.7", "stream-pair": "^1.0.3" }, "dependencies": { @@ -49,7 +48,6 @@ "multistream-select": "^0.6.1", "peer-id": "^0.3.3", "peer-info": "^0.3.2", - "protocol-buffers-stream": "^1.2.0", - "spdy-stream-muxer": "^0.6.0" + "protocol-buffers-stream": "^1.2.0" } } diff --git a/src/identify/index.js b/src/identify.js similarity index 100% rename from src/identify/index.js rename to src/identify.js diff --git a/src/identify/identify.proto b/src/identify.proto similarity index 100% rename from src/identify/identify.proto rename to src/identify.proto diff --git a/src/index.js b/src/index.js index 5e3bb3994c..54c254265a 100644 --- a/src/index.js +++ b/src/index.js @@ -1,3 +1,343 @@ -var Swarm = require('./swarm') +var multistream = require('multistream-select') +var async = require('async') +var identify = require('./identify') exports = module.exports = Swarm + +function Swarm (peerInfo) { + var self = this + + if (!(self instanceof Swarm)) { + throw new Error('Swarm must be called with new') + } + + if (!peerInfo) { + throw new Error('You must provide a value for `peerInfo`') + } + + self.peerInfo = peerInfo + + // peerIdB58: { conn: } + self.conns = {} + + // peerIdB58: { + // muxer: , + // socket: socket // so we can extract the info we need for identify + // } + self.muxedConns = {} + + // transportName: { transport: transport, + // dialOptions: dialOptions, + // listenOptions: listenOptions, + // listeners: [] } + self.transports = {} + + // transportName: listener + self.listeners = {} + + // protocolName: handlerFunc + self.protocols = {} + + // muxerName: { Muxer: Muxer // Muxer is a constructor + // options: options } + self.muxers = {} + + // for connection reuse + self.identify = false + + // public interface + + self.addTransport = function (name, transport, options, dialOptions, listenOptions, callback) { + // set up the transport and add the list of incoming streams + // add transport to the list of transports + + var multiaddr = options.multiaddr + if (multiaddr) { + // no need to pass that to the transports + delete options.multiaddr + } + + var listener = transport.createListener(options, listen) + + listener.listen(listenOptions, function ready () { + self.transports[name] = { + transport: transport, + options: options, + dialOptions: dialOptions, + listenOptions: listenOptions, + listener: listener + } + + // If a known multiaddr is passed, then add to our list of multiaddrs + if (multiaddr) { + self.peerInfo.multiaddrs.push(multiaddr) + } + + callback() + }) + } + + self.addUpgrade = function (ConnUpgrade, options) {} + + self.addStreamMuxer = function (name, StreamMuxer, options) { + self.muxers[name] = { + Muxer: StreamMuxer, + options: options + } + } + + self.dial = function (peerInfo, options, protocol, callback) { + // 1. check if we have transports we support + // 2. check if we have a conn waiting + // 3. check if we have a stream muxer available + + if (typeof protocol === 'function') { + callback = protocol + protocol = undefined + } + + // check if a conn is waiting + // if it is and protocol was selected, jump into multistreamHandshake + // if it is and no protocol was selected, do nothing and call and empty callback + + if (self.conns[peerInfo.id.toB58String()]) { + if (protocol) { + if (self.muxers['spdy']) { + // TODO upgrade this conn to a muxer + console.log('TODO: upgrade a warm conn to muxer that was added after') + } else { + multistreamHandshake(self.conns[peerInfo.id.toB58String()]) + } + self.conns[peerInfo.id.toB58String()] = undefined + delete self.conns[peerInfo.id.toB58String()] + return + } else { + return callback() + } + } + + // check if a stream muxer for this peer is available + if (self.muxedConns[peerInfo.id.toB58String()]) { + if (protocol) { + return openMuxedStream(self.muxedConns[peerInfo.id.toB58String()].muxer) + } else { + return callback() + } + } + + // Creating a new conn with this peer routine + + // TODO - check if there is a preference in protocol to use on + // options.protocol + var supportedTransports = Object.keys(self.transports) + var multiaddrs = peerInfo.multiaddrs.filter(function (multiaddr) { + return multiaddr.protoNames().some(function (proto) { + return supportedTransports.indexOf(proto) >= 0 + }) + }) + + if (!multiaddrs.length) { + callback(new Error("The swarm doesn't support any of the peer transports")) + return + } + + var conn + + async.eachSeries(multiaddrs, function (multiaddr, next) { + if (conn) { + return next() + } + + var transportName = getTransportNameForMultiaddr(multiaddr) + var transport = self.transports[transportName] + var dialOptions = clone(transport.dialOptions) + dialOptions.ready = connected + + var connTry = transport.transport.dial(multiaddr, dialOptions) + + connTry.once('error', function (err) { + if (err) { + return console.log(err) // TODO handle error + } + next() // try next multiaddr + }) + + function connected () { + conn = connTry + next() + } + + function getTransportNameForMultiaddr (multiaddr) { + // this works for all those ip + transport + port tripplets + return multiaddr.protoNames()[1] + } + + function clone (obj) { + var target = {} + for (var i in obj) { + if (obj.hasOwnProperty(i)) { + target[i] = obj[i] + } + } + return target + } + }, done) + + function done () { + // TODO apply upgrades + // apply stream muxer + // if no protocol is selected, save it in the pool + // if protocol is selected, multistream that protocol + if (!conn) { + callback(new Error('Unable to open a connection')) + return + } + + if (self.muxers['spdy']) { + var spdy = new self.muxers['spdy'].Muxer(self.muxers['spdy'].options) + spdy.attach(conn, false, function (err, muxer) { + if (err) { + return console.log(err) // TODO Treat error + } + + muxer.on('stream', userProtocolMuxer) + + self.muxedConns[peerInfo.id.toB58String()] = { + muxer: muxer, + socket: conn + } + + if (protocol) { + openMuxedStream(muxer) + } else { + callback() + } + }) + } else { + if (protocol) { + multistreamHandshake(conn) + } else { + self.conns[peerInfo.id.toB58String()] = conn + callback() + } + } + } + + function openMuxedStream (muxer) { + // 1. create a new stream on this muxedConn and pass that to + // multistreamHanshake + muxer.dialStream(function (err, conn) { + if (err) { + return console.log(err) // TODO Treat error + } + multistreamHandshake(conn) + }) + } + + function multistreamHandshake (conn) { + var msI = new multistream.Interactive() + msI.handle(conn, function () { + msI.select(protocol, callback) + }) + } + } + + self.closeListener = function (transportName, callback) { + self.transports[transportName].listener.close(closed) + + // only gets called when all the streams on this transport are closed too + function closed () { + delete self.transports[transportName] + callback() + } + } + + // Iterates all the listeners closing them + // one by one. It calls back once all are closed. + self.closeAllListeners = function (callback) { + var transportNames = Object.keys(self.transports) + + async.each(transportNames, self.closeListener, callback) + } + + self.closeConns = function (callback) { + // close warmed up cons so that the listener can gracefully exit + Object.keys(self.conns).forEach(function (conn) { + self.conns[conn].destroy() + }) + self.conns = {} + + callback() + } + + // Closes both transport listeners and + // connections. It calls back once everything + // is closed + self.close = function (callback) { + async.parallel([ + self.closeAllListeners, + self.closeConns + ], callback) + } + + self.enableIdentify = function () { + // set flag to true + // add identify to the list of handled protocols + self.identify = true + + // we pass muxedConns so that identify can access the socket of the other + // peer + self.handleProtocol(identify.protoId, + identify.getHandlerFunction(self.peerInfo, self.muxedConns)) + } + + self.handleProtocol = function (protocol, handlerFunction) { + self.protocols[protocol] = handlerFunction + } + + // internals + + function listen (conn) { + // TODO apply upgrades + // add StreamMuxer if available (and point streams from muxer to userProtocolMuxer) + + if (self.muxers['spdy']) { + var spdy = new self.muxers['spdy'].Muxer(self.muxers['spdy'].options) + spdy.attach(conn, true, function (err, muxer) { + if (err) { + return console.log(err) // TODO treat error + } + + // TODO This muxer has to be identified! + // pass to identify a reference of + // our muxedConn list + // ourselves (peerInfo) + // the conn, which is the socket + // and a stream it can send stuff + if (self.identify) { + muxer.dialStream(function (err, stream) { + if (err) { + return console.log(err) // TODO Treat error + } + // conn === socket at this point + identify(self.muxedConns, self.peerInfo, conn, stream, muxer) + }) + } + + muxer.on('stream', userProtocolMuxer) + }) + } else { + // if no stream muxer, then + userProtocolMuxer(conn) + } + } + + // Handle user given protocols + function userProtocolMuxer (conn) { + var msS = new multistream.Select() + msS.handle(conn) + Object.keys(self.protocols).forEach(function (protocol) { + msS.addHandler(protocol, self.protocols[protocol]) + }) + } +} diff --git a/src/swarm.js b/src/swarm.js deleted file mode 100644 index 54c254265a..0000000000 --- a/src/swarm.js +++ /dev/null @@ -1,343 +0,0 @@ -var multistream = require('multistream-select') -var async = require('async') -var identify = require('./identify') - -exports = module.exports = Swarm - -function Swarm (peerInfo) { - var self = this - - if (!(self instanceof Swarm)) { - throw new Error('Swarm must be called with new') - } - - if (!peerInfo) { - throw new Error('You must provide a value for `peerInfo`') - } - - self.peerInfo = peerInfo - - // peerIdB58: { conn: } - self.conns = {} - - // peerIdB58: { - // muxer: , - // socket: socket // so we can extract the info we need for identify - // } - self.muxedConns = {} - - // transportName: { transport: transport, - // dialOptions: dialOptions, - // listenOptions: listenOptions, - // listeners: [] } - self.transports = {} - - // transportName: listener - self.listeners = {} - - // protocolName: handlerFunc - self.protocols = {} - - // muxerName: { Muxer: Muxer // Muxer is a constructor - // options: options } - self.muxers = {} - - // for connection reuse - self.identify = false - - // public interface - - self.addTransport = function (name, transport, options, dialOptions, listenOptions, callback) { - // set up the transport and add the list of incoming streams - // add transport to the list of transports - - var multiaddr = options.multiaddr - if (multiaddr) { - // no need to pass that to the transports - delete options.multiaddr - } - - var listener = transport.createListener(options, listen) - - listener.listen(listenOptions, function ready () { - self.transports[name] = { - transport: transport, - options: options, - dialOptions: dialOptions, - listenOptions: listenOptions, - listener: listener - } - - // If a known multiaddr is passed, then add to our list of multiaddrs - if (multiaddr) { - self.peerInfo.multiaddrs.push(multiaddr) - } - - callback() - }) - } - - self.addUpgrade = function (ConnUpgrade, options) {} - - self.addStreamMuxer = function (name, StreamMuxer, options) { - self.muxers[name] = { - Muxer: StreamMuxer, - options: options - } - } - - self.dial = function (peerInfo, options, protocol, callback) { - // 1. check if we have transports we support - // 2. check if we have a conn waiting - // 3. check if we have a stream muxer available - - if (typeof protocol === 'function') { - callback = protocol - protocol = undefined - } - - // check if a conn is waiting - // if it is and protocol was selected, jump into multistreamHandshake - // if it is and no protocol was selected, do nothing and call and empty callback - - if (self.conns[peerInfo.id.toB58String()]) { - if (protocol) { - if (self.muxers['spdy']) { - // TODO upgrade this conn to a muxer - console.log('TODO: upgrade a warm conn to muxer that was added after') - } else { - multistreamHandshake(self.conns[peerInfo.id.toB58String()]) - } - self.conns[peerInfo.id.toB58String()] = undefined - delete self.conns[peerInfo.id.toB58String()] - return - } else { - return callback() - } - } - - // check if a stream muxer for this peer is available - if (self.muxedConns[peerInfo.id.toB58String()]) { - if (protocol) { - return openMuxedStream(self.muxedConns[peerInfo.id.toB58String()].muxer) - } else { - return callback() - } - } - - // Creating a new conn with this peer routine - - // TODO - check if there is a preference in protocol to use on - // options.protocol - var supportedTransports = Object.keys(self.transports) - var multiaddrs = peerInfo.multiaddrs.filter(function (multiaddr) { - return multiaddr.protoNames().some(function (proto) { - return supportedTransports.indexOf(proto) >= 0 - }) - }) - - if (!multiaddrs.length) { - callback(new Error("The swarm doesn't support any of the peer transports")) - return - } - - var conn - - async.eachSeries(multiaddrs, function (multiaddr, next) { - if (conn) { - return next() - } - - var transportName = getTransportNameForMultiaddr(multiaddr) - var transport = self.transports[transportName] - var dialOptions = clone(transport.dialOptions) - dialOptions.ready = connected - - var connTry = transport.transport.dial(multiaddr, dialOptions) - - connTry.once('error', function (err) { - if (err) { - return console.log(err) // TODO handle error - } - next() // try next multiaddr - }) - - function connected () { - conn = connTry - next() - } - - function getTransportNameForMultiaddr (multiaddr) { - // this works for all those ip + transport + port tripplets - return multiaddr.protoNames()[1] - } - - function clone (obj) { - var target = {} - for (var i in obj) { - if (obj.hasOwnProperty(i)) { - target[i] = obj[i] - } - } - return target - } - }, done) - - function done () { - // TODO apply upgrades - // apply stream muxer - // if no protocol is selected, save it in the pool - // if protocol is selected, multistream that protocol - if (!conn) { - callback(new Error('Unable to open a connection')) - return - } - - if (self.muxers['spdy']) { - var spdy = new self.muxers['spdy'].Muxer(self.muxers['spdy'].options) - spdy.attach(conn, false, function (err, muxer) { - if (err) { - return console.log(err) // TODO Treat error - } - - muxer.on('stream', userProtocolMuxer) - - self.muxedConns[peerInfo.id.toB58String()] = { - muxer: muxer, - socket: conn - } - - if (protocol) { - openMuxedStream(muxer) - } else { - callback() - } - }) - } else { - if (protocol) { - multistreamHandshake(conn) - } else { - self.conns[peerInfo.id.toB58String()] = conn - callback() - } - } - } - - function openMuxedStream (muxer) { - // 1. create a new stream on this muxedConn and pass that to - // multistreamHanshake - muxer.dialStream(function (err, conn) { - if (err) { - return console.log(err) // TODO Treat error - } - multistreamHandshake(conn) - }) - } - - function multistreamHandshake (conn) { - var msI = new multistream.Interactive() - msI.handle(conn, function () { - msI.select(protocol, callback) - }) - } - } - - self.closeListener = function (transportName, callback) { - self.transports[transportName].listener.close(closed) - - // only gets called when all the streams on this transport are closed too - function closed () { - delete self.transports[transportName] - callback() - } - } - - // Iterates all the listeners closing them - // one by one. It calls back once all are closed. - self.closeAllListeners = function (callback) { - var transportNames = Object.keys(self.transports) - - async.each(transportNames, self.closeListener, callback) - } - - self.closeConns = function (callback) { - // close warmed up cons so that the listener can gracefully exit - Object.keys(self.conns).forEach(function (conn) { - self.conns[conn].destroy() - }) - self.conns = {} - - callback() - } - - // Closes both transport listeners and - // connections. It calls back once everything - // is closed - self.close = function (callback) { - async.parallel([ - self.closeAllListeners, - self.closeConns - ], callback) - } - - self.enableIdentify = function () { - // set flag to true - // add identify to the list of handled protocols - self.identify = true - - // we pass muxedConns so that identify can access the socket of the other - // peer - self.handleProtocol(identify.protoId, - identify.getHandlerFunction(self.peerInfo, self.muxedConns)) - } - - self.handleProtocol = function (protocol, handlerFunction) { - self.protocols[protocol] = handlerFunction - } - - // internals - - function listen (conn) { - // TODO apply upgrades - // add StreamMuxer if available (and point streams from muxer to userProtocolMuxer) - - if (self.muxers['spdy']) { - var spdy = new self.muxers['spdy'].Muxer(self.muxers['spdy'].options) - spdy.attach(conn, true, function (err, muxer) { - if (err) { - return console.log(err) // TODO treat error - } - - // TODO This muxer has to be identified! - // pass to identify a reference of - // our muxedConn list - // ourselves (peerInfo) - // the conn, which is the socket - // and a stream it can send stuff - if (self.identify) { - muxer.dialStream(function (err, stream) { - if (err) { - return console.log(err) // TODO Treat error - } - // conn === socket at this point - identify(self.muxedConns, self.peerInfo, conn, stream, muxer) - }) - } - - muxer.on('stream', userProtocolMuxer) - }) - } else { - // if no stream muxer, then - userProtocolMuxer(conn) - } - } - - // Handle user given protocols - function userProtocolMuxer (conn) { - var msS = new multistream.Select() - msS.handle(conn) - Object.keys(self.protocols).forEach(function (protocol) { - msS.addHandler(protocol, self.protocols[protocol]) - }) - } -} diff --git a/tests/multistream-and-muxer-old.js b/tests/multistream-and-muxer-old.js deleted file mode 100644 index a7e294773e..0000000000 --- a/tests/multistream-and-muxer-old.js +++ /dev/null @@ -1,139 +0,0 @@ -var Lab = require('lab') -var Code = require('code') -var lab = exports.lab = Lab.script() - -var experiment = lab.experiment -var test = lab.test -var beforeEach = lab.beforeEach -var afterEach = lab.afterEach -var expect = Code.expect - -var Muxer = require('./../src/stream-muxer.js') -var multistream = require('multistream-select') -var Interactive = multistream.Interactive -var Select = multistream.Select -var streamPair = require('stream-pair') - -beforeEach(function (done) { - done() -}) - -afterEach(function (done) { - done() -}) - -experiment('MULTISTREAM AND STREAM MUXER', function () { - test('Open a socket and multistream-select it into spdy', function (done) { - var pair = streamPair.create() - - var msI = new Interactive() - var msS = new Select() - - var dialerMuxer = new Muxer() - var listenerMuxer = new Muxer() - - msS.handle(pair.other) - - msS.addHandler('/spdy/0.3.1', function (stream) { - var listenerConn = listenerMuxer.attach(stream, true) - expect(typeof listenerConn).to.be.equal('object') - done() - }) - - msI.handle(pair, function () { - msI.select('/spdy/0.3.1', function (err, stream) { - expect(err).to.not.be.instanceof(Error) - var dialerConn = dialerMuxer.attach(stream, false) - expect(typeof dialerConn).to.be.equal('object') - }) - }) - }) - - test('socket->ms-select into spdy->stream from dialer->ms-select into other protocol', function (done) { - var pair = streamPair.create() - - var msI = new Interactive() - var msS = new Select() - - var dialerMuxer = new Muxer() - var listenerMuxer = new Muxer() - - msS.handle(pair.other) - - msS.addHandler('/spdy/0.3.1', function (stream) { - var listenerConn = listenerMuxer.attach(stream, true) - listenerConn.on('stream', function (stream) { - stream.on('data', function (chunk) { - expect(chunk.toString()).to.equal('mux all the streams') - done() - }) - }) - }) - - msI.handle(pair, function () { - msI.select('/spdy/0.3.1', function (err, stream) { - expect(err).to.not.be.instanceof(Error) - var dialerConn = dialerMuxer.attach(stream, false) - dialerConn.dialStream(function (err, stream) { - expect(err).to.not.be.instanceof(Error) - stream.write('mux all the streams') - }) - }) - }) - }) - - test('socket->ms-select into spdy->stream from listener->ms-select into another protocol', function (done) { - var pair = streamPair.create() - - var msI = new Interactive() - var msS = new Select() - - var dialerMuxer = new Muxer() - var listenerMuxer = new Muxer() - - msS.handle(pair.other) - - msS.addHandler('/spdy/0.3.1', function (stream) { - var listenerConn = listenerMuxer.attach(stream, true) - listenerConn.on('stream', function (stream) { - stream.on('data', function (chunk) { - expect(chunk.toString()).to.equal('mux all the streams') - - listenerConn.dialStream(function (err, stream) { - expect(err).to.not.be.instanceof(Error) - var msI2 = new Interactive() - msI2.handle(stream, function () { - msI2.select('/other/protocol', function (err, stream) { - expect(err).to.not.be.instanceof(Error) - stream.write('the other protocol') - }) - }) - }) - }) - }) - }) - - msI.handle(pair, function () { - msI.select('/spdy/0.3.1', function (err, stream) { - expect(err).to.not.be.instanceof(Error) - var dialerConn = dialerMuxer.attach(stream, false) - dialerConn.dialStream(function (err, stream) { - expect(err).to.not.be.instanceof(Error) - stream.write('mux all the streams') - }) - - dialerConn.on('stream', function (stream) { - var msS2 = new Select() - msS2.handle(stream) - msS2.addHandler('/other/protocol', function (stream) { - stream.on('data', function (chunk) { - expect(chunk.toString()).to.equal('the other protocol') - done() - }) - }) - }) - }) - }) - - }) -}) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 07a20147e5..0acd765406 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -1,13 +1,7 @@ -var Lab = require('lab') -var Code = require('code') -var lab = exports.lab = Lab.script() -var async = require('async') +/* eslint-env mocha */ -var experiment = lab.experiment -var test = lab.test -var beforeEach = lab.beforeEach -var afterEach = lab.afterEach -var expect = Code.expect +var async = require('async') +var expect = require('chai').expect var multiaddr = require('multiaddr') var Id = require('peer-id') @@ -21,15 +15,15 @@ process.on('uncaughtException', function (err) { console.log('Caught exception: ' + err) }) -experiment('Basics', function () { - test('enforces creation with new', function (done) { +describe('Basics', function () { + it('enforces creation with new', function (done) { expect(function () { Swarm() }).to.throw() done() }) - test('it throws an exception without peerSelf', function (done) { + it('it throws an exception without peerSelf', function (done) { expect(function () { var sw = new Swarm() sw.close() @@ -38,24 +32,24 @@ experiment('Basics', function () { }) }) -experiment('When dialing', function () { - experiment('if the swarm does add any of the peer transports', function () { - test('it returns an error', function (done) { +describe('When dialing', function () { + describe('if the swarm does add any of the peer transports', function () { + it('it returns an error', function (done) { var peerOne = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8090')]) var peerTwo = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8091')]) var swarm = new Swarm(peerOne) swarm.dial(peerTwo, {}, function (err) { - expect(err).to.exist() + expect(err).to.exist done() }) }) }) }) -experiment('Without a Stream Muxer', function () { - experiment('and one swarm over tcp', function () { - test('add the transport', function (done) { +describe('Without a Stream Muxer', function () { + describe('and one swarm over tcp', function () { + it('add the transport', function (done) { var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') var p = new Peer(Id.create(), []) var sw = new Swarm(p) @@ -73,7 +67,7 @@ experiment('Without a Stream Muxer', function () { }) }) - experiment('and two swarms over tcp', function () { + describe('and two swarms over tcp', function () { var mh1, p1, sw1, mh2, p2, sw2 beforeEach(function (done) { @@ -99,7 +93,7 @@ experiment('Without a Stream Muxer', function () { async.parallel([sw1.close, sw2.close], done) }) - test('dial a conn', function (done) { + it('dial a conn', function (done) { sw1.dial(p2, {}, function (err) { expect(err).to.equal(undefined) expect(Object.keys(sw1.conns).length).to.equal(1) @@ -107,7 +101,7 @@ experiment('Without a Stream Muxer', function () { }) }) - test('dial a conn on a protocol', function (done) { + it('dial a conn on a protocol', function (done) { sw2.handleProtocol('/sparkles/1.0.0', function (conn) { conn.end() conn.on('end', done) @@ -120,7 +114,7 @@ experiment('Without a Stream Muxer', function () { }) }) - test('dial a protocol on a previous created conn', function (done) { + it('dial a protocol on a previous created conn', function (done) { sw2.handleProtocol('/sparkles/1.0.0', function (conn) { conn.end() conn.on('end', done) @@ -139,46 +133,46 @@ experiment('Without a Stream Muxer', function () { }) }) - // test('add an upgrade', function (done) { done() }) - // test('dial a conn on top of a upgrade', function (done) { done() }) - // test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + // it('add an upgrade', function (done) { done() }) + // it('dial a conn on top of a upgrade', function (done) { done() }) + // it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) }) /* TODO - experiment('udp', function () { - test('add the transport', function (done) { done() }) - test('dial a conn', function (done) { done() }) - test('dial a conn on a protocol', function (done) { done() }) - test('add an upgrade', function (done) { done() }) - test('dial a conn on top of a upgrade', function (done) { done() }) - test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + describe('udp', function () { + it('add the transport', function (done) { done() }) + it('dial a conn', function (done) { done() }) + it('dial a conn on a protocol', function (done) { done() }) + it('add an upgrade', function (done) { done() }) + it('dial a conn on top of a upgrade', function (done) { done() }) + it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) }) */ /* TODO - experiment('udt', function () { - test('add the transport', function (done) { done() }) - test('dial a conn', function (done) { done() }) - test('dial a conn on a protocol', function (done) { done() }) - test('add an upgrade', function (done) { done() }) - test('dial a conn on top of a upgrade', function (done) { done() }) - test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + describe('udt', function () { + it('add the transport', function (done) { done() }) + it('dial a conn', function (done) { done() }) + it('dial a conn on a protocol', function (done) { done() }) + it('add an upgrade', function (done) { done() }) + it('dial a conn on top of a upgrade', function (done) { done() }) + it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) }) */ /* TODO -experiment('utp', function () { - test('add the transport', function (done) { done() }) - test('dial a conn', function (done) { done() }) - test('dial a conn on a protocol', function (done) { done() }) - test('add an upgrade', function (done) { done() }) - test('dial a conn on top of a upgrade', function (done) { done() }) - test('dial a conn on a protocol on top of a upgrade', function (done) { done() }) +describe('utp', function () { + it('add the transport', function (done) { done() }) + it('dial a conn', function (done) { done() }) + it('dial a conn on a protocol', function (done) { done() }) + it('add an upgrade', function (done) { done() }) + it('dial a conn on top of a upgrade', function (done) { done() }) + it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) }) */ }) -experiment('With a SPDY Stream Muxer', function () { - experiment('and one swarm over tcp', function () { - // TODO: What is the test here? - test('add Stream Muxer', function (done) { +describe('With a SPDY Stream Muxer', function () { + describe('and one swarm over tcp', function () { + // TODO: What is the it here? + it('add Stream Muxer', function (done) { // var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') var p = new Peer(Id.create(), []) var sw = new Swarm(p) @@ -188,7 +182,7 @@ experiment('With a SPDY Stream Muxer', function () { }) }) - experiment('and two swarms over tcp', function () { + describe('and two swarms over tcp', function () { var mh1, p1, sw1, mh2, p2, sw2 beforeEach(function (done) { @@ -233,7 +227,7 @@ experiment('With a SPDY Stream Muxer', function () { } } - test('dial a conn on a protocol', function (done) { + it('dial a conn on a protocol', function (done) { sw2.handleProtocol('/sparkles/1.0.0', function (conn) { // formallity so that the conn starts flowing conn.on('data', function (chunk) {}) @@ -254,7 +248,7 @@ experiment('With a SPDY Stream Muxer', function () { }) }) - test('dial two conns (transport reuse)', function (done) { + it('dial two conns (transport reuse)', function (done) { sw2.handleProtocol('/sparkles/1.0.0', function (conn) { // formality so that the conn starts flowing conn.on('data', function (chunk) {}) @@ -287,7 +281,7 @@ experiment('With a SPDY Stream Muxer', function () { }) }) - experiment('and two identity enabled swarms over tcp', function () { + describe('and two identity enabled swarms over tcp', function () { var mh1, p1, sw1, mh2, p2, sw2 beforeEach(function (done) { @@ -337,7 +331,7 @@ experiment('With a SPDY Stream Muxer', function () { } }) - test('identify', function (done) { + it('identify', function (done) { sw2.handleProtocol('/sparkles/1.0.0', function (conn) { // formallity so that the conn starts flowing conn.on('data', function (chunk) {}) From c651dd2aece2b42009afe60bd3910946d78939ee Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 3 Mar 2016 12:18:00 +0000 Subject: [PATCH 086/634] remove coverage folder and solve new linting issue --- .gitignore | 3 ++- src/identify.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 554436631f..e3386ccc26 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ + # Logs logs *.log @@ -27,6 +28,6 @@ build/Release node_modules - +coverage .jshintrc .jshintignore diff --git a/src/identify.js b/src/identify.js index 63684631e1..beded68b12 100644 --- a/src/identify.js +++ b/src/identify.js @@ -6,7 +6,8 @@ var Interactive = require('multistream-select').Interactive var protobufs = require('protocol-buffers-stream') var fs = require('fs') -var schema = fs.readFileSync(__dirname + '/identify.proto') +var path = require('path') +var schema = fs.readFileSync(path.join(__dirname, '/identify.proto')) var Address6 = require('ip-address').Address6 var Id = require('peer-id') var multiaddr = require('multiaddr') From 1fd6a3885dc0ec89f413039c1be711bdeb1204a7 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 3 Mar 2016 12:20:54 +0000 Subject: [PATCH 087/634] remove coverage folder and solve new linting issue --- package.json | 2 +- src/identify.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 2574c65826..180bb5bca4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.5.5", + "version": "0.5.6", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { diff --git a/src/identify.js b/src/identify.js index beded68b12..bf78cd89dc 100644 --- a/src/identify.js +++ b/src/identify.js @@ -7,7 +7,7 @@ var Interactive = require('multistream-select').Interactive var protobufs = require('protocol-buffers-stream') var fs = require('fs') var path = require('path') -var schema = fs.readFileSync(path.join(__dirname, '/identify.proto')) +var schema = fs.readFileSync(path.join(__dirname, 'identify.proto')) var Address6 = require('ip-address').Address6 var Id = require('peer-id') var multiaddr = require('multiaddr') From c8f2fdd0775129971033e0067090be7890c577fa Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 3 Mar 2016 16:35:05 +0000 Subject: [PATCH 088/634] internal transport interface + libp2p-tcp tests --- README.md | 13 +- package.json | 14 +- src/index.js | 162 +++++++++++++- tests/swarm-old.js | 354 ++++++++++++++++++++++++++++++ tests/swarm-test.js | 522 ++++++++++++++++++-------------------------- 5 files changed, 732 insertions(+), 333 deletions(-) create mode 100644 tests/swarm-old.js diff --git a/README.md b/README.md index dc4200c125..3b8225f24a 100644 --- a/README.md +++ b/README.md @@ -18,18 +18,23 @@ libp2p-swarm is used by libp2p but it can be also used as a standalone module. libp2p-swarm is available on npm and so, like any other npm module, just: ```bash -$ npm install libp2p-swarm --save +> npm install libp2p-swarm --save ``` And use it in your Node.js code as: ```JavaScript -var Swarm = require('libp2p-swarm') +const Swarm = require('libp2p-swarm') -var sw = new Swarm(peerInfoSelf) +const sw = new Swarm(peerInfo) ``` -peerInfoSelf is a [PeerInfo](https://github.com/diasdavid/js-peer-info) object that represents the peer creating this swarm instance. +peerInfo is a [PeerInfo](https://github.com/diasdavid/js-peer-info) object that represents the peer creating this swarm instance. + + +---------- +BELOW NEEDS AN UPDATE + ### Support a transport diff --git a/package.json b/package.json index 180bb5bca4..f69871191a 100644 --- a/package.json +++ b/package.json @@ -26,28 +26,26 @@ "test" ], "engines": { - "node": "^4.0.0" + "node": "^4.3.0" }, "devDependencies": { + "bl": "^1.1.2", "chai": "^3.5.0", "istanbul": "^0.4.2", "libp2p-spdy": "^0.1.0", - "libp2p-tcp": "^0.1.1", + "libp2p-tcp": "^0.2.1", "mocha": "^2.4.5", + "multiaddr": "^1.1.1", + "peer-id": "^0.5.1", + "peer-info": "^0.5.2", "pre-commit": "^1.1.2", - "sinon": "^1.15.4", "standard": "^6.0.7", "stream-pair": "^1.0.3" }, "dependencies": { "async": "^1.3.0", "ip-address": "^5.0.2", - "ipfs-logger": "^0.1.0", - "multiaddr": "^1.0.0", - "multiplex-stream-muxer": "^0.2.0", "multistream-select": "^0.6.1", - "peer-id": "^0.3.3", - "peer-info": "^0.3.2", "protocol-buffers-stream": "^1.2.0" } } diff --git a/src/index.js b/src/index.js index 54c254265a..f933418cd9 100644 --- a/src/index.js +++ b/src/index.js @@ -1,14 +1,161 @@ -var multistream = require('multistream-select') -var async = require('async') -var identify = require('./identify') +// const multistream = require('multistream-select') +// const async = require('async') +// const identify = require('./identify') +const PassThrough = require('stream').PassThrough exports = module.exports = Swarm function Swarm (peerInfo) { - var self = this + if (!(this instanceof Swarm)) { + return new Swarm(peerInfo) + } + + if (!peerInfo) { + throw new Error('You must provide a value for `peerInfo`') + } + + // transports -- + + // { key: transport }; e.g { tcp: } + this.transports = {} + + this.transport = {} + + this.transport.add = (key, transport, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + if (!callback) { callback = noop } + + if (this.transports[key]) { + throw new Error('There is already a transport with this key') + } + this.transports[key] = transport + callback() + } - if (!(self instanceof Swarm)) { - throw new Error('Swarm must be called with new') + this.transport.dial = (key, multiaddrs, callback) => { + const t = this.transports[key] + + // a) if multiaddrs.length = 1, return the conn from the + // transport, otherwise, create a passthrough + if (!Array.isArray(multiaddrs)) { + multiaddrs = [multiaddrs] + } + if (multiaddrs.length === 1) { + const conn = t.dial(multiaddrs.shift(), {ready: () => { + const cb = callback + callback = noop // this is done to avoid connection drops as connect errors + cb(null, conn) + }}) + conn.once('error', () => { + callback(new Error('failed to connect to every multiaddr')) + }) + return conn + } + + // b) multiaddrs should already be a filtered list + // specific for the transport we are using + const pt = new PassThrough() + + next(multiaddrs.shift()) + return pt + function next (multiaddr) { + const conn = t.dial(multiaddr, {ready: () => { + pt.pipe(conn).pipe(pt) + const cb = callback + callback = noop // this is done to avoid connection drops as connect errors + cb(null, pt) + }}) + + conn.once('error', () => { + if (multiaddrs.length === 0) { + return callback(new Error('failed to connect to every multiaddr')) + } + next(multiaddrs.shift()) + }) + } + } + + this.transport.listen = (key, options, handler, callback) => { + // if no callback is passed, we pass conns to connHandler + if (!handler) { handler = connHandler } + + const multiaddrs = peerInfo.multiaddrs.filter((m) => { + if (m.toString().indexOf('tcp') !== -1) { + return m + } + }) + + this.transports[key].createListener(multiaddrs, handler, (err, maUpdate) => { + if (err) { + return callback(err) + } + if (maUpdate) { + // because we can listen on port 0... + peerInfo.multiaddr.replace(multiaddrs, maUpdate) + } + + callback() + }) + } + + this.transport.close = (key, callback) => { + this.transports[key].close(callback) + } + + // connections -- + + // { peerIdB58: { conn: }} + this.conns = {} + + // { + // peerIdB58: { + // muxerName: { + // muxer: + // rawSocket: socket // to abstract info required for the Identify Protocol + // } + // } + this.muxedConns = {} + + this.connection = {} + this.connection.addUpgrade = () => {} + this.connection.addStreamMuxer = () => {} + + // enable the Identify protocol + this.connection.reuse = () => {} + + // main API - higher level abstractions -- + + this.dial = () => { + // TODO + } + this.handle = (protocol, callback) => { + // TODO + } + this.close = (callback) => { + var count = 0 + + Object.keys(this.transports).forEach((key) => { + this.transports[key].close(() => { + if (++count === Object.keys(this.transports).length) { + callback() + } + }) + }) + } + + function connHandler (conn) { + // do all the multistream select handshakes (this should be probably done recursively + } +} + +/* +function Swarm (peerInfo) { + var self = this + if (!(this instanceof Swarm)) { + return new Swarm(peerInfo) } if (!peerInfo) { @@ -341,3 +488,6 @@ function Swarm (peerInfo) { }) } } +*/ + +function noop () {} diff --git a/tests/swarm-old.js b/tests/swarm-old.js new file mode 100644 index 0000000000..0acd765406 --- /dev/null +++ b/tests/swarm-old.js @@ -0,0 +1,354 @@ +/* eslint-env mocha */ + +var async = require('async') +var expect = require('chai').expect + +var multiaddr = require('multiaddr') +var Id = require('peer-id') +var Peer = require('peer-info') +var Swarm = require('../src') +var tcp = require('libp2p-tcp') +var Spdy = require('libp2p-spdy') + +// because of Travis-CI +process.on('uncaughtException', function (err) { + console.log('Caught exception: ' + err) +}) + +describe('Basics', function () { + it('enforces creation with new', function (done) { + expect(function () { + Swarm() + }).to.throw() + done() + }) + + it('it throws an exception without peerSelf', function (done) { + expect(function () { + var sw = new Swarm() + sw.close() + }).to.throw(Error) + done() + }) +}) + +describe('When dialing', function () { + describe('if the swarm does add any of the peer transports', function () { + it('it returns an error', function (done) { + var peerOne = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8090')]) + var peerTwo = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8091')]) + var swarm = new Swarm(peerOne) + + swarm.dial(peerTwo, {}, function (err) { + expect(err).to.exist + done() + }) + }) + }) +}) + +describe('Without a Stream Muxer', function () { + describe('and one swarm over tcp', function () { + it('add the transport', function (done) { + var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') + var p = new Peer(Id.create(), []) + var sw = new Swarm(p) + + sw.addTransport('tcp', tcp, { multiaddr: mh }, {}, {port: 8010}, ready) + + function ready () { + expect(sw.transports['tcp'].options).to.deep.equal({}) + expect(sw.transports['tcp'].dialOptions).to.deep.equal({}) + expect(sw.transports['tcp'].listenOptions).to.deep.equal({port: 8010}) + expect(sw.transports['tcp'].transport).to.deep.equal(tcp) + + sw.close(done) + } + }) + }) + + describe('and two swarms over tcp', function () { + var mh1, p1, sw1, mh2, p2, sw2 + + beforeEach(function (done) { + mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + p1 = new Peer(Id.create(), []) + sw1 = new Swarm(p1) + + mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + p2 = new Peer(Id.create(), []) + sw2 = new Swarm(p2) + + async.parallel([ + function (cb) { + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) + }, + function (cb) { + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) + } + ], done) + }) + + afterEach(function (done) { + async.parallel([sw1.close, sw2.close], done) + }) + + it('dial a conn', function (done) { + sw1.dial(p2, {}, function (err) { + expect(err).to.equal(undefined) + expect(Object.keys(sw1.conns).length).to.equal(1) + done() + }) + }) + + it('dial a conn on a protocol', function (done) { + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + conn.end() + conn.on('end', done) + }) + + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) + }) + + it('dial a protocol on a previous created conn', function (done) { + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + conn.end() + conn.on('end', done) + }) + + sw1.dial(p2, {}, function (err) { + expect(err).to.equal(undefined) + expect(Object.keys(sw1.conns).length).to.equal(1) + + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + + conn.end() + }) + }) + }) + + // it('add an upgrade', function (done) { done() }) + // it('dial a conn on top of a upgrade', function (done) { done() }) + // it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + }) + + /* TODO + describe('udp', function () { + it('add the transport', function (done) { done() }) + it('dial a conn', function (done) { done() }) + it('dial a conn on a protocol', function (done) { done() }) + it('add an upgrade', function (done) { done() }) + it('dial a conn on top of a upgrade', function (done) { done() }) + it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + }) */ + + /* TODO + describe('udt', function () { + it('add the transport', function (done) { done() }) + it('dial a conn', function (done) { done() }) + it('dial a conn on a protocol', function (done) { done() }) + it('add an upgrade', function (done) { done() }) + it('dial a conn on top of a upgrade', function (done) { done() }) + it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + }) */ + +/* TODO +describe('utp', function () { + it('add the transport', function (done) { done() }) + it('dial a conn', function (done) { done() }) + it('dial a conn on a protocol', function (done) { done() }) + it('add an upgrade', function (done) { done() }) + it('dial a conn on top of a upgrade', function (done) { done() }) + it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) +}) */ +}) + +describe('With a SPDY Stream Muxer', function () { + describe('and one swarm over tcp', function () { + // TODO: What is the it here? + it('add Stream Muxer', function (done) { + // var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') + var p = new Peer(Id.create(), []) + var sw = new Swarm(p) + sw.addStreamMuxer('spdy', Spdy, {}) + + done() + }) + }) + + describe('and two swarms over tcp', function () { + var mh1, p1, sw1, mh2, p2, sw2 + + beforeEach(function (done) { + mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + p1 = new Peer(Id.create(), []) + sw1 = new Swarm(p1) + sw1.addStreamMuxer('spdy', Spdy, {}) + + mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + p2 = new Peer(Id.create(), []) + sw2 = new Swarm(p2) + sw2.addStreamMuxer('spdy', Spdy, {}) + + async.parallel([ + function (cb) { + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) + }, + function (cb) { + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) + } + ], done) + }) + + function afterEach (done) { + var cleaningCounter = 0 + sw1.closeConns(cleaningUp) + sw2.closeConns(cleaningUp) + + sw1.closeListener('tcp', cleaningUp) + sw2.closeListener('tcp', cleaningUp) + + function cleaningUp () { + cleaningCounter++ + // TODO FIX: here should be 4, but because super wrapping of + // streams, it makes it so hard to properly close the muxed + // streams - https://github.com/indutny/spdy-transport/issues/14 + if (cleaningCounter < 3) { + return + } + + done() + } + } + + it('dial a conn on a protocol', function (done) { + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + // formallity so that the conn starts flowing + conn.on('data', function (chunk) {}) + + conn.end() + conn.on('end', function () { + expect(Object.keys(sw1.muxedConns).length).to.equal(1) + expect(Object.keys(sw2.muxedConns).length).to.equal(0) + afterEach(done) + }) + }) + + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) + }) + + it('dial two conns (transport reuse)', function (done) { + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + // formality so that the conn starts flowing + conn.on('data', function (chunk) {}) + + conn.end() + conn.on('end', function () { + expect(Object.keys(sw1.muxedConns).length).to.equal(1) + expect(Object.keys(sw2.muxedConns).length).to.equal(0) + + afterEach(done) + }) + }) + + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + // TODO Improve clarity + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + + conn.end() + }) + + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + + conn.end() + }) + }) + }) + + describe('and two identity enabled swarms over tcp', function () { + var mh1, p1, sw1, mh2, p2, sw2 + + beforeEach(function (done) { + mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') + p1 = new Peer(Id.create(), []) + sw1 = new Swarm(p1) + sw1.addStreamMuxer('spdy', Spdy, {}) + sw1.enableIdentify() + + mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') + p2 = new Peer(Id.create(), []) + sw2 = new Swarm(p2) + sw2.addStreamMuxer('spdy', Spdy, {}) + sw2.enableIdentify() + + async.parallel([ + function (cb) { + sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) + }, + function (cb) { + sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) + } + ], done) + }) + + afterEach(function (done) { + var cleaningCounter = 0 + sw1.closeConns(cleaningUp) + sw2.closeConns(cleaningUp) + + sw1.closeListener('tcp', cleaningUp) + sw2.closeListener('tcp', cleaningUp) + + function cleaningUp () { + cleaningCounter++ + // TODO FIX: here should be 4, but because super wrapping of + // streams, it makes it so hard to properly close the muxed + // streams - https://github.com/indutny/spdy-transport/issues/14 + if (cleaningCounter < 3) { + return + } + // give time for identify to finish + setTimeout(function () { + expect(Object.keys(sw2.muxedConns).length).to.equal(1) + done() + }, 500) + } + }) + + it('identify', function (done) { + sw2.handleProtocol('/sparkles/1.0.0', function (conn) { + // formallity so that the conn starts flowing + conn.on('data', function (chunk) {}) + + conn.end() + conn.on('end', function () { + expect(Object.keys(sw1.muxedConns).length).to.equal(1) + done() + }) + }) + + sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { + conn.on('data', function () {}) + expect(err).to.equal(null) + expect(Object.keys(sw1.conns).length).to.equal(0) + conn.end() + }) + }) + }) +}) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 0acd765406..a9ecfb2680 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -1,354 +1,246 @@ /* eslint-env mocha */ -var async = require('async') -var expect = require('chai').expect - -var multiaddr = require('multiaddr') -var Id = require('peer-id') -var Peer = require('peer-info') -var Swarm = require('../src') -var tcp = require('libp2p-tcp') -var Spdy = require('libp2p-spdy') - -// because of Travis-CI -process.on('uncaughtException', function (err) { - console.log('Caught exception: ' + err) -}) - -describe('Basics', function () { - it('enforces creation with new', function (done) { - expect(function () { - Swarm() - }).to.throw() - done() - }) - - it('it throws an exception without peerSelf', function (done) { - expect(function () { - var sw = new Swarm() - sw.close() - }).to.throw(Error) +const expect = require('chai').expect +// const async = require('async') + +const multiaddr = require('multiaddr') +// const Id = require('peer-id') +const Peer = require('peer-info') +const Swarm = require('../src') +const TCP = require('libp2p-tcp') +const bl = require('bl') +// var SPDY = require('libp2p-spdy') + +describe('basics', () => { + it('throws on missing peerInfo', (done) => { + expect(Swarm).to.throw(Error) done() }) }) -describe('When dialing', function () { - describe('if the swarm does add any of the peer transports', function () { - it('it returns an error', function (done) { - var peerOne = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8090')]) - var peerTwo = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8091')]) - var swarm = new Swarm(peerOne) - - swarm.dial(peerTwo, {}, function (err) { - expect(err).to.exist - done() - }) - }) - }) -}) - -describe('Without a Stream Muxer', function () { - describe('and one swarm over tcp', function () { - it('add the transport', function (done) { - var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') - var p = new Peer(Id.create(), []) - var sw = new Swarm(p) - - sw.addTransport('tcp', tcp, { multiaddr: mh }, {}, {port: 8010}, ready) +describe('transport - tcp', function () { + this.timeout(10000) - function ready () { - expect(sw.transports['tcp'].options).to.deep.equal({}) - expect(sw.transports['tcp'].dialOptions).to.deep.equal({}) - expect(sw.transports['tcp'].listenOptions).to.deep.equal({port: 8010}) - expect(sw.transports['tcp'].transport).to.deep.equal(tcp) + var swarmA + var swarmB + var peerA = new Peer() + var peerB = new Peer() - sw.close(done) - } - }) + before((done) => { + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9888')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9999')) + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + done() }) - describe('and two swarms over tcp', function () { - var mh1, p1, sw1, mh2, p2, sw2 - - beforeEach(function (done) { - mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - p1 = new Peer(Id.create(), []) - sw1 = new Swarm(p1) - - mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - p2 = new Peer(Id.create(), []) - sw2 = new Swarm(p2) - - async.parallel([ - function (cb) { - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) - }, - function (cb) { - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) - } - ], done) - }) - - afterEach(function (done) { - async.parallel([sw1.close, sw2.close], done) + it('add', (done) => { + swarmA.transport.add('tcp', new TCP()) + expect(Object.keys(swarmA.transports).length).to.equal(1) + swarmB.transport.add('tcp', new TCP(), () => { + expect(Object.keys(swarmB.transports).length).to.equal(1) + done() }) + }) - it('dial a conn', function (done) { - sw1.dial(p2, {}, function (err) { - expect(err).to.equal(undefined) - expect(Object.keys(sw1.conns).length).to.equal(1) + it('listen', (done) => { + var count = 0 + swarmA.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + swarmB.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + if (++count === 2) { + expect(peerA.multiaddrs.length).to.equal(1) + expect(peerA.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9888')) + expect(peerB.multiaddrs.length).to.equal(1) + expect(peerB.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9999')) done() - }) - }) + } + } + }) - it('dial a conn on a protocol', function (done) { - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - conn.end() - conn.on('end', done) - }) - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) + it('dial to a multiaddr', (done) => { + const conn = swarmA.transport.dial('tcp', multiaddr('/ip4/127.0.0.1/tcp/9999'), (err, conn) => { + expect(err).to.not.exist }) + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + done() + })) + conn.write('hey') + conn.end() + }) - it('dial a protocol on a previous created conn', function (done) { - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - conn.end() - conn.on('end', done) - }) - - sw1.dial(p2, {}, function (err) { - expect(err).to.equal(undefined) - expect(Object.keys(sw1.conns).length).to.equal(1) - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - - conn.end() - }) - }) + it('dial to set of multiaddr, only one is available', (done) => { + const conn = swarmA.transport.dial('tcp', [ + multiaddr('/ip4/127.0.0.1/tcp/9910'), + multiaddr('/ip4/127.0.0.1/tcp/9999'), + multiaddr('/ip4/127.0.0.1/tcp/9309') + ], (err, conn) => { + expect(err).to.not.exist }) - - // it('add an upgrade', function (done) { done() }) - // it('dial a conn on top of a upgrade', function (done) { done() }) - // it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + done() + })) + conn.write('hey') + conn.end() }) - /* TODO - describe('udp', function () { - it('add the transport', function (done) { done() }) - it('dial a conn', function (done) { done() }) - it('dial a conn on a protocol', function (done) { done() }) - it('add an upgrade', function (done) { done() }) - it('dial a conn on top of a upgrade', function (done) { done() }) - it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) - }) */ - - /* TODO - describe('udt', function () { - it('add the transport', function (done) { done() }) - it('dial a conn', function (done) { done() }) - it('dial a conn on a protocol', function (done) { done() }) - it('add an upgrade', function (done) { done() }) - it('dial a conn on top of a upgrade', function (done) { done() }) - it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) - }) */ - -/* TODO -describe('utp', function () { - it('add the transport', function (done) { done() }) - it('dial a conn', function (done) { done() }) - it('dial a conn on a protocol', function (done) { done() }) - it('add an upgrade', function (done) { done() }) - it('dial a conn on top of a upgrade', function (done) { done() }) - it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) -}) */ -}) - -describe('With a SPDY Stream Muxer', function () { - describe('and one swarm over tcp', function () { - // TODO: What is the it here? - it('add Stream Muxer', function (done) { - // var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') - var p = new Peer(Id.create(), []) - var sw = new Swarm(p) - sw.addStreamMuxer('spdy', Spdy, {}) + it('close', (done) => { + var count = 0 + swarmA.transport.close('tcp', closed) + swarmB.transport.close('tcp', closed) - done() - }) + function closed () { + if (++count === 2) { + done() + } + } }) - describe('and two swarms over tcp', function () { - var mh1, p1, sw1, mh2, p2, sw2 - - beforeEach(function (done) { - mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - p1 = new Peer(Id.create(), []) - sw1 = new Swarm(p1) - sw1.addStreamMuxer('spdy', Spdy, {}) - - mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - p2 = new Peer(Id.create(), []) - sw2 = new Swarm(p2) - sw2.addStreamMuxer('spdy', Spdy, {}) - - async.parallel([ - function (cb) { - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) - }, - function (cb) { - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) - } - ], done) - }) - - function afterEach (done) { - var cleaningCounter = 0 - sw1.closeConns(cleaningUp) - sw2.closeConns(cleaningUp) + it('support port 0', (done) => { + var swarm + var peer = new Peer() + peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) + swarm = new Swarm(peer) + swarm.transport.add('tcp', new TCP()) + swarm.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + expect(peer.multiaddrs.length).to.equal(1) + expect(peer.multiaddrs[0]).to.not.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/0')) + swarm.close(done) + } + }) - sw1.closeListener('tcp', cleaningUp) - sw2.closeListener('tcp', cleaningUp) + it('support addr /ip4/0.0.0.0/tcp/9050', (done) => { + var swarm + var peer = new Peer() + peer.multiaddr.add(multiaddr('/ip4/0.0.0.0/tcp/9050')) + swarm = new Swarm(peer) + swarm.transport.add('tcp', new TCP()) + swarm.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + expect(peer.multiaddrs.length).to.equal(1) + expect(peer.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/0.0.0.0/tcp/9050')) + swarm.close(done) + } + }) - function cleaningUp () { - cleaningCounter++ - // TODO FIX: here should be 4, but because super wrapping of - // streams, it makes it so hard to properly close the muxed - // streams - https://github.com/indutny/spdy-transport/issues/14 - if (cleaningCounter < 3) { - return - } + it('support addr /ip4/0.0.0.0/tcp/0', (done) => { + var swarm + var peer = new Peer() + peer.multiaddr.add(multiaddr('/ip4/0.0.0.0/tcp/0')) + swarm = new Swarm(peer) + swarm.transport.add('tcp', new TCP()) + swarm.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + expect(peer.multiaddrs.length).to.equal(1) + expect(peer.multiaddrs[0]).to.not.deep.equal(multiaddr('/ip4/0.0.0.0/tcp/0')) + swarm.close(done) + } + }) - done() - } + it('listen in several addrs', (done) => { + var swarm + var peer = new Peer() + peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) + peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) + peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9003')) + swarm = new Swarm(peer) + swarm.transport.add('tcp', new TCP()) + swarm.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + expect(peer.multiaddrs.length).to.equal(3) + swarm.close(done) } + }) +}) - it('dial a conn on a protocol', function (done) { - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - // formallity so that the conn starts flowing - conn.on('data', function (chunk) {}) - - conn.end() - conn.on('end', function () { - expect(Object.keys(sw1.muxedConns).length).to.equal(1) - expect(Object.keys(sw2.muxedConns).length).to.equal(0) - afterEach(done) - }) - }) - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) - }) +describe('transport - udt', () => { + before((done) => { done() }) - it('dial two conns (transport reuse)', function (done) { - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - // formality so that the conn starts flowing - conn.on('data', function (chunk) {}) + it.skip('add', (done) => {}) + it.skip('listen', (done) => {}) + it.skip('dial', (done) => {}) + it.skip('close', (done) => {}) +}) - conn.end() - conn.on('end', function () { - expect(Object.keys(sw1.muxedConns).length).to.equal(1) - expect(Object.keys(sw2.muxedConns).length).to.equal(0) +describe('transport - websockets', () => { + before((done) => { done() }) - afterEach(done) - }) - }) + it.skip('add', (done) => {}) + it.skip('listen', (done) => {}) + it.skip('dial', (done) => {}) + it.skip('close', (done) => {}) +}) - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - // TODO Improve clarity - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) +describe('high level API - 1st stage, without stream multiplexing (on TCP)', () => { + it.skip('handle a protocol', (done) => {}) + it.skip('dial on protocol', (done) => {}) + it.skip('dial to warm conn', (done) => {}) + it.skip('dial on protocol, reuse warmed conn', (done) => {}) + it.skip('close', (done) => {}) +}) - conn.end() - }) +describe('stream muxing (on TCP)', () => { + describe('multiplex', () => { + before((done) => { done() }) + it.skip('add', (done) => {}) + it.skip('handle + dial on protocol', (done) => {}) + it.skip('dial to warm conn', (done) => {}) + it.skip('dial on protocol, reuse warmed conn', (done) => {}) + it.skip('enable identify to reuse incomming muxed conn', (done) => {}) + it.skip('close', (done) => {}) + }) + describe('spdy', () => { + it.skip('add', (done) => {}) + it.skip('handle + dial on protocol', (done) => {}) + it.skip('dial to warm conn', (done) => {}) + it.skip('dial on protocol, reuse warmed conn', (done) => {}) + it.skip('enable identify to reuse incomming muxed conn', (done) => {}) + it.skip('close', (done) => {}) + }) +}) - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) +describe('conn upgrades', () => { + describe('secio on tcp', () => { + before((done) => { done() }) - conn.end() - }) - }) + it.skip('add', (done) => {}) + it.skip('dial', (done) => {}) + it.skip('tls on a muxed stream (not the full conn)', (done) => {}) }) + describe('tls on tcp', () => { + before((done) => { done() }) - describe('and two identity enabled swarms over tcp', function () { - var mh1, p1, sw1, mh2, p2, sw2 - - beforeEach(function (done) { - mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - p1 = new Peer(Id.create(), []) - sw1 = new Swarm(p1) - sw1.addStreamMuxer('spdy', Spdy, {}) - sw1.enableIdentify() - - mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - p2 = new Peer(Id.create(), []) - sw2 = new Swarm(p2) - sw2.addStreamMuxer('spdy', Spdy, {}) - sw2.enableIdentify() - - async.parallel([ - function (cb) { - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) - }, - function (cb) { - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) - } - ], done) - }) + it.skip('add', (done) => {}) + it.skip('dial', (done) => {}) + it.skip('tls on a muxed stream (not the full conn)', (done) => {}) + }) +}) - afterEach(function (done) { - var cleaningCounter = 0 - sw1.closeConns(cleaningUp) - sw2.closeConns(cleaningUp) - - sw1.closeListener('tcp', cleaningUp) - sw2.closeListener('tcp', cleaningUp) - - function cleaningUp () { - cleaningCounter++ - // TODO FIX: here should be 4, but because super wrapping of - // streams, it makes it so hard to properly close the muxed - // streams - https://github.com/indutny/spdy-transport/issues/14 - if (cleaningCounter < 3) { - return - } - // give time for identify to finish - setTimeout(function () { - expect(Object.keys(sw2.muxedConns).length).to.equal(1) - done() - }, 500) - } - }) +describe('high level API = 2nd stage, with everything all together!', () => { + before((done) => { done() }) - it('identify', function (done) { - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - // formallity so that the conn starts flowing - conn.on('data', function (chunk) {}) - - conn.end() - conn.on('end', function () { - expect(Object.keys(sw1.muxedConns).length).to.equal(1) - done() - }) - }) - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) - }) - }) + it.skip('add tcp', (done) => {}) + it.skip('add utp', (done) => {}) + it.skip('add websockets', (done) => {}) + it.skip('dial', (done) => {}) }) From 9d8ee67c61d2c57b6a68cd04fabc8e4b8afdfeea Mon Sep 17 00:00:00 2001 From: David Dias Date: Sun, 6 Mar 2016 08:40:49 +0000 Subject: [PATCH 089/634] high level API working + tests --- package.json | 2 +- src/index.js | 433 +++++++++++--------------------------------- tests/swarm-test.js | 115 ++++++++++-- 3 files changed, 207 insertions(+), 343 deletions(-) diff --git a/package.json b/package.json index f69871191a..70983e973d 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "libp2p-tcp": "^0.2.1", "mocha": "^2.4.5", "multiaddr": "^1.1.1", - "peer-id": "^0.5.1", + "peer-id": "^0.5.3", "peer-info": "^0.5.2", "pre-commit": "^1.1.2", "standard": "^6.0.7", diff --git a/src/index.js b/src/index.js index f933418cd9..a465871330 100644 --- a/src/index.js +++ b/src/index.js @@ -1,4 +1,4 @@ -// const multistream = require('multistream-select') +const multistream = require('multistream-select') // const async = require('async') // const identify = require('./identify') const PassThrough = require('stream').PassThrough @@ -38,11 +38,14 @@ function Swarm (peerInfo) { this.transport.dial = (key, multiaddrs, callback) => { const t = this.transports[key] - // a) if multiaddrs.length = 1, return the conn from the - // transport, otherwise, create a passthrough if (!Array.isArray(multiaddrs)) { multiaddrs = [multiaddrs] } + + // TODO a) filter the multiaddrs that are actually valid for this transport (use a func from the transport itself) + + // b) if multiaddrs.length = 1, return the conn from the + // transport, otherwise, create a passthrough if (multiaddrs.length === 1) { const conn = t.dial(multiaddrs.shift(), {ready: () => { const cb = callback @@ -55,7 +58,7 @@ function Swarm (peerInfo) { return conn } - // b) multiaddrs should already be a filtered list + // c) multiaddrs should already be a filtered list // specific for the transport we are using const pt = new PassThrough() @@ -112,382 +115,154 @@ function Swarm (peerInfo) { // { // peerIdB58: { - // muxerName: { - // muxer: - // rawSocket: socket // to abstract info required for the Identify Protocol - // } + // muxer: + // rawSocket: socket // to abstract info required for the Identify Protocol // } + // } this.muxedConns = {} + // { protocol: handler } + this.protocols = {} + this.connection = {} this.connection.addUpgrade = () => {} - this.connection.addStreamMuxer = () => {} - - // enable the Identify protocol - this.connection.reuse = () => {} - // main API - higher level abstractions -- - - this.dial = () => { - // TODO - } - this.handle = (protocol, callback) => { + // { muxerCodec: } e.g { '/spdy/0.3.1': spdy } + this.muxers = {} + this.connection.addStreamMuxer = (muxer) => { // TODO + // .handle(protocol, () => { + // after attaching the stream muxer, check if identify is enabled + // }) + // TODO add to the list of muxers available } - this.close = (callback) => { - var count = 0 - Object.keys(this.transports).forEach((key) => { - this.transports[key].close(() => { - if (++count === Object.keys(this.transports).length) { - callback() - } - }) - }) - } - - function connHandler (conn) { - // do all the multistream select handshakes (this should be probably done recursively - } -} - -/* -function Swarm (peerInfo) { - var self = this - if (!(this instanceof Swarm)) { - return new Swarm(peerInfo) - } - - if (!peerInfo) { - throw new Error('You must provide a value for `peerInfo`') + // enable the Identify protocol + this.connection.reuse = () => { + // TODO identify } - self.peerInfo = peerInfo - - // peerIdB58: { conn: } - self.conns = {} - - // peerIdB58: { - // muxer: , - // socket: socket // so we can extract the info we need for identify - // } - self.muxedConns = {} - - // transportName: { transport: transport, - // dialOptions: dialOptions, - // listenOptions: listenOptions, - // listeners: [] } - self.transports = {} - - // transportName: listener - self.listeners = {} + const self = this // couldn't get rid of this - // protocolName: handlerFunc - self.protocols = {} - - // muxerName: { Muxer: Muxer // Muxer is a constructor - // options: options } - self.muxers = {} - - // for connection reuse - self.identify = false - - // public interface - - self.addTransport = function (name, transport, options, dialOptions, listenOptions, callback) { - // set up the transport and add the list of incoming streams - // add transport to the list of transports - - var multiaddr = options.multiaddr - if (multiaddr) { - // no need to pass that to the transports - delete options.multiaddr - } - - var listener = transport.createListener(options, listen) - - listener.listen(listenOptions, function ready () { - self.transports[name] = { - transport: transport, - options: options, - dialOptions: dialOptions, - listenOptions: listenOptions, - listener: listener - } - - // If a known multiaddr is passed, then add to our list of multiaddrs - if (multiaddr) { - self.peerInfo.multiaddrs.push(multiaddr) - } - - callback() + // incomming connection handler + function connHandler (conn) { + var msS = new multistream.Select() + msS.handle(conn) + Object.keys(self.protocols).forEach(function (protocol) { + msS.addHandler(protocol, self.protocols[protocol]) }) } - self.addUpgrade = function (ConnUpgrade, options) {} - - self.addStreamMuxer = function (name, StreamMuxer, options) { - self.muxers[name] = { - Muxer: StreamMuxer, - options: options - } - } - - self.dial = function (peerInfo, options, protocol, callback) { - // 1. check if we have transports we support - // 2. check if we have a conn waiting - // 3. check if we have a stream muxer available - + // higher level (public) API + this.dial = (pi, protocol, callback) => { + var pt = null if (typeof protocol === 'function') { callback = protocol - protocol = undefined - } - - // check if a conn is waiting - // if it is and protocol was selected, jump into multistreamHandshake - // if it is and no protocol was selected, do nothing and call and empty callback - - if (self.conns[peerInfo.id.toB58String()]) { - if (protocol) { - if (self.muxers['spdy']) { - // TODO upgrade this conn to a muxer - console.log('TODO: upgrade a warm conn to muxer that was added after') - } else { - multistreamHandshake(self.conns[peerInfo.id.toB58String()]) - } - self.conns[peerInfo.id.toB58String()] = undefined - delete self.conns[peerInfo.id.toB58String()] - return - } else { - return callback() - } + protocol = null + } else { + pt = new PassThrough() } - // check if a stream muxer for this peer is available - if (self.muxedConns[peerInfo.id.toB58String()]) { - if (protocol) { - return openMuxedStream(self.muxedConns[peerInfo.id.toB58String()].muxer) + const b58Id = pi.id.toB58String() + if (!this.muxedConns[b58Id]) { + if (!this.conns[b58Id]) { + attemptDial(pi, (err, conn) => { + if (err) { + return callback(err) + } + gotWarmedUpConn(conn) + }) } else { - return callback() + const conn = this.conns[b58Id] + this.conns[b58Id] = undefined + gotWarmedUpConn(conn) } + } else { + gotMuxer(this.muxedConns[b58Id].muxer) } - // Creating a new conn with this peer routine - - // TODO - check if there is a preference in protocol to use on - // options.protocol - var supportedTransports = Object.keys(self.transports) - var multiaddrs = peerInfo.multiaddrs.filter(function (multiaddr) { - return multiaddr.protoNames().some(function (proto) { - return supportedTransports.indexOf(proto) >= 0 - }) - }) - - if (!multiaddrs.length) { - callback(new Error("The swarm doesn't support any of the peer transports")) - return - } - - var conn - - async.eachSeries(multiaddrs, function (multiaddr, next) { - if (conn) { - return next() - } - - var transportName = getTransportNameForMultiaddr(multiaddr) - var transport = self.transports[transportName] - var dialOptions = clone(transport.dialOptions) - dialOptions.ready = connected - - var connTry = transport.transport.dial(multiaddr, dialOptions) + function gotWarmedUpConn (conn) { + attemptMuxerUpgrade(conn, (err, muxer) => { + if (!protocol) { + if (err) { + self.conns[b58Id] = conn + } + return callback() + } - connTry.once('error', function (err) { if (err) { - return console.log(err) // TODO handle error + // couldn't upgrade to Muxer, it is ok + protocolHandshake(conn, protocol, callback) + } else { + gotMuxer(muxer) } - next() // try next multiaddr }) + } - function connected () { - conn = connTry - next() - } + function gotMuxer (muxer) { + openConnInMuxedConn(muxer, (conn) => { + protocolHandshake(conn, protocol, callback) + }) + } - function getTransportNameForMultiaddr (multiaddr) { - // this works for all those ip + transport + port tripplets - return multiaddr.protoNames()[1] - } + function attemptDial (pi, cb) { + const tKeys = Object.keys(self.transports) + nextTransport(tKeys.shift()) - function clone (obj) { - var target = {} - for (var i in obj) { - if (obj.hasOwnProperty(i)) { - target[i] = obj[i] - } - } - return target - } - }, done) - - function done () { - // TODO apply upgrades - // apply stream muxer - // if no protocol is selected, save it in the pool - // if protocol is selected, multistream that protocol - if (!conn) { - callback(new Error('Unable to open a connection')) - return - } - - if (self.muxers['spdy']) { - var spdy = new self.muxers['spdy'].Muxer(self.muxers['spdy'].options) - spdy.attach(conn, false, function (err, muxer) { + function nextTransport (key) { + const multiaddrs = pi.multiaddrs.slice() + self.transport.dial(key, multiaddrs, (err, conn) => { if (err) { - return console.log(err) // TODO Treat error - } - - muxer.on('stream', userProtocolMuxer) - - self.muxedConns[peerInfo.id.toB58String()] = { - muxer: muxer, - socket: conn - } - - if (protocol) { - openMuxedStream(muxer) - } else { - callback() + if (tKeys.length === 0) { + return cb(new Error('Could not dial in any of the transports')) + } + return nextTransport(tKeys.shift()) } + cb(null, conn) }) - } else { - if (protocol) { - multistreamHandshake(conn) - } else { - self.conns[peerInfo.id.toB58String()] = conn - callback() - } } } - function openMuxedStream (muxer) { - // 1. create a new stream on this muxedConn and pass that to - // multistreamHanshake - muxer.dialStream(function (err, conn) { - if (err) { - return console.log(err) // TODO Treat error - } - multistreamHandshake(conn) - }) + function attemptMuxerUpgrade (conn, cb) { + if (Object.keys(self.muxers).length === 0) { + return cb(new Error('no muxers available')) + } + // TODO add muxer to the muxedConns object for the peerId + // TODO if it succeeds, add incomming open coons to connHandler + } + function openConnInMuxedConn (muxer, cb) { + // TODO open a conn in this muxer } - function multistreamHandshake (conn) { + function protocolHandshake (conn, protocol, cb) { var msI = new multistream.Interactive() msI.handle(conn, function () { - msI.select(protocol, callback) + msI.select(protocol, (err, conn) => { + if (err) { + return callback(err) + } + pt.pipe(conn).pipe(pt) + callback(null, pt) + }) }) } } - self.closeListener = function (transportName, callback) { - self.transports[transportName].listener.close(closed) - - // only gets called when all the streams on this transport are closed too - function closed () { - delete self.transports[transportName] - callback() - } - } - - // Iterates all the listeners closing them - // one by one. It calls back once all are closed. - self.closeAllListeners = function (callback) { - var transportNames = Object.keys(self.transports) - - async.each(transportNames, self.closeListener, callback) + this.handle = (protocol, handler) => { + this.protocols[protocol] = handler } - self.closeConns = function (callback) { - // close warmed up cons so that the listener can gracefully exit - Object.keys(self.conns).forEach(function (conn) { - self.conns[conn].destroy() - }) - self.conns = {} - - callback() - } - - // Closes both transport listeners and - // connections. It calls back once everything - // is closed - self.close = function (callback) { - async.parallel([ - self.closeAllListeners, - self.closeConns - ], callback) - } - - self.enableIdentify = function () { - // set flag to true - // add identify to the list of handled protocols - self.identify = true - - // we pass muxedConns so that identify can access the socket of the other - // peer - self.handleProtocol(identify.protoId, - identify.getHandlerFunction(self.peerInfo, self.muxedConns)) - } - - self.handleProtocol = function (protocol, handlerFunction) { - self.protocols[protocol] = handlerFunction - } - - // internals - - function listen (conn) { - // TODO apply upgrades - // add StreamMuxer if available (and point streams from muxer to userProtocolMuxer) - - if (self.muxers['spdy']) { - var spdy = new self.muxers['spdy'].Muxer(self.muxers['spdy'].options) - spdy.attach(conn, true, function (err, muxer) { - if (err) { - return console.log(err) // TODO treat error - } + this.close = (callback) => { + var count = 0 - // TODO This muxer has to be identified! - // pass to identify a reference of - // our muxedConn list - // ourselves (peerInfo) - // the conn, which is the socket - // and a stream it can send stuff - if (self.identify) { - muxer.dialStream(function (err, stream) { - if (err) { - return console.log(err) // TODO Treat error - } - // conn === socket at this point - identify(self.muxedConns, self.peerInfo, conn, stream, muxer) - }) + Object.keys(this.transports).forEach((key) => { + this.transports[key].close(() => { + if (++count === Object.keys(this.transports).length) { + callback() } - - muxer.on('stream', userProtocolMuxer) }) - } else { - // if no stream muxer, then - userProtocolMuxer(conn) - } - } - - // Handle user given protocols - function userProtocolMuxer (conn) { - var msS = new multistream.Select() - msS.handle(conn) - Object.keys(self.protocols).forEach(function (protocol) { - msS.addHandler(protocol, self.protocols[protocol]) }) } } -*/ function noop () {} diff --git a/tests/swarm-test.js b/tests/swarm-test.js index a9ecfb2680..02e752cdf3 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -173,7 +173,9 @@ describe('transport - tcp', function () { }) }) -describe('transport - udt', () => { +describe('transport - udt', function () { + this.timeout(10000) + before((done) => { done() }) it.skip('add', (done) => {}) @@ -182,7 +184,9 @@ describe('transport - udt', () => { it.skip('close', (done) => {}) }) -describe('transport - websockets', () => { +describe('transport - websockets', function () { + this.timeout(10000) + before((done) => { done() }) it.skip('add', (done) => {}) @@ -191,37 +195,119 @@ describe('transport - websockets', () => { it.skip('close', (done) => {}) }) -describe('high level API - 1st stage, without stream multiplexing (on TCP)', () => { - it.skip('handle a protocol', (done) => {}) - it.skip('dial on protocol', (done) => {}) - it.skip('dial to warm conn', (done) => {}) - it.skip('dial on protocol, reuse warmed conn', (done) => {}) - it.skip('close', (done) => {}) +describe('high level API - 1st without stream multiplexing (on TCP)', function () { + this.timeout(10000) + + var swarmA + var peerA + var swarmB + var peerB + + before((done) => { + peerA = new Peer() + peerB = new Peer() + + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) + + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + + swarmA.transport.add('tcp', new TCP()) + swarmA.transport.listen('tcp', {}, null, ready) + + swarmB.transport.add('tcp', new TCP()) + swarmB.transport.listen('tcp', {}, null, ready) + + var counter = 0 + + function ready () { + if (++counter === 2) { + done() + } + } + }) + + after((done) => { + var counter = 0 + + swarmA.close(closed) + swarmB.close(closed) + + function closed () { + if (++counter === 2) { + done() + } + } + }) + + it('handle a protocol', (done) => { + swarmB.handle('/bananas/1.0.0', (conn) => { + conn.pipe(conn) + }) + expect(Object.keys(swarmB.protocols).length).to.equal(1) + done() + }) + + it('dial on protocol', (done) => { + swarmB.handle('/pineapple/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmA.dial(peerB, '/pineapple/1.0.0', (err, conn) => { + expect(err).to.not.exist + conn.end() + conn.on('end', done) + }) + }) + + it('dial to warm a conn', (done) => { + swarmA.dial(peerB, (err) => { + expect(err).to.not.exist + done() + }) + }) + + it('dial on protocol, reuse warmed conn', (done) => { + swarmA.dial(peerB, '/bananas/1.0.0', (err, conn) => { + expect(err).to.not.exist + conn.end() + conn.on('end', done) + }) + }) }) -describe('stream muxing (on TCP)', () => { +describe('stream muxing (on TCP)', function () { + this.timeout(10000) + describe('multiplex', () => { before((done) => { done() }) + after((done) => { done() }) + it.skip('add', (done) => {}) it.skip('handle + dial on protocol', (done) => {}) it.skip('dial to warm conn', (done) => {}) it.skip('dial on protocol, reuse warmed conn', (done) => {}) it.skip('enable identify to reuse incomming muxed conn', (done) => {}) - it.skip('close', (done) => {}) }) describe('spdy', () => { + before((done) => { done() }) + after((done) => { done() }) + it.skip('add', (done) => {}) it.skip('handle + dial on protocol', (done) => {}) it.skip('dial to warm conn', (done) => {}) it.skip('dial on protocol, reuse warmed conn', (done) => {}) it.skip('enable identify to reuse incomming muxed conn', (done) => {}) - it.skip('close', (done) => {}) }) }) -describe('conn upgrades', () => { +describe('conn upgrades', function () { + this.timeout(10000) + describe('secio on tcp', () => { before((done) => { done() }) + after((done) => { done() }) it.skip('add', (done) => {}) it.skip('dial', (done) => {}) @@ -229,6 +315,7 @@ describe('conn upgrades', () => { }) describe('tls on tcp', () => { before((done) => { done() }) + after((done) => { done() }) it.skip('add', (done) => {}) it.skip('dial', (done) => {}) @@ -236,7 +323,9 @@ describe('conn upgrades', () => { }) }) -describe('high level API = 2nd stage, with everything all together!', () => { +describe('high level API - with everything mixed all together!', function () { + this.timeout(10000) + before((done) => { done() }) it.skip('add tcp', (done) => {}) From f8e14e4ddf050eb6626decc1146b7bde9e045648 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 7 Mar 2016 12:47:11 +0000 Subject: [PATCH 090/634] stream multiplexing done, starting on identify refactor --- package.json | 2 +- src/identify.js | 63 ++++---- src/index.js | 64 ++++++-- tests/swarm-old.js | 354 -------------------------------------------- tests/swarm-test.js | 136 ++++++++++++++--- 5 files changed, 208 insertions(+), 411 deletions(-) delete mode 100644 tests/swarm-old.js diff --git a/package.json b/package.json index 70983e973d..8c13358965 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "bl": "^1.1.2", "chai": "^3.5.0", "istanbul": "^0.4.2", - "libp2p-spdy": "^0.1.0", + "libp2p-spdy": "^0.2.3", "libp2p-tcp": "^0.2.1", "mocha": "^2.4.5", "multiaddr": "^1.1.1", diff --git a/src/identify.js b/src/identify.js index bf78cd89dc..7bad7eb7e0 100644 --- a/src/identify.js +++ b/src/identify.js @@ -1,30 +1,47 @@ /* - * Identify is one of the protocols swarms speaks in order to broadcast and learn - * about the ip:port pairs a specific peer is available through + * Identify is one of the protocols swarms speaks in order to + * broadcast and learn about the ip:port pairs a specific peer + * is available through */ -var Interactive = require('multistream-select').Interactive -var protobufs = require('protocol-buffers-stream') -var fs = require('fs') -var path = require('path') -var schema = fs.readFileSync(path.join(__dirname, 'identify.proto')) -var Address6 = require('ip-address').Address6 -var Id = require('peer-id') -var multiaddr = require('multiaddr') - -exports = module.exports = identify - -var protoId = '/ipfs/identify/1.0.0' +// var multistream = require('multistream-select') +// var protobufs = require('protocol-buffers-stream') +// var fs = require('fs') +// var path = require('path') +// var protobufs = require('protocol-buffers-stream') +// var schema = fs.readFileSync(path.join(__dirname, 'identify.proto')) +// var Address6 = require('ip-address').Address6 +// var Id = require('peer-id') +// var multiaddr = require('multiaddr') + +exports = module.exports + +exports.multicodec = '/ipfs/identify/1.0.0' + +exports.exec = (muxedConn, callback) => { + // TODO + // 1. open a stream + // 2. multistream into identify + // 3. send what I see from this other peer + // 4. receive what the other peer sees from me + // 4. callback with (err, peerInfo) +} -exports.protoId = protoId -var createProtoStream = protobufs(schema) +exports.handler = (peerInfo) => { + return function (conn) { + // TODO + // 1. receive incoming observed info about me + // 2. send back what I see from the other + } +} +/* function identify (muxedConns, peerInfoSelf, socket, conn, muxer) { var msi = new Interactive() msi.handle(conn, function () { msi.select(protoId, function (err, ds) { if (err) { - return console.log(err) // TODO Treat error + return console.log(err) } var ps = createProtoStream() @@ -39,16 +56,6 @@ function identify (muxedConns, peerInfoSelf, socket, conn, muxer) { socket: socket } - // TODO: Pass the new discovered info about the peer that contacted us - // to something like the Kademlia Router, so the peerInfo for this peer - // is fresh - // - before this was exectued through a event emitter - // self.emit('peer-update', { - // peerId: peerId, - // listenAddrs: msg.listenAddrs.map(function (mhb) {return multiaddr(mhb)}) - // }) - }) - var mh = getMultiaddr(socket) ps.identify({ @@ -156,4 +163,4 @@ function updateSelf (peerSelf, observedAddr) { peerSelf.multiaddrs.push(omh) } } -} +}*/ diff --git a/src/index.js b/src/index.js index a465871330..4e3f3f6df1 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,6 @@ const multistream = require('multistream-select') // const async = require('async') -// const identify = require('./identify') +const identify = require('./identify') const PassThrough = require('stream').PassThrough exports = module.exports = Swarm @@ -130,16 +130,28 @@ function Swarm (peerInfo) { // { muxerCodec: } e.g { '/spdy/0.3.1': spdy } this.muxers = {} this.connection.addStreamMuxer = (muxer) => { - // TODO - // .handle(protocol, () => { - // after attaching the stream muxer, check if identify is enabled - // }) - // TODO add to the list of muxers available + // for dialing + this.muxers[muxer.multicodec] = muxer + + // for listening + this.handle(muxer.multicodec, (conn) => { + const muxedConn = muxer(conn, true) + muxedConn.on('stream', connHandler) + + if (this.identify) { + identify.exec(muxedConn, (err, pi) => { + if (err) {} + // TODO muxedConns[pi.id.toB58String()].muxer = muxedConn + }) + } + }) } // enable the Identify protocol + this.identify = false this.connection.reuse = () => { - // TODO identify + this.identify = true + this.handle(identify.multicodec, identify.handler(peerInfo)) } const self = this // couldn't get rid of this @@ -224,14 +236,40 @@ function Swarm (peerInfo) { } function attemptMuxerUpgrade (conn, cb) { - if (Object.keys(self.muxers).length === 0) { + const muxers = Object.keys(self.muxers) + if (muxers.length === 0) { return cb(new Error('no muxers available')) } - // TODO add muxer to the muxedConns object for the peerId - // TODO if it succeeds, add incomming open coons to connHandler + + // 1. try to handshake in one of the muxers available + // 2. if succeeds + // - add the muxedConn to the list of muxedConns + // - add incomming new streams to connHandler + + nextMuxer(muxers.shift()) + + function nextMuxer (key) { + var msI = new multistream.Interactive() + msI.handle(conn, function () { + msI.select(key, (err, conn) => { + if (err) { + if (muxers.length === 0) { + cb(new Error('could not upgrade to stream muxing')) + } else { + nextMuxer(muxers.shift()) + } + } + + const muxedConn = self.muxers[key](conn, false) + self.muxedConns[b58Id] = {} + self.muxedConns[b58Id].muxer = muxedConn + cb(null, muxedConn) + }) + }) + } } function openConnInMuxedConn (muxer, cb) { - // TODO open a conn in this muxer + cb(muxer.newStream()) } function protocolHandshake (conn, protocol, cb) { @@ -255,6 +293,10 @@ function Swarm (peerInfo) { this.close = (callback) => { var count = 0 + Object.keys(this.muxedConns).forEach((key) => { + this.muxedConns[key].muxer.end() + }) + Object.keys(this.transports).forEach((key) => { this.transports[key].close(() => { if (++count === Object.keys(this.transports).length) { diff --git a/tests/swarm-old.js b/tests/swarm-old.js deleted file mode 100644 index 0acd765406..0000000000 --- a/tests/swarm-old.js +++ /dev/null @@ -1,354 +0,0 @@ -/* eslint-env mocha */ - -var async = require('async') -var expect = require('chai').expect - -var multiaddr = require('multiaddr') -var Id = require('peer-id') -var Peer = require('peer-info') -var Swarm = require('../src') -var tcp = require('libp2p-tcp') -var Spdy = require('libp2p-spdy') - -// because of Travis-CI -process.on('uncaughtException', function (err) { - console.log('Caught exception: ' + err) -}) - -describe('Basics', function () { - it('enforces creation with new', function (done) { - expect(function () { - Swarm() - }).to.throw() - done() - }) - - it('it throws an exception without peerSelf', function (done) { - expect(function () { - var sw = new Swarm() - sw.close() - }).to.throw(Error) - done() - }) -}) - -describe('When dialing', function () { - describe('if the swarm does add any of the peer transports', function () { - it('it returns an error', function (done) { - var peerOne = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8090')]) - var peerTwo = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/8091')]) - var swarm = new Swarm(peerOne) - - swarm.dial(peerTwo, {}, function (err) { - expect(err).to.exist - done() - }) - }) - }) -}) - -describe('Without a Stream Muxer', function () { - describe('and one swarm over tcp', function () { - it('add the transport', function (done) { - var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') - var p = new Peer(Id.create(), []) - var sw = new Swarm(p) - - sw.addTransport('tcp', tcp, { multiaddr: mh }, {}, {port: 8010}, ready) - - function ready () { - expect(sw.transports['tcp'].options).to.deep.equal({}) - expect(sw.transports['tcp'].dialOptions).to.deep.equal({}) - expect(sw.transports['tcp'].listenOptions).to.deep.equal({port: 8010}) - expect(sw.transports['tcp'].transport).to.deep.equal(tcp) - - sw.close(done) - } - }) - }) - - describe('and two swarms over tcp', function () { - var mh1, p1, sw1, mh2, p2, sw2 - - beforeEach(function (done) { - mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - p1 = new Peer(Id.create(), []) - sw1 = new Swarm(p1) - - mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - p2 = new Peer(Id.create(), []) - sw2 = new Swarm(p2) - - async.parallel([ - function (cb) { - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) - }, - function (cb) { - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) - } - ], done) - }) - - afterEach(function (done) { - async.parallel([sw1.close, sw2.close], done) - }) - - it('dial a conn', function (done) { - sw1.dial(p2, {}, function (err) { - expect(err).to.equal(undefined) - expect(Object.keys(sw1.conns).length).to.equal(1) - done() - }) - }) - - it('dial a conn on a protocol', function (done) { - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - conn.end() - conn.on('end', done) - }) - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) - }) - - it('dial a protocol on a previous created conn', function (done) { - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - conn.end() - conn.on('end', done) - }) - - sw1.dial(p2, {}, function (err) { - expect(err).to.equal(undefined) - expect(Object.keys(sw1.conns).length).to.equal(1) - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - - conn.end() - }) - }) - }) - - // it('add an upgrade', function (done) { done() }) - // it('dial a conn on top of a upgrade', function (done) { done() }) - // it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) - }) - - /* TODO - describe('udp', function () { - it('add the transport', function (done) { done() }) - it('dial a conn', function (done) { done() }) - it('dial a conn on a protocol', function (done) { done() }) - it('add an upgrade', function (done) { done() }) - it('dial a conn on top of a upgrade', function (done) { done() }) - it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) - }) */ - - /* TODO - describe('udt', function () { - it('add the transport', function (done) { done() }) - it('dial a conn', function (done) { done() }) - it('dial a conn on a protocol', function (done) { done() }) - it('add an upgrade', function (done) { done() }) - it('dial a conn on top of a upgrade', function (done) { done() }) - it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) - }) */ - -/* TODO -describe('utp', function () { - it('add the transport', function (done) { done() }) - it('dial a conn', function (done) { done() }) - it('dial a conn on a protocol', function (done) { done() }) - it('add an upgrade', function (done) { done() }) - it('dial a conn on top of a upgrade', function (done) { done() }) - it('dial a conn on a protocol on top of a upgrade', function (done) { done() }) -}) */ -}) - -describe('With a SPDY Stream Muxer', function () { - describe('and one swarm over tcp', function () { - // TODO: What is the it here? - it('add Stream Muxer', function (done) { - // var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') - var p = new Peer(Id.create(), []) - var sw = new Swarm(p) - sw.addStreamMuxer('spdy', Spdy, {}) - - done() - }) - }) - - describe('and two swarms over tcp', function () { - var mh1, p1, sw1, mh2, p2, sw2 - - beforeEach(function (done) { - mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - p1 = new Peer(Id.create(), []) - sw1 = new Swarm(p1) - sw1.addStreamMuxer('spdy', Spdy, {}) - - mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - p2 = new Peer(Id.create(), []) - sw2 = new Swarm(p2) - sw2.addStreamMuxer('spdy', Spdy, {}) - - async.parallel([ - function (cb) { - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) - }, - function (cb) { - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) - } - ], done) - }) - - function afterEach (done) { - var cleaningCounter = 0 - sw1.closeConns(cleaningUp) - sw2.closeConns(cleaningUp) - - sw1.closeListener('tcp', cleaningUp) - sw2.closeListener('tcp', cleaningUp) - - function cleaningUp () { - cleaningCounter++ - // TODO FIX: here should be 4, but because super wrapping of - // streams, it makes it so hard to properly close the muxed - // streams - https://github.com/indutny/spdy-transport/issues/14 - if (cleaningCounter < 3) { - return - } - - done() - } - } - - it('dial a conn on a protocol', function (done) { - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - // formallity so that the conn starts flowing - conn.on('data', function (chunk) {}) - - conn.end() - conn.on('end', function () { - expect(Object.keys(sw1.muxedConns).length).to.equal(1) - expect(Object.keys(sw2.muxedConns).length).to.equal(0) - afterEach(done) - }) - }) - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) - }) - - it('dial two conns (transport reuse)', function (done) { - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - // formality so that the conn starts flowing - conn.on('data', function (chunk) {}) - - conn.end() - conn.on('end', function () { - expect(Object.keys(sw1.muxedConns).length).to.equal(1) - expect(Object.keys(sw2.muxedConns).length).to.equal(0) - - afterEach(done) - }) - }) - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - // TODO Improve clarity - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - - conn.end() - }) - - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - - conn.end() - }) - }) - }) - - describe('and two identity enabled swarms over tcp', function () { - var mh1, p1, sw1, mh2, p2, sw2 - - beforeEach(function (done) { - mh1 = multiaddr('/ip4/127.0.0.1/tcp/8010') - p1 = new Peer(Id.create(), []) - sw1 = new Swarm(p1) - sw1.addStreamMuxer('spdy', Spdy, {}) - sw1.enableIdentify() - - mh2 = multiaddr('/ip4/127.0.0.1/tcp/8020') - p2 = new Peer(Id.create(), []) - sw2 = new Swarm(p2) - sw2.addStreamMuxer('spdy', Spdy, {}) - sw2.enableIdentify() - - async.parallel([ - function (cb) { - sw1.addTransport('tcp', tcp, { multiaddr: mh1 }, {}, {port: 8010}, cb) - }, - function (cb) { - sw2.addTransport('tcp', tcp, { multiaddr: mh2 }, {}, {port: 8020}, cb) - } - ], done) - }) - - afterEach(function (done) { - var cleaningCounter = 0 - sw1.closeConns(cleaningUp) - sw2.closeConns(cleaningUp) - - sw1.closeListener('tcp', cleaningUp) - sw2.closeListener('tcp', cleaningUp) - - function cleaningUp () { - cleaningCounter++ - // TODO FIX: here should be 4, but because super wrapping of - // streams, it makes it so hard to properly close the muxed - // streams - https://github.com/indutny/spdy-transport/issues/14 - if (cleaningCounter < 3) { - return - } - // give time for identify to finish - setTimeout(function () { - expect(Object.keys(sw2.muxedConns).length).to.equal(1) - done() - }, 500) - } - }) - - it('identify', function (done) { - sw2.handleProtocol('/sparkles/1.0.0', function (conn) { - // formallity so that the conn starts flowing - conn.on('data', function (chunk) {}) - - conn.end() - conn.on('end', function () { - expect(Object.keys(sw1.muxedConns).length).to.equal(1) - done() - }) - }) - - sw1.dial(p2, {}, '/sparkles/1.0.0', function (err, conn) { - conn.on('data', function () {}) - expect(err).to.equal(null) - expect(Object.keys(sw1.conns).length).to.equal(0) - conn.end() - }) - }) - }) -}) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 02e752cdf3..06e1161210 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -9,7 +9,7 @@ const Peer = require('peer-info') const Swarm = require('../src') const TCP = require('libp2p-tcp') const bl = require('bl') -// var SPDY = require('libp2p-spdy') +const spdy = require('libp2p-spdy') describe('basics', () => { it('throws on missing peerInfo', (done) => { @@ -196,7 +196,7 @@ describe('transport - websockets', function () { }) describe('high level API - 1st without stream multiplexing (on TCP)', function () { - this.timeout(10000) + this.timeout(20000) var swarmA var peerA @@ -278,7 +278,7 @@ describe('high level API - 1st without stream multiplexing (on TCP)', function ( }) describe('stream muxing (on TCP)', function () { - this.timeout(10000) + this.timeout(20000) describe('multiplex', () => { before((done) => { done() }) @@ -291,31 +291,131 @@ describe('stream muxing (on TCP)', function () { it.skip('enable identify to reuse incomming muxed conn', (done) => {}) }) describe('spdy', () => { - before((done) => { done() }) - after((done) => { done() }) + var swarmA + var peerA + var swarmB + var peerB + var swarmC + var peerC - it.skip('add', (done) => {}) - it.skip('handle + dial on protocol', (done) => {}) - it.skip('dial to warm conn', (done) => {}) - it.skip('dial on protocol, reuse warmed conn', (done) => {}) - it.skip('enable identify to reuse incomming muxed conn', (done) => {}) + before((done) => { + peerA = new Peer() + peerB = new Peer() + peerC = new Peer() + + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) + peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9003')) + + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + swarmC = new Swarm(peerC) + + swarmA.transport.add('tcp', new TCP()) + swarmA.transport.listen('tcp', {}, null, ready) + + swarmB.transport.add('tcp', new TCP()) + swarmB.transport.listen('tcp', {}, null, ready) + + swarmC.transport.add('tcp', new TCP()) + swarmC.transport.listen('tcp', {}, null, ready) + + var counter = 0 + + function ready () { + if (++counter === 3) { + done() + } + } + }) + + after((done) => { + var counter = 0 + + swarmA.close(closed) + swarmB.close(closed) + swarmC.close(closed) + + function closed () { + if (++counter === 3) { + done() + } + } + }) + + it('add', (done) => { + swarmA.connection.addStreamMuxer(spdy) + swarmB.connection.addStreamMuxer(spdy) + swarmC.connection.addStreamMuxer(spdy) + done() + }) + + it('handle + dial on protocol', (done) => { + swarmB.handle('/abacaxi/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmA.dial(peerB, '/abacaxi/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmA.muxedConns).length).to.equal(1) + conn.end() + conn.on('end', done) + }) + }) + + it('dial to warm conn', (done) => { + swarmB.dial(peerA, (err) => { + expect(err).to.not.exist + expect(Object.keys(swarmB.conns).length).to.equal(0) + expect(Object.keys(swarmB.muxedConns).length).to.equal(1) + done() + }) + }) + + it('dial on protocol, reuse warmed conn', (done) => { + swarmA.handle('/papaia/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmB.dial(peerA, '/papaia/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmB.conns).length).to.equal(0) + expect(Object.keys(swarmB.muxedConns).length).to.equal(1) + conn.end() + conn.on('end', done) + }) + }) + + it.skip('enable identify to reuse incomming muxed conn', (done) => { + swarmA.connection.reuse() + swarmC.connection.reuse() + + swarmC.dial(peerA, (err) => { + expect(err).to.not.exist + setTimeout(() => { + expect(Object.keys(swarmC.muxedConns).length).to.equal(1) + expect(Object.keys(swarmA.muxedConns).length).to.equal(2) + }, 100) + }) + }) }) }) +/* describe('conn upgrades', function () { - this.timeout(10000) + this.timeout(20000) describe('secio on tcp', () => { - before((done) => { done() }) - after((done) => { done() }) + // before((done) => { done() }) + // after((done) => { done() }) it.skip('add', (done) => {}) it.skip('dial', (done) => {}) it.skip('tls on a muxed stream (not the full conn)', (done) => {}) }) describe('tls on tcp', () => { - before((done) => { done() }) - after((done) => { done() }) + // before((done) => { done() }) + // after((done) => { done() }) it.skip('add', (done) => {}) it.skip('dial', (done) => {}) @@ -324,12 +424,14 @@ describe('conn upgrades', function () { }) describe('high level API - with everything mixed all together!', function () { - this.timeout(10000) + this.timeout(20000) - before((done) => { done() }) + // before((done) => { done() }) + // after((done) => { done() }) it.skip('add tcp', (done) => {}) it.skip('add utp', (done) => {}) it.skip('add websockets', (done) => {}) it.skip('dial', (done) => {}) }) +*/ From e8de55bc2875f86a754cbd05734da367a2f95195 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 10 Mar 2016 14:38:22 +0000 Subject: [PATCH 091/634] update the docs --- README.md | 99 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 3b8225f24a..fd4df14875 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ libp2p-swarm is used by libp2p but it can be also used as a standalone module. # Usage -### Install and create a Swarm +## Install libp2p-swarm is available on npm and so, like any other npm module, just: @@ -21,6 +21,10 @@ libp2p-swarm is available on npm and so, like any other npm module, just: > npm install libp2p-swarm --save ``` +## API + +#### Create a libp2p Swarm + And use it in your Node.js code as: ```JavaScript @@ -31,68 +35,77 @@ const sw = new Swarm(peerInfo) peerInfo is a [PeerInfo](https://github.com/diasdavid/js-peer-info) object that represents the peer creating this swarm instance. +### Transports ----------- -BELOW NEEDS AN UPDATE +##### `swarm.transport.add(key, transport, options, callback)` +libp2p-swarm expects transports that implement [interface-transport](https://github.com/diasdavid/abstract-transport). For example [libp2p-tcp](https://github.com/diasdavid/js-libp2p-tcp). -### Support a transport +- `key` - the transport identifier +- `transport` - +- `options` +- `callback` -libp2p-swarm expects transports that implement [abstract-transport](https://github.com/diasdavid/abstract-transport). For example [libp2p-tcp](https://github.com/diasdavid/js-libp2p-tcp), a simple shim on top of the `net` module to make it work with swarm expectations. +##### `swarm.transport.dial(key, multiaddrs, callback)` -```JavaScript -sw.addTransport(transport, [options, dialOptions, listenOptions]) -``` +Dial to a peer on a specific transport. -### Add a connection upgrade +- `key` +- `multiaddrs` +- `callback` -A connection upgrade must be able to receive and return something that implements the [abstract-connection](https://github.com/diasdavid/abstract-connection) interface. +##### `swarm.transport.listen(key, options, handler, callback)` -```JavaScript -sw.addUpgrade(connUpgrade, [options]) -``` +Set a transport to start listening mode. -Upgrading a connection to use a stream muxer is still considered an upgrade, but a special case since once this connection is applied, the returned obj will implement the [abstract-stream-muxer](https://github.com/diasdavid/abstract-stream-muxer) interface. +- `key` +- `options` +- `handler` +- `callback` -```JavaScript -sw.addStreamMuxer(streamMuxer, [options]) -``` +##### `swarm.transport.close(key, callback)` -### Dial to another peer +Close the listeners of a given transport. -```JavaScript -sw.dial(PeerInfo, options, protocol, callback) -sw.dial(PeerInfo, options, callback) -``` +- `key` +- `callback` -dial uses the best transport (whatever works first, in the future we can have some criteria), and jump starts the connection until the point we have to negotiate the protocol. If a muxer is available, then drop the muxer onto that connection. Good to warm up connections or to check for connectivity. If we have already a muxer for that peerInfo, than do nothing. +### Connection -### Accept requests on a specific protocol +##### `swarm.connection.addUpgrade()` -```JavaScript -sw.handleProtocol(protocol, handlerFunction) -``` +A connection upgrade must be able to receive and return something that implements the [interface-connection](https://github.com/diasdavid/interface-connection) specification. -### Cleaning up before exiting +> **WIP** -Each time you add a transport or dial you create connections. Be sure to close -them up before exiting. To do so you can: +##### `swarm.connection.addStreamMuxer(muxer)` -Close a transport listener: +Upgrading a connection to use a stream muxer is still considered an upgrade, but a special case since once this connection is applied, the returned obj will implement the [interface-stream-muxer](https://github.com/diasdavid/interface-stream-muxer) spec. -```js -sw.closeListener(transportName, callback) -sw.closeAllListeners(callback) -``` +- `muxer` -Close all open connections +##### `swarm.connection.reuse()` -```js -sw.closeConns(callback) -``` +Enable the identify protocol -Close everything! +### `swarm.dial(pi, protocol, callback)` + +dial uses the best transport (whatever works first, in the future we can have some criteria), and jump starts the connection until the point we have to negotiate the protocol. If a muxer is available, then drop the muxer onto that connection. Good to warm up connections or to check for connectivity. If we have already a muxer for that peerInfo, than do nothing. + +- `pi` - peer info project +- `protocol` +- `callback` + +### `swarm.handle(protocol, handler)` + +handle a new protocol. + +- `protocol` +- `handler` - function called when we receive a dial on `protocol. Signature must be `function (conn) {}` + +### `swarm.close(callback)` + +close all the listeners and muxers. + +- `callback` -```js -sw.close(callback) -``` From 366b6ef382ac40172b3e133b86acbb41b01801a7 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 10 Mar 2016 15:17:07 +0000 Subject: [PATCH 092/634] design notes --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index fd4df14875..9a7b56b85f 100644 --- a/README.md +++ b/README.md @@ -109,3 +109,44 @@ close all the listeners and muxers. - `callback` +# Design + +## Multitransport + +libp2p is designed to support multiple transports at the same time. While peers are identified by their ID (which are generated from their public keys), the addresses of each pair may vary, depending the device where they are being run or the network in which they are accessible through. + +In order for a transport to be supported, it has to follow the [interface-transport](https://github.com/diasdavid/interface-transport) spec. + +## Connection upgrades + +Each connection in libp2p follows the [interface-connection](https://github.com/diasdavid/interface-connection) spec. This design decision enables libp2p to have upgradable transports. + +We think of `upgrade` as a very important notion when we are talking about connections, we can see mechanisms like: stream multiplexing, congestion control, encrypted channels, multipath, simulcast, etc, as `upgrades` to a connection. A connection can be a simple and with no guarantees, drop a packet on the network with a destination thing, a transport in the other hand can be a connection and or a set of different upgrades that are mounted on top of each other, giving extra functionality to that connection and therefore `upgrading` it. + +Types of upgrades to a connection: + +- encrypted channel (with TLS for e.g) +- congestion flow (some transports don't have it by default) +- multipath (open several connections and abstract it as a single connection) +- simulcast (still really thinking this one through, it might be interesting to send a packet through different connections under some hard network circumstances) +- stream-muxer - this a special case, because once we upgrade a connection to a stream-muxer, we can open more streams (multiplex them) on a single stream, also enabling us to reuse the underlying dialed transport + +We also want to enable flexibility when it comes to upgrading a connection, for example, we might that all dialed transports pass through the encrypted channel upgrade, but not the congestion flow, specially when a transport might have already some underlying properties (UDP vs TCP vs WebRTC vs every other transport protocol) + +## Identify + +Identify is a protocol that Swarms mounts on top of itself, to identify the connections between any two peers. E.g: + +- a) peer A dials a conn to peer B +- b) that conn gets upgraded to a stream multiplexer that both peers agree +- c) peer B executes de identify protocol +- d) peer B now can open streams to peer A, knowing which is the identity of peer A + +In addition to this, we also share the 'observed addresses' by the other peer, which is extremely useful information for different kinds of network topologies. + +## Notes + +To avoid the confusion between connection, stream, transport, and other names that represent an abstraction of data flow between two points, we use terms as: + +- connection - something that implements the transversal expectations of a stream between two peers, including the benefits of using a stream plus having a way to do half duplex, full duplex +- transport - something that as a dial/listen interface and return objs that implement a connection interface From 990111980b2773b921538c663706cd9f15cdc00e Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 10 Mar 2016 20:28:58 +0000 Subject: [PATCH 093/634] woot --- examples/peerA.js | 25 ------- examples/peerB.js | 14 ---- package.json | 6 +- src/identify.js | 175 +++++++++++++------------------------------- src/index.js | 34 ++++++--- tests/swarm-test.js | 9 ++- 6 files changed, 85 insertions(+), 178 deletions(-) delete mode 100644 examples/peerA.js delete mode 100644 examples/peerB.js diff --git a/examples/peerA.js b/examples/peerA.js deleted file mode 100644 index 9bba479311..0000000000 --- a/examples/peerA.js +++ /dev/null @@ -1,25 +0,0 @@ -// var Identify = require('./../src/identify') -var Swarm = require('./../src') -var Peer = require('ipfs-peer') -var Id = require('ipfs-peer-id') -var multiaddr = require('multiaddr') - -var a = new Swarm() -a.port = 4000 -// a.listen() -// var peerA = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/' + a.port)]) - -// attention, peerB Id isn't going to match, but whateves -var peerB = new Peer(Id.create(), [multiaddr('/ip4/127.0.0.1/tcp/4001')]) - -// var i = new Identify(a, peerA) -// i.on('thenews', function (news) { -// console.log('such news') -// }) - -a.openStream(peerB, '/ipfs/sparkles/1.2.3', function (err, stream) { - if (err) { - return console.log(err) - } - console.log('WoHoo, dialed a stream') -}) diff --git a/examples/peerB.js b/examples/peerB.js deleted file mode 100644 index 95ca3ff620..0000000000 --- a/examples/peerB.js +++ /dev/null @@ -1,14 +0,0 @@ -var Swarm = require('./../src') - -var Peer = require('peer-info') -var Id = require('peer-id') -var multiaddr = require('multiaddr') -var tcp = require('libp2p-tcp') - -var mh = multiaddr('/ip4/127.0.0.1/tcp/8010') -var p = new Peer(Id.create(), []) -var sw = new Swarm(p) - -sw.addTransport('tcp', tcp, { multiaddr: mh }, {}, {port: 8010}, function () { - console.log('transport added') -}) diff --git a/package.json b/package.json index 8c13358965..e2e81a6cd6 100644 --- a/package.json +++ b/package.json @@ -33,11 +33,11 @@ "chai": "^3.5.0", "istanbul": "^0.4.2", "libp2p-spdy": "^0.2.3", - "libp2p-tcp": "^0.2.1", + "libp2p-tcp": "^0.3.0", "mocha": "^2.4.5", "multiaddr": "^1.1.1", - "peer-id": "^0.5.3", - "peer-info": "^0.5.2", + "peer-id": "^0.6.0", + "peer-info": "^0.6.0", "pre-commit": "^1.1.2", "standard": "^6.0.7", "stream-pair": "^1.0.3" diff --git a/src/identify.js b/src/identify.js index 7bad7eb7e0..0ef04ae9cb 100644 --- a/src/identify.js +++ b/src/identify.js @@ -1,166 +1,93 @@ /* * Identify is one of the protocols swarms speaks in order to * broadcast and learn about the ip:port pairs a specific peer - * is available through + * is available through and to know when a new stream muxer is + * established, so a conn can be reused */ -// var multistream = require('multistream-select') -// var protobufs = require('protocol-buffers-stream') -// var fs = require('fs') -// var path = require('path') -// var protobufs = require('protocol-buffers-stream') -// var schema = fs.readFileSync(path.join(__dirname, 'identify.proto')) -// var Address6 = require('ip-address').Address6 -// var Id = require('peer-id') -// var multiaddr = require('multiaddr') +const multistream = require('multistream-select') +const fs = require('fs') +const path = require('path') +const pbStream = require('protocol-buffers-stream')( + fs.readFileSync(path.join(__dirname, 'identify.proto'))) +const Info = require('peer-info') +const Id = require('peer-id') +const multiaddr = require('multiaddr') exports = module.exports - exports.multicodec = '/ipfs/identify/1.0.0' -exports.exec = (muxedConn, callback) => { - // TODO +exports.exec = (rawConn, muxer, peerInfo, callback) => { // 1. open a stream // 2. multistream into identify - // 3. send what I see from this other peer + // 3. send what I see from this other peer (extract fro conn) // 4. receive what the other peer sees from me // 4. callback with (err, peerInfo) -} -exports.handler = (peerInfo) => { - return function (conn) { - // TODO - // 1. receive incoming observed info about me - // 2. send back what I see from the other - } -} + const conn = muxer.newStream() -/* -function identify (muxedConns, peerInfoSelf, socket, conn, muxer) { - var msi = new Interactive() - msi.handle(conn, function () { - msi.select(protoId, function (err, ds) { + var msI = new multistream.Interactive() + msI.handle(conn, () => { + msI.select(exports.multicodec, (err, ds) => { if (err) { - return console.log(err) + return callback(err) } - var ps = createProtoStream() + var pbs = pbStream() - ps.on('identify', function (msg) { - var peerId = Id.createFromPubKey(msg.publicKey) + pbs.on('identify', (msg) => { + peerInfo.multiaddr.addSafe(msg.observedAddr) - updateSelf(peerInfoSelf, msg.observedAddr) + const peerId = Id.createFromPubKey(msg.publicKey) + const otherPeerInfo = new Info(peerId) + msg.listenAddrs.forEach((ma) => { + otherPeerInfo.multiaddr.add(multiaddr(ma)) + }) - muxedConns[peerId.toB58String()] = { - muxer: muxer, - socket: socket - } + callback(null, otherPeerInfo) + }) - var mh = getMultiaddr(socket) + const obsMultiaddr = rawConn.getObservedAddrs()[0] - ps.identify({ + pbs.identify({ protocolVersion: 'na', agentVersion: 'na', - publicKey: peerInfoSelf.id.pubKey, - listenAddrs: peerInfoSelf.multiaddrs.map(function (mh) { - return mh.buffer - }), - observedAddr: mh.buffer + publicKey: peerInfo.id.pubKey, + listenAddrs: peerInfo.multiaddrs.map((mh) => { return mh.buffer }), + observedAddr: obsMultiaddr ? obsMultiaddr.buffer : null }) - ps.pipe(ds).pipe(ps) - ps.finalize() + pbs.pipe(ds).pipe(pbs) + pbs.finalize() }) }) } -exports.getHandlerFunction = function (peerInfoSelf, muxedConns) { +exports.handler = (peerInfo, swarm) => { return function (conn) { - // wait for the other peer to identify itself - // update our multiaddr with observed addr list - // then get the socket from our list of muxedConns and send the reply back - - var ps = createProtoStream() - - ps.on('identify', function (msg) { - updateSelf(peerInfoSelf, msg.observedAddr) - - var peerId = Id.createFromPubKey(msg.publicKey) + // 1. receive incoming observed info about me + // 2. update my own information (on peerInfo) + // 3. send back what I see from the other (get from swarm.muxedConns[incPeerID].conn.getObservedAddrs() + var pbs = pbStream() - var socket = muxedConns[peerId.toB58String()].socket + pbs.on('identify', function (msg) { + peerInfo.multiaddr.addSafe(msg.observedAddr) - var mh = getMultiaddr(socket) + const peerId = Id.createFromPubKey(msg.publicKey) + const conn = swarm.muxedConns[peerId.toB58String()].conn + const obsMultiaddr = conn.getObservedAddrs()[0] - ps.identify({ + pbs.identify({ protocolVersion: 'na', agentVersion: 'na', - publicKey: peerInfoSelf.id.pubKey, - listenAddrs: peerInfoSelf.multiaddrs.map(function (mh) { - return mh.buffer + publicKey: peerInfo.id.pubKey, + listenAddrs: peerInfo.multiaddrs.map(function (ma) { + return ma.buffer }), - observedAddr: mh.buffer + observedAddr: obsMultiaddr ? obsMultiaddr.buffer : null }) - - // TODO: Pass the new discovered info about the peer that contacted us - // to something like the Kademlia Router, so the peerInfo for this peer - // is fresh - // - before this was exectued through a event emitter - // self.emit('peer-update', { - // peerId: peerId, - // listenAddrs: msg.listenAddrs.map(function (mhb) { - // return multiaddr(mhb) - // }) - // }) - - ps.finalize() + pbs.finalize() }) - ps.pipe(conn).pipe(ps) + pbs.pipe(conn).pipe(pbs) } } - -function getMultiaddr (socket) { - var mh - if (socket.remoteFamily === 'IPv6') { - var addr = new Address6(socket.remoteAddress) - if (addr.v4) { - var ip4 = addr.to4().correctForm() - mh = multiaddr('/ip4/' + ip4 + '/tcp/' + socket.remotePort) - } else { - mh = multiaddr('/ip6/' + socket.remoteAddress + '/tcp/' + socket.remotePort) - } - } else { - mh = multiaddr('/ip4/' + socket.remoteAddress + '/tcp/' + socket.remotePort) - } - return mh -} - -function updateSelf (peerSelf, observedAddr) { - var omh = multiaddr(observedAddr) - - if (!peerSelf.previousObservedAddrs) { - peerSelf.previousObservedAddrs = [] - } - - for (var i = 0; i < peerSelf.previousObservedAddrs.length; i++) { - if (peerSelf.previousObservedAddrs[i].toString() === omh.toString()) { - peerSelf.previousObservedAddrs.splice(i, 1) - addToSelf() - return - } - } - - peerSelf.previousObservedAddrs.push(omh) - - function addToSelf () { - var isIn = false - peerSelf.multiaddrs.forEach(function (mh) { - if (mh.toString() === omh.toString()) { - isIn = true - } - }) - - if (!isIn) { - peerSelf.multiaddrs.push(omh) - } - } -}*/ diff --git a/src/index.js b/src/index.js index 4e3f3f6df1..34427f55d5 100644 --- a/src/index.js +++ b/src/index.js @@ -42,7 +42,7 @@ function Swarm (peerInfo) { multiaddrs = [multiaddrs] } - // TODO a) filter the multiaddrs that are actually valid for this transport (use a func from the transport itself) + // TODO a) filter the multiaddrs that are actually valid for this transport (use a func from the transport itself) (maybe even make the transport do that) // b) if multiaddrs.length = 1, return the conn from the // transport, otherwise, create a passthrough @@ -116,7 +116,7 @@ function Swarm (peerInfo) { // { // peerIdB58: { // muxer: - // rawSocket: socket // to abstract info required for the Identify Protocol + // conn: // to extract info required for the Identify Protocol // } // } this.muxedConns = {} @@ -136,12 +136,19 @@ function Swarm (peerInfo) { // for listening this.handle(muxer.multicodec, (conn) => { const muxedConn = muxer(conn, true) - muxedConn.on('stream', connHandler) + muxedConn.on('stream', (conn) => { + connHandler(conn) + }) + // if identify is enabled, attempt to do it for muxer reuse if (this.identify) { - identify.exec(muxedConn, (err, pi) => { - if (err) {} - // TODO muxedConns[pi.id.toB58String()].muxer = muxedConn + identify.exec(conn, muxedConn, peerInfo, (err, pi) => { + if (err) { + return console.log('Identify exec failed', err) + } + this.muxedConns[pi.id.toB58String()] = {} + this.muxedConns[pi.id.toB58String()].muxer = muxedConn + this.muxedConns[pi.id.toB58String()].conn = conn // to be able to extract addrs }) } }) @@ -151,18 +158,19 @@ function Swarm (peerInfo) { this.identify = false this.connection.reuse = () => { this.identify = true - this.handle(identify.multicodec, identify.handler(peerInfo)) + this.handle(identify.multicodec, identify.handler(peerInfo, this)) } - const self = this // couldn't get rid of this + const self = this // prefered this to bind // incomming connection handler function connHandler (conn) { var msS = new multistream.Select() - msS.handle(conn) - Object.keys(self.protocols).forEach(function (protocol) { + Object.keys(self.protocols).forEach((protocol) => { + if (!protocol) { return } msS.addHandler(protocol, self.protocols[protocol]) }) + msS.handle(conn) } // higher level (public) API @@ -258,11 +266,17 @@ function Swarm (peerInfo) { } else { nextMuxer(muxers.shift()) } + return } const muxedConn = self.muxers[key](conn, false) self.muxedConns[b58Id] = {} self.muxedConns[b58Id].muxer = muxedConn + self.muxedConns[b58Id].conn = conn + + // in case identify is on + muxedConn.on('stream', connHandler) + cb(null, muxedConn) }) }) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 06e1161210..13c2889450 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -303,6 +303,10 @@ describe('stream muxing (on TCP)', function () { peerB = new Peer() peerC = new Peer() + // console.log('peer A', peerA.id.toB58String()) + // console.log('peer B', peerB.id.toB58String()) + // console.log('peer C', peerC.id.toB58String()) + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9003')) @@ -386,7 +390,7 @@ describe('stream muxing (on TCP)', function () { }) }) - it.skip('enable identify to reuse incomming muxed conn', (done) => { + it('enable identify to reuse incomming muxed conn', (done) => { swarmA.connection.reuse() swarmC.connection.reuse() @@ -395,7 +399,8 @@ describe('stream muxing (on TCP)', function () { setTimeout(() => { expect(Object.keys(swarmC.muxedConns).length).to.equal(1) expect(Object.keys(swarmA.muxedConns).length).to.equal(2) - }, 100) + done() + }, 500) }) }) }) From 849d38e8e00cc73f1c302fb3f4547c4851678e9e Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 10 Mar 2016 21:00:15 +0000 Subject: [PATCH 094/634] update readme badges --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a7b56b85f..107fd71854 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ libp2p-swarm JavaScript implementation ====================================== -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) [![Build Status](https://img.shields.io/travis/diasdavid/js-ipfs-swarm/master.svg?style=flat-square)](https://travis-ci.org/diasdavid/js-ipfs-swarm) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) +[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![Build Status](https://img.shields.io/travis/diasdavid/js-libp2p-swarm/master.svg?style=flat-square)](https://travis-ci.org/diasdavid/js-libp2p-swarm) > libp2p swarm implementation in JavaScript From 69bd386afceaa115fc8ca66f5078e430f7e46271 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 10 Mar 2016 21:00:28 +0000 Subject: [PATCH 095/634] Release v0.6.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e2e81a6cd6..8f0777618d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.5.6", + "version": "0.6.0", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { From 412bda731b8beac812bd51e4376efea573cdb3bc Mon Sep 17 00:00:00 2001 From: David Dias Date: Fri, 11 Mar 2016 14:47:39 +0000 Subject: [PATCH 096/634] fixed --- package.json | 2 +- src/index.js | 12 ++++++------ tests/swarm-test.js | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 8f0777618d..1ba28ee6c3 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "stream-pair": "^1.0.3" }, "dependencies": { - "async": "^1.3.0", + "duplex-passthrough": "github:diasdavid/duplex-passthrough", "ip-address": "^5.0.2", "multistream-select": "^0.6.1", "protocol-buffers-stream": "^1.2.0" diff --git a/src/index.js b/src/index.js index 34427f55d5..c454b8236c 100644 --- a/src/index.js +++ b/src/index.js @@ -1,7 +1,6 @@ const multistream = require('multistream-select') -// const async = require('async') const identify = require('./identify') -const PassThrough = require('stream').PassThrough +const DuplexPassThrough = require('duplex-passthrough') exports = module.exports = Swarm @@ -60,13 +59,13 @@ function Swarm (peerInfo) { // c) multiaddrs should already be a filtered list // specific for the transport we are using - const pt = new PassThrough() + const pt = new DuplexPassThrough() next(multiaddrs.shift()) return pt function next (multiaddr) { const conn = t.dial(multiaddr, {ready: () => { - pt.pipe(conn).pipe(pt) + pt.wrapStream(conn) const cb = callback callback = noop // this is done to avoid connection drops as connect errors cb(null, pt) @@ -180,7 +179,7 @@ function Swarm (peerInfo) { callback = protocol protocol = null } else { - pt = new PassThrough() + pt = new DuplexPassThrough() } const b58Id = pi.id.toB58String() @@ -293,7 +292,8 @@ function Swarm (peerInfo) { if (err) { return callback(err) } - pt.pipe(conn).pipe(pt) + + pt.wrapStream(conn) callback(null, pt) }) }) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 13c2889450..18b2e7cae3 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -257,6 +257,7 @@ describe('high level API - 1st without stream multiplexing (on TCP)', function ( swarmA.dial(peerB, '/pineapple/1.0.0', (err, conn) => { expect(err).to.not.exist conn.end() + conn.on('data', () => {}) // let it flow.. let it flooooow conn.on('end', done) }) }) @@ -272,6 +273,7 @@ describe('high level API - 1st without stream multiplexing (on TCP)', function ( swarmA.dial(peerB, '/bananas/1.0.0', (err, conn) => { expect(err).to.not.exist conn.end() + conn.on('data', () => {}) // let it flow.. let it flooooow conn.on('end', done) }) }) @@ -363,6 +365,8 @@ describe('stream muxing (on TCP)', function () { expect(err).to.not.exist expect(Object.keys(swarmA.muxedConns).length).to.equal(1) conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow conn.on('end', done) }) }) @@ -386,6 +390,8 @@ describe('stream muxing (on TCP)', function () { expect(Object.keys(swarmB.conns).length).to.equal(0) expect(Object.keys(swarmB.muxedConns).length).to.equal(1) conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow conn.on('end', done) }) }) @@ -406,7 +412,6 @@ describe('stream muxing (on TCP)', function () { }) }) -/* describe('conn upgrades', function () { this.timeout(20000) @@ -439,4 +444,3 @@ describe('high level API - with everything mixed all together!', function () { it.skip('add websockets', (done) => {}) it.skip('dial', (done) => {}) }) -*/ From 4036ea4b1adb4f1261e58cfc5b170872db64bd2d Mon Sep 17 00:00:00 2001 From: David Dias Date: Fri, 11 Mar 2016 14:56:16 +0000 Subject: [PATCH 097/634] Release v0.7.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1ba28ee6c3..69116f1d5f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.6.0", + "version": "0.7.0", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { From b83e5dd8dc0109db746995f0cc6f2620b41bcd61 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 15 Mar 2016 09:57:06 +0000 Subject: [PATCH 098/634] complete --- package.json | 4 ++-- src/index.js | 3 ++- tests/swarm-test.js | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 69116f1d5f..e9ccf2fa5f 100644 --- a/package.json +++ b/package.json @@ -33,9 +33,9 @@ "chai": "^3.5.0", "istanbul": "^0.4.2", "libp2p-spdy": "^0.2.3", - "libp2p-tcp": "^0.3.0", + "libp2p-tcp": "^0.4.0", "mocha": "^2.4.5", - "multiaddr": "^1.1.1", + "multiaddr": "^1.3.0", "peer-id": "^0.6.0", "peer-info": "^0.6.0", "pre-commit": "^1.1.2", diff --git a/src/index.js b/src/index.js index c454b8236c..d756609329 100644 --- a/src/index.js +++ b/src/index.js @@ -41,7 +41,8 @@ function Swarm (peerInfo) { multiaddrs = [multiaddrs] } - // TODO a) filter the multiaddrs that are actually valid for this transport (use a func from the transport itself) (maybe even make the transport do that) + // a) filter the multiaddrs that are actually valid for this transport (use a func from the transport itself) (maybe even make the transport do that) + multiaddrs = t.filter(multiaddrs) // b) if multiaddrs.length = 1, return the conn from the // transport, otherwise, create a passthrough diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 18b2e7cae3..1d5107e6d6 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -77,6 +77,7 @@ describe('transport - tcp', function () { it('dial to set of multiaddr, only one is available', (done) => { const conn = swarmA.transport.dial('tcp', [ + multiaddr('/ip4/127.0.0.1/tcp/9910/websockets'), // not valid on purpose multiaddr('/ip4/127.0.0.1/tcp/9910'), multiaddr('/ip4/127.0.0.1/tcp/9999'), multiaddr('/ip4/127.0.0.1/tcp/9309') From 6bcf48ff39a9fd66727cdb8ca96b74f34a135889 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 15 Mar 2016 10:25:27 +0000 Subject: [PATCH 099/634] Release v0.8.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e9ccf2fa5f..aa0fb69c67 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.7.0", + "version": "0.8.0", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { From eda78d7a7636bf6089a4edaeea4311350ef75845 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 15 Mar 2016 11:14:55 +0000 Subject: [PATCH 100/634] add websockets to the battery of tests, everything checks out --- package.json | 1 + tests/swarm-test.js | 74 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index aa0fb69c67..0baae0b3c3 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "istanbul": "^0.4.2", "libp2p-spdy": "^0.2.3", "libp2p-tcp": "^0.4.0", + "libp2p-websockets": "^0.2.0", "mocha": "^2.4.5", "multiaddr": "^1.3.0", "peer-id": "^0.6.0", diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 1d5107e6d6..69dfd350af 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -8,6 +8,7 @@ const multiaddr = require('multiaddr') const Peer = require('peer-info') const Swarm = require('../src') const TCP = require('libp2p-tcp') +const WebSockets = require('libp2p-websockets') const bl = require('bl') const spdy = require('libp2p-spdy') @@ -174,18 +175,77 @@ describe('transport - tcp', function () { }) }) -describe('transport - udt', function () { +describe('transport - websockets', function () { this.timeout(10000) - before((done) => { done() }) + var swarmA + var swarmB + var peerA = new Peer() + var peerB = new Peer() - it.skip('add', (done) => {}) - it.skip('listen', (done) => {}) - it.skip('dial', (done) => {}) - it.skip('close', (done) => {}) + before((done) => { + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9888/websockets')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9999/websockets')) + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + done() + }) + + it('add', (done) => { + swarmA.transport.add('ws', new WebSockets()) + expect(Object.keys(swarmA.transports).length).to.equal(1) + swarmB.transport.add('ws', new WebSockets(), () => { + expect(Object.keys(swarmB.transports).length).to.equal(1) + done() + }) + }) + + it('listen', (done) => { + var count = 0 + swarmA.transport.listen('ws', {}, (conn) => { + conn.pipe(conn) + }, ready) + swarmB.transport.listen('ws', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + if (++count === 2) { + expect(peerA.multiaddrs.length).to.equal(1) + expect(peerA.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9888/websockets')) + expect(peerB.multiaddrs.length).to.equal(1) + expect(peerB.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9999/websockets')) + done() + } + } + }) + + it('dial', (done) => { + const conn = swarmA.transport.dial('ws', multiaddr('/ip4/127.0.0.1/tcp/9999/websockets'), (err, conn) => { + expect(err).to.not.exist + }) + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + done() + })) + conn.write('hey') + conn.end() + }) + + it('close', (done) => { + var count = 0 + swarmA.transport.close('ws', closed) + swarmB.transport.close('ws', closed) + + function closed () { + if (++count === 2) { + done() + } + } + }) }) -describe('transport - websockets', function () { +describe('transport - utp', function () { this.timeout(10000) before((done) => { done() }) From 63569849c0af1329de0fab0a9e297bcb4406b2b1 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 15 Mar 2016 11:21:54 +0000 Subject: [PATCH 101/634] Release v0.8.1. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0baae0b3c3..852876a1a5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.8.0", + "version": "0.8.1", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { From 9a29b01ad3aceaab9425325b50a454b46e74bfde Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 15 Mar 2016 12:20:42 +0000 Subject: [PATCH 102/634] =?UTF-8?q?it=20wooorks=E2=84=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/index.js | 6 +- tests/swarm-test.js | 133 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 125 insertions(+), 14 deletions(-) diff --git a/src/index.js b/src/index.js index d756609329..27eb70bf71 100644 --- a/src/index.js +++ b/src/index.js @@ -85,11 +85,7 @@ function Swarm (peerInfo) { // if no callback is passed, we pass conns to connHandler if (!handler) { handler = connHandler } - const multiaddrs = peerInfo.multiaddrs.filter((m) => { - if (m.toString().indexOf('tcp') !== -1) { - return m - } - }) + const multiaddrs = this.transports[key].filter(peerInfo.multiaddrs) this.transports[key].createListener(multiaddrs, handler, (err, maUpdate) => { if (err) { diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 69dfd350af..03a6333e09 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -477,16 +477,16 @@ describe('conn upgrades', function () { this.timeout(20000) describe('secio on tcp', () => { - // before((done) => { done() }) - // after((done) => { done() }) + before((done) => { done() }) + after((done) => { done() }) it.skip('add', (done) => {}) it.skip('dial', (done) => {}) it.skip('tls on a muxed stream (not the full conn)', (done) => {}) }) describe('tls on tcp', () => { - // before((done) => { done() }) - // after((done) => { done() }) + before((done) => { done() }) + after((done) => { done() }) it.skip('add', (done) => {}) it.skip('dial', (done) => {}) @@ -497,11 +497,126 @@ describe('conn upgrades', function () { describe('high level API - with everything mixed all together!', function () { this.timeout(20000) - // before((done) => { done() }) - // after((done) => { done() }) + var swarmA // tcp + var peerA + var swarmB // tcp+ws + var peerB + var swarmC // tcp+ws + var peerC + + before((done) => { + peerA = new Peer() + peerB = new Peer() + peerC = new Peer() + + // console.log('peer A', peerA.id.toB58String()) + // console.log('peer B', peerB.id.toB58String()) + // console.log('peer C', peerC.id.toB58String()) + + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + swarmC = new Swarm(peerC) + + done() + }) + + after((done) => { + var counter = 0 + + swarmA.close(closed) + swarmB.close(closed) + swarmC.close(closed) + + function closed () { + if (++counter === 3) { + done() + } + } + }) + + it('add tcp', (done) => { + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9011')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9021')) + peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9031')) + + swarmA.transport.add('tcp', new TCP()) + swarmA.transport.listen('tcp', {}, null, ready) + + swarmB.transport.add('tcp', new TCP()) + swarmB.transport.listen('tcp', {}, null, ready) + + swarmC.transport.add('tcp', new TCP()) + swarmC.transport.listen('tcp', {}, null, ready) + + var counter = 0 + + function ready () { + if (++counter === 3) { + done() + } + } + }) - it.skip('add tcp', (done) => {}) it.skip('add utp', (done) => {}) - it.skip('add websockets', (done) => {}) - it.skip('dial', (done) => {}) + + it('add websockets', (done) => { + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9022/websockets')) + peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9032/websockets')) + + swarmB.transport.add('ws', new WebSockets()) + swarmB.transport.listen('ws', {}, null, ready) + + swarmC.transport.add('ws', new WebSockets()) + swarmC.transport.listen('ws', {}, null, ready) + + var counter = 0 + + function ready () { + if (++counter === 2) { + done() + } + } + }) + + it('add spdy', (done) => { + swarmA.connection.addStreamMuxer(spdy) + swarmB.connection.addStreamMuxer(spdy) + swarmC.connection.addStreamMuxer(spdy) + swarmA.connection.reuse() + swarmB.connection.reuse() + swarmC.connection.reuse() + done() + }) + + it.skip('add multiplex', (done) => {}) + + it('dial from tcp to tcp+ws', (done) => { + swarmB.handle('/anona/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmA.dial(peerB, '/anona/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmA.muxedConns).length).to.equal(1) + conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) + + it('dial from tcp+ws to tcp+ws', (done) => { + swarmC.handle('/mamao/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmA.dial(peerC, '/mamao/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmA.muxedConns).length).to.equal(2) + conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) }) From 9d149caf15e58d28a2c1093f54408381a333f783 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 15 Mar 2016 12:57:14 +0000 Subject: [PATCH 103/634] Release v0.9.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 852876a1a5..60e1aef704 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.8.1", + "version": "0.9.0", "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { From f2df5586d5c4d1c1fdf71a44a01f9fc9ec0f37d0 Mon Sep 17 00:00:00 2001 From: Francisco Baio Dias Date: Sun, 20 Mar 2016 17:57:14 +0000 Subject: [PATCH 104/634] Update libp2p-websockets and add test for callback's conn --- package.json | 2 +- tests/swarm-test.js | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 60e1aef704..3cda3a1410 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "istanbul": "^0.4.2", "libp2p-spdy": "^0.2.3", "libp2p-tcp": "^0.4.0", - "libp2p-websockets": "^0.2.0", + "libp2p-websockets": "^0.2.1", "mocha": "^2.4.5", "multiaddr": "^1.3.0", "peer-id": "^0.6.0", diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 03a6333e09..fbd892c03d 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -232,6 +232,19 @@ describe('transport - websockets', function () { conn.end() }) + it('dial (conn from callback)', (done) => { + swarmA.transport.dial('ws', multiaddr('/ip4/127.0.0.1/tcp/9999/websockets'), (err, conn) => { + expect(err).to.not.exist + + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + done() + })) + conn.write('hey') + conn.end() + }) + }) + it('close', (done) => { var count = 0 swarmA.transport.close('ws', closed) From c726c252df4c23849a29eca2af28f1d09eef8a3e Mon Sep 17 00:00:00 2001 From: Francisco Baio Dias Date: Sun, 20 Mar 2016 18:11:16 +0000 Subject: [PATCH 105/634] Return conn on swarm.dial --- src/index.js | 2 ++ tests/swarm-test.js | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/index.js b/src/index.js index 27eb70bf71..eddc858bb4 100644 --- a/src/index.js +++ b/src/index.js @@ -197,6 +197,8 @@ function Swarm (peerInfo) { gotMuxer(this.muxedConns[b58Id].muxer) } + return pt + function gotWarmedUpConn (conn) { attemptMuxerUpgrade(conn, (err, muxer) => { if (!protocol) { diff --git a/tests/swarm-test.js b/tests/swarm-test.js index fbd892c03d..6c0c3b4242 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -336,6 +336,19 @@ describe('high level API - 1st without stream multiplexing (on TCP)', function ( }) }) + it('dial on protocol (returned conn)', (done) => { + swarmB.handle('/apples/1.0.0', (conn) => { + conn.pipe(conn) + }) + + const conn = swarmA.dial(peerB, '/apples/1.0.0', (err) => { + expect(err).to.not.exist + }) + conn.end() + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + it('dial to warm a conn', (done) => { swarmA.dial(peerB, (err) => { expect(err).to.not.exist @@ -618,6 +631,21 @@ describe('high level API - with everything mixed all together!', function () { }) }) + it('dial from tcp to tcp+ws (returned conn)', (done) => { + swarmB.handle('/grapes/1.0.0', (conn) => { + conn.pipe(conn) + }) + + const conn = swarmA.dial(peerB, '/grapes/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmA.muxedConns).length).to.equal(1) + }) + conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + it('dial from tcp+ws to tcp+ws', (done) => { swarmC.handle('/mamao/1.0.0', (conn) => { conn.pipe(conn) From caf3a0180aa9685d90d8ffe63ef32880c29740ad Mon Sep 17 00:00:00 2001 From: Francisco Baio Dias Date: Wed, 16 Mar 2016 00:32:13 +0000 Subject: [PATCH 106/634] Add browser tests --- .travis.yml | 7 +++++++ karma.conf.js | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 15 ++++++++++++-- src/identify.js | 10 ++++++++-- tests/browser.js | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ tests/karma.js | 33 +++++++++++++++++++++++++++++++ 6 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 karma.conf.js create mode 100644 tests/browser.js create mode 100644 tests/karma.js diff --git a/.travis.yml b/.travis.yml index fdd1203c0c..b0136dec54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,3 +10,10 @@ before_install: script: - npm run lint - npm test + +addons: + firefox: 'latest' + +before_script: + - export DISPLAY=:99.0 + - sh -e /etc/init.d/xvfb start diff --git a/karma.conf.js b/karma.conf.js new file mode 100644 index 0000000000..07cb4fceaa --- /dev/null +++ b/karma.conf.js @@ -0,0 +1,51 @@ +module.exports = function (config) { + var path = require('path') + var nodeForgePath = path.resolve(__dirname, 'node_modules/peer-id/deps/forge.bundle.js') + + config.set({ + basePath: '', + frameworks: ['mocha'], + + files: [ + nodeForgePath, + 'tests/browser.js' + ], + + preprocessors: { + 'tests/*': ['webpack'] + }, + + webpack: { + resolve: { + extensions: ['', '.js', '.json'] + }, + externals: { + fs: '{}', + 'node-forge': 'forge' + }, + node: { + Buffer: true + }, + module: { + loaders: [ + { test: /\.json$/, loader: 'json' } + ] + } + }, + + webpackMiddleware: { + noInfo: true, + stats: { + colors: true + } + }, + reporters: ['spec'], + port: 9876, + colors: true, + logLevel: config.LOG_INFO, + autoWatch: false, + browsers: process.env.TRAVIS ? ['Firefox'] : ['Chrome'], + captureTimeout: 60000, + singleRun: true + }) +} diff --git a/package.json b/package.json index 3cda3a1410..ecb670b364 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,9 @@ "description": "libp2p swarm implementation in Node.js", "main": "src/index.js", "scripts": { - "test": "mocha tests/*-test.js", + "test:node": "mocha tests/*-test.js", + "test:browser": "node tests/karma.js", + "test": "npm run test:node && npm run test:browser", "coverage": "istanbul cover --print both -- _mocha tests/*-test.js", "lint": "standard" }, @@ -30,8 +32,16 @@ }, "devDependencies": { "bl": "^1.1.2", + "buffer-loader": "0.0.1", "chai": "^3.5.0", "istanbul": "^0.4.2", + "json-loader": "^0.5.4", + "karma": "^0.13.22", + "karma-chrome-launcher": "^0.2.2", + "karma-firefox-launcher": "^0.1.7", + "karma-mocha": "^0.2.2", + "karma-spec-reporter": "0.0.24", + "karma-webpack": "^1.7.0", "libp2p-spdy": "^0.2.3", "libp2p-tcp": "^0.4.0", "libp2p-websockets": "^0.2.1", @@ -41,7 +51,8 @@ "peer-info": "^0.6.0", "pre-commit": "^1.1.2", "standard": "^6.0.7", - "stream-pair": "^1.0.3" + "stream-pair": "^1.0.3", + "webpack": "^2.1.0-beta.4" }, "dependencies": { "duplex-passthrough": "github:diasdavid/duplex-passthrough", diff --git a/src/identify.js b/src/identify.js index 0ef04ae9cb..078d844435 100644 --- a/src/identify.js +++ b/src/identify.js @@ -8,12 +8,18 @@ const multistream = require('multistream-select') const fs = require('fs') const path = require('path') -const pbStream = require('protocol-buffers-stream')( - fs.readFileSync(path.join(__dirname, 'identify.proto'))) const Info = require('peer-info') const Id = require('peer-id') const multiaddr = require('multiaddr') +const isNode = !global.window + +const identity = isNode + ? fs.readFileSync(path.join(__dirname, 'identify.proto')) + : require('buffer!./identify.proto') + +const pbStream = require('protocol-buffers-stream')(identity) + exports = module.exports exports.multicodec = '/ipfs/identify/1.0.0' diff --git a/tests/browser.js b/tests/browser.js new file mode 100644 index 0000000000..62d81f1f77 --- /dev/null +++ b/tests/browser.js @@ -0,0 +1,51 @@ +/* eslint-env mocha */ + +const expect = require('chai').expect +// const async = require('async') + +const multiaddr = require('multiaddr') +// const Id = require('peer-id') +const Peer = require('peer-info') +const Swarm = require('../src') +const WebSockets = require('libp2p-websockets') +const bl = require('bl') + +describe('basics', () => { + it('throws on missing peerInfo', (done) => { + expect(Swarm).to.throw(Error) + done() + }) +}) + +describe('transport - websockets', function () { + this.timeout(10000) + + var swarmB + var peerB = new Peer() + + before((done) => { + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9999/websockets')) + swarmB = new Swarm(peerB) + done() + }) + + it('add', (done) => { + swarmB.transport.add('ws', new WebSockets(), () => { + expect(Object.keys(swarmB.transports).length).to.equal(1) + done() + }) + }) + + it('dial', (done) => { + const conn = swarmB.transport.dial('ws', multiaddr('/ip4/127.0.0.1/tcp/9888/websockets'), (err, conn) => { + expect(err).to.not.exist + }) + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + expect(data.toString()).to.equal('hey') + done() + })) + conn.write('hey') + conn.end() + }) +}) diff --git a/tests/karma.js b/tests/karma.js new file mode 100644 index 0000000000..d41ea25e4d --- /dev/null +++ b/tests/karma.js @@ -0,0 +1,33 @@ +const Server = require('karma').Server +const path = require('path') + +const Peer = require('peer-info') +const WebSockets = require('libp2p-websockets') +const Swarm = require('../src') +const multiaddr = require('multiaddr') + +var swarmA +var peerA + +function createServer (done) { + peerA = new Peer() + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9888/websockets')) + swarmA = new Swarm(peerA) + swarmA.transport.add('ws', new WebSockets()) + swarmA.transport.listen('ws', {}, (conn) => { + conn.pipe(conn) + }, done) +} + +function stopServer (done) { + swarmA.transport.close('ws', done) +} + +function runTests (done) { + new Server({ + configFile: path.join(__dirname, '/../karma.conf.js'), + singleRun: true + }, done).start() +} + +createServer(() => runTests(() => stopServer(() => null))) From e41694d3081ca194db9857857abda9ab73afb660 Mon Sep 17 00:00:00 2001 From: Francisco Baio Dias Date: Sun, 20 Mar 2016 18:50:30 +0000 Subject: [PATCH 107/634] Add more tests --- karma.conf.js | 1 + tests/browser.js | 98 ++++++++++++++++++++++++++++++++++++++++++++---- tests/karma.js | 55 +++++++++++++++++++++------ 3 files changed, 135 insertions(+), 19 deletions(-) diff --git a/karma.conf.js b/karma.conf.js index 07cb4fceaa..50ef8cb0d5 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -46,6 +46,7 @@ module.exports = function (config) { autoWatch: false, browsers: process.env.TRAVIS ? ['Firefox'] : ['Chrome'], captureTimeout: 60000, + browserNoActivityTimeout: 20000, singleRun: true }) } diff --git a/tests/browser.js b/tests/browser.js index 62d81f1f77..5b67917063 100644 --- a/tests/browser.js +++ b/tests/browser.js @@ -4,12 +4,20 @@ const expect = require('chai').expect // const async = require('async') const multiaddr = require('multiaddr') -// const Id = require('peer-id') +const PeerId = require('peer-id') const Peer = require('peer-info') const Swarm = require('../src') const WebSockets = require('libp2p-websockets') const bl = require('bl') +const PEER_ID_SERVER_A = 'QmWg2L4Fucx1x4KXJTfKHGixBJvveubzcd7DdhB2Mqwfh1' +const PEER_ID_SERVER_B = 'QmRy1iU6BHmG5Hd8rnPhPL98cy1W1przUSTAMcGDq9yAAV' +const MULTIADDR_SERVER_A = '/ip4/127.0.0.1/tcp/9888/websockets' +const MULTIADDR_SERVER_B = '/ip4/127.0.0.1/tcp/9999/websockets' + +// random id to be used on the tests +const PEER_ID_A = 'QmYzgdesgjdvD3okTPGZT9NPmh1BuH5FfTVNKjsvaAprhb' + describe('basics', () => { it('throws on missing peerInfo', (done) => { expect(Swarm).to.throw(Error) @@ -20,24 +28,24 @@ describe('basics', () => { describe('transport - websockets', function () { this.timeout(10000) - var swarmB - var peerB = new Peer() + var swarm + var peerId = PeerId.createFromB58String(PEER_ID_A) + var peer = new Peer(peerId) before((done) => { - peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9999/websockets')) - swarmB = new Swarm(peerB) + swarm = new Swarm(peer) done() }) it('add', (done) => { - swarmB.transport.add('ws', new WebSockets(), () => { - expect(Object.keys(swarmB.transports).length).to.equal(1) + swarm.transport.add('ws', new WebSockets(), () => { + expect(Object.keys(swarm.transports).length).to.equal(1) done() }) }) it('dial', (done) => { - const conn = swarmB.transport.dial('ws', multiaddr('/ip4/127.0.0.1/tcp/9888/websockets'), (err, conn) => { + const conn = swarm.transport.dial('ws', multiaddr(MULTIADDR_SERVER_A), (err, conn) => { expect(err).to.not.exist }) conn.pipe(bl((err, data) => { @@ -49,3 +57,77 @@ describe('transport - websockets', function () { conn.end() }) }) + +describe('high level API - 1st without stream multiplexing (on websockets)', function () { + this.timeout(10000) + + var swarm + var peerSelf + var peerServerA + var peerServerB + + before((done) => { + const peerServerAId = PeerId.createFromB58String(PEER_ID_SERVER_A) + peerServerA = new Peer(peerServerAId) + const peerServerBId = PeerId.createFromB58String(PEER_ID_SERVER_B) + peerServerB = new Peer(peerServerBId) + const peerSelfId = PeerId.createFromB58String(PEER_ID_A) + peerSelf = new Peer(peerSelfId) + + peerServerA.multiaddr.add(multiaddr(MULTIADDR_SERVER_A)) + peerServerB.multiaddr.add(multiaddr(MULTIADDR_SERVER_B)) + + swarm = new Swarm(peerSelf) + + swarm.transport.add('ws', new WebSockets()) + expect(Object.keys(swarm.transports).length).to.equal(1) + done() + }) + + // after((done) => { + // swarm.close(done) + // }) + + it('dial using transport interface', (done) => { + const conn = swarm.transport.dial('ws', peerServerA.multiaddrs, (err, conn) => { + expect(err).to.not.exist + }) + + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + expect(data.toString()).to.equal('hey') + done() + })) + conn.write('hey') + conn.end() + }) + + it('dial on protocol', (done) => { + swarm.dial(peerServerB, '/pineapple/1.0.0', (err, conn) => { + expect(err).to.not.exist + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + expect(data.toString()).to.equal('yo') + done() + })) + conn.write('yo') + conn.end() + }) + }) + + it('dial to warm a conn', (done) => { + swarm.dial(peerServerB, (err) => { + expect(err).to.not.exist + done() + }) + }) + + it('dial on protocol, reuse warmed conn', (done) => { + swarm.dial(peerServerB, '/pineapple/1.0.0', (err, conn) => { + expect(err).to.not.exist + conn.end() + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) +}) diff --git a/tests/karma.js b/tests/karma.js index d41ea25e4d..b86a2f8148 100644 --- a/tests/karma.js +++ b/tests/karma.js @@ -2,25 +2,58 @@ const Server = require('karma').Server const path = require('path') const Peer = require('peer-info') +const PeerId = require('peer-id') const WebSockets = require('libp2p-websockets') const Swarm = require('../src') const multiaddr = require('multiaddr') +const PEER_ID_SERVER_A = 'QmWg2L4Fucx1x4KXJTfKHGixBJvveubzcd7DdhB2Mqwfh1' +const PEER_ID_SERVER_B = 'QmRy1iU6BHmG5Hd8rnPhPL98cy1W1przUSTAMcGDq9yAAV' +const MULTIADDR_SERVER_A = '/ip4/127.0.0.1/tcp/9888/websockets' +const MULTIADDR_SERVER_B = '/ip4/127.0.0.1/tcp/9999/websockets' + var swarmA var peerA +var swarmB +var peerB + +function createServers (done) { + function createServerA (cb) { + const id = PeerId.createFromB58String(PEER_ID_SERVER_A) + peerA = new Peer(id) + peerA.multiaddr.add(multiaddr(MULTIADDR_SERVER_A)) + swarmA = new Swarm(peerA) + swarmA.transport.add('ws', new WebSockets()) + swarmA.transport.listen('ws', {}, (conn) => { + conn.pipe(conn) + }, cb) + } + + function createServerB (cb) { + const id = PeerId.createFromB58String(PEER_ID_SERVER_B) + peerB = new Peer(id) + peerB.multiaddr.add(multiaddr(MULTIADDR_SERVER_B)) + swarmB = new Swarm(peerB) + swarmB.transport.add('ws', new WebSockets()) + swarmB.handle('/pineapple/1.0.0', (conn) => { + conn.pipe(conn) + }) + swarmB.transport.listen('ws', {}, null, cb) + } -function createServer (done) { - peerA = new Peer() - peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9888/websockets')) - swarmA = new Swarm(peerA) - swarmA.transport.add('ws', new WebSockets()) - swarmA.transport.listen('ws', {}, (conn) => { - conn.pipe(conn) - }, done) + var count = 0 + const ready = () => ++count === 2 ? done() : null + + createServerA(ready) + createServerB(ready) } -function stopServer (done) { - swarmA.transport.close('ws', done) +function stopServers (done) { + var count = 0 + const ready = () => ++count === 2 ? done() : null + + swarmA.transport.close('ws', ready) + swarmB.transport.close('ws', ready) } function runTests (done) { @@ -30,4 +63,4 @@ function runTests (done) { }, done).start() } -createServer(() => runTests(() => stopServer(() => null))) +createServers(() => runTests(() => stopServers(() => null))) From 8c6ef5250402ace28219a8ca63e4e4a06442ebd2 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 22 Mar 2016 10:20:22 +0000 Subject: [PATCH 108/634] clean browser tests, add badgers --- README.md | 3 + karma.conf.js | 10 ++- package.json | 4 +- tests/browser-nodejs/browser.js | 123 +++++++++++++++++++++++++++++ tests/browser-nodejs/test.js | 65 ++++++++++++++++ tests/browser.js | 133 -------------------------------- tests/karma.js | 66 ---------------- 7 files changed, 199 insertions(+), 205 deletions(-) create mode 100644 tests/browser-nodejs/browser.js create mode 100644 tests/browser-nodejs/test.js delete mode 100644 tests/browser.js delete mode 100644 tests/karma.js diff --git a/README.md b/README.md index 107fd71854..25113352b9 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,9 @@ libp2p-swarm JavaScript implementation [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) [![Build Status](https://img.shields.io/travis/diasdavid/js-libp2p-swarm/master.svg?style=flat-square)](https://travis-ci.org/diasdavid/js-libp2p-swarm) +![](https://img.shields.io/badge/coverage-%3F-yellow.svg?style=flat-square) +[![Dependency Status](https://david-dm.org/ipfs/js-libp2p-swarm.svg?style=flat-square)](https://david-dm.org/ipfs/js-libp2p-swarm) +[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/feross/standard) > libp2p swarm implementation in JavaScript diff --git a/karma.conf.js b/karma.conf.js index 50ef8cb0d5..42617833e3 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -1,6 +1,7 @@ +const path = require('path') + module.exports = function (config) { - var path = require('path') - var nodeForgePath = path.resolve(__dirname, 'node_modules/peer-id/deps/forge.bundle.js') + const nodeForgePath = path.resolve(__dirname, 'node_modules/peer-id/deps/forge.bundle.js') config.set({ basePath: '', @@ -8,11 +9,12 @@ module.exports = function (config) { files: [ nodeForgePath, - 'tests/browser.js' + 'tests/browser-nodejs/browser.js' ], preprocessors: { - 'tests/*': ['webpack'] + 'tests/*': ['webpack'], + 'tests/browser-nodejs/*': ['webpack'] }, webpack: { diff --git a/package.json b/package.json index ecb670b364..47ff5d13b7 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,11 @@ { "name": "libp2p-swarm", "version": "0.9.0", - "description": "libp2p swarm implementation in Node.js", + "description": "libp2p swarm implementation in JavaScript", "main": "src/index.js", "scripts": { "test:node": "mocha tests/*-test.js", - "test:browser": "node tests/karma.js", + "test:browser": "node tests/browser-nodejs/test.js", "test": "npm run test:node && npm run test:browser", "coverage": "istanbul cover --print both -- _mocha tests/*-test.js", "lint": "standard" diff --git a/tests/browser-nodejs/browser.js b/tests/browser-nodejs/browser.js new file mode 100644 index 0000000000..6b6a7ed037 --- /dev/null +++ b/tests/browser-nodejs/browser.js @@ -0,0 +1,123 @@ +/* eslint-env mocha */ + +const expect = require('chai').expect +const multiaddr = require('multiaddr') +const Id = require('peer-id') +const Peer = require('peer-info') +const Swarm = require('./../../src') +const WebSockets = require('libp2p-websockets') +const bl = require('bl') + +describe('basics', () => { + it('throws on missing peerInfo', (done) => { + expect(Swarm).to.throw(Error) + done() + }) +}) + +describe('transport - websockets', function () { + this.timeout(10000) + + var swarm + + before((done) => { + const b58IdSrc = 'QmYzgdesgjdvD3okTPGZT9NPmh1BuH5FfTVNKjsvaAprhb' + // use a pre generated Id to save time + const idSrc = Id.createFromB58String(b58IdSrc) + const peerSrc = new Peer(idSrc) + swarm = new Swarm(peerSrc) + + done() + }) + + it('add', (done) => { + swarm.transport.add('ws', new WebSockets(), () => { + expect(Object.keys(swarm.transports).length).to.equal(1) + done() + }) + }) + + it('dial', (done) => { + const ma = multiaddr('/ip4/127.0.0.1/tcp/9100/websockets') + + const conn = swarm.transport.dial('ws', ma, (err, conn) => { + expect(err).to.not.exist + }) + + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + expect(data.toString()).to.equal('hey') + done() + })) + conn.write('hey') + conn.end() + }) +}) + +describe('high level API - 1st without stream multiplexing (on websockets)', function () { + this.timeout(10000) + + var swarm + var peerDst + + before((done) => { + const b58IdSrc = 'QmYzgdesgjdvD3okTPGZT9NPmh1BuH5FfTVNKjsvaAprhb' + // use a pre generated Id to save time + const idSrc = Id.createFromB58String(b58IdSrc) + const peerSrc = new Peer(idSrc) + swarm = new Swarm(peerSrc) + + done() + }) + + after((done) => { + done() + // swarm.close(done) + }) + + it('add ws', (done) => { + swarm.transport.add('ws', new WebSockets()) + expect(Object.keys(swarm.transports).length).to.equal(1) + done() + }) + + it('create Dst peer info', (done) => { + const b58IdDst = 'QmYzgdesgjdvD3okTPGZT9NPmh1BuH5FfTVNKjsvaAprhb' + // use a pre generated Id to save time + const idDst = Id.createFromB58String(b58IdDst) + peerDst = new Peer(idDst) + + const ma = multiaddr('/ip4/127.0.0.1/tcp/9200/websockets') + peerDst.multiaddr.add(ma) + done() + }) + + it('dial on protocol', (done) => { + swarm.dial(peerDst, '/echo/1.0.0', (err, conn) => { + expect(err).to.not.exist + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + expect(data.toString()).to.equal('hey') + done() + })) + conn.write('hey') + conn.end() + }) + }) + + it('dial to warm a conn', (done) => { + swarm.dial(peerDst, (err) => { + expect(err).to.not.exist + done() + }) + }) + + it('dial on protocol, reuse warmed conn', (done) => { + swarm.dial(peerDst, '/echo/1.0.0', (err, conn) => { + expect(err).to.not.exist + conn.end() + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) +}) diff --git a/tests/browser-nodejs/test.js b/tests/browser-nodejs/test.js new file mode 100644 index 0000000000..9f54de2aef --- /dev/null +++ b/tests/browser-nodejs/test.js @@ -0,0 +1,65 @@ +const Server = require('karma').Server +const path = require('path') + +const Peer = require('peer-info') +const Id = require('peer-id') +const WebSockets = require('libp2p-websockets') +const Swarm = require('./../../src') +const multiaddr = require('multiaddr') + +var swarmA +var swarmB + +function createListeners (done) { + function createListenerA (cb) { + const b58IdA = 'QmWg2L4Fucx1x4KXJTfKHGixBJvveubzcd7DdhB2Mqwfh1' + const peerA = new Peer(Id.createFromB58String(b58IdA)) + const maA = multiaddr('/ip4/127.0.0.1/tcp/9100/websockets') + + peerA.multiaddr.add(maA) + swarmA = new Swarm(peerA) + swarmA.transport.add('ws', new WebSockets()) + swarmA.transport.listen('ws', {}, echo, cb) + } + + function createListenerB (cb) { + const b58IdB = 'QmRy1iU6BHmG5Hd8rnPhPL98cy1W1przUSTAMcGDq9yAAV' + const maB = multiaddr('/ip4/127.0.0.1/tcp/9200/websockets') + const peerB = new Peer(Id.createFromB58String(b58IdB)) + peerB.multiaddr.add(maB) + swarmB = new Swarm(peerB) + + swarmB.transport.add('ws', new WebSockets()) + swarmB.transport.listen('ws', {}, null, cb) + + swarmB.handle('/echo/1.0.0', echo) + } + + var count = 0 + const ready = () => ++count === 2 ? done() : null + + createListenerA(ready) + createListenerB(ready) + + function echo (conn) { + conn.pipe(conn) + } +} + +function stop (done) { + var count = 0 + const ready = () => ++count === 2 ? done() : null + + swarmA.transport.close('ws', ready) + swarmB.transport.close('ws', ready) +} + +function run (done) { + const karma = new Server({ + configFile: path.join(__dirname, '../../karma.conf.js') + }, done) + + karma.start() +} + +createListeners(() => run(() => stop(() => null))) diff --git a/tests/browser.js b/tests/browser.js deleted file mode 100644 index 5b67917063..0000000000 --- a/tests/browser.js +++ /dev/null @@ -1,133 +0,0 @@ -/* eslint-env mocha */ - -const expect = require('chai').expect -// const async = require('async') - -const multiaddr = require('multiaddr') -const PeerId = require('peer-id') -const Peer = require('peer-info') -const Swarm = require('../src') -const WebSockets = require('libp2p-websockets') -const bl = require('bl') - -const PEER_ID_SERVER_A = 'QmWg2L4Fucx1x4KXJTfKHGixBJvveubzcd7DdhB2Mqwfh1' -const PEER_ID_SERVER_B = 'QmRy1iU6BHmG5Hd8rnPhPL98cy1W1przUSTAMcGDq9yAAV' -const MULTIADDR_SERVER_A = '/ip4/127.0.0.1/tcp/9888/websockets' -const MULTIADDR_SERVER_B = '/ip4/127.0.0.1/tcp/9999/websockets' - -// random id to be used on the tests -const PEER_ID_A = 'QmYzgdesgjdvD3okTPGZT9NPmh1BuH5FfTVNKjsvaAprhb' - -describe('basics', () => { - it('throws on missing peerInfo', (done) => { - expect(Swarm).to.throw(Error) - done() - }) -}) - -describe('transport - websockets', function () { - this.timeout(10000) - - var swarm - var peerId = PeerId.createFromB58String(PEER_ID_A) - var peer = new Peer(peerId) - - before((done) => { - swarm = new Swarm(peer) - done() - }) - - it('add', (done) => { - swarm.transport.add('ws', new WebSockets(), () => { - expect(Object.keys(swarm.transports).length).to.equal(1) - done() - }) - }) - - it('dial', (done) => { - const conn = swarm.transport.dial('ws', multiaddr(MULTIADDR_SERVER_A), (err, conn) => { - expect(err).to.not.exist - }) - conn.pipe(bl((err, data) => { - expect(err).to.not.exist - expect(data.toString()).to.equal('hey') - done() - })) - conn.write('hey') - conn.end() - }) -}) - -describe('high level API - 1st without stream multiplexing (on websockets)', function () { - this.timeout(10000) - - var swarm - var peerSelf - var peerServerA - var peerServerB - - before((done) => { - const peerServerAId = PeerId.createFromB58String(PEER_ID_SERVER_A) - peerServerA = new Peer(peerServerAId) - const peerServerBId = PeerId.createFromB58String(PEER_ID_SERVER_B) - peerServerB = new Peer(peerServerBId) - const peerSelfId = PeerId.createFromB58String(PEER_ID_A) - peerSelf = new Peer(peerSelfId) - - peerServerA.multiaddr.add(multiaddr(MULTIADDR_SERVER_A)) - peerServerB.multiaddr.add(multiaddr(MULTIADDR_SERVER_B)) - - swarm = new Swarm(peerSelf) - - swarm.transport.add('ws', new WebSockets()) - expect(Object.keys(swarm.transports).length).to.equal(1) - done() - }) - - // after((done) => { - // swarm.close(done) - // }) - - it('dial using transport interface', (done) => { - const conn = swarm.transport.dial('ws', peerServerA.multiaddrs, (err, conn) => { - expect(err).to.not.exist - }) - - conn.pipe(bl((err, data) => { - expect(err).to.not.exist - expect(data.toString()).to.equal('hey') - done() - })) - conn.write('hey') - conn.end() - }) - - it('dial on protocol', (done) => { - swarm.dial(peerServerB, '/pineapple/1.0.0', (err, conn) => { - expect(err).to.not.exist - conn.pipe(bl((err, data) => { - expect(err).to.not.exist - expect(data.toString()).to.equal('yo') - done() - })) - conn.write('yo') - conn.end() - }) - }) - - it('dial to warm a conn', (done) => { - swarm.dial(peerServerB, (err) => { - expect(err).to.not.exist - done() - }) - }) - - it('dial on protocol, reuse warmed conn', (done) => { - swarm.dial(peerServerB, '/pineapple/1.0.0', (err, conn) => { - expect(err).to.not.exist - conn.end() - conn.on('data', () => {}) // let it flow.. let it flooooow - conn.on('end', done) - }) - }) -}) diff --git a/tests/karma.js b/tests/karma.js deleted file mode 100644 index b86a2f8148..0000000000 --- a/tests/karma.js +++ /dev/null @@ -1,66 +0,0 @@ -const Server = require('karma').Server -const path = require('path') - -const Peer = require('peer-info') -const PeerId = require('peer-id') -const WebSockets = require('libp2p-websockets') -const Swarm = require('../src') -const multiaddr = require('multiaddr') - -const PEER_ID_SERVER_A = 'QmWg2L4Fucx1x4KXJTfKHGixBJvveubzcd7DdhB2Mqwfh1' -const PEER_ID_SERVER_B = 'QmRy1iU6BHmG5Hd8rnPhPL98cy1W1przUSTAMcGDq9yAAV' -const MULTIADDR_SERVER_A = '/ip4/127.0.0.1/tcp/9888/websockets' -const MULTIADDR_SERVER_B = '/ip4/127.0.0.1/tcp/9999/websockets' - -var swarmA -var peerA -var swarmB -var peerB - -function createServers (done) { - function createServerA (cb) { - const id = PeerId.createFromB58String(PEER_ID_SERVER_A) - peerA = new Peer(id) - peerA.multiaddr.add(multiaddr(MULTIADDR_SERVER_A)) - swarmA = new Swarm(peerA) - swarmA.transport.add('ws', new WebSockets()) - swarmA.transport.listen('ws', {}, (conn) => { - conn.pipe(conn) - }, cb) - } - - function createServerB (cb) { - const id = PeerId.createFromB58String(PEER_ID_SERVER_B) - peerB = new Peer(id) - peerB.multiaddr.add(multiaddr(MULTIADDR_SERVER_B)) - swarmB = new Swarm(peerB) - swarmB.transport.add('ws', new WebSockets()) - swarmB.handle('/pineapple/1.0.0', (conn) => { - conn.pipe(conn) - }) - swarmB.transport.listen('ws', {}, null, cb) - } - - var count = 0 - const ready = () => ++count === 2 ? done() : null - - createServerA(ready) - createServerB(ready) -} - -function stopServers (done) { - var count = 0 - const ready = () => ++count === 2 ? done() : null - - swarmA.transport.close('ws', ready) - swarmB.transport.close('ws', ready) -} - -function runTests (done) { - new Server({ - configFile: path.join(__dirname, '/../karma.conf.js'), - singleRun: true - }, done).start() -} - -createServers(() => runTests(() => stopServers(() => null))) From 2bd536e861dfb2faeb541a92abccc1d34f3caeea Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 22 Mar 2016 17:31:58 +0000 Subject: [PATCH 109/634] fix dirty identify bug --- src/identify.js | 12 ++++++---- tests/swarm-test.js | 58 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/src/identify.js b/src/identify.js index 078d844435..35a2de206c 100644 --- a/src/identify.js +++ b/src/identify.js @@ -42,7 +42,9 @@ exports.exec = (rawConn, muxer, peerInfo, callback) => { var pbs = pbStream() pbs.on('identify', (msg) => { - peerInfo.multiaddr.addSafe(msg.observedAddr) + if (msg.observedAddr.length > 0) { + peerInfo.multiaddr.addSafe(msg.observedAddr) + } const peerId = Id.createFromPubKey(msg.publicKey) const otherPeerInfo = new Info(peerId) @@ -60,7 +62,7 @@ exports.exec = (rawConn, muxer, peerInfo, callback) => { agentVersion: 'na', publicKey: peerInfo.id.pubKey, listenAddrs: peerInfo.multiaddrs.map((mh) => { return mh.buffer }), - observedAddr: obsMultiaddr ? obsMultiaddr.buffer : null + observedAddr: obsMultiaddr ? obsMultiaddr.buffer : new Buffer('') }) pbs.pipe(ds).pipe(pbs) @@ -77,7 +79,9 @@ exports.handler = (peerInfo, swarm) => { var pbs = pbStream() pbs.on('identify', function (msg) { - peerInfo.multiaddr.addSafe(msg.observedAddr) + if (msg.observedAddr.length > 0) { + peerInfo.multiaddr.addSafe(msg.observedAddr) + } const peerId = Id.createFromPubKey(msg.publicKey) const conn = swarm.muxedConns[peerId.toB58String()].conn @@ -90,7 +94,7 @@ exports.handler = (peerInfo, swarm) => { listenAddrs: peerInfo.multiaddrs.map(function (ma) { return ma.buffer }), - observedAddr: obsMultiaddr ? obsMultiaddr.buffer : null + observedAddr: obsMultiaddr ? obsMultiaddr.buffer : new Buffer('') }) pbs.finalize() }) diff --git a/tests/swarm-test.js b/tests/swarm-test.js index 6c0c3b4242..5c86cb7e1b 100644 --- a/tests/swarm-test.js +++ b/tests/swarm-test.js @@ -529,11 +529,17 @@ describe('high level API - with everything mixed all together!', function () { var peerB var swarmC // tcp+ws var peerC + var swarmD // ws + var peerD + var swarmE // ws + var peerE before((done) => { peerA = new Peer() peerB = new Peer() peerC = new Peer() + peerD = new Peer() + peerE = new Peer() // console.log('peer A', peerA.id.toB58String()) // console.log('peer B', peerB.id.toB58String()) @@ -542,6 +548,8 @@ describe('high level API - with everything mixed all together!', function () { swarmA = new Swarm(peerA) swarmB = new Swarm(peerB) swarmC = new Swarm(peerC) + swarmD = new Swarm(peerD) + swarmE = new Swarm(peerE) done() }) @@ -552,18 +560,20 @@ describe('high level API - with everything mixed all together!', function () { swarmA.close(closed) swarmB.close(closed) swarmC.close(closed) + swarmD.close(closed) + swarmE.close(closed) function closed () { - if (++counter === 3) { + if (++counter === 4) { done() } } }) it('add tcp', (done) => { - peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9011')) - peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9021')) - peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9031')) + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) + peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) swarmA.transport.add('tcp', new TCP()) swarmA.transport.listen('tcp', {}, null, ready) @@ -586,8 +596,10 @@ describe('high level API - with everything mixed all together!', function () { it.skip('add utp', (done) => {}) it('add websockets', (done) => { - peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9022/websockets')) - peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9032/websockets')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9012/websockets')) + peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9022/websockets')) + peerD.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9032/websockets')) + peerE.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9042/websockets')) swarmB.transport.add('ws', new WebSockets()) swarmB.transport.listen('ws', {}, null, ready) @@ -595,10 +607,16 @@ describe('high level API - with everything mixed all together!', function () { swarmC.transport.add('ws', new WebSockets()) swarmC.transport.listen('ws', {}, null, ready) + swarmD.transport.add('ws', new WebSockets()) + swarmD.transport.listen('ws', {}, null, ready) + + swarmE.transport.add('ws', new WebSockets()) + swarmE.transport.listen('ws', {}, null, ready) + var counter = 0 function ready () { - if (++counter === 2) { + if (++counter === 4) { done() } } @@ -608,9 +626,15 @@ describe('high level API - with everything mixed all together!', function () { swarmA.connection.addStreamMuxer(spdy) swarmB.connection.addStreamMuxer(spdy) swarmC.connection.addStreamMuxer(spdy) + swarmD.connection.addStreamMuxer(spdy) + swarmE.connection.addStreamMuxer(spdy) + swarmA.connection.reuse() swarmB.connection.reuse() swarmC.connection.reuse() + swarmD.connection.reuse() + swarmE.connection.reuse() + done() }) @@ -631,6 +655,26 @@ describe('high level API - with everything mixed all together!', function () { }) }) + it('dial from ws to ws', (done) => { + swarmE.handle('/abacaxi/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmD.dial(peerE, '/abacaxi/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmD.muxedConns).length).to.equal(1) + + conn.end() + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', () => { + setTimeout(() => { + expect(Object.keys(swarmE.muxedConns).length).to.equal(1) + done() + }, 1000) + }) + }) + }) + it('dial from tcp to tcp+ws (returned conn)', (done) => { swarmB.handle('/grapes/1.0.0', (conn) => { conn.pipe(conn) From 28bd4ab1879b901ceebb52ba1d3e4d72962da9f0 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Mar 2016 07:01:54 +0000 Subject: [PATCH 110/634] Release v0.9.1. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 47ff5d13b7..2747e0ad3e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.9.0", + "version": "0.9.1", "description": "libp2p swarm implementation in JavaScript", "main": "src/index.js", "scripts": { From 5bfbd8d97292c849fe1759c00105c294ac7258fd Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Mar 2016 08:14:06 +0000 Subject: [PATCH 111/634] restructure test files --- README.md | 2 +- package.json | 1 + tests/00-basic-test.js | 12 + tests/01-transport-tcp-test.js | 165 ++++++ tests/02-transport-utp-test.js | 12 + tests/03-transport-websockets-test.js | 92 ++++ tests/04-muxing-multiplex-test.js | 14 + tests/05-muxing-spdy-test.js | 130 +++++ tests/06-conn-upgrade-secio-test.js | 12 + tests/07-conn-upgrade-tls-test.js | 10 + tests/08-swarm-without-muxing-test.js | 105 ++++ tests/09-swarm-with-muxing-test.js | 196 +++++++ tests/swarm-test.js | 707 -------------------------- 13 files changed, 750 insertions(+), 708 deletions(-) create mode 100644 tests/00-basic-test.js create mode 100644 tests/01-transport-tcp-test.js create mode 100644 tests/02-transport-utp-test.js create mode 100644 tests/03-transport-websockets-test.js create mode 100644 tests/04-muxing-multiplex-test.js create mode 100644 tests/05-muxing-spdy-test.js create mode 100644 tests/06-conn-upgrade-secio-test.js create mode 100644 tests/07-conn-upgrade-tls-test.js create mode 100644 tests/08-swarm-without-muxing-test.js create mode 100644 tests/09-swarm-with-muxing-test.js delete mode 100644 tests/swarm-test.js diff --git a/README.md b/README.md index 25113352b9..0b6ede9930 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ libp2p-swarm JavaScript implementation [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) [![Build Status](https://img.shields.io/travis/diasdavid/js-libp2p-swarm/master.svg?style=flat-square)](https://travis-ci.org/diasdavid/js-libp2p-swarm) ![](https://img.shields.io/badge/coverage-%3F-yellow.svg?style=flat-square) -[![Dependency Status](https://david-dm.org/ipfs/js-libp2p-swarm.svg?style=flat-square)](https://david-dm.org/ipfs/js-libp2p-swarm) +[![Dependency Status](https://david-dm.org/diasdavid/js-libp2p-swarm.svg?style=flat-square)](https://david-dm.org/ipfs/js-libp2p-swarm) [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/feross/standard) > libp2p swarm implementation in JavaScript diff --git a/package.json b/package.json index 2747e0ad3e..f7cfbf2516 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "karma-mocha": "^0.2.2", "karma-spec-reporter": "0.0.24", "karma-webpack": "^1.7.0", + "libp2p-multiplex": "^0.2.1", "libp2p-spdy": "^0.2.3", "libp2p-tcp": "^0.4.0", "libp2p-websockets": "^0.2.1", diff --git a/tests/00-basic-test.js b/tests/00-basic-test.js new file mode 100644 index 0000000000..ef5d96ea22 --- /dev/null +++ b/tests/00-basic-test.js @@ -0,0 +1,12 @@ +/* eslint-env mocha */ + +const expect = require('chai').expect + +const Swarm = require('../src') + +describe('basics', () => { + it('throws on missing peerInfo', (done) => { + expect(Swarm).to.throw(Error) + done() + }) +}) diff --git a/tests/01-transport-tcp-test.js b/tests/01-transport-tcp-test.js new file mode 100644 index 0000000000..641f6323c5 --- /dev/null +++ b/tests/01-transport-tcp-test.js @@ -0,0 +1,165 @@ +/* eslint-env mocha */ + +const expect = require('chai').expect + +const multiaddr = require('multiaddr') +const Peer = require('peer-info') +const Swarm = require('../src') +const TCP = require('libp2p-tcp') +const bl = require('bl') + +describe('transport - tcp', function () { + this.timeout(10000) + + var swarmA + var swarmB + var peerA = new Peer() + var peerB = new Peer() + + before((done) => { + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9888')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9999')) + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + done() + }) + + it('add', (done) => { + swarmA.transport.add('tcp', new TCP()) + expect(Object.keys(swarmA.transports).length).to.equal(1) + swarmB.transport.add('tcp', new TCP(), () => { + expect(Object.keys(swarmB.transports).length).to.equal(1) + done() + }) + }) + + it('listen', (done) => { + var count = 0 + swarmA.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + swarmB.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + if (++count === 2) { + expect(peerA.multiaddrs.length).to.equal(1) + expect(peerA.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9888')) + expect(peerB.multiaddrs.length).to.equal(1) + expect(peerB.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9999')) + done() + } + } + }) + + it('dial to a multiaddr', (done) => { + const conn = swarmA.transport.dial('tcp', multiaddr('/ip4/127.0.0.1/tcp/9999'), (err, conn) => { + expect(err).to.not.exist + }) + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + done() + })) + conn.write('hey') + conn.end() + }) + + it('dial to set of multiaddr, only one is available', (done) => { + const conn = swarmA.transport.dial('tcp', [ + multiaddr('/ip4/127.0.0.1/tcp/9910/websockets'), // not valid on purpose + multiaddr('/ip4/127.0.0.1/tcp/9910'), + multiaddr('/ip4/127.0.0.1/tcp/9999'), + multiaddr('/ip4/127.0.0.1/tcp/9309') + ], (err, conn) => { + expect(err).to.not.exist + }) + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + done() + })) + conn.write('hey') + conn.end() + }) + + it('close', (done) => { + var count = 0 + swarmA.transport.close('tcp', closed) + swarmB.transport.close('tcp', closed) + + function closed () { + if (++count === 2) { + done() + } + } + }) + + it('support port 0', (done) => { + var swarm + var peer = new Peer() + peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) + swarm = new Swarm(peer) + swarm.transport.add('tcp', new TCP()) + swarm.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + expect(peer.multiaddrs.length).to.equal(1) + expect(peer.multiaddrs[0]).to.not.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/0')) + swarm.close(done) + } + }) + + it('support addr /ip4/0.0.0.0/tcp/9050', (done) => { + var swarm + var peer = new Peer() + peer.multiaddr.add(multiaddr('/ip4/0.0.0.0/tcp/9050')) + swarm = new Swarm(peer) + swarm.transport.add('tcp', new TCP()) + swarm.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + expect(peer.multiaddrs.length).to.equal(1) + expect(peer.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/0.0.0.0/tcp/9050')) + swarm.close(done) + } + }) + + it('support addr /ip4/0.0.0.0/tcp/0', (done) => { + var swarm + var peer = new Peer() + peer.multiaddr.add(multiaddr('/ip4/0.0.0.0/tcp/0')) + swarm = new Swarm(peer) + swarm.transport.add('tcp', new TCP()) + swarm.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + expect(peer.multiaddrs.length).to.equal(1) + expect(peer.multiaddrs[0]).to.not.deep.equal(multiaddr('/ip4/0.0.0.0/tcp/0')) + swarm.close(done) + } + }) + + it('listen in several addrs', (done) => { + var swarm + var peer = new Peer() + peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) + peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) + peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9003')) + swarm = new Swarm(peer) + swarm.transport.add('tcp', new TCP()) + swarm.transport.listen('tcp', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + expect(peer.multiaddrs.length).to.equal(3) + swarm.close(done) + } + }) +}) diff --git a/tests/02-transport-utp-test.js b/tests/02-transport-utp-test.js new file mode 100644 index 0000000000..7b3714dbe5 --- /dev/null +++ b/tests/02-transport-utp-test.js @@ -0,0 +1,12 @@ +/* eslint-env mocha */ + +describe('transport - utp', function () { + this.timeout(10000) + + before((done) => { done() }) + + it.skip('add', (done) => {}) + it.skip('listen', (done) => {}) + it.skip('dial', (done) => {}) + it.skip('close', (done) => {}) +}) diff --git a/tests/03-transport-websockets-test.js b/tests/03-transport-websockets-test.js new file mode 100644 index 0000000000..2a6246fe19 --- /dev/null +++ b/tests/03-transport-websockets-test.js @@ -0,0 +1,92 @@ +/* eslint-env mocha */ + +const expect = require('chai').expect + +const multiaddr = require('multiaddr') +const Peer = require('peer-info') +const Swarm = require('../src') +const WebSockets = require('libp2p-websockets') +const bl = require('bl') + +describe('transport - websockets', function () { + this.timeout(10000) + + var swarmA + var swarmB + var peerA = new Peer() + var peerB = new Peer() + + before((done) => { + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9888/websockets')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9999/websockets')) + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + done() + }) + + it('add', (done) => { + swarmA.transport.add('ws', new WebSockets()) + expect(Object.keys(swarmA.transports).length).to.equal(1) + swarmB.transport.add('ws', new WebSockets(), () => { + expect(Object.keys(swarmB.transports).length).to.equal(1) + done() + }) + }) + + it('listen', (done) => { + var count = 0 + swarmA.transport.listen('ws', {}, (conn) => { + conn.pipe(conn) + }, ready) + swarmB.transport.listen('ws', {}, (conn) => { + conn.pipe(conn) + }, ready) + + function ready () { + if (++count === 2) { + expect(peerA.multiaddrs.length).to.equal(1) + expect(peerA.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9888/websockets')) + expect(peerB.multiaddrs.length).to.equal(1) + expect(peerB.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9999/websockets')) + done() + } + } + }) + + it('dial', (done) => { + const conn = swarmA.transport.dial('ws', multiaddr('/ip4/127.0.0.1/tcp/9999/websockets'), (err, conn) => { + expect(err).to.not.exist + }) + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + done() + })) + conn.write('hey') + conn.end() + }) + + it('dial (conn from callback)', (done) => { + swarmA.transport.dial('ws', multiaddr('/ip4/127.0.0.1/tcp/9999/websockets'), (err, conn) => { + expect(err).to.not.exist + + conn.pipe(bl((err, data) => { + expect(err).to.not.exist + done() + })) + conn.write('hey') + conn.end() + }) + }) + + it('close', (done) => { + var count = 0 + swarmA.transport.close('ws', closed) + swarmB.transport.close('ws', closed) + + function closed () { + if (++count === 2) { + done() + } + } + }) +}) diff --git a/tests/04-muxing-multiplex-test.js b/tests/04-muxing-multiplex-test.js new file mode 100644 index 0000000000..acc962d588 --- /dev/null +++ b/tests/04-muxing-multiplex-test.js @@ -0,0 +1,14 @@ +/* eslint-env mocha */ + +describe('stream muxing with multiplex (on TCP)', function () { + this.timeout(20000) + + before((done) => { done() }) + after((done) => { done() }) + + it.skip('add', (done) => {}) + it.skip('handle + dial on protocol', (done) => {}) + it.skip('dial to warm conn', (done) => {}) + it.skip('dial on protocol, reuse warmed conn', (done) => {}) + it.skip('enable identify to reuse incomming muxed conn', (done) => {}) +}) diff --git a/tests/05-muxing-spdy-test.js b/tests/05-muxing-spdy-test.js new file mode 100644 index 0000000000..859616728b --- /dev/null +++ b/tests/05-muxing-spdy-test.js @@ -0,0 +1,130 @@ +/* eslint-env mocha */ + +const expect = require('chai').expect + +const multiaddr = require('multiaddr') +const Peer = require('peer-info') +const Swarm = require('../src') +const TCP = require('libp2p-tcp') +const spdy = require('libp2p-spdy') + +describe('stream muxing with spdy (on TCP)', function () { + this.timeout(20000) + + var swarmA + var peerA + var swarmB + var peerB + var swarmC + var peerC + + before((done) => { + peerA = new Peer() + peerB = new Peer() + peerC = new Peer() + + // console.log('peer A', peerA.id.toB58String()) + // console.log('peer B', peerB.id.toB58String()) + // console.log('peer C', peerC.id.toB58String()) + + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) + peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9003')) + + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + swarmC = new Swarm(peerC) + + swarmA.transport.add('tcp', new TCP()) + swarmA.transport.listen('tcp', {}, null, ready) + + swarmB.transport.add('tcp', new TCP()) + swarmB.transport.listen('tcp', {}, null, ready) + + swarmC.transport.add('tcp', new TCP()) + swarmC.transport.listen('tcp', {}, null, ready) + + var counter = 0 + + function ready () { + if (++counter === 3) { + done() + } + } + }) + + after((done) => { + var counter = 0 + + swarmA.close(closed) + swarmB.close(closed) + swarmC.close(closed) + + function closed () { + if (++counter === 3) { + done() + } + } + }) + + it('add', (done) => { + swarmA.connection.addStreamMuxer(spdy) + swarmB.connection.addStreamMuxer(spdy) + swarmC.connection.addStreamMuxer(spdy) + done() + }) + + it('handle + dial on protocol', (done) => { + swarmB.handle('/abacaxi/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmA.dial(peerB, '/abacaxi/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmA.muxedConns).length).to.equal(1) + conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) + + it('dial to warm conn', (done) => { + swarmB.dial(peerA, (err) => { + expect(err).to.not.exist + expect(Object.keys(swarmB.conns).length).to.equal(0) + expect(Object.keys(swarmB.muxedConns).length).to.equal(1) + done() + }) + }) + + it('dial on protocol, reuse warmed conn', (done) => { + swarmA.handle('/papaia/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmB.dial(peerA, '/papaia/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmB.conns).length).to.equal(0) + expect(Object.keys(swarmB.muxedConns).length).to.equal(1) + conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) + + it('enable identify to reuse incomming muxed conn', (done) => { + swarmA.connection.reuse() + swarmC.connection.reuse() + + swarmC.dial(peerA, (err) => { + expect(err).to.not.exist + setTimeout(() => { + expect(Object.keys(swarmC.muxedConns).length).to.equal(1) + expect(Object.keys(swarmA.muxedConns).length).to.equal(2) + done() + }, 500) + }) + }) +}) diff --git a/tests/06-conn-upgrade-secio-test.js b/tests/06-conn-upgrade-secio-test.js new file mode 100644 index 0000000000..18e4887fd7 --- /dev/null +++ b/tests/06-conn-upgrade-secio-test.js @@ -0,0 +1,12 @@ +/* eslint-env mocha */ + +describe('secio conn upgrade (on TCP)', function () { + this.timeout(20000) + + before((done) => { done() }) + after((done) => { done() }) + + it.skip('add', (done) => {}) + it.skip('dial', (done) => {}) + it.skip('tls on a muxed stream (not the full conn)', (done) => {}) +}) diff --git a/tests/07-conn-upgrade-tls-test.js b/tests/07-conn-upgrade-tls-test.js new file mode 100644 index 0000000000..2fe9c4e815 --- /dev/null +++ b/tests/07-conn-upgrade-tls-test.js @@ -0,0 +1,10 @@ +/* eslint-env mocha */ + +describe('tls conn upgrade (on TCP)', function () { + before((done) => { done() }) + after((done) => { done() }) + + it.skip('add', (done) => {}) + it.skip('dial', (done) => {}) + it.skip('tls on a muxed stream (not the full conn)', (done) => {}) +}) diff --git a/tests/08-swarm-without-muxing-test.js b/tests/08-swarm-without-muxing-test.js new file mode 100644 index 0000000000..c41813fba8 --- /dev/null +++ b/tests/08-swarm-without-muxing-test.js @@ -0,0 +1,105 @@ +/* eslint-env mocha */ + +const expect = require('chai').expect + +const multiaddr = require('multiaddr') +const Peer = require('peer-info') +const Swarm = require('../src') +const TCP = require('libp2p-tcp') + +describe('high level API - 1st without stream multiplexing (on TCP)', function () { + this.timeout(20000) + + var swarmA + var peerA + var swarmB + var peerB + + before((done) => { + peerA = new Peer() + peerB = new Peer() + + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) + + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + + swarmA.transport.add('tcp', new TCP()) + swarmA.transport.listen('tcp', {}, null, ready) + + swarmB.transport.add('tcp', new TCP()) + swarmB.transport.listen('tcp', {}, null, ready) + + var counter = 0 + + function ready () { + if (++counter === 2) { + done() + } + } + }) + + after((done) => { + var counter = 0 + + swarmA.close(closed) + swarmB.close(closed) + + function closed () { + if (++counter === 2) { + done() + } + } + }) + + it('handle a protocol', (done) => { + swarmB.handle('/bananas/1.0.0', (conn) => { + conn.pipe(conn) + }) + expect(Object.keys(swarmB.protocols).length).to.equal(1) + done() + }) + + it('dial on protocol', (done) => { + swarmB.handle('/pineapple/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmA.dial(peerB, '/pineapple/1.0.0', (err, conn) => { + expect(err).to.not.exist + conn.end() + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) + + it('dial on protocol (returned conn)', (done) => { + swarmB.handle('/apples/1.0.0', (conn) => { + conn.pipe(conn) + }) + + const conn = swarmA.dial(peerB, '/apples/1.0.0', (err) => { + expect(err).to.not.exist + }) + conn.end() + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + + it('dial to warm a conn', (done) => { + swarmA.dial(peerB, (err) => { + expect(err).to.not.exist + done() + }) + }) + + it('dial on protocol, reuse warmed conn', (done) => { + swarmA.dial(peerB, '/bananas/1.0.0', (err, conn) => { + expect(err).to.not.exist + conn.end() + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) +}) diff --git a/tests/09-swarm-with-muxing-test.js b/tests/09-swarm-with-muxing-test.js new file mode 100644 index 0000000000..998a6ee258 --- /dev/null +++ b/tests/09-swarm-with-muxing-test.js @@ -0,0 +1,196 @@ +/* eslint-env mocha */ + +const expect = require('chai').expect + +const multiaddr = require('multiaddr') +const Peer = require('peer-info') +const Swarm = require('../src') +const TCP = require('libp2p-tcp') +const WebSockets = require('libp2p-websockets') +const spdy = require('libp2p-spdy') + +describe('high level API - with everything mixed all together!', function () { + this.timeout(20000) + + var swarmA // tcp + var peerA + var swarmB // tcp+ws + var peerB + var swarmC // tcp+ws + var peerC + var swarmD // ws + var peerD + var swarmE // ws + var peerE + + before((done) => { + peerA = new Peer() + peerB = new Peer() + peerC = new Peer() + peerD = new Peer() + peerE = new Peer() + + // console.log('peer A', peerA.id.toB58String()) + // console.log('peer B', peerB.id.toB58String()) + // console.log('peer C', peerC.id.toB58String()) + + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + swarmC = new Swarm(peerC) + swarmD = new Swarm(peerD) + swarmE = new Swarm(peerE) + + done() + }) + + after((done) => { + var counter = 0 + + swarmA.close(closed) + swarmB.close(closed) + swarmC.close(closed) + swarmD.close(closed) + swarmE.close(closed) + + function closed () { + if (++counter === 4) { + done() + } + } + }) + + it('add tcp', (done) => { + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) + peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) + + swarmA.transport.add('tcp', new TCP()) + swarmA.transport.listen('tcp', {}, null, ready) + + swarmB.transport.add('tcp', new TCP()) + swarmB.transport.listen('tcp', {}, null, ready) + + swarmC.transport.add('tcp', new TCP()) + swarmC.transport.listen('tcp', {}, null, ready) + + var counter = 0 + + function ready () { + if (++counter === 3) { + done() + } + } + }) + + it.skip('add utp', (done) => {}) + + it('add websockets', (done) => { + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9012/websockets')) + peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9022/websockets')) + peerD.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9032/websockets')) + peerE.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9042/websockets')) + + swarmB.transport.add('ws', new WebSockets()) + swarmB.transport.listen('ws', {}, null, ready) + + swarmC.transport.add('ws', new WebSockets()) + swarmC.transport.listen('ws', {}, null, ready) + + swarmD.transport.add('ws', new WebSockets()) + swarmD.transport.listen('ws', {}, null, ready) + + swarmE.transport.add('ws', new WebSockets()) + swarmE.transport.listen('ws', {}, null, ready) + + var counter = 0 + + function ready () { + if (++counter === 4) { + done() + } + } + }) + + it('add spdy', (done) => { + swarmA.connection.addStreamMuxer(spdy) + swarmB.connection.addStreamMuxer(spdy) + swarmC.connection.addStreamMuxer(spdy) + swarmD.connection.addStreamMuxer(spdy) + swarmE.connection.addStreamMuxer(spdy) + + swarmA.connection.reuse() + swarmB.connection.reuse() + swarmC.connection.reuse() + swarmD.connection.reuse() + swarmE.connection.reuse() + + done() + }) + + it.skip('add multiplex', (done) => {}) + + it('dial from tcp to tcp+ws', (done) => { + swarmB.handle('/anona/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmA.dial(peerB, '/anona/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmA.muxedConns).length).to.equal(1) + conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) + + it('dial from ws to ws', (done) => { + swarmE.handle('/abacaxi/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmD.dial(peerE, '/abacaxi/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmD.muxedConns).length).to.equal(1) + + conn.end() + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', () => { + setTimeout(() => { + expect(Object.keys(swarmE.muxedConns).length).to.equal(1) + done() + }, 1000) + }) + }) + }) + + it('dial from tcp to tcp+ws (returned conn)', (done) => { + swarmB.handle('/grapes/1.0.0', (conn) => { + conn.pipe(conn) + }) + + const conn = swarmA.dial(peerB, '/grapes/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmA.muxedConns).length).to.equal(1) + }) + conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + + it('dial from tcp+ws to tcp+ws', (done) => { + swarmC.handle('/mamao/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmA.dial(peerC, '/mamao/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmA.muxedConns).length).to.equal(2) + conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) +}) diff --git a/tests/swarm-test.js b/tests/swarm-test.js deleted file mode 100644 index 5c86cb7e1b..0000000000 --- a/tests/swarm-test.js +++ /dev/null @@ -1,707 +0,0 @@ -/* eslint-env mocha */ - -const expect = require('chai').expect -// const async = require('async') - -const multiaddr = require('multiaddr') -// const Id = require('peer-id') -const Peer = require('peer-info') -const Swarm = require('../src') -const TCP = require('libp2p-tcp') -const WebSockets = require('libp2p-websockets') -const bl = require('bl') -const spdy = require('libp2p-spdy') - -describe('basics', () => { - it('throws on missing peerInfo', (done) => { - expect(Swarm).to.throw(Error) - done() - }) -}) - -describe('transport - tcp', function () { - this.timeout(10000) - - var swarmA - var swarmB - var peerA = new Peer() - var peerB = new Peer() - - before((done) => { - peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9888')) - peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9999')) - swarmA = new Swarm(peerA) - swarmB = new Swarm(peerB) - done() - }) - - it('add', (done) => { - swarmA.transport.add('tcp', new TCP()) - expect(Object.keys(swarmA.transports).length).to.equal(1) - swarmB.transport.add('tcp', new TCP(), () => { - expect(Object.keys(swarmB.transports).length).to.equal(1) - done() - }) - }) - - it('listen', (done) => { - var count = 0 - swarmA.transport.listen('tcp', {}, (conn) => { - conn.pipe(conn) - }, ready) - swarmB.transport.listen('tcp', {}, (conn) => { - conn.pipe(conn) - }, ready) - - function ready () { - if (++count === 2) { - expect(peerA.multiaddrs.length).to.equal(1) - expect(peerA.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9888')) - expect(peerB.multiaddrs.length).to.equal(1) - expect(peerB.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9999')) - done() - } - } - }) - - it('dial to a multiaddr', (done) => { - const conn = swarmA.transport.dial('tcp', multiaddr('/ip4/127.0.0.1/tcp/9999'), (err, conn) => { - expect(err).to.not.exist - }) - conn.pipe(bl((err, data) => { - expect(err).to.not.exist - done() - })) - conn.write('hey') - conn.end() - }) - - it('dial to set of multiaddr, only one is available', (done) => { - const conn = swarmA.transport.dial('tcp', [ - multiaddr('/ip4/127.0.0.1/tcp/9910/websockets'), // not valid on purpose - multiaddr('/ip4/127.0.0.1/tcp/9910'), - multiaddr('/ip4/127.0.0.1/tcp/9999'), - multiaddr('/ip4/127.0.0.1/tcp/9309') - ], (err, conn) => { - expect(err).to.not.exist - }) - conn.pipe(bl((err, data) => { - expect(err).to.not.exist - done() - })) - conn.write('hey') - conn.end() - }) - - it('close', (done) => { - var count = 0 - swarmA.transport.close('tcp', closed) - swarmB.transport.close('tcp', closed) - - function closed () { - if (++count === 2) { - done() - } - } - }) - - it('support port 0', (done) => { - var swarm - var peer = new Peer() - peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) - swarm = new Swarm(peer) - swarm.transport.add('tcp', new TCP()) - swarm.transport.listen('tcp', {}, (conn) => { - conn.pipe(conn) - }, ready) - - function ready () { - expect(peer.multiaddrs.length).to.equal(1) - expect(peer.multiaddrs[0]).to.not.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/0')) - swarm.close(done) - } - }) - - it('support addr /ip4/0.0.0.0/tcp/9050', (done) => { - var swarm - var peer = new Peer() - peer.multiaddr.add(multiaddr('/ip4/0.0.0.0/tcp/9050')) - swarm = new Swarm(peer) - swarm.transport.add('tcp', new TCP()) - swarm.transport.listen('tcp', {}, (conn) => { - conn.pipe(conn) - }, ready) - - function ready () { - expect(peer.multiaddrs.length).to.equal(1) - expect(peer.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/0.0.0.0/tcp/9050')) - swarm.close(done) - } - }) - - it('support addr /ip4/0.0.0.0/tcp/0', (done) => { - var swarm - var peer = new Peer() - peer.multiaddr.add(multiaddr('/ip4/0.0.0.0/tcp/0')) - swarm = new Swarm(peer) - swarm.transport.add('tcp', new TCP()) - swarm.transport.listen('tcp', {}, (conn) => { - conn.pipe(conn) - }, ready) - - function ready () { - expect(peer.multiaddrs.length).to.equal(1) - expect(peer.multiaddrs[0]).to.not.deep.equal(multiaddr('/ip4/0.0.0.0/tcp/0')) - swarm.close(done) - } - }) - - it('listen in several addrs', (done) => { - var swarm - var peer = new Peer() - peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) - peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) - peer.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9003')) - swarm = new Swarm(peer) - swarm.transport.add('tcp', new TCP()) - swarm.transport.listen('tcp', {}, (conn) => { - conn.pipe(conn) - }, ready) - - function ready () { - expect(peer.multiaddrs.length).to.equal(3) - swarm.close(done) - } - }) -}) - -describe('transport - websockets', function () { - this.timeout(10000) - - var swarmA - var swarmB - var peerA = new Peer() - var peerB = new Peer() - - before((done) => { - peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9888/websockets')) - peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9999/websockets')) - swarmA = new Swarm(peerA) - swarmB = new Swarm(peerB) - done() - }) - - it('add', (done) => { - swarmA.transport.add('ws', new WebSockets()) - expect(Object.keys(swarmA.transports).length).to.equal(1) - swarmB.transport.add('ws', new WebSockets(), () => { - expect(Object.keys(swarmB.transports).length).to.equal(1) - done() - }) - }) - - it('listen', (done) => { - var count = 0 - swarmA.transport.listen('ws', {}, (conn) => { - conn.pipe(conn) - }, ready) - swarmB.transport.listen('ws', {}, (conn) => { - conn.pipe(conn) - }, ready) - - function ready () { - if (++count === 2) { - expect(peerA.multiaddrs.length).to.equal(1) - expect(peerA.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9888/websockets')) - expect(peerB.multiaddrs.length).to.equal(1) - expect(peerB.multiaddrs[0]).to.deep.equal(multiaddr('/ip4/127.0.0.1/tcp/9999/websockets')) - done() - } - } - }) - - it('dial', (done) => { - const conn = swarmA.transport.dial('ws', multiaddr('/ip4/127.0.0.1/tcp/9999/websockets'), (err, conn) => { - expect(err).to.not.exist - }) - conn.pipe(bl((err, data) => { - expect(err).to.not.exist - done() - })) - conn.write('hey') - conn.end() - }) - - it('dial (conn from callback)', (done) => { - swarmA.transport.dial('ws', multiaddr('/ip4/127.0.0.1/tcp/9999/websockets'), (err, conn) => { - expect(err).to.not.exist - - conn.pipe(bl((err, data) => { - expect(err).to.not.exist - done() - })) - conn.write('hey') - conn.end() - }) - }) - - it('close', (done) => { - var count = 0 - swarmA.transport.close('ws', closed) - swarmB.transport.close('ws', closed) - - function closed () { - if (++count === 2) { - done() - } - } - }) -}) - -describe('transport - utp', function () { - this.timeout(10000) - - before((done) => { done() }) - - it.skip('add', (done) => {}) - it.skip('listen', (done) => {}) - it.skip('dial', (done) => {}) - it.skip('close', (done) => {}) -}) - -describe('high level API - 1st without stream multiplexing (on TCP)', function () { - this.timeout(20000) - - var swarmA - var peerA - var swarmB - var peerB - - before((done) => { - peerA = new Peer() - peerB = new Peer() - - peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) - peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) - - swarmA = new Swarm(peerA) - swarmB = new Swarm(peerB) - - swarmA.transport.add('tcp', new TCP()) - swarmA.transport.listen('tcp', {}, null, ready) - - swarmB.transport.add('tcp', new TCP()) - swarmB.transport.listen('tcp', {}, null, ready) - - var counter = 0 - - function ready () { - if (++counter === 2) { - done() - } - } - }) - - after((done) => { - var counter = 0 - - swarmA.close(closed) - swarmB.close(closed) - - function closed () { - if (++counter === 2) { - done() - } - } - }) - - it('handle a protocol', (done) => { - swarmB.handle('/bananas/1.0.0', (conn) => { - conn.pipe(conn) - }) - expect(Object.keys(swarmB.protocols).length).to.equal(1) - done() - }) - - it('dial on protocol', (done) => { - swarmB.handle('/pineapple/1.0.0', (conn) => { - conn.pipe(conn) - }) - - swarmA.dial(peerB, '/pineapple/1.0.0', (err, conn) => { - expect(err).to.not.exist - conn.end() - conn.on('data', () => {}) // let it flow.. let it flooooow - conn.on('end', done) - }) - }) - - it('dial on protocol (returned conn)', (done) => { - swarmB.handle('/apples/1.0.0', (conn) => { - conn.pipe(conn) - }) - - const conn = swarmA.dial(peerB, '/apples/1.0.0', (err) => { - expect(err).to.not.exist - }) - conn.end() - conn.on('data', () => {}) // let it flow.. let it flooooow - conn.on('end', done) - }) - - it('dial to warm a conn', (done) => { - swarmA.dial(peerB, (err) => { - expect(err).to.not.exist - done() - }) - }) - - it('dial on protocol, reuse warmed conn', (done) => { - swarmA.dial(peerB, '/bananas/1.0.0', (err, conn) => { - expect(err).to.not.exist - conn.end() - conn.on('data', () => {}) // let it flow.. let it flooooow - conn.on('end', done) - }) - }) -}) - -describe('stream muxing (on TCP)', function () { - this.timeout(20000) - - describe('multiplex', () => { - before((done) => { done() }) - after((done) => { done() }) - - it.skip('add', (done) => {}) - it.skip('handle + dial on protocol', (done) => {}) - it.skip('dial to warm conn', (done) => {}) - it.skip('dial on protocol, reuse warmed conn', (done) => {}) - it.skip('enable identify to reuse incomming muxed conn', (done) => {}) - }) - describe('spdy', () => { - var swarmA - var peerA - var swarmB - var peerB - var swarmC - var peerC - - before((done) => { - peerA = new Peer() - peerB = new Peer() - peerC = new Peer() - - // console.log('peer A', peerA.id.toB58String()) - // console.log('peer B', peerB.id.toB58String()) - // console.log('peer C', peerC.id.toB58String()) - - peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) - peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) - peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9003')) - - swarmA = new Swarm(peerA) - swarmB = new Swarm(peerB) - swarmC = new Swarm(peerC) - - swarmA.transport.add('tcp', new TCP()) - swarmA.transport.listen('tcp', {}, null, ready) - - swarmB.transport.add('tcp', new TCP()) - swarmB.transport.listen('tcp', {}, null, ready) - - swarmC.transport.add('tcp', new TCP()) - swarmC.transport.listen('tcp', {}, null, ready) - - var counter = 0 - - function ready () { - if (++counter === 3) { - done() - } - } - }) - - after((done) => { - var counter = 0 - - swarmA.close(closed) - swarmB.close(closed) - swarmC.close(closed) - - function closed () { - if (++counter === 3) { - done() - } - } - }) - - it('add', (done) => { - swarmA.connection.addStreamMuxer(spdy) - swarmB.connection.addStreamMuxer(spdy) - swarmC.connection.addStreamMuxer(spdy) - done() - }) - - it('handle + dial on protocol', (done) => { - swarmB.handle('/abacaxi/1.0.0', (conn) => { - conn.pipe(conn) - }) - - swarmA.dial(peerB, '/abacaxi/1.0.0', (err, conn) => { - expect(err).to.not.exist - expect(Object.keys(swarmA.muxedConns).length).to.equal(1) - conn.end() - - conn.on('data', () => {}) // let it flow.. let it flooooow - conn.on('end', done) - }) - }) - - it('dial to warm conn', (done) => { - swarmB.dial(peerA, (err) => { - expect(err).to.not.exist - expect(Object.keys(swarmB.conns).length).to.equal(0) - expect(Object.keys(swarmB.muxedConns).length).to.equal(1) - done() - }) - }) - - it('dial on protocol, reuse warmed conn', (done) => { - swarmA.handle('/papaia/1.0.0', (conn) => { - conn.pipe(conn) - }) - - swarmB.dial(peerA, '/papaia/1.0.0', (err, conn) => { - expect(err).to.not.exist - expect(Object.keys(swarmB.conns).length).to.equal(0) - expect(Object.keys(swarmB.muxedConns).length).to.equal(1) - conn.end() - - conn.on('data', () => {}) // let it flow.. let it flooooow - conn.on('end', done) - }) - }) - - it('enable identify to reuse incomming muxed conn', (done) => { - swarmA.connection.reuse() - swarmC.connection.reuse() - - swarmC.dial(peerA, (err) => { - expect(err).to.not.exist - setTimeout(() => { - expect(Object.keys(swarmC.muxedConns).length).to.equal(1) - expect(Object.keys(swarmA.muxedConns).length).to.equal(2) - done() - }, 500) - }) - }) - }) -}) - -describe('conn upgrades', function () { - this.timeout(20000) - - describe('secio on tcp', () => { - before((done) => { done() }) - after((done) => { done() }) - - it.skip('add', (done) => {}) - it.skip('dial', (done) => {}) - it.skip('tls on a muxed stream (not the full conn)', (done) => {}) - }) - describe('tls on tcp', () => { - before((done) => { done() }) - after((done) => { done() }) - - it.skip('add', (done) => {}) - it.skip('dial', (done) => {}) - it.skip('tls on a muxed stream (not the full conn)', (done) => {}) - }) -}) - -describe('high level API - with everything mixed all together!', function () { - this.timeout(20000) - - var swarmA // tcp - var peerA - var swarmB // tcp+ws - var peerB - var swarmC // tcp+ws - var peerC - var swarmD // ws - var peerD - var swarmE // ws - var peerE - - before((done) => { - peerA = new Peer() - peerB = new Peer() - peerC = new Peer() - peerD = new Peer() - peerE = new Peer() - - // console.log('peer A', peerA.id.toB58String()) - // console.log('peer B', peerB.id.toB58String()) - // console.log('peer C', peerC.id.toB58String()) - - swarmA = new Swarm(peerA) - swarmB = new Swarm(peerB) - swarmC = new Swarm(peerC) - swarmD = new Swarm(peerD) - swarmE = new Swarm(peerE) - - done() - }) - - after((done) => { - var counter = 0 - - swarmA.close(closed) - swarmB.close(closed) - swarmC.close(closed) - swarmD.close(closed) - swarmE.close(closed) - - function closed () { - if (++counter === 4) { - done() - } - } - }) - - it('add tcp', (done) => { - peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) - peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) - peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/0')) - - swarmA.transport.add('tcp', new TCP()) - swarmA.transport.listen('tcp', {}, null, ready) - - swarmB.transport.add('tcp', new TCP()) - swarmB.transport.listen('tcp', {}, null, ready) - - swarmC.transport.add('tcp', new TCP()) - swarmC.transport.listen('tcp', {}, null, ready) - - var counter = 0 - - function ready () { - if (++counter === 3) { - done() - } - } - }) - - it.skip('add utp', (done) => {}) - - it('add websockets', (done) => { - peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9012/websockets')) - peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9022/websockets')) - peerD.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9032/websockets')) - peerE.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9042/websockets')) - - swarmB.transport.add('ws', new WebSockets()) - swarmB.transport.listen('ws', {}, null, ready) - - swarmC.transport.add('ws', new WebSockets()) - swarmC.transport.listen('ws', {}, null, ready) - - swarmD.transport.add('ws', new WebSockets()) - swarmD.transport.listen('ws', {}, null, ready) - - swarmE.transport.add('ws', new WebSockets()) - swarmE.transport.listen('ws', {}, null, ready) - - var counter = 0 - - function ready () { - if (++counter === 4) { - done() - } - } - }) - - it('add spdy', (done) => { - swarmA.connection.addStreamMuxer(spdy) - swarmB.connection.addStreamMuxer(spdy) - swarmC.connection.addStreamMuxer(spdy) - swarmD.connection.addStreamMuxer(spdy) - swarmE.connection.addStreamMuxer(spdy) - - swarmA.connection.reuse() - swarmB.connection.reuse() - swarmC.connection.reuse() - swarmD.connection.reuse() - swarmE.connection.reuse() - - done() - }) - - it.skip('add multiplex', (done) => {}) - - it('dial from tcp to tcp+ws', (done) => { - swarmB.handle('/anona/1.0.0', (conn) => { - conn.pipe(conn) - }) - - swarmA.dial(peerB, '/anona/1.0.0', (err, conn) => { - expect(err).to.not.exist - expect(Object.keys(swarmA.muxedConns).length).to.equal(1) - conn.end() - - conn.on('data', () => {}) // let it flow.. let it flooooow - conn.on('end', done) - }) - }) - - it('dial from ws to ws', (done) => { - swarmE.handle('/abacaxi/1.0.0', (conn) => { - conn.pipe(conn) - }) - - swarmD.dial(peerE, '/abacaxi/1.0.0', (err, conn) => { - expect(err).to.not.exist - expect(Object.keys(swarmD.muxedConns).length).to.equal(1) - - conn.end() - conn.on('data', () => {}) // let it flow.. let it flooooow - conn.on('end', () => { - setTimeout(() => { - expect(Object.keys(swarmE.muxedConns).length).to.equal(1) - done() - }, 1000) - }) - }) - }) - - it('dial from tcp to tcp+ws (returned conn)', (done) => { - swarmB.handle('/grapes/1.0.0', (conn) => { - conn.pipe(conn) - }) - - const conn = swarmA.dial(peerB, '/grapes/1.0.0', (err, conn) => { - expect(err).to.not.exist - expect(Object.keys(swarmA.muxedConns).length).to.equal(1) - }) - conn.end() - - conn.on('data', () => {}) // let it flow.. let it flooooow - conn.on('end', done) - }) - - it('dial from tcp+ws to tcp+ws', (done) => { - swarmC.handle('/mamao/1.0.0', (conn) => { - conn.pipe(conn) - }) - - swarmA.dial(peerC, '/mamao/1.0.0', (err, conn) => { - expect(err).to.not.exist - expect(Object.keys(swarmA.muxedConns).length).to.equal(2) - conn.end() - - conn.on('data', () => {}) // let it flow.. let it flooooow - conn.on('end', done) - }) - }) -}) From 52519cd4e9a4f635db39f7f90219bcfb2750d705 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Mar 2016 08:17:16 +0000 Subject: [PATCH 112/634] add multiplex tests --- tests/04-muxing-multiplex-test.js | 130 ++++++++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 7 deletions(-) diff --git a/tests/04-muxing-multiplex-test.js b/tests/04-muxing-multiplex-test.js index acc962d588..a16872f9b5 100644 --- a/tests/04-muxing-multiplex-test.js +++ b/tests/04-muxing-multiplex-test.js @@ -1,14 +1,130 @@ /* eslint-env mocha */ +const expect = require('chai').expect + +const multiaddr = require('multiaddr') +const Peer = require('peer-info') +const Swarm = require('../src') +const TCP = require('libp2p-tcp') +const multiplex = require('libp2p-spdy') + describe('stream muxing with multiplex (on TCP)', function () { this.timeout(20000) - before((done) => { done() }) - after((done) => { done() }) + var swarmA + var peerA + var swarmB + var peerB + var swarmC + var peerC + + before((done) => { + peerA = new Peer() + peerB = new Peer() + peerC = new Peer() + + // console.log('peer A', peerA.id.toB58String()) + // console.log('peer B', peerB.id.toB58String()) + // console.log('peer C', peerC.id.toB58String()) + + peerA.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9001')) + peerB.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9002')) + peerC.multiaddr.add(multiaddr('/ip4/127.0.0.1/tcp/9003')) + + swarmA = new Swarm(peerA) + swarmB = new Swarm(peerB) + swarmC = new Swarm(peerC) + + swarmA.transport.add('tcp', new TCP()) + swarmA.transport.listen('tcp', {}, null, ready) + + swarmB.transport.add('tcp', new TCP()) + swarmB.transport.listen('tcp', {}, null, ready) + + swarmC.transport.add('tcp', new TCP()) + swarmC.transport.listen('tcp', {}, null, ready) + + var counter = 0 + + function ready () { + if (++counter === 3) { + done() + } + } + }) + + after((done) => { + var counter = 0 + + swarmA.close(closed) + swarmB.close(closed) + swarmC.close(closed) + + function closed () { + if (++counter === 3) { + done() + } + } + }) + + it('add', (done) => { + swarmA.connection.addStreamMuxer(multiplex) + swarmB.connection.addStreamMuxer(multiplex) + swarmC.connection.addStreamMuxer(multiplex) + done() + }) + + it('handle + dial on protocol', (done) => { + swarmB.handle('/abacaxi/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmA.dial(peerB, '/abacaxi/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmA.muxedConns).length).to.equal(1) + conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) + + it('dial to warm conn', (done) => { + swarmB.dial(peerA, (err) => { + expect(err).to.not.exist + expect(Object.keys(swarmB.conns).length).to.equal(0) + expect(Object.keys(swarmB.muxedConns).length).to.equal(1) + done() + }) + }) + + it('dial on protocol, reuse warmed conn', (done) => { + swarmA.handle('/papaia/1.0.0', (conn) => { + conn.pipe(conn) + }) + + swarmB.dial(peerA, '/papaia/1.0.0', (err, conn) => { + expect(err).to.not.exist + expect(Object.keys(swarmB.conns).length).to.equal(0) + expect(Object.keys(swarmB.muxedConns).length).to.equal(1) + conn.end() + + conn.on('data', () => {}) // let it flow.. let it flooooow + conn.on('end', done) + }) + }) + + it('enable identify to reuse incomming muxed conn', (done) => { + swarmA.connection.reuse() + swarmC.connection.reuse() - it.skip('add', (done) => {}) - it.skip('handle + dial on protocol', (done) => {}) - it.skip('dial to warm conn', (done) => {}) - it.skip('dial on protocol, reuse warmed conn', (done) => {}) - it.skip('enable identify to reuse incomming muxed conn', (done) => {}) + swarmC.dial(peerA, (err) => { + expect(err).to.not.exist + setTimeout(() => { + expect(Object.keys(swarmC.muxedConns).length).to.equal(1) + expect(Object.keys(swarmA.muxedConns).length).to.equal(2) + done() + }, 500) + }) + }) }) From d7a0246d54803949de787960273f982a681bfcdd Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 23 Mar 2016 08:32:07 +0000 Subject: [PATCH 113/634] Release v0.9.2. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f7cfbf2516..20a443c9e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.9.1", + "version": "0.9.2", "description": "libp2p swarm implementation in JavaScript", "main": "src/index.js", "scripts": { From 71c6242a102571f04fe158c2c26cbb61f70ccfa5 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 23 Mar 2016 16:10:12 +0100 Subject: [PATCH 114/634] Use dignified.js --- .gitignore | 5 +- .travis.yml | 3 +- tests/browser-nodejs/test.js => gulpfile.js | 33 +++++------- karma.conf.js | 54 ------------------- package.json | 35 ++++++------ src/identify.js | 2 + src/index.js | 2 + .../00-basic-test.js => test/00-basic.node.js | 1 + .../01-transport-tcp.node.js | 1 + .../02-transport-utp.node.js | 1 + .../03-transport-websockets.node.js | 1 + .../04-muxing-multiplex.node.js | 1 + .../05-muxing-spdy.node.js | 1 + .../06-conn-upgrade-secio.node.js | 1 + .../07-conn-upgrade-tls.node.js | 1 + .../08-swarm-without-muxing.node.js | 1 + .../09-swarm-with-muxing.node.js | 1 + {tests/browser-nodejs => test}/browser.js | 4 +- test/node.js | 12 +++++ 19 files changed, 66 insertions(+), 94 deletions(-) rename tests/browser-nodejs/test.js => gulpfile.js (75%) delete mode 100644 karma.conf.js rename tests/00-basic-test.js => test/00-basic.node.js (94%) rename tests/01-transport-tcp-test.js => test/01-transport-tcp.node.js (99%) rename tests/02-transport-utp-test.js => test/02-transport-utp.node.js (95%) rename tests/03-transport-websockets-test.js => test/03-transport-websockets.node.js (99%) rename tests/04-muxing-multiplex-test.js => test/04-muxing-multiplex.node.js (99%) rename tests/05-muxing-spdy-test.js => test/05-muxing-spdy.node.js (99%) rename tests/06-conn-upgrade-secio-test.js => test/06-conn-upgrade-secio.node.js (95%) rename tests/07-conn-upgrade-tls-test.js => test/07-conn-upgrade-tls.node.js (95%) rename tests/08-swarm-without-muxing-test.js => test/08-swarm-without-muxing.node.js (99%) rename tests/09-swarm-with-muxing-test.js => test/09-swarm-with-muxing.node.js (99%) rename {tests/browser-nodejs => test}/browser.js (98%) create mode 100644 test/node.js diff --git a/.gitignore b/.gitignore index e3386ccc26..d5c8745647 100644 --- a/.gitignore +++ b/.gitignore @@ -29,5 +29,6 @@ node_modules coverage -.jshintrc -.jshintignore + +dist +lib \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index b0136dec54..d689c7950d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ sudo: false language: node_js node_js: - - "4.0" + - 4 + - 5 # Make sure we have new NPM. before_install: diff --git a/tests/browser-nodejs/test.js b/gulpfile.js similarity index 75% rename from tests/browser-nodejs/test.js rename to gulpfile.js index 9f54de2aef..b8b7f37547 100644 --- a/tests/browser-nodejs/test.js +++ b/gulpfile.js @@ -1,16 +1,17 @@ -const Server = require('karma').Server -const path = require('path') +'use strict' +const gulp = require('gulp') const Peer = require('peer-info') const Id = require('peer-id') const WebSockets = require('libp2p-websockets') -const Swarm = require('./../../src') + +const Swarm = require('./src') const multiaddr = require('multiaddr') -var swarmA -var swarmB +let swarmA +let swarmB -function createListeners (done) { +gulp.task('test:browser:before', (done) => { function createListenerA (cb) { const b58IdA = 'QmWg2L4Fucx1x4KXJTfKHGixBJvveubzcd7DdhB2Mqwfh1' const peerA = new Peer(Id.createFromB58String(b58IdA)) @@ -35,7 +36,7 @@ function createListeners (done) { swarmB.handle('/echo/1.0.0', echo) } - var count = 0 + let count = 0 const ready = () => ++count === 2 ? done() : null createListenerA(ready) @@ -44,22 +45,14 @@ function createListeners (done) { function echo (conn) { conn.pipe(conn) } -} +}) -function stop (done) { - var count = 0 +gulp.task('test:browser:after', (done) => { + let count = 0 const ready = () => ++count === 2 ? done() : null swarmA.transport.close('ws', ready) swarmB.transport.close('ws', ready) -} - -function run (done) { - const karma = new Server({ - configFile: path.join(__dirname, '../../karma.conf.js') - }, done) - - karma.start() -} +}) -createListeners(() => run(() => stop(() => null))) +require('dignified.js/gulp')(gulp) diff --git a/karma.conf.js b/karma.conf.js deleted file mode 100644 index 42617833e3..0000000000 --- a/karma.conf.js +++ /dev/null @@ -1,54 +0,0 @@ -const path = require('path') - -module.exports = function (config) { - const nodeForgePath = path.resolve(__dirname, 'node_modules/peer-id/deps/forge.bundle.js') - - config.set({ - basePath: '', - frameworks: ['mocha'], - - files: [ - nodeForgePath, - 'tests/browser-nodejs/browser.js' - ], - - preprocessors: { - 'tests/*': ['webpack'], - 'tests/browser-nodejs/*': ['webpack'] - }, - - webpack: { - resolve: { - extensions: ['', '.js', '.json'] - }, - externals: { - fs: '{}', - 'node-forge': 'forge' - }, - node: { - Buffer: true - }, - module: { - loaders: [ - { test: /\.json$/, loader: 'json' } - ] - } - }, - - webpackMiddleware: { - noInfo: true, - stats: { - colors: true - } - }, - reporters: ['spec'], - port: 9876, - colors: true, - logLevel: config.LOG_INFO, - autoWatch: false, - browsers: process.env.TRAVIS ? ['Firefox'] : ['Chrome'], - captureTimeout: 60000, - browserNoActivityTimeout: 20000, - singleRun: true - }) -} diff --git a/package.json b/package.json index 20a443c9e1..69240af3ad 100644 --- a/package.json +++ b/package.json @@ -4,11 +4,13 @@ "description": "libp2p swarm implementation in JavaScript", "main": "src/index.js", "scripts": { - "test:node": "mocha tests/*-test.js", - "test:browser": "node tests/browser-nodejs/test.js", - "test": "npm run test:node && npm run test:browser", - "coverage": "istanbul cover --print both -- _mocha tests/*-test.js", - "lint": "standard" + "lint": "dignified-lint", + "build": "dignified-build", + "test": "gulp test", + "test:node": "gulp test:node", + "test:browser": "gulp test:browser", + "release": "dignified-release", + "coverage": "istanbul cover --print both -- _mocha test/node.js" }, "repository": { "type": "git", @@ -34,31 +36,32 @@ "bl": "^1.1.2", "buffer-loader": "0.0.1", "chai": "^3.5.0", + "dignified.js": "github:dignifiedquire/dignified.js", + "gulp": "^3.9.1", "istanbul": "^0.4.2", - "json-loader": "^0.5.4", - "karma": "^0.13.22", - "karma-chrome-launcher": "^0.2.2", - "karma-firefox-launcher": "^0.1.7", - "karma-mocha": "^0.2.2", - "karma-spec-reporter": "0.0.24", - "karma-webpack": "^1.7.0", "libp2p-multiplex": "^0.2.1", "libp2p-spdy": "^0.2.3", "libp2p-tcp": "^0.4.0", "libp2p-websockets": "^0.2.1", - "mocha": "^2.4.5", "multiaddr": "^1.3.0", "peer-id": "^0.6.0", "peer-info": "^0.6.0", "pre-commit": "^1.1.2", - "standard": "^6.0.7", - "stream-pair": "^1.0.3", - "webpack": "^2.1.0-beta.4" + "stream-pair": "^1.0.3" }, "dependencies": { "duplex-passthrough": "github:diasdavid/duplex-passthrough", "ip-address": "^5.0.2", "multistream-select": "^0.6.1", "protocol-buffers-stream": "^1.2.0" + }, + "dignified": { + "webpack": { + "resolve": { + "alias": { + "node-forge": "../deps/forge.bundle.js" + } + } + } } } diff --git a/src/identify.js b/src/identify.js index 35a2de206c..516c6aa85b 100644 --- a/src/identify.js +++ b/src/identify.js @@ -5,6 +5,8 @@ * established, so a conn can be reused */ +'use strict' + const multistream = require('multistream-select') const fs = require('fs') const path = require('path') diff --git a/src/index.js b/src/index.js index eddc858bb4..89ca24c81c 100644 --- a/src/index.js +++ b/src/index.js @@ -1,3 +1,5 @@ +'use strict' + const multistream = require('multistream-select') const identify = require('./identify') const DuplexPassThrough = require('duplex-passthrough') diff --git a/tests/00-basic-test.js b/test/00-basic.node.js similarity index 94% rename from tests/00-basic-test.js rename to test/00-basic.node.js index ef5d96ea22..4c94654671 100644 --- a/tests/00-basic-test.js +++ b/test/00-basic.node.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +'use strict' const expect = require('chai').expect diff --git a/tests/01-transport-tcp-test.js b/test/01-transport-tcp.node.js similarity index 99% rename from tests/01-transport-tcp-test.js rename to test/01-transport-tcp.node.js index 641f6323c5..3d020fad9d 100644 --- a/tests/01-transport-tcp-test.js +++ b/test/01-transport-tcp.node.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +'use strict' const expect = require('chai').expect diff --git a/tests/02-transport-utp-test.js b/test/02-transport-utp.node.js similarity index 95% rename from tests/02-transport-utp-test.js rename to test/02-transport-utp.node.js index 7b3714dbe5..f8f046dace 100644 --- a/tests/02-transport-utp-test.js +++ b/test/02-transport-utp.node.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +'use strict' describe('transport - utp', function () { this.timeout(10000) diff --git a/tests/03-transport-websockets-test.js b/test/03-transport-websockets.node.js similarity index 99% rename from tests/03-transport-websockets-test.js rename to test/03-transport-websockets.node.js index 2a6246fe19..ab4c11264b 100644 --- a/tests/03-transport-websockets-test.js +++ b/test/03-transport-websockets.node.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +'use strict' const expect = require('chai').expect diff --git a/tests/04-muxing-multiplex-test.js b/test/04-muxing-multiplex.node.js similarity index 99% rename from tests/04-muxing-multiplex-test.js rename to test/04-muxing-multiplex.node.js index a16872f9b5..d8a8507ad7 100644 --- a/tests/04-muxing-multiplex-test.js +++ b/test/04-muxing-multiplex.node.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +'use strict' const expect = require('chai').expect diff --git a/tests/05-muxing-spdy-test.js b/test/05-muxing-spdy.node.js similarity index 99% rename from tests/05-muxing-spdy-test.js rename to test/05-muxing-spdy.node.js index 859616728b..cbfc5fa629 100644 --- a/tests/05-muxing-spdy-test.js +++ b/test/05-muxing-spdy.node.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +'use strict' const expect = require('chai').expect diff --git a/tests/06-conn-upgrade-secio-test.js b/test/06-conn-upgrade-secio.node.js similarity index 95% rename from tests/06-conn-upgrade-secio-test.js rename to test/06-conn-upgrade-secio.node.js index 18e4887fd7..1f33224ce7 100644 --- a/tests/06-conn-upgrade-secio-test.js +++ b/test/06-conn-upgrade-secio.node.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +'use strict' describe('secio conn upgrade (on TCP)', function () { this.timeout(20000) diff --git a/tests/07-conn-upgrade-tls-test.js b/test/07-conn-upgrade-tls.node.js similarity index 95% rename from tests/07-conn-upgrade-tls-test.js rename to test/07-conn-upgrade-tls.node.js index 2fe9c4e815..b3be730ee7 100644 --- a/tests/07-conn-upgrade-tls-test.js +++ b/test/07-conn-upgrade-tls.node.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +'use strict' describe('tls conn upgrade (on TCP)', function () { before((done) => { done() }) diff --git a/tests/08-swarm-without-muxing-test.js b/test/08-swarm-without-muxing.node.js similarity index 99% rename from tests/08-swarm-without-muxing-test.js rename to test/08-swarm-without-muxing.node.js index c41813fba8..41950581a4 100644 --- a/tests/08-swarm-without-muxing-test.js +++ b/test/08-swarm-without-muxing.node.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +'use strict' const expect = require('chai').expect diff --git a/tests/09-swarm-with-muxing-test.js b/test/09-swarm-with-muxing.node.js similarity index 99% rename from tests/09-swarm-with-muxing-test.js rename to test/09-swarm-with-muxing.node.js index 998a6ee258..0039ecd136 100644 --- a/tests/09-swarm-with-muxing-test.js +++ b/test/09-swarm-with-muxing.node.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +'use strict' const expect = require('chai').expect diff --git a/tests/browser-nodejs/browser.js b/test/browser.js similarity index 98% rename from tests/browser-nodejs/browser.js rename to test/browser.js index 6b6a7ed037..bcd92cbdb7 100644 --- a/tests/browser-nodejs/browser.js +++ b/test/browser.js @@ -1,13 +1,15 @@ /* eslint-env mocha */ +'use strict' const expect = require('chai').expect const multiaddr = require('multiaddr') const Id = require('peer-id') const Peer = require('peer-info') -const Swarm = require('./../../src') const WebSockets = require('libp2p-websockets') const bl = require('bl') +const Swarm = require('../src') + describe('basics', () => { it('throws on missing peerInfo', (done) => { expect(Swarm).to.throw(Error) diff --git a/test/node.js b/test/node.js new file mode 100644 index 0000000000..3598479010 --- /dev/null +++ b/test/node.js @@ -0,0 +1,12 @@ +/* eslint-env mocha */ +'use strict' + +const fs = require('fs') + +describe('libp2p-swarm', () => { + fs.readdirSync(__dirname) + .filter((file) => file.match(/\.node\.js$/)) + .forEach((file) => { + require(`./${file}`) + }) +}) From cea42f54cd3f1d786bbf3daf71c15a7e11bbde7d Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 7 Apr 2016 21:21:42 +0200 Subject: [PATCH 115/634] Use dignified.js 1.0.0 --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 69240af3ad..0ca8c7a1bc 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,8 @@ "name": "libp2p-swarm", "version": "0.9.2", "description": "libp2p swarm implementation in JavaScript", - "main": "src/index.js", + "main": "lib/index.js", + "jsnext:main": "src/index.js", "scripts": { "lint": "dignified-lint", "build": "dignified-build", @@ -36,7 +37,7 @@ "bl": "^1.1.2", "buffer-loader": "0.0.1", "chai": "^3.5.0", - "dignified.js": "github:dignifiedquire/dignified.js", + "dignified.js": "^1.0.0", "gulp": "^3.9.1", "istanbul": "^0.4.2", "libp2p-multiplex": "^0.2.1", From f8cbc89d1abfb2e782557e827e18e6d70b88d3db Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 14 Apr 2016 02:34:07 +0100 Subject: [PATCH 116/634] chore: release version v0.9.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0ca8c7a1bc..9470c527f2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "libp2p-swarm", - "version": "0.9.2", + "version": "0.9.3", "description": "libp2p swarm implementation in JavaScript", "main": "lib/index.js", "jsnext:main": "src/index.js", From 01bd659ee8f80c9372b48f9bf3cd613f0e34fa68 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 14 Apr 2016 03:03:54 +0100 Subject: [PATCH 117/634] vendor node-forge, update websockets dep --- package.json | 4 +- vendor/forge.bundle.js | 29552 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 29554 insertions(+), 2 deletions(-) create mode 100644 vendor/forge.bundle.js diff --git a/package.json b/package.json index 9470c527f2..386e421d8a 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "libp2p-multiplex": "^0.2.1", "libp2p-spdy": "^0.2.3", "libp2p-tcp": "^0.4.0", - "libp2p-websockets": "^0.2.1", + "libp2p-websockets": "^0.3.1", "multiaddr": "^1.3.0", "peer-id": "^0.6.0", "peer-info": "^0.6.0", @@ -60,7 +60,7 @@ "webpack": { "resolve": { "alias": { - "node-forge": "../deps/forge.bundle.js" + "node-forge": "../vendor/forge.bundle.js" } } } diff --git a/vendor/forge.bundle.js b/vendor/forge.bundle.js new file mode 100644 index 0000000000..0e03bcb006 --- /dev/null +++ b/vendor/forge.bundle.js @@ -0,0 +1,29552 @@ +(function(root, factory) { + if(typeof define === 'function' && define.amd) { + define([], factory); + } else { + root.forge = factory(); + } +})(this, function() { +/** + * @license almond 0.2.9 Copyright (c) 2011-2014, The Dojo Foundation All Rights Reserved. + * Available via the MIT or new BSD license. + * see: http://github.com/jrburke/almond for details + */ +//Going sloppy to avoid 'use strict' string cost, but strict practices should +//be followed. +/*jslint sloppy: true */ +/*global setTimeout: false */ + +var requirejs, require, define; +(function (undef) { + var main, req, makeMap, handlers, + defined = {}, + waiting = {}, + config = {}, + defining = {}, + hasOwn = Object.prototype.hasOwnProperty, + aps = [].slice, + jsSuffixRegExp = /\.js$/; + + function hasProp(obj, prop) { + return hasOwn.call(obj, prop); + } + + /** + * Given a relative module name, like ./something, normalize it to + * a real name that can be mapped to a path. + * @param {String} name the relative name + * @param {String} baseName a real name that the name arg is relative + * to. + * @returns {String} normalized name + */ + function normalize(name, baseName) { + var nameParts, nameSegment, mapValue, foundMap, lastIndex, + foundI, foundStarMap, starI, i, j, part, + baseParts = baseName && baseName.split("/"), + map = config.map, + starMap = (map && map['*']) || {}; + + //Adjust any relative paths. + if (name && name.charAt(0) === ".") { + //If have a base name, try to normalize against it, + //otherwise, assume it is a top-level require that will + //be relative to baseUrl in the end. + if (baseName) { + //Convert baseName to array, and lop off the last part, + //so that . matches that "directory" and not name of the baseName's + //module. For instance, baseName of "one/two/three", maps to + //"one/two/three.js", but we want the directory, "one/two" for + //this normalization. + baseParts = baseParts.slice(0, baseParts.length - 1); + name = name.split('/'); + lastIndex = name.length - 1; + + // Node .js allowance: + if (config.nodeIdCompat && jsSuffixRegExp.test(name[lastIndex])) { + name[lastIndex] = name[lastIndex].replace(jsSuffixRegExp, ''); + } + + name = baseParts.concat(name); + + //start trimDots + for (i = 0; i < name.length; i += 1) { + part = name[i]; + if (part === ".") { + name.splice(i, 1); + i -= 1; + } else if (part === "..") { + if (i === 1 && (name[2] === '..' || name[0] === '..')) { + //End of the line. Keep at least one non-dot + //path segment at the front so it can be mapped + //correctly to disk. Otherwise, there is likely + //no path mapping for a path starting with '..'. + //This can still fail, but catches the most reasonable + //uses of .. + break; + } else if (i > 0) { + name.splice(i - 1, 2); + i -= 2; + } + } + } + //end trimDots + + name = name.join("/"); + } else if (name.indexOf('./') === 0) { + // No baseName, so this is ID is resolved relative + // to baseUrl, pull off the leading dot. + name = name.substring(2); + } + } + + //Apply map config if available. + if ((baseParts || starMap) && map) { + nameParts = name.split('/'); + + for (i = nameParts.length; i > 0; i -= 1) { + nameSegment = nameParts.slice(0, i).join("/"); + + if (baseParts) { + //Find the longest baseName segment match in the config. + //So, do joins on the biggest to smallest lengths of baseParts. + for (j = baseParts.length; j > 0; j -= 1) { + mapValue = map[baseParts.slice(0, j).join('/')]; + + //baseName segment has config, find if it has one for + //this name. + if (mapValue) { + mapValue = mapValue[nameSegment]; + if (mapValue) { + //Match, update name to the new value. + foundMap = mapValue; + foundI = i; + break; + } + } + } + } + + if (foundMap) { + break; + } + + //Check for a star map match, but just hold on to it, + //if there is a shorter segment match later in a matching + //config, then favor over this star map. + if (!foundStarMap && starMap && starMap[nameSegment]) { + foundStarMap = starMap[nameSegment]; + starI = i; + } + } + + if (!foundMap && foundStarMap) { + foundMap = foundStarMap; + foundI = starI; + } + + if (foundMap) { + nameParts.splice(0, foundI, foundMap); + name = nameParts.join('/'); + } + } + + return name; + } + + function makeRequire(relName, forceSync) { + return function () { + //A version of a require function that passes a moduleName + //value for items that may need to + //look up paths relative to the moduleName + return req.apply(undef, aps.call(arguments, 0).concat([relName, forceSync])); + }; + } + + function makeNormalize(relName) { + return function (name) { + return normalize(name, relName); + }; + } + + function makeLoad(depName) { + return function (value) { + defined[depName] = value; + }; + } + + function callDep(name) { + if (hasProp(waiting, name)) { + var args = waiting[name]; + delete waiting[name]; + defining[name] = true; + main.apply(undef, args); + } + + if (!hasProp(defined, name) && !hasProp(defining, name)) { + throw new Error('No ' + name); + } + return defined[name]; + } + + //Turns a plugin!resource to [plugin, resource] + //with the plugin being undefined if the name + //did not have a plugin prefix. + function splitPrefix(name) { + var prefix, + index = name ? name.indexOf('!') : -1; + if (index > -1) { + prefix = name.substring(0, index); + name = name.substring(index + 1, name.length); + } + return [prefix, name]; + } + + /** + * Makes a name map, normalizing the name, and using a plugin + * for normalization if necessary. Grabs a ref to plugin + * too, as an optimization. + */ + makeMap = function (name, relName) { + var plugin, + parts = splitPrefix(name), + prefix = parts[0]; + + name = parts[1]; + + if (prefix) { + prefix = normalize(prefix, relName); + plugin = callDep(prefix); + } + + //Normalize according + if (prefix) { + if (plugin && plugin.normalize) { + name = plugin.normalize(name, makeNormalize(relName)); + } else { + name = normalize(name, relName); + } + } else { + name = normalize(name, relName); + parts = splitPrefix(name); + prefix = parts[0]; + name = parts[1]; + if (prefix) { + plugin = callDep(prefix); + } + } + + //Using ridiculous property names for space reasons + return { + f: prefix ? prefix + '!' + name : name, //fullName + n: name, + pr: prefix, + p: plugin + }; + }; + + function makeConfig(name) { + return function () { + return (config && config.config && config.config[name]) || {}; + }; + } + + handlers = { + require: function (name) { + return makeRequire(name); + }, + exports: function (name) { + var e = defined[name]; + if (typeof e !== 'undefined') { + return e; + } else { + return (defined[name] = {}); + } + }, + module: function (name) { + return { + id: name, + uri: '', + exports: defined[name], + config: makeConfig(name) + }; + } + }; + + main = function (name, deps, callback, relName) { + var cjsModule, depName, ret, map, i, + args = [], + callbackType = typeof callback, + usingExports; + + //Use name if no relName + relName = relName || name; + + //Call the callback to define the module, if necessary. + if (callbackType === 'undefined' || callbackType === 'function') { + //Pull out the defined dependencies and pass the ordered + //values to the callback. + //Default to [require, exports, module] if no deps + deps = !deps.length && callback.length ? ['require', 'exports', 'module'] : deps; + for (i = 0; i < deps.length; i += 1) { + map = makeMap(deps[i], relName); + depName = map.f; + + //Fast path CommonJS standard dependencies. + if (depName === "require") { + args[i] = handlers.require(name); + } else if (depName === "exports") { + //CommonJS module spec 1.1 + args[i] = handlers.exports(name); + usingExports = true; + } else if (depName === "module") { + //CommonJS module spec 1.1 + cjsModule = args[i] = handlers.module(name); + } else if (hasProp(defined, depName) || + hasProp(waiting, depName) || + hasProp(defining, depName)) { + args[i] = callDep(depName); + } else if (map.p) { + map.p.load(map.n, makeRequire(relName, true), makeLoad(depName), {}); + args[i] = defined[depName]; + } else { + throw new Error(name + ' missing ' + depName); + } + } + + ret = callback ? callback.apply(defined[name], args) : undefined; + + if (name) { + //If setting exports via "module" is in play, + //favor that over return value and exports. After that, + //favor a non-undefined return value over exports use. + if (cjsModule && cjsModule.exports !== undef && + cjsModule.exports !== defined[name]) { + defined[name] = cjsModule.exports; + } else if (ret !== undef || !usingExports) { + //Use the return value from the function. + defined[name] = ret; + } + } + } else if (name) { + //May just be an object definition for the module. Only + //worry about defining if have a module name. + defined[name] = callback; + } + }; + + requirejs = require = req = function (deps, callback, relName, forceSync, alt) { + if (typeof deps === "string") { + if (handlers[deps]) { + //callback in this case is really relName + return handlers[deps](callback); + } + //Just return the module wanted. In this scenario, the + //deps arg is the module name, and second arg (if passed) + //is just the relName. + //Normalize module name, if it contains . or .. + return callDep(makeMap(deps, callback).f); + } else if (!deps.splice) { + //deps is a config object, not an array. + config = deps; + if (config.deps) { + req(config.deps, config.callback); + } + if (!callback) { + return; + } + + if (callback.splice) { + //callback is an array, which means it is a dependency list. + //Adjust args if there are dependencies + deps = callback; + callback = relName; + relName = null; + } else { + deps = undef; + } + } + + //Support require(['a']) + callback = callback || function () {}; + + //If relName is a function, it is an errback handler, + //so remove it. + if (typeof relName === 'function') { + relName = forceSync; + forceSync = alt; + } + + //Simulate async callback; + if (forceSync) { + main(undef, deps, callback, relName); + } else { + //Using a non-zero value because of concern for what old browsers + //do, and latest browsers "upgrade" to 4 if lower value is used: + //http://www.whatwg.org/specs/web-apps/current-work/multipage/timers.html#dom-windowtimers-settimeout: + //If want a value immediately, use require('id') instead -- something + //that works in almond on the global level, but not guaranteed and + //unlikely to work in other AMD implementations. + setTimeout(function () { + main(undef, deps, callback, relName); + }, 4); + } + + return req; + }; + + /** + * Just drops the config on the floor, but returns req in case + * the config return value is used. + */ + req.config = function (cfg) { + return req(cfg); + }; + + /** + * Expose module registry for debugging and tooling + */ + requirejs._defined = defined; + + define = function (name, deps, callback) { + + //This module may not have dependencies + if (!deps.splice) { + //deps is not an array, so probably means + //an object literal or factory function for + //the value. Adjust args. + callback = deps; + deps = []; + } + + if (!hasProp(defined, name) && !hasProp(waiting, name)) { + waiting[name] = [name, deps, callback]; + } + }; + + define.amd = { + jQuery: true + }; +}()); + +define("node_modules/almond/almond", function(){}); + +/** + * Utility functions for web applications. + * + * @author Dave Longley + * + * Copyright (c) 2010-2014 Digital Bazaar, Inc. + */ +(function() { +/* ########## Begin module implementation ########## */ +function initModule(forge) { + +/* Utilities API */ +var util = forge.util = forge.util || {}; + +// define setImmediate and nextTick +(function() { + // use native nextTick + if(typeof process !== 'undefined' && process.nextTick) { + util.nextTick = process.nextTick; + if(typeof setImmediate === 'function') { + util.setImmediate = setImmediate; + } else { + // polyfill setImmediate with nextTick, older versions of node + // (those w/o setImmediate) won't totally starve IO + util.setImmediate = util.nextTick; + } + return; + } + + // polyfill nextTick with native setImmediate + if(typeof setImmediate === 'function') { + util.setImmediate = setImmediate; + util.nextTick = function(callback) { + return setImmediate(callback); + }; + return; + } + + /* Note: A polyfill upgrade pattern is used here to allow combining + polyfills. For example, MutationObserver is fast, but blocks UI updates, + so it needs to allow UI updates periodically, so it falls back on + postMessage or setTimeout. */ + + // polyfill with setTimeout + util.setImmediate = function(callback) { + setTimeout(callback, 0); + }; + + // upgrade polyfill to use postMessage + if(typeof window !== 'undefined' && + typeof window.postMessage === 'function') { + var msg = 'forge.setImmediate'; + var callbacks = []; + util.setImmediate = function(callback) { + callbacks.push(callback); + // only send message when one hasn't been sent in + // the current turn of the event loop + if(callbacks.length === 1) { + window.postMessage(msg, '*'); + } + }; + function handler(event) { + if(event.source === window && event.data === msg) { + event.stopPropagation(); + var copy = callbacks.slice(); + callbacks.length = 0; + copy.forEach(function(callback) { + callback(); + }); + } + } + window.addEventListener('message', handler, true); + } + + // upgrade polyfill to use MutationObserver + if(typeof MutationObserver !== 'undefined') { + // polyfill with MutationObserver + var now = Date.now(); + var attr = true; + var div = document.createElement('div'); + var callbacks = []; + new MutationObserver(function() { + var copy = callbacks.slice(); + callbacks.length = 0; + copy.forEach(function(callback) { + callback(); + }); + }).observe(div, {attributes: true}); + var oldSetImmediate = util.setImmediate; + util.setImmediate = function(callback) { + if(Date.now() - now > 15) { + now = Date.now(); + oldSetImmediate(callback); + } else { + callbacks.push(callback); + // only trigger observer when it hasn't been triggered in + // the current turn of the event loop + if(callbacks.length === 1) { + div.setAttribute('a', attr = !attr); + } + } + }; + } + + util.nextTick = util.setImmediate; +})(); + +// define isArray +util.isArray = Array.isArray || function(x) { + return Object.prototype.toString.call(x) === '[object Array]'; +}; + +// define isArrayBuffer +util.isArrayBuffer = function(x) { + return typeof ArrayBuffer !== 'undefined' && x instanceof ArrayBuffer; +}; + +// define isArrayBufferView +util.isArrayBufferView = function(x) { + return x && util.isArrayBuffer(x.buffer) && x.byteLength !== undefined; +}; + +// TODO: set ByteBuffer to best available backing +util.ByteBuffer = ByteStringBuffer; + +/** Buffer w/BinaryString backing */ + +/** + * Constructor for a binary string backed byte buffer. + * + * @param [b] the bytes to wrap (either encoded as string, one byte per + * character, or as an ArrayBuffer or Typed Array). + */ +function ByteStringBuffer(b) { + // TODO: update to match DataBuffer API + + // the data in this buffer + this.data = ''; + // the pointer for reading from this buffer + this.read = 0; + + if(typeof b === 'string') { + this.data = b; + } else if(util.isArrayBuffer(b) || util.isArrayBufferView(b)) { + // convert native buffer to forge buffer + // FIXME: support native buffers internally instead + var arr = new Uint8Array(b); + try { + this.data = String.fromCharCode.apply(null, arr); + } catch(e) { + for(var i = 0; i < arr.length; ++i) { + this.putByte(arr[i]); + } + } + } else if(b instanceof ByteStringBuffer || + (typeof b === 'object' && typeof b.data === 'string' && + typeof b.read === 'number')) { + // copy existing buffer + this.data = b.data; + this.read = b.read; + } + + // used for v8 optimization + this._constructedStringLength = 0; +} +util.ByteStringBuffer = ByteStringBuffer; + +/* Note: This is an optimization for V8-based browsers. When V8 concatenates + a string, the strings are only joined logically using a "cons string" or + "constructed/concatenated string". These containers keep references to one + another and can result in very large memory usage. For example, if a 2MB + string is constructed by concatenating 4 bytes together at a time, the + memory usage will be ~44MB; so ~22x increase. The strings are only joined + together when an operation requiring their joining takes place, such as + substr(). This function is called when adding data to this buffer to ensure + these types of strings are periodically joined to reduce the memory + footprint. */ +var _MAX_CONSTRUCTED_STRING_LENGTH = 4096; +util.ByteStringBuffer.prototype._optimizeConstructedString = function(x) { + this._constructedStringLength += x; + if(this._constructedStringLength > _MAX_CONSTRUCTED_STRING_LENGTH) { + // this substr() should cause the constructed string to join + this.data.substr(0, 1); + this._constructedStringLength = 0; + } +}; + +/** + * Gets the number of bytes in this buffer. + * + * @return the number of bytes in this buffer. + */ +util.ByteStringBuffer.prototype.length = function() { + return this.data.length - this.read; +}; + +/** + * Gets whether or not this buffer is empty. + * + * @return true if this buffer is empty, false if not. + */ +util.ByteStringBuffer.prototype.isEmpty = function() { + return this.length() <= 0; +}; + +/** + * Puts a byte in this buffer. + * + * @param b the byte to put. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putByte = function(b) { + return this.putBytes(String.fromCharCode(b)); +}; + +/** + * Puts a byte in this buffer N times. + * + * @param b the byte to put. + * @param n the number of bytes of value b to put. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.fillWithByte = function(b, n) { + b = String.fromCharCode(b); + var d = this.data; + while(n > 0) { + if(n & 1) { + d += b; + } + n >>>= 1; + if(n > 0) { + b += b; + } + } + this.data = d; + this._optimizeConstructedString(n); + return this; +}; + +/** + * Puts bytes in this buffer. + * + * @param bytes the bytes (as a UTF-8 encoded string) to put. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putBytes = function(bytes) { + this.data += bytes; + this._optimizeConstructedString(bytes.length); + return this; +}; + +/** + * Puts a UTF-16 encoded string into this buffer. + * + * @param str the string to put. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putString = function(str) { + return this.putBytes(util.encodeUtf8(str)); +}; + +/** + * Puts a 16-bit integer in this buffer in big-endian order. + * + * @param i the 16-bit integer. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putInt16 = function(i) { + return this.putBytes( + String.fromCharCode(i >> 8 & 0xFF) + + String.fromCharCode(i & 0xFF)); +}; + +/** + * Puts a 24-bit integer in this buffer in big-endian order. + * + * @param i the 24-bit integer. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putInt24 = function(i) { + return this.putBytes( + String.fromCharCode(i >> 16 & 0xFF) + + String.fromCharCode(i >> 8 & 0xFF) + + String.fromCharCode(i & 0xFF)); +}; + +/** + * Puts a 32-bit integer in this buffer in big-endian order. + * + * @param i the 32-bit integer. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putInt32 = function(i) { + return this.putBytes( + String.fromCharCode(i >> 24 & 0xFF) + + String.fromCharCode(i >> 16 & 0xFF) + + String.fromCharCode(i >> 8 & 0xFF) + + String.fromCharCode(i & 0xFF)); +}; + +/** + * Puts a 16-bit integer in this buffer in little-endian order. + * + * @param i the 16-bit integer. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putInt16Le = function(i) { + return this.putBytes( + String.fromCharCode(i & 0xFF) + + String.fromCharCode(i >> 8 & 0xFF)); +}; + +/** + * Puts a 24-bit integer in this buffer in little-endian order. + * + * @param i the 24-bit integer. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putInt24Le = function(i) { + return this.putBytes( + String.fromCharCode(i & 0xFF) + + String.fromCharCode(i >> 8 & 0xFF) + + String.fromCharCode(i >> 16 & 0xFF)); +}; + +/** + * Puts a 32-bit integer in this buffer in little-endian order. + * + * @param i the 32-bit integer. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putInt32Le = function(i) { + return this.putBytes( + String.fromCharCode(i & 0xFF) + + String.fromCharCode(i >> 8 & 0xFF) + + String.fromCharCode(i >> 16 & 0xFF) + + String.fromCharCode(i >> 24 & 0xFF)); +}; + +/** + * Puts an n-bit integer in this buffer in big-endian order. + * + * @param i the n-bit integer. + * @param n the number of bits in the integer. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putInt = function(i, n) { + var bytes = ''; + do { + n -= 8; + bytes += String.fromCharCode((i >> n) & 0xFF); + } while(n > 0); + return this.putBytes(bytes); +}; + +/** + * Puts a signed n-bit integer in this buffer in big-endian order. Two's + * complement representation is used. + * + * @param i the n-bit integer. + * @param n the number of bits in the integer. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putSignedInt = function(i, n) { + if(i < 0) { + i += 2 << (n - 1); + } + return this.putInt(i, n); +}; + +/** + * Puts the given buffer into this buffer. + * + * @param buffer the buffer to put into this one. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.putBuffer = function(buffer) { + return this.putBytes(buffer.getBytes()); +}; + +/** + * Gets a byte from this buffer and advances the read pointer by 1. + * + * @return the byte. + */ +util.ByteStringBuffer.prototype.getByte = function() { + return this.data.charCodeAt(this.read++); +}; + +/** + * Gets a uint16 from this buffer in big-endian order and advances the read + * pointer by 2. + * + * @return the uint16. + */ +util.ByteStringBuffer.prototype.getInt16 = function() { + var rval = ( + this.data.charCodeAt(this.read) << 8 ^ + this.data.charCodeAt(this.read + 1)); + this.read += 2; + return rval; +}; + +/** + * Gets a uint24 from this buffer in big-endian order and advances the read + * pointer by 3. + * + * @return the uint24. + */ +util.ByteStringBuffer.prototype.getInt24 = function() { + var rval = ( + this.data.charCodeAt(this.read) << 16 ^ + this.data.charCodeAt(this.read + 1) << 8 ^ + this.data.charCodeAt(this.read + 2)); + this.read += 3; + return rval; +}; + +/** + * Gets a uint32 from this buffer in big-endian order and advances the read + * pointer by 4. + * + * @return the word. + */ +util.ByteStringBuffer.prototype.getInt32 = function() { + var rval = ( + this.data.charCodeAt(this.read) << 24 ^ + this.data.charCodeAt(this.read + 1) << 16 ^ + this.data.charCodeAt(this.read + 2) << 8 ^ + this.data.charCodeAt(this.read + 3)); + this.read += 4; + return rval; +}; + +/** + * Gets a uint16 from this buffer in little-endian order and advances the read + * pointer by 2. + * + * @return the uint16. + */ +util.ByteStringBuffer.prototype.getInt16Le = function() { + var rval = ( + this.data.charCodeAt(this.read) ^ + this.data.charCodeAt(this.read + 1) << 8); + this.read += 2; + return rval; +}; + +/** + * Gets a uint24 from this buffer in little-endian order and advances the read + * pointer by 3. + * + * @return the uint24. + */ +util.ByteStringBuffer.prototype.getInt24Le = function() { + var rval = ( + this.data.charCodeAt(this.read) ^ + this.data.charCodeAt(this.read + 1) << 8 ^ + this.data.charCodeAt(this.read + 2) << 16); + this.read += 3; + return rval; +}; + +/** + * Gets a uint32 from this buffer in little-endian order and advances the read + * pointer by 4. + * + * @return the word. + */ +util.ByteStringBuffer.prototype.getInt32Le = function() { + var rval = ( + this.data.charCodeAt(this.read) ^ + this.data.charCodeAt(this.read + 1) << 8 ^ + this.data.charCodeAt(this.read + 2) << 16 ^ + this.data.charCodeAt(this.read + 3) << 24); + this.read += 4; + return rval; +}; + +/** + * Gets an n-bit integer from this buffer in big-endian order and advances the + * read pointer by n/8. + * + * @param n the number of bits in the integer. + * + * @return the integer. + */ +util.ByteStringBuffer.prototype.getInt = function(n) { + var rval = 0; + do { + rval = (rval << 8) + this.data.charCodeAt(this.read++); + n -= 8; + } while(n > 0); + return rval; +}; + +/** + * Gets a signed n-bit integer from this buffer in big-endian order, using + * two's complement, and advances the read pointer by n/8. + * + * @param n the number of bits in the integer. + * + * @return the integer. + */ +util.ByteStringBuffer.prototype.getSignedInt = function(n) { + var x = this.getInt(n); + var max = 2 << (n - 2); + if(x >= max) { + x -= max << 1; + } + return x; +}; + +/** + * Reads bytes out into a UTF-8 string and clears them from the buffer. + * + * @param count the number of bytes to read, undefined or null for all. + * + * @return a UTF-8 string of bytes. + */ +util.ByteStringBuffer.prototype.getBytes = function(count) { + var rval; + if(count) { + // read count bytes + count = Math.min(this.length(), count); + rval = this.data.slice(this.read, this.read + count); + this.read += count; + } else if(count === 0) { + rval = ''; + } else { + // read all bytes, optimize to only copy when needed + rval = (this.read === 0) ? this.data : this.data.slice(this.read); + this.clear(); + } + return rval; +}; + +/** + * Gets a UTF-8 encoded string of the bytes from this buffer without modifying + * the read pointer. + * + * @param count the number of bytes to get, omit to get all. + * + * @return a string full of UTF-8 encoded characters. + */ +util.ByteStringBuffer.prototype.bytes = function(count) { + return (typeof(count) === 'undefined' ? + this.data.slice(this.read) : + this.data.slice(this.read, this.read + count)); +}; + +/** + * Gets a byte at the given index without modifying the read pointer. + * + * @param i the byte index. + * + * @return the byte. + */ +util.ByteStringBuffer.prototype.at = function(i) { + return this.data.charCodeAt(this.read + i); +}; + +/** + * Puts a byte at the given index without modifying the read pointer. + * + * @param i the byte index. + * @param b the byte to put. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.setAt = function(i, b) { + this.data = this.data.substr(0, this.read + i) + + String.fromCharCode(b) + + this.data.substr(this.read + i + 1); + return this; +}; + +/** + * Gets the last byte without modifying the read pointer. + * + * @return the last byte. + */ +util.ByteStringBuffer.prototype.last = function() { + return this.data.charCodeAt(this.data.length - 1); +}; + +/** + * Creates a copy of this buffer. + * + * @return the copy. + */ +util.ByteStringBuffer.prototype.copy = function() { + var c = util.createBuffer(this.data); + c.read = this.read; + return c; +}; + +/** + * Compacts this buffer. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.compact = function() { + if(this.read > 0) { + this.data = this.data.slice(this.read); + this.read = 0; + } + return this; +}; + +/** + * Clears this buffer. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.clear = function() { + this.data = ''; + this.read = 0; + return this; +}; + +/** + * Shortens this buffer by triming bytes off of the end of this buffer. + * + * @param count the number of bytes to trim off. + * + * @return this buffer. + */ +util.ByteStringBuffer.prototype.truncate = function(count) { + var len = Math.max(0, this.length() - count); + this.data = this.data.substr(this.read, len); + this.read = 0; + return this; +}; + +/** + * Converts this buffer to a hexadecimal string. + * + * @return a hexadecimal string. + */ +util.ByteStringBuffer.prototype.toHex = function() { + var rval = ''; + for(var i = this.read; i < this.data.length; ++i) { + var b = this.data.charCodeAt(i); + if(b < 16) { + rval += '0'; + } + rval += b.toString(16); + } + return rval; +}; + +/** + * Converts this buffer to a UTF-16 string (standard JavaScript string). + * + * @return a UTF-16 string. + */ +util.ByteStringBuffer.prototype.toString = function() { + return util.decodeUtf8(this.bytes()); +}; + +/** End Buffer w/BinaryString backing */ + + +/** Buffer w/UInt8Array backing */ + +/** + * FIXME: Experimental. Do not use yet. + * + * Constructor for an ArrayBuffer-backed byte buffer. + * + * The buffer may be constructed from a string, an ArrayBuffer, DataView, or a + * TypedArray. + * + * If a string is given, its encoding should be provided as an option, + * otherwise it will default to 'binary'. A 'binary' string is encoded such + * that each character is one byte in length and size. + * + * If an ArrayBuffer, DataView, or TypedArray is given, it will be used + * *directly* without any copying. Note that, if a write to the buffer requires + * more space, the buffer will allocate a new backing ArrayBuffer to + * accommodate. The starting read and write offsets for the buffer may be + * given as options. + * + * @param [b] the initial bytes for this buffer. + * @param options the options to use: + * [readOffset] the starting read offset to use (default: 0). + * [writeOffset] the starting write offset to use (default: the + * length of the first parameter). + * [growSize] the minimum amount, in bytes, to grow the buffer by to + * accommodate writes (default: 1024). + * [encoding] the encoding ('binary', 'utf8', 'utf16', 'hex') for the + * first parameter, if it is a string (default: 'binary'). + */ +function DataBuffer(b, options) { + // default options + options = options || {}; + + // pointers for read from/write to buffer + this.read = options.readOffset || 0; + this.growSize = options.growSize || 1024; + + var isArrayBuffer = util.isArrayBuffer(b); + var isArrayBufferView = util.isArrayBufferView(b); + if(isArrayBuffer || isArrayBufferView) { + // use ArrayBuffer directly + if(isArrayBuffer) { + this.data = new DataView(b); + } else { + // TODO: adjust read/write offset based on the type of view + // or specify that this must be done in the options ... that the + // offsets are byte-based + this.data = new DataView(b.buffer, b.byteOffset, b.byteLength); + } + this.write = ('writeOffset' in options ? + options.writeOffset : this.data.byteLength); + return; + } + + // initialize to empty array buffer and add any given bytes using putBytes + this.data = new DataView(new ArrayBuffer(0)); + this.write = 0; + + if(b !== null && b !== undefined) { + this.putBytes(b); + } + + if('writeOffset' in options) { + this.write = options.writeOffset; + } +} +util.DataBuffer = DataBuffer; + +/** + * Gets the number of bytes in this buffer. + * + * @return the number of bytes in this buffer. + */ +util.DataBuffer.prototype.length = function() { + return this.write - this.read; +}; + +/** + * Gets whether or not this buffer is empty. + * + * @return true if this buffer is empty, false if not. + */ +util.DataBuffer.prototype.isEmpty = function() { + return this.length() <= 0; +}; + +/** + * Ensures this buffer has enough empty space to accommodate the given number + * of bytes. An optional parameter may be given that indicates a minimum + * amount to grow the buffer if necessary. If the parameter is not given, + * the buffer will be grown by some previously-specified default amount + * or heuristic. + * + * @param amount the number of bytes to accommodate. + * @param [growSize] the minimum amount, in bytes, to grow the buffer by if + * necessary. + */ +util.DataBuffer.prototype.accommodate = function(amount, growSize) { + if(this.length() >= amount) { + return this; + } + growSize = Math.max(growSize || this.growSize, amount); + + // grow buffer + var src = new Uint8Array( + this.data.buffer, this.data.byteOffset, this.data.byteLength); + var dst = new Uint8Array(this.length() + growSize); + dst.set(src); + this.data = new DataView(dst.buffer); + + return this; +}; + +/** + * Puts a byte in this buffer. + * + * @param b the byte to put. + * + * @return this buffer. + */ +util.DataBuffer.prototype.putByte = function(b) { + this.accommodate(1); + this.data.setUint8(this.write++, b); + return this; +}; + +/** + * Puts a byte in this buffer N times. + * + * @param b the byte to put. + * @param n the number of bytes of value b to put. + * + * @return this buffer. + */ +util.DataBuffer.prototype.fillWithByte = function(b, n) { + this.accommodate(n); + for(var i = 0; i < n; ++i) { + this.data.setUint8(b); + } + return this; +}; + +/** + * Puts bytes in this buffer. The bytes may be given as a string, an + * ArrayBuffer, a DataView, or a TypedArray. + * + * @param bytes the bytes to put. + * @param [encoding] the encoding for the first parameter ('binary', 'utf8', + * 'utf16', 'hex'), if it is a string (default: 'binary'). + * + * @return this buffer. + */ +util.DataBuffer.prototype.putBytes = function(bytes, encoding) { + if(util.isArrayBufferView(bytes)) { + var src = new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength); + var len = src.byteLength - src.byteOffset; + this.accommodate(len); + var dst = new Uint8Array(this.data.buffer, this.write); + dst.set(src); + this.write += len; + return this; + } + + if(util.isArrayBuffer(bytes)) { + var src = new Uint8Array(bytes); + this.accommodate(src.byteLength); + var dst = new Uint8Array(this.data.buffer); + dst.set(src, this.write); + this.write += src.byteLength; + return this; + } + + // bytes is a util.DataBuffer or equivalent + if(bytes instanceof util.DataBuffer || + (typeof bytes === 'object' && + typeof bytes.read === 'number' && typeof bytes.write === 'number' && + util.isArrayBufferView(bytes.data))) { + var src = new Uint8Array(bytes.data.byteLength, bytes.read, bytes.length()); + this.accommodate(src.byteLength); + var dst = new Uint8Array(bytes.data.byteLength, this.write); + dst.set(src); + this.write += src.byteLength; + return this; + } + + if(bytes instanceof util.ByteStringBuffer) { + // copy binary string and process as the same as a string parameter below + bytes = bytes.data; + encoding = 'binary'; + } + + // string conversion + encoding = encoding || 'binary'; + if(typeof bytes === 'string') { + var view; + + // decode from string + if(encoding === 'hex') { + this.accommodate(Math.ceil(bytes.length / 2)); + view = new Uint8Array(this.data.buffer, this.write); + this.write += util.binary.hex.decode(bytes, view, this.write); + return this; + } + if(encoding === 'base64') { + this.accommodate(Math.ceil(bytes.length / 4) * 3); + view = new Uint8Array(this.data.buffer, this.write); + this.write += util.binary.base64.decode(bytes, view, this.write); + return this; + } + + // encode text as UTF-8 bytes + if(encoding === 'utf8') { + // encode as UTF-8 then decode string as raw binary + bytes = util.encodeUtf8(bytes); + encoding = 'binary'; + } + + // decode string as raw binary + if(encoding === 'binary' || encoding === 'raw') { + // one byte per character + this.accommodate(bytes.length); + view = new Uint8Array(this.data.buffer, this.write); + this.write += util.binary.raw.decode(view); + return this; + } + + // encode text as UTF-16 bytes + if(encoding === 'utf16') { + // two bytes per character + this.accommodate(bytes.length * 2); + view = new Uint16Array(this.data.buffer, this.write); + this.write += util.text.utf16.encode(view); + return this; + } + + throw new Error('Invalid encoding: ' + encoding); + } + + throw Error('Invalid parameter: ' + bytes); +}; + +/** + * Puts the given buffer into this buffer. + * + * @param buffer the buffer to put into this one. + * + * @return this buffer. + */ +util.DataBuffer.prototype.putBuffer = function(buffer) { + this.putBytes(buffer); + buffer.clear(); + return this; +}; + +/** + * Puts a string into this buffer. + * + * @param str the string to put. + * @param [encoding] the encoding for the string (default: 'utf16'). + * + * @return this buffer. + */ +util.DataBuffer.prototype.putString = function(str) { + return this.putBytes(str, 'utf16'); +}; + +/** + * Puts a 16-bit integer in this buffer in big-endian order. + * + * @param i the 16-bit integer. + * + * @return this buffer. + */ +util.DataBuffer.prototype.putInt16 = function(i) { + this.accommodate(2); + this.data.setInt16(this.write, i); + this.write += 2; + return this; +}; + +/** + * Puts a 24-bit integer in this buffer in big-endian order. + * + * @param i the 24-bit integer. + * + * @return this buffer. + */ +util.DataBuffer.prototype.putInt24 = function(i) { + this.accommodate(3); + this.data.setInt16(this.write, i >> 8 & 0xFFFF); + this.data.setInt8(this.write, i >> 16 & 0xFF); + this.write += 3; + return this; +}; + +/** + * Puts a 32-bit integer in this buffer in big-endian order. + * + * @param i the 32-bit integer. + * + * @return this buffer. + */ +util.DataBuffer.prototype.putInt32 = function(i) { + this.accommodate(4); + this.data.setInt32(this.write, i); + this.write += 4; + return this; +}; + +/** + * Puts a 16-bit integer in this buffer in little-endian order. + * + * @param i the 16-bit integer. + * + * @return this buffer. + */ +util.DataBuffer.prototype.putInt16Le = function(i) { + this.accommodate(2); + this.data.setInt16(this.write, i, true); + this.write += 2; + return this; +}; + +/** + * Puts a 24-bit integer in this buffer in little-endian order. + * + * @param i the 24-bit integer. + * + * @return this buffer. + */ +util.DataBuffer.prototype.putInt24Le = function(i) { + this.accommodate(3); + this.data.setInt8(this.write, i >> 16 & 0xFF); + this.data.setInt16(this.write, i >> 8 & 0xFFFF, true); + this.write += 3; + return this; +}; + +/** + * Puts a 32-bit integer in this buffer in little-endian order. + * + * @param i the 32-bit integer. + * + * @return this buffer. + */ +util.DataBuffer.prototype.putInt32Le = function(i) { + this.accommodate(4); + this.data.setInt32(this.write, i, true); + this.write += 4; + return this; +}; + +/** + * Puts an n-bit integer in this buffer in big-endian order. + * + * @param i the n-bit integer. + * @param n the number of bits in the integer. + * + * @return this buffer. + */ +util.DataBuffer.prototype.putInt = function(i, n) { + this.accommodate(n / 8); + do { + n -= 8; + this.data.setInt8(this.write++, (i >> n) & 0xFF); + } while(n > 0); + return this; +}; + +/** + * Puts a signed n-bit integer in this buffer in big-endian order. Two's + * complement representation is used. + * + * @param i the n-bit integer. + * @param n the number of bits in the integer. + * + * @return this buffer. + */ +util.DataBuffer.prototype.putSignedInt = function(i, n) { + this.accommodate(n / 8); + if(i < 0) { + i += 2 << (n - 1); + } + return this.putInt(i, n); +}; + +/** + * Gets a byte from this buffer and advances the read pointer by 1. + * + * @return the byte. + */ +util.DataBuffer.prototype.getByte = function() { + return this.data.getInt8(this.read++); +}; + +/** + * Gets a uint16 from this buffer in big-endian order and advances the read + * pointer by 2. + * + * @return the uint16. + */ +util.DataBuffer.prototype.getInt16 = function() { + var rval = this.data.getInt16(this.read); + this.read += 2; + return rval; +}; + +/** + * Gets a uint24 from this buffer in big-endian order and advances the read + * pointer by 3. + * + * @return the uint24. + */ +util.DataBuffer.prototype.getInt24 = function() { + var rval = ( + this.data.getInt16(this.read) << 8 ^ + this.data.getInt8(this.read + 2)); + this.read += 3; + return rval; +}; + +/** + * Gets a uint32 from this buffer in big-endian order and advances the read + * pointer by 4. + * + * @return the word. + */ +util.DataBuffer.prototype.getInt32 = function() { + var rval = this.data.getInt32(this.read); + this.read += 4; + return rval; +}; + +/** + * Gets a uint16 from this buffer in little-endian order and advances the read + * pointer by 2. + * + * @return the uint16. + */ +util.DataBuffer.prototype.getInt16Le = function() { + var rval = this.data.getInt16(this.read, true); + this.read += 2; + return rval; +}; + +/** + * Gets a uint24 from this buffer in little-endian order and advances the read + * pointer by 3. + * + * @return the uint24. + */ +util.DataBuffer.prototype.getInt24Le = function() { + var rval = ( + this.data.getInt8(this.read) ^ + this.data.getInt16(this.read + 1, true) << 8); + this.read += 3; + return rval; +}; + +/** + * Gets a uint32 from this buffer in little-endian order and advances the read + * pointer by 4. + * + * @return the word. + */ +util.DataBuffer.prototype.getInt32Le = function() { + var rval = this.data.getInt32(this.read, true); + this.read += 4; + return rval; +}; + +/** + * Gets an n-bit integer from this buffer in big-endian order and advances the + * read pointer by n/8. + * + * @param n the number of bits in the integer. + * + * @return the integer. + */ +util.DataBuffer.prototype.getInt = function(n) { + var rval = 0; + do { + rval = (rval << 8) + this.data.getInt8(this.read++); + n -= 8; + } while(n > 0); + return rval; +}; + +/** + * Gets a signed n-bit integer from this buffer in big-endian order, using + * two's complement, and advances the read pointer by n/8. + * + * @param n the number of bits in the integer. + * + * @return the integer. + */ +util.DataBuffer.prototype.getSignedInt = function(n) { + var x = this.getInt(n); + var max = 2 << (n - 2); + if(x >= max) { + x -= max << 1; + } + return x; +}; + +/** + * Reads bytes out into a UTF-8 string and clears them from the buffer. + * + * @param count the number of bytes to read, undefined or null for all. + * + * @return a UTF-8 string of bytes. + */ +util.DataBuffer.prototype.getBytes = function(count) { + // TODO: deprecate this method, it is poorly named and + // this.toString('binary') replaces it + // add a toTypedArray()/toArrayBuffer() function + var rval; + if(count) { + // read count bytes + count = Math.min(this.length(), count); + rval = this.data.slice(this.read, this.read + count); + this.read += count; + } else if(count === 0) { + rval = ''; + } else { + // read all bytes, optimize to only copy when needed + rval = (this.read === 0) ? this.data : this.data.slice(this.read); + this.clear(); + } + return rval; +}; + +/** + * Gets a UTF-8 encoded string of the bytes from this buffer without modifying + * the read pointer. + * + * @param count the number of bytes to get, omit to get all. + * + * @return a string full of UTF-8 encoded characters. + */ +util.DataBuffer.prototype.bytes = function(count) { + // TODO: deprecate this method, it is poorly named, add "getString()" + return (typeof(count) === 'undefined' ? + this.data.slice(this.read) : + this.data.slice(this.read, this.read + count)); +}; + +/** + * Gets a byte at the given index without modifying the read pointer. + * + * @param i the byte index. + * + * @return the byte. + */ +util.DataBuffer.prototype.at = function(i) { + return this.data.getUint8(this.read + i); +}; + +/** + * Puts a byte at the given index without modifying the read pointer. + * + * @param i the byte index. + * @param b the byte to put. + * + * @return this buffer. + */ +util.DataBuffer.prototype.setAt = function(i, b) { + this.data.setUint8(i, b); + return this; +}; + +/** + * Gets the last byte without modifying the read pointer. + * + * @return the last byte. + */ +util.DataBuffer.prototype.last = function() { + return this.data.getUint8(this.write - 1); +}; + +/** + * Creates a copy of this buffer. + * + * @return the copy. + */ +util.DataBuffer.prototype.copy = function() { + return new util.DataBuffer(this); +}; + +/** + * Compacts this buffer. + * + * @return this buffer. + */ +util.DataBuffer.prototype.compact = function() { + if(this.read > 0) { + var src = new Uint8Array(this.data.buffer, this.read); + var dst = new Uint8Array(src.byteLength); + dst.set(src); + this.data = new DataView(dst); + this.write -= this.read; + this.read = 0; + } + return this; +}; + +/** + * Clears this buffer. + * + * @return this buffer. + */ +util.DataBuffer.prototype.clear = function() { + this.data = new DataView(new ArrayBuffer(0)); + this.read = this.write = 0; + return this; +}; + +/** + * Shortens this buffer by triming bytes off of the end of this buffer. + * + * @param count the number of bytes to trim off. + * + * @return this buffer. + */ +util.DataBuffer.prototype.truncate = function(count) { + this.write = Math.max(0, this.length() - count); + this.read = Math.min(this.read, this.write); + return this; +}; + +/** + * Converts this buffer to a hexadecimal string. + * + * @return a hexadecimal string. + */ +util.DataBuffer.prototype.toHex = function() { + var rval = ''; + for(var i = this.read; i < this.data.byteLength; ++i) { + var b = this.data.getUint8(i); + if(b < 16) { + rval += '0'; + } + rval += b.toString(16); + } + return rval; +}; + +/** + * Converts this buffer to a string, using the given encoding. If no + * encoding is given, 'utf8' (UTF-8) is used. + * + * @param [encoding] the encoding to use: 'binary', 'utf8', 'utf16', 'hex', + * 'base64' (default: 'utf8'). + * + * @return a string representation of the bytes in this buffer. + */ +util.DataBuffer.prototype.toString = function(encoding) { + var view = new Uint8Array(this.data, this.read, this.length()); + encoding = encoding || 'utf8'; + + // encode to string + if(encoding === 'binary' || encoding === 'raw') { + return util.binary.raw.encode(view); + } + if(encoding === 'hex') { + return util.binary.hex.encode(view); + } + if(encoding === 'base64') { + return util.binary.base64.encode(view); + } + + // decode to text + if(encoding === 'utf8') { + return util.text.utf8.decode(view); + } + if(encoding === 'utf16') { + return util.text.utf16.decode(view); + } + + throw new Error('Invalid encoding: ' + encoding); +}; + +/** End Buffer w/UInt8Array backing */ + + +/** + * Creates a buffer that stores bytes. A value may be given to put into the + * buffer that is either a string of bytes or a UTF-16 string that will + * be encoded using UTF-8 (to do the latter, specify 'utf8' as the encoding). + * + * @param [input] the bytes to wrap (as a string) or a UTF-16 string to encode + * as UTF-8. + * @param [encoding] (default: 'raw', other: 'utf8'). + */ +util.createBuffer = function(input, encoding) { + // TODO: deprecate, use new ByteBuffer() instead + encoding = encoding || 'raw'; + if(input !== undefined && encoding === 'utf8') { + input = util.encodeUtf8(input); + } + return new util.ByteBuffer(input); +}; + +/** + * Fills a string with a particular value. If you want the string to be a byte + * string, pass in String.fromCharCode(theByte). + * + * @param c the character to fill the string with, use String.fromCharCode + * to fill the string with a byte value. + * @param n the number of characters of value c to fill with. + * + * @return the filled string. + */ +util.fillString = function(c, n) { + var s = ''; + while(n > 0) { + if(n & 1) { + s += c; + } + n >>>= 1; + if(n > 0) { + c += c; + } + } + return s; +}; + +/** + * Performs a per byte XOR between two byte strings and returns the result as a + * string of bytes. + * + * @param s1 first string of bytes. + * @param s2 second string of bytes. + * @param n the number of bytes to XOR. + * + * @return the XOR'd result. + */ +util.xorBytes = function(s1, s2, n) { + var s3 = ''; + var b = ''; + var t = ''; + var i = 0; + var c = 0; + for(; n > 0; --n, ++i) { + b = s1.charCodeAt(i) ^ s2.charCodeAt(i); + if(c >= 10) { + s3 += t; + t = ''; + c = 0; + } + t += String.fromCharCode(b); + ++c; + } + s3 += t; + return s3; +}; + +/** + * Converts a hex string into a 'binary' encoded string of bytes. + * + * @param hex the hexadecimal string to convert. + * + * @return the binary-encoded string of bytes. + */ +util.hexToBytes = function(hex) { + // TODO: deprecate: "Deprecated. Use util.binary.hex.decode instead." + var rval = ''; + var i = 0; + if(hex.length & 1 == 1) { + // odd number of characters, convert first character alone + i = 1; + rval += String.fromCharCode(parseInt(hex[0], 16)); + } + // convert 2 characters (1 byte) at a time + for(; i < hex.length; i += 2) { + rval += String.fromCharCode(parseInt(hex.substr(i, 2), 16)); + } + return rval; +}; + +/** + * Converts a 'binary' encoded string of bytes to hex. + * + * @param bytes the byte string to convert. + * + * @return the string of hexadecimal characters. + */ +util.bytesToHex = function(bytes) { + // TODO: deprecate: "Deprecated. Use util.binary.hex.encode instead." + return util.createBuffer(bytes).toHex(); +}; + +/** + * Converts an 32-bit integer to 4-big-endian byte string. + * + * @param i the integer. + * + * @return the byte string. + */ +util.int32ToBytes = function(i) { + return ( + String.fromCharCode(i >> 24 & 0xFF) + + String.fromCharCode(i >> 16 & 0xFF) + + String.fromCharCode(i >> 8 & 0xFF) + + String.fromCharCode(i & 0xFF)); +}; + +// base64 characters, reverse mapping +var _base64 = + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='; +var _base64Idx = [ +/*43 -43 = 0*/ +/*'+', 1, 2, 3,'/' */ + 62, -1, -1, -1, 63, + +/*'0','1','2','3','4','5','6','7','8','9' */ + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + +/*15, 16, 17,'=', 19, 20, 21 */ + -1, -1, -1, 64, -1, -1, -1, + +/*65 - 43 = 22*/ +/*'A','B','C','D','E','F','G','H','I','J','K','L','M', */ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + +/*'N','O','P','Q','R','S','T','U','V','W','X','Y','Z' */ + 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + +/*91 - 43 = 48 */ +/*48, 49, 50, 51, 52, 53 */ + -1, -1, -1, -1, -1, -1, + +/*97 - 43 = 54*/ +/*'a','b','c','d','e','f','g','h','i','j','k','l','m' */ + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + +/*'n','o','p','q','r','s','t','u','v','w','x','y','z' */ + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 +]; + +/** + * Base64 encodes a 'binary' encoded string of bytes. + * + * @param input the binary encoded string of bytes to base64-encode. + * @param maxline the maximum number of encoded characters per line to use, + * defaults to none. + * + * @return the base64-encoded output. + */ +util.encode64 = function(input, maxline) { + // TODO: deprecate: "Deprecated. Use util.binary.base64.encode instead." + var line = ''; + var output = ''; + var chr1, chr2, chr3; + var i = 0; + while(i < input.length) { + chr1 = input.charCodeAt(i++); + chr2 = input.charCodeAt(i++); + chr3 = input.charCodeAt(i++); + + // encode 4 character group + line += _base64.charAt(chr1 >> 2); + line += _base64.charAt(((chr1 & 3) << 4) | (chr2 >> 4)); + if(isNaN(chr2)) { + line += '=='; + } else { + line += _base64.charAt(((chr2 & 15) << 2) | (chr3 >> 6)); + line += isNaN(chr3) ? '=' : _base64.charAt(chr3 & 63); + } + + if(maxline && line.length > maxline) { + output += line.substr(0, maxline) + '\r\n'; + line = line.substr(maxline); + } + } + output += line; + return output; +}; + +/** + * Base64 decodes a string into a 'binary' encoded string of bytes. + * + * @param input the base64-encoded input. + * + * @return the binary encoded string. + */ +util.decode64 = function(input) { + // TODO: deprecate: "Deprecated. Use util.binary.base64.decode instead." + + // remove all non-base64 characters + input = input.replace(/[^A-Za-z0-9\+\/\=]/g, ''); + + var output = ''; + var enc1, enc2, enc3, enc4; + var i = 0; + + while(i < input.length) { + enc1 = _base64Idx[input.charCodeAt(i++) - 43]; + enc2 = _base64Idx[input.charCodeAt(i++) - 43]; + enc3 = _base64Idx[input.charCodeAt(i++) - 43]; + enc4 = _base64Idx[input.charCodeAt(i++) - 43]; + + output += String.fromCharCode((enc1 << 2) | (enc2 >> 4)); + if(enc3 !== 64) { + // decoded at least 2 bytes + output += String.fromCharCode(((enc2 & 15) << 4) | (enc3 >> 2)); + if(enc4 !== 64) { + // decoded 3 bytes + output += String.fromCharCode(((enc3 & 3) << 6) | enc4); + } + } + } + + return output; +}; + +/** + * UTF-8 encodes the given UTF-16 encoded string (a standard JavaScript + * string). Non-ASCII characters will be encoded as multiple bytes according + * to UTF-8. + * + * @param str the string to encode. + * + * @return the UTF-8 encoded string. + */ +util.encodeUtf8 = function(str) { + return unescape(encodeURIComponent(str)); +}; + +/** + * Decodes a UTF-8 encoded string into a UTF-16 string. + * + * @param str the string to decode. + * + * @return the UTF-16 encoded string (standard JavaScript string). + */ +util.decodeUtf8 = function(str) { + return decodeURIComponent(escape(str)); +}; + +// binary encoding/decoding tools +// FIXME: Experimental. Do not use yet. +util.binary = { + raw: {}, + hex: {}, + base64: {} +}; + +/** + * Encodes a Uint8Array as a binary-encoded string. This encoding uses + * a value between 0 and 255 for each character. + * + * @param bytes the Uint8Array to encode. + * + * @return the binary-encoded string. + */ +util.binary.raw.encode = function(bytes) { + return String.fromCharCode.apply(null, bytes); +}; + +/** + * Decodes a binary-encoded string to a Uint8Array. This encoding uses + * a value between 0 and 255 for each character. + * + * @param str the binary-encoded string to decode. + * @param [output] an optional Uint8Array to write the output to; if it + * is too small, an exception will be thrown. + * @param [offset] the start offset for writing to the output (default: 0). + * + * @return the Uint8Array or the number of bytes written if output was given. + */ +util.binary.raw.decode = function(str, output, offset) { + var out = output; + if(!out) { + out = new Uint8Array(str.length); + } + offset = offset || 0; + var j = offset; + for(var i = 0; i < str.length; ++i) { + out[j++] = str.charCodeAt(i); + } + return output ? (j - offset) : out; +}; + +/** + * Encodes a 'binary' string, ArrayBuffer, DataView, TypedArray, or + * ByteBuffer as a string of hexadecimal characters. + * + * @param bytes the bytes to convert. + * + * @return the string of hexadecimal characters. + */ +util.binary.hex.encode = util.bytesToHex; + +/** + * Decodes a hex-encoded string to a Uint8Array. + * + * @param hex the hexadecimal string to convert. + * @param [output] an optional Uint8Array to write the output to; if it + * is too small, an exception will be thrown. + * @param [offset] the start offset for writing to the output (default: 0). + * + * @return the Uint8Array or the number of bytes written if output was given. + */ +util.binary.hex.decode = function(hex, output, offset) { + var out = output; + if(!out) { + out = new Uint8Array(Math.ceil(hex.length / 2)); + } + offset = offset || 0; + var i = 0, j = offset; + if(hex.length & 1) { + // odd number of characters, convert first character alone + i = 1; + out[j++] = parseInt(hex[0], 16); + } + // convert 2 characters (1 byte) at a time + for(; i < hex.length; i += 2) { + out[j++] = parseInt(hex.substr(i, 2), 16); + } + return output ? (j - offset) : out; +}; + +/** + * Base64-encodes a Uint8Array. + * + * @param input the Uint8Array to encode. + * @param maxline the maximum number of encoded characters per line to use, + * defaults to none. + * + * @return the base64-encoded output string. + */ +util.binary.base64.encode = function(input, maxline) { + var line = ''; + var output = ''; + var chr1, chr2, chr3; + var i = 0; + while(i < input.byteLength) { + chr1 = input[i++]; + chr2 = input[i++]; + chr3 = input[i++]; + + // encode 4 character group + line += _base64.charAt(chr1 >> 2); + line += _base64.charAt(((chr1 & 3) << 4) | (chr2 >> 4)); + if(isNaN(chr2)) { + line += '=='; + } else { + line += _base64.charAt(((chr2 & 15) << 2) | (chr3 >> 6)); + line += isNaN(chr3) ? '=' : _base64.charAt(chr3 & 63); + } + + if(maxline && line.length > maxline) { + output += line.substr(0, maxline) + '\r\n'; + line = line.substr(maxline); + } + } + output += line; + return output; +}; + +/** + * Decodes a base64-encoded string to a Uint8Array. + * + * @param input the base64-encoded input string. + * @param [output] an optional Uint8Array to write the output to; if it + * is too small, an exception will be thrown. + * @param [offset] the start offset for writing to the output (default: 0). + * + * @return the Uint8Array or the number of bytes written if output was given. + */ +util.binary.base64.decode = function(input, output, offset) { + var out = output; + if(!out) { + out = new Uint8Array(Math.ceil(input.length / 4) * 3); + } + + // remove all non-base64 characters + input = input.replace(/[^A-Za-z0-9\+\/\=]/g, ''); + + offset = offset || 0; + var enc1, enc2, enc3, enc4; + var i = 0, j = offset; + + while(i < input.length) { + enc1 = _base64Idx[input.charCodeAt(i++) - 43]; + enc2 = _base64Idx[input.charCodeAt(i++) - 43]; + enc3 = _base64Idx[input.charCodeAt(i++) - 43]; + enc4 = _base64Idx[input.charCodeAt(i++) - 43]; + + out[j++] = (enc1 << 2) | (enc2 >> 4); + if(enc3 !== 64) { + // decoded at least 2 bytes + out[j++] = ((enc2 & 15) << 4) | (enc3 >> 2); + if(enc4 !== 64) { + // decoded 3 bytes + out[j++] = ((enc3 & 3) << 6) | enc4; + } + } + } + + // make sure result is the exact decoded length + return output ? + (j - offset) : + out.subarray(0, j); +}; + +// text encoding/decoding tools +// FIXME: Experimental. Do not use yet. +util.text = { + utf8: {}, + utf16: {} +}; + +/** + * Encodes the given string as UTF-8 in a Uint8Array. + * + * @param str the string to encode. + * @param [output] an optional Uint8Array to write the output to; if it + * is too small, an exception will be thrown. + * @param [offset] the start offset for writing to the output (default: 0). + * + * @return the Uint8Array or the number of bytes written if output was given. + */ +util.text.utf8.encode = function(str, output, offset) { + str = util.encodeUtf8(str); + var out = output; + if(!out) { + out = new Uint8Array(str.length); + } + offset = offset || 0; + var j = offset; + for(var i = 0; i < str.length; ++i) { + out[j++] = str.charCodeAt(i); + } + return output ? (j - offset) : out; +}; + +/** + * Decodes the UTF-8 contents from a Uint8Array. + * + * @param bytes the Uint8Array to decode. + * + * @return the resulting string. + */ +util.text.utf8.decode = function(bytes) { + return util.decodeUtf8(String.fromCharCode.apply(null, bytes)); +}; + +/** + * Encodes the given string as UTF-16 in a Uint8Array. + * + * @param str the string to encode. + * @param [output] an optional Uint8Array to write the output to; if it + * is too small, an exception will be thrown. + * @param [offset] the start offset for writing to the output (default: 0). + * + * @return the Uint8Array or the number of bytes written if output was given. + */ +util.text.utf16.encode = function(str, output, offset) { + var out = output; + if(!out) { + out = new Uint8Array(str.length * 2); + } + var view = new Uint16Array(out.buffer); + offset = offset || 0; + var j = offset; + var k = offset; + for(var i = 0; i < str.length; ++i) { + view[k++] = str.charCodeAt(i); + j += 2; + } + return output ? (j - offset) : out; +}; + +/** + * Decodes the UTF-16 contents from a Uint8Array. + * + * @param bytes the Uint8Array to decode. + * + * @return the resulting string. + */ +util.text.utf16.decode = function(bytes) { + return String.fromCharCode.apply(null, new Uint16Array(bytes.buffer)); +}; + +/** + * Deflates the given data using a flash interface. + * + * @param api the flash interface. + * @param bytes the data. + * @param raw true to return only raw deflate data, false to include zlib + * header and trailer. + * + * @return the deflated data as a string. + */ +util.deflate = function(api, bytes, raw) { + bytes = util.decode64(api.deflate(util.encode64(bytes)).rval); + + // strip zlib header and trailer if necessary + if(raw) { + // zlib header is 2 bytes (CMF,FLG) where FLG indicates that + // there is a 4-byte DICT (alder-32) block before the data if + // its 5th bit is set + var start = 2; + var flg = bytes.charCodeAt(1); + if(flg & 0x20) { + start = 6; + } + // zlib trailer is 4 bytes of adler-32 + bytes = bytes.substring(start, bytes.length - 4); + } + + return bytes; +}; + +/** + * Inflates the given data using a flash interface. + * + * @param api the flash interface. + * @param bytes the data. + * @param raw true if the incoming data has no zlib header or trailer and is + * raw DEFLATE data. + * + * @return the inflated data as a string, null on error. + */ +util.inflate = function(api, bytes, raw) { + // TODO: add zlib header and trailer if necessary/possible + var rval = api.inflate(util.encode64(bytes)).rval; + return (rval === null) ? null : util.decode64(rval); +}; + +/** + * Sets a storage object. + * + * @param api the storage interface. + * @param id the storage ID to use. + * @param obj the storage object, null to remove. + */ +var _setStorageObject = function(api, id, obj) { + if(!api) { + throw new Error('WebStorage not available.'); + } + + var rval; + if(obj === null) { + rval = api.removeItem(id); + } else { + // json-encode and base64-encode object + obj = util.encode64(JSON.stringify(obj)); + rval = api.setItem(id, obj); + } + + // handle potential flash error + if(typeof(rval) !== 'undefined' && rval.rval !== true) { + var error = new Error(rval.error.message); + error.id = rval.error.id; + error.name = rval.error.name; + throw error; + } +}; + +/** + * Gets a storage object. + * + * @param api the storage interface. + * @param id the storage ID to use. + * + * @return the storage object entry or null if none exists. + */ +var _getStorageObject = function(api, id) { + if(!api) { + throw new Error('WebStorage not available.'); + } + + // get the existing entry + var rval = api.getItem(id); + + /* Note: We check api.init because we can't do (api == localStorage) + on IE because of "Class doesn't support Automation" exception. Only + the flash api has an init method so this works too, but we need a + better solution in the future. */ + + // flash returns item wrapped in an object, handle special case + if(api.init) { + if(rval.rval === null) { + if(rval.error) { + var error = new Error(rval.error.message); + error.id = rval.error.id; + error.name = rval.error.name; + throw error; + } + // no error, but also no item + rval = null; + } else { + rval = rval.rval; + } + } + + // handle decoding + if(rval !== null) { + // base64-decode and json-decode data + rval = JSON.parse(util.decode64(rval)); + } + + return rval; +}; + +/** + * Stores an item in local storage. + * + * @param api the storage interface. + * @param id the storage ID to use. + * @param key the key for the item. + * @param data the data for the item (any javascript object/primitive). + */ +var _setItem = function(api, id, key, data) { + // get storage object + var obj = _getStorageObject(api, id); + if(obj === null) { + // create a new storage object + obj = {}; + } + // update key + obj[key] = data; + + // set storage object + _setStorageObject(api, id, obj); +}; + +/** + * Gets an item from local storage. + * + * @param api the storage interface. + * @param id the storage ID to use. + * @param key the key for the item. + * + * @return the item. + */ +var _getItem = function(api, id, key) { + // get storage object + var rval = _getStorageObject(api, id); + if(rval !== null) { + // return data at key + rval = (key in rval) ? rval[key] : null; + } + + return rval; +}; + +/** + * Removes an item from local storage. + * + * @param api the storage interface. + * @param id the storage ID to use. + * @param key the key for the item. + */ +var _removeItem = function(api, id, key) { + // get storage object + var obj = _getStorageObject(api, id); + if(obj !== null && key in obj) { + // remove key + delete obj[key]; + + // see if entry has no keys remaining + var empty = true; + for(var prop in obj) { + empty = false; + break; + } + if(empty) { + // remove entry entirely if no keys are left + obj = null; + } + + // set storage object + _setStorageObject(api, id, obj); + } +}; + +/** + * Clears the local disk storage identified by the given ID. + * + * @param api the storage interface. + * @param id the storage ID to use. + */ +var _clearItems = function(api, id) { + _setStorageObject(api, id, null); +}; + +/** + * Calls a storage function. + * + * @param func the function to call. + * @param args the arguments for the function. + * @param location the location argument. + * + * @return the return value from the function. + */ +var _callStorageFunction = function(func, args, location) { + var rval = null; + + // default storage types + if(typeof(location) === 'undefined') { + location = ['web', 'flash']; + } + + // apply storage types in order of preference + var type; + var done = false; + var exception = null; + for(var idx in location) { + type = location[idx]; + try { + if(type === 'flash' || type === 'both') { + if(args[0] === null) { + throw new Error('Flash local storage not available.'); + } + rval = func.apply(this, args); + done = (type === 'flash'); + } + if(type === 'web' || type === 'both') { + args[0] = localStorage; + rval = func.apply(this, args); + done = true; + } + } catch(ex) { + exception = ex; + } + if(done) { + break; + } + } + + if(!done) { + throw exception; + } + + return rval; +}; + +/** + * Stores an item on local disk. + * + * The available types of local storage include 'flash', 'web', and 'both'. + * + * The type 'flash' refers to flash local storage (SharedObject). In order + * to use flash local storage, the 'api' parameter must be valid. The type + * 'web' refers to WebStorage, if supported by the browser. The type 'both' + * refers to storing using both 'flash' and 'web', not just one or the + * other. + * + * The location array should list the storage types to use in order of + * preference: + * + * ['flash']: flash only storage + * ['web']: web only storage + * ['both']: try to store in both + * ['flash','web']: store in flash first, but if not available, 'web' + * ['web','flash']: store in web first, but if not available, 'flash' + * + * The location array defaults to: ['web', 'flash'] + * + * @param api the flash interface, null to use only WebStorage. + * @param id the storage ID to use. + * @param key the key for the item. + * @param data the data for the item (any javascript object/primitive). + * @param location an array with the preferred types of storage to use. + */ +util.setItem = function(api, id, key, data, location) { + _callStorageFunction(_setItem, arguments, location); +}; + +/** + * Gets an item on local disk. + * + * Set setItem() for details on storage types. + * + * @param api the flash interface, null to use only WebStorage. + * @param id the storage ID to use. + * @param key the key for the item. + * @param location an array with the preferred types of storage to use. + * + * @return the item. + */ +util.getItem = function(api, id, key, location) { + return _callStorageFunction(_getItem, arguments, location); +}; + +/** + * Removes an item on local disk. + * + * Set setItem() for details on storage types. + * + * @param api the flash interface. + * @param id the storage ID to use. + * @param key the key for the item. + * @param location an array with the preferred types of storage to use. + */ +util.removeItem = function(api, id, key, location) { + _callStorageFunction(_removeItem, arguments, location); +}; + +/** + * Clears the local disk storage identified by the given ID. + * + * Set setItem() for details on storage types. + * + * @param api the flash interface if flash is available. + * @param id the storage ID to use. + * @param location an array with the preferred types of storage to use. + */ +util.clearItems = function(api, id, location) { + _callStorageFunction(_clearItems, arguments, location); +}; + +/** + * Parses the scheme, host, and port from an http(s) url. + * + * @param str the url string. + * + * @return the parsed url object or null if the url is invalid. + */ +util.parseUrl = function(str) { + // FIXME: this regex looks a bit broken + var regex = /^(https?):\/\/([^:&^\/]*):?(\d*)(.*)$/g; + regex.lastIndex = 0; + var m = regex.exec(str); + var url = (m === null) ? null : { + full: str, + scheme: m[1], + host: m[2], + port: m[3], + path: m[4] + }; + if(url) { + url.fullHost = url.host; + if(url.port) { + if(url.port !== 80 && url.scheme === 'http') { + url.fullHost += ':' + url.port; + } else if(url.port !== 443 && url.scheme === 'https') { + url.fullHost += ':' + url.port; + } + } else if(url.scheme === 'http') { + url.port = 80; + } else if(url.scheme === 'https') { + url.port = 443; + } + url.full = url.scheme + '://' + url.fullHost; + } + return url; +}; + +/* Storage for query variables */ +var _queryVariables = null; + +/** + * Returns the window location query variables. Query is parsed on the first + * call and the same object is returned on subsequent calls. The mapping + * is from keys to an array of values. Parameters without values will have + * an object key set but no value added to the value array. Values are + * unescaped. + * + * ...?k1=v1&k2=v2: + * { + * "k1": ["v1"], + * "k2": ["v2"] + * } + * + * ...?k1=v1&k1=v2: + * { + * "k1": ["v1", "v2"] + * } + * + * ...?k1=v1&k2: + * { + * "k1": ["v1"], + * "k2": [] + * } + * + * ...?k1=v1&k1: + * { + * "k1": ["v1"] + * } + * + * ...?k1&k1: + * { + * "k1": [] + * } + * + * @param query the query string to parse (optional, default to cached + * results from parsing window location search query). + * + * @return object mapping keys to variables. + */ +util.getQueryVariables = function(query) { + var parse = function(q) { + var rval = {}; + var kvpairs = q.split('&'); + for(var i = 0; i < kvpairs.length; i++) { + var pos = kvpairs[i].indexOf('='); + var key; + var val; + if(pos > 0) { + key = kvpairs[i].substring(0, pos); + val = kvpairs[i].substring(pos + 1); + } else { + key = kvpairs[i]; + val = null; + } + if(!(key in rval)) { + rval[key] = []; + } + // disallow overriding object prototype keys + if(!(key in Object.prototype) && val !== null) { + rval[key].push(unescape(val)); + } + } + return rval; + }; + + var rval; + if(typeof(query) === 'undefined') { + // set cached variables if needed + if(_queryVariables === null) { + if(typeof(window) !== 'undefined' && window.location && window.location.search) { + // parse window search query + _queryVariables = parse(window.location.search.substring(1)); + } else { + // no query variables available + _queryVariables = {}; + } + } + rval = _queryVariables; + } else { + // parse given query + rval = parse(query); + } + return rval; +}; + +/** + * Parses a fragment into a path and query. This method will take a URI + * fragment and break it up as if it were the main URI. For example: + * /bar/baz?a=1&b=2 + * results in: + * { + * path: ["bar", "baz"], + * query: {"k1": ["v1"], "k2": ["v2"]} + * } + * + * @return object with a path array and query object. + */ +util.parseFragment = function(fragment) { + // default to whole fragment + var fp = fragment; + var fq = ''; + // split into path and query if possible at the first '?' + var pos = fragment.indexOf('?'); + if(pos > 0) { + fp = fragment.substring(0, pos); + fq = fragment.substring(pos + 1); + } + // split path based on '/' and ignore first element if empty + var path = fp.split('/'); + if(path.length > 0 && path[0] === '') { + path.shift(); + } + // convert query into object + var query = (fq === '') ? {} : util.getQueryVariables(fq); + + return { + pathString: fp, + queryString: fq, + path: path, + query: query + }; +}; + +/** + * Makes a request out of a URI-like request string. This is intended to + * be used where a fragment id (after a URI '#') is parsed as a URI with + * path and query parts. The string should have a path beginning and + * delimited by '/' and optional query parameters following a '?'. The + * query should be a standard URL set of key value pairs delimited by + * '&'. For backwards compatibility the initial '/' on the path is not + * required. The request object has the following API, (fully described + * in the method code): + * { + * path: . + * query: , + * getPath(i): get part or all of the split path array, + * getQuery(k, i): get part or all of a query key array, + * getQueryLast(k, _default): get last element of a query key array. + * } + * + * @return object with request parameters. + */ +util.makeRequest = function(reqString) { + var frag = util.parseFragment(reqString); + var req = { + // full path string + path: frag.pathString, + // full query string + query: frag.queryString, + /** + * Get path or element in path. + * + * @param i optional path index. + * + * @return path or part of path if i provided. + */ + getPath: function(i) { + return (typeof(i) === 'undefined') ? frag.path : frag.path[i]; + }, + /** + * Get query, values for a key, or value for a key index. + * + * @param k optional query key. + * @param i optional query key index. + * + * @return query, values for a key, or value for a key index. + */ + getQuery: function(k, i) { + var rval; + if(typeof(k) === 'undefined') { + rval = frag.query; + } else { + rval = frag.query[k]; + if(rval && typeof(i) !== 'undefined') { + rval = rval[i]; + } + } + return rval; + }, + getQueryLast: function(k, _default) { + var rval; + var vals = req.getQuery(k); + if(vals) { + rval = vals[vals.length - 1]; + } else { + rval = _default; + } + return rval; + } + }; + return req; +}; + +/** + * Makes a URI out of a path, an object with query parameters, and a + * fragment. Uses jQuery.param() internally for query string creation. + * If the path is an array, it will be joined with '/'. + * + * @param path string path or array of strings. + * @param query object with query parameters. (optional) + * @param fragment fragment string. (optional) + * + * @return string object with request parameters. + */ +util.makeLink = function(path, query, fragment) { + // join path parts if needed + path = jQuery.isArray(path) ? path.join('/') : path; + + var qstr = jQuery.param(query || {}); + fragment = fragment || ''; + return path + + ((qstr.length > 0) ? ('?' + qstr) : '') + + ((fragment.length > 0) ? ('#' + fragment) : ''); +}; + +/** + * Follows a path of keys deep into an object hierarchy and set a value. + * If a key does not exist or it's value is not an object, create an + * object in it's place. This can be destructive to a object tree if + * leaf nodes are given as non-final path keys. + * Used to avoid exceptions from missing parts of the path. + * + * @param object the starting object. + * @param keys an array of string keys. + * @param value the value to set. + */ +util.setPath = function(object, keys, value) { + // need to start at an object + if(typeof(object) === 'object' && object !== null) { + var i = 0; + var len = keys.length; + while(i < len) { + var next = keys[i++]; + if(i == len) { + // last + object[next] = value; + } else { + // more + var hasNext = (next in object); + if(!hasNext || + (hasNext && typeof(object[next]) !== 'object') || + (hasNext && object[next] === null)) { + object[next] = {}; + } + object = object[next]; + } + } + } +}; + +/** + * Follows a path of keys deep into an object hierarchy and return a value. + * If a key does not exist, create an object in it's place. + * Used to avoid exceptions from missing parts of the path. + * + * @param object the starting object. + * @param keys an array of string keys. + * @param _default value to return if path not found. + * + * @return the value at the path if found, else default if given, else + * undefined. + */ +util.getPath = function(object, keys, _default) { + var i = 0; + var len = keys.length; + var hasNext = true; + while(hasNext && i < len && + typeof(object) === 'object' && object !== null) { + var next = keys[i++]; + hasNext = next in object; + if(hasNext) { + object = object[next]; + } + } + return (hasNext ? object : _default); +}; + +/** + * Follow a path of keys deep into an object hierarchy and delete the + * last one. If a key does not exist, do nothing. + * Used to avoid exceptions from missing parts of the path. + * + * @param object the starting object. + * @param keys an array of string keys. + */ +util.deletePath = function(object, keys) { + // need to start at an object + if(typeof(object) === 'object' && object !== null) { + var i = 0; + var len = keys.length; + while(i < len) { + var next = keys[i++]; + if(i == len) { + // last + delete object[next]; + } else { + // more + if(!(next in object) || + (typeof(object[next]) !== 'object') || + (object[next] === null)) { + break; + } + object = object[next]; + } + } + } +}; + +/** + * Check if an object is empty. + * + * Taken from: + * http://stackoverflow.com/questions/679915/how-do-i-test-for-an-empty-javascript-object-from-json/679937#679937 + * + * @param object the object to check. + */ +util.isEmpty = function(obj) { + for(var prop in obj) { + if(obj.hasOwnProperty(prop)) { + return false; + } + } + return true; +}; + +/** + * Format with simple printf-style interpolation. + * + * %%: literal '%' + * %s,%o: convert next argument into a string. + * + * @param format the string to format. + * @param ... arguments to interpolate into the format string. + */ +util.format = function(format) { + var re = /%./g; + // current match + var match; + // current part + var part; + // current arg index + var argi = 0; + // collected parts to recombine later + var parts = []; + // last index found + var last = 0; + // loop while matches remain + while((match = re.exec(format))) { + part = format.substring(last, re.lastIndex - 2); + // don't add empty strings (ie, parts between %s%s) + if(part.length > 0) { + parts.push(part); + } + last = re.lastIndex; + // switch on % code + var code = match[0][1]; + switch(code) { + case 's': + case 'o': + // check if enough arguments were given + if(argi < arguments.length) { + parts.push(arguments[argi++ + 1]); + } else { + parts.push(''); + } + break; + // FIXME: do proper formating for numbers, etc + //case 'f': + //case 'd': + case '%': + parts.push('%'); + break; + default: + parts.push('<%' + code + '?>'); + } + } + // add trailing part of format string + parts.push(format.substring(last)); + return parts.join(''); +}; + +/** + * Formats a number. + * + * http://snipplr.com/view/5945/javascript-numberformat--ported-from-php/ + */ +util.formatNumber = function(number, decimals, dec_point, thousands_sep) { + // http://kevin.vanzonneveld.net + // + original by: Jonas Raoni Soares Silva (http://www.jsfromhell.com) + // + improved by: Kevin van Zonneveld (http://kevin.vanzonneveld.net) + // + bugfix by: Michael White (http://crestidg.com) + // + bugfix by: Benjamin Lupton + // + bugfix by: Allan Jensen (http://www.winternet.no) + // + revised by: Jonas Raoni Soares Silva (http://www.jsfromhell.com) + // * example 1: number_format(1234.5678, 2, '.', ''); + // * returns 1: 1234.57 + + var n = number, c = isNaN(decimals = Math.abs(decimals)) ? 2 : decimals; + var d = dec_point === undefined ? ',' : dec_point; + var t = thousands_sep === undefined ? + '.' : thousands_sep, s = n < 0 ? '-' : ''; + var i = parseInt((n = Math.abs(+n || 0).toFixed(c)), 10) + ''; + var j = (i.length > 3) ? i.length % 3 : 0; + return s + (j ? i.substr(0, j) + t : '') + + i.substr(j).replace(/(\d{3})(?=\d)/g, '$1' + t) + + (c ? d + Math.abs(n - i).toFixed(c).slice(2) : ''); +}; + +/** + * Formats a byte size. + * + * http://snipplr.com/view/5949/format-humanize-file-byte-size-presentation-in-javascript/ + */ +util.formatSize = function(size) { + if(size >= 1073741824) { + size = util.formatNumber(size / 1073741824, 2, '.', '') + ' GiB'; + } else if(size >= 1048576) { + size = util.formatNumber(size / 1048576, 2, '.', '') + ' MiB'; + } else if(size >= 1024) { + size = util.formatNumber(size / 1024, 0) + ' KiB'; + } else { + size = util.formatNumber(size, 0) + ' bytes'; + } + return size; +}; + +/** + * Converts an IPv4 or IPv6 string representation into bytes (in network order). + * + * @param ip the IPv4 or IPv6 address to convert. + * + * @return the 4-byte IPv6 or 16-byte IPv6 address or null if the address can't + * be parsed. + */ +util.bytesFromIP = function(ip) { + if(ip.indexOf('.') !== -1) { + return util.bytesFromIPv4(ip); + } + if(ip.indexOf(':') !== -1) { + return util.bytesFromIPv6(ip); + } + return null; +}; + +/** + * Converts an IPv4 string representation into bytes (in network order). + * + * @param ip the IPv4 address to convert. + * + * @return the 4-byte address or null if the address can't be parsed. + */ +util.bytesFromIPv4 = function(ip) { + ip = ip.split('.'); + if(ip.length !== 4) { + return null; + } + var b = util.createBuffer(); + for(var i = 0; i < ip.length; ++i) { + var num = parseInt(ip[i], 10); + if(isNaN(num)) { + return null; + } + b.putByte(num); + } + return b.getBytes(); +}; + +/** + * Converts an IPv6 string representation into bytes (in network order). + * + * @param ip the IPv6 address to convert. + * + * @return the 16-byte address or null if the address can't be parsed. + */ +util.bytesFromIPv6 = function(ip) { + var blanks = 0; + ip = ip.split(':').filter(function(e) { + if(e.length === 0) ++blanks; + return true; + }); + var zeros = (8 - ip.length + blanks) * 2; + var b = util.createBuffer(); + for(var i = 0; i < 8; ++i) { + if(!ip[i] || ip[i].length === 0) { + b.fillWithByte(0, zeros); + zeros = 0; + continue; + } + var bytes = util.hexToBytes(ip[i]); + if(bytes.length < 2) { + b.putByte(0); + } + b.putBytes(bytes); + } + return b.getBytes(); +}; + +/** + * Converts 4-bytes into an IPv4 string representation or 16-bytes into + * an IPv6 string representation. The bytes must be in network order. + * + * @param bytes the bytes to convert. + * + * @return the IPv4 or IPv6 string representation if 4 or 16 bytes, + * respectively, are given, otherwise null. + */ +util.bytesToIP = function(bytes) { + if(bytes.length === 4) { + return util.bytesToIPv4(bytes); + } + if(bytes.length === 16) { + return util.bytesToIPv6(bytes); + } + return null; +}; + +/** + * Converts 4-bytes into an IPv4 string representation. The bytes must be + * in network order. + * + * @param bytes the bytes to convert. + * + * @return the IPv4 string representation or null for an invalid # of bytes. + */ +util.bytesToIPv4 = function(bytes) { + if(bytes.length !== 4) { + return null; + } + var ip = []; + for(var i = 0; i < bytes.length; ++i) { + ip.push(bytes.charCodeAt(i)); + } + return ip.join('.'); +}; + +/** + * Converts 16-bytes into an IPv16 string representation. The bytes must be + * in network order. + * + * @param bytes the bytes to convert. + * + * @return the IPv16 string representation or null for an invalid # of bytes. + */ +util.bytesToIPv6 = function(bytes) { + if(bytes.length !== 16) { + return null; + } + var ip = []; + var zeroGroups = []; + var zeroMaxGroup = 0; + for(var i = 0; i < bytes.length; i += 2) { + var hex = util.bytesToHex(bytes[i] + bytes[i + 1]); + // canonicalize zero representation + while(hex[0] === '0' && hex !== '0') { + hex = hex.substr(1); + } + if(hex === '0') { + var last = zeroGroups[zeroGroups.length - 1]; + var idx = ip.length; + if(!last || idx !== last.end + 1) { + zeroGroups.push({start: idx, end: idx}); + } else { + last.end = idx; + if((last.end - last.start) > + (zeroGroups[zeroMaxGroup].end - zeroGroups[zeroMaxGroup].start)) { + zeroMaxGroup = zeroGroups.length - 1; + } + } + } + ip.push(hex); + } + if(zeroGroups.length > 0) { + var group = zeroGroups[zeroMaxGroup]; + // only shorten group of length > 0 + if(group.end - group.start > 0) { + ip.splice(group.start, group.end - group.start + 1, ''); + if(group.start === 0) { + ip.unshift(''); + } + if(group.end === 7) { + ip.push(''); + } + } + } + return ip.join(':'); +}; + +/** + * Estimates the number of processes that can be run concurrently. If + * creating Web Workers, keep in mind that the main JavaScript process needs + * its own core. + * + * @param options the options to use: + * update true to force an update (not use the cached value). + * @param callback(err, max) called once the operation completes. + */ +util.estimateCores = function(options, callback) { + if(typeof options === 'function') { + callback = options; + options = {}; + } + options = options || {}; + if('cores' in util && !options.update) { + return callback(null, util.cores); + } + if(typeof navigator !== 'undefined' && + 'hardwareConcurrency' in navigator && + navigator.hardwareConcurrency > 0) { + util.cores = navigator.hardwareConcurrency; + return callback(null, util.cores); + } + if(typeof Worker === 'undefined') { + // workers not available + util.cores = 1; + return callback(null, util.cores); + } + if(typeof Blob === 'undefined') { + // can't estimate, default to 2 + util.cores = 2; + return callback(null, util.cores); + } + + // create worker concurrency estimation code as blob + var blobUrl = URL.createObjectURL(new Blob(['(', + function() { + self.addEventListener('message', function(e) { + // run worker for 4 ms + var st = Date.now(); + var et = st + 4; + while(Date.now() < et); + self.postMessage({st: st, et: et}); + }); + }.toString(), + ')()'], {type: 'application/javascript'})); + + // take 5 samples using 16 workers + sample([], 5, 16); + + function sample(max, samples, numWorkers) { + if(samples === 0) { + // get overlap average + var avg = Math.floor(max.reduce(function(avg, x) { + return avg + x; + }, 0) / max.length); + util.cores = Math.max(1, avg); + URL.revokeObjectURL(blobUrl); + return callback(null, util.cores); + } + map(numWorkers, function(err, results) { + max.push(reduce(numWorkers, results)); + sample(max, samples - 1, numWorkers); + }); + } + + function map(numWorkers, callback) { + var workers = []; + var results = []; + for(var i = 0; i < numWorkers; ++i) { + var worker = new Worker(blobUrl); + worker.addEventListener('message', function(e) { + results.push(e.data); + if(results.length === numWorkers) { + for(var i = 0; i < numWorkers; ++i) { + workers[i].terminate(); + } + callback(null, results); + } + }); + workers.push(worker); + } + for(var i = 0; i < numWorkers; ++i) { + workers[i].postMessage(i); + } + } + + function reduce(numWorkers, results) { + // find overlapping time windows + var overlaps = []; + for(var n = 0; n < numWorkers; ++n) { + var r1 = results[n]; + var overlap = overlaps[n] = []; + for(var i = 0; i < numWorkers; ++i) { + if(n === i) { + continue; + } + var r2 = results[i]; + if((r1.st > r2.st && r1.st < r2.et) || + (r2.st > r1.st && r2.st < r1.et)) { + overlap.push(i); + } + } + } + // get maximum overlaps ... don't include overlapping worker itself + // as the main JS process was also being scheduled during the work and + // would have to be subtracted from the estimate anyway + return overlaps.reduce(function(max, overlap) { + return Math.max(max, overlap.length); + }, 0); + } +}; + +} // end module implementation + +/* ########## Begin module wrapper ########## */ +var name = 'util'; +if(typeof define !== 'function') { + // NodeJS -> AMD + if(typeof module === 'object' && module.exports) { + var nodeJS = true; + define = function(ids, factory) { + factory(require, module); + }; + } else { + //