From 6cf8f71d109d8908cf05b093c7cc17aac7dd5062 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 2 Apr 2019 10:24:43 -0400 Subject: [PATCH 01/32] chore: callbacks -> async / await BREAKING CHANGE: All places in the API that used callbacks are now replaced with async/await --- .travis.yml | 53 +++++++++++------------ package.json | 24 ++++++----- src/adapter.js | 80 +++++++++++++++++++++++++++++++++++ src/constants.js | 8 ++++ src/index.js | 63 +++++++++++++++------------- src/listener.js | 81 ++++++++++++++++++------------------ src/socket.js | 65 +++++++++++++++++++++++++++++ test/compliance.spec.js | 7 +--- test/connection-wrap.spec.js | 2 +- test/connection.spec.js | 2 +- test/listen-dial.js | 2 +- 11 files changed, 274 insertions(+), 113 deletions(-) create mode 100644 src/adapter.js create mode 100644 src/constants.js create mode 100644 src/socket.js diff --git a/.travis.yml b/.travis.yml index 5102ee5..99fe0ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,32 +1,33 @@ -# Warning: This file is automatically synced from https://github.com/ipfs/ci-sync so if you want to change it, please change it there and ask someone to sync all repositories. -sudo: false language: node_js +cache: npm -matrix: - include: - - node_js: 6 - env: CXX=g++-4.8 - - node_js: 8 - env: CXX=g++-4.8 - # - node_js: stable - # env: CXX=g++-4.8 +stages: + - check + - test + - cov + +node_js: + - '10' -script: - - npm run lint - - npm run test - - npm run coverage +os: + - linux + - osx -before_script: - - export DISPLAY=:99.0 - - sh -e /etc/init.d/xvfb start +script: npx nyc -s npm run test:node -- --bail +after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov + +jobs: + include: + - os: windows + filter_secrets: false + cache: false -after_success: - - npm run coverage-publish + - stage: check + script: + - npx aegir build --bundlesize + - npx aegir commitlint --travis + - npx aegir dep-check + - npm run lint -addons: - firefox: 'latest' - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-4.8 +notifications: + email: false diff --git a/package.json b/package.json index a07eaf3..6b374ed 100644 --- a/package.json +++ b/package.json @@ -6,12 +6,15 @@ "main": "src/index.js", "scripts": { "lint": "aegir lint", - "test": "aegir test -t node -f test/**/*.js", - "release": "aegir release -t node --no-build", - "release-minor": "aegir release -t node --type minor --no-build", - "release-major": "aegir-release -t node --type major --no-build", + "test": "aegir test -t node", + "test:node": "aegir test -t node", + "build": "aegir build", + "docs": "aegir docs", + "release": "aegir release --docs -t node", + "release-minor": "aegir release --type minor --docs -t node", + "release-major": "aegir release --type major --docs -t node", "coverage": "aegir coverage", - "coverage-publish": "aegir coverage --provider coveralls" + "coverage-publish": "aegir-coverage publish" }, "pre-push": [ "lint", @@ -34,7 +37,7 @@ "npm": ">=3.0.0" }, "devDependencies": { - "aegir": "^15.1.0", + "aegir": "^18.1.1", "chai": "^4.1.2", "dirty-chai": "^2.0.1", "interface-transport": "~0.3.6", @@ -42,16 +45,17 @@ "pull-stream": "^3.6.9" }, "dependencies": { + "async-iterator-to-pull-stream": "^1.3.0", "class-is": "^1.1.0", "debug": "^3.1.0", - "interface-connection": "~0.3.2", + "err-code": "^1.1.2", + "interface-connection": "~0.3.3", + "interface-transport": "libp2p/interface-transport#feat/async-await", "ip-address": "^5.8.9", "lodash.includes": "^4.3.0", "lodash.isfunction": "^3.0.9", "mafmt": "^6.0.2", - "multiaddr": "^5.0.0", - "once": "^1.4.0", - "stream-to-pull-stream": "^1.7.2" + "multiaddr": "^5.0.0" }, "contributors": [ "Alan Shaw ", diff --git a/src/adapter.js b/src/adapter.js new file mode 100644 index 0000000..f1cfa6f --- /dev/null +++ b/src/adapter.js @@ -0,0 +1,80 @@ +'use strict' + +const { Connection } = require('interface-connection') +const withIs = require('class-is') +const toPull = require('async-iterator-to-pull-stream') +const error = require('pull-stream/sources/error') +const drain = require('pull-stream/sinks/drain') +const TCP = require('./') +const noop = () => {} + +function callbackify (fn) { + return async function (...args) { + let cb = args.pop() + if (typeof cb !== 'function') { + args.push(cb) + cb = noop + } + let res + try { + res = await fn(...args) + } catch (err) { + return cb(err) + } + cb(null, res) + } +} + +// Legacy adapter to old transport & connection interface +class TcpAdapter extends TCP { + dial (ma, options, callback) { + if (typeof options === 'function') { + callback = options + options = {} + } + + callback = callback || noop + + const conn = new Connection() + + super.dial(ma, options) + .then(socket => { + conn.setInnerConn(toPull.duplex(socket)) + conn.getObservedAddrs = callbackify(socket.getObservedAddrs.bind(socket)) + conn.close = callbackify(socket.close.bind(socket)) + callback(null, conn) + }) + .catch(err => { + conn.setInnerConn({ sink: drain(), source: error(err) }) + callback(err) + }) + + return conn + } + + createListener (options, handler) { + if (typeof options === 'function') { + handler = options + options = {} + } + + const server = super.createListener(options, socket => { + const conn = new Connection(toPull.duplex(socket)) + conn.getObservedAddrs = callbackify(socket.getObservedAddrs.bind(socket)) + handler(conn) + }) + + const proxy = { + listen: callbackify(server.listen.bind(server)), + close: callbackify(server.close.bind(server)), + getAddrs: callbackify(server.getAddrs.bind(server)), + getObservedAddrs: callbackify(() => server.getObservedAddrs()) + } + + return new Proxy(server, { get: (_, prop) => proxy[prop] || server[prop] }) + } +} +module.exports = withIs(TcpAdapter, { + className: 'TCP', + symbolName: '@libp2p/js-libp2p-tcp/tcp' +}) diff --git a/src/constants.js b/src/constants.js new file mode 100644 index 0000000..3e5cb6e --- /dev/null +++ b/src/constants.js @@ -0,0 +1,8 @@ +'use strict' + +// IPFS multi-address code +module.exports.IPFS_MA_CODE = 421 + +// Time to wait for a connection to close gracefully before destroying it +// manually +module.exports.CLOSE_TIMEOUT = 2000 diff --git a/src/index.js b/src/index.js index b04c223..51a8706 100644 --- a/src/index.js +++ b/src/index.js @@ -1,55 +1,61 @@ 'use strict' const net = require('net') -const toPull = require('stream-to-pull-stream') const mafmt = require('mafmt') const withIs = require('class-is') const includes = require('lodash.includes') const isFunction = require('lodash.isfunction') -const Connection = require('interface-connection').Connection -const once = require('once') +const errcode = require('err-code') const debug = require('debug') const log = debug('libp2p:tcp:dial') +const Libp2pSocket = require('./socket') const createListener = require('./listener') function noop () {} class TCP { - dial (ma, options, callback) { - if (isFunction(options)) { - callback = options - options = {} - } - - callback = once(callback || noop) - + async dial (ma, options) { const cOpts = ma.toOptions() - log('Connecting to %s %s', cOpts.port, cOpts.host) + log('Connecting to %s:%s', cOpts.host, cOpts.port) - const rawSocket = net.connect(cOpts) + const rawSocket = await this._connect(cOpts) + return new Libp2pSocket(rawSocket, ma, options) + } - rawSocket.once('timeout', () => { - log('timeout') - rawSocket.emit('error', new Error('Timeout')) - }) + _connect (cOpts) { + return new Promise((resolve, reject) => { + const start = Date.now() + const rawSocket = net.connect(cOpts) - rawSocket.once('error', callback) + const onError = (err) => { + const msg = `Error connecting to ${cOpts.host}:${cOpts.port}: ${err.message}` + done(errcode(msg, err.code)) + } - rawSocket.once('connect', () => { - rawSocket.removeListener('error', callback) - callback() - }) + const onTimeout = () => { + log('Timeout connecting to %s:%s', cOpts.host, cOpts.port) + const err = errcode(`Timeout after ${Date.now() - start}ms`, 'ETIMEDOUT') + rawSocket.emit('error', err) + } - const socket = toPull.duplex(rawSocket) + const onConnect = () => { + log('Connected to %s:%s', cOpts.host, cOpts.port) + done(null, rawSocket) + } - const conn = new Connection(socket) + const done = (err, res) => { + rawSocket.removeListener('error', onError) + rawSocket.removeListener('timeout', onTimeout) + rawSocket.removeListener('connect', onConnect) - conn.getObservedAddrs = (callback) => { - return callback(null, [ma]) - } + err ? reject(err) : resolve(res) + } - return conn + rawSocket.once('error', onError) + rawSocket.once('timeout', onTimeout) + rawSocket.once('connect', onConnect) + }) } createListener (options, handler) { @@ -59,7 +65,6 @@ class TCP { } handler = handler || noop - return createListener(handler) } diff --git a/src/listener.js b/src/listener.js index 30b3076..672ac9a 100644 --- a/src/listener.js +++ b/src/listener.js @@ -1,19 +1,16 @@ 'use strict' const multiaddr = require('multiaddr') -const Connection = require('interface-connection').Connection const os = require('os') const includes = require('lodash.includes') const net = require('net') -const toPull = require('stream-to-pull-stream') const EventEmitter = require('events').EventEmitter const debug = require('debug') const log = debug('libp2p:tcp:listen') +const Libp2pSocket = require('./socket') const getMultiaddr = require('./get-multiaddr') - -const IPFS_CODE = 421 -const CLOSE_TIMEOUT = 2000 +const c = require('./constants') function noop () {} @@ -21,7 +18,7 @@ module.exports = (handler) => { const listener = new EventEmitter() const server = net.createServer((socket) => { - // Avoid uncaught errors cause by unstable connections + // Avoid uncaught errors caused by unstable connections socket.on('error', noop) const addr = getMultiaddr(socket) @@ -36,17 +33,11 @@ module.exports = (handler) => { log('new connection', addr.toString()) - const s = toPull.duplex(socket) - - s.getObservedAddrs = (cb) => { - cb(null, [addr]) - } - + const s = new Libp2pSocket(socket) trackSocket(server, socket) - const conn = new Connection(s) - handler(conn) - listener.emit('connection', conn) + handler && handler(s) + listener.emit('connection', s) }) server.on('listening', () => listener.emit('listening')) @@ -56,33 +47,35 @@ module.exports = (handler) => { // Keep track of open connections to destroy in case of timeout server.__connections = {} - listener.close = (options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} + listener.close = (options = {}) => { + if (!server.listening) { + return } - callback = callback || noop - options = options || {} - - const timeout = setTimeout(() => { - log('unable to close graciously, destroying conns') - Object.keys(server.__connections).forEach((key) => { - log('destroying %s', key) - server.__connections[key].destroy() - }) - }, options.timeout || CLOSE_TIMEOUT) - server.close(callback) + return new Promise((resolve, reject) => { + const start = Date.now() - server.once('close', () => { - clearTimeout(timeout) + // Attempt to stop the server. If it takes longer than the timeout, + // destroy all the underlying sockets manually. + const timeout = setTimeout(() => { + log('Timeout closing server after %dms, destroying connections manually', Date.now() - start) + Object.keys(server.__connections).forEach((key) => { + log('destroying %s', key) + server.__connections[key].destroy() + }) + resolve() + }, options.timeout || c.CLOSE_TIMEOUT) + + server.once('close', () => clearTimeout(timeout)) + + server.close((err) => err ? reject(err) : resolve()) }) } let ipfsId let listeningAddr - listener.listen = (ma, callback) => { + listener.listen = (ma) => { listeningAddr = ma if (includes(ma.protoNames(), 'ipfs')) { ipfsId = getIpfsId(ma) @@ -90,16 +83,24 @@ module.exports = (handler) => { } const lOpts = listeningAddr.toOptions() - log('Listening on %s %s', lOpts.port, lOpts.host) - return server.listen(lOpts.port, lOpts.host, callback) + return new Promise((resolve, reject) => { + server.listen(lOpts.port, lOpts.host, (err) => { + if (err) { + return reject(err) + } + + log('Listening on %s %s', lOpts.port, lOpts.host) + resolve() + }) + }) } - listener.getAddrs = (callback) => { + listener.getAddrs = () => { const multiaddrs = [] const address = server.address() if (!address) { - return callback(new Error('Listener is not ready yet')) + throw new Error('Listener is not ready yet') } // Because TCP will only return the IPv6 version @@ -134,7 +135,7 @@ module.exports = (handler) => { multiaddrs.push(ma) } - callback(null, multiaddrs) + return multiaddrs } return listener @@ -142,7 +143,7 @@ module.exports = (handler) => { function getIpfsId (ma) { return ma.stringTuples().filter((tuple) => { - return tuple[0] === IPFS_CODE + return tuple[0] === c.IPFS_MA_CODE })[0][1] } @@ -150,7 +151,7 @@ function trackSocket (server, socket) { const key = `${socket.remoteAddress}:${socket.remotePort}` server.__connections[key] = socket - socket.on('close', () => { + socket.once('close', () => { delete server.__connections[key] }) } diff --git a/src/socket.js b/src/socket.js new file mode 100644 index 0000000..345484e --- /dev/null +++ b/src/socket.js @@ -0,0 +1,65 @@ +'use strict' + +const debug = require('debug') +const log = debug('libp2p:tcp:socket') + +const c = require('./constants') + +class Libp2pSocket { + constructor (rawSocket, ma, opts) { + this._rawSocket = rawSocket + this._ma = ma + + this.sink = this._sink(opts) + this.source = rawSocket + } + + _sink (opts = {}) { + // By default, close when the source is exhausted + const closeOnEnd = opts.closeOnEnd !== false + return (source) => this._write(source, closeOnEnd) + } + + async _write (source, closeOnEnd) { + for await (const data of source) { + const flushed = this._rawSocket.write(data) + if (!flushed) { + await new Promise((resolve) => this._rawSocket.once('drain', resolve)) + } + } + + if (closeOnEnd) { + await this.close() + } + } + + close (opts = {}) { + if (this._rawSocket.pending || this._rawSocket.destroyed) { + return + } + + return new Promise((resolve, reject) => { + const start = Date.now() + + // Attempt to end the socket. If it takes longer to close than the + // timeout, destroy it manually. + const timeout = setTimeout(() => { + const cOpts = this._ma.toOptions() + log('Timeout closing socket to %s:%s after %dms, destroying it manually', + cOpts.host, cOpts.port, Date.now() - start) + this._rawSocket.destroy() + resolve() + }, opts.timeout || c.CLOSE_TIMEOUT) + + this._rawSocket.once('close', () => clearTimeout(timeout)) + + this._rawSocket.end((err) => err ? reject(err) : resolve()) + }) + } + + getObservedAddrs () { + return [this._ma] + } +} + +module.exports = Libp2pSocket diff --git a/test/compliance.spec.js b/test/compliance.spec.js index 1339086..54cedee 100644 --- a/test/compliance.spec.js +++ b/test/compliance.spec.js @@ -7,7 +7,7 @@ const TCP = require('../src') describe('interface-transport compliance', () => { tests({ - setup (cb) { + setup () { const tcp = new TCP() const addrs = [ multiaddr('/ip4/127.0.0.1/tcp/9091'), @@ -15,10 +15,7 @@ describe('interface-transport compliance', () => { multiaddr('/ip4/127.0.0.1/tcp/9093'), multiaddr('/dns4/ipfs.io') ] - cb(null, tcp, addrs) - }, - teardown (cb) { - cb() + return { transport: tcp, addrs } } }) }) diff --git a/test/connection-wrap.spec.js b/test/connection-wrap.spec.js index ff141fb..5403bae 100644 --- a/test/connection-wrap.spec.js +++ b/test/connection-wrap.spec.js @@ -6,7 +6,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) -const TCP = require('../src') +const TCP = require('../src/adapter') const multiaddr = require('multiaddr') const Connection = require('interface-connection').Connection diff --git a/test/connection.spec.js b/test/connection.spec.js index 5100cb1..bef1964 100644 --- a/test/connection.spec.js +++ b/test/connection.spec.js @@ -6,7 +6,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) -const TCP = require('../src') +const TCP = require('../src/adapter') const multiaddr = require('multiaddr') describe('valid Connection', () => { diff --git a/test/listen-dial.js b/test/listen-dial.js index 7d8f119..7b7bf4c 100644 --- a/test/listen-dial.js +++ b/test/listen-dial.js @@ -6,7 +6,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) -const TCP = require('../src') +const TCP = require('../src/adapter') const net = require('net') const multiaddr = require('multiaddr') const isCI = process.env.CI From 65c1888f543ccddb596a2c6f159ffecb37d556f2 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 8 Apr 2019 22:52:12 -0400 Subject: [PATCH 02/32] feat: cancel dials with AbortController --- package.json | 5 +-- src/adapter.js | 75 ++++------------------------------------- src/index.js | 17 ++++++++-- test/compliance.spec.js | 24 ++++++++++++- 4 files changed, 47 insertions(+), 74 deletions(-) diff --git a/package.json b/package.json index 6b374ed..baf84d2 100644 --- a/package.json +++ b/package.json @@ -42,10 +42,11 @@ "dirty-chai": "^2.0.1", "interface-transport": "~0.3.6", "lodash.isfunction": "^3.0.9", - "pull-stream": "^3.6.9" + "pull-stream": "^3.6.9", + "sinon": "^7.3.1" }, "dependencies": { - "async-iterator-to-pull-stream": "^1.3.0", + "abort-controller": "^3.0.0", "class-is": "^1.1.0", "debug": "^3.1.0", "err-code": "^1.1.2", diff --git a/src/adapter.js b/src/adapter.js index f1cfa6f..e206a8b 100644 --- a/src/adapter.js +++ b/src/adapter.js @@ -1,79 +1,16 @@ 'use strict' -const { Connection } = require('interface-connection') +const { Adapter } = require('interface-transport') const withIs = require('class-is') -const toPull = require('async-iterator-to-pull-stream') -const error = require('pull-stream/sources/error') -const drain = require('pull-stream/sinks/drain') -const TCP = require('./') -const noop = () => {} - -function callbackify (fn) { - return async function (...args) { - let cb = args.pop() - if (typeof cb !== 'function') { - args.push(cb) - cb = noop - } - let res - try { - res = await fn(...args) - } catch (err) { - return cb(err) - } - cb(null, res) - } -} +const TCP = require('.') // Legacy adapter to old transport & connection interface -class TcpAdapter extends TCP { - dial (ma, options, callback) { - if (typeof options === 'function') { - callback = options - options = {} - } - - callback = callback || noop - - const conn = new Connection() - - super.dial(ma, options) - .then(socket => { - conn.setInnerConn(toPull.duplex(socket)) - conn.getObservedAddrs = callbackify(socket.getObservedAddrs.bind(socket)) - conn.close = callbackify(socket.close.bind(socket)) - callback(null, conn) - }) - .catch(err => { - conn.setInnerConn({ sink: drain(), source: error(err) }) - callback(err) - }) - - return conn - } - - createListener (options, handler) { - if (typeof options === 'function') { - handler = options - options = {} - } - - const server = super.createListener(options, socket => { - const conn = new Connection(toPull.duplex(socket)) - conn.getObservedAddrs = callbackify(socket.getObservedAddrs.bind(socket)) - handler(conn) - }) - - const proxy = { - listen: callbackify(server.listen.bind(server)), - close: callbackify(server.close.bind(server)), - getAddrs: callbackify(server.getAddrs.bind(server)), - getObservedAddrs: callbackify(() => server.getObservedAddrs()) - } - - return new Proxy(server, { get: (_, prop) => proxy[prop] || server[prop] }) +class TcpAdapter extends Adapter { + constructor () { + super(new TCP()) } } + module.exports = withIs(TcpAdapter, { className: 'TCP', symbolName: '@libp2p/js-libp2p-tcp/tcp' diff --git a/src/index.js b/src/index.js index 51a8706..e59da74 100644 --- a/src/index.js +++ b/src/index.js @@ -11,6 +11,7 @@ const log = debug('libp2p:tcp:dial') const Libp2pSocket = require('./socket') const createListener = require('./listener') +const { AbortError } = require('interface-transport') function noop () {} @@ -19,12 +20,16 @@ class TCP { const cOpts = ma.toOptions() log('Connecting to %s:%s', cOpts.host, cOpts.port) - const rawSocket = await this._connect(cOpts) + const rawSocket = await this._connect(cOpts, options) return new Libp2pSocket(rawSocket, ma, options) } - _connect (cOpts) { + _connect (cOpts, options = {}) { return new Promise((resolve, reject) => { + if ((options.signal || {}).aborted) { + return reject(new AbortError()) + } + const start = Date.now() const rawSocket = net.connect(cOpts) @@ -44,10 +49,17 @@ class TCP { done(null, rawSocket) } + const onAbort = () => { + log('Connect to %s:%s aborted', cOpts.host, cOpts.port) + rawSocket.destroy() + done(new AbortError()) + } + const done = (err, res) => { rawSocket.removeListener('error', onError) rawSocket.removeListener('timeout', onTimeout) rawSocket.removeListener('connect', onConnect) + options.signal && options.signal.removeEventListener(onAbort) err ? reject(err) : resolve(res) } @@ -55,6 +67,7 @@ class TCP { rawSocket.once('error', onError) rawSocket.once('timeout', onTimeout) rawSocket.once('connect', onConnect) + options.signal && options.signal.addEventListener('abort', onAbort) }) } diff --git a/test/compliance.spec.js b/test/compliance.spec.js index 54cedee..11fd244 100644 --- a/test/compliance.spec.js +++ b/test/compliance.spec.js @@ -1,8 +1,10 @@ /* eslint-env mocha */ 'use strict' +const sinon = require('sinon') const tests = require('interface-transport') const multiaddr = require('multiaddr') +const net = require('net') const TCP = require('../src') describe('interface-transport compliance', () => { @@ -15,7 +17,27 @@ describe('interface-transport compliance', () => { multiaddr('/ip4/127.0.0.1/tcp/9093'), multiaddr('/dns4/ipfs.io') ] - return { transport: tcp, addrs } + + // Used by the dial tests to simulate a delayed connect + const connector = { + delay (delayMs) { + const netConnect = net.connect + sinon.replace(net, 'connect', (opts) => { + const socket = netConnect(opts) + const socketEmit = socket.emit.bind(socket) + sinon.replace(socket, 'emit', (...args) => { + const time = args[0] === 'connect' ? delayMs : 0 + setTimeout(() => socketEmit(...args), time) + }) + return socket + }) + }, + restore () { + sinon.restore() + } + } + + return { transport: tcp, addrs, connector } } }) }) From 776a5246b5a8ccf433238a55cba367aae19d3ff7 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 12 Apr 2019 15:53:17 +0800 Subject: [PATCH 03/32] test: add async/await tests --- src/listener.js | 2 +- test/{ => adapter}/connection-wrap.spec.js | 2 +- test/adapter/connection.spec.js | 111 ++++++++ .../listen-dial.spec.js} | 2 +- test/connection.spec.js | 108 ++------ test/listen-dial.spec.js | 239 ++++++++++++++++++ 6 files changed, 379 insertions(+), 85 deletions(-) rename test/{ => adapter}/connection-wrap.spec.js (98%) create mode 100644 test/adapter/connection.spec.js rename test/{listen-dial.js => adapter/listen-dial.spec.js} (99%) create mode 100644 test/listen-dial.spec.js diff --git a/src/listener.js b/src/listener.js index 672ac9a..b7764b9 100644 --- a/src/listener.js +++ b/src/listener.js @@ -33,7 +33,7 @@ module.exports = (handler) => { log('new connection', addr.toString()) - const s = new Libp2pSocket(socket) + const s = new Libp2pSocket(socket, addr) trackSocket(server, socket) handler && handler(s) diff --git a/test/connection-wrap.spec.js b/test/adapter/connection-wrap.spec.js similarity index 98% rename from test/connection-wrap.spec.js rename to test/adapter/connection-wrap.spec.js index 5403bae..0077417 100644 --- a/test/connection-wrap.spec.js +++ b/test/adapter/connection-wrap.spec.js @@ -6,7 +6,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) -const TCP = require('../src/adapter') +const TCP = require('../../src/adapter') const multiaddr = require('multiaddr') const Connection = require('interface-connection').Connection diff --git a/test/adapter/connection.spec.js b/test/adapter/connection.spec.js new file mode 100644 index 0000000..559143d --- /dev/null +++ b/test/adapter/connection.spec.js @@ -0,0 +1,111 @@ +/* eslint-env mocha */ +'use strict' + +const pull = require('pull-stream') +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) +const TCP = require('../../src/adapter') +const multiaddr = require('multiaddr') + +describe('valid Connection', () => { + let tcp + + beforeEach(() => { + tcp = new TCP() + }) + + const ma = multiaddr('/ip4/127.0.0.1/tcp/9090') + + it('get observed addrs', (done) => { + let dialerObsAddrs + + const listener = tcp.createListener((conn) => { + expect(conn).to.exist() + conn.getObservedAddrs((err, addrs) => { + expect(err).to.not.exist() + dialerObsAddrs = addrs + pull(pull.empty(), conn) + }) + }) + + listener.listen(ma, () => { + const conn = tcp.dial(ma) + pull( + conn, + pull.onEnd(endHandler) + ) + + function endHandler () { + conn.getObservedAddrs((err, addrs) => { + expect(err).to.not.exist() + pull(pull.empty(), conn) + closeAndAssert(listener, addrs) + }) + } + + function closeAndAssert (listener, addrs) { + listener.close(() => { + expect(addrs[0]).to.deep.equal(ma) + expect(dialerObsAddrs.length).to.equal(1) + done() + }) + } + }) + }) + + it('get Peer Info', (done) => { + const listener = tcp.createListener((conn) => { + expect(conn).to.exist() + conn.getPeerInfo((err, peerInfo) => { + expect(err).to.exist() + expect(peerInfo).to.not.exist() + pull(pull.empty(), conn) + }) + }) + + listener.listen(ma, () => { + const conn = tcp.dial(ma) + + pull(conn, pull.onEnd(endHandler)) + + function endHandler () { + conn.getPeerInfo((err, peerInfo) => { + expect(err).to.exist() + expect(peerInfo).to.not.exist() + + listener.close(done) + }) + } + }) + }) + + it('set Peer Info', (done) => { + const listener = tcp.createListener((conn) => { + expect(conn).to.exist() + conn.setPeerInfo('batatas') + conn.getPeerInfo((err, peerInfo) => { + expect(err).to.not.exist() + expect(peerInfo).to.equal('batatas') + pull(pull.empty(), conn) + }) + }) + + listener.listen(ma, () => { + const conn = tcp.dial(ma) + + pull(conn, pull.onEnd(endHandler)) + + function endHandler () { + conn.setPeerInfo('arroz') + conn.getPeerInfo((err, peerInfo) => { + expect(err).to.not.exist() + expect(peerInfo).to.equal('arroz') + + listener.close(done) + }) + } + }) + }) +}) diff --git a/test/listen-dial.js b/test/adapter/listen-dial.spec.js similarity index 99% rename from test/listen-dial.js rename to test/adapter/listen-dial.spec.js index 7b7bf4c..97e6980 100644 --- a/test/listen-dial.js +++ b/test/adapter/listen-dial.spec.js @@ -6,7 +6,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) -const TCP = require('../src/adapter') +const TCP = require('../../src/adapter') const net = require('net') const multiaddr = require('multiaddr') const isCI = process.env.CI diff --git a/test/connection.spec.js b/test/connection.spec.js index bef1964..4d8f385 100644 --- a/test/connection.spec.js +++ b/test/connection.spec.js @@ -1,12 +1,11 @@ /* eslint-env mocha */ 'use strict' -const pull = require('pull-stream') const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) -const TCP = require('../src/adapter') +const TCP = require('../src') const multiaddr = require('multiaddr') describe('valid Connection', () => { @@ -18,94 +17,39 @@ describe('valid Connection', () => { const ma = multiaddr('/ip4/127.0.0.1/tcp/9090') - it('get observed addrs', (done) => { - let dialerObsAddrs - - const listener = tcp.createListener((conn) => { - expect(conn).to.exist() - conn.getObservedAddrs((err, addrs) => { - expect(err).to.not.exist() - dialerObsAddrs = addrs - pull(pull.empty(), conn) - }) - }) - - listener.listen(ma, () => { - const conn = tcp.dial(ma) - pull( - conn, - pull.onEnd(endHandler) - ) - - function endHandler () { - conn.getObservedAddrs((err, addrs) => { - expect(err).to.not.exist() - pull(pull.empty(), conn) - closeAndAssert(listener, addrs) - }) - } - - function closeAndAssert (listener, addrs) { - listener.close(() => { - expect(addrs[0]).to.deep.equal(ma) - expect(dialerObsAddrs.length).to.equal(1) - done() - }) - } + it('get observed addrs', async () => { + // Create a Promise that resolves when a connection is handled + let handled + const handlerPromise = new Promise((resolve) => { + handled = resolve }) - }) - it('get Peer Info', (done) => { - const listener = tcp.createListener((conn) => { + const handler = async (conn) => { expect(conn).to.exist() - conn.getPeerInfo((err, peerInfo) => { - expect(err).to.exist() - expect(peerInfo).to.not.exist() - pull(pull.empty(), conn) - }) - }) + const dialerObsAddrs = await conn.getObservedAddrs() + handled(dialerObsAddrs) + } - listener.listen(ma, () => { - const conn = tcp.dial(ma) + // Create a listener with the handler + const listener = tcp.createListener(handler) - pull(conn, pull.onEnd(endHandler)) + // Listen on the multi-address + await listener.listen(ma) - function endHandler () { - conn.getPeerInfo((err, peerInfo) => { - expect(err).to.exist() - expect(peerInfo).to.not.exist() + // Dial to that same address + const conn = await tcp.dial(ma) + const addrs = await conn.getObservedAddrs() - listener.close(done) - }) - } - }) - }) - - it('set Peer Info', (done) => { - const listener = tcp.createListener((conn) => { - expect(conn).to.exist() - conn.setPeerInfo('batatas') - conn.getPeerInfo((err, peerInfo) => { - expect(err).to.not.exist() - expect(peerInfo).to.equal('batatas') - pull(pull.empty(), conn) - }) - }) - - listener.listen(ma, () => { - const conn = tcp.dial(ma) + // Wait for the incoming dial to be handled + const dialerObsAddrs = await handlerPromise - pull(conn, pull.onEnd(endHandler)) + // Close the listener + await listener.close() - function endHandler () { - conn.setPeerInfo('arroz') - conn.getPeerInfo((err, peerInfo) => { - expect(err).to.not.exist() - expect(peerInfo).to.equal('arroz') - - listener.close(done) - }) - } - }) + // The addresses should match + expect(addrs.length).to.equal(1) + expect(addrs[0]).to.deep.equal(ma) + expect(dialerObsAddrs.length).to.equal(1) + expect(dialerObsAddrs[0]).to.exist() }) }) diff --git a/test/listen-dial.spec.js b/test/listen-dial.spec.js new file mode 100644 index 0000000..e14e759 --- /dev/null +++ b/test/listen-dial.spec.js @@ -0,0 +1,239 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) + +const TCP = require('../src') +const net = require('net') +const multiaddr = require('multiaddr') +const pipe = require('it-pipe') +const { collect, map } = require('streaming-iterables') +const isCI = process.env.CI + +describe('listen', () => { + let tcp + + beforeEach(() => { + tcp = new TCP() + }) + + it('close listener with connections, through timeout', async () => { + const mh = multiaddr('/ip4/127.0.0.1/tcp/9090/ipfs/Qmb6owHp6eaWArVbcJJbQSyifyJBttMMjYV76N2hMbf5Vw') + const listener = tcp.createListener((conn) => { + pipe(conn, conn) + }) + + await listener.listen(mh) + + const socket1 = net.connect(9090) + const socket2 = net.connect(9090) + + socket1.write('Some data that is never handled') + socket1.end() + socket1.on('error', () => {}) + socket2.on('error', () => {}) + + await new Promise((resolve) => { + socket1.on('connect', async () => { + await listener.close() + resolve() + }) + }) + }) + + it('listen on port 0', async () => { + const mh = multiaddr('/ip4/127.0.0.1/tcp/0') + const listener = tcp.createListener((conn) => {}) + await listener.listen(mh) + await listener.close() + }) + + it('listen on IPv6 addr', async () => { + if (isCI) { + return + } + const mh = multiaddr('/ip6/::/tcp/9090') + const listener = tcp.createListener((conn) => {}) + await listener.listen(mh) + await listener.close() + }) + + it('listen on any Interface', async () => { + const mh = multiaddr('/ip4/0.0.0.0/tcp/9090') + const listener = tcp.createListener((conn) => {}) + await listener.listen(mh) + await listener.close() + }) + + it('getAddrs', async () => { + const mh = multiaddr('/ip4/127.0.0.1/tcp/9090') + const listener = tcp.createListener((conn) => {}) + await listener.listen(mh) + + const multiaddrs = listener.getAddrs() + expect(multiaddrs.length).to.equal(1) + expect(multiaddrs[0]).to.deep.equal(mh) + + await listener.close() + }) + + it('getAddrs on port 0 listen', async () => { + const mh = multiaddr('/ip4/127.0.0.1/tcp/0') + const listener = tcp.createListener((conn) => {}) + await listener.listen(mh) + + const multiaddrs = listener.getAddrs() + expect(multiaddrs.length).to.equal(1) + + await listener.close() + }) + + it('getAddrs from listening on 0.0.0.0', async () => { + const mh = multiaddr('/ip4/0.0.0.0/tcp/9090') + const listener = tcp.createListener((conn) => {}) + await listener.listen(mh) + + const multiaddrs = listener.getAddrs() + expect(multiaddrs.length > 0).to.equal(true) + expect(multiaddrs[0].toString().indexOf('0.0.0.0')).to.equal(-1) + + await listener.close() + }) + + it('getAddrs from listening on 0.0.0.0 and port 0', async () => { + const mh = multiaddr('/ip4/0.0.0.0/tcp/0') + const listener = tcp.createListener((conn) => {}) + await listener.listen(mh) + + const multiaddrs = listener.getAddrs() + expect(multiaddrs.length > 0).to.equal(true) + expect(multiaddrs[0].toString().indexOf('0.0.0.0')).to.equal(-1) + + await listener.close() + }) + + it('getAddrs preserves IPFS Id', async () => { + const mh = multiaddr('/ip4/127.0.0.1/tcp/9090/ipfs/Qmb6owHp6eaWArVbcJJbQSyifyJBttMMjYV76N2hMbf5Vw') + const listener = tcp.createListener((conn) => {}) + await listener.listen(mh) + + const multiaddrs = listener.getAddrs() + expect(multiaddrs.length).to.equal(1) + expect(multiaddrs[0]).to.deep.equal(mh) + + await listener.close() + }) +}) + +describe('dial', () => { + let tcp + let listener + const ma = multiaddr('/ip4/127.0.0.1/tcp/9090') + + beforeEach(async () => { + tcp = new TCP() + listener = tcp.createListener((conn) => { + pipe( + conn, + map((x) => Buffer.from(x.toString() + '!')), + conn + ) + }) + await listener.listen(ma) + }) + + afterEach(() => listener.close()) + + it('dial on IPv4', async () => { + const values = await pipe( + ['hey'], + await tcp.dial(ma), + collect + ) + expect(values).to.eql([Buffer.from('hey!')]) + }) + + it('dial on IPv6', async () => { + if (isCI) { + return + } + + const ma = multiaddr('/ip6/::/tcp/9066') + const listener = tcp.createListener((conn) => { + pipe(conn, conn) + }) + await listener.listen(ma) + + const values = await pipe( + ['hey'], + await tcp.dial(ma), + collect + ) + expect(values).to.be.eql([Buffer.from('hey')]) + + await listener.close() + }) + + it('dial and destroy on listener', async () => { + let handled + const handledPromise = new Promise((resolve) => { + handled = resolve + }) + + const ma = multiaddr('/ip6/::/tcp/9067') + + const listener = tcp.createListener(async (conn) => { + await pipe( + [], + conn + ) + handled() + }) + + await listener.listen(ma) + await pipe(await tcp.dial(ma)) + + await handledPromise + await listener.close() + }) + + it('dial and destroy on dialer', async () => { + if (isCI) { + return + } + + let handled + const handledPromise = new Promise((resolve) => { + handled = resolve + }) + + const ma = multiaddr('/ip6/::/tcp/9068') + + const listener = tcp.createListener(async (conn) => { + // pull(conn, pull.onEnd(destroyed)) + await pipe(conn) + handled() + }) + + await listener.listen(ma) + await pipe([], await tcp.dial(ma)) + + await handledPromise + await listener.close() + }) + + it('dial on IPv4 with IPFS Id', async () => { + const ma = multiaddr('/ip4/127.0.0.1/tcp/9090/ipfs/Qmb6owHp6eaWArVbcJJbQSyifyJBttMMjYV76N2hMbf5Vw') + const conn = await tcp.dial(ma) + + const res = await pipe( + ['hey'], + conn, + collect + ) + expect(res).to.be.eql([Buffer.from('hey!')]) + }) +}) From 1ffd549dab82177da47b7b6af87f2a809b99d6c3 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 12 Apr 2019 15:53:33 +0800 Subject: [PATCH 04/32] chore: update README --- README.md | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index fe041d8..60519ca 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ [![](https://raw.githubusercontent.com/libp2p/interface-connection/master/img/badge.png)](https://github.com/libp2p/interface-connection) -> JavaScript implementation of the TCP module for libp2p. It exposes the [interface-transport](https://github.com/libp2p/interface-connection) for dial/listen. `libp2p-tcp` is a very thin shim that adds support for dialing to a `multiaddr`. This small shim will enable libp2p to use other different transports. +> JavaScript implementation of the TCP module for libp2p. It exposes the [interface-transport](https://github.com/libp2p/interface-connection) for dial/listen. `libp2p-tcp` is a very thin shim that adds support for dialing to a `multiaddr`. This small shim will enable libp2p to use other transports. ## Lead Maintainer @@ -41,7 +41,8 @@ ```js const TCP = require('libp2p-tcp') const multiaddr = require('multiaddr') -const pull = require('pull-stream') +const pipe = require('it-pipe') +const { collect } = require('streaming-iterables') const mh = multiaddr('/ip4/127.0.0.1/tcp/9090') @@ -49,29 +50,24 @@ const tcp = new TCP() const listener = tcp.createListener((socket) => { console.log('new connection opened') - pull( - pull.values(['hello']), + pipe( + ['hello'], socket ) }) -listener.listen(mh, () => { - console.log('listening') - - pull( - tcp.dial(mh), - pull.collect((err, values) => { - if (!err) { - console.log(`Value: ${values.toString()}`) - } else { - console.log(`Error: ${err}`) - } - - // Close connection after reading - listener.close() - }), - ) -}) +await listener.listen(mh) +console.log('listening') + +const socket = await tcp.dial(mh) +const values = await pipe( + socket, + collect +) +console.log(`Value: ${values.toString()}`) + +// Close connection after reading +await listener.close() ``` Outputs: @@ -88,12 +84,12 @@ Value: hello [![](https://raw.githubusercontent.com/libp2p/interface-transport/master/img/badge.png)](https://github.com/libp2p/interface-transport) -`libp2p-tcp` accepts TCP addresses both IPFS and non IPFS encapsulated addresses, i.e: +`libp2p-tcp` accepts TCP addresses as both IPFS and non IPFS encapsulated addresses, i.e: `/ip4/127.0.0.1/tcp/4001` `/ip4/127.0.0.1/tcp/4001/ipfs/QmHash` -Both for dialing and listening. +(both for dialing and listening) ### Connection From 5009c2ca49f5e3f1ce8e8fe5f0bd88edbc13b5d8 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 22 Apr 2019 16:00:06 +0200 Subject: [PATCH 05/32] feat: listen to array of multiaddrs (#104) * feat: support listen on array * chore: fix missing deps * chore: update interface version * docs: update readme for array listen * test: use port 0 * docs: add some more jsdocs * chore: fix travis support for ip6 on linux * refactor: clean up some code --- .travis.yml | 11 +- README.md | 6 +- package.json | 11 +- src/index.js | 2 +- src/listener.js | 289 ++++++++++++++++++++----------- test/adapter/listen-dial.spec.js | 4 +- test/listen-dial.spec.js | 16 +- 7 files changed, 215 insertions(+), 124 deletions(-) diff --git a/.travis.yml b/.travis.yml index 99fe0ef..d6481b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,20 +9,23 @@ stages: node_js: - '10' -os: - - linux - - osx - script: npx nyc -s npm run test:node -- --bail after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov jobs: include: + - os: linux + sudo: false + before_script: sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6' + - os: windows filter_secrets: false cache: false + - os: osx + - stage: check + os: linux script: - npx aegir build --bundlesize - npx aegir commitlint --travis diff --git a/README.md b/README.md index 60519ca..b01f9a2 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ const multiaddr = require('multiaddr') const pipe = require('it-pipe') const { collect } = require('streaming-iterables') -const mh = multiaddr('/ip4/127.0.0.1/tcp/9090') +const addr = multiaddr('/ip4/127.0.0.1/tcp/9090') const tcp = new TCP() @@ -56,10 +56,10 @@ const listener = tcp.createListener((socket) => { ) }) -await listener.listen(mh) +await listener.listen([addr]) console.log('listening') -const socket = await tcp.dial(mh) +const socket = await tcp.dial(addr) const values = await pipe( socket, collect diff --git a/package.json b/package.json index baf84d2..defd978 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,6 @@ "aegir": "^18.1.1", "chai": "^4.1.2", "dirty-chai": "^2.0.1", - "interface-transport": "~0.3.6", "lodash.isfunction": "^3.0.9", "pull-stream": "^3.6.9", "sinon": "^7.3.1" @@ -48,15 +47,17 @@ "dependencies": { "abort-controller": "^3.0.0", "class-is": "^1.1.0", - "debug": "^3.1.0", + "debug": "^4.1.1", "err-code": "^1.1.2", "interface-connection": "~0.3.3", - "interface-transport": "libp2p/interface-transport#feat/async-await", + "interface-transport": "~0.4.0", "ip-address": "^5.8.9", + "it-pipe": "^1.0.0", "lodash.includes": "^4.3.0", "lodash.isfunction": "^3.0.9", - "mafmt": "^6.0.2", - "multiaddr": "^5.0.0" + "mafmt": "^6.0.7", + "multiaddr": "^6.0.6", + "streaming-iterables": "^4.1.0" }, "contributors": [ "Alan Shaw ", diff --git a/src/index.js b/src/index.js index e59da74..288c41e 100644 --- a/src/index.js +++ b/src/index.js @@ -78,7 +78,7 @@ class TCP { } handler = handler || noop - return createListener(handler) + return createListener(options, handler) } filter (multiaddrs) { diff --git a/src/listener.js b/src/listener.js index b7764b9..ec40998 100644 --- a/src/listener.js +++ b/src/listener.js @@ -2,11 +2,12 @@ const multiaddr = require('multiaddr') const os = require('os') -const includes = require('lodash.includes') const net = require('net') const EventEmitter = require('events').EventEmitter +const { AllListenersFailedError } = require('interface-transport') const debug = require('debug') const log = debug('libp2p:tcp:listen') +log.error = debug('libp2p:tcp:listen:error') const Libp2pSocket = require('./socket') const getMultiaddr = require('./get-multiaddr') @@ -14,144 +15,228 @@ const c = require('./constants') function noop () {} -module.exports = (handler) => { - const listener = new EventEmitter() +class Listener extends EventEmitter { + /** + * @constructor + * @param {object} options + * @param {function(Connection)} handler + */ + constructor (options, handler) { + super() + this._options = options + this._connectionHandler = handler + this._servers = new Set() + this.__connections = new Map() + } - const server = net.createServer((socket) => { - // Avoid uncaught errors caused by unstable connections - socket.on('error', noop) + /** + * Whether or not there is currently at least 1 server listening + * @private + * @returns {boolean} + */ + _isListening () { + return [...this._servers].filter(server => server.listening).length > 0 + } - const addr = getMultiaddr(socket) - if (!addr) { - if (socket.remoteAddress === undefined) { - log('connection closed before p2p connection made') - } else { - log('error interpreting incoming p2p connection') - } + /** + * Closes all open servers + * @param {object} options + * @param {number} options.timeout how long before closure is forced, defaults to 2000 ms + * @returns {Promise} + */ + close (options = {}) { + if (!this._isListening()) { return } - log('new connection', addr.toString()) - - const s = new Libp2pSocket(socket, addr) - trackSocket(server, socket) + // Close all running servers in parallel + return Promise.all( + [...this._servers].map(server => { + return new Promise((resolve) => { + const start = Date.now() + + // Attempt to stop the server. If it takes longer than the timeout, + // resolve the promise. Any remaining connections will be destroyed after + const timeout = setTimeout(() => { + log('Timeout closing server after %dms, destroying connections manually', Date.now() - start) + resolve() + }, options.timeout || c.CLOSE_TIMEOUT) + + // Just clear the timeout, cleanup listeners are added on server creation + server.once('close', () => clearTimeout(timeout)) + server.close((err) => { + // log the error and resolve so we don't exit early + err && log.error('an error occurred closing the server', err) + resolve() + }) + }) + }) + ).then(() => { + // Destroy all remaining connections + this.__connections.forEach((connection, key) => { + log('destroying %s', key) + connection.destroy() + }) + this.__connections.clear() + this._servers.clear() + }) + } - handler && handler(s) - listener.emit('connection', s) - }) + /** + * Creates servers listening on the given `addrs` + * @async + * @param {Array} addrs + */ + async listen (addrs) { + addrs = Array.isArray(addrs) ? addrs : [addrs] + + let listeners = [] + let errors = [] + + // Filter out duplicate ports, unless it's port 0 + addrs = uniqueBy(addrs, (addr) => { + const port = Number(addr.toOptions().port) + return isNaN(port) || port === 0 ? addr.toString() : port + }) - server.on('listening', () => listener.emit('listening')) - server.on('error', (err) => listener.emit('error', err)) - server.on('close', () => listener.emit('close')) + for (const ma of addrs) { + const lOpts = ma.toOptions() - // Keep track of open connections to destroy in case of timeout - server.__connections = {} + listeners.push( + new Promise((resolve) => { + const server = net.createServer(this._onSocket.bind(this)) + this._servers.add(server) - listener.close = (options = {}) => { - if (!server.listening) { - return - } + server.on('listening', () => this.emit('listening')) + server.on('close', () => { + this._removeServer(server) + }) + server.on('error', (err) => this.emit('error', err)) - return new Promise((resolve, reject) => { - const start = Date.now() + server.listen(lOpts.port, lOpts.host, (err) => { + if (err) { + errors.push(err) + return resolve() + } - // Attempt to stop the server. If it takes longer than the timeout, - // destroy all the underlying sockets manually. - const timeout = setTimeout(() => { - log('Timeout closing server after %dms, destroying connections manually', Date.now() - start) - Object.keys(server.__connections).forEach((key) => { - log('destroying %s', key) - server.__connections[key].destroy() + log('Listening on %s %s', lOpts.port, lOpts.host) + resolve() + }) }) - resolve() - }, options.timeout || c.CLOSE_TIMEOUT) - - server.once('close', () => clearTimeout(timeout)) - - server.close((err) => err ? reject(err) : resolve()) - }) - } - - let ipfsId - let listeningAddr - - listener.listen = (ma) => { - listeningAddr = ma - if (includes(ma.protoNames(), 'ipfs')) { - ipfsId = getIpfsId(ma) - listeningAddr = ma.decapsulate('ipfs') + ) } - const lOpts = listeningAddr.toOptions() - return new Promise((resolve, reject) => { - server.listen(lOpts.port, lOpts.host, (err) => { - if (err) { - return reject(err) - } + return Promise.all(listeners) + .then(() => { + errors.forEach((err) => { + log.error('received an error while attempting to listen', err) + }) - log('Listening on %s %s', lOpts.port, lOpts.host) - resolve() + // All servers failed to listen, throw an error + if (errors.length === listeners.length) { + throw new AllListenersFailedError() + } }) - }) } - listener.getAddrs = () => { - const multiaddrs = [] - const address = server.address() - - if (!address) { - throw new Error('Listener is not ready yet') + /** + * Removes the server from tracking and performs cleanup. + * If all servers have been closed, `close` will be emitted by + * the listener. + * @private + * @param {net.Server} server + */ + _removeServer (server) { + // only emit if we're not listening + if (!this._isListening()) { + this.emit('close') } + this._servers.delete(server) + server.removeAllListeners() + } - // Because TCP will only return the IPv6 version - // we need to capture from the passed multiaddr - if (listeningAddr.toString().indexOf('ip4') !== -1) { - let m = listeningAddr.decapsulate('tcp') - m = m.encapsulate('/tcp/' + address.port) - if (ipfsId) { - m = m.encapsulate('/ipfs/' + ipfsId) - } + /** + * Return the addresses we are listening on + * @throws + * @returns {Array} + */ + getAddrs () { + const multiaddrs = [] + this._servers.forEach(server => { + const address = server.address() - if (m.toString().indexOf('0.0.0.0') !== -1) { + if (address.address === '0.0.0.0') { const netInterfaces = os.networkInterfaces() Object.keys(netInterfaces).forEach((niKey) => { netInterfaces[niKey].forEach((ni) => { - if (ni.family === 'IPv4') { - multiaddrs.push(multiaddr(m.toString().replace('0.0.0.0', ni.address))) + if (address.family === ni.family) { + multiaddrs.push( + multiaddr.fromNodeAddress({ + ...address, + address: ni.address + }, 'tcp') + ) } }) }) } else { - multiaddrs.push(m) - } - } - - if (address.family === 'IPv6') { - let ma = multiaddr('/ip6/' + address.address + '/tcp/' + address.port) - if (ipfsId) { - ma = ma.encapsulate('/ipfs/' + ipfsId) + multiaddrs.push(multiaddr.fromNodeAddress(address, 'tcp')) } + }) - multiaddrs.push(ma) + if (multiaddrs.length === 0) { + throw new Error('Listener is not ready yet') } return multiaddrs } - return listener -} + /** + * Handler for new sockets from `net.createServer` + * @param {net.Socket} socket + */ + _onSocket (socket) { + // Avoid uncaught errors caused by unstable connections + socket.on('error', noop) + + const addr = getMultiaddr(socket) + if (!addr) { + if (socket.remoteAddress === undefined) { + log('connection closed before p2p connection made') + } else { + log('error interpreting incoming p2p connection') + } + return + } -function getIpfsId (ma) { - return ma.stringTuples().filter((tuple) => { - return tuple[0] === c.IPFS_MA_CODE - })[0][1] + log('new connection', addr.toString()) + + const s = new Libp2pSocket(socket, addr) + + // Track the connection + const key = `${socket.remoteAddress}:${socket.remotePort}` + this.__connections.set(key, socket) + socket.once('close', () => { + this.__connections.delete(key) + socket.removeAllListeners() + }) + + this._connectionHandler(s) + this.emit('connection', s) + } } -function trackSocket (server, socket) { - const key = `${socket.remoteAddress}:${socket.remotePort}` - server.__connections[key] = socket +module.exports = (options, handler) => { + return new Listener(options, handler) +} - socket.once('close', () => { - delete server.__connections[key] - }) +/** + * Get unique values from `arr` using `getValue` to determine + * what is used for uniqueness + * @param {Array} arr The array to get unique values for + * @param {function(value)} getValue The function to determine what is compared + * @returns {Array} + */ +function uniqueBy (arr, getValue) { + return [...new Map(arr.map((i) => [getValue(i), i])).values()] } diff --git a/test/adapter/listen-dial.spec.js b/test/adapter/listen-dial.spec.js index 97e6980..72c3cfc 100644 --- a/test/adapter/listen-dial.spec.js +++ b/test/adapter/listen-dial.spec.js @@ -114,14 +114,14 @@ describe('listen', () => { }) }) - it('getAddrs preserves IPFS Id', (done) => { + it('getAddrs does not preserve IPFS Id', (done) => { const mh = multiaddr('/ip4/127.0.0.1/tcp/9090/ipfs/Qmb6owHp6eaWArVbcJJbQSyifyJBttMMjYV76N2hMbf5Vw') const listener = tcp.createListener((conn) => {}) listener.listen(mh, () => { listener.getAddrs((err, multiaddrs) => { expect(err).to.not.exist() expect(multiaddrs.length).to.equal(1) - expect(multiaddrs[0]).to.deep.equal(mh) + expect(multiaddrs[0]).to.deep.equal(mh.decapsulate('ipfs')) listener.close(done) }) }) diff --git a/test/listen-dial.spec.js b/test/listen-dial.spec.js index e14e759..83a1876 100644 --- a/test/listen-dial.spec.js +++ b/test/listen-dial.spec.js @@ -38,7 +38,7 @@ describe('listen', () => { await new Promise((resolve) => { socket1.on('connect', async () => { - await listener.close() + await listener.close({ timeout: 100 }) resolve() }) }) @@ -115,14 +115,14 @@ describe('listen', () => { await listener.close() }) - it('getAddrs preserves IPFS Id', async () => { + it('getAddrs does not preserve IPFS Id', async () => { const mh = multiaddr('/ip4/127.0.0.1/tcp/9090/ipfs/Qmb6owHp6eaWArVbcJJbQSyifyJBttMMjYV76N2hMbf5Vw') const listener = tcp.createListener((conn) => {}) await listener.listen(mh) const multiaddrs = listener.getAddrs() expect(multiaddrs.length).to.equal(1) - expect(multiaddrs[0]).to.deep.equal(mh) + expect(multiaddrs[0]).to.deep.equal(mh.decapsulate('ipfs')) await listener.close() }) @@ -183,7 +183,7 @@ describe('dial', () => { handled = resolve }) - const ma = multiaddr('/ip6/::/tcp/9067') + const ma = multiaddr('/ip6/::/tcp/0') const listener = tcp.createListener(async (conn) => { await pipe( @@ -194,7 +194,8 @@ describe('dial', () => { }) await listener.listen(ma) - await pipe(await tcp.dial(ma)) + const addrs = listener.getAddrs() + await pipe(await tcp.dial(addrs[0])) await handledPromise await listener.close() @@ -210,7 +211,7 @@ describe('dial', () => { handled = resolve }) - const ma = multiaddr('/ip6/::/tcp/9068') + const ma = multiaddr('/ip6/::/tcp/0') const listener = tcp.createListener(async (conn) => { // pull(conn, pull.onEnd(destroyed)) @@ -219,7 +220,8 @@ describe('dial', () => { }) await listener.listen(ma) - await pipe([], await tcp.dial(ma)) + const addrs = listener.getAddrs() + await pipe([], await tcp.dial(addrs[0])) await handledPromise await listener.close() From 02bff187de0f3335231e3828119e6e9d4f18752a Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 23 Apr 2019 15:48:45 +0800 Subject: [PATCH 06/32] feat: abort after connect --- package.json | 1 + src/index.js | 12 +++++++----- src/listener.js | 2 +- src/socket.js | 29 +++++++++++++++++++++++++---- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index defd978..af8d858 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ }, "dependencies": { "abort-controller": "^3.0.0", + "abortable-iterator": "^2.0.0", "class-is": "^1.1.0", "debug": "^4.1.1", "err-code": "^1.1.2", diff --git a/src/index.js b/src/index.js index 288c41e..46f47b8 100644 --- a/src/index.js +++ b/src/index.js @@ -18,7 +18,7 @@ function noop () {} class TCP { async dial (ma, options) { const cOpts = ma.toOptions() - log('Connecting to %s:%s', cOpts.host, cOpts.port) + log('Dialing %s:%s', cOpts.host, cOpts.port) const rawSocket = await this._connect(cOpts, options) return new Libp2pSocket(rawSocket, ma, options) @@ -34,13 +34,14 @@ class TCP { const rawSocket = net.connect(cOpts) const onError = (err) => { - const msg = `Error connecting to ${cOpts.host}:${cOpts.port}: ${err.message}` + const msg = `Error dialing ${cOpts.host}:${cOpts.port}: ${err.message}` done(errcode(msg, err.code)) } const onTimeout = () => { - log('Timeout connecting to %s:%s', cOpts.host, cOpts.port) + log('Timeout dialing %s:%s', cOpts.host, cOpts.port) const err = errcode(`Timeout after ${Date.now() - start}ms`, 'ETIMEDOUT') + // Note: this will result in onError() being called rawSocket.emit('error', err) } @@ -50,7 +51,7 @@ class TCP { } const onAbort = () => { - log('Connect to %s:%s aborted', cOpts.host, cOpts.port) + log('Dial to %s:%s aborted', cOpts.host, cOpts.port) rawSocket.destroy() done(new AbortError()) } @@ -59,7 +60,8 @@ class TCP { rawSocket.removeListener('error', onError) rawSocket.removeListener('timeout', onTimeout) rawSocket.removeListener('connect', onConnect) - options.signal && options.signal.removeEventListener(onAbort) + + options.signal && options.signal.removeEventListener('abort', onAbort) err ? reject(err) : resolve(res) } diff --git a/src/listener.js b/src/listener.js index ec40998..8093360 100644 --- a/src/listener.js +++ b/src/listener.js @@ -119,7 +119,7 @@ class Listener extends EventEmitter { return resolve() } - log('Listening on %s %s', lOpts.port, lOpts.host) + log('Listening on %s:%s', lOpts.host, lOpts.port) resolve() }) }) diff --git a/src/socket.js b/src/socket.js index 345484e..c8b1507 100644 --- a/src/socket.js +++ b/src/socket.js @@ -1,27 +1,48 @@ 'use strict' +const abortable = require('abortable-iterator') const debug = require('debug') const log = debug('libp2p:tcp:socket') const c = require('./constants') class Libp2pSocket { - constructor (rawSocket, ma, opts) { + constructor (rawSocket, ma, opts = {}) { this._rawSocket = rawSocket this._ma = ma this.sink = this._sink(opts) - this.source = rawSocket + this.source = opts.signal ? abortable(rawSocket, opts.signal) : rawSocket } - _sink (opts = {}) { + _sink (opts) { // By default, close when the source is exhausted const closeOnEnd = opts.closeOnEnd !== false - return (source) => this._write(source, closeOnEnd) + + return async (source) => { + try { + const src = opts.signal ? abortable(source, opts.signal) : source + await this._write(src, closeOnEnd) + } catch (err) { + // If the connection is aborted just close the socket + if (err.type === 'aborted') { + return this.close() + } + + throw err + } + } } async _write (source, closeOnEnd) { for await (const data of source) { + if (this._rawSocket.destroyed) { + const cOpts = this._ma.toOptions() + log('Cannot write %d bytes to destroyed socket %s:%s', + data.length, cOpts.host, cOpts.port) + return + } + const flushed = this._rawSocket.write(data) if (!flushed) { await new Promise((resolve) => this._rawSocket.once('drain', resolve)) From 6bfa500a4543ae193cc504dc4e865ee3934aa47a Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 23 Apr 2019 19:58:55 +0800 Subject: [PATCH 07/32] chore: fix package.json --- package.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index af8d858..106a417 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,9 @@ "test:node": "aegir test -t node", "build": "aegir build", "docs": "aegir docs", - "release": "aegir release --docs -t node", - "release-minor": "aegir release --type minor --docs -t node", - "release-major": "aegir release --type major --docs -t node", + "release": "aegir release -t node --no-build", + "release-minor": "aegir release -t node --type minor --no-build", + "release-major": "aegir-release -t node --type major --no-build", "coverage": "aegir coverage", "coverage-publish": "aegir-coverage publish" }, @@ -45,7 +45,6 @@ "sinon": "^7.3.1" }, "dependencies": { - "abort-controller": "^3.0.0", "abortable-iterator": "^2.0.0", "class-is": "^1.1.0", "debug": "^4.1.1", From b108c9fdb138762516c8be1dda1e0540c7516859 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 29 Apr 2019 22:45:37 +0800 Subject: [PATCH 08/32] revert: "feat: listen to array of multiaddrs (#104)" This reverts commit 5009c2ca49f5e3f1ce8e8fe5f0bd88edbc13b5d8. --- .travis.yml | 11 +- README.md | 6 +- package.json | 11 +- src/index.js | 2 +- src/listener.js | 289 +++++++++++-------------------- test/adapter/listen-dial.spec.js | 4 +- test/listen-dial.spec.js | 16 +- 7 files changed, 124 insertions(+), 215 deletions(-) diff --git a/.travis.yml b/.travis.yml index d6481b1..99fe0ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,23 +9,20 @@ stages: node_js: - '10' +os: + - linux + - osx + script: npx nyc -s npm run test:node -- --bail after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov jobs: include: - - os: linux - sudo: false - before_script: sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6' - - os: windows filter_secrets: false cache: false - - os: osx - - stage: check - os: linux script: - npx aegir build --bundlesize - npx aegir commitlint --travis diff --git a/README.md b/README.md index b01f9a2..60519ca 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ const multiaddr = require('multiaddr') const pipe = require('it-pipe') const { collect } = require('streaming-iterables') -const addr = multiaddr('/ip4/127.0.0.1/tcp/9090') +const mh = multiaddr('/ip4/127.0.0.1/tcp/9090') const tcp = new TCP() @@ -56,10 +56,10 @@ const listener = tcp.createListener((socket) => { ) }) -await listener.listen([addr]) +await listener.listen(mh) console.log('listening') -const socket = await tcp.dial(addr) +const socket = await tcp.dial(mh) const values = await pipe( socket, collect diff --git a/package.json b/package.json index 106a417..c55d438 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "aegir": "^18.1.1", "chai": "^4.1.2", "dirty-chai": "^2.0.1", + "interface-transport": "~0.3.6", "lodash.isfunction": "^3.0.9", "pull-stream": "^3.6.9", "sinon": "^7.3.1" @@ -47,17 +48,15 @@ "dependencies": { "abortable-iterator": "^2.0.0", "class-is": "^1.1.0", - "debug": "^4.1.1", + "debug": "^3.1.0", "err-code": "^1.1.2", "interface-connection": "~0.3.3", - "interface-transport": "~0.4.0", + "interface-transport": "libp2p/interface-transport#feat/async-await", "ip-address": "^5.8.9", - "it-pipe": "^1.0.0", "lodash.includes": "^4.3.0", "lodash.isfunction": "^3.0.9", - "mafmt": "^6.0.7", - "multiaddr": "^6.0.6", - "streaming-iterables": "^4.1.0" + "mafmt": "^6.0.2", + "multiaddr": "^5.0.0" }, "contributors": [ "Alan Shaw ", diff --git a/src/index.js b/src/index.js index 46f47b8..000ac4d 100644 --- a/src/index.js +++ b/src/index.js @@ -80,7 +80,7 @@ class TCP { } handler = handler || noop - return createListener(options, handler) + return createListener(handler) } filter (multiaddrs) { diff --git a/src/listener.js b/src/listener.js index 8093360..b7764b9 100644 --- a/src/listener.js +++ b/src/listener.js @@ -2,12 +2,11 @@ const multiaddr = require('multiaddr') const os = require('os') +const includes = require('lodash.includes') const net = require('net') const EventEmitter = require('events').EventEmitter -const { AllListenersFailedError } = require('interface-transport') const debug = require('debug') const log = debug('libp2p:tcp:listen') -log.error = debug('libp2p:tcp:listen:error') const Libp2pSocket = require('./socket') const getMultiaddr = require('./get-multiaddr') @@ -15,228 +14,144 @@ const c = require('./constants') function noop () {} -class Listener extends EventEmitter { - /** - * @constructor - * @param {object} options - * @param {function(Connection)} handler - */ - constructor (options, handler) { - super() - this._options = options - this._connectionHandler = handler - this._servers = new Set() - this.__connections = new Map() - } +module.exports = (handler) => { + const listener = new EventEmitter() - /** - * Whether or not there is currently at least 1 server listening - * @private - * @returns {boolean} - */ - _isListening () { - return [...this._servers].filter(server => server.listening).length > 0 - } + const server = net.createServer((socket) => { + // Avoid uncaught errors caused by unstable connections + socket.on('error', noop) - /** - * Closes all open servers - * @param {object} options - * @param {number} options.timeout how long before closure is forced, defaults to 2000 ms - * @returns {Promise} - */ - close (options = {}) { - if (!this._isListening()) { + const addr = getMultiaddr(socket) + if (!addr) { + if (socket.remoteAddress === undefined) { + log('connection closed before p2p connection made') + } else { + log('error interpreting incoming p2p connection') + } return } - // Close all running servers in parallel - return Promise.all( - [...this._servers].map(server => { - return new Promise((resolve) => { - const start = Date.now() - - // Attempt to stop the server. If it takes longer than the timeout, - // resolve the promise. Any remaining connections will be destroyed after - const timeout = setTimeout(() => { - log('Timeout closing server after %dms, destroying connections manually', Date.now() - start) - resolve() - }, options.timeout || c.CLOSE_TIMEOUT) - - // Just clear the timeout, cleanup listeners are added on server creation - server.once('close', () => clearTimeout(timeout)) - server.close((err) => { - // log the error and resolve so we don't exit early - err && log.error('an error occurred closing the server', err) - resolve() - }) - }) - }) - ).then(() => { - // Destroy all remaining connections - this.__connections.forEach((connection, key) => { - log('destroying %s', key) - connection.destroy() - }) - this.__connections.clear() - this._servers.clear() - }) - } - - /** - * Creates servers listening on the given `addrs` - * @async - * @param {Array} addrs - */ - async listen (addrs) { - addrs = Array.isArray(addrs) ? addrs : [addrs] - - let listeners = [] - let errors = [] - - // Filter out duplicate ports, unless it's port 0 - addrs = uniqueBy(addrs, (addr) => { - const port = Number(addr.toOptions().port) - return isNaN(port) || port === 0 ? addr.toString() : port - }) + log('new connection', addr.toString()) - for (const ma of addrs) { - const lOpts = ma.toOptions() + const s = new Libp2pSocket(socket, addr) + trackSocket(server, socket) - listeners.push( - new Promise((resolve) => { - const server = net.createServer(this._onSocket.bind(this)) - this._servers.add(server) + handler && handler(s) + listener.emit('connection', s) + }) - server.on('listening', () => this.emit('listening')) - server.on('close', () => { - this._removeServer(server) - }) - server.on('error', (err) => this.emit('error', err)) + server.on('listening', () => listener.emit('listening')) + server.on('error', (err) => listener.emit('error', err)) + server.on('close', () => listener.emit('close')) - server.listen(lOpts.port, lOpts.host, (err) => { - if (err) { - errors.push(err) - return resolve() - } + // Keep track of open connections to destroy in case of timeout + server.__connections = {} - log('Listening on %s:%s', lOpts.host, lOpts.port) - resolve() - }) - }) - ) + listener.close = (options = {}) => { + if (!server.listening) { + return } - return Promise.all(listeners) - .then(() => { - errors.forEach((err) => { - log.error('received an error while attempting to listen', err) + return new Promise((resolve, reject) => { + const start = Date.now() + + // Attempt to stop the server. If it takes longer than the timeout, + // destroy all the underlying sockets manually. + const timeout = setTimeout(() => { + log('Timeout closing server after %dms, destroying connections manually', Date.now() - start) + Object.keys(server.__connections).forEach((key) => { + log('destroying %s', key) + server.__connections[key].destroy() }) + resolve() + }, options.timeout || c.CLOSE_TIMEOUT) - // All servers failed to listen, throw an error - if (errors.length === listeners.length) { - throw new AllListenersFailedError() - } - }) + server.once('close', () => clearTimeout(timeout)) + + server.close((err) => err ? reject(err) : resolve()) + }) } - /** - * Removes the server from tracking and performs cleanup. - * If all servers have been closed, `close` will be emitted by - * the listener. - * @private - * @param {net.Server} server - */ - _removeServer (server) { - // only emit if we're not listening - if (!this._isListening()) { - this.emit('close') + let ipfsId + let listeningAddr + + listener.listen = (ma) => { + listeningAddr = ma + if (includes(ma.protoNames(), 'ipfs')) { + ipfsId = getIpfsId(ma) + listeningAddr = ma.decapsulate('ipfs') } - this._servers.delete(server) - server.removeAllListeners() + + const lOpts = listeningAddr.toOptions() + return new Promise((resolve, reject) => { + server.listen(lOpts.port, lOpts.host, (err) => { + if (err) { + return reject(err) + } + + log('Listening on %s %s', lOpts.port, lOpts.host) + resolve() + }) + }) } - /** - * Return the addresses we are listening on - * @throws - * @returns {Array} - */ - getAddrs () { + listener.getAddrs = () => { const multiaddrs = [] - this._servers.forEach(server => { - const address = server.address() + const address = server.address() - if (address.address === '0.0.0.0') { + if (!address) { + throw new Error('Listener is not ready yet') + } + + // Because TCP will only return the IPv6 version + // we need to capture from the passed multiaddr + if (listeningAddr.toString().indexOf('ip4') !== -1) { + let m = listeningAddr.decapsulate('tcp') + m = m.encapsulate('/tcp/' + address.port) + if (ipfsId) { + m = m.encapsulate('/ipfs/' + ipfsId) + } + + if (m.toString().indexOf('0.0.0.0') !== -1) { const netInterfaces = os.networkInterfaces() Object.keys(netInterfaces).forEach((niKey) => { netInterfaces[niKey].forEach((ni) => { - if (address.family === ni.family) { - multiaddrs.push( - multiaddr.fromNodeAddress({ - ...address, - address: ni.address - }, 'tcp') - ) + if (ni.family === 'IPv4') { + multiaddrs.push(multiaddr(m.toString().replace('0.0.0.0', ni.address))) } }) }) } else { - multiaddrs.push(multiaddr.fromNodeAddress(address, 'tcp')) + multiaddrs.push(m) } - }) - - if (multiaddrs.length === 0) { - throw new Error('Listener is not ready yet') } - return multiaddrs - } - - /** - * Handler for new sockets from `net.createServer` - * @param {net.Socket} socket - */ - _onSocket (socket) { - // Avoid uncaught errors caused by unstable connections - socket.on('error', noop) - - const addr = getMultiaddr(socket) - if (!addr) { - if (socket.remoteAddress === undefined) { - log('connection closed before p2p connection made') - } else { - log('error interpreting incoming p2p connection') + if (address.family === 'IPv6') { + let ma = multiaddr('/ip6/' + address.address + '/tcp/' + address.port) + if (ipfsId) { + ma = ma.encapsulate('/ipfs/' + ipfsId) } - return - } - log('new connection', addr.toString()) - - const s = new Libp2pSocket(socket, addr) - - // Track the connection - const key = `${socket.remoteAddress}:${socket.remotePort}` - this.__connections.set(key, socket) - socket.once('close', () => { - this.__connections.delete(key) - socket.removeAllListeners() - }) + multiaddrs.push(ma) + } - this._connectionHandler(s) - this.emit('connection', s) + return multiaddrs } + + return listener } -module.exports = (options, handler) => { - return new Listener(options, handler) +function getIpfsId (ma) { + return ma.stringTuples().filter((tuple) => { + return tuple[0] === c.IPFS_MA_CODE + })[0][1] } -/** - * Get unique values from `arr` using `getValue` to determine - * what is used for uniqueness - * @param {Array} arr The array to get unique values for - * @param {function(value)} getValue The function to determine what is compared - * @returns {Array} - */ -function uniqueBy (arr, getValue) { - return [...new Map(arr.map((i) => [getValue(i), i])).values()] +function trackSocket (server, socket) { + const key = `${socket.remoteAddress}:${socket.remotePort}` + server.__connections[key] = socket + + socket.once('close', () => { + delete server.__connections[key] + }) } diff --git a/test/adapter/listen-dial.spec.js b/test/adapter/listen-dial.spec.js index 72c3cfc..97e6980 100644 --- a/test/adapter/listen-dial.spec.js +++ b/test/adapter/listen-dial.spec.js @@ -114,14 +114,14 @@ describe('listen', () => { }) }) - it('getAddrs does not preserve IPFS Id', (done) => { + it('getAddrs preserves IPFS Id', (done) => { const mh = multiaddr('/ip4/127.0.0.1/tcp/9090/ipfs/Qmb6owHp6eaWArVbcJJbQSyifyJBttMMjYV76N2hMbf5Vw') const listener = tcp.createListener((conn) => {}) listener.listen(mh, () => { listener.getAddrs((err, multiaddrs) => { expect(err).to.not.exist() expect(multiaddrs.length).to.equal(1) - expect(multiaddrs[0]).to.deep.equal(mh.decapsulate('ipfs')) + expect(multiaddrs[0]).to.deep.equal(mh) listener.close(done) }) }) diff --git a/test/listen-dial.spec.js b/test/listen-dial.spec.js index 83a1876..e14e759 100644 --- a/test/listen-dial.spec.js +++ b/test/listen-dial.spec.js @@ -38,7 +38,7 @@ describe('listen', () => { await new Promise((resolve) => { socket1.on('connect', async () => { - await listener.close({ timeout: 100 }) + await listener.close() resolve() }) }) @@ -115,14 +115,14 @@ describe('listen', () => { await listener.close() }) - it('getAddrs does not preserve IPFS Id', async () => { + it('getAddrs preserves IPFS Id', async () => { const mh = multiaddr('/ip4/127.0.0.1/tcp/9090/ipfs/Qmb6owHp6eaWArVbcJJbQSyifyJBttMMjYV76N2hMbf5Vw') const listener = tcp.createListener((conn) => {}) await listener.listen(mh) const multiaddrs = listener.getAddrs() expect(multiaddrs.length).to.equal(1) - expect(multiaddrs[0]).to.deep.equal(mh.decapsulate('ipfs')) + expect(multiaddrs[0]).to.deep.equal(mh) await listener.close() }) @@ -183,7 +183,7 @@ describe('dial', () => { handled = resolve }) - const ma = multiaddr('/ip6/::/tcp/0') + const ma = multiaddr('/ip6/::/tcp/9067') const listener = tcp.createListener(async (conn) => { await pipe( @@ -194,8 +194,7 @@ describe('dial', () => { }) await listener.listen(ma) - const addrs = listener.getAddrs() - await pipe(await tcp.dial(addrs[0])) + await pipe(await tcp.dial(ma)) await handledPromise await listener.close() @@ -211,7 +210,7 @@ describe('dial', () => { handled = resolve }) - const ma = multiaddr('/ip6/::/tcp/0') + const ma = multiaddr('/ip6/::/tcp/9068') const listener = tcp.createListener(async (conn) => { // pull(conn, pull.onEnd(destroyed)) @@ -220,8 +219,7 @@ describe('dial', () => { }) await listener.listen(ma) - const addrs = listener.getAddrs() - await pipe([], await tcp.dial(addrs[0])) + await pipe([], await tcp.dial(ma)) await handledPromise await listener.close() From 5869c0440d1d450cf70db16612f79d5f964bb7cd Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 29 Apr 2019 23:02:58 +0800 Subject: [PATCH 09/32] chore: fix travis file --- .travis.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 99fe0ef..d6481b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,20 +9,23 @@ stages: node_js: - '10' -os: - - linux - - osx - script: npx nyc -s npm run test:node -- --bail after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov jobs: include: + - os: linux + sudo: false + before_script: sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6' + - os: windows filter_secrets: false cache: false + - os: osx + - stage: check + os: linux script: - npx aegir build --bundlesize - npx aegir commitlint --travis From bfbfd07607c4d7a76984c0a8936ce01d3153d55c Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 29 Apr 2019 23:03:32 +0800 Subject: [PATCH 10/32] chore: nicer addr name in README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 60519ca..5269938 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ const multiaddr = require('multiaddr') const pipe = require('it-pipe') const { collect } = require('streaming-iterables') -const mh = multiaddr('/ip4/127.0.0.1/tcp/9090') +const addr = multiaddr('/ip4/127.0.0.1/tcp/9090') const tcp = new TCP() @@ -56,10 +56,10 @@ const listener = tcp.createListener((socket) => { ) }) -await listener.listen(mh) +await listener.listen(addr) console.log('listening') -const socket = await tcp.dial(mh) +const socket = await tcp.dial(addr) const values = await pipe( socket, collect From 4c3c8111e7bfaaa5e2cf03f9671bed053c1b96d3 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 29 Apr 2019 23:03:45 +0800 Subject: [PATCH 11/32] chore: update packages --- package.json | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index c55d438..106a417 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,6 @@ "aegir": "^18.1.1", "chai": "^4.1.2", "dirty-chai": "^2.0.1", - "interface-transport": "~0.3.6", "lodash.isfunction": "^3.0.9", "pull-stream": "^3.6.9", "sinon": "^7.3.1" @@ -48,15 +47,17 @@ "dependencies": { "abortable-iterator": "^2.0.0", "class-is": "^1.1.0", - "debug": "^3.1.0", + "debug": "^4.1.1", "err-code": "^1.1.2", "interface-connection": "~0.3.3", - "interface-transport": "libp2p/interface-transport#feat/async-await", + "interface-transport": "~0.4.0", "ip-address": "^5.8.9", + "it-pipe": "^1.0.0", "lodash.includes": "^4.3.0", "lodash.isfunction": "^3.0.9", - "mafmt": "^6.0.2", - "multiaddr": "^5.0.0" + "mafmt": "^6.0.7", + "multiaddr": "^6.0.6", + "streaming-iterables": "^4.1.0" }, "contributors": [ "Alan Shaw ", From 5fde9cd6447a08b5caf689acacba5d2e332a80f9 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 29 Apr 2019 23:04:12 +0800 Subject: [PATCH 12/32] test: fix listen addresses in tests --- test/listen-dial.spec.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/listen-dial.spec.js b/test/listen-dial.spec.js index e14e759..ad414fa 100644 --- a/test/listen-dial.spec.js +++ b/test/listen-dial.spec.js @@ -38,7 +38,7 @@ describe('listen', () => { await new Promise((resolve) => { socket1.on('connect', async () => { - await listener.close() + await listener.close({ timeout: 100 }) resolve() }) }) @@ -183,7 +183,7 @@ describe('dial', () => { handled = resolve }) - const ma = multiaddr('/ip6/::/tcp/9067') + const ma = multiaddr('/ip6/::/tcp/0') const listener = tcp.createListener(async (conn) => { await pipe( @@ -194,7 +194,8 @@ describe('dial', () => { }) await listener.listen(ma) - await pipe(await tcp.dial(ma)) + const addrs = listener.getAddrs() + await pipe(await tcp.dial(addrs[0])) await handledPromise await listener.close() @@ -210,7 +211,7 @@ describe('dial', () => { handled = resolve }) - const ma = multiaddr('/ip6/::/tcp/9068') + const ma = multiaddr('/ip6/::/tcp/0') const listener = tcp.createListener(async (conn) => { // pull(conn, pull.onEnd(destroyed)) @@ -219,7 +220,8 @@ describe('dial', () => { }) await listener.listen(ma) - await pipe([], await tcp.dial(ma)) + const addrs = listener.getAddrs() + await pipe(await tcp.dial(addrs[0])) await handledPromise await listener.close() From 3ff9ed4321a0f07fe75cbfef53c4184ab7ca3df6 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 29 Apr 2019 23:14:33 +0800 Subject: [PATCH 13/32] chore: update interface-transport --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 106a417..80861dd 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "debug": "^4.1.1", "err-code": "^1.1.2", "interface-connection": "~0.3.3", - "interface-transport": "~0.4.0", + "interface-transport": "~0.5.0", "ip-address": "^5.8.9", "it-pipe": "^1.0.0", "lodash.includes": "^4.3.0", From d9230e80050e5ca39ba51b2b12f1d2a294bd1a77 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 1 May 2019 16:32:07 +0800 Subject: [PATCH 14/32] fix: throw error on write to destroyed socket --- src/socket.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/socket.js b/src/socket.js index c8b1507..19a4128 100644 --- a/src/socket.js +++ b/src/socket.js @@ -38,9 +38,8 @@ class Libp2pSocket { for await (const data of source) { if (this._rawSocket.destroyed) { const cOpts = this._ma.toOptions() - log('Cannot write %d bytes to destroyed socket %s:%s', + throw new Error('Cannot write %d bytes to destroyed socket %s:%s', data.length, cOpts.host, cOpts.port) - return } const flushed = this._rawSocket.write(data) From c61e9ea2af19885267cb17d1fd91b8f2d8028b12 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 1 May 2019 19:53:14 +0800 Subject: [PATCH 15/32] chore: log errors emited by server handler socket --- src/listener.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/listener.js b/src/listener.js index b7764b9..604b55d 100644 --- a/src/listener.js +++ b/src/listener.js @@ -7,19 +7,20 @@ const net = require('net') const EventEmitter = require('events').EventEmitter const debug = require('debug') const log = debug('libp2p:tcp:listen') +const logError = debug('libp2p:tcp:listen:error') const Libp2pSocket = require('./socket') const getMultiaddr = require('./get-multiaddr') const c = require('./constants') -function noop () {} - module.exports = (handler) => { const listener = new EventEmitter() const server = net.createServer((socket) => { // Avoid uncaught errors caused by unstable connections - socket.on('error', noop) + socket.on('error', (err) => { + logError('Error emitted by server handler socket: ' + err.message) + }) const addr = getMultiaddr(socket) if (!addr) { From a658537f86e2d4dc57397b3b247179b30ef021a6 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 6 Sep 2019 17:57:03 +0100 Subject: [PATCH 16/32] refactor: many refactor, maybe work? License: MIT Signed-off-by: Alan Shaw --- .travis.yml | 1 + appveyor.yml | 29 --------------- ci/Jenkinsfile | 2 - circle.yml | 15 -------- package.json | 8 ++-- src/constants.js | 7 ++-- src/get-multiaddr.js | 33 ----------------- src/index.js | 83 ++++++++++++++++++++---------------------- src/listener.js | 75 +++++++++++++++----------------------- src/socket-to-conn.js | 82 +++++++++++++++++++++++++++++++++++++++++ src/socket.js | 85 ------------------------------------------- 11 files changed, 159 insertions(+), 261 deletions(-) delete mode 100644 appveyor.yml delete mode 100644 ci/Jenkinsfile delete mode 100644 circle.yml delete mode 100644 src/get-multiaddr.js create mode 100644 src/socket-to-conn.js delete mode 100644 src/socket.js diff --git a/.travis.yml b/.travis.yml index d6481b1..395da31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ stages: node_js: - '10' + - '12' script: npx nyc -s npm run test:node -- --bail after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 046bf91..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Warning: This file is automatically synced from https://github.com/ipfs/ci-sync so if you want to change it, please change it there and ask someone to sync all repositories. -version: "{build}" - -environment: - matrix: - - nodejs_version: "6" - - nodejs_version: "8" - -matrix: - fast_finish: true - -install: - # Install Node.js - - ps: Install-Product node $env:nodejs_version - - # Upgrade npm - - npm install -g npm - - # Output our current versions for debugging - - node --version - - npm --version - - # Install our package dependencies - - npm install - -test_script: - - npm run test:node - -build: off diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile deleted file mode 100644 index a7da2e5..0000000 --- a/ci/Jenkinsfile +++ /dev/null @@ -1,2 +0,0 @@ -// Warning: This file is automatically synced from https://github.com/ipfs/ci-sync so if you want to change it, please change it there and ask someone to sync all repositories. -javascript() diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 0009693..0000000 --- a/circle.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Warning: This file is automatically synced from https://github.com/ipfs/ci-sync so if you want to change it, please change it there and ask someone to sync all repositories. -machine: - node: - version: stable - -dependencies: - pre: - - google-chrome --version - - curl -L -o google-chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - - sudo dpkg -i google-chrome.deb || true - - sudo apt-get update - - sudo apt-get install -f - - sudo apt-get install --only-upgrade lsb-base - - sudo dpkg -i google-chrome.deb - - google-chrome --version diff --git a/package.json b/package.json index 80861dd..840f520 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "npm": ">=3.0.0" }, "devDependencies": { - "aegir": "^18.1.1", + "aegir": "^20.0.0", "chai": "^4.1.2", "dirty-chai": "^2.0.1", "lodash.isfunction": "^3.0.9", @@ -48,15 +48,15 @@ "abortable-iterator": "^2.0.0", "class-is": "^1.1.0", "debug": "^4.1.1", - "err-code": "^1.1.2", + "err-code": "^2.0.0", "interface-connection": "~0.3.3", "interface-transport": "~0.5.0", - "ip-address": "^5.8.9", + "ip-address": "^6.1.0", "it-pipe": "^1.0.0", - "lodash.includes": "^4.3.0", "lodash.isfunction": "^3.0.9", "mafmt": "^6.0.7", "multiaddr": "^6.0.6", + "stream-to-it": "^0.1.1", "streaming-iterables": "^4.1.0" }, "contributors": [ diff --git a/src/constants.js b/src/constants.js index 3e5cb6e..067be58 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,8 +1,7 @@ 'use strict' // IPFS multi-address code -module.exports.IPFS_MA_CODE = 421 +exports.IPFS_MA_CODE = 421 -// Time to wait for a connection to close gracefully before destroying it -// manually -module.exports.CLOSE_TIMEOUT = 2000 +// Time to wait for a connection to close gracefully before destroying it manually +exports.CLOSE_TIMEOUT = 2000 diff --git a/src/get-multiaddr.js b/src/get-multiaddr.js deleted file mode 100644 index 4be7243..0000000 --- a/src/get-multiaddr.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict' - -const multiaddr = require('multiaddr') -const Address6 = require('ip-address').Address6 -const debug = require('debug') -const log = debug('libp2p:tcp:get-multiaddr') - -module.exports = (socket) => { - let ma - - try { - if (socket.remoteFamily === 'IPv6') { - const addr = new Address6(socket.remoteAddress) - - if (addr.v4) { - const ip4 = addr.to4().correctForm() - ma = multiaddr('/ip4/' + ip4 + - '/tcp/' + socket.remotePort - ) - } else { - ma = multiaddr('/ip6/' + socket.remoteAddress + - '/tcp/' + socket.remotePort - ) - } - } else { - ma = multiaddr('/ip4/' + socket.remoteAddress + - '/tcp/' + socket.remotePort) - } - } catch (err) { - log(err) - } - return ma -} diff --git a/src/index.js b/src/index.js index 000ac4d..8feb5fa 100644 --- a/src/index.js +++ b/src/index.js @@ -3,97 +3,94 @@ const net = require('net') const mafmt = require('mafmt') const withIs = require('class-is') -const includes = require('lodash.includes') -const isFunction = require('lodash.isfunction') -const errcode = require('err-code') -const debug = require('debug') -const log = debug('libp2p:tcp:dial') - -const Libp2pSocket = require('./socket') +const errCode = require('err-code') +const log = require('debug')('libp2p:tcp') +const toConnection = require('./socket-to-conn') const createListener = require('./listener') const { AbortError } = require('interface-transport') -function noop () {} - class TCP { - async dial (ma, options) { - const cOpts = ma.toOptions() - log('Dialing %s:%s', cOpts.host, cOpts.port) + constructor ({ upgrader }) { + this._upgrader = upgrader + } - const rawSocket = await this._connect(cOpts, options) - return new Libp2pSocket(rawSocket, ma, options) + async dial (ma, options) { + const socket = await this._connect(ma, options) + return this._upgrader.upgradeOutbound(toConnection(socket, options)) } - _connect (cOpts, options = {}) { - return new Promise((resolve, reject) => { - if ((options.signal || {}).aborted) { - return reject(new AbortError()) - } + _connect (ma, options = {}) { + if (options.signal && options.signal.aborted) { + throw new AbortError() + } + return new Promise((resolve, reject) => { const start = Date.now() + const cOpts = ma.toOptions() + + log('dialing %s:%s', cOpts.host, cOpts.port) const rawSocket = net.connect(cOpts) - const onError = (err) => { - const msg = `Error dialing ${cOpts.host}:${cOpts.port}: ${err.message}` - done(errcode(msg, err.code)) + const onError = err => { + err.message = `connection error ${cOpts.host}:${cOpts.port}: ${err.message}` + done(err) } const onTimeout = () => { - log('Timeout dialing %s:%s', cOpts.host, cOpts.port) - const err = errcode(`Timeout after ${Date.now() - start}ms`, 'ETIMEDOUT') + log('connnection timeout %s:%s', cOpts.host, cOpts.port) + const err = errCode(new Error(`connection timeout after ${Date.now() - start}ms`), 'ETIMEDOUT') // Note: this will result in onError() being called rawSocket.emit('error', err) } const onConnect = () => { - log('Connected to %s:%s', cOpts.host, cOpts.port) - done(null, rawSocket) + log('connection opened %s:%s', cOpts.host, cOpts.port) + done() } const onAbort = () => { - log('Dial to %s:%s aborted', cOpts.host, cOpts.port) + log('connection aborted %s:%s', cOpts.host, cOpts.port) rawSocket.destroy() done(new AbortError()) } - const done = (err, res) => { + const done = err => { rawSocket.removeListener('error', onError) rawSocket.removeListener('timeout', onTimeout) rawSocket.removeListener('connect', onConnect) - options.signal && options.signal.removeEventListener('abort', onAbort) - err ? reject(err) : resolve(res) + if (err) return reject(err) + resolve(rawSocket) } - rawSocket.once('error', onError) - rawSocket.once('timeout', onTimeout) - rawSocket.once('connect', onConnect) + rawSocket.on('error', onError) + rawSocket.on('timeout', onTimeout) + rawSocket.on('connect', onConnect) options.signal && options.signal.addEventListener('abort', onAbort) }) } createListener (options, handler) { - if (isFunction(options)) { + if (typeof options === 'function') { handler = options options = {} } - - handler = handler || noop - return createListener(handler) + options = options || {} + return createListener({ handler, upgrader: this._upgrader }, options) } filter (multiaddrs) { - if (!Array.isArray(multiaddrs)) { - multiaddrs = [multiaddrs] - } + multiaddrs = Array.isArray(multiaddrs) ? multiaddrs : [multiaddrs] + + return multiaddrs.filter(ma => { + const protos = ma.protoNames() - return multiaddrs.filter((ma) => { - if (includes(ma.protoNames(), 'p2p-circuit')) { + if (protos.includes('p2p-circuit')) { return false } - if (includes(ma.protoNames(), 'ipfs')) { + if (protos.includes('ipfs')) { ma = ma.decapsulate('ipfs') } diff --git a/src/listener.js b/src/listener.js index 604b55d..690f45f 100644 --- a/src/listener.js +++ b/src/listener.js @@ -2,48 +2,33 @@ const multiaddr = require('multiaddr') const os = require('os') -const includes = require('lodash.includes') const net = require('net') -const EventEmitter = require('events').EventEmitter -const debug = require('debug') -const log = debug('libp2p:tcp:listen') -const logError = debug('libp2p:tcp:listen:error') +const EventEmitter = require('events') +const log = require('debug')('libp2p:tcp:listener') +const toConnection = require('./socket-to-conn') +const { IPFS_MA_CODE, CLOSE_TIMEOUT } = require('./constants') -const Libp2pSocket = require('./socket') -const getMultiaddr = require('./get-multiaddr') -const c = require('./constants') - -module.exports = (handler) => { +module.exports = ({ handler, upgrader }, options) => { const listener = new EventEmitter() - const server = net.createServer((socket) => { + const server = net.createServer(async socket => { // Avoid uncaught errors caused by unstable connections - socket.on('error', (err) => { - logError('Error emitted by server handler socket: ' + err.message) - }) - - const addr = getMultiaddr(socket) - if (!addr) { - if (socket.remoteAddress === undefined) { - log('connection closed before p2p connection made') - } else { - log('error interpreting incoming p2p connection') - } - return - } + socket.on('error', err => log('socket error', err)) - log('new connection', addr.toString()) + const maConn = toConnection(socket) + const conn = upgrader.upgradeInbound(maConn) + log('new connection %s', conn.remoteAddr) - const s = new Libp2pSocket(socket, addr) - trackSocket(server, socket) + trackConn(server, maConn) - handler && handler(s) - listener.emit('connection', s) + if (handler) handler(conn) + listener.emit('connection', conn) }) - server.on('listening', () => listener.emit('listening')) - server.on('error', (err) => listener.emit('error', err)) - server.on('close', () => listener.emit('close')) + server + .on('listening', () => listener.emit('listening')) + .on('error', err => listener.emit('error', err)) + .on('close', () => listener.emit('close')) // Keep track of open connections to destroy in case of timeout server.__connections = {} @@ -60,16 +45,16 @@ module.exports = (handler) => { // destroy all the underlying sockets manually. const timeout = setTimeout(() => { log('Timeout closing server after %dms, destroying connections manually', Date.now() - start) - Object.keys(server.__connections).forEach((key) => { + Object.keys(server.__connections).forEach(key => { log('destroying %s', key) - server.__connections[key].destroy() + server.__connections[key].conn.destroy() }) resolve() - }, options.timeout || c.CLOSE_TIMEOUT) - - server.once('close', () => clearTimeout(timeout)) + }, options.timeout || CLOSE_TIMEOUT) - server.close((err) => err ? reject(err) : resolve()) + server + .once('close', () => clearTimeout(timeout)) + .close(err => err ? reject(err) : resolve()) }) } @@ -78,7 +63,7 @@ module.exports = (handler) => { listener.listen = (ma) => { listeningAddr = ma - if (includes(ma.protoNames(), 'ipfs')) { + if (ma.protoNames().includes('ipfs')) { ipfsId = getIpfsId(ma) listeningAddr = ma.decapsulate('ipfs') } @@ -143,16 +128,14 @@ module.exports = (handler) => { } function getIpfsId (ma) { - return ma.stringTuples().filter((tuple) => { - return tuple[0] === c.IPFS_MA_CODE - })[0][1] + return ma.stringTuples().filter(tuple => tuple[0] === IPFS_MA_CODE)[0][1] } -function trackSocket (server, socket) { - const key = `${socket.remoteAddress}:${socket.remotePort}` - server.__connections[key] = socket +function trackConn (server, maConn) { + const key = maConn.remoteAddr.toString() + server.__connections[key] = maConn - socket.once('close', () => { + maConn.conn.once('close', () => { delete server.__connections[key] }) } diff --git a/src/socket-to-conn.js b/src/socket-to-conn.js new file mode 100644 index 0000000..4ded6f4 --- /dev/null +++ b/src/socket-to-conn.js @@ -0,0 +1,82 @@ +'use strict' + +const abortable = require('abortable-iterator') +const log = require('debug')('libp2p:tcp:socket') +const toIterable = require('stream-to-it') +const Multiaddr = require('multiaddr') +const { Address6 } = require('ip-address') +const { CLOSE_TIMEOUT } = require('./constants') + +// Convert a socket into a MultiaddrConnection +// https://github.com/libp2p/interface-transport#multiaddrconnection +module.exports = function toConnection (socket, options) { + options = options || {} + + const duplex = toIterable.duplex(socket) + const { sink } = duplex + + duplex.sink = async source => { + try { + await sink(source) + } catch (err) { + // If aborted we can safely ignore + if (err.type !== 'aborted') { + // If the source errored the socket will already have been destroyed by + // toIterable.duplex(). If the socket errored it will already be + // destroyed. There's nothing to do here except log the error & return. + log(err) + } + } + } + + const conn = options.signal ? abortable.duplex(duplex, options.signal) : duplex + + conn.conn = socket + + conn.localAddr = (() => { + const ip = socket.localAddress + const port = socket.localPort + return Multiaddr(`/ip4/${ip}/tcp/${port}`) // FIXME: ip6?? + })() + + conn.remoteAddr = (() => { + let proto = 'ip4' + let ip = socket.remoteAddress + + if (socket.remoteFamily === 'IPv6') { + const ip6 = new Address6(ip) + + if (ip6.is4()) { + ip = ip6.to4().correctForm() + } else { + proto = 'ip6' + } + } + + return Multiaddr(`/${proto}/${ip}/tcp/${socket.remotePort}`) + })() + + conn.close = async options => { + options = options || {} + if (socket.destroyed) return + + return new Promise((resolve, reject) => { + const start = Date.now() + + // Attempt to end the socket. If it takes longer to close than the + // timeout, destroy it manually. + const timeout = setTimeout(() => { + const { host, port } = conn.remoteAddr.toOptions() + log('Timeout closing socket to %s:%s after %dms, destroying it manually', + host, port, Date.now() - start) + socket.destroy() + resolve() + }, options.timeout || CLOSE_TIMEOUT) + + socket.once('close', () => clearTimeout(timeout)) + socket.end(err => err ? reject(err) : resolve()) + }) + } + + return conn +} diff --git a/src/socket.js b/src/socket.js deleted file mode 100644 index 19a4128..0000000 --- a/src/socket.js +++ /dev/null @@ -1,85 +0,0 @@ -'use strict' - -const abortable = require('abortable-iterator') -const debug = require('debug') -const log = debug('libp2p:tcp:socket') - -const c = require('./constants') - -class Libp2pSocket { - constructor (rawSocket, ma, opts = {}) { - this._rawSocket = rawSocket - this._ma = ma - - this.sink = this._sink(opts) - this.source = opts.signal ? abortable(rawSocket, opts.signal) : rawSocket - } - - _sink (opts) { - // By default, close when the source is exhausted - const closeOnEnd = opts.closeOnEnd !== false - - return async (source) => { - try { - const src = opts.signal ? abortable(source, opts.signal) : source - await this._write(src, closeOnEnd) - } catch (err) { - // If the connection is aborted just close the socket - if (err.type === 'aborted') { - return this.close() - } - - throw err - } - } - } - - async _write (source, closeOnEnd) { - for await (const data of source) { - if (this._rawSocket.destroyed) { - const cOpts = this._ma.toOptions() - throw new Error('Cannot write %d bytes to destroyed socket %s:%s', - data.length, cOpts.host, cOpts.port) - } - - const flushed = this._rawSocket.write(data) - if (!flushed) { - await new Promise((resolve) => this._rawSocket.once('drain', resolve)) - } - } - - if (closeOnEnd) { - await this.close() - } - } - - close (opts = {}) { - if (this._rawSocket.pending || this._rawSocket.destroyed) { - return - } - - return new Promise((resolve, reject) => { - const start = Date.now() - - // Attempt to end the socket. If it takes longer to close than the - // timeout, destroy it manually. - const timeout = setTimeout(() => { - const cOpts = this._ma.toOptions() - log('Timeout closing socket to %s:%s after %dms, destroying it manually', - cOpts.host, cOpts.port, Date.now() - start) - this._rawSocket.destroy() - resolve() - }, opts.timeout || c.CLOSE_TIMEOUT) - - this._rawSocket.once('close', () => clearTimeout(timeout)) - - this._rawSocket.end((err) => err ? reject(err) : resolve()) - }) - } - - getObservedAddrs () { - return [this._ma] - } -} - -module.exports = Libp2pSocket From ebcfdd6de331744b87880380f481eae95cb5e3c2 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 8 Sep 2019 13:59:40 +0100 Subject: [PATCH 17/32] fix: await on upgrade License: MIT Signed-off-by: Alan Shaw --- src/index.js | 6 +++++- src/listener.js | 30 +++++++++++++++++++----------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/index.js b/src/index.js index 8feb5fa..3f51374 100644 --- a/src/index.js +++ b/src/index.js @@ -16,7 +16,11 @@ class TCP { async dial (ma, options) { const socket = await this._connect(ma, options) - return this._upgrader.upgradeOutbound(toConnection(socket, options)) + const maConn = toConnection(socket) + log('new outbound connection %s', maConn.remoteAddr) + const conn = await this._upgrader.upgradeOutbound(maConn) + log('outbound connection %s upgraded', maConn.remoteAddr) + return conn } _connect (ma, options = {}) { diff --git a/src/listener.js b/src/listener.js index 690f45f..6a847c3 100644 --- a/src/listener.js +++ b/src/listener.js @@ -16,8 +16,10 @@ module.exports = ({ handler, upgrader }, options) => { socket.on('error', err => log('socket error', err)) const maConn = toConnection(socket) - const conn = upgrader.upgradeInbound(maConn) - log('new connection %s', conn.remoteAddr) + log('new inbound connection %s', maConn.remoteAddr) + + const conn = await upgrader.upgradeInbound(maConn) + log('inbound connection %s upgraded', maConn.remoteAddr) trackConn(server, maConn) @@ -31,7 +33,7 @@ module.exports = ({ handler, upgrader }, options) => { .on('close', () => listener.emit('close')) // Keep track of open connections to destroy in case of timeout - server.__connections = {} + server.__connections = [] listener.close = (options = {}) => { if (!server.listening) { @@ -45,10 +47,13 @@ module.exports = ({ handler, upgrader }, options) => { // destroy all the underlying sockets manually. const timeout = setTimeout(() => { log('Timeout closing server after %dms, destroying connections manually', Date.now() - start) - Object.keys(server.__connections).forEach(key => { - log('destroying %s', key) - server.__connections[key].conn.destroy() + server.__connections.forEach(maConn => { + log('destroying %s', maConn.remoteAddr) + if (!maConn.conn.destroyed) { + maConn.conn.destroy() + } }) + server.__connections = [] resolve() }, options.timeout || CLOSE_TIMEOUT) @@ -132,10 +137,13 @@ function getIpfsId (ma) { } function trackConn (server, maConn) { - const key = maConn.remoteAddr.toString() - server.__connections[key] = maConn + server.__connections.push(maConn) - maConn.conn.once('close', () => { - delete server.__connections[key] - }) + const untrackConn = () => { + server.__connections = server.__connections.filter(c => c !== maConn) + } + + maConn.conn + .on('close', untrackConn) + .on('error', untrackConn) } From a24d847ea10fe27088cb463016ef4aca5484ac1c Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 8 Sep 2019 14:18:40 +0100 Subject: [PATCH 18/32] refactor: simplify connection addresss accessors License: MIT Signed-off-by: Alan Shaw --- src/socket-to-conn.js | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/src/socket-to-conn.js b/src/socket-to-conn.js index 4ded6f4..c87e8fe 100644 --- a/src/socket-to-conn.js +++ b/src/socket-to-conn.js @@ -7,6 +7,21 @@ const Multiaddr = require('multiaddr') const { Address6 } = require('ip-address') const { CLOSE_TIMEOUT } = require('./constants') +function toMultiaddr (ip, port) { + let proto = 'ip4' + const ip6 = new Address6(ip) + + if (ip6.isValid) { + if (ip6.is4()) { + ip = ip6.to4().correctForm() + } else { + proto = 'ip6' + } + } + + return Multiaddr(`/${proto}/${ip}/tcp/${port}`) +} + // Convert a socket into a MultiaddrConnection // https://github.com/libp2p/interface-transport#multiaddrconnection module.exports = function toConnection (socket, options) { @@ -33,30 +48,10 @@ module.exports = function toConnection (socket, options) { conn.conn = socket - conn.localAddr = (() => { - const ip = socket.localAddress - const port = socket.localPort - return Multiaddr(`/ip4/${ip}/tcp/${port}`) // FIXME: ip6?? - })() - - conn.remoteAddr = (() => { - let proto = 'ip4' - let ip = socket.remoteAddress - - if (socket.remoteFamily === 'IPv6') { - const ip6 = new Address6(ip) - - if (ip6.is4()) { - ip = ip6.to4().correctForm() - } else { - proto = 'ip6' - } - } - - return Multiaddr(`/${proto}/${ip}/tcp/${socket.remotePort}`) - })() + conn.localAddr = toMultiaddr(socket.localAddress, socket.localPort) + conn.remoteAddr = toMultiaddr(socket.remoteAddress, socket.remotePort) - conn.close = async options => { + conn.close = options => { options = options || {} if (socket.destroyed) return From c493d02d1073b153c393b9eb06f475540309e464 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 8 Sep 2019 14:32:19 +0100 Subject: [PATCH 19/32] fix: remove unused dependencies License: MIT Signed-off-by: Alan Shaw --- package.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/package.json b/package.json index 840f520..1e972e8 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,6 @@ "aegir": "^20.0.0", "chai": "^4.1.2", "dirty-chai": "^2.0.1", - "lodash.isfunction": "^3.0.9", "pull-stream": "^3.6.9", "sinon": "^7.3.1" }, @@ -49,15 +48,11 @@ "class-is": "^1.1.0", "debug": "^4.1.1", "err-code": "^2.0.0", - "interface-connection": "~0.3.3", "interface-transport": "~0.5.0", "ip-address": "^6.1.0", - "it-pipe": "^1.0.0", - "lodash.isfunction": "^3.0.9", "mafmt": "^6.0.7", "multiaddr": "^6.0.6", - "stream-to-it": "^0.1.1", - "streaming-iterables": "^4.1.0" + "stream-to-it": "^0.1.1" }, "contributors": [ "Alan Shaw ", From 5c6e6af45f1913a1c75ecc35462808aa8cc68e85 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 9 Sep 2019 12:48:06 +0100 Subject: [PATCH 20/32] fix: isValid is a function not a property The docs for this module are just terrible. License: MIT Signed-off-by: Alan Shaw --- src/socket-to-conn.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/socket-to-conn.js b/src/socket-to-conn.js index c87e8fe..20f6958 100644 --- a/src/socket-to-conn.js +++ b/src/socket-to-conn.js @@ -11,7 +11,7 @@ function toMultiaddr (ip, port) { let proto = 'ip4' const ip6 = new Address6(ip) - if (ip6.isValid) { + if (ip6.isValid()) { if (ip6.is4()) { ip = ip6.to4().correctForm() } else { From dea67b397afbba38b0172cd14a6957a1aeff2c4c Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 9 Sep 2019 22:07:44 +0100 Subject: [PATCH 21/32] refactor: much refactors License: MIT Signed-off-by: Alan Shaw --- src/adapter.js | 4 +- src/index.js | 2 +- src/listener.js | 101 ++++++++++++---------------------- src/socket-to-conn.js | 123 ++++++++++++++++++++++++------------------ 4 files changed, 107 insertions(+), 123 deletions(-) diff --git a/src/adapter.js b/src/adapter.js index e206a8b..d879996 100644 --- a/src/adapter.js +++ b/src/adapter.js @@ -6,8 +6,8 @@ const TCP = require('.') // Legacy adapter to old transport & connection interface class TcpAdapter extends Adapter { - constructor () { - super(new TCP()) + constructor (config) { + super(new TCP(config)) } } diff --git a/src/index.js b/src/index.js index 3f51374..c769c66 100644 --- a/src/index.js +++ b/src/index.js @@ -16,7 +16,7 @@ class TCP { async dial (ma, options) { const socket = await this._connect(ma, options) - const maConn = toConnection(socket) + const maConn = toConnection(socket, { remoteAddr: ma }) log('new outbound connection %s', maConn.remoteAddr) const conn = await this._upgrader.upgradeOutbound(maConn) log('outbound connection %s upgraded', maConn.remoteAddr) diff --git a/src/listener.js b/src/listener.js index 6a847c3..d3bb57c 100644 --- a/src/listener.js +++ b/src/listener.js @@ -6,7 +6,8 @@ const net = require('net') const EventEmitter = require('events') const log = require('debug')('libp2p:tcp:listener') const toConnection = require('./socket-to-conn') -const { IPFS_MA_CODE, CLOSE_TIMEOUT } = require('./constants') +const { IPFS_MA_CODE } = require('./constants') +const ProtoFamily = { ip4: 'IPv4', ip6: 'IPv6' } module.exports = ({ handler, upgrader }, options) => { const listener = new EventEmitter() @@ -35,59 +36,36 @@ module.exports = ({ handler, upgrader }, options) => { // Keep track of open connections to destroy in case of timeout server.__connections = [] - listener.close = (options = {}) => { - if (!server.listening) { - return - } + listener.close = () => { + if (!server.listening) return return new Promise((resolve, reject) => { - const start = Date.now() - - // Attempt to stop the server. If it takes longer than the timeout, - // destroy all the underlying sockets manually. - const timeout = setTimeout(() => { - log('Timeout closing server after %dms, destroying connections manually', Date.now() - start) - server.__connections.forEach(maConn => { - log('destroying %s', maConn.remoteAddr) - if (!maConn.conn.destroyed) { - maConn.conn.destroy() - } - }) - server.__connections = [] - resolve() - }, options.timeout || CLOSE_TIMEOUT) - - server - .once('close', () => clearTimeout(timeout)) - .close(err => err ? reject(err) : resolve()) + server.__connections.forEach(maConn => maConn.close()) + server.close(err => err ? reject(err) : resolve()) }) } - let ipfsId - let listeningAddr + let ipfsId, listeningAddr - listener.listen = (ma) => { + listener.listen = ma => { listeningAddr = ma if (ma.protoNames().includes('ipfs')) { ipfsId = getIpfsId(ma) listeningAddr = ma.decapsulate('ipfs') } - const lOpts = listeningAddr.toOptions() return new Promise((resolve, reject) => { - server.listen(lOpts.port, lOpts.host, (err) => { - if (err) { - return reject(err) - } - - log('Listening on %s %s', lOpts.port, lOpts.host) + const { host, port } = listeningAddr.toOptions() + server.listen(port, host, err => { + if (err) return reject(err) + log('Listening on %s %s', port, host) resolve() }) }) } listener.getAddrs = () => { - const multiaddrs = [] + let addrs = [] const address = server.address() if (!address) { @@ -96,42 +74,33 @@ module.exports = ({ handler, upgrader }, options) => { // Because TCP will only return the IPv6 version // we need to capture from the passed multiaddr - if (listeningAddr.toString().indexOf('ip4') !== -1) { - let m = listeningAddr.decapsulate('tcp') - m = m.encapsulate('/tcp/' + address.port) - if (ipfsId) { - m = m.encapsulate('/ipfs/' + ipfsId) - } - - if (m.toString().indexOf('0.0.0.0') !== -1) { - const netInterfaces = os.networkInterfaces() - Object.keys(netInterfaces).forEach((niKey) => { - netInterfaces[niKey].forEach((ni) => { - if (ni.family === 'IPv4') { - multiaddrs.push(multiaddr(m.toString().replace('0.0.0.0', ni.address))) - } - }) - }) - } else { - multiaddrs.push(m) - } - } - - if (address.family === 'IPv6') { - let ma = multiaddr('/ip6/' + address.address + '/tcp/' + address.port) - if (ipfsId) { - ma = ma.encapsulate('/ipfs/' + ipfsId) - } - - multiaddrs.push(ma) + if (listeningAddr.toString().startsWith('/ip4')) { + addrs = addrs.concat(getMulitaddrs('ip4', address.address, address.port)) + } else if (address.family === 'IPv6') { + addrs = addrs.concat(getMulitaddrs('ip6', address.address, address.port)) } - return multiaddrs + return addrs.map(ma => ipfsId ? ma.encapsulate(`/ipfs/${ipfsId}`) : ma) } return listener } +function getMulitaddrs (proto, ip, port) { + const toMa = ip => multiaddr(`/${proto}/${ip}/tcp/${port}`) + return (isAnyAddr(ip) ? getNetworkAddrs(ProtoFamily[proto]) : [ip]).map(toMa) +} + +function isAnyAddr (ip) { + return ['0.0.0.0', '::'].includes(ip) +} + +function getNetworkAddrs (family) { + return [].concat(Object.values(os.networkInterfaces())) + .filter((netAddr) => netAddr.family === family) + .map(({ address }) => address) +} + function getIpfsId (ma) { return ma.stringTuples().filter(tuple => tuple[0] === IPFS_MA_CODE)[0][1] } @@ -143,7 +112,5 @@ function trackConn (server, maConn) { server.__connections = server.__connections.filter(c => c !== maConn) } - maConn.conn - .on('close', untrackConn) - .on('error', untrackConn) + maConn.conn.on('close', untrackConn).on('error', untrackConn) } diff --git a/src/socket-to-conn.js b/src/socket-to-conn.js index 20f6958..473e9d4 100644 --- a/src/socket-to-conn.js +++ b/src/socket-to-conn.js @@ -7,71 +7,88 @@ const Multiaddr = require('multiaddr') const { Address6 } = require('ip-address') const { CLOSE_TIMEOUT } = require('./constants') -function toMultiaddr (ip, port) { - let proto = 'ip4' - const ip6 = new Address6(ip) - - if (ip6.isValid()) { - if (ip6.is4()) { - ip = ip6.to4().correctForm() - } else { - proto = 'ip6' - } - } - - return Multiaddr(`/${proto}/${ip}/tcp/${port}`) -} - // Convert a socket into a MultiaddrConnection // https://github.com/libp2p/interface-transport#multiaddrconnection -module.exports = function toConnection (socket, options) { +module.exports = (socket, options) => { options = options || {} - const duplex = toIterable.duplex(socket) - const { sink } = duplex - - duplex.sink = async source => { - try { - await sink(source) - } catch (err) { - // If aborted we can safely ignore - if (err.type !== 'aborted') { - // If the source errored the socket will already have been destroyed by - // toIterable.duplex(). If the socket errored it will already be - // destroyed. There's nothing to do here except log the error & return. - log(err) + const { sink, source } = toIterable.duplex(socket) + const localAddr = toMultiaddr(socket.localAddress, socket.localPort) + + // If the remote address was passed, use it - it may have the peer ID encapsulated + const remoteAddr = options.remoteAddr + ? options.remoteAddr + : toMultiaddr(socket.remoteAddress, socket.remotePort) + + return { + async sink (source) { + if (options.signal) { + source = abortable(source, options.signal) } - } - } - const conn = options.signal ? abortable.duplex(duplex, options.signal) : duplex + try { + await sink((async function * () { + for await (const chunk of source) { + // Convert BufferList to Buffer + yield Buffer.isBuffer(chunk) ? chunk : chunk.slice() + } + })()) + } catch (err) { + // If aborted we can safely ignore + if (err.type !== 'aborted') { + // If the source errored the socket will already have been destroyed by + // toIterable.duplex(). If the socket errored it will already be + // destroyed. There's nothing to do here except log the error & return. + log(err) + } + } + }, + + source: options.signal ? abortable(source, options.signal) : source, + conn: socket, + localAddr, + remoteAddr, - conn.conn = socket + close () { + if (socket.destroyed) return - conn.localAddr = toMultiaddr(socket.localAddress, socket.localPort) - conn.remoteAddr = toMultiaddr(socket.remoteAddress, socket.remotePort) + return new Promise((resolve, reject) => { + const start = Date.now() - conn.close = options => { - options = options || {} - if (socket.destroyed) return + // Attempt to end the socket. If it takes longer to close than the + // timeout, destroy it manually. + const timeout = setTimeout(() => { + const { host, port } = remoteAddr.toOptions() + log('timeout closing socket to %s:%s after %dms, destroying it manually', + host, port, Date.now() - start) - return new Promise((resolve, reject) => { - const start = Date.now() + if (socket.destroyed) { + log('%s:%s is already destroyed', host, port) + } else { + socket.destroy() + } - // Attempt to end the socket. If it takes longer to close than the - // timeout, destroy it manually. - const timeout = setTimeout(() => { - const { host, port } = conn.remoteAddr.toOptions() - log('Timeout closing socket to %s:%s after %dms, destroying it manually', - host, port, Date.now() - start) - socket.destroy() - resolve() - }, options.timeout || CLOSE_TIMEOUT) + resolve() + }, CLOSE_TIMEOUT) + + socket.once('close', () => clearTimeout(timeout)) + socket.end(err => err ? reject(err) : resolve()) + }) + } + } +} + +function toMultiaddr (ip, port) { + let proto = 'ip4' + const ip6 = new Address6(ip) - socket.once('close', () => clearTimeout(timeout)) - socket.end(err => err ? reject(err) : resolve()) - }) + if (ip6.isValid()) { + if (ip6.is4()) { + ip = ip6.to4().correctForm() + } else { + proto = 'ip6' + } } - return conn + return Multiaddr(`/${proto}/${ip}/tcp/${port}`) } From 9c88c26a791762cb964d3e0c7e96315f2d20ea39 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Tue, 10 Sep 2019 19:06:10 +0200 Subject: [PATCH 22/32] fix: flatten the network address arrays License: MIT Signed-off-by: Jacob Heun --- src/listener.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/listener.js b/src/listener.js index d3bb57c..63ea042 100644 --- a/src/listener.js +++ b/src/listener.js @@ -95,10 +95,19 @@ function isAnyAddr (ip) { return ['0.0.0.0', '::'].includes(ip) } +/** + * @private + * @param {string} family One of ['IPv6', 'IPv4'] + * @returns {string[]} an array of ip address strings + */ function getNetworkAddrs (family) { - return [].concat(Object.values(os.networkInterfaces())) - .filter((netAddr) => netAddr.family === family) - .map(({ address }) => address) + return Object.values(os.networkInterfaces()).reduce((addresses, netAddrs) => { + netAddrs.forEach(netAddr => { + // Add the ip of each matching network interface + if (netAddr.family === family) addresses.push(netAddr.address) + }) + return addresses + }, []) } function getIpfsId (ma) { From 989426e10a7c05610eb773be7bd6ea0ea64e1e19 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Tue, 10 Sep 2019 19:06:42 +0200 Subject: [PATCH 23/32] fix: use latest multiaddr to improve filter License: MIT Signed-off-by: Jacob Heun --- package.json | 8 ++++---- src/constants.js | 5 +++-- src/index.js | 11 +++-------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 1e972e8..ba800e3 100644 --- a/package.json +++ b/package.json @@ -44,14 +44,14 @@ "sinon": "^7.3.1" }, "dependencies": { - "abortable-iterator": "^2.0.0", + "abortable-iterator": "^2.1.0", "class-is": "^1.1.0", "debug": "^4.1.1", "err-code": "^2.0.0", - "interface-transport": "~0.5.0", + "interface-transport": "~0.6.0", "ip-address": "^6.1.0", - "mafmt": "^6.0.7", - "multiaddr": "^6.0.6", + "mafmt": "^6.0.9", + "multiaddr": "^7.1.0", "stream-to-it": "^0.1.1" }, "contributors": [ diff --git a/src/constants.js b/src/constants.js index 067be58..b7ab8fe 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,7 +1,8 @@ 'use strict' -// IPFS multi-address code -exports.IPFS_MA_CODE = 421 +// p2p multi-address code +exports.CODE_P2P = 421 +exports.CODE_CIRCUIT = 290 // Time to wait for a connection to close gracefully before destroying it manually exports.CLOSE_TIMEOUT = 2000 diff --git a/src/index.js b/src/index.js index c769c66..173b411 100644 --- a/src/index.js +++ b/src/index.js @@ -8,6 +8,7 @@ const log = require('debug')('libp2p:tcp') const toConnection = require('./socket-to-conn') const createListener = require('./listener') const { AbortError } = require('interface-transport') +const { CODE_CIRCUIT, CODE_P2P } = require('./constants') class TCP { constructor ({ upgrader }) { @@ -88,17 +89,11 @@ class TCP { multiaddrs = Array.isArray(multiaddrs) ? multiaddrs : [multiaddrs] return multiaddrs.filter(ma => { - const protos = ma.protoNames() - - if (protos.includes('p2p-circuit')) { + if (ma.protoCodes().includes(CODE_CIRCUIT)) { return false } - if (protos.includes('ipfs')) { - ma = ma.decapsulate('ipfs') - } - - return mafmt.TCP.matches(ma) + return mafmt.TCP.matches(ma.decapsulateCode(CODE_P2P)) }) } } From a1fe98b67b88c66fa2a7855e7b1a797ce56a0279 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 13 Sep 2019 19:23:59 +0100 Subject: [PATCH 24/32] fix: almost all the tests pass License: MIT Signed-off-by: Alan Shaw --- .gitignore | 46 +---- package.json | 6 +- src/adapter.js | 17 -- src/index.js | 7 +- src/listener.js | 17 +- test/adapter/connection-wrap.spec.js | 121 ------------- test/adapter/connection.spec.js | 111 ------------ test/adapter/listen-dial.spec.js | 259 --------------------------- test/compliance.spec.js | 4 +- test/connection.spec.js | 41 ++--- test/constructor.spec.js | 15 -- test/filter.spec.js | 2 +- test/get-multiaddr.spec.js | 2 +- test/listen-dial.spec.js | 14 +- test/turbolence.spec.js | 16 -- 15 files changed, 52 insertions(+), 626 deletions(-) delete mode 100644 src/adapter.js delete mode 100644 test/adapter/connection-wrap.spec.js delete mode 100644 test/adapter/connection.spec.js delete mode 100644 test/adapter/listen-dial.spec.js delete mode 100644 test/constructor.spec.js delete mode 100644 test/turbolence.spec.js diff --git a/.gitignore b/.gitignore index f427b41..9faa8e0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,46 +1,4 @@ -docs -**/node_modules/ -**/*.log -test/repo-tests* -**/bundle.js - -# yarn -yarn.lock - -# Logs -logs -*.log - -coverage - -# Runtime data -pids -*.pid -*.seed - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage - -# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# node-waf configuration -.lock-wscript - -build - -# Dependency directory -# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git node_modules - -lib -dist -test/test-data/go-ipfs-repo/LOCK -test/test-data/go-ipfs-repo/LOG -test/test-data/go-ipfs-repo/LOG.old - -# while testing npm5 package-lock.json +coverage +.nyc_output diff --git a/package.json b/package.json index ba800e3..14b9d13 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,7 @@ "release": "aegir release -t node --no-build", "release-minor": "aegir release -t node --type minor --no-build", "release-major": "aegir-release -t node --type major --no-build", - "coverage": "aegir coverage", - "coverage-publish": "aegir-coverage publish" + "coverage": "nyc --reporter=text --reporter=lcov npm run test:node" }, "pre-push": [ "lint", @@ -40,7 +39,7 @@ "aegir": "^20.0.0", "chai": "^4.1.2", "dirty-chai": "^2.0.1", - "pull-stream": "^3.6.9", + "interface-transport": "^0.6.0", "sinon": "^7.3.1" }, "dependencies": { @@ -48,7 +47,6 @@ "class-is": "^1.1.0", "debug": "^4.1.1", "err-code": "^2.0.0", - "interface-transport": "~0.6.0", "ip-address": "^6.1.0", "mafmt": "^6.0.9", "multiaddr": "^7.1.0", diff --git a/src/adapter.js b/src/adapter.js deleted file mode 100644 index d879996..0000000 --- a/src/adapter.js +++ /dev/null @@ -1,17 +0,0 @@ -'use strict' - -const { Adapter } = require('interface-transport') -const withIs = require('class-is') -const TCP = require('.') - -// Legacy adapter to old transport & connection interface -class TcpAdapter extends Adapter { - constructor (config) { - super(new TCP(config)) - } -} - -module.exports = withIs(TcpAdapter, { - className: 'TCP', - symbolName: '@libp2p/js-libp2p-tcp/tcp' -}) diff --git a/src/index.js b/src/index.js index 173b411..dbb4205 100644 --- a/src/index.js +++ b/src/index.js @@ -7,7 +7,7 @@ const errCode = require('err-code') const log = require('debug')('libp2p:tcp') const toConnection = require('./socket-to-conn') const createListener = require('./listener') -const { AbortError } = require('interface-transport') +const { AbortError } = require('abortable-iterator') const { CODE_CIRCUIT, CODE_P2P } = require('./constants') class TCP { @@ -16,8 +16,9 @@ class TCP { } async dial (ma, options) { + options = options || {} const socket = await this._connect(ma, options) - const maConn = toConnection(socket, { remoteAddr: ma }) + const maConn = toConnection(socket, { remoteAddr: ma, signal: options.signal }) log('new outbound connection %s', maConn.remoteAddr) const conn = await this._upgrader.upgradeOutbound(maConn) log('outbound connection %s upgraded', maConn.remoteAddr) @@ -43,7 +44,7 @@ class TCP { const onTimeout = () => { log('connnection timeout %s:%s', cOpts.host, cOpts.port) - const err = errCode(new Error(`connection timeout after ${Date.now() - start}ms`), 'ETIMEDOUT') + const err = errCode(new Error(`connection timeout after ${Date.now() - start}ms`), 'ERR_CONNECT_TIMEOUT') // Note: this will result in onError() being called rawSocket.emit('error', err) } diff --git a/src/listener.js b/src/listener.js index 63ea042..fabc270 100644 --- a/src/listener.js +++ b/src/listener.js @@ -6,7 +6,7 @@ const net = require('net') const EventEmitter = require('events') const log = require('debug')('libp2p:tcp:listener') const toConnection = require('./socket-to-conn') -const { IPFS_MA_CODE } = require('./constants') +const { CODE_P2P } = require('./constants') const ProtoFamily = { ip4: 'IPv4', ip6: 'IPv6' } module.exports = ({ handler, upgrader }, options) => { @@ -45,13 +45,14 @@ module.exports = ({ handler, upgrader }, options) => { }) } - let ipfsId, listeningAddr + let peerId, listeningAddr listener.listen = ma => { listeningAddr = ma - if (ma.protoNames().includes('ipfs')) { - ipfsId = getIpfsId(ma) - listeningAddr = ma.decapsulate('ipfs') + peerId = ma.getPeerId() + + if (peerId) { + listeningAddr = ma.decapsulateCode(CODE_P2P) } return new Promise((resolve, reject) => { @@ -80,7 +81,7 @@ module.exports = ({ handler, upgrader }, options) => { addrs = addrs.concat(getMulitaddrs('ip6', address.address, address.port)) } - return addrs.map(ma => ipfsId ? ma.encapsulate(`/ipfs/${ipfsId}`) : ma) + return addrs.map(ma => peerId ? ma.encapsulate(`/p2p/${peerId}`) : ma) } return listener @@ -110,10 +111,6 @@ function getNetworkAddrs (family) { }, []) } -function getIpfsId (ma) { - return ma.stringTuples().filter(tuple => tuple[0] === IPFS_MA_CODE)[0][1] -} - function trackConn (server, maConn) { server.__connections.push(maConn) diff --git a/test/adapter/connection-wrap.spec.js b/test/adapter/connection-wrap.spec.js deleted file mode 100644 index 0077417..0000000 --- a/test/adapter/connection-wrap.spec.js +++ /dev/null @@ -1,121 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const pull = require('pull-stream') -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) -const TCP = require('../../src/adapter') -const multiaddr = require('multiaddr') -const Connection = require('interface-connection').Connection - -describe('Connection Wrap', () => { - let tcp - let listener - const ma = multiaddr('/ip4/127.0.0.1/tcp/9090') - - beforeEach((done) => { - tcp = new TCP() - listener = tcp.createListener((conn) => { - pull(conn, conn) - }) - listener.on('listening', done) - listener.listen(ma) - }) - - afterEach((done) => { - listener.close(done) - }) - - it('simple wrap', (done) => { - const conn = tcp.dial(ma) - conn.setPeerInfo('peerInfo') - const connWrap = new Connection(conn) - pull( - pull.values(['hey']), - connWrap, - pull.collect((err, chunks) => { - expect(err).to.not.exist() - expect(chunks).to.be.eql([Buffer.from('hey')]) - - connWrap.getPeerInfo((err, peerInfo) => { - expect(err).to.not.exist() - expect(peerInfo).to.equal('peerInfo') - done() - }) - }) - ) - }) - - it('buffer wrap', (done) => { - const conn = tcp.dial(ma) - const connWrap = new Connection() - pull( - pull.values(['hey']), - connWrap, - pull.collect((err, chunks) => { - expect(err).to.not.exist() - expect(chunks).to.be.eql([Buffer.from('hey')]) - done() - }) - ) - - connWrap.setInnerConn(conn) - }) - - it('overload wrap', (done) => { - const conn = tcp.dial(ma) - const connWrap = new Connection(conn) - connWrap.getPeerInfo = (callback) => { - callback(null, 'none') - } - conn.getPeerInfo((err, peerInfo) => { - expect(err).to.exist() - }) - connWrap.getPeerInfo((err, peerInfo) => { - expect(err).to.not.exist() - expect(peerInfo).to.equal('none') - }) - pull( - pull.values(['hey']), - connWrap, - pull.collect((err, chunks) => { - expect(err).to.not.exist() - expect(chunks).to.be.eql([Buffer.from('hey')]) - done() - }) - ) - }) - - it('dial error', (done) => { - tcp.dial(multiaddr('/ip4/127.0.0.1/tcp/22234'), (err) => { - expect(err).to.exist() - done() - }) - }) - - it('matryoshka wrap', (done) => { - const conn = tcp.dial(ma) - const connWrap1 = new Connection(conn) - const connWrap2 = new Connection(connWrap1) - const connWrap3 = new Connection(connWrap2) - - conn.getPeerInfo = (callback) => { - callback(null, 'inner doll') - } - pull( - pull.values(['hey']), - connWrap3, - pull.collect((err, chunks) => { - expect(err).to.not.exist() - expect(chunks).to.eql([Buffer.from('hey')]) - connWrap3.getPeerInfo((err, peerInfo) => { - expect(err).to.not.exist() - expect(peerInfo).to.equal('inner doll') - done() - }) - }) - ) - }) -}) diff --git a/test/adapter/connection.spec.js b/test/adapter/connection.spec.js deleted file mode 100644 index 559143d..0000000 --- a/test/adapter/connection.spec.js +++ /dev/null @@ -1,111 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const pull = require('pull-stream') -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) -const TCP = require('../../src/adapter') -const multiaddr = require('multiaddr') - -describe('valid Connection', () => { - let tcp - - beforeEach(() => { - tcp = new TCP() - }) - - const ma = multiaddr('/ip4/127.0.0.1/tcp/9090') - - it('get observed addrs', (done) => { - let dialerObsAddrs - - const listener = tcp.createListener((conn) => { - expect(conn).to.exist() - conn.getObservedAddrs((err, addrs) => { - expect(err).to.not.exist() - dialerObsAddrs = addrs - pull(pull.empty(), conn) - }) - }) - - listener.listen(ma, () => { - const conn = tcp.dial(ma) - pull( - conn, - pull.onEnd(endHandler) - ) - - function endHandler () { - conn.getObservedAddrs((err, addrs) => { - expect(err).to.not.exist() - pull(pull.empty(), conn) - closeAndAssert(listener, addrs) - }) - } - - function closeAndAssert (listener, addrs) { - listener.close(() => { - expect(addrs[0]).to.deep.equal(ma) - expect(dialerObsAddrs.length).to.equal(1) - done() - }) - } - }) - }) - - it('get Peer Info', (done) => { - const listener = tcp.createListener((conn) => { - expect(conn).to.exist() - conn.getPeerInfo((err, peerInfo) => { - expect(err).to.exist() - expect(peerInfo).to.not.exist() - pull(pull.empty(), conn) - }) - }) - - listener.listen(ma, () => { - const conn = tcp.dial(ma) - - pull(conn, pull.onEnd(endHandler)) - - function endHandler () { - conn.getPeerInfo((err, peerInfo) => { - expect(err).to.exist() - expect(peerInfo).to.not.exist() - - listener.close(done) - }) - } - }) - }) - - it('set Peer Info', (done) => { - const listener = tcp.createListener((conn) => { - expect(conn).to.exist() - conn.setPeerInfo('batatas') - conn.getPeerInfo((err, peerInfo) => { - expect(err).to.not.exist() - expect(peerInfo).to.equal('batatas') - pull(pull.empty(), conn) - }) - }) - - listener.listen(ma, () => { - const conn = tcp.dial(ma) - - pull(conn, pull.onEnd(endHandler)) - - function endHandler () { - conn.setPeerInfo('arroz') - conn.getPeerInfo((err, peerInfo) => { - expect(err).to.not.exist() - expect(peerInfo).to.equal('arroz') - - listener.close(done) - }) - } - }) - }) -}) diff --git a/test/adapter/listen-dial.spec.js b/test/adapter/listen-dial.spec.js deleted file mode 100644 index 97e6980..0000000 --- a/test/adapter/listen-dial.spec.js +++ /dev/null @@ -1,259 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const pull = require('pull-stream') -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) -const TCP = require('../../src/adapter') -const net = require('net') -const multiaddr = require('multiaddr') -const isCI = process.env.CI - -describe('listen', () => { - let tcp - - beforeEach(() => { - tcp = new TCP() - }) - - it('close listener with connections, through timeout', (done) => { - const mh = multiaddr('/ip4/127.0.0.1/tcp/9090/ipfs/Qmb6owHp6eaWArVbcJJbQSyifyJBttMMjYV76N2hMbf5Vw') - const listener = tcp.createListener((conn) => { - pull(conn, conn) - }) - - listener.listen(mh, () => { - const socket1 = net.connect(9090) - const socket2 = net.connect(9090) - - socket1.write('Some data that is never handled') - socket1.end() - socket1.on('error', () => {}) - socket2.on('error', () => {}) - socket1.on('connect', () => { - listener.close(done) - }) - }) - }) - - it('listen on port 0', (done) => { - const mh = multiaddr('/ip4/127.0.0.1/tcp/0') - const listener = tcp.createListener((conn) => {}) - listener.listen(mh, () => { - listener.close(done) - }) - }) - - it('listen on IPv6 addr', (done) => { - if (isCI) { return done() } - const mh = multiaddr('/ip6/::/tcp/9090') - const listener = tcp.createListener((conn) => {}) - listener.listen(mh, () => { - listener.close(done) - }) - }) - - it('listen on any Interface', (done) => { - const mh = multiaddr('/ip4/0.0.0.0/tcp/9090') - const listener = tcp.createListener((conn) => {}) - listener.listen(mh, () => { - listener.close(done) - }) - }) - - it('getAddrs', (done) => { - const mh = multiaddr('/ip4/127.0.0.1/tcp/9090') - const listener = tcp.createListener((conn) => {}) - listener.listen(mh, () => { - listener.getAddrs((err, multiaddrs) => { - expect(err).to.not.exist() - expect(multiaddrs.length).to.equal(1) - expect(multiaddrs[0]).to.deep.equal(mh) - listener.close(done) - }) - }) - }) - - it('getAddrs on port 0 listen', (done) => { - const mh = multiaddr('/ip4/127.0.0.1/tcp/0') - const listener = tcp.createListener((conn) => {}) - listener.listen(mh, () => { - listener.getAddrs((err, multiaddrs) => { - expect(err).to.not.exist() - expect(multiaddrs.length).to.equal(1) - listener.close(done) - }) - }) - }) - - it('getAddrs from listening on 0.0.0.0', (done) => { - const mh = multiaddr('/ip4/0.0.0.0/tcp/9090') - const listener = tcp.createListener((conn) => {}) - listener.listen(mh, () => { - listener.getAddrs((err, multiaddrs) => { - expect(err).to.not.exist() - expect(multiaddrs.length > 0).to.equal(true) - expect(multiaddrs[0].toString().indexOf('0.0.0.0')).to.equal(-1) - listener.close(done) - }) - }) - }) - - it('getAddrs from listening on 0.0.0.0 and port 0', (done) => { - const mh = multiaddr('/ip4/0.0.0.0/tcp/0') - const listener = tcp.createListener((conn) => {}) - listener.listen(mh, () => { - listener.getAddrs((err, multiaddrs) => { - expect(err).to.not.exist() - expect(multiaddrs.length > 0).to.equal(true) - expect(multiaddrs[0].toString().indexOf('0.0.0.0')).to.equal(-1) - listener.close(done) - }) - }) - }) - - it('getAddrs preserves IPFS Id', (done) => { - const mh = multiaddr('/ip4/127.0.0.1/tcp/9090/ipfs/Qmb6owHp6eaWArVbcJJbQSyifyJBttMMjYV76N2hMbf5Vw') - const listener = tcp.createListener((conn) => {}) - listener.listen(mh, () => { - listener.getAddrs((err, multiaddrs) => { - expect(err).to.not.exist() - expect(multiaddrs.length).to.equal(1) - expect(multiaddrs[0]).to.deep.equal(mh) - listener.close(done) - }) - }) - }) -}) - -describe('dial', () => { - let tcp - let listener - const ma = multiaddr('/ip4/127.0.0.1/tcp/9090') - - beforeEach((done) => { - tcp = new TCP() - listener = tcp.createListener((conn) => { - pull( - conn, - pull.map((x) => Buffer.from(x.toString() + '!')), - conn - ) - }) - listener.listen(ma, done) - }) - - afterEach((done) => { - listener.close(done) - }) - - it('dial on IPv4', (done) => { - pull( - pull.values(['hey']), - tcp.dial(ma), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([Buffer.from('hey!')]) - done() - }) - ) - }) - - it('dial to non existent listener', (done) => { - const ma = multiaddr('/ip4/127.0.0.1/tcp/8989') - pull( - tcp.dial(ma), - pull.onEnd((err) => { - expect(err).to.exist() - done() - }) - ) - }) - - it('dial on IPv6', (done) => { - if (isCI) { return done() } - - const ma = multiaddr('/ip6/::/tcp/9066') - const listener = tcp.createListener((conn) => { - pull(conn, conn) - }) - listener.listen(ma, () => { - pull( - pull.values(['hey']), - tcp.dial(ma), - pull.collect((err, values) => { - expect(err).to.not.exist() - - expect(values).to.be.eql([Buffer.from('hey')]) - - listener.close(done) - }) - ) - }) - }) - - it('dial and destroy on listener', (done) => { - let count = 0 - const closed = () => ++count === 2 ? finish() : null - - const ma = multiaddr('/ip6/::/tcp/9067') - - const listener = tcp.createListener((conn) => { - pull( - pull.empty(), - conn, - pull.onEnd(closed) - ) - }) - - listener.listen(ma, () => { - pull(tcp.dial(ma), pull.onEnd(closed)) - }) - - function finish () { - listener.close(done) - } - }) - - it('dial and destroy on dialer', (done) => { - if (isCI) { return done() } - - let count = 0 - const destroyed = () => ++count === 2 ? finish() : null - - const ma = multiaddr('/ip6/::/tcp/9068') - - const listener = tcp.createListener((conn) => { - pull(conn, pull.onEnd(destroyed)) - }) - - listener.listen(ma, () => { - pull( - pull.empty(), - tcp.dial(ma), - pull.onEnd(destroyed) - ) - }) - - function finish () { - listener.close(done) - } - }) - - it('dial on IPv4 with IPFS Id', (done) => { - const ma = multiaddr('/ip4/127.0.0.1/tcp/9090/ipfs/Qmb6owHp6eaWArVbcJJbQSyifyJBttMMjYV76N2hMbf5Vw') - const conn = tcp.dial(ma) - - pull( - pull.values(['hey']), - conn, - pull.collect((err, res) => { - expect(err).to.not.exist() - expect(res).to.be.eql([Buffer.from('hey!')]) - done() - }) - ) - }) -}) diff --git a/test/compliance.spec.js b/test/compliance.spec.js index 11fd244..b337d3f 100644 --- a/test/compliance.spec.js +++ b/test/compliance.spec.js @@ -9,8 +9,8 @@ const TCP = require('../src') describe('interface-transport compliance', () => { tests({ - setup () { - const tcp = new TCP() + setup ({ upgrader }) { + const tcp = new TCP({ upgrader }) const addrs = [ multiaddr('/ip4/127.0.0.1/tcp/9091'), multiaddr('/ip4/127.0.0.1/tcp/9092'), diff --git a/test/connection.spec.js b/test/connection.spec.js index 4d8f385..89e5973 100644 --- a/test/connection.spec.js +++ b/test/connection.spec.js @@ -11,24 +11,23 @@ const multiaddr = require('multiaddr') describe('valid Connection', () => { let tcp + const mockUpgrader = { + upgradeInbound: maConn => maConn, + upgradeOutbound: maConn => maConn + } + beforeEach(() => { - tcp = new TCP() + tcp = new TCP({ upgrader: mockUpgrader }) }) - const ma = multiaddr('/ip4/127.0.0.1/tcp/9090') + const ma = multiaddr('/ip4/127.0.0.1/tcp/0') - it('get observed addrs', async () => { + it('should resolve port 0', async () => { // Create a Promise that resolves when a connection is handled let handled - const handlerPromise = new Promise((resolve) => { - handled = resolve - }) + const handlerPromise = new Promise(resolve => { handled = resolve }) - const handler = async (conn) => { - expect(conn).to.exist() - const dialerObsAddrs = await conn.getObservedAddrs() - handled(dialerObsAddrs) - } + const handler = conn => handled(conn) // Create a listener with the handler const listener = tcp.createListener(handler) @@ -36,20 +35,22 @@ describe('valid Connection', () => { // Listen on the multi-address await listener.listen(ma) - // Dial to that same address - const conn = await tcp.dial(ma) - const addrs = await conn.getObservedAddrs() + const localAddrs = listener.getAddrs() + expect(localAddrs.length).to.equal(1) + + // Dial to that address + const dialerConn = await tcp.dial(localAddrs[0]) // Wait for the incoming dial to be handled - const dialerObsAddrs = await handlerPromise + const listenerConn = await handlerPromise // Close the listener await listener.close() - // The addresses should match - expect(addrs.length).to.equal(1) - expect(addrs[0]).to.deep.equal(ma) - expect(dialerObsAddrs.length).to.equal(1) - expect(dialerObsAddrs[0]).to.exist() + expect(dialerConn.localAddr.toString()) + .to.equal(listenerConn.remoteAddr.toString()) + + expect(dialerConn.remoteAddr.toString()) + .to.equal(listenerConn.localAddr.toString()) }) }) diff --git a/test/constructor.spec.js b/test/constructor.spec.js deleted file mode 100644 index 6183ad1..0000000 --- a/test/constructor.spec.js +++ /dev/null @@ -1,15 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) -const TCP = require('../src') - -describe('Constructor', () => { - it('create an instance', () => { - const tcp = new TCP() - expect(tcp).to.exist() - }) -}) diff --git a/test/filter.spec.js b/test/filter.spec.js index b6ce629..9539d1c 100644 --- a/test/filter.spec.js +++ b/test/filter.spec.js @@ -15,7 +15,7 @@ describe('filter addrs', () => { let tcp before(() => { - tcp = new TCP() + tcp = new TCP({ upgrader: {} }) }) it('filter valid addrs for this transport', () => { diff --git a/test/get-multiaddr.spec.js b/test/get-multiaddr.spec.js index d7199ac..7ef0fff 100644 --- a/test/get-multiaddr.spec.js +++ b/test/get-multiaddr.spec.js @@ -5,7 +5,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) -const getMultiaddr = require('../src/get-multiaddr') +// const getMultiaddr = require('../src/get-multiaddr') const goodSocket4 = { remoteAddress: '127.0.0.1', diff --git a/test/listen-dial.spec.js b/test/listen-dial.spec.js index ad414fa..9c2f057 100644 --- a/test/listen-dial.spec.js +++ b/test/listen-dial.spec.js @@ -17,7 +17,12 @@ describe('listen', () => { let tcp beforeEach(() => { - tcp = new TCP() + tcp = new TCP({ + upgrader: { + upgradeOutbound: maConn => maConn, + upgradeInbound: maConn => maConn + } + }) }) it('close listener with connections, through timeout', async () => { @@ -134,7 +139,12 @@ describe('dial', () => { const ma = multiaddr('/ip4/127.0.0.1/tcp/9090') beforeEach(async () => { - tcp = new TCP() + tcp = new TCP({ + upgrader: { + upgradeOutbound: maConn => maConn, + upgradeInbound: maConn => maConn + } + }) listener = tcp.createListener((conn) => { pipe( conn, diff --git a/test/turbolence.spec.js b/test/turbolence.spec.js deleted file mode 100644 index e54bae0..0000000 --- a/test/turbolence.spec.js +++ /dev/null @@ -1,16 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) -// const TCP = require('../src') - -describe.skip('turbolence', () => { - it('dialer - emits error on the other end is terminated abruptly', (done) => { - expect('ok').to.equal('ok') - }) - - it('listener - emits error on the other end is terminated abruptly', (done) => {}) -}) From 3e0c84425cf64775c03ef3e0d3cd347550b041fe Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sat, 14 Sep 2019 14:23:32 +0100 Subject: [PATCH 25/32] fix: tests License: MIT Signed-off-by: Alan Shaw --- src/ip-port-to-multiaddr.js | 30 +++++++++++++++++ src/socket-to-conn.js | 45 ++++++++++---------------- test/connection.spec.js | 2 +- test/get-multiaddr.spec.js | 54 ------------------------------- test/ip-port-to-multiaddr.spec.js | 50 ++++++++++++++++++++++++++++ 5 files changed, 98 insertions(+), 83 deletions(-) create mode 100644 src/ip-port-to-multiaddr.js delete mode 100644 test/get-multiaddr.spec.js create mode 100644 test/ip-port-to-multiaddr.spec.js diff --git a/src/ip-port-to-multiaddr.js b/src/ip-port-to-multiaddr.js new file mode 100644 index 0000000..6d980c4 --- /dev/null +++ b/src/ip-port-to-multiaddr.js @@ -0,0 +1,30 @@ +'use strict' + +const multiaddr = require('multiaddr') +const { Address4, Address6 } = require('ip-address') + +module.exports = (ip, port) => { + if (typeof ip !== 'string') { + throw new Error('invalid ip') + } + + port = parseInt(port) + + if (isNaN(port)) { + throw new Error('invalid port') + } + + if (new Address4(ip).isValid()) { + return multiaddr(`/ip4/${ip}/tcp/${port}`) + } + + const ip6 = new Address6(ip) + + if (ip6.isValid()) { + return ip6.is4() + ? multiaddr(`/ip4/${ip6.to4().correctForm()}/tcp/${port}`) + : multiaddr(`/ip6/${ip}/tcp/${port}`) + } + + throw new Error('invalid ip') +} diff --git a/src/socket-to-conn.js b/src/socket-to-conn.js index 473e9d4..83e9a4d 100644 --- a/src/socket-to-conn.js +++ b/src/socket-to-conn.js @@ -3,8 +3,7 @@ const abortable = require('abortable-iterator') const log = require('debug')('libp2p:tcp:socket') const toIterable = require('stream-to-it') -const Multiaddr = require('multiaddr') -const { Address6 } = require('ip-address') +const toMultiaddr = require('./ip-port-to-multiaddr') const { CLOSE_TIMEOUT } = require('./constants') // Convert a socket into a MultiaddrConnection @@ -13,14 +12,7 @@ module.exports = (socket, options) => { options = options || {} const { sink, source } = toIterable.duplex(socket) - const localAddr = toMultiaddr(socket.localAddress, socket.localPort) - - // If the remote address was passed, use it - it may have the peer ID encapsulated - const remoteAddr = options.remoteAddr - ? options.remoteAddr - : toMultiaddr(socket.remoteAddress, socket.remotePort) - - return { + const maConn = { async sink (source) { if (options.signal) { source = abortable(source, options.signal) @@ -45,9 +37,15 @@ module.exports = (socket, options) => { }, source: options.signal ? abortable(source, options.signal) : source, + conn: socket, - localAddr, - remoteAddr, + + localAddr: toMultiaddr(socket.localAddress, socket.localPort), + + // If the remote address was passed, use it - it may have the peer ID encapsulated + remoteAddr: options.remoteAddr || toMultiaddr(socket.remoteAddress, socket.remotePort), + + timeline: { open: Date.now() }, close () { if (socket.destroyed) return @@ -58,7 +56,7 @@ module.exports = (socket, options) => { // Attempt to end the socket. If it takes longer to close than the // timeout, destroy it manually. const timeout = setTimeout(() => { - const { host, port } = remoteAddr.toOptions() + const { host, port } = maConn.remoteAddr.toOptions() log('timeout closing socket to %s:%s after %dms, destroying it manually', host, port, Date.now() - start) @@ -72,23 +70,14 @@ module.exports = (socket, options) => { }, CLOSE_TIMEOUT) socket.once('close', () => clearTimeout(timeout)) - socket.end(err => err ? reject(err) : resolve()) + socket.end(err => { + maConn.timeline.close = Date.now() + if (err) return reject(err) + resolve() + }) }) } } -} - -function toMultiaddr (ip, port) { - let proto = 'ip4' - const ip6 = new Address6(ip) - - if (ip6.isValid()) { - if (ip6.is4()) { - ip = ip6.to4().correctForm() - } else { - proto = 'ip6' - } - } - return Multiaddr(`/${proto}/${ip}/tcp/${port}`) + return maConn } diff --git a/test/connection.spec.js b/test/connection.spec.js index 89e5973..870e0ac 100644 --- a/test/connection.spec.js +++ b/test/connection.spec.js @@ -8,7 +8,7 @@ chai.use(dirtyChai) const TCP = require('../src') const multiaddr = require('multiaddr') -describe('valid Connection', () => { +describe('valid localAddr and remoteAddr', () => { let tcp const mockUpgrader = { diff --git a/test/get-multiaddr.spec.js b/test/get-multiaddr.spec.js deleted file mode 100644 index 7ef0fff..0000000 --- a/test/get-multiaddr.spec.js +++ /dev/null @@ -1,54 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) -// const getMultiaddr = require('../src/get-multiaddr') - -const goodSocket4 = { - remoteAddress: '127.0.0.1', - remotePort: '9090', - remoteFamily: 'IPv4' -} - -const goodSocket6 = { - remoteAddress: '::1', - remotePort: '9090', - remoteFamily: 'IPv6' -} - -const badSocket = {} - -const badSocketData = { - remoteAddress: 'aewmrn4awoew', - remotePort: '234', - remoteFamily: 'Hufflepuff' -} - -describe('getMultiaddr multiaddr creation', () => { - it('creates multiaddr from valid socket data', (done) => { - expect(getMultiaddr(goodSocket4)) - .to.exist() - done() - }) - - it('creates multiaddr from valid IPv6 socket data', (done) => { - expect(getMultiaddr(goodSocket6)) - .to.exist() - done() - }) - - it('returns undefined multiaddr from missing socket data', (done) => { - expect(getMultiaddr(badSocket)) - .to.equal(undefined) - done() - }) - - it('returns undefined multiaddr from unparseable socket data', (done) => { - expect(getMultiaddr(badSocketData)) - .to.equal(undefined) - done() - }) -}) diff --git a/test/ip-port-to-multiaddr.spec.js b/test/ip-port-to-multiaddr.spec.js new file mode 100644 index 0000000..657e569 --- /dev/null +++ b/test/ip-port-to-multiaddr.spec.js @@ -0,0 +1,50 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) +const toMultiaddr = require('../src/ip-port-to-multiaddr') + +describe('IP and port to Multiaddr', () => { + it('creates multiaddr from valid IPv4 IP and port', () => { + const ip = '127.0.0.1' + const port = '9090' + expect(toMultiaddr(ip, port).toString()).to.equal(`/ip4/${ip}/tcp/${port}`) + }) + + it('creates multiaddr from valid IPv4 IP and numeric port', () => { + const ip = '127.0.0.1' + const port = 9090 + expect(toMultiaddr(ip, port).toString()).to.equal(`/ip4/${ip}/tcp/${port}`) + }) + + it('creates multiaddr from valid IPv4 in IPv6 IP and port', () => { + const ip = '0:0:0:0:0:0:101.45.75.219' + const port = '9090' + expect(toMultiaddr(ip, port).toString()).to.equal(`/ip4/101.45.75.219/tcp/${port}`) + }) + + it('creates multiaddr from valid IPv6 IP and port', () => { + const ip = '::1' + const port = '9090' + expect(toMultiaddr(ip, port).toString()).to.equal(`/ip6/${ip}/tcp/${port}`) + }) + + it('throws for missing IP address', () => { + expect(() => toMultiaddr()).to.throw('invalid ip') + }) + + it('throws for invalid IP address', () => { + const ip = 'aewmrn4awoew' + const port = '234' + expect(() => toMultiaddr(ip, port)).to.throw('invalid ip') + }) + + it('throws for invalid port', () => { + const ip = '127.0.0.1' + const port = 'garbage' + expect(() => toMultiaddr(ip, port)).to.throw('invalid port') + }) +}) From b2d52869cb7a1ee325516b5bc4c24bbf6ec03e96 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 16 Sep 2019 13:36:38 +0200 Subject: [PATCH 26/32] chore: remove commitlint License: MIT Signed-off-by: Jacob Heun --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 430f0df..d78ede7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,6 @@ jobs: os: linux script: - npx aegir build --bundlesize - - npx aegir commitlint --travis - npx aegir dep-check - npm run lint From 588c24ddb16b246c6658bbfa6511596369de251b Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 16 Sep 2019 13:37:07 +0200 Subject: [PATCH 27/32] chore: remove pre push test License: MIT Signed-off-by: Jacob Heun --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index f0265c6..ea65d67 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,7 @@ "coverage": "nyc --reporter=text --reporter=lcov npm run test:node" }, "pre-push": [ - "lint", - "test" + "lint" ], "repository": { "type": "git", From ea9ac904c6e4442e9268a44e7ac9ab426adbed06 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 16 Sep 2019 13:58:01 +0200 Subject: [PATCH 28/32] docs: add some jsdocs License: MIT Signed-off-by: Jacob Heun --- src/index.js | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/index.js b/src/index.js index dbb4205..df5ce9c 100644 --- a/src/index.js +++ b/src/index.js @@ -10,11 +10,26 @@ const createListener = require('./listener') const { AbortError } = require('abortable-iterator') const { CODE_CIRCUIT, CODE_P2P } = require('./constants') +/** + * @class TCP + */ class TCP { + /** + * @constructor + * @param {object} options + * @param {Upgrader} options.upgrader + */ constructor ({ upgrader }) { this._upgrader = upgrader } + /** + * @async + * @param {Multiaddr} ma + * @param {object} options + * @param {AbortSignal} options.signal Used to abort dial requests + * @returns {Connection} An upgraded Connection + */ async dial (ma, options) { options = options || {} const socket = await this._connect(ma, options) @@ -25,6 +40,13 @@ class TCP { return conn } + /** + * @private + * @param {Multiaddr} ma + * @param {object} options + * @param {AbortSignal} options.signal Used to abort dial requests + * @returns {Promise} Resolves a TCP Socket + */ _connect (ma, options = {}) { if (options.signal && options.signal.aborted) { throw new AbortError() @@ -77,6 +99,14 @@ class TCP { }) } + /** + * Creates a TCP listener. The provided `handler` function will be called + * anytime a new incoming Connection has been successfully upgraded via + * `upgrader.upgradeInbound`. + * @param {*} [options] + * @param {function(Connection)} handler + * @returns {Listener} A TCP listener + */ createListener (options, handler) { if (typeof options === 'function') { handler = options @@ -86,6 +116,11 @@ class TCP { return createListener({ handler, upgrader: this._upgrader }, options) } + /** + * Takes a list of `Multiaddr`s and returns only valid TCP addresses + * @param {Multiaddr[]} multiaddrs + * @returns {Multiaddr[]} Valid TCP multiaddrs + */ filter (multiaddrs) { multiaddrs = Array.isArray(multiaddrs) ? multiaddrs : [multiaddrs] From e37af101087e3454914acdfbf8424ba35f310787 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 16 Sep 2019 15:15:20 +0200 Subject: [PATCH 29/32] chore: update dep License: MIT Signed-off-by: Jacob Heun --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ea65d67..7d404dc 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "aegir": "^20.0.0", "chai": "^4.2.0", "dirty-chai": "^2.0.1", - "interface-transport": "^0.6.0", + "interface-transport": "github:libp2p/interface-transport#test/close", "sinon": "^7.3.1" }, "dependencies": { From b29b368b44060d3a72c9ad0baf2a1e361351e57f Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 16 Sep 2019 15:29:04 +0200 Subject: [PATCH 30/32] fix: untrack only needs the close event License: MIT Signed-off-by: Jacob Heun --- src/listener.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/listener.js b/src/listener.js index fabc270..ac675f8 100644 --- a/src/listener.js +++ b/src/listener.js @@ -118,5 +118,5 @@ function trackConn (server, maConn) { server.__connections = server.__connections.filter(c => c !== maConn) } - maConn.conn.on('close', untrackConn).on('error', untrackConn) + maConn.conn.once('close', untrackConn) } From 208ef899a282d6f4c1c9f99b5243b1abeefb4761 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 16 Sep 2019 16:02:23 +0200 Subject: [PATCH 31/32] chore: update interface dep License: MIT Signed-off-by: Jacob Heun --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7d404dc..13738cb 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "aegir": "^20.0.0", "chai": "^4.2.0", "dirty-chai": "^2.0.1", - "interface-transport": "github:libp2p/interface-transport#test/close", + "interface-transport": "^0.6.1", "sinon": "^7.3.1" }, "dependencies": { From 4a959d8554cef0458e299cc29eeea4ed0f56dd78 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 16 Sep 2019 16:56:05 +0200 Subject: [PATCH 32/32] refactor: assert upgrader presence License: MIT Signed-off-by: Jacob Heun --- src/index.js | 2 ++ test/listen-dial.spec.js | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/index.js b/src/index.js index df5ce9c..56eabac 100644 --- a/src/index.js +++ b/src/index.js @@ -9,6 +9,7 @@ const toConnection = require('./socket-to-conn') const createListener = require('./listener') const { AbortError } = require('abortable-iterator') const { CODE_CIRCUIT, CODE_P2P } = require('./constants') +const assert = require('assert') /** * @class TCP @@ -20,6 +21,7 @@ class TCP { * @param {Upgrader} options.upgrader */ constructor ({ upgrader }) { + assert(upgrader, 'An upgrader must be provided. See https://github.com/libp2p/interface-transport#upgrader.') this._upgrader = upgrader } diff --git a/test/listen-dial.spec.js b/test/listen-dial.spec.js index 3fdc0d2..4f90413 100644 --- a/test/listen-dial.spec.js +++ b/test/listen-dial.spec.js @@ -12,6 +12,12 @@ const pipe = require('it-pipe') const { collect, map } = require('streaming-iterables') const isCI = process.env.CI +describe('construction', () => { + it('requires an upgrader', () => { + expect(() => new TCP()).to.throw() + }) +}) + describe('listen', () => { let tcp