refactor: decouple HTTP servers from cli/commands/daemon (#1950)

* refactor: decouple HttpApi from cli/commands/daemon

In the past API was exposed via HTTP Server only when jsipfs daemon was run
from the commandline, so src/http/index.js was also responsible for
orchestration that is not related to HTTP itself.

This refactor moves code that is not related to HTTP Servers into
standalone-daemon.js, which is easier to reason about, and unlocks use
of HttpApi in contexts other than commandline jsipfs daemon,
such as Firefox with libdweb or Chromium-based web browser with chrome.sockets APIs.

Refs.
ipfs/ipfs-companion#664

License: MIT
Signed-off-by: Marcin Rataj <lidel@lidel.org>

* fix: print HTTP listeners only when run as daemon

This changes behavior in web browser. Instead of printing to
console.log, it uses proper debug-based logger.

Old behavior in terminal (when run via `jsipfs daemon`) does not change.

License: MIT
Signed-off-by: Marcin Rataj <lidel@lidel.org>

* test: use StandaloneDaemon in test/http-api,gateway

This replaces durect use of HttpApi with StandaloneDaemon, restoring all
existing tests to operational state.

License: MIT
Signed-off-by: Marcin Rataj <lidel@lidel.org>

* refactor: rename StandaloneDaemon to Daemon

License: MIT
Signed-off-by: Marcin Rataj <lidel@lidel.org>

.gitignore

@@ -6,6 +6,8 @@ docs
 # Logs
 logs
 *.log
+# npm pack
+*.tgz
 
 coverage
 

src/cli/commands/daemon.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const os = require('os')
+const toUri = require('multiaddr-to-uri')
 const { getRepoPath, print, ipfsPathHelp } = require('../utils')
 
 module.exports = {
@@ -44,8 +45,8 @@ module.exports = {
       const repoPath = getRepoPath()
 
       // Required inline to reduce startup time
-      const HttpApi = require('../../http')
-      const api = new HttpApi({
+      const Daemon = require('../../cli/daemon')
+      const daemon = new Daemon({
         silent: argv.silent,
         repo: process.env.IPFS_PATH,
         offline: argv.offline,
@@ -60,7 +61,16 @@ module.exports = {
       })
 
       try {
-        await api.start()
+        await daemon.start()
+        daemon._httpApi._apiServers.forEach(apiServer => {
+          print(`API listening on ${apiServer.info.ma.toString()}`)
+        })
+        daemon._httpApi._gatewayServers.forEach(gatewayServer => {
+          print(`Gateway (read only) listening on ${gatewayServer.info.ma.toString()}`)
+        })
+        daemon._httpApi._apiServers.forEach(apiServer => {
+          print(`Web UI available at ${toUri(apiServer.info.ma)}/webui`)
+        })
       } catch (err) {
         if (err.code === 'ENOENT' && err.message.match(/uninitialized/i)) {
           print('Error: no initialized ipfs repo found in ' + repoPath)
@@ -74,7 +84,7 @@ module.exports = {
 
       const cleanup = async () => {
         print(`Received interrupt signal, shutting down...`)
-        await api.stop()
+        await daemon.stop()
         process.exit(0)
       }
 

src/cli/daemon.js

@@ -0,0 +1,96 @@
+'use strict'
+
+const debug = require('debug')
+
+const IPFS = require('../core')
+const HttpApi = require('../http')
+const WStar = require('libp2p-webrtc-star')
+const TCP = require('libp2p-tcp')
+const MulticastDNS = require('libp2p-mdns')
+const WS = require('libp2p-websockets')
+const Bootstrap = require('libp2p-bootstrap')
+const promisify = require('promisify-es6')
+
+class Daemon {
+  constructor (options) {
+    this._options = options || {}
+    this._log = debug('ipfs:daemon')
+    this._log.error = debug('ipfs:daemon:error')
+
+    if (process.env.IPFS_MONITORING) {
+      // Setup debug metrics collection
+      const prometheusClient = require('prom-client')
+      const prometheusGcStats = require('prometheus-gc-stats')
+      const collectDefaultMetrics = prometheusClient.collectDefaultMetrics
+      collectDefaultMetrics({ timeout: 5000 })
+      prometheusGcStats(prometheusClient.register)()
+    }
+  }
+
+  async start () {
+    this._log('starting')
+
+    const libp2p = { modules: {} }
+
+    // Attempt to use any of the WebRTC versions available globally
+    let electronWebRTC
+    let wrtc
+    try {
+      electronWebRTC = require('electron-webrtc')()
+    } catch (err) {
+      this._log('failed to load optional electron-webrtc dependency')
+    }
+    try {
+      wrtc = require('wrtc')
+    } catch (err) {
+      this._log('failed to load optional webrtc dependency')
+    }
+
+    if (wrtc || electronWebRTC) {
+      const using = wrtc ? 'wrtc' : 'electron-webrtc'
+      this._log(`Using ${using} for webrtc support`)
+      const wstar = new WStar({ wrtc: (wrtc || electronWebRTC) })
+      libp2p.modules.transport = [TCP, WS, wstar]
+      libp2p.modules.peerDiscovery = [MulticastDNS, Bootstrap, wstar.discovery]
+    }
+
+    // start the daemon
+    const ipfsOpts = Object.assign({ init: false }, this._options, { start: true, libp2p })
+    const ipfs = new IPFS(ipfsOpts)
+
+    await new Promise((resolve, reject) => {
+      ipfs.once('error', err => {
+        this._log('error starting core', err)
+        err.code = 'ENOENT'
+        reject(err)
+      })
+      ipfs.once('start', resolve)
+    })
+
+    this._ipfs = ipfs
+
+    // start HTTP servers (if API or Gateway is enabled in options)
+    const httpApi = new HttpApi(ipfs, ipfsOpts)
+    this._httpApi = await httpApi.start()
+
+    // for the CLI to know the where abouts of the API
+    if (this._httpApi._apiServers.length) {
+      await promisify(ipfs._repo.apiAddr.set)(this._httpApi._apiServers[0].info.ma)
+    }
+
+    this._log('started')
+    return this
+  }
+
+  async stop () {
+    this._log('stopping')
+    await Promise.all([
+      this._httpApi && this._httpApi.stop(),
+      this._ipfs && this._ipfs.stop()
+    ])
+    this._log('stopped')
+    return this
+  }
+}
+
+module.exports = Daemon

src/http/index.js

@@ -4,16 +4,8 @@ const Hapi = require('hapi')
 const Pino = require('hapi-pino')
 const debug = require('debug')
 const multiaddr = require('multiaddr')
-const promisify = require('promisify-es6')
-const toUri = require('multiaddr-to-uri')
 const toMultiaddr = require('uri-to-multiaddr')
 
-const IPFS = require('../core')
-const WStar = require('libp2p-webrtc-star')
-const TCP = require('libp2p-tcp')
-const MulticastDNS = require('libp2p-mdns')
-const WS = require('libp2p-websockets')
-const Bootstrap = require('libp2p-bootstrap')
 const errorHandler = require('./error-handler')
 const LOG = 'ipfs:http-api'
 const LOG_ERROR = 'ipfs:http-api:error'
@@ -48,7 +40,8 @@ function serverCreator (serverAddrs, createServer, ipfs) {
 }
 
 class HttpApi {
-  constructor (options) {
+  constructor (ipfs, options) {
+    this._ipfs = ipfs
     this._options = options || {}
     this._log = debug(LOG)
     this._log.error = debug(LOG_ERROR)
@@ -66,68 +59,17 @@ class HttpApi {
   async start () {
     this._log('starting')
 
-    const libp2p = { modules: {} }
-
-    // Attempt to use any of the WebRTC versions available globally
-    let electronWebRTC
-    let wrtc
-    try {
-      electronWebRTC = require('electron-webrtc')()
-    } catch (err) {
-      this._log('failed to load optional electron-webrtc dependency')
-    }
-    try {
-      wrtc = require('wrtc')
-    } catch (err) {
-      this._log('failed to load optional webrtc dependency')
-    }
-
-    if (wrtc || electronWebRTC) {
-      const using = wrtc ? 'wrtc' : 'electron-webrtc'
-      this._log(`Using ${using} for webrtc support`)
-      const wstar = new WStar({ wrtc: (wrtc || electronWebRTC) })
-      libp2p.modules.transport = [TCP, WS, wstar]
-      libp2p.modules.peerDiscovery = [MulticastDNS, Bootstrap, wstar.discovery]
-    }
-
-    // start the daemon
-    const ipfsOpts = Object.assign({ init: false }, this._options, { start: true, libp2p })
-    const ipfs = new IPFS(ipfsOpts)
-
-    await new Promise((resolve, reject) => {
-      ipfs.once('error', err => {
-        this._log('error starting core', err)
-        err.code = 'ENOENT'
-        reject(err)
-      })
-      ipfs.once('start', resolve)
-    })
-
-    this._ipfs = ipfs
+    const ipfs = this._ipfs
 
     const config = await ipfs.config.get()
     config.Addresses = config.Addresses || {}
 
     const apiAddrs = config.Addresses.API
     this._apiServers = await serverCreator(apiAddrs, this._createApiServer, ipfs)
 
-    // for the CLI to know the where abouts of the API
-    if (this._apiServers.length) {
-      await promisify(ipfs._repo.apiAddr.set)(this._apiServers[0].info.ma)
-    }
-
     const gatewayAddrs = config.Addresses.Gateway
     this._gatewayServers = await serverCreator(gatewayAddrs, this._createGatewayServer, ipfs)
 
-    this._apiServers.forEach(apiServer => {
-      ipfs._print('API listening on %s', apiServer.info.ma)
-    })
-    this._gatewayServers.forEach(gatewayServer => {
-      ipfs._print('Gateway (read only) listening on %s', gatewayServer.info.ma)
-    })
-    this._apiServers.forEach(apiServer => {
-      ipfs._print('Web UI available at %s', toUri(apiServer.info.ma) + '/webui')
-    })
     this._log('started')
     return this
   }
@@ -208,8 +150,7 @@ class HttpApi {
     const stopServers = servers => Promise.all((servers || []).map(s => s.stop()))
     await Promise.all([
       stopServers(this._apiServers),
-      stopServers(this._gatewayServers),
-      this._ipfs && this._ipfs.stop()
+      stopServers(this._gatewayServers)
     ])
     this._log('stopped')
     return this

test/gateway/index.js

@@ -5,7 +5,7 @@ const chai = require('chai')
 const dirtyChai = require('dirty-chai')
 const expect = chai.expect
 chai.use(dirtyChai)
-const API = require('../../src/http')
+const Daemon = require('../../src/cli/daemon')
 const loadFixture = require('aegir/fixtures')
 const os = require('os')
 const path = require('path')
@@ -33,7 +33,7 @@ describe('HTTP Gateway', function () {
     this.timeout(60 * 1000)
     const repoPath = path.join(os.tmpdir(), '/ipfs-' + hat())
 
-    http.api = new API({
+    http.api = new Daemon({
       repo: repoPath,
       init: true,
       config: {
@@ -60,7 +60,7 @@ describe('HTTP Gateway', function () {
 
     await http.api.start()
 
-    gateway = http.api._gatewayServers[0]
+    gateway = http.api._httpApi._gatewayServers[0]
 
     // QmbQD7EMEL1zeebwBsWEfA3ndgSS6F7S6iTuwuqasPgVRi
     await http.api._ipfs.add([

test/http-api/inject/bitswap.js

@@ -12,7 +12,7 @@ module.exports = (http) => {
     let api
 
     before(() => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     before(async function () {

test/http-api/inject/block.js

@@ -13,7 +13,7 @@ module.exports = (http) => {
     let api
 
     before(() => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     describe('/block/put', () => {

test/http-api/inject/bootstrap.js

@@ -11,7 +11,7 @@ module.exports = (http) => {
     let api
 
     before(() => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
       return api.inject({
         method: 'GET',
         url: '/api/v0/bootstrap/add/default'

test/http-api/inject/config.js

@@ -17,7 +17,7 @@ module.exports = (http) => {
 
     before(() => {
       updatedConfig = () => JSON.parse(fs.readFileSync(configPath, 'utf8'))
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     after(() => {

test/http-api/inject/dag.js

@@ -33,7 +33,7 @@ module.exports = (http) => {
     let api
 
     before(() => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     describe('/dag/get', () => {

test/http-api/inject/dht.js

@@ -12,7 +12,7 @@ module.exports = (http) => {
     let api
 
     before(() => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     describe('/findpeer', () => {

test/http-api/inject/dns.js

@@ -8,7 +8,7 @@ module.exports = (http) => {
     let api
 
     before(() => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     it('resolve ipfs.io dns', async () => {

test/http-api/inject/files.js

@@ -13,7 +13,7 @@ module.exports = (http) => {
     let api
 
     before(() => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     describe('/add', () => {

test/http-api/inject/id.js

@@ -8,7 +8,7 @@ module.exports = (http) => {
     let api
 
     before(() => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     it('get the id', async () => {

test/http-api/inject/name.js

@@ -16,7 +16,7 @@ module.exports = (http) => {
     let api
 
     before(() => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     it('should publish a record', async function () {

test/http-api/inject/object.js

@@ -17,7 +17,7 @@ module.exports = (http) => {
     let api
 
     before('api', () => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     describe('/new', () => {

test/http-api/inject/pin.js

@@ -37,7 +37,7 @@ module.exports = (http) => {
     let api
 
     before(() => {
-      api = http.api._apiServers[0]
+      api = http.api._httpApi._apiServers[0]
     })
 
     describe('rm', () => {