feat: stats API (stats.bitswap and stats.repo) (#1198)

README.md

@@ -256,11 +256,6 @@ A complete API definition is in the works. Meanwhile, you can learn how to you u
   - [`ipfs.block.put(block, cid, [callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/BLOCK.md#put)
   - [`ipfs.block.stat(cid, [callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/BLOCK.md#stat)
 
-- [repo](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/)
-  - `ipfs.repo.init`
-  - `ipfs.repo.version`
-  - `ipfs.repo.gc` (not implemented, yet!)
-
 #### `Graph`
 
 - [dag](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/DAG.md)
@@ -334,6 +329,17 @@ A complete API definition is in the works. Meanwhile, you can learn how to you u
   - `ipfs.start([callback])`
   - [`ipfs.stop([callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/MISCELLANEOUS.md#stop)
   - `ipfs.isOnline()`
+
+- [repo](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/REPO.md)
+  - `ipfs.repo.init`
+  - [`ipfs.repo.stat([options, callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/REPO.md#stat)
+  - [`ipfs.repo.version([callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/REPO.md#version)
+  - `ipfs.repo.gc([options, callback])` (not implemented, yet!)
+
+- [stats](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/STATS.md)
+  - [`ipfs.stats.bitswap([callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/STATS.md#bitswap)
+  - [`ipfs.stats.bw([options, callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/STATS.md#bw) (not implemented, yet!)
+  - [`ipfs.stats.repo([options, callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/STATS.md#repo)
 
 - [config](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/CONFIG.md)
   - [`ipfs.config.get([key, callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/CONFIG.md#configget)

package.json

@@ -73,7 +73,7 @@
     "form-data": "^2.3.2",
     "go-ipfs-dep": "^0.4.13",
     "hat": "0.0.3",
-    "interface-ipfs-core": "~0.51.0",
+    "interface-ipfs-core": "~0.52.0",
     "ipfsd-ctl": "~0.28.0",
     "left-pad": "^1.2.0",
     "lodash": "^4.17.5",
@@ -90,6 +90,7 @@
   },
   "dependencies": {
     "async": "^2.6.0",
+    "big.js": "^5.0.3",
     "binary-querystring": "~0.1.2",
     "bl": "^1.2.1",
     "boom": "^7.1.1",
@@ -107,6 +108,7 @@
     "hoek": "^5.0.3",
     "ipfs-api": "^18.0.0",
     "ipfs-bitswap": "~0.19.0",
+    "human-to-milliseconds": "^1.0.0",
     "ipfs-block": "~0.6.1",
     "ipfs-block-service": "~0.13.0",
     "ipfs-multipart": "~0.1.0",

src/cli/commands/bitswap/stat.js

@@ -16,22 +16,22 @@ module.exports = {
         throw err
       }
 
-      stats.Wantlist = stats.Wantlist || []
-      stats.Wantlist = stats.Wantlist.map((entry) => {
+      stats.wantlist = stats.wantlist || []
+      stats.wantlist = stats.wantlist.map((entry) => {
         const buf = Buffer.from(entry.cid.hash.data)
         const cid = new CID(entry.cid.version, entry.cid.codec, buf)
         return cid.toBaseEncodedString()
       })
-      stats.Peers = stats.Peers || []
+      stats.peers = stats.peers || []
 
       print(`bitswap status
-  blocks received: ${stats.BlocksReceived}
-  dup blocks received: ${stats.DupBlksReceived}
-  dup data received: ${stats.DupDataReceived}B
-  wantlist [${stats.Wantlist.length} keys]
-    ${stats.Wantlist.join('\n    ')}
-  partners [${stats.Peers.length}]
-    ${stats.Peers.join('\n    ')}`)
+  blocks received: ${stats.blocksReceived}
+  dup blocks received: ${stats.dupBlksReceived}
+  dup data received: ${stats.dupDataReceived}B
+  wantlist [${stats.wantlist.length} keys]
+    ${stats.wantlist.join('\n    ')}
+  partners [${stats.peers.length}]
+    ${stats.peers.join('\n    ')}`)
     })
   }
 }

src/cli/commands/repo/stat.js

@@ -0,0 +1,31 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'stat',
+
+  describe: 'Get stats for the currently used repo',
+
+  builder: {
+    human: {
+      type: 'boolean',
+      default: false
+    }
+  },
+
+  handler (argv) {
+    argv.ipfs.repo.stat({human: argv.human}, (err, stats) => {
+      if (err) {
+        throw err
+      }
+
+      print(`repo status
+  number of objects: ${stats.numObjects}
+  repo size: ${stats.repoSize}
+  repo path: ${stats.repoPath}
+  version: ${stats.version}
+  maximum storage: ${stats.storageMax}`)
+    })
+  }
+}

src/cli/commands/repo/version.js

@@ -10,10 +10,11 @@ module.exports = {
   builder: {},
 
   handler (argv) {
-    argv.ipfs.repo.version(function (err, version) {
+    argv.ipfs.repo.version((err, version) => {
       if (err) {
         throw err
       }
+
       print(version)
     })
   }

src/cli/commands/stats.js

@@ -0,0 +1,14 @@
+'use strict'
+
+module.exports = {
+  command: 'stats <command>',
+
+  description: 'Query IPFS statistics.',
+
+  builder (yargs) {
+    return yargs.commandDir('stats')
+  },
+
+  handler (argv) {
+  }
+}

src/cli/commands/stats/bitswap.js

@@ -0,0 +1,37 @@
+'use strict'
+
+const CID = require('cids')
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'bitswap',
+
+  describe: 'Show some diagnostic information on the bitswap agent.',
+
+  builder: {},
+
+  handler (argv) {
+    argv.ipfs.stats.bitswap((err, stats) => {
+      if (err) {
+        throw err
+      }
+
+      stats.wantlist = stats.wantlist || []
+      stats.wantlist = stats.wantlist.map((entry) => {
+        const buf = Buffer.from(entry.cid.hash.data)
+        const cid = new CID(entry.cid.version, entry.cid.codec, buf)
+        return cid.toBaseEncodedString()
+      })
+      stats.peers = stats.peers || []
+
+      print(`bitswap status
+  blocks received: ${stats.blocksReceived}
+  dup blocks received: ${stats.dupBlksReceived}
+  dup data received: ${stats.dupDataReceived}B
+  wantlist [${stats.wantlist.length} keys]
+    ${stats.wantlist.join('\n    ')}
+  partners [${stats.peers.length}]
+    ${stats.peers.join('\n    ')}`)
+    })
+  }
+}

src/cli/commands/stats/repo.js

@@ -0,0 +1,31 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'repo',
+
+  describe: 'Get stats for the currently used repo',
+
+  builder: {
+    human: {
+      type: 'boolean',
+      default: false
+    }
+  },
+
+  handler (argv) {
+    argv.ipfs.stats.repo({human: argv.human}, (err, stats) => {
+      if (err) {
+        throw err
+      }
+
+      print(`repo status
+  number of objects: ${stats.numObjects}
+  repo size: ${stats.repoSize}
+  repo path: ${stats.repoPath}
+  version: ${stats.version}
+  maximum storage: ${stats.storageMax}`)
+    })
+  }
+}

src/core/components/bitswap.js

@@ -1,6 +1,9 @@
 'use strict'
 
 const OFFLINE_ERROR = require('../utils').OFFLINE_ERROR
+const promisify = require('promisify-es6')
+const setImmediate = require('async/setImmediate')
+const Big = require('big.js')
 
 function formatWantlist (list) {
   return Array.from(list).map((e) => e[1])
@@ -16,16 +19,27 @@ module.exports = function bitswap (self) {
       const list = self._bitswap.getWantlist()
       return formatWantlist(list)
     },
-    stat: () => {
+
+    stat: promisify((callback) => {
       if (!self.isOnline()) {
-        throw new Error(OFFLINE_ERROR)
+        return setImmediate(() => callback(new Error(OFFLINE_ERROR)))
       }
 
-      return Object.assign({}, self._bitswap.stat().snapshot, {
+      const snapshot = self._bitswap.stat().snapshot
+
+      callback(null, {
+        provideBufLen: parseInt(snapshot.providesBufferLength.toString()),
+        blocksReceived: new Big(snapshot.blocksReceived),
         wantlist: formatWantlist(self._bitswap.getWantlist()),
-        peers: self._bitswap.peers().map((id) => id.toB58String())
+        peers: self._bitswap.peers().map((id) => id.toB58String()),
+        dupBlksReceived: new Big(snapshot.dupBlksReceived),
+        dupDataReceived: new Big(snapshot.dupDataReceived),
+        dataReceived: new Big(snapshot.dataReceived),
+        blocksSent: new Big(snapshot.blocksSent),
+        dataSent: new Big(snapshot.dataSent)
       })
-    },
+    }),
+
     unwant: (key) => {
       if (!self.isOnline()) {
         throw new Error(OFFLINE_ERROR)

src/core/components/index.js

@@ -22,3 +22,4 @@ exports.pubsub = require('./pubsub')
 exports.dht = require('./dht')
 exports.dns = require('./dns')
 exports.key = require('./key')
+exports.stats = require('./stats')

src/core/components/repo.js

@@ -1,16 +1,37 @@
 'use strict'
 
+const promisify = require('promisify-es6')
+
 module.exports = function repo (self) {
   return {
     init: (bits, empty, callback) => {
       // 1. check if repo already exists
     },
 
-    version: (callback) => {
+    version: promisify((callback) => {
       self._repo.version.get(callback)
-    },
+    }),
+
+    gc: () => {},
+
+    stat: promisify((options, callback) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
+      self._repo.stat(options, (err, stats) => {
+        if (err) return callback(err)
 
-    gc: function () {},
+        callback(null, {
+          numObjects: stats.numObjects,
+          repoSize: stats.repoSize,
+          repoPath: stats.repoPath,
+          version: stats.version.toString(),
+          storageMax: stats.storageMax
+        })
+      })
+    }),
 
     path: () => self._repo.path
   }

src/core/components/start.js

@@ -45,7 +45,7 @@ module.exports = (self) => {
       self._bitswap = new Bitswap(
         self._libp2pNode,
         self._repo.blocks,
-        self._peerInfoBook
+        { statsEnabled: true }
       )
 
       self._bitswap.start()

src/core/components/stats.js

@@ -0,0 +1,8 @@
+'use strict'
+
+module.exports = function stats (self) {
+  return {
+    bitswap: require('./bitswap')(self).stat,
+    repo: require('./repo')(self).stat
+  }
+}

src/core/index.js

@@ -97,6 +97,7 @@ class IPFS extends EventEmitter {
     this.dht = components.dht(this)
     this.dns = components.dns(this)
     this.key = components.key(this)
+    this.stats = components.stats(this)
 
     if (this._options.EXPERIMENTAL.pubsub) {
       this.log('EXPERIMENTAL pubsub is enabled')

src/http/api/resources/bitswap.js

@@ -21,19 +21,27 @@ exports.wantlist = (request, reply) => {
 }
 
 exports.stat = (request, reply) => {
-  let stats
-  try {
-    stats = request.server.app.ipfs.bitswap.stat()
-  } catch (err) {
-    return reply(boom.badRequest(err))
-  }
-
-  reply({
-    BlocksReceived: stats.blocksReceived,
-    Wantlist: stats.wantlist,
-    Peers: stats.peers,
-    DupBlksReceived: stats.dupBlksReceived,
-    DupDataReceived: stats.dupDataReceived
+  const ipfs = request.server.app.ipfs
+
+  ipfs.bitswap.stat((err, stats) => {
+    if (err) {
+      return reply({
+        Message: err.toString(),
+        Code: 0
+      }).code(500)
+    }
+
+    reply({
+      ProvideBufLen: stats.provideBufLen,
+      BlocksReceived: stats.blocksReceived,
+      Wantlist: stats.wantlist,
+      Peers: stats.peers,
+      DupBlksReceived: stats.dupBlksReceived,
+      DupDataReceived: stats.dupDataReceived,
+      DataReceived: stats.dataReceived,
+      BlocksSent: stats.blocksSent,
+      DataSent: stats.dataSent
+    })
   })
 }
 

src/http/api/resources/index.js

@@ -15,3 +15,4 @@ exports.files = require('./files')
 exports.pubsub = require('./pubsub')
 exports.dns = require('./dns')
 exports.key = require('./key')
+exports.stats = require('./stats')