feat: Add config profile endpoint and CLI (#2165)

* feat: config profile
* fix: increase command count
* feat: ipfs init --profile option
* chore: update dependencies

License: MIT
Signed-off-by: Alan Shaw <alan.shaw@protocol.ai>

* fix: make sure default-config still works
* chore: add http api tests
* chore: address PR comments
* chore: fix linting
* test: let internals of config profiles be internal
* chore: add test for listing profiles
* chore: turn profile list into list outside of core
* chore: expose profile list over http
* fix: use chai exported from interface tests

Workaround for chaijs/chai#1298

* chore: fix linting
* chore: update deps and make keys agree with spec
* chore: update interface tests
* chore: udpate test skips/includes
* chore: fix up interface tests

README.md

@@ -9,9 +9,9 @@
 <p align="center">
   <a href="http://protocol.ai"><img src="https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat" /></a>
   <a href="http://ipfs.io/"><img src="https://img.shields.io/badge/project-IPFS-blue.svg?style=flat" /></a>
-</p>  
+</p>
 
-<p align="center">  
+<p align="center">
   <a href="https://riot.im/app/#/room/#ipfs-dev:matrix.org"><img src="https://img.shields.io/badge/matrix-%23ipfs%3Amatrix.org-blue.svg?style=flat" /> </a>
   <a href="http://webchat.freenode.net/?channels=%23ipfs"><img src="https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat" /></a>
   <a href="https://discord.gg/24fmuwR"><img src="https://img.shields.io/discord/475789330380488707?color=blueviolet&label=discord&style=flat" /></a>

package.json

@@ -99,7 +99,7 @@
     "ipfs-bitswap": "^0.26.0",
     "ipfs-block": "~0.8.1",
     "ipfs-block-service": "~0.16.0",
-    "ipfs-http-client": "^38.0.1",
+    "ipfs-http-client": "^38.1.0",
     "ipfs-http-response": "~0.3.1",
     "ipfs-mfs": "^0.13.0",
     "ipfs-multipart": "^0.2.0",
@@ -124,6 +124,7 @@
     "iso-url": "~0.4.6",
     "it-pipe": "^1.0.1",
     "it-to-stream": "^0.1.1",
+    "jsondiffpatch": "~0.3.11",
     "just-safe-set": "^2.1.0",
     "kind-of": "^6.0.2",
     "ky": "^0.14.0",
@@ -202,7 +203,7 @@
     "execa": "^2.0.4",
     "form-data": "^2.5.1",
     "hat": "0.0.3",
-    "interface-ipfs-core": "^0.115.3",
+    "interface-ipfs-core": "^0.117.2",
     "ipfs-interop": "~0.1.0",
     "ipfsd-ctl": "^0.47.2",
     "libp2p-websocket-star": "~0.10.2",

src/cli/commands/config/profile.js

@@ -0,0 +1,15 @@
+'use strict'
+
+module.exports = {
+  command: 'profile <command>',
+
+  description: 'Interact with config profiles.',
+
+  builder (yargs) {
+    return yargs
+      .commandDir('profile')
+  },
+
+  handler (argv) {
+  }
+}

src/cli/commands/config/profile/apply.js

@@ -0,0 +1,35 @@
+'use strict'
+
+const JSONDiff = require('jsondiffpatch')
+
+module.exports = {
+  command: 'apply <profile>',
+
+  describe: 'Apply profile to config',
+
+  builder: {
+    'dry-run': {
+      type: 'boolean',
+      describe: 'print difference between the current config and the config that would be generated.'
+    }
+  },
+
+  handler (argv) {
+    argv.resolve((async () => {
+      const ipfs = await argv.getIpfs()
+      const diff = await ipfs.config.profiles.apply(argv.profile, { dryRun: argv.dryRun })
+      const delta = JSONDiff.diff(diff.original, diff.updated)
+      const res = JSONDiff.formatters.console.format(delta, diff.original)
+
+      if (res) {
+        argv.print(res)
+
+        if (ipfs.send) {
+          argv.print('\nThe IPFS daemon is running in the background, you may need to restart it for changes to take effect.')
+        }
+      } else {
+        argv.print(`IPFS config already contains the settings from the '${argv.profile}' profile`)
+      }
+    })())
+  }
+}

src/cli/commands/config/profile/ls.js

@@ -0,0 +1,21 @@
+'use strict'
+
+module.exports = {
+  command: 'ls',
+
+  describe: 'List available config profiles',
+
+  builder: {},
+
+  handler (argv) {
+    argv.resolve(
+      (async () => {
+        const ipfs = await argv.getIpfs()
+
+        for (const profile of await ipfs.config.profiles.list()) {
+          argv.print(`${profile.name}:\n ${profile.description}`)
+        }
+      })()
+    )
+  }
+}

src/cli/commands/init.js

@@ -6,12 +6,15 @@ const { ipfsPathHelp } = require('../utils')
 
 module.exports = {
   command: 'init [default-config] [options]',
-  describe: 'Initialize a local IPFS node',
+  describe: 'Initialize a local IPFS node\n\n' +
+    'If you are going to run IPFS in a server environment, you may want to ' +
+    'initialize it using the \'server\' profile.\n\n' +
+    'For the list of available profiles run `jsipfs config profile ls`',
   builder (yargs) {
     return yargs
       .epilog(ipfsPathHelp)
       .positional('default-config', {
-        describe: 'Initialize with the given configuration. Path to the config file. Check https://github.com/ipfs/js-ipfs#optionsconfig',
+        describe: 'Node config, this should be a path to a file or JSON and will be merged with the default config. See https://github.com/ipfs/js-ipfs#optionsconfig',
         type: 'string'
       })
       .option('bits', {
@@ -30,6 +33,14 @@ module.exports = {
         type: 'string',
         describe: 'Pre-generated private key to use for the repo'
       })
+      .option('profile', {
+        alias: 'p',
+        type: 'string',
+        describe: 'Apply profile settings to config. Multiple profiles can be separated by \',\'',
+        coerce: (value) => {
+          return (value || '').split(',')
+        }
+      })
   },
 
   handler (argv) {
@@ -66,6 +77,7 @@ module.exports = {
           bits: argv.bits,
           privateKey: argv.privateKey,
           emptyRepo: argv.emptyRepo,
+          profiles: argv.profile,
           pass: argv.pass,
           log: argv.print
         })

src/core/components/config.js

@@ -1,11 +1,129 @@
 'use strict'
 
 const callbackify = require('callbackify')
+const getDefaultConfig = require('../runtime/config-nodejs.js')
+const log = require('debug')('ipfs:core:config')
 
 module.exports = function config (self) {
   return {
     get: callbackify.variadic(self._repo.config.get),
     set: callbackify(self._repo.config.set),
-    replace: callbackify.variadic(self._repo.config.set)
+    replace: callbackify.variadic(self._repo.config.set),
+    profiles: {
+      apply: callbackify.variadic(applyProfile),
+      list: callbackify.variadic(listProfiles)
+    }
+  }
+
+  async function applyProfile (profileName, opts) {
+    opts = opts || {}
+    const { dryRun } = opts
+
+    const profile = profiles[profileName]
+
+    if (!profile) {
+      throw new Error(`No profile with name '${profileName}' exists`)
+    }
+
+    try {
+      const oldCfg = await self.config.get()
+      let newCfg = JSON.parse(JSON.stringify(oldCfg)) // clone
+      newCfg = profile.transform(newCfg)
+
+      if (!dryRun) {
+        await self.config.replace(newCfg)
+      }
+
+      // Scrub private key from output
+      delete oldCfg.Identity.PrivKey
+      delete newCfg.Identity.PrivKey
+
+      return { original: oldCfg, updated: newCfg }
+    } catch (err) {
+      log(err)
+
+      throw new Error(`Could not apply profile '${profileName}' to config: ${err.message}`)
+    }
+  }
+}
+
+async function listProfiles (options) { // eslint-disable-line require-await
+  return Object.keys(profiles).map(name => ({
+    name,
+    description: profiles[name].description
+  }))
+}
+
+const profiles = {
+  server: {
+    description: 'Disables local host discovery - recommended when running IPFS on machines with public IPv4 addresses.',
+    transform: (config) => {
+      config.Discovery.MDNS.Enabled = false
+      config.Discovery.webRTCStar.Enabled = false
+
+      return config
+    }
+  },
+  'local-discovery': {
+    description: 'Enables local host discovery - inverse of "server" profile.',
+    transform: (config) => {
+      config.Discovery.MDNS.Enabled = true
+      config.Discovery.webRTCStar.Enabled = true
+
+      return config
+    }
+  },
+  lowpower: {
+    description: 'Reduces daemon overhead on the system - recommended for low power systems.',
+    transform: (config) => {
+      config.Swarm = config.Swarm || {}
+      config.Swarm.ConnMgr = config.Swarm.ConnMgr || {}
+      config.Swarm.ConnMgr.LowWater = 20
+      config.Swarm.ConnMgr.HighWater = 40
+
+      return config
+    }
+  },
+  'default-power': {
+    description: 'Inverse of "lowpower" profile.',
+    transform: (config) => {
+      const defaultConfig = getDefaultConfig()
+
+      config.Swarm = defaultConfig.Swarm
+
+      return config
+    }
+  },
+  test: {
+    description: 'Reduces external interference of IPFS daemon - for running the daemon in test environments.',
+    transform: (config) => {
+      const defaultConfig = getDefaultConfig()
+
+      config.Addresses.API = defaultConfig.Addresses.API ? '/ip4/127.0.0.1/tcp/0' : ''
+      config.Addresses.Gateway = defaultConfig.Addresses.Gateway ? '/ip4/127.0.0.1/tcp/0' : ''
+      config.Addresses.Swarm = defaultConfig.Addresses.Swarm.length ? ['/ip4/127.0.0.1/tcp/0'] : []
+      config.Bootstrap = []
+      config.Discovery.MDNS.Enabled = false
+      config.Discovery.webRTCStar.Enabled = false
+
+      return config
+    }
+  },
+  'default-networking': {
+    description: 'Restores default network settings - inverse of "test" profile.',
+    transform: (config) => {
+      const defaultConfig = getDefaultConfig()
+
+      config.Addresses.API = defaultConfig.Addresses.API
+      config.Addresses.Gateway = defaultConfig.Addresses.Gateway
+      config.Addresses.Swarm = defaultConfig.Addresses.Swarm
+      config.Bootstrap = defaultConfig.Bootstrap
+      config.Discovery.MDNS.Enabled = defaultConfig.Discovery.MDNS.Enabled
+      config.Discovery.webRTCStar.Enabled = defaultConfig.Discovery.webRTCStar.Enabled
+
+      return config
+    }
   }
 }
+
+module.exports.profiles = profiles

src/core/components/files-mfs.js

@@ -46,7 +46,7 @@ module.exports = (/** @type { import("../index") } */ ipfs) => {
 
     if (paths.length) {
       const options = args[args.length - 1]
-      if (options.preload !== false) {
+      if (options && options.preload !== false) {
         paths.forEach(path => ipfs._preload(path))
       }
     }

src/core/components/init.js

@@ -16,6 +16,7 @@ const IPNS = require('../ipns')
 const OfflineDatastore = require('../ipns/routing/offline-datastore')
 
 const addDefaultAssets = require('./init-assets')
+const { profiles } = require('./config')
 
 function createPeerId (self, opts) {
   if (opts.privateKey) {
@@ -55,6 +56,8 @@ async function createRepo (self, opts) {
 
   const config = mergeOptions(defaultConfig(), self._options.config)
 
+  applyProfile(self, config, opts)
+
   // Verify repo does not exist yet
   const exists = await self._repo.exists()
   self.log('repo exists?', exists)
@@ -128,8 +131,24 @@ async function addRepoAssets (self, privateKey, opts) {
   }
 }
 
+// Apply profiles (eg "server,lowpower") to config
+function applyProfile (self, config, opts) {
+  if (opts.profiles) {
+    for (const name of opts.profiles) {
+      const profile = profiles[name]
+
+      if (!profile) {
+        throw new Error(`Could not find profile with name '${name}'`)
+      }
+
+      self.log(`applying profile ${name}`)
+      profile.transform(config)
+    }
+  }
+}
+
 module.exports = function init (self) {
-  return callbackify(async (opts) => {
+  return callbackify.variadic(async (opts) => {
     opts = opts || {}
 
     await createRepo(self, opts)

src/http/api/resources/config.js

@@ -7,6 +7,8 @@ const log = debug('ipfs:http-api:config')
 log.error = debug('ipfs:http-api:config:error')
 const multipart = require('ipfs-multipart')
 const Boom = require('@hapi/boom')
+const Joi = require('@hapi/joi')
+const { profiles } = require('../../../core/components/config')
 const all = require('async-iterator-all')
 
 exports.getOrSet = {
@@ -163,3 +165,53 @@ exports.replace = {
     return h.response()
   }
 }
+
+exports.profiles = {
+  apply: {
+    validate: {
+      query: Joi.object().keys({
+        'dry-run': Joi.boolean().default(false)
+      }).unknown()
+    },
+
+    // pre request handler that parses the args and returns `profile` which is assigned to `request.pre.args`
+    parseArgs: function (request, h) {
+      if (!request.query.arg) {
+        throw Boom.badRequest("Argument 'profile' is required")
+      }
+
+      if (!profiles[request.query.arg]) {
+        throw Boom.badRequest("Argument 'profile' is not a valid profile name")
+      }
+
+      return { profile: request.query.arg }
+    },
+
+    handler: async function (request, h) {
+      const { ipfs } = request.server.app
+      const { profile } = request.pre.args
+      const dryRun = request.query['dry-run']
+
+      try {
+        const diff = await ipfs.config.profiles.apply(profile, { dryRun })
+
+        return h.response({ OldCfg: diff.original, NewCfg: diff.updated })
+      } catch (err) {
+        throw Boom.boomify(err, { message: 'Failed to apply profile' })
+      }
+    }
+  },
+  list: {
+    handler: async function (request, h) {
+      const { ipfs } = request.server.app
+      const list = await ipfs.config.profiles.list()
+
+      return h.response(
+        list.map(profile => ({
+          Name: profile.name,
+          Description: profile.description
+        }))
+      )
+    }
+  }
+}