feat: integrate ipfs-repo-migrations tool (#2527)

Resolves: #1115

Supersedes: #2044

Depends on:

* [x] ipfs/js-ipfs-repo#202

.aegir.js

@@ -11,7 +11,7 @@ const preloadNode = MockPreloadNode.createNode()
 const echoServer = EchoServer.createServer()
 
 module.exports = {
-  bundlesize: { maxSize: '685kB' },
+  bundlesize: { maxSize: '650kB' },
   webpack: {
     resolve: {
       mainFields: ['browser', 'main'],

README.md

@@ -284,6 +284,18 @@ Example:
 const node = await IPFS.create({ repo: '/var/ipfs/data' })
 ```
 
+##### `options.repoAutoMigrate`
+
+| Type | Default |
+|------|---------|
+| boolean | `true` |
+
+`js-ipfs` comes bundled with a tool that automatically migrates your IPFS repository when a new version is available.
+
+**For apps that build on top of `js-ipfs` and run in the browser environment, be aware that disabling automatic
+migrations leaves the user with no way to run the migrations because there is no CLI in the browser. In such
+a case, you should provide a way to trigger migrations manually.**
+
 ##### `options.init`
 
 | Type | Default |

package.json

@@ -104,7 +104,7 @@
     "ipfs-http-response": "~0.4.0",
     "ipfs-mfs": "^0.13.0",
     "ipfs-multipart": "^0.2.0",
-    "ipfs-repo": "^0.28.1",
+    "ipfs-repo": "^0.29.0",
     "ipfs-unixfs": "~0.1.16",
     "ipfs-unixfs-exporter": "^0.38.0",
     "ipfs-unixfs-importer": "^0.40.0",

src/cli/bin.js

@@ -26,6 +26,7 @@ if (!semver.satisfies(process.versions.node, pkg.engines.node)) {
 const YargsPromise = require('yargs-promise')
 const updateNotifier = require('update-notifier')
 const debug = require('debug')('ipfs:cli')
+const { errors: { InvalidRepoVersionError } } = require('ipfs-repo')
 const parser = require('./parser')
 const commandAlias = require('./command-alias')
 const { print } = require('./utils')
@@ -50,13 +51,19 @@ cli
   })
   .catch(({ error, argv }) => {
     getIpfs = argv && argv.getIpfs
+
+    if (error.code === InvalidRepoVersionError.code) {
+      error.message = 'Incompatible repo version. Migration needed. Pass --migrate for automatic migration'
+    }
+
     if (error.message) {
       print(error.message)
       debug(error)
     } else {
       print('Unknown error, please re-run the command with DEBUG=ipfs:cli to see debug output')
       debug(error)
     }
+
     process.exit(1)
   })
   .finally(() => {

src/cli/commands/daemon.js

@@ -73,6 +73,7 @@ module.exports = {
         config,
         silent: argv.silent,
         repo: process.env.IPFS_PATH,
+        repoAutoMigrate: argv.migrate,
         offline: argv.offline,
         pass: argv.pass,
         preload: { enabled: argv.enablePreload },
@@ -96,10 +97,8 @@ module.exports = {
           print(`Web UI available at ${toUri(apiServer.info.ma)}/webui`)
         })
       } catch (err) {
-        if (err.code === 'ENOENT' && err.message.match(/uninitialized/i)) {
-          print('Error: no initialized ipfs repo found in ' + repoPath)
-          print('please run: jsipfs init')
-          process.exit(1)
+        if (err.code === 'ERR_REPO_NOT_INITIALIZED' || err.message.match(/uninitialized/i)) {
+          err.message = 'no initialized ipfs repo found in ' + repoPath + '\nplease run: jsipfs init'
         }
         throw err
       }

src/cli/daemon.js

@@ -54,17 +54,8 @@ class Daemon {
     }
 
     // start the daemon
-    const ipfsOpts = Object.assign({ }, { init: true, start: true, libp2p }, this._options)
-    const ipfs = new IPFS(ipfsOpts)
-
-    await new Promise((resolve, reject) => {
-      ipfs.once('error', err => {
-        this._log('error starting core', err)
-        err.code = 'ENOENT'
-        reject(err)
-      })
-      ipfs.once('start', resolve)
-    })
+    const ipfsOpts = Object.assign({}, { init: true, start: true, libp2p }, this._options)
+    const ipfs = await IPFS.create(ipfsOpts)
 
     this._ipfs = ipfs
 

src/cli/parser.js

@@ -19,6 +19,11 @@ const parser = yargs
     type: 'string',
     default: ''
   })
+  .option('migrate', {
+    desc: 'Enable/disable automatic repo migrations',
+    type: 'boolean',
+    default: false
+  })
   .epilog(utils.ipfsPathHelp)
   .demandCommand(1)
   .fail((msg, err, yargs) => {

src/cli/utils.js

@@ -4,9 +4,7 @@ const fs = require('fs')
 const os = require('os')
 const multiaddr = require('multiaddr')
 const path = require('path')
-const debug = require('debug')
-const log = debug('cli')
-log.error = debug('cli:error')
+const log = require('debug')('ipfs:cli:utils')
 const Progress = require('progress')
 const byteman = require('byteman')
 const promisify = require('promisify-es6')
@@ -47,6 +45,7 @@ exports.getIPFS = (argv, callback) => {
   const IPFS = require('../core')
   const node = new IPFS({
     silent: argv.silent,
+    repoAutoMigrate: argv.migrate,
     repo: exports.getRepoPath(),
     init: false,
     start: false,
@@ -60,7 +59,7 @@ exports.getIPFS = (argv, callback) => {
   })
 
   node.on('error', (err) => {
-    throw err
+    callback(err)
   })
 
   node.once('ready', () => {

src/core/config.js

@@ -28,6 +28,7 @@ const s = superstruct({
 const configSchema = s({
   repo: optional(s('object|string')),
   repoOwner: 'boolean?',
+  repoAutoMigrate: 'boolean?',
   preload: s({
     enabled: 'boolean?',
     addresses: optional(s(['multiaddr'])),

src/core/index.js

@@ -70,7 +70,7 @@ class IPFS extends EventEmitter {
 
     if (typeof options.repo === 'string' ||
         options.repo === undefined) {
-      this._repo = defaultRepo(options.repo)
+      this._repo = defaultRepo(options)
     } else {
       this._repo = options.repo
     }

src/core/runtime/repo-browser.js

@@ -2,7 +2,7 @@
 
 const IPFSRepo = require('ipfs-repo')
 
-module.exports = (dir) => {
-  const repoPath = dir || 'ipfs'
-  return new IPFSRepo(repoPath)
+module.exports = (options) => {
+  const repoPath = options.repo || 'ipfs'
+  return new IPFSRepo(repoPath, { autoMigrate: options.repoAutoMigrate })
 }

src/core/runtime/repo-nodejs.js

@@ -4,8 +4,8 @@ const os = require('os')
 const IPFSRepo = require('ipfs-repo')
 const path = require('path')
 
-module.exports = (dir) => {
-  const repoPath = dir || path.join(os.homedir(), '.jsipfs')
+module.exports = (options) => {
+  const repoPath = options.repo || path.join(os.homedir(), '.jsipfs')
 
-  return new IPFSRepo(repoPath)
+  return new IPFSRepo(repoPath, { autoMigrate: options.repoAutoMigrate })
 }

test/cli/general.js

@@ -1,8 +1,17 @@
 /* eslint-env mocha */
 'use strict'
 
+const os = require('os')
+const fs = require('fs').promises
+const path = require('path')
+const hat = require('hat')
 const { expect } = require('interface-ipfs-core/src/utils/mocha')
+const { repoVersion } = require('ipfs-repo')
+const promisify = require('promisify-es6')
+const ncp = promisify(require('ncp').ncp)
 const runOnAndOff = require('../utils/on-and-off')
+const ipfsExec = require('../utils/ipfs-exec')
+const clean = require('../utils/clean')
 
 describe('general cli options', () => runOnAndOff.off((thing) => {
   it('should handle --silent flag', async () => {
@@ -17,3 +26,83 @@ describe('general cli options', () => runOnAndOff.off((thing) => {
     expect(out).to.include('again')
   })
 }))
+
+describe('--migrate', () => {
+  let ipfs, repoPath
+
+  async function setRepoVersion (version) {
+    await fs.writeFile(path.join(repoPath, 'version'), version)
+  }
+
+  async function getRepoVersion () {
+    return parseInt(await fs.readFile(path.join(repoPath, 'version'), 'utf8'))
+  }
+
+  beforeEach(async () => {
+    repoPath = path.join(os.tmpdir(), `ipfs-${hat()}`)
+    const v7RepoPath = path.join(__dirname, '../fixtures/v7-repo')
+    await ncp(v7RepoPath, repoPath)
+    ipfs = ipfsExec(repoPath)
+  })
+
+  afterEach(() => clean(repoPath))
+
+  it('should not migrate for daemon command when --migrate flag not set', async () => {
+    // There are no migrations prior to 7 so it's safe to set version to 5 since
+    // the repo is the same. We set to 5 because version 6 & 7 are considered
+    // the same in repo.version.check.
+    await setRepoVersion(5)
+    const err = await ipfs.fail('daemon')
+    expect(err.stdout).to.include('Pass --migrate for automatic migration')
+    const version = await getRepoVersion()
+    expect(version).to.equal(5) // Should not have migrated
+  })
+
+  it('should not migrate for other commands when --migrate flag not set', async () => {
+    // There are no migrations prior to 7 so it's safe to set version to 5 since
+    // the repo is the same. We set to 5 because version 6 & 7 are considered
+    // the same in repo.version.check.
+    await setRepoVersion(5)
+    const err = await ipfs.fail('files ls')
+    expect(err.stdout).to.include('Pass --migrate for automatic migration')
+    const version = await getRepoVersion()
+    expect(version).to.equal(5) // Should not have migrated
+  })
+
+  it('should migrate for daemon command when --migrate flag set', async () => {
+    // There are no migrations prior to 7 so it's safe to set version to 5 since
+    // the repo is the same. We set to 5 because version 6 & 7 are considered
+    // the same in repo.version.check.
+    await setRepoVersion(5)
+
+    const daemon = ipfs('daemon --migrate')
+    let stdout = ''
+
+    daemon.stdout.on('data', data => {
+      stdout += data.toString('utf8')
+
+      if (stdout.includes('Daemon is ready')) {
+        daemon.kill()
+      }
+    })
+
+    await expect(daemon)
+      .to.eventually.be.rejected()
+      .and.to.include({
+        killed: true
+      })
+
+    const version = await getRepoVersion()
+    expect(version).to.equal(repoVersion) // Should have migrated to latest
+  })
+
+  it('should migrate for other commands when --migrate flag set', async () => {
+    // There are no migrations prior to 7 so it's safe to set version to 5 since
+    // the repo is the same. We set to 5 because version 6 & 7 are considered
+    // the same in repo.version.check.
+    await setRepoVersion(5)
+    await ipfs('files ls --migrate')
+    const version = await getRepoVersion()
+    expect(version).to.equal(repoVersion) // Should have migrated to latest
+  })
+})

test/fixtures/v7-repo/blocks/7J/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data

@@ -0,0 +1,55 @@
+
+��
+                  IPFS -- Inter-Planetary File system
+
+IPFS is a global, versioned, peer-to-peer filesystem. It combines good ideas
+from Git, BitTorrent, Kademlia, SFS, and the Web. It is like a single bit-
+torrent swarm, exchanging git objects. IPFS provides an interface as simple
+as the HTTP web, but with permanence built in. You can also mount the world
+at /ipfs.
+
+IPFS is a protocol:
+- defines a content-addressed file system
+- coordinates content delivery
+- combines Kademlia + BitTorrent + Git
+
+IPFS is a filesystem:
+- has directories and files
+- mountable filesystem (via FUSE)
+
+IPFS is a web:
+- can be used to view documents like the web
+- files accessible via HTTP at `http://ipfs.io/<path>`
+- browsers or extensions can learn to use `ipfs://` directly
+- hash-addressed content guarantees authenticity
+
+IPFS is modular:
+- connection layer over any network protocol
+- routing layer
+- uses a routing layer DHT (kademlia/coral)
+- uses a path-based naming service
+- uses bittorrent-inspired block exchange
+
+IPFS uses crypto:
+- cryptographic-hash content addressing
+- block-level deduplication
+- file integrity + versioning
+- filesystem-level encryption + signing support
+
+IPFS is p2p:
+- worldwide peer-to-peer file transfers
+- completely decentralized architecture
+- **no** central point of failure
+
+IPFS is a cdn:
+- add a file to the filesystem locally, and it's now available to the world
+- caching-friendly (content-hash naming)
+- bittorrent-based bandwidth distribution
+
+IPFS has a name service:
+- IPNS, an SFS inspired name system
+- global namespace based on PKI
+- serves to build trust chains
+- compatible with other NSes
+- can map DNS, .onion, .bit, etc to IPNS
+�

test/fixtures/v7-repo/blocks/EX/CIQKA7ZA5YM6KOJE3HVPZNQ6C4TJVSVGFTWWOTHP7GWZYGDUP5HIEXY.data

@@ -0,0 +1,4 @@
+/
+" �I%�s�!@��<��'����8���@:�шo_�direct�V2
+" �e��I�\(&PD�
+�	�� 2
�hO.߃o�	recursive�V

test/fixtures/v7-repo/blocks/IL/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data

@@ -0,0 +1,8 @@
+
+�
�
Come hang out in our IRC chat room if you have any questions.
+
+Contact the ipfs dev team:
+- Bugs: https://github.com/ipfs/go-ipfs/issues
+- Help: irc.freenode.org/#ipfs
+- Email: dev@ipfs.io
+�

test/fixtures/v7-repo/blocks/LG/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data

@@ -0,0 +1,9 @@
+
+��Some helpful resources for finding your way around ipfs:
+
+- quick-start: a quick show of various ipfs features.
+- ipfs commands: a list of all commands
+- ipfs --help: every command describes itself
+- https://github.com/ipfs/go-ipfs -- the src repository
+- #ipfs on irc.freenode.org -- the community irc channel
+�