diff --git a/migrations/index.js b/migrations/index.js index 7d8f06c..182e78f 100644 --- a/migrations/index.js +++ b/migrations/index.js @@ -11,11 +11,12 @@ const emptyMigration = { } module.exports = [ - Object.assign({}, emptyMigration, { version: 7, revert: undefined }), - Object.assign({}, emptyMigration, { version: 6, revert: undefined }), - Object.assign({}, emptyMigration, { version: 5, revert: undefined }), - Object.assign({}, emptyMigration, { version: 4, revert: undefined }), - Object.assign({}, emptyMigration, { version: 3, revert: undefined }), - Object.assign({}, emptyMigration, { version: 2, revert: undefined }), - Object.assign({}, emptyMigration, { version: 1, revert: undefined }) + Object.assign({version: 1}, emptyMigration), + Object.assign({version: 2}, emptyMigration), + Object.assign({version: 3}, emptyMigration), + Object.assign({version: 4}, emptyMigration), + Object.assign({version: 5}, emptyMigration), + Object.assign({version: 6}, emptyMigration), + Object.assign({version: 7}, emptyMigration), + require('./migration-8') ] diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js new file mode 100644 index 0000000..69febe9 --- /dev/null +++ b/migrations/migration-8/index.js @@ -0,0 +1,88 @@ +'use strict' + +const path = require('path') +const CID = require('cids') +const Key = require('interface-datastore').Key +const core = require('datastore-core') +const ShardingStore = core.ShardingDatastore +const mb = require('multibase') +const utils = require('../../src/utils') +const log = require('debug')('ipfs-repo-migrations:migration-8') + +// This function in js-ipfs-repo defaults to not using sharding +// but the default value of the options.sharding is true hence this +// function defaults to use sharding. +async function maybeWithSharding (filestore, options) { + if (options.sharding === false) { + return filestore + } + + const shard = new core.shard.NextToLast(2) + + return ShardingStore.createOrOpen(filestore, shard) +} + +function keyToMultihash (key) { + const buf = mb.decode(`b${key.toString().slice(1)}`) + + // Extract multihash from CID + let multihash = new CID(buf).multihash + + // Encode and slice off multibase codec + multihash = mb.encode('base32', multihash).slice(1) + + // Should be uppercase for interop with go + multihash = multihash.toString().toUpperCase() + + return new Key(`/${multihash}`, false) +} + +function keyToCid (key) { + const buf = mb.decode(`b${key.toString().slice(1)}`) + + // CID to Key + const multihash = mb.encode('base32', new CID(1, 'raw', buf).buffer).slice(1) + + return new Key(`/${multihash}`.toUpperCase(), false) +} + +async function process (repoPath, options, keyFunction){ + const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'blocks') + + const baseStore = new StorageBackend(path.join(repoPath, 'blocks'), storageOptions) + await baseStore.open() + const store = await maybeWithSharding(baseStore, storageOptions) + await store.open() + + try { + let counter = 0 + + for await (const block of store.query({})) { + const newKey = keyFunction(block.key) + + // If the Key is base32 CIDv0 then there's nothing to do + if(newKey.toString() !== block.key.toString()) { + counter += 1 + + log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`) + await store.delete(block.key) + await store.put(newKey, block.value) + } + } + + log(`Changed ${ counter } blocks`) + } finally { + await store.close() + } +} + +module.exports = { + version: 8, + description: 'Transforms key names into base32 encoding and converts Block store to use bare multihashes encoded as base32', + migrate: (repoPath, options = {}) => { + return process(repoPath, options, keyToMultihash) + }, + revert: (repoPath, options = {}) => { + return process(repoPath, options, keyToCid) + } +} diff --git a/package.json b/package.json index 74fa306..d1decf4 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "main": "src/index.js", "browser": { "./src/repo/lock.js": "./src/repo/lock-memory.js", - "datastore-fs": "datastore-idb" + "datastore-fs": "datastore-level" }, "bin": { "jsipfs-migrations": "./src/cli.js" @@ -46,16 +46,19 @@ "dependencies": { "buffer": "^5.6.0", "chalk": "^4.0.0", + "cids": "^0.8.3", + "datastore-core": "^1.1.0", "datastore-fs": "^1.0.0", - "datastore-idb": "^1.0.2", + "datastore-level": "^1.1.0", "debug": "^4.1.0", - "interface-datastore": "^1.0.4", + "interface-datastore": "^1.0.2", + "multibase": "^1.0.1", "proper-lockfile": "^4.1.1", "yargs": "^15.3.1", "yargs-promise": "^1.1.0" }, "devDependencies": { - "aegir": "^25.0.0", + "aegir": "^23.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "dirty-chai": "^2.0.1", diff --git a/test/browser.js b/test/browser.js index cb707f7..a8f55dd 100644 --- a/test/browser.js +++ b/test/browser.js @@ -3,7 +3,7 @@ const { Buffer } = require('buffer') const loadFixture = require('aegir/fixtures') -const Datastore = require('datastore-idb') +const Datastore = require('datastore-level') const Key = require('interface-datastore').Key const CONFIG_KEY = new Key('config') @@ -44,6 +44,10 @@ describe('Browser specific tests', () => { require('./version-test')(createRepo, repoCleanup) }) + describe('migrations tests', () => { + require('./migrations/migration-8-test')(createRepo, repoCleanup) + }) + describe('init tests', () => { require('./init-test')(createRepo, repoCleanup) }) diff --git a/test/migrations/migration-8-test.js b/test/migrations/migration-8-test.js new file mode 100644 index 0000000..aee36b0 --- /dev/null +++ b/test/migrations/migration-8-test.js @@ -0,0 +1,77 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const chaiAsPromised = require('chai-as-promised') +chai.use(chaiAsPromised) +const expect = chai.expect + +const path = require('path') +const migration = require('../../migrations/migration-8') +const Key = require('interface-datastore').Key +const Datastore = require('datastore-fs') +const core = require('datastore-core') +const ShardingStore = core.ShardingDatastore + +const blocksFixtures = [ + ['AFKREIBFG77IKIKDMBDUFDCSPK7H5TE5LNPMCSXYLPML27WSTT5YA5IUNU', + 'CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I'] +] + +async function bootstrapBlocks (dir, encoded) { + const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: true }) + const shard = new core.shard.NextToLast(2) + + await baseStore.open() + const store = await ShardingStore.createOrOpen(baseStore, shard) + + let name + for (const blocksNames of blocksFixtures) { + name = encoded ? blocksNames[1] : blocksNames[0] + await store.put(new Key(name), '') + } + + await store.close() +} + +async function validateBlocks (dir, shouldBeEncoded) { + const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: false }) + const shard = new core.shard.NextToLast(2) + + await baseStore.open() + const store = await ShardingStore.createOrOpen(baseStore, shard) + + let newName, oldName + for (const blockNames of blocksFixtures) { + newName = shouldBeEncoded ? blockNames[1] : blockNames[0] + oldName = shouldBeEncoded ? blockNames[0] : blockNames[1] + expect(await store.has(new Key(`/${oldName}`))).to.be.false(`${oldName} was not migrated to ${newName}`) + expect(await store.has(new Key(`/${newName}`))).to.be.true(`${newName} was not removed`) + } + + await store.close() +} + +module.exports = (setup, cleanup) => { + describe('migration 8', () => { + let dir + + beforeEach(async () => { + dir = await setup() + }) + afterEach(() => cleanup(dir)) + + it('should migrate blocks forward', async () => { + await bootstrapBlocks(dir, false) + await migration.migrate(dir) + await validateBlocks(dir, true) + }) + + it('should migrate blocks backward', async () => { + await bootstrapBlocks(dir, true) + await migration.revert(dir) + await validateBlocks(dir, false) + }) + }) +} diff --git a/test/node.js b/test/node.js index 032889c..584275d 100644 --- a/test/node.js +++ b/test/node.js @@ -43,6 +43,10 @@ describe('Node specific tests', () => { require('./version-test')(createRepo, repoCleanup) }) + describe('migrations tests', () => { + require('./migrations/migration-8-test')(createRepo, repoCleanup) + }) + describe('init tests', () => { require('./init-test')(createRepo, repoCleanup) })