From 1c42f42541fea316833555ea6cc8ade86aea9c48 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Sun, 14 Aug 2022 09:18:12 +0100 Subject: [PATCH] deps: update interface-datastore and friends (#409) --- .github/workflows/js-test-and-release.yml | 2 +- .gitignore | 1 + packages/ipfs-repo-migrations/package.json | 16 +- .../test/fixtures/repo.js | 25 ++- .../test/migrations/migration-10-test.js | 196 +++++++++++------- packages/ipfs-repo-migrations/test/node.js | 68 ++++-- packages/ipfs-repo-migrations/test/types.ts | 17 +- packages/ipfs-repo/package.json | 10 +- 8 files changed, 224 insertions(+), 111 deletions(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 5a88d9ca..70e7ac2b 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -25,7 +25,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest] node: [16] fail-fast: true steps: diff --git a/.gitignore b/.gitignore index b9eda9fb..4208fdba 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ logs *.log coverage +.coverage # Runtime data pids diff --git a/packages/ipfs-repo-migrations/package.json b/packages/ipfs-repo-migrations/package.json index 65062bcc..0dfa5569 100644 --- a/packages/ipfs-repo-migrations/package.json +++ b/packages/ipfs-repo-migrations/package.json @@ -173,11 +173,11 @@ "dependencies": { "@ipld/dag-pb": "^2.1.0", "cborg": "^1.3.4", - "datastore-core": "^7.0.0", + "datastore-core": "^8.0.1", "debug": "^4.1.0", "fnv1a": "^1.0.1", - "interface-blockstore": "^2.0.2", - "interface-datastore": "^6.0.2", + "interface-blockstore": "^3.0.0", + "interface-datastore": "^7.0.0", "it-length": "^1.0.1", "multiaddr": "^10.0.1", "multiformats": "^9.0.4", @@ -192,11 +192,11 @@ "@types/varint": "^6.0.0", "aegir": "^37.5.0", "aws-sdk": "^2.884.0", - "blockstore-core": "^1.0.2", - "blockstore-datastore-adapter": "^2.0.1", - "datastore-fs": "^7.0.0", - "datastore-level": "^8.0.0", - "datastore-s3": "^9.0.0", + "blockstore-core": "^2.0.0", + "blockstore-datastore-adapter": "^3.0.1", + "datastore-fs": "^8.0.0", + "datastore-level": "^9.0.0", + "datastore-s3": "^10.0.0", "protobufjs-cli": "^1.0.0", "just-safe-set": "^4.1.1", "level-5": "npm:level@^5.0.0", diff --git a/packages/ipfs-repo-migrations/test/fixtures/repo.js b/packages/ipfs-repo-migrations/test/fixtures/repo.js index 7e84c05b..9241af00 100644 --- a/packages/ipfs-repo-migrations/test/fixtures/repo.js +++ b/packages/ipfs-repo-migrations/test/fixtures/repo.js @@ -1,26 +1,35 @@ - import loadFixture from 'aegir/fixtures' import { CONFIG_KEY, VERSION_KEY } from '../../src/utils.js' +import fs from 'fs/promises' /** * @typedef {import('../../src/types').Backends} Backends + * @typedef {import('../types').SetupOptions} SetupOptions + * @typedef {import('../types').CreateBackends} CreateBackends */ /** - * - * @param {(dir: string) => import('../../src/types').Backends} createBackends - * @param {*} prefix - * @returns + * @param {CreateBackends} createBackends + * @param {SetupOptions} [opts] */ -export async function createRepo (createBackends, prefix) { - const dir = `${prefix ? `${prefix}/` : ''}test-repo-for-${Date.now()}` - const backends = createBackends(dir) +export async function createRepo (createBackends, opts = {}) { + let dir = opts.dir + const prefix = opts.prefix ?? '' + + if (dir == null) { + dir = [prefix, `test-repo-for-${Date.now()}`].filter(Boolean).join('/') + await fs.mkdir(dir, { + recursive: true + }) + } + const backends = createBackends(dir, opts.createBackends) await backends.root.open() await backends.root.close() return { dir, + prefix, backends } } diff --git a/packages/ipfs-repo-migrations/test/migrations/migration-10-test.js b/packages/ipfs-repo-migrations/test/migrations/migration-10-test.js index 03cc4bec..a7a5c847 100644 --- a/packages/ipfs-repo-migrations/test/migrations/migration-10-test.js +++ b/packages/ipfs-repo-migrations/test/migrations/migration-10-test.js @@ -53,64 +53,19 @@ async function validate (store) { if (store instanceof BaseBlockstore) { const key = CID.parse(`b${name.toLowerCase()}`) - expect(await store.has(key)).to.be.true(`Could not read key ${name}`) - expect(equals(await store.get(key), keys[name])).to.be.true(`Could not read value for key ${keys[name]}`) + await expect(store.has(key)).to.eventually.be.true(`Could not read key ${name} from blockstore`) + expect(equals(await store.get(key), keys[name])).to.be.true(`Could not read value for key ${keys[name]} from blockstore`) } else { const key = new Key(`/${name}`) - await expect(store.has(key)).to.eventually.be.true(`Could not read key ${name}`) - expect(equals(await store.get(key), keys[name])).to.be.true(`Could not read value for key ${keys[name]}`) + await expect(store.has(key)).to.eventually.be.true(`Could not read key ${name} from datastore`) + expect(equals(await store.get(key), keys[name])).to.be.true(`Could not read value for key ${keys[name]} from datastore`) } } await store.close() } -/** - * @param {Backends} backends - * @param {*} LevelImpl - * @returns {Backends} - */ -function withLevels (backends, LevelImpl) { - const output = { - ...backends - } - - Object.entries(backends) - .forEach(([key, value]) => { - // @ts-ignore it's ok - output[key] = withLevel(value, LevelImpl) - }) - - return output -} - -/** - * @param {Datastore} store - * @param {*} LevelImpl - */ -function withLevel (store, LevelImpl) { - let parent = { - child: store - } - - while (parent.child) { - if (parent.child.constructor.name === 'LevelDatastore') { - // @ts-ignore undocumented properties - parent.child.database = LevelImpl - // @ts-ignore undocumented properties - delete parent.child.db - - return store - } - - // @ts-ignore undocumented properties - parent = parent.child - } - - return store -} - /** * @param {import('../types').SetupFunction} setup * @param {import('../types').CleanupFunction} cleanup @@ -118,63 +73,164 @@ function withLevel (store, LevelImpl) { export function test (setup, cleanup) { describe('migration 10', function () { this.timeout(1024 * 1000) - /** @type {string} */ - let dir - /** @type {import('../../src/types').Backends} */ - let backends - - beforeEach(async () => { - ({ dir, backends } = await setup()) - }) - - afterEach(async () => { - await cleanup(dir) - }) describe('forwards', () => { + /** @type {string} */ + let dir + /** @type {import('../../src/types').Backends} */ + let backends + /** @type {string} */ + let prefix + beforeEach(async () => { + ({ dir, prefix, backends } = await setup({ + createBackends: { + createLevel: (path) => new Level5(path, { + valueEncoding: 'binary' + }) + } + })) + for (const backend of Object.values(backends)) { - await bootstrap(withLevel(backend, Level5)) + await bootstrap(backend) + } + }) + + afterEach(async () => { + if (dir != null) { + await cleanup(dir) } }) it('should migrate keys and values forward', async () => { - await migration.migrate(withLevels(backends, Level6), () => {}) + ({ backends } = await setup({ + dir, + prefix, + createBackends: { + createLevel: (path) => new Level6(path, { + valueEncoding: 'binary' + }) + } + })) + + await migration.migrate(backends, () => {}) for (const backend of Object.values(backends)) { - await validate(withLevel(backend, Level6)) + await validate(backend) } }) }) describe('backwards using level@6.x.x', () => { + /** @type {string} */ + let dir + /** @type {import('../../src/types').Backends} */ + let backends + /** @type {string} */ + let prefix + beforeEach(async () => { + ({ dir, prefix, backends } = await setup({ + createBackends: { + createLevel: (path) => new Level6(path, { + valueEncoding: 'binary' + }) + } + })) + for (const backend of Object.values(backends)) { - await bootstrap(withLevel(backend, Level6)) + await bootstrap(backend) + } + }) + + afterEach(async () => { + if (dir != null) { + await cleanup(dir) } }) it('should migrate keys and values backward', async () => { - await migration.revert(withLevels(backends, Level6), () => {}) + ({ backends } = await setup({ + dir, + prefix, + createBackends: { + createLevel: (path) => new Level6(path, { + valueEncoding: 'binary' + }) + } + })) + + await migration.revert(backends, () => {}) + + ;({ backends } = await setup({ + dir, + prefix, + createBackends: { + createLevel: (path) => new Level5(path, { + valueEncoding: 'binary' + }) + } + })) for (const backend of Object.values(backends)) { - await validate(withLevel(backend, Level5)) + await validate(backend) } }) }) describe('backwards using level@5.x.x', () => { + /** @type {string} */ + let dir + /** @type {import('../../src/types').Backends} */ + let backends + /** @type {string} */ + let prefix + beforeEach(async () => { + ({ dir, prefix, backends } = await setup({ + createBackends: { + createLevel: (path) => new Level6(path, { + valueEncoding: 'binary' + }) + } + })) + for (const backend of Object.values(backends)) { - await bootstrap(withLevel(backend, Level6)) + await bootstrap(backend) + } + }) + + afterEach(async () => { + if (dir != null) { + await cleanup(dir) } }) it('should migrate keys and values backward', async () => { - await migration.revert(withLevels(backends, Level5), () => {}) + ({ backends } = await setup({ + dir, + prefix, + createBackends: { + createLevel: (path) => new Level5(path, { + valueEncoding: 'binary' + }) + } + })) + + await migration.revert(backends, () => {}) + + ;({ backends } = await setup({ + dir, + prefix, + createBackends: { + createLevel: (path) => new Level5(path, { + valueEncoding: 'binary' + }) + } + })) for (const backend of Object.values(backends)) { - await validate(withLevel(backend, Level5)) + await validate(backend) } }) }) diff --git a/packages/ipfs-repo-migrations/test/node.js b/packages/ipfs-repo-migrations/test/node.js index fe3001fc..fe54bec0 100644 --- a/packages/ipfs-repo-migrations/test/node.js +++ b/packages/ipfs-repo-migrations/test/node.js @@ -16,21 +16,41 @@ import { test as migrationTests } from './migrations/index.js' import { test as initTests } from './init-test.js' import { test as integrationTests } from './integration-test.js' +/** + * @typedef {import('./types').CreateBackendsOptions} CreateBackendsOptions + * @typedef {import('./types').CreateBackends} CreateBackends + */ + +/** @type {S3 | null} */ +let s3Instance + /** * @param {string} dir */ async function cleanup (dir) { - await rimraf.sync(dir) + /** @type {Promise} */ + const p = new Promise((resolve, reject) => { + rimraf(dir, (err) => { + if (err) { + reject(err) + return + } + + resolve() + }) + }) + await p } const CONFIGURATIONS = [{ name: 'with sharding', cleanup, /** - * @param {string} prefix - * @returns {import('../src/types').Backends} + * @type {CreateBackends} */ - createBackends: (prefix) => { + createBackends: (prefix, opts = {}) => { + const createLevel = opts.createLevel ?? ((path) => path) + return { root: new FsDatastore(prefix), blocks: new BlockstoreDatastoreAdapter( @@ -40,9 +60,9 @@ const CONFIGURATIONS = [{ }), new NextToLast(2)) ), - datastore: new LevelDatastore(`${prefix}/datastore`), + datastore: new LevelDatastore(createLevel(`${prefix}/datastore`)), keys: new FsDatastore(`${prefix}/keys`), - pins: new LevelDatastore(`${prefix}/pins`) + pins: new LevelDatastore(createLevel(`${prefix}/pins`)) } } }, { @@ -50,9 +70,12 @@ const CONFIGURATIONS = [{ cleanup, /** * @param {string} prefix + * @param {CreateBackendsOptions} [opts] * @returns {import('../src/types').Backends} */ - createBackends: (prefix) => { + createBackends: (prefix, opts = {}) => { + const createLevel = opts.createLevel ?? ((path) => path) + return { root: new FsDatastore(prefix), blocks: new BlockstoreDatastoreAdapter( @@ -60,25 +83,30 @@ const CONFIGURATIONS = [{ extension: '.data' }) ), - datastore: new LevelDatastore(`${prefix}/datastore`), + datastore: new LevelDatastore(createLevel(`${prefix}/datastore`)), keys: new FsDatastore(`${prefix}/keys`), - pins: new LevelDatastore(`${prefix}/pins`) + pins: new LevelDatastore(createLevel(`${prefix}/pins`)) } } }, { name: 'with s3', - cleanup: async () => {}, + cleanup: async () => { + s3Instance = null + }, /** * @param {string} prefix + * @param {CreateBackendsOptions} [opts] * @returns {import('../src/types').Backends} */ - createBackends: (prefix) => { - const s3Instance = new S3({ - params: { - Bucket: 'test' - } - }) - mockS3(s3Instance) + createBackends: (prefix, opts = {}) => { + if (s3Instance == null) { + s3Instance = new S3({ + params: { + Bucket: 'test' + } + }) + mockS3(s3Instance) + } return { root: new S3Datastore(prefix, { @@ -111,7 +139,11 @@ const CONFIGURATIONS = [{ }] CONFIGURATIONS.forEach(({ name, createBackends, cleanup }) => { - const setup = () => createRepo(createBackends, os.tmpdir()) + /** @type {import('./types').SetupFunction} */ + const setup = (opts = {}) => createRepo(createBackends, { + ...opts, + prefix: opts.prefix ?? os.tmpdir() + }) describe(name, () => { describe('version tests', () => { diff --git a/packages/ipfs-repo-migrations/test/types.ts b/packages/ipfs-repo-migrations/test/types.ts index 2c9e993e..1756bb3d 100644 --- a/packages/ipfs-repo-migrations/test/types.ts +++ b/packages/ipfs-repo-migrations/test/types.ts @@ -1,4 +1,19 @@ import type { Backends } from '../src/types' +import type { Level } from 'level' -export interface SetupFunction { (prefix?: string): Promise<{ dir: string, backends: Backends}> } +export interface SetupOptions { + dir?: string + prefix?: string + createBackends?: CreateBackendsOptions +} + +export interface CreateBackendsOptions { + createLevel?: (path: string) => string | Level +} + +export interface SetupFunction { (opts?: SetupOptions): Promise<{ dir: string, prefix: string, backends: Backends}> } export interface CleanupFunction { (dir: string): Promise } + +export interface CreateBackends { + (prefix: string, opts?: CreateBackendsOptions): Backends +} diff --git a/packages/ipfs-repo/package.json b/packages/ipfs-repo/package.json index 40ac4d2d..ff6b6ea0 100644 --- a/packages/ipfs-repo/package.json +++ b/packages/ipfs-repo/package.json @@ -183,11 +183,11 @@ "@ipld/dag-pb": "^2.1.0", "bytes": "^3.1.0", "cborg": "^1.3.4", - "datastore-core": "^7.0.0", + "datastore-core": "^8.0.1", "debug": "^4.1.0", "err-code": "^3.0.1", - "interface-blockstore": "^2.0.2", - "interface-datastore": "^6.0.2", + "interface-blockstore": "^3.0.0", + "interface-datastore": "^7.0.0", "ipfs-repo-migrations": "^13.0.0", "it-drain": "^1.0.1", "it-filter": "^1.0.2", @@ -214,8 +214,8 @@ "@types/proper-lockfile": "^4.1.1", "@types/rimraf": "^3.0.2", "aegir": "^37.5.0", - "blockstore-core": "^1.0.2", - "blockstore-datastore-adapter": "^2.0.1", + "blockstore-core": "^2.0.0", + "blockstore-datastore-adapter": "^3.0.1", "ipfs-utils": "^9.0.7", "it-all": "^1.0.2", "just-range": "^4.1.1",