diff --git a/benches/DB1KiB.ts b/benches/DB1KiB.ts index 98f916b4..ee328a1f 100644 --- a/benches/DB1KiB.ts +++ b/benches/DB1KiB.ts @@ -20,20 +20,20 @@ async function main() { const summary = await b.suite( 'DB1KiB', b.add('get 1 KiB of data', async () => { - await db.put([], '1kib', data1KiB, true); + await db.put('1kib', data1KiB, true); return async () => { - await db.get([], '1kib', true); + await db.get('1kib', true); }; }), b.add('put 1 KiB of data', async () => { - await db.put([], '1kib', data1KiB, true); + await db.put('1kib', data1KiB, true); }), b.add('put zero data', async () => { - await db.put([], '0', data0, true); + await db.put('0', data0, true); }), b.add('put zero data then del', async () => { - await db.put([], '0', data0, true); - await db.del([], '0'); + await db.put('0', data0, true); + await db.del('0'); }), b.cycle(), b.complete(), diff --git a/benches/DB1MiB.ts b/benches/DB1MiB.ts index 580d5979..40afd435 100644 --- a/benches/DB1MiB.ts +++ b/benches/DB1MiB.ts @@ -20,20 +20,20 @@ async function main() { const summary = await b.suite( 'DB1MiB', b.add('get 1 MiB of data', async () => { - await db.put([], '1kib', data1MiB, true); + await db.put('1kib', data1MiB, true); return async () => { - await db.get([], '1kib', true); + await db.get('1kib', true); }; }), b.add('put 1 MiB of data', async () => { - await db.put([], '1kib', data1MiB, true); + await db.put('1kib', data1MiB, true); }), b.add('put zero data', async () => { - await db.put([], '0', data0, true); + await db.put('0', data0, true); }), b.add('put zero data then del', async () => { - await db.put([], '0', data0, true); - await db.del([], '0'); + await db.put('0', data0, true); + await db.del('0'); }), b.cycle(), b.complete(), diff --git a/benches/DBLevel.ts b/benches/DBLevel.ts deleted file mode 100644 index 2d2ac000..00000000 --- a/benches/DBLevel.ts +++ /dev/null @@ -1,86 +0,0 @@ -import os from 'os'; -import path from 'path'; -import fs from 'fs'; -import b from 'benny'; -import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; -import DB from '@/DB'; -import packageJson from '../package.json'; - -const logger = new Logger('DBLevel Bench', LogLevel.WARN, [ - new StreamHandler(), -]); - -async function main() { - const dataDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), 'encryptedfs-benches-'), - ); - const dbPath = `${dataDir}/db`; - const db = await DB.createDB({ dbPath, logger }); - const summary = await b.suite( - 'DBLevel', - b.add('create 1 sublevels', async () => { - let level; - for (let i = 0; i < 1; i++) { - level = await db.level(`level${i}`, level); - } - }), - b.add('create 2 sublevels', async () => { - let level; - for (let i = 0; i < 2; i++) { - level = await db.level(`level${i}`, level); - } - }), - b.add('create 3 sublevels', async () => { - let level; - for (let i = 0; i < 3; i++) { - level = await db.level(`level${i}`, level); - } - }), - b.add('create 4 sublevels', async () => { - let level; - for (let i = 0; i < 4; i++) { - level = await db.level(`level${i}`, level); - } - }), - b.add('get via sublevel', async () => { - await db.put(['level0'], 'hello', 'world'); - return async () => { - const level = await db.level('level0'); - await level.get('hello'); - }; - }), - b.add('get via key path concatenation', async () => { - await db.put(['level0'], 'hello', 'world'); - return async () => { - await db.get(['level0'], 'hello'); - }; - }), - b.cycle(), - b.complete(), - b.save({ - file: 'DBLevel', - folder: 'benches/results', - version: packageJson.version, - details: true, - }), - b.save({ - file: 'DBLevel', - folder: 'benches/results', - format: 'chart.html', - }), - ); - await db.stop(); - await fs.promises.rm(dataDir, { - force: true, - recursive: true, - }); - return summary; -} - -if (require.main === module) { - (async () => { - await main(); - })(); -} - -export default main; diff --git a/package-lock.json b/package-lock.json index 88b5f477..c0bc7889 100644 --- a/package-lock.json +++ b/package-lock.json @@ -924,8 +924,7 @@ "@types/abstract-leveldown": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@types/abstract-leveldown/-/abstract-leveldown-7.2.0.tgz", - "integrity": "sha512-q5veSX6zjUy/DlDhR4Y4cU0k2Ar+DT2LUraP00T19WLmTO6Se1djepCCaqU6nQrwcJ5Hyo/CWqxTzrrFg8eqbQ==", - "dev": true + "integrity": "sha512-q5veSX6zjUy/DlDhR4Y4cU0k2Ar+DT2LUraP00T19WLmTO6Se1djepCCaqU6nQrwcJ5Hyo/CWqxTzrrFg8eqbQ==" }, "@types/babel__core": { "version": "7.1.15", @@ -1100,17 +1099,6 @@ "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==", "dev": true }, - "@types/subleveldown": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@types/subleveldown/-/subleveldown-4.1.1.tgz", - "integrity": "sha512-cnup7ibJZjwHIwDATB9hdtvfM3j6hkRquBz/Ld/PJC50r1u7V+bXhm+tof665A2D56shExosPqybbET770GBEg==", - "dev": true, - "requires": { - "@types/abstract-leveldown": "*", - "@types/level-codec": "*", - "@types/levelup": "*" - } - }, "@types/yargs": { "version": "15.0.14", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.14.tgz", @@ -1246,19 +1234,6 @@ "integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q==", "dev": true }, - "abstract-leveldown": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-7.2.0.tgz", - "integrity": "sha512-DnhQwcFEaYsvYDnACLZhMmCWd3rkOeEvglpa4q5i/5Jlm3UIsWaxVzuXvDLFCSCWRO3yy2/+V/G7FusFgejnfQ==", - "requires": { - "buffer": "^6.0.3", - "catering": "^2.0.0", - "is-buffer": "^2.0.5", - "level-concat-iterator": "^3.0.0", - "level-supports": "^2.0.1", - "queue-microtask": "^1.2.3" - } - }, "acorn": { "version": "7.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", @@ -2025,15 +2000,6 @@ "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==", "dev": true }, - "deferred-leveldown": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-7.0.0.tgz", - "integrity": "sha512-QKN8NtuS3BC6m0B8vAnBls44tX1WXAFATUsJlruyAYbZpysWV3siH6o/i3g9DCHauzodksO60bdj5NazNbjCmg==", - "requires": { - "abstract-leveldown": "^7.2.0", - "inherits": "^2.0.3" - } - }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -2084,11 +2050,6 @@ } } }, - "defined": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/defined/-/defined-0.0.0.tgz", - "integrity": "sha1-817qfXBekzuvE7LwOz+D2SFAOz4=" - }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -2166,17 +2127,6 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, - "encoding-down": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-7.1.0.tgz", - "integrity": "sha512-ky47X5jP84ryk5EQmvedQzELwVJPjCgXDQZGeb9F6r4PdChByCGHTBrVcF3h8ynKVJ1wVbkxTsDC8zBROPypgQ==", - "requires": { - "abstract-leveldown": "^7.2.0", - "inherits": "^2.0.3", - "level-codec": "^10.0.0", - "level-errors": "^3.0.0" - } - }, "end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -4415,33 +4365,11 @@ "leveldown": "^6.1.0" } }, - "level-codec": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-10.0.0.tgz", - "integrity": "sha512-QW3VteVNAp6c/LuV6nDjg7XDXx9XHK4abmQarxZmlRSDyXYk20UdaJTSX6yzVvQ4i0JyWSB7jert0DsyD/kk6g==", - "requires": { - "buffer": "^6.0.3" - } - }, "level-concat-iterator": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.0.0.tgz", "integrity": "sha512-UHGiIdj+uiFQorOrURRvJF3Ei0uHc89ciM/aRi0qsWDV2f0HXypeXUPhJKL6DsONgSR76Pc0AI4sKYEYYRn2Dg==" }, - "level-errors": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-3.0.1.tgz", - "integrity": "sha512-tqTL2DxzPDzpwl0iV5+rBCv65HWbHp6eutluHNcVIftKZlQN//b6GEnZDM2CvGZvzGYMwyPtYppYnydBQd2SMQ==" - }, - "level-iterator-stream": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-5.0.0.tgz", - "integrity": "sha512-wnb1+o+CVFUDdiSMR/ZymE2prPs3cjVLlXuDeSq9Zb8o032XrabGEXcTCsBxprAtseO3qvFeGzh6406z9sOTRA==", - "requires": { - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, "level-js": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/level-js/-/level-js-6.1.0.tgz", @@ -4474,14 +4402,6 @@ } } }, - "level-option-wrap": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/level-option-wrap/-/level-option-wrap-1.1.0.tgz", - "integrity": "sha1-rSDmjZ88IsiJdTHMaqevWWse0Sk=", - "requires": { - "defined": "~0.0.0" - } - }, "level-packager": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-6.0.1.tgz", @@ -4566,11 +4486,6 @@ } } }, - "level-supports": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.1.0.tgz", - "integrity": "sha512-E486g1NCjW5cF78KGPrMDRBYzPuueMZ6VBXHT6gC7A8UYWGiM14fGgp+s/L1oFfDWSPV/+SFkYCmZ0SiESkRKA==" - }, "leveldown": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-6.1.0.tgz", @@ -4601,19 +4516,6 @@ } } }, - "levelup": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/levelup/-/levelup-5.1.1.tgz", - "integrity": "sha512-0mFCcHcEebOwsQuk00WJwjLI6oCjbBuEYdh/RaRqhjnyVlzqf41T1NnDtCedumZ56qyIh8euLFDqV1KfzTAVhg==", - "requires": { - "catering": "^2.0.0", - "deferred-leveldown": "^7.0.0", - "level-errors": "^3.0.1", - "level-iterator-stream": "^5.0.0", - "level-supports": "^2.0.1", - "queue-microtask": "^1.2.3" - } - }, "leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -5355,11 +5257,6 @@ "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" }, - "reachdown": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/reachdown/-/reachdown-1.1.0.tgz", - "integrity": "sha512-6LsdRe4cZyOjw4NnvbhUd/rGG7WQ9HMopPr+kyL018Uci4kijtxcGR5kVb5Ln13k4PEE+fEFQbjfOvNw7cnXmA==" - }, "react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", @@ -6189,24 +6086,6 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, - "sublevel-prefixer": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/sublevel-prefixer/-/sublevel-prefixer-1.0.0.tgz", - "integrity": "sha1-TuRZ72Y6yFvyj8ZJ17eWX9ppEHM=" - }, - "subleveldown": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/subleveldown/-/subleveldown-6.0.1.tgz", - "integrity": "sha512-Cnf+cn2wISXU2xflY1SFIqfX4hG2d6lFk2P5F8RDQLmiqN9Ir4ExNfUFH6xnmizMseM/t+nMsDUKjN9Kw6ShFA==", - "requires": { - "abstract-leveldown": "^7.2.0", - "encoding-down": "^7.1.0", - "inherits": "^2.0.3", - "level-option-wrap": "^1.1.0", - "levelup": "^5.1.1", - "reachdown": "^1.1.0" - } - }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", diff --git a/package.json b/package.json index b76cb583..036c804c 100644 --- a/package.json +++ b/package.json @@ -24,22 +24,16 @@ "@matrixai/logger": "^2.1.0", "@matrixai/resources": "^1.0.0", "@matrixai/workers": "^1.2.5", - "abstract-leveldown": "^7.2.0", + "@types/abstract-leveldown": "^7.2.0", "level": "7.0.1", - "levelup": "^5.1.1", - "sublevel-prefixer": "^1.0.0", - "subleveldown": "^6.0.1", "threads": "^1.6.5", "ts-custom-error": "^3.2.0" }, "devDependencies": { - "@types/abstract-leveldown": "^7.2.0", "@types/jest": "^26.0.20", "@types/level": "^6.0.0", - "@types/levelup": "^5.1.0", "@types/node": "^14.14.35", "@types/node-forge": "^0.10.4", - "@types/subleveldown": "^4.1.1", "@typescript-eslint/eslint-plugin": "^5.4.0", "@typescript-eslint/parser": "^5.4.0", "benny": "^3.6.15", diff --git a/src/DB.ts b/src/DB.ts index 8dd41a93..738d9fd2 100644 --- a/src/DB.ts +++ b/src/DB.ts @@ -5,17 +5,15 @@ import type { import type { LevelDB } from 'level'; import type { ResourceAcquire } from '@matrixai/resources'; import type { - POJO, + KeyPath, + LevelPath, FileSystem, Crypto, DBWorkerManagerInterface, - DBDomain, - DBLevel, DBIterator, DBOps, } from './types'; import level from 'level'; -import subleveldown from 'subleveldown'; import { Transfer } from 'threads'; import Logger from '@matrixai/logger'; import { @@ -70,8 +68,6 @@ class DB { protected logger: Logger; protected workerManager?: DBWorkerManagerInterface; protected _db: LevelDB; - protected _dataDb: DBLevel; - protected _transactionsDb: DBLevel; protected transactionCounter: number = 0; constructor({ @@ -94,18 +90,10 @@ class DB { this.fs = fs; } - get db(): LevelDB { + get db(): Readonly> { return this._db; } - get dataDb(): DBLevel { - return this._dataDb; - } - - get transactionsDb(): DBLevel { - return this._transactionsDb; - } - public async start({ fresh = false, }: { @@ -124,16 +112,14 @@ class DB { } } const db = await this.setupDb(this.dbPath); - const { dataDb, transactionsDb } = await this.setupRootLevels(db); + await this.setupRootLevels(db); this._db = db; - this._dataDb = dataDb; - this._transactionsDb = transactionsDb; this.logger.info(`Started ${this.constructor.name}`); } public async stop(): Promise { this.logger.info(`Stopping ${this.constructor.name}`); - await this.db.close(); + await this._db.close(); this.logger.info(`Stopped ${this.constructor.name}`); } @@ -167,14 +153,9 @@ class DB { public transaction(): ResourceAcquire { return async () => { const transactionId = this.transactionCounter++; - const transactionDb = await this._level( - transactionId.toString(), - this.transactionsDb, - ); const tran = await DBTransaction.createTransaction({ db: this, transactionId, - transactionDb, logger: this.logger, }); return [ @@ -200,93 +181,47 @@ class DB { }; } - @ready(new errors.ErrorDBNotRunning()) - public async level( - domain: string, - dbLevel: DBLevel = this._dataDb, - ): ReturnType { - return this._level(domain, dbLevel); - } - - public iterator( - options: AbstractIteratorOptions & { key: false; value: false }, - dbLevel: DBLevel, - ): DBIterator; - public iterator( - options: AbstractIteratorOptions & { key: false }, - dbLevel: DBLevel, - ): DBIterator; - public iterator( - options: AbstractIteratorOptions & { value: false }, - dbLevel: DBLevel, - ): DBIterator; - public iterator( - options?: AbstractIteratorOptions, - dbLevel?: DBLevel, - ): DBIterator; - @ready(new errors.ErrorDBNotRunning()) - public iterator( - options?: AbstractIteratorOptions, - dbLevel: DBLevel = this._dataDb, - ): DBIterator { - const iterator = dbLevel.iterator(options); - const next = iterator.next.bind(iterator); - // @ts-ignore AbstractIterator type is outdated - iterator.next = async (cb) => { - const kv = await next(cb); - if (kv != null) { - kv[1] = await this.deserializeDecrypt(kv[1], true); - } - return kv; - }; - return iterator as unknown as DBIterator; - } - - @ready(new errors.ErrorDBNotRunning()) - public async clear(dbLevel: DBLevel = this._dataDb): Promise { - await dbLevel.clear(); - } - - @ready(new errors.ErrorDBNotRunning()) - public async count(dbLevel: DBLevel = this._dataDb): Promise { - let count = 0; - for await (const _ of dbLevel.createKeyStream()) { - count++; - } - return count; - } - - @ready(new errors.ErrorDBNotRunning()) - public async dump(dbLevel: DBLevel = this._dataDb): Promise { - const records = {}; - for await (const o of dbLevel.createReadStream()) { - const key = (o as any).key.toString(); - const data = (o as any).value as Buffer; - const value = await this.deserializeDecrypt(data, false); - records[key] = value; - } - return records; - } - + /** + * Gets a value from the DB + * Use raw to return the raw decrypted buffer + */ public async get( - domain: DBDomain, - key: string | Buffer, + keyPath: KeyPath | string | Buffer, raw?: false, ): Promise; public async get( - domain: DBDomain, - key: string | Buffer, + keyPath: KeyPath | string | Buffer, raw: true, ): Promise; @ready(new errors.ErrorDBNotRunning()) public async get( - domain: DBDomain, - key: string | Buffer, + keyPath: KeyPath | string | Buffer, + raw: boolean = false, + ): Promise { + if (!Array.isArray(keyPath)) { + keyPath = [keyPath] as KeyPath; + } + keyPath = ['data', ...keyPath]; + if (utils.checkSepKeyPath(keyPath)) { + throw new errors.ErrorDBLevelSep(); + } + return this._get(keyPath, raw as any); + } + + /** + * Get from root level + * @internal + */ + public async _get(keyPath: KeyPath, raw?: false): Promise; + public async _get(keyPath: KeyPath, raw: true): Promise; + public async _get( + keyPath: KeyPath, raw: boolean = false, ): Promise { let data; try { - data = await this._dataDb.get(utils.domainPath(domain, key)); + const key = utils.keyPathToKey(keyPath); + data = await this._db.get(key); } catch (e) { if (e.notFound) { return undefined; @@ -296,49 +231,99 @@ class DB { return this.deserializeDecrypt(data, raw as any); } + /** + * Put a key and value into the DB + * Use raw to put raw encrypted buffer + */ public async put( - domain: DBDomain, - key: string | Buffer, + keyPath: KeyPath | string | Buffer, value: any, raw?: false, ): Promise; public async put( - domain: DBDomain, - key: string | Buffer, + keyPath: KeyPath | string | Buffer, value: Buffer, raw: true, ): Promise; @ready(new errors.ErrorDBNotRunning()) public async put( - domain: DBDomain, - key: string | Buffer, + keyPath: KeyPath | string | Buffer, + value: any, + raw: boolean = false, + ): Promise { + if (!Array.isArray(keyPath)) { + keyPath = [keyPath] as KeyPath; + } + keyPath = ['data', ...keyPath]; + if (utils.checkSepKeyPath(keyPath)) { + throw new errors.ErrorDBLevelSep(); + } + return this._put(keyPath, value, raw as any); + } + + /** + * Put from root level + * @internal + */ + public async _put(keyPath: KeyPath, value: any, raw?: false): Promise; + public async _put(keyPath: KeyPath, value: Buffer, raw: true): Promise; + public async _put( + keyPath: KeyPath, value: any, raw: boolean = false, ): Promise { const data = await this.serializeEncrypt(value, raw as any); - return this._dataDb.put(utils.domainPath(domain, key), data); + return this._db.put(utils.keyPathToKey(keyPath), data); } + /** + * Deletes a key from the DB + */ @ready(new errors.ErrorDBNotRunning()) - public async del(domain: DBDomain, key: string | Buffer): Promise { - return this._dataDb.del(utils.domainPath(domain, key)); + public async del(keyPath: KeyPath | string | Buffer): Promise { + if (!Array.isArray(keyPath)) { + keyPath = [keyPath] as KeyPath; + } + keyPath = ['data', ...keyPath]; + if (utils.checkSepKeyPath(keyPath)) { + throw new errors.ErrorDBLevelSep(); + } + return this._del(keyPath); + } + + /** + * Delete from root level + * @internal + */ + public async _del(keyPath: KeyPath): Promise { + return this._db.del(utils.keyPathToKey(keyPath)); } + /** + * Batches operations together atomically + */ @ready(new errors.ErrorDBNotRunning()) public async batch(ops: Readonly): Promise { const opsP: Array | AbstractBatch> = []; for (const op of ops) { + if (!Array.isArray(op.keyPath)) { + op.keyPath = [op.keyPath] as KeyPath; + } + op.keyPath = ['data', ...op.keyPath]; + if (utils.checkSepKeyPath(op.keyPath)) { + throw new errors.ErrorDBLevelSep(); + } if (op.type === 'del') { opsP.push({ type: op.type, - key: utils.domainPath(op.domain, op.key), + key: utils.keyPathToKey(op.keyPath), }); } else { opsP.push( this.serializeEncrypt(op.value, (op.raw === true) as any).then( (data) => ({ type: op.type, - key: utils.domainPath(op.domain, op.key), + key: utils.keyPathToKey(op.keyPath as KeyPath), value: data, }), ), @@ -346,7 +331,233 @@ class DB { } } const opsB = await Promise.all(opsP); - return this._dataDb.batch(opsB); + return this._db.batch(opsB); + } + + /** + * Batch from root level + * @internal + */ + public async _batch(ops: Readonly): Promise { + const opsP: Array | AbstractBatch> = []; + for (const op of ops) { + if (!Array.isArray(op.keyPath)) { + op.keyPath = [op.keyPath] as KeyPath; + } + if (op.type === 'del') { + opsP.push({ + type: op.type, + key: utils.keyPathToKey(op.keyPath as KeyPath), + }); + } else { + opsP.push( + this.serializeEncrypt(op.value, (op.raw === true) as any).then( + (data) => ({ + type: op.type, + key: utils.keyPathToKey(op.keyPath as KeyPath), + value: data, + }), + ), + ); + } + } + const opsB = await Promise.all(opsP); + return this._db.batch(opsB); + } + + /** + * Public iterator that works from the data level + * If keys and values are both false, this iterator will not run at all + * You must have at least one of them being true or undefined + */ + public iterator( + options: AbstractIteratorOptions & { keys: false; values: false }, + levelPath?: LevelPath, + ): DBIterator; + public iterator( + options: AbstractIteratorOptions & { keys: false }, + levelPath?: LevelPath, + ): DBIterator; + public iterator( + options: AbstractIteratorOptions & { values: false }, + levelPath?: LevelPath, + ): DBIterator; + public iterator( + options?: AbstractIteratorOptions, + levelPath?: LevelPath, + ): DBIterator; + @ready(new errors.ErrorDBNotRunning()) + public iterator( + options?: AbstractIteratorOptions, + levelPath: LevelPath = [], + ): DBIterator { + levelPath = ['data', ...levelPath]; + if (utils.checkSepLevelPath(levelPath)) { + throw new errors.ErrorDBLevelSep(); + } + return this._iterator(this._db, options, levelPath); + } + + /** + * Iterator from root level + * @internal + */ + public _iterator( + db: LevelDB, + options: AbstractIteratorOptions & { keys: false; values: false }, + levelPath?: LevelPath, + ): DBIterator; + public _iterator( + db: LevelDB, + options: AbstractIteratorOptions & { keys: false }, + levelPath?: LevelPath, + ): DBIterator; + public _iterator( + db: LevelDB, + options: AbstractIteratorOptions & { values: false }, + levelPath?: LevelPath, + ): DBIterator; + public _iterator( + db: LevelDB, + options?: AbstractIteratorOptions, + levelPath?: LevelPath, + ): DBIterator; + public _iterator( + db: LevelDB, + options?: AbstractIteratorOptions, + levelPath: LevelPath = [], + ): DBIterator { + options = options ?? {}; + const levelKeyStart = utils.levelPathToKey(levelPath); + if (options.gt != null) { + options.gt = Buffer.concat([ + levelKeyStart, + typeof options.gt === 'string' ? Buffer.from(options.gt) : options.gt, + ]); + } + if (options.gte != null) { + options.gte = Buffer.concat([ + levelKeyStart, + typeof options.gte === 'string' + ? Buffer.from(options.gte) + : options.gte, + ]); + } + if (options.gt == null && options.gte == null) { + options.gt = levelKeyStart; + } + if (options?.lt != null) { + options.lt = Buffer.concat([ + levelKeyStart, + typeof options.lt === 'string' ? Buffer.from(options.lt) : options.lt, + ]); + } + if (options?.lte != null) { + options.lte = Buffer.concat([ + levelKeyStart, + typeof options.lte === 'string' + ? Buffer.from(options.lte) + : options.lte, + ]); + } + if (options.lt == null && options.lte == null) { + const levelKeyEnd = Buffer.from(levelKeyStart); + levelKeyEnd[levelKeyEnd.length - 1] += 1; + options.lt = levelKeyEnd; + } + const iterator = db.iterator(options); + const seek = iterator.seek.bind(iterator); + const next = iterator.next.bind(iterator); + // @ts-ignore AbstractIterator type is outdated + iterator.seek = (k: Buffer | string): void => { + seek(utils.keyPathToKey([...levelPath, k] as unknown as KeyPath)); + }; + // @ts-ignore AbstractIterator type is outdated + iterator.next = async () => { + const kv = await next(); + // If kv is undefined, we have reached the end of iteration + if (kv != null) { + // Handle keys: false + if (kv[0] != null) { + // Truncate level path so the returned key is relative to the level path + const keyPath = utils.parseKey(kv[0]).slice(levelPath.length); + kv[0] = utils.keyPathToKey(keyPath as unknown as KeyPath); + } + // Handle values: false + if (kv[1] != null) { + kv[1] = await this.deserializeDecrypt(kv[1], true); + } + } + return kv; + }; + return iterator as unknown as DBIterator; + } + + /** + * Clear all key values for a specific level + * This is not atomic, it will iterate over a snapshot of the DB + */ + @ready(new errors.ErrorDBNotRunning()) + public async clear(levelPath: LevelPath = []): Promise { + levelPath = ['data', ...levelPath]; + if (utils.checkSepLevelPath(levelPath)) { + throw new errors.ErrorDBLevelSep(); + } + await this._clear(this._db, levelPath); + } + + /** + * Clear from root level + * @internal + */ + public async _clear(db: LevelDB, levelPath: LevelPath = []): Promise { + for await (const [k] of this._iterator(db, { values: false }, levelPath)) { + await db.del(utils.keyPathToKey([...levelPath, k] as unknown as KeyPath)); + } + } + + @ready(new errors.ErrorDBNotRunning()) + public async count(levelPath: LevelPath = []): Promise { + let count = 0; + for await (const _ of this.iterator({ values: false }, levelPath)) { + count++; + } + return count; + } + + /** + * Dump from root level + * It is intended for diagnostics + */ + public async dump( + levelPath?: LevelPath, + raw?: false, + ): Promise>; + public async dump( + levelPath: LevelPath | undefined, + raw: true, + ): Promise>; + @ready(new errors.ErrorDBNotRunning()) + public async dump( + levelPath: LevelPath = [], + raw: boolean = false, + ): Promise> { + if (utils.checkSepLevelPath(levelPath)) { + throw new errors.ErrorDBLevelSep(); + } + const records: Array<[string | Buffer, any]> = []; + for await (const [k, v] of this._iterator(this._db, undefined, levelPath)) { + let key: string | Buffer, value: any; + if (raw) { + key = k; + value = v; + } else { + key = k.toString('utf-8'); + value = utils.deserialize(v); + } + records.push([key, value]); + } + return records; } public async serializeEncrypt(value: any, raw: false): Promise; @@ -460,51 +671,9 @@ class DB { return db; } - protected async setupRootLevels( - db: LevelDB, - ): Promise<{ - dataDb: DBLevel; - transactionsDb: DBLevel; - }> { - const dataDb = await this._level('data', db); - const transactionsDb = await this._level('transactions', db); - // Clear any dirty state in the transactions - await transactionsDb.clear(); - return { - dataDb, - transactionsDb, - }; - } - - protected async _level(domain: string, dbLevel: DBLevel): Promise { - try { - return await new Promise((resolve, reject) => { - const dbLevelNew = subleveldown(dbLevel, domain, { - keyEncoding: 'binary', - valueEncoding: 'binary', - open: (cb) => { - // This `cb` is defaulted (hardcoded) to a function that emits an error event - // When using `level`, we are able to provide a callback that overrides this `cb` - // However `subleveldown` does not provide a callback parameter - // It provides this `open` option, which requires us to call `cb` to finish - // If we provide an exception as a parameter, it will be received by the `error` event handler - cb(undefined); - resolve(dbLevelNew); - }, - }); - // @ts-ignore error event for subleveldown - dbLevelNew.on('error', (e) => { - // Errors during construction of the sublevel will be emitted as events - reject(e); - }); - }); - } catch (e) { - if (e instanceof RangeError) { - // Some domain prefixes will conflict with the separator - throw new errors.ErrorDBLevelPrefix(); - } - throw e; - } + protected async setupRootLevels(db: LevelDB): Promise { + // Clear any dirty state in transactions + await this._clear(db, ['transactions']); } } diff --git a/src/DBTransaction.ts b/src/DBTransaction.ts index fa8ee213..480b73e8 100644 --- a/src/DBTransaction.ts +++ b/src/DBTransaction.ts @@ -1,10 +1,10 @@ import type { AbstractIteratorOptions } from 'abstract-leveldown'; import type DB from './DB'; -import type { POJO, DBDomain, DBLevel, DBIterator, DBOps } from './types'; +import type { KeyPath, LevelPath, DBIterator, DBOps } from './types'; import Logger from '@matrixai/logger'; import { CreateDestroy, ready } from '@matrixai/async-init/dist/CreateDestroy'; -import * as dbUtils from './utils'; -import * as dbErrors from './errors'; +import * as utils from './utils'; +import * as errors from './errors'; /** * Minimal read-committed transaction system @@ -32,27 +32,24 @@ interface DBTransaction extends CreateDestroy {} class DBTransaction { public static async createTransaction({ db, - transactionDb, transactionId, logger = new Logger(this.name), }: { db: DB; - transactionDb: DBLevel; transactionId: number; logger?: Logger; }): Promise { return new this({ db, - transactionDb, transactionId, logger, }); } public readonly transactionId: number; + public readonly transactionPath: LevelPath; protected db: DB; - protected transactionDb: DBLevel; protected logger: Logger; protected _ops: DBOps = []; protected _callbacksSuccess: Array<() => any> = []; @@ -62,23 +59,21 @@ class DBTransaction { public constructor({ db, - transactionDb, transactionId, logger, }: { db: DB; - transactionDb: DBLevel; transactionId: number; logger: Logger; }) { this.logger = logger; this.db = db; - this.transactionDb = transactionDb; this.transactionId = transactionId; + this.transactionPath = ['transactions', this.transactionId.toString()]; } public async destroy() { - await this.transactionDb.clear(); + await this.db._clear(this.db.db, this.transactionPath); } get ops(): Readonly { @@ -101,60 +96,128 @@ class DBTransaction { return this._rollbacked; } - @ready(new dbErrors.ErrorDBTransactionDestroyed()) - public async dump(domain: DBDomain = []): Promise { - let transactionLevel = this.transactionDb; - for (const d of domain) { - transactionLevel = await this.db.level(d, transactionLevel); + public async get( + keyPath: KeyPath | string | Buffer, + raw?: false, + ): Promise; + public async get( + keyPath: KeyPath | string | Buffer, + raw: true, + ): Promise; + @ready(new errors.ErrorDBTransactionDestroyed()) + public async get( + keyPath: KeyPath | string | Buffer, + raw: boolean = false, + ): Promise { + if (!Array.isArray(keyPath)) { + keyPath = [keyPath] as KeyPath; + } + if (utils.checkSepKeyPath(keyPath as KeyPath)) { + throw new errors.ErrorDBLevelSep(); } - const records = {}; - for await (const o of transactionLevel.createReadStream()) { - const key = (o as any).key.toString(); - const data = (o as any).value as Buffer; - const value = await this.db.deserializeDecrypt(data, false); - records[key] = value; + let value = await this.db._get( + [...this.transactionPath, ...keyPath] as unknown as KeyPath, + raw as any, + ); + if (value === undefined) { + value = await this.db.get(keyPath, raw as any); + // Don't set it in the transaction DB + // Because this is not a repeatable-read "snapshot" } - return records; + return value; } - public async iterator( - options: AbstractIteratorOptions & { values: false }, - domain?: DBDomain, - ): Promise>; - public async iterator( - options?: AbstractIteratorOptions, - domain?: DBDomain, - ): Promise>; - @ready(new dbErrors.ErrorDBTransactionDestroyed()) - public async iterator( - options?: AbstractIteratorOptions, - domain: DBDomain = [], - ): Promise { - let dataLevel = this.db.dataDb; - for (const d of domain) { - dataLevel = await this.db.level(d, dataLevel); + public async put( + keyPath: KeyPath | string | Buffer, + value: any, + raw?: false, + ): Promise; + public async put( + keyPath: KeyPath | string | Buffer, + value: Buffer, + raw: true, + ): Promise; + @ready(new errors.ErrorDBTransactionDestroyed()) + public async put( + keyPath: KeyPath | string | Buffer, + value: any, + raw: boolean = false, + ): Promise { + if (!Array.isArray(keyPath)) { + keyPath = [keyPath] as KeyPath; + } + if (utils.checkSepKeyPath(keyPath as KeyPath)) { + throw new errors.ErrorDBLevelSep(); } - const dataIterator = dataLevel.iterator({ - ...options, - keys: true, - keyAsBuffer: true, - valuesAsBuffer: true, + await this.db._put( + [...this.transactionPath, ...keyPath] as unknown as KeyPath, + value, + raw as any, + ); + this._ops.push({ + type: 'put', + keyPath, + value, + raw, }); - let transactionLevel = this.transactionDb; - for (const d of domain) { - transactionLevel = await this.db.level(d, transactionLevel); + } + + @ready(new errors.ErrorDBTransactionDestroyed()) + public async del(keyPath: KeyPath | string | Buffer): Promise { + if (!Array.isArray(keyPath)) { + keyPath = [keyPath] as KeyPath; + } + if (utils.checkSepKeyPath(keyPath as KeyPath)) { + throw new errors.ErrorDBLevelSep(); } - const tranIterator = transactionLevel.iterator({ - ...options, - keys: true, - keyAsBuffer: true, - valuesAsBuffer: true, + await this.db._del([ + ...this.transactionPath, + ...keyPath, + ] as unknown as KeyPath); + this._ops.push({ + type: 'del', + keyPath, }); + } + + public iterator( + options: AbstractIteratorOptions & { values: false }, + levelPath?: LevelPath, + ): DBIterator; + public iterator( + options?: AbstractIteratorOptions, + levelPath?: LevelPath, + ): DBIterator; + @ready(new errors.ErrorDBTransactionDestroyed()) + public iterator( + options?: AbstractIteratorOptions, + levelPath: LevelPath = [], + ): DBIterator { + const dataIterator = this.db._iterator( + this.db.db, + { + ...options, + keys: true, + keyAsBuffer: true, + valueAsBuffer: true, + }, + ['data', ...levelPath], + ); + const tranIterator = this.db._iterator( + this.db.db, + { + ...options, + keys: true, + keyAsBuffer: true, + valueAsBuffer: true, + }, + [...this.transactionPath, ...levelPath], + ); const order = options?.reverse ? 'desc' : 'asc'; const iterator = { _ended: false, _nexting: false, - seek: (k: Buffer | string) => { + seek: (k: Buffer | string): void => { if (iterator._ended) { throw new Error('cannot call seek() after end()'); } @@ -167,17 +230,15 @@ class DBTransaction { dataIterator.seek(k); tranIterator.seek(k); }, - end: async () => { + end: async (): Promise => { if (iterator._ended) { throw new Error('end() already called on iterator'); } iterator._ended = true; - // @ts-ignore AbstractIterator type is outdated await dataIterator.end(); - // @ts-ignore AbstractIterator type is outdated await tranIterator.end(); }, - next: async () => { + next: async (): Promise<[Buffer, Buffer | undefined] | undefined> => { if (iterator._ended) { throw new Error('cannot call next() after end()'); } @@ -187,21 +248,10 @@ class DBTransaction { ); } iterator._nexting = true; - const decryptKV = async ([key, data]: [ - Buffer, - Buffer | undefined, - ]): Promise<[Buffer, Buffer | undefined]> => { - if (data != null) { - data = await this.db.deserializeDecrypt(data, true); - } - return [key, data]; - }; try { - // @ts-ignore AbstractIterator type is outdated const tranKV = (await tranIterator.next()) as | [Buffer, Buffer | undefined] | undefined; - // @ts-ignore AbstractIterator type is outdated const dataKV = (await dataIterator.next()) as | [Buffer, Buffer | undefined] | undefined; @@ -212,12 +262,12 @@ class DBTransaction { // If tranIterator is not finished but dataIterator is finished // continue with tranIterator if (tranKV != null && dataKV == null) { - return decryptKV(tranKV); + return tranKV; } // If tranIterator is finished but dataIterator is not finished // continue with the dataIterator if (tranKV == null && dataKV != null) { - return decryptKV(dataKV); + return dataKV; } const [tranKey, tranData] = tranKV as [Buffer, Buffer | undefined]; const [dataKey, dataData] = dataKV as [Buffer, Buffer | undefined]; @@ -225,21 +275,21 @@ class DBTransaction { if (keyCompare < 0) { if (order === 'asc') { dataIterator.seek(tranKey); - return decryptKV([tranKey, tranData]); + return [tranKey, tranData]; } else if (order === 'desc') { tranIterator.seek(dataKey); - return decryptKV([dataKey, dataData]); + return [dataKey, dataData]; } } else if (keyCompare > 0) { if (order === 'asc') { tranIterator.seek(dataKey); - return decryptKV([dataKey, dataData]); + return [dataKey, dataData]; } else if (order === 'desc') { dataIterator.seek(tranKey); - return decryptKV([tranKey, tranData]); + return [tranKey, tranData]; } } else { - return decryptKV([tranKey, tranData]); + return [tranKey, tranData]; } } finally { iterator._nexting = false; @@ -259,111 +309,59 @@ class DBTransaction { return iterator; } - @ready(new dbErrors.ErrorDBTransactionDestroyed()) - public async clear(domain: DBDomain = []): Promise { - for await (const [k] of await this.iterator({ values: false }, domain)) { - await this.del(domain, k); + @ready(new errors.ErrorDBTransactionDestroyed()) + public async clear(levelPath: LevelPath = []): Promise { + for await (const [k] of await this.iterator({ values: false }, levelPath)) { + await this.del([...levelPath, k] as unknown as KeyPath); } } - @ready(new dbErrors.ErrorDBTransactionDestroyed()) - public async count(domain: DBDomain = []): Promise { + @ready(new errors.ErrorDBTransactionDestroyed()) + public async count(levelPath: LevelPath = []): Promise { let count = 0; - for await (const _ of await this.iterator({ values: false }, domain)) { + for await (const _ of await this.iterator({ values: false }, levelPath)) { count++; } return count; } - public async get( - domain: DBDomain, - key: string | Buffer, + /** + * Dump from transaction level + * It is intended for diagnostics + */ + public async dump( + levelPath?: LevelPath, raw?: false, - ): Promise; - public async get( - domain: DBDomain, - key: string | Buffer, + ): Promise>; + public async dump( + levelPath: LevelPath | undefined, raw: true, - ): Promise; - @ready(new dbErrors.ErrorDBTransactionDestroyed()) - public async get( - domain: DBDomain, - key: string | Buffer, + ): Promise>; + @ready(new errors.ErrorDBTransactionDestroyed()) + public async dump( + levelPath: LevelPath = [], raw: boolean = false, - ): Promise { - const path = dbUtils.domainPath(domain, key); - let value: T | undefined; - try { - const data = await this.transactionDb.get(path); - value = await this.db.deserializeDecrypt(data, raw as any); - } catch (e) { - if (e.notFound) { - value = await this.db.get(domain, key, raw as any); - // Don't set it in the transaction DB - // Because this is not a repeatable-read "snapshot" - } else { - throw e; - } - } - return value; - } - - public async put( - domain: DBDomain, - key: string | Buffer, - value: any, - raw?: false, - ): Promise; - public async put( - domain: DBDomain, - key: string | Buffer, - value: Buffer, - raw: true, - ): Promise; - @ready(new dbErrors.ErrorDBTransactionDestroyed()) - public async put( - domain: DBDomain, - key: string | Buffer, - value: any, - raw: boolean = false, - ): Promise { - const path = dbUtils.domainPath(domain, key); - const data = await this.db.serializeEncrypt(value, raw as any); - await this.transactionDb.put(path, data); - this._ops.push({ - type: 'put', - domain, - key, - value, - raw, - }); - } - - @ready(new dbErrors.ErrorDBTransactionDestroyed()) - public async del(domain: DBDomain, key: string | Buffer): Promise { - const path = dbUtils.domainPath(domain, key); - await this.transactionDb.del(path); - this._ops.push({ - type: 'del', - domain, - key, - }); + ): Promise> { + return await this.db.dump( + [...this.transactionPath, ...levelPath], + raw as any, + ); } - @ready(new dbErrors.ErrorDBTransactionDestroyed()) + @ready(new errors.ErrorDBTransactionDestroyed()) public queueSuccess(f: () => any): void { this._callbacksSuccess.push(f); } - @ready(new dbErrors.ErrorDBTransactionDestroyed()) + @ready(new errors.ErrorDBTransactionDestroyed()) public queueFailure(f: () => any): void { this._callbacksFailure.push(f); } - @ready(new dbErrors.ErrorDBTransactionDestroyed()) + @ready(new errors.ErrorDBTransactionDestroyed()) public async commit(): Promise { if (this._rollbacked) { - throw new dbErrors.ErrorDBTransactionRollbacked(); + throw new errors.ErrorDBTransactionRollbacked(); } if (this._committed) { return; @@ -377,10 +375,10 @@ class DBTransaction { } } - @ready(new dbErrors.ErrorDBTransactionDestroyed()) + @ready(new errors.ErrorDBTransactionDestroyed()) public async rollback(): Promise { if (this._committed) { - throw new dbErrors.ErrorDBTransactionCommitted(); + throw new errors.ErrorDBTransactionCommitted(); } if (this._rollbacked) { return; @@ -391,13 +389,13 @@ class DBTransaction { } } - @ready(new dbErrors.ErrorDBTransactionDestroyed()) + @ready(new errors.ErrorDBTransactionDestroyed()) public async finalize(): Promise { if (this._rollbacked) { - throw new dbErrors.ErrorDBTransactionRollbacked(); + throw new errors.ErrorDBTransactionRollbacked(); } if (!this._committed) { - throw new dbErrors.ErrorDBTransactionNotCommited(); + throw new errors.ErrorDBTransactionNotCommited(); } for (const f of this._callbacksSuccess) { await f(); diff --git a/src/errors.ts b/src/errors.ts index af0bba17..fd30a300 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -24,11 +24,13 @@ class ErrorDBCreate extends ErrorDB {} class ErrorDBDelete extends ErrorDB {} -class ErrorDBLevelPrefix extends ErrorDB {} +class ErrorDBLevelSep extends ErrorDB {} class ErrorDBDecrypt extends ErrorDB {} -class ErrorDBParse extends ErrorDB {} +class ErrorDBParseKey extends ErrorDB {} + +class ErrorDBParseValue extends ErrorDB {} class ErrorDBTransactionDestroyed extends ErrorDB {} @@ -45,9 +47,10 @@ export { ErrorDBDestroyed, ErrorDBCreate, ErrorDBDelete, - ErrorDBLevelPrefix, + ErrorDBLevelSep, ErrorDBDecrypt, - ErrorDBParse, + ErrorDBParseKey, + ErrorDBParseValue, ErrorDBTransactionDestroyed, ErrorDBTransactionCommitted, ErrorDBTransactionNotCommited, diff --git a/src/types.ts b/src/types.ts index 8bb952e4..d8d3a962 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,4 @@ import type fs from 'fs'; -import type { AbstractLevelDOWN, AbstractIterator } from 'abstract-leveldown'; -import type { LevelUp } from 'levelup'; import type { WorkerManagerInterface } from '@matrixai/workers'; /** @@ -8,6 +6,11 @@ import type { WorkerManagerInterface } from '@matrixai/workers'; */ type POJO = { [key: string]: any }; +/** + * Non-empty array + */ +type NonEmptyArray = [T, ...T[]]; + interface FileSystem { promises: { rm: typeof fs.promises.rm; @@ -29,12 +32,17 @@ type Crypto = { type DBWorkerManagerInterface = WorkerManagerInterface; -type DBDomain = Readonly>; +/** + * Path to a key + * This must be an non-empty array + */ +type KeyPath = Readonly>; -type DBLevel = LevelUp< - AbstractLevelDOWN, - AbstractIterator ->; +/** + * Path to a DB level + * Empty level path refers to the root level + */ +type LevelPath = Readonly>; /** * Custom type for our iterator @@ -49,14 +57,12 @@ type DBIterator = { type DBOp_ = | { - domain: DBDomain; - key: string | Buffer; + keyPath: KeyPath | string | Buffer; value: any; raw?: false; } | { - domain: DBDomain; - key: string | Buffer; + keyPath: KeyPath | string | Buffer; value: Buffer; raw: true; }; @@ -73,11 +79,12 @@ type DBOps = Array; export type { POJO, + NonEmptyArray, FileSystem, Crypto, DBWorkerManagerInterface, - DBDomain, - DBLevel, + KeyPath, + LevelPath, DBIterator, DBOp, DBOps, diff --git a/src/utils.ts b/src/utils.ts index b7ceee97..50929143 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,20 +1,153 @@ -import type { DBDomain } from './types'; -import sublevelprefixer from 'sublevel-prefixer'; -import * as dbErrors from './errors'; +import type { NonEmptyArray, KeyPath, LevelPath } from './types'; +import * as errors from './errors'; -const prefix = '!'; +/** + * Separator is a single null byte + * During iteration acquiring a sublevel requires iterating + * between 0x00 and 0x01 + */ +const sep = Buffer.from([0]); + +/** + * Converts KeyPath to key buffer + * e.g. ['A', 'B'] => !A!B (where ! is the sep) + * The key path must not be empty + * Level parts must not contain the separator + * Key actual part is allowed to contain the separator + */ +function keyPathToKey(keyPath: KeyPath): Buffer { + const keyPart = keyPath.slice(-1)[0]; + const levelPath = keyPath.slice(0, -1); + return Buffer.concat([ + levelPathToKey(levelPath), + typeof keyPart === 'string' ? Buffer.from(keyPart, 'utf-8') : keyPart, + ]); +} -const prefixer = sublevelprefixer(prefix); +/** + * Converts LevelPath to key buffer + * e.g. ['A', 'B'] => !A!!B! (where ! is the sep) + * Level parts must not contain the separator + */ +function levelPathToKey(levelPath: LevelPath): Buffer { + return Buffer.concat( + levelPath.map((p) => + Buffer.concat([ + sep, + typeof p === 'string' ? Buffer.from(p, 'utf-8') : p, + sep, + ]), + ), + ); +} -function domainPath(levels: DBDomain, key: string | Buffer): string | Buffer { - if (!levels.length) { - return key; +/** + * Converts key buffer back into KeyPath + * e.g. !A!!B!C => ['A', 'B', 'C'] (where ! is the sep) + * Returned parts are always buffers + * + * BNF grammar of key buffer: + * path => levels:ls keyActual:k -> [...ls, k] | keyActual + * levels => level:l levels:ls finalKey -> [l, ...ls] | '' -> [] + * level => sep [^sep]+:l sep -> l + * sep => '!' + * keyActual => .+ + */ +function parseKey(key: Buffer): KeyPath { + const [bufs] = parsePath(key); + if (!isNonEmptyArray(bufs)) { + throw new TypeError('Buffer is not a key'); } - let prefix = key; - for (let i = levels.length - 1; i >= 0; i--) { - prefix = prefixer(levels[i], prefix); + return bufs; +} + +function parsePath(input: Buffer): [Array, Buffer] { + try { + let output: Array = []; + let input_: Buffer = input; + let output_: Array; + [output_, input_] = parseLevels(input_); + output = output.concat(output_); + [output_, input_] = parseKeyActual(input_); + output = output.concat(output_); + return [output, input_]; + } catch (e) { + let output: Array = []; + let input_: Buffer = input; + let output_: Array; + // eslint-disable-next-line prefer-const + [output_, input_] = parseKeyActual(input_); + output = output.concat(output_); + return [output, input_]; + } +} + +function parseLevels( + input: Buffer, +): [output: Array, remaining: Buffer] { + let output: Array = []; + try { + let input_: Buffer = input; + let output_: Array; + [output_, input_] = parseLevel(input_); + output = output.concat(output_); + [output_, input_] = parseLevels(input_); + output = output.concat(output_); + parseKeyActual(input_); + return [output, input_]; + } catch (e) { + return [[], input]; + } +} + +function parseLevel(input: Buffer): [Array, Buffer] { + const sepStart = input.indexOf(sep); + if (sepStart === -1) { + throw new errors.ErrorDBParseKey('Missing separator start'); + } + const sepEnd = input.indexOf(sep, sepStart + 1); + if (sepEnd === -1) { + throw new errors.ErrorDBParseKey('Missing separator end'); + } + const level = input.subarray(sepStart + 1, sepEnd); + const remaining = input.subarray(sepEnd + 1); + + return [[level], remaining]; +} + +function parseKeyActual(input: Buffer): [Array, Buffer] { + if (input.byteLength < 1) { + throw new errors.ErrorDBParseKey('Key cannot be empty'); + } + return [[input], input.subarray(input.byteLength)]; +} + +/** + * Checks if the KeyPath contains the separator + * This only checks the LevelPath part + */ +function checkSepKeyPath(keyPath: KeyPath): boolean { + const levelPath = keyPath.slice(0, -1); + return checkSepLevelPath(levelPath); +} + +/** + * Checks if LevelPath contains the separator + */ +function checkSepLevelPath(levelPath: LevelPath): boolean { + return levelPath.some(sepExists); +} + +/** + * Checks if the separator exists in a string or buffer + * This only needs to applied to the LevelPath, not the final key + */ +function sepExists(data: string | Buffer): boolean { + if (typeof data === 'string') { + return data.includes(sep.toString('utf-8')); + } else { + return data.includes(sep); } - return prefix; } function serialize(value: T): Buffer { @@ -26,7 +159,7 @@ function deserialize(value_: Buffer): T { return JSON.parse(value_.toString('utf-8')); } catch (e) { if (e instanceof SyntaxError) { - throw new dbErrors.ErrorDBParse(); + throw new errors.ErrorDBParseValue(); } throw e; } @@ -50,9 +183,22 @@ function fromArrayBuffer( return Buffer.from(b, offset, length); } +/** + * Type guard for NonEmptyArray + */ +function isNonEmptyArray(arr: T[]): arr is NonEmptyArray { + return arr.length > 0; +} + export { - prefix, - domainPath, + sep, + keyPathToKey, + levelPathToKey, + parseKey, + checkSepKeyPath, + checkSepLevelPath, + sepExists, + isNonEmptyArray, serialize, deserialize, toArrayBuffer, diff --git a/tests/DB.test.ts b/tests/DB.test.ts index 71933fc9..deedbbd8 100644 --- a/tests/DB.test.ts +++ b/tests/DB.test.ts @@ -1,4 +1,3 @@ -import type { DBOp } from '@/types'; import type { DBWorkerModule } from './workers/dbWorkerModule'; import os from 'os'; import path from 'path'; @@ -9,6 +8,7 @@ import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; import { WorkerManager } from '@matrixai/workers'; import { spawn, Worker } from 'threads'; import DB from '@/DB'; +import * as errors from '@/errors'; import * as utils from '@/utils'; import * as testUtils from './utils'; @@ -54,10 +54,10 @@ describe(DB.name, () => { const db = await DB.createDB({ dbPath, crypto, logger }); // This is a noop await db.start(); - await db.put([], 'a', 'value0'); + await db.put('a', 'value0'); await db.stop(); await db.start(); - expect(await db.get([], 'a')).toBe('value0'); + expect(await db.get('a')).toBe('value0'); await db.stop(); }); test('async start and stop preserves state without crypto', async () => { @@ -65,195 +65,188 @@ describe(DB.name, () => { const db = await DB.createDB({ dbPath, logger }); // This is a noop await db.start(); - await db.put([], 'a', 'value0'); + await db.put('a', 'value0'); await db.stop(); await db.start(); - expect(await db.get([], 'a')).toBe('value0'); - await db.stop(); - }); - test('async start and stop requires recreation of db levels', async () => { - const dbPath = `${dataDir}/db`; - const db = await DB.createDB({ dbPath, logger }); - await db.start(); - let level1 = await db.level('level1'); - await db.put(['level1'], 'key', 'value'); - await db.stop(); - await db.start(); - // The `level1` has to be recreated after `await db.stop()` - await expect(db.level('level2', level1)).rejects.toThrow( - /Inner database is not open/, - ); - level1 = await db.level('level1'); - await db.level('level2', level1); - expect(await db.get(['level1'], 'key')).toBe('value'); + expect(await db.get('a')).toBe('value0'); await db.stop(); }); test('creating fresh db', async () => { const dbPath = `${dataDir}/db`; const db1 = await DB.createDB({ dbPath, logger }); - await db1.put([], 'key', 'value'); + await db1.put('key', 'value'); await db1.stop(); const db2 = await DB.createDB({ dbPath, logger, fresh: true }); - expect(await db2.get([], 'key')).toBeUndefined(); + expect(await db2.get('key')).toBeUndefined(); await db2.stop(); }); - test('get and put and del', async () => { + test('start wipes dirty transaction state', async () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); + const data = await db.serializeEncrypt('bar', false); + // Put in dirty transaction state + await db.db.put(utils.keyPathToKey(['transactions', 'foo']), data); + expect(await db.dump(['transactions'])).toStrictEqual([['foo', 'bar']]); + await db.stop(); + // Should wipe the transaction state await db.start(); - await db.put([], 'a', 'value0'); - expect(await db.get([], 'a')).toBe('value0'); - await db.del([], 'a'); - expect(await db.get([], 'a')).toBeUndefined(); - await db.level('level1'); - await db.put(['level1'], 'a', 'value1'); - expect(await db.get(['level1'], 'a')).toBe('value1'); - await db.del(['level1'], 'a'); - expect(await db.get(['level1'], 'a')).toBeUndefined(); + expect(await db.dump(['transactions'])).toStrictEqual([]); + await db.dump(); await db.stop(); }); - test('batch put and del', async () => { + test('get and put and del', async () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); - await db.start(); - await db.batch([ - { - type: 'put', - domain: [], - key: 'a', - value: 'value0', - raw: false, - }, - { - type: 'put', - domain: [], - key: 'b', - value: 'value1', - raw: false, - }, - { - type: 'put', - domain: [], - key: 'c', - value: 'value2', - raw: false, - }, - { - type: 'del', - domain: [], - key: 'a', - }, - { - type: 'put', - domain: [], - key: 'd', - value: 'value3', - raw: false, - }, - ]); - expect(await db.get([], 'a')).toBeUndefined(); - expect(await db.get([], 'b')).toBe('value1'); - expect(await db.get([], 'c')).toBe('value2'); - expect(await db.get([], 'd')).toBe('value3'); + await db.put('a', 'value0'); + expect(await db.get('a')).toBe('value0'); + await db.del('a'); + expect(await db.get('a')).toBeUndefined(); + await db.put(['level1', 'a'], 'value1'); + expect(await db.get(['level1', 'a'])).toBe('value1'); + await db.del(['level1', 'a']); + expect(await db.get(['level1', 'a'])).toBeUndefined(); await db.stop(); }); - test('db levels are leveldbs', async () => { + test('get and put and del on string and buffer keys', async () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); - await db.start(); - await db.dataDb.put('a', await db.serializeEncrypt('value0', false)); - expect(await db.get([], 'a')).toBe('value0'); - await db.put([], 'b', 'value0'); - expect(await db.deserializeDecrypt(await db.dataDb.get('b'), false)).toBe( - 'value0', - ); - const level1 = await db.level('level1'); - await level1.put('a', await db.serializeEncrypt('value1', false)); - expect(await db.get(['level1'], 'a')).toBe('value1'); - await db.put(['level1'], 'b', 'value1'); - expect(await db.deserializeDecrypt(await level1.get('b'), false)).toBe( - 'value1', - ); + // 'string' is the same as Buffer.from('string') + // even across levels + await db.put('string', 'value1'); + expect(await db.get('string')).toBe('value1'); + expect(await db.get(Buffer.from('string'))).toBe('value1'); + await db.del('string'); + expect(await db.get('string')).toBeUndefined(); + expect(await db.get(Buffer.from('string'))).toBeUndefined(); + // Now using buffer keys across levels that are always strings + await db.put(['level1', 'string'], 'value2'); + expect(await db.get(['level1', 'string'])).toBe('value2'); + expect(await db.get(['level1', Buffer.from('string')])).toBe('value2'); + await db.del(['level1', Buffer.from('string')]); + expect(await db.get(['level1', 'string'])).toBeUndefined(); + expect(await db.get(['level1', Buffer.from('string')])).toBeUndefined(); await db.stop(); }); - test('db levels are just ephemeral abstractions', async () => { + test('levels cannot contain separator buffer', async () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); - await db.start(); - // There's no need to actually create a sublevel instance - // if you are always going to directly use the root - // however it is useful if you need to iterate over a sublevel - // plus you do not need to "destroy" a sublevel - // clearing the entries is sufficient - await db.put(['level1'], 'a', 'value1'); - expect(await db.get(['level1'], 'a')).toBe('value1'); - await db.del(['level1'], 'a'); - expect(await db.get(['level1'], 'a')).toBeUndefined(); + await expect( + db.put( + [Buffer.concat([utils.sep, Buffer.from('level')]), 'key'], + 'value', + ), + ).rejects.toThrow(errors.ErrorDBLevelSep); + await expect( + db.get([Buffer.concat([Buffer.from('level'), utils.sep]), 'key']), + ).rejects.toThrow(errors.ErrorDBLevelSep); + await expect( + db.del([ + Buffer.concat([utils.sep, Buffer.from('level'), utils.sep]), + 'key', + ]), + ).rejects.toThrow(errors.ErrorDBLevelSep); + expect(() => db.iterator(undefined, [utils.sep])).toThrow( + errors.ErrorDBLevelSep, + ); await db.stop(); }); - test('db levels are facilitated by key prefixes', async () => { + test('keys can contain separator buffer', async () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); - await db.start(); - const level1 = await db.level('level1'); - const level2a = await db.level('100', level1); - const level2b = await db.level('200', level1); - let count; - // Expect level1 to be empty - count = 0; - for await (const _ of level1.createKeyStream()) { - count++; - } - expect(count).toBe(0); - await level2a.put('a', await db.serializeEncrypt('value1', false)); - await level2b.put('b', await db.serializeEncrypt('value2', false)); - // There should be 2 entries at level1 - // because there is 1 entry for each sublevel - count = 0; - let keyToTest: string; - for await (const k of level1.createKeyStream()) { - // All keys are buffers - keyToTest = k.toString('utf-8'); - count++; - } - expect(count).toBe(2); - // It is possible to access sublevel entries from the upper level - const valueToTest = await db.get(['level1'], keyToTest!); - expect(valueToTest).toBeDefined(); - // The level separator is set to `!` - expect(keyToTest!).toBe('!200!b'); + await db.put(utils.sep, 'value'); + await db.put( + ['level', Buffer.concat([utils.sep, Buffer.from('key')])], + 'value', + ); + await db.put( + ['level', Buffer.concat([Buffer.from('key'), utils.sep])], + 'value', + ); + await db.put( + ['level', Buffer.concat([utils.sep, Buffer.from('key'), utils.sep])], + 'value', + ); + await db.put(['level', utils.sep], 'value'); + await db.put( + ['level', Buffer.concat([utils.sep, Buffer.from('key')])], + 'value', + ); + await db.put( + ['level', Buffer.concat([Buffer.from('key'), utils.sep])], + 'value', + ); + await db.put( + ['level', Buffer.concat([utils.sep, Buffer.from('key'), utils.sep])], + 'value', + ); + expect(await db.get(utils.sep)).toBe('value'); + expect( + await db.get(['level', Buffer.concat([utils.sep, Buffer.from('key')])]), + ).toBe('value'); + expect( + await db.get(['level', Buffer.concat([Buffer.from('key'), utils.sep])]), + ).toBe('value'); + expect( + await db.get([ + 'level', + Buffer.concat([utils.sep, Buffer.from('key'), utils.sep]), + ]), + ).toBe('value'); + expect(await db.get(['level', utils.sep])).toBe('value'); + expect( + await db.get(['level', Buffer.concat([utils.sep, Buffer.from('key')])]), + ).toBe('value'); + expect( + await db.get(['level', Buffer.concat([Buffer.from('key'), utils.sep])]), + ).toBe('value'); + expect( + await db.get([ + 'level', + Buffer.concat([utils.sep, Buffer.from('key'), utils.sep]), + ]), + ).toBe('value'); + await db.del(utils.sep); + await db.del(['level', Buffer.concat([utils.sep, Buffer.from('key')])]); + await db.del(['level', Buffer.concat([Buffer.from('key'), utils.sep])]); + await db.del([ + 'level', + Buffer.concat([utils.sep, Buffer.from('key'), utils.sep]), + ]); + await db.del(['level', utils.sep]); + await db.del(['level', Buffer.concat([utils.sep, Buffer.from('key')])]); + await db.del(['level', Buffer.concat([Buffer.from('key'), utils.sep])]); + await db.del([ + 'level', + Buffer.concat([utils.sep, Buffer.from('key'), utils.sep]), + ]); await db.stop(); }); test('clearing a db level clears all sublevels', async () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); - await db.start(); - const level1 = await db.level('level1'); - await db.level('level2', level1); - await db.put([], 'a', 'value0'); - await db.put(['level1'], 'a', 'value1'); - await db.put(['level1', 'level2'], 'a', 'value2'); - expect(await db.get([], 'a')).toBe('value0'); - expect(await db.get(['level1'], 'a')).toBe('value1'); - expect(await db.get(['level1', 'level2'], 'a')).toBe('value2'); - await level1.clear(); - expect(await db.get([], 'a')).toBe('value0'); - expect(await db.get(['level1'], 'a')).toBeUndefined(); - expect(await db.get(['level1', 'level2'], 'a')).toBeUndefined(); + await db.put(['a'], 'value0'); + await db.put(['level1', 'a'], 'value1'); + await db.put(['level1', 'level2', 'a'], 'value2'); + expect(await db.get('a')).toBe('value0'); + expect(await db.get(['level1', 'a'])).toBe('value1'); + expect(await db.get(['level1', 'level2', 'a'])).toBe('value2'); + await db.clear(['level1']); + expect(await db.get(['a'])).toBe('value0'); + expect(await db.get(['level1', 'a'])).toBeUndefined(); + expect(await db.get(['level1', 'level2', 'a'])).toBeUndefined(); await db.stop(); }); test('lexicographic iteration order', async () => { // Leveldb stores keys in lexicographic order const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); - await db.start(); // Sorted order [ 'AQ', 'L', 'Q', 'fP' ] const keys = ['Q', 'fP', 'AQ', 'L']; for (const k of keys) { - await db.put([], k, 'value'); + await db.put(k, 'value'); } const keysIterated: Array = []; - for await (const k of db.dataDb.createKeyStream()) { + for await (const [k] of db.iterator({ values: false })) { // Keys are buffers due to key encoding keysIterated.push(k.toString('utf-8')); } @@ -268,11 +261,11 @@ describe(DB.name, () => { nodeCrypto.randomBytes(3), ); for (const k of keys) { - await db.put([], k, 'value'); + await db.put(k, 'value'); } const keysIterated: Array = []; - for await (const k of db.dataDb.createKeyStream()) { - keysIterated.push(k as Buffer); + for await (const [k] of db.iterator({ values: false })) { + keysIterated.push(k); } expect(keys).not.toStrictEqual(keysIterated); expect(keys.sort(Buffer.compare)).toStrictEqual(keysIterated); @@ -288,14 +281,13 @@ describe(DB.name, () => { // Using the lexicographic-integer encoding const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); - await db.start(); // Sorted order should be [3, 4, 42, 100] const keys = [100, 3, 4, 42]; for (const k of keys) { - await db.put([], Buffer.from(lexi.pack(k)), 'value'); + await db.put(Buffer.from(lexi.pack(k)), 'value'); } const keysIterated: Array = []; - for await (const k of db.dataDb.createKeyStream()) { + for await (const [k] of db.iterator({ values: false })) { // Keys are buffers due to key encoding keysIterated.push(lexi.unpack([...k])); } @@ -304,32 +296,28 @@ describe(DB.name, () => { expect(keys.sort((a, b) => a - b)).toEqual(keysIterated); await db.stop(); }); - test('db level lexicographic iteration', async () => { + test('sublevel lexicographic iteration', async () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); - await db.start(); - const level1 = await db.level('level1'); + // Const level1 = await db.level('level1'); const keys1 = ['Q', 'fP', 'AQ', 'L']; for (const k of keys1) { - await level1.put(k, await db.serializeEncrypt('value1', false)); + await db.put(['level1', k], 'value1'); } const keysIterated1: Array = []; - for await (const k of level1.createKeyStream()) { + for await (const [k] of db.iterator({ values: false }, ['level1'])) { // Keys are buffers due to key encoding keysIterated1.push(k.toString('utf-8')); } expect(keys1).not.toEqual(keysIterated1); expect(keys1.sort()).toEqual(keysIterated1); - const level2 = await db.level('level2'); + // Const level2 = await db.level('level2'); const keys2 = [100, 3, 4, 42]; for (const k of keys2) { - await level2.put( - Buffer.from(lexi.pack(k)), - await db.serializeEncrypt('value2', false), - ); + await db.put(['level2', Buffer.from(lexi.pack(k))], 'value2'); } const keysIterated2: Array = []; - for await (const k of level2.createKeyStream()) { + for await (const [k] of db.iterator({ values: false }, ['level2'])) { // Keys are buffers due to key encoding keysIterated2.push(lexi.unpack([...k])); } @@ -338,102 +326,57 @@ describe(DB.name, () => { expect(keys2.sort((a, b) => a - b)).toEqual(keysIterated2); await db.stop(); }); - test('get and put and del on string and buffer keys', async () => { - const dbPath = `${dataDir}/db`; - const db = await DB.createDB({ dbPath, crypto, logger }); - await db.start(); - // 'string' is the same as Buffer.from('string') - // even across levels - await db.put([], 'string', 'value1'); - expect(await db.get([], 'string')).toBe('value1'); - expect(await db.get([], Buffer.from('string'))).toBe('value1'); - await db.del([], 'string'); - expect(await db.get([], 'string')).toBeUndefined(); - expect(await db.get([], Buffer.from('string'))).toBeUndefined(); - // Now using buffer keys across levels that are always strings - await db.level('level1'); - await db.put(['level1'], 'string', 'value2'); - expect(await db.get(['level1'], 'string')).toBe('value2'); - // Level1 has been typed to use string keys - // however the reality is that you can always use buffer keys - // since strings and buffers get turned into buffers - // so we can use buffer keys starting from root - // we use this key type to enforce opaque types that are actually strings or buffers - expect(await db.get(['level1'], Buffer.from('string'))).toBe('value2'); - await db.del(['level1'], Buffer.from('string')); - expect(await db.get(['level1'], 'string')).toBeUndefined(); - expect(await db.get(['level1'], Buffer.from('string'))).toBeUndefined(); - await db.stop(); - }); - test('streams can be consumed with promises', async () => { - const dbPath = `${dataDir}/db`; - const db = await DB.createDB({ dbPath, crypto, logger }); - await db.start(); - await db.put([], 'a', 'value0'); - await db.put([], 'b', 'value1'); - await db.put([], 'c', 'value2'); - await db.put([], 'd', 'value3'); - const keyStream = db.dataDb.createKeyStream(); - const ops = await new Promise>((resolve, reject) => { - const ops: Array = []; - keyStream.on('data', (k) => { - ops.push({ - type: 'del', - domain: [], - key: k, - }); - }); - keyStream.on('end', () => { - resolve(ops); - }); - keyStream.on('error', (e) => { - reject(e); - }); - }); - // Here we batch up the deletion - await db.batch(ops); - expect(await db.get([], 'a')).toBeUndefined(); - expect(await db.get([], 'b')).toBeUndefined(); - expect(await db.get([], 'c')).toBeUndefined(); - expect(await db.get([], 'd')).toBeUndefined(); - await db.stop(); - }); test('iterating sublevels', async () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); - await db.put([], 'a', 'value0'); - await db.put([], 'b', 'value1'); - await db.put([], 'c', 'value2'); - await db.put([], 'd', 'value3'); - await db.put(['level1'], 'a', 'value0'); - await db.put(['level1'], 'b', 'value1'); - await db.put(['level1'], 'c', 'value2'); - await db.put(['level1'], 'd', 'value3'); + await db.put('a', 'value0'); + await db.put('b', 'value1'); + await db.put(['level1', 'a'], 'value0'); + await db.put(['level1', 'b'], 'value1'); + await db.put(['level1', 'level2', 'a'], 'value0'); + await db.put(['level1', 'level2', 'b'], 'value1'); let results: Array<[string, string]>; results = []; for await (const [k, v] of db.iterator()) { results.push([k.toString(), JSON.parse(v.toString())]); } expect(results).toStrictEqual([ - [`${utils.prefix}level1${utils.prefix}a`, 'value0'], - [`${utils.prefix}level1${utils.prefix}b`, 'value1'], - [`${utils.prefix}level1${utils.prefix}c`, 'value2'], - [`${utils.prefix}level1${utils.prefix}d`, 'value3'], + [ + `${utils.sep.toString('binary')}level1${utils.sep.toString( + 'binary', + )}${utils.sep.toString('binary')}level2${utils.sep.toString( + 'binary', + )}a`, + 'value0', + ], + [ + `${utils.sep.toString('binary')}level1${utils.sep.toString( + 'binary', + )}${utils.sep.toString('binary')}level2${utils.sep.toString( + 'binary', + )}b`, + 'value1', + ], + [ + `${utils.sep.toString('binary')}level1${utils.sep.toString('binary')}a`, + 'value0', + ], + [ + `${utils.sep.toString('binary')}level1${utils.sep.toString('binary')}b`, + 'value1', + ], ['a', 'value0'], ['b', 'value1'], - ['c', 'value2'], - ['d', 'value3'], ]); results = []; - const level1 = await db.level('level1'); - for await (const [k, v] of db.iterator(undefined, level1)) { + for await (const [k, v] of db.iterator(undefined, ['level1'])) { results.push([k.toString(), JSON.parse(v.toString())]); } expect(results).toStrictEqual([ + [`${utils.sep}level2${utils.sep}a`, 'value0'], + [`${utils.sep}level2${utils.sep}b`, 'value1'], ['a', 'value0'], ['b', 'value1'], - ['c', 'value2'], - ['d', 'value3'], ]); await db.stop(); }); @@ -441,28 +384,25 @@ describe(DB.name, () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); await db.start(); - await db.put([], 'a', 'value0'); - await db.put([], 'b', 'value1'); - await db.put([], 'c', 'value2'); - await db.put([], 'd', 'value3'); - await db.put(['level1'], 'a', 'value0'); - await db.put(['level1'], 'b', 'value1'); - await db.put(['level1'], 'c', 'value2'); - await db.put(['level1'], 'd', 'value3'); - await db.put(['level1', 'level11'], 'a', 'value0'); - await db.put(['level1', 'level11'], 'b', 'value1'); - await db.put(['level1', 'level11'], 'c', 'value2'); - await db.put(['level1', 'level11'], 'd', 'value3'); - await db.put(['level2'], 'a', 'value0'); - await db.put(['level2'], 'b', 'value1'); - await db.put(['level2'], 'c', 'value2'); - await db.put(['level2'], 'd', 'value3'); - const level1 = await db.level('level1'); - const level11 = await db.level('level11', level1); - const level2 = await db.level('level2'); - expect(await db.count(level1)).toBe(8); - expect(await db.count(level11)).toBe(4); - expect(await db.count(level2)).toBe(4); + await db.put('a', 'value0'); + await db.put('b', 'value1'); + await db.put('c', 'value2'); + await db.put('d', 'value3'); + await db.put(['level1', 'a'], 'value0'); + await db.put(['level1', 'b'], 'value1'); + await db.put(['level1', 'c'], 'value2'); + await db.put(['level1', 'd'], 'value3'); + await db.put(['level1', 'level11', 'a'], 'value0'); + await db.put(['level1', 'level11', 'b'], 'value1'); + await db.put(['level1', 'level11', 'c'], 'value2'); + await db.put(['level1', 'level11', 'd'], 'value3'); + await db.put(['level2', 'a'], 'value0'); + await db.put(['level2', 'b'], 'value1'); + await db.put(['level2', 'c'], 'value2'); + await db.put(['level2', 'd'], 'value3'); + expect(await db.count(['level1'])).toBe(8); + expect(await db.count(['level1', 'level11'])).toBe(4); + expect(await db.count(['level2'])).toBe(4); expect(await db.count()).toBe(16); await db.stop(); }); @@ -477,18 +417,57 @@ describe(DB.name, () => { }); db.setWorkerManager(workerManager); await db.start(); - await db.put([], 'a', 'value0'); - expect(await db.get([], 'a')).toBe('value0'); - await db.del([], 'a'); - expect(await db.get([], 'a')).toBeUndefined(); - await db.level('level1'); - await db.put(['level1'], 'a', 'value1'); - expect(await db.get(['level1'], 'a')).toBe('value1'); - await db.del(['level1'], 'a'); - expect(await db.get(['level1'], 'a')).toBeUndefined(); + await db.put('a', 'value0'); + expect(await db.get('a')).toBe('value0'); + await db.del('a'); + expect(await db.get('a')).toBeUndefined(); + await db.put(['level1', 'a'], 'value1'); + expect(await db.get(['level1', 'a'])).toBe('value1'); + await db.del(['level1', 'a']); + expect(await db.get(['level1', 'a'])).toBeUndefined(); await db.stop(); await workerManager.destroy(); }); + test('batch put and del', async () => { + const dbPath = `${dataDir}/db`; + const db = await DB.createDB({ dbPath, crypto, logger }); + await db.start(); + await db.batch([ + { + type: 'put', + keyPath: ['a'], + value: 'value0', + raw: false, + }, + { + type: 'put', + keyPath: 'b', + value: 'value1', + raw: false, + }, + { + type: 'put', + keyPath: ['c'], + value: 'value2', + raw: false, + }, + { + type: 'del', + keyPath: 'a', + }, + { + type: 'put', + keyPath: 'd', + value: 'value3', + raw: false, + }, + ]); + expect(await db.get('a')).toBeUndefined(); + expect(await db.get('b')).toBe('value1'); + expect(await db.get('c')).toBe('value2'); + expect(await db.get('d')).toBe('value3'); + await db.stop(); + }); test('parallelized batch put and del', async () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, crypto, logger }); @@ -503,42 +482,37 @@ describe(DB.name, () => { await db.batch([ { type: 'put', - domain: [], - key: 'a', + keyPath: ['a'], value: 'value0', raw: false, }, { type: 'put', - domain: [], - key: 'b', + keyPath: 'b', value: 'value1', raw: false, }, { type: 'put', - domain: [], - key: 'c', + keyPath: ['c'], value: 'value2', raw: false, }, { type: 'del', - domain: [], - key: 'a', + keyPath: 'a', }, { type: 'put', - domain: [], - key: 'd', + keyPath: 'd', value: 'value3', raw: false, }, ]); - expect(await db.get([], 'a')).toBeUndefined(); - expect(await db.get([], 'b')).toBe('value1'); - expect(await db.get([], 'c')).toBe('value2'); - expect(await db.get([], 'd')).toBe('value3'); + expect(await db.get('a')).toBeUndefined(); + expect(await db.get('b')).toBe('value1'); + expect(await db.get('c')).toBe('value2'); + expect(await db.get('d')).toBe('value3'); await db.stop(); await workerManager.destroy(); }); @@ -546,54 +520,48 @@ describe(DB.name, () => { const dbPath = `${dataDir}/db`; const db = await DB.createDB({ dbPath, logger }); await db.start(); - await db.put([], 'a', 'value0'); - expect(await db.get([], 'a')).toBe('value0'); - await db.del([], 'a'); - expect(await db.get([], 'a')).toBeUndefined(); - await db.level('level1'); - await db.put(['level1'], 'a', 'value1'); - expect(await db.get(['level1'], 'a')).toBe('value1'); - await db.del(['level1'], 'a'); - expect(await db.get(['level1'], 'a')).toBeUndefined(); + await db.put('a', 'value0'); + expect(await db.get('a')).toBe('value0'); + await db.del('a'); + expect(await db.get('a')).toBeUndefined(); + await db.put(['level1', 'a'], 'value1'); + expect(await db.get(['level1', 'a'])).toBe('value1'); + await db.del(['level1', 'a']); + expect(await db.get(['level1', 'a'])).toBeUndefined(); await db.batch([ { type: 'put', - domain: [], - key: 'a', + keyPath: ['a'], value: 'value0', raw: false, }, { type: 'put', - domain: [], - key: 'b', + keyPath: ['b'], value: 'value1', raw: false, }, { type: 'put', - domain: [], - key: 'c', + keyPath: ['c'], value: 'value2', raw: false, }, { type: 'del', - domain: [], - key: 'a', + keyPath: ['a'], }, { type: 'put', - domain: [], - key: 'd', + keyPath: ['d'], value: 'value3', raw: false, }, ]); - expect(await db.get([], 'a')).toBeUndefined(); - expect(await db.get([], 'b')).toBe('value1'); - expect(await db.get([], 'c')).toBe('value2'); - expect(await db.get([], 'd')).toBe('value3'); + expect(await db.get('a')).toBeUndefined(); + expect(await db.get('b')).toBe('value1'); + expect(await db.get('c')).toBe('value2'); + expect(await db.get('d')).toBe('value3'); await db.stop(); }); }); diff --git a/tests/DBTransaction.test.ts b/tests/DBTransaction.test.ts index 7d126465..555ec616 100644 --- a/tests/DBTransaction.test.ts +++ b/tests/DBTransaction.test.ts @@ -38,138 +38,136 @@ describe(DBTransaction.name, () => { test('snapshot state is cleared after releasing transactions', async () => { const acquireTran1 = db.transaction(); const [releaseTran1, tran1] = await acquireTran1(); - await tran1!.put([], 'hello', 'world'); + await tran1!.put('hello', 'world'); const acquireTran2 = db.transaction(); const [releaseTran2, tran2] = await acquireTran2(); - await tran2!.put([], 'hello', 'world'); - expect(await db.dump(db.transactionsDb)).toStrictEqual({ - [`${utils.prefix}0${utils.prefix}hello`]: 'world', - [`${utils.prefix}1${utils.prefix}hello`]: 'world', - }); + await tran2!.put('hello', 'world'); + expect(await db.dump(['transactions'])).toStrictEqual([ + [`${utils.sep}0${utils.sep}hello`, 'world'], + [`${utils.sep}1${utils.sep}hello`, 'world'], + ]); await releaseTran1(); - expect(await db.dump(db.transactionsDb)).toStrictEqual({ - [`${utils.prefix}1${utils.prefix}hello`]: 'world', - }); + expect(await db.dump(['transactions'])).toStrictEqual([ + [`${utils.sep}1${utils.sep}hello`, 'world'], + ]); await releaseTran2(); - expect(await db.dump(db.transactionsDb)).toStrictEqual({}); + expect(await db.dump(['transactions'])).toStrictEqual([]); }); test('get, put and del', async () => { const p = withF([db.transaction()], async ([tran]) => { - expect(await tran.get([], 'foo')).toBeUndefined(); + expect(await tran.get('foo')).toBeUndefined(); // Add foo -> bar to the transaction - await tran.put([], 'foo', 'bar'); + await tran.put('foo', 'bar'); // Add hello -> world to the transaction - await tran.put([], 'hello', 'world'); - expect(await tran.get([], 'foo')).toBe('bar'); - expect(await tran.get([], 'hello')).toBe('world'); - expect(await tran.dump()).toStrictEqual({ - foo: 'bar', - hello: 'world', - }); + await tran.put('hello', 'world'); + expect(await tran.get('foo')).toBe('bar'); + expect(await tran.get('hello')).toBe('world'); + expect(await tran.dump()).toStrictEqual([ + ['foo', 'bar'], + ['hello', 'world'], + ]); // Delete hello -> world - await tran.del([], 'hello'); + await tran.del('hello'); // Transaction state should be used - expect(Object.entries(await db.dump(db.transactionsDb)).length > 0).toBe( + expect(Object.entries(await db.dump(['transactions'])).length > 0).toBe( true, ); }); // While the transaction is executed, there is no data - expect(await db.dump()).toStrictEqual({}); + expect(await db.dump(['data'])).toStrictEqual([]); await p; // Now the state should be applied to the DB - expect(await db.dump()).toStrictEqual({ - foo: 'bar', - }); + expect(await db.dump(['data'])).toStrictEqual([['foo', 'bar']]); // Transaction state is cleared - expect(await db.dump(db.transactionsDb)).toStrictEqual({}); + expect(await db.dump(['transactions'])).toStrictEqual([]); }); test('transactional clear', async () => { - await db.put([], '1', '1'); - await db.put([], '2', '2'); - await db.put([], '3', '3'); + await db.put('1', '1'); + await db.put('2', '2'); + await db.put('3', '3'); + // Transactional clear, clears all values await withF([db.transaction()], async ([tran]) => { await tran.clear(); }); - expect(await db.dump()).toStrictEqual({}); + expect(await db.dump(['data'])).toStrictEqual([]); + // Noop await db.clear(); - await db.put([], '1', '1'); - await db.put(['level1'], '2', '2'); - await db.put(['level1', 'level2'], '3', '3'); + await db.put('1', '1'); + await db.put(['level1', '2'], '2'); + await db.put(['level1', 'level2', '3'], '3'); await withF([db.transaction()], async ([tran]) => { await tran.clear(['level1']); }); - expect(await db.dump()).toStrictEqual({ - '1': '1', - }); + expect(await db.dump(['data'])).toStrictEqual([['1', '1']]); }); test('transactional count', async () => { - await db.put([], '1', '1'); - await db.put([], '2', '2'); - await db.put([], '3', '3'); + await db.put('1', '1'); + await db.put('2', '2'); + await db.put('3', '3'); await withF([db.transaction()], async ([tran]) => { expect(await tran.count()).toBe(3); }); await db.clear(); - await db.put([], '1', '1'); - await db.put(['level1'], '2', '2'); - await db.put(['level1', 'level2'], '3', '3'); + await db.put('1', '1'); + await db.put(['level1', '2'], '2'); + await db.put(['level1', 'level2', '3'], '3'); await withF([db.transaction()], async ([tran]) => { expect(await tran.count(['level1'])).toBe(2); }); }); test('no dirty reads', async () => { await withF([db.transaction()], async ([tran1]) => { - expect(await tran1.get([], 'hello')).toBeUndefined(); + expect(await tran1.get('hello')).toBeUndefined(); await withF([db.transaction()], async ([tran2]) => { - await tran2.put([], 'hello', 'world'); + await tran2.put('hello', 'world'); // `tran2` has not yet committed - expect(await tran1.get([], 'hello')).toBeUndefined(); + expect(await tran1.get('hello')).toBeUndefined(); }); }); await db.clear(); await withF([db.transaction()], async ([tran1]) => { - expect(await tran1.get([], 'hello')).toBeUndefined(); - await tran1.put([], 'hello', 'foo'); + expect(await tran1.get('hello')).toBeUndefined(); + await tran1.put('hello', 'foo'); await withF([db.transaction()], async ([tran2]) => { // `tran1` has not yet committed - expect(await tran2.get([], 'hello')).toBeUndefined(); - await tran2.put([], 'hello', 'bar'); + expect(await tran2.get('hello')).toBeUndefined(); + await tran2.put('hello', 'bar'); // `tran2` has not yet committed - expect(await tran1.get([], 'hello')).toBe('foo'); + expect(await tran1.get('hello')).toBe('foo'); }); }); }); test('non-repeatable reads', async () => { await withF([db.transaction()], async ([tran1]) => { - expect(await tran1.get([], 'hello')).toBeUndefined(); + expect(await tran1.get('hello')).toBeUndefined(); await withF([db.transaction()], async ([tran2]) => { - await tran2.put([], 'hello', 'world'); + await tran2.put('hello', 'world'); }); // `tran2` is now committed - expect(await tran1.get([], 'hello')).toBe('world'); + expect(await tran1.get('hello')).toBe('world'); }); await db.clear(); await withF([db.transaction()], async ([tran1]) => { - expect(await tran1.get([], 'hello')).toBeUndefined(); - await tran1.put([], 'hello', 'foo'); + expect(await tran1.get('hello')).toBeUndefined(); + await tran1.put('hello', 'foo'); await withF([db.transaction()], async ([tran2]) => { // `tran1` has not yet committed - expect(await tran2.get([], 'hello')).toBeUndefined(); - await tran2.put([], 'hello', 'bar'); + expect(await tran2.get('hello')).toBeUndefined(); + await tran2.put('hello', 'bar'); }); // `tran2` is now committed // however because `foo` has been written in tran1, it stays as `foo` - expect(await tran1.get([], 'hello')).toBe('foo'); + expect(await tran1.get('hello')).toBe('foo'); }); }); test('phantom reads', async () => { - await db.put([], '1', '1'); - await db.put([], '2', '2'); - await db.put([], '3', '3'); + await db.put('1', '1'); + await db.put('2', '2'); + await db.put('3', '3'); let rows: Array<[string, string]>; await withF([db.transaction()], async ([tran1]) => { rows = []; - for await (const [k, v] of await tran1.iterator()) { + for await (const [k, v] of tran1.iterator()) { rows.push([k.toString(), JSON.parse(v.toString())]); } expect(rows).toStrictEqual([ @@ -178,10 +176,10 @@ describe(DBTransaction.name, () => { ['3', '3'], ]); await withF([db.transaction()], async ([tran2]) => { - await tran2.del([], '1'); - await tran2.put([], '4', '4'); + await tran2.del('1'); + await tran2.put('4', '4'); rows = []; - for await (const [k, v] of await tran1.iterator()) { + for await (const [k, v] of tran1.iterator()) { rows.push([k.toString(), JSON.parse(v.toString())]); } expect(rows).toStrictEqual([ @@ -191,7 +189,7 @@ describe(DBTransaction.name, () => { ]); }); rows = []; - for await (const [k, v] of await tran1.iterator()) { + for await (const [k, v] of tran1.iterator()) { rows.push([k.toString(), JSON.parse(v.toString())]); } expect(rows).toStrictEqual([ @@ -203,14 +201,14 @@ describe(DBTransaction.name, () => { }); test('lost updates', async () => { await withF([db.transaction()], async ([tran1]) => { - await tran1.put([], 'hello', 'foo'); + await tran1.put('hello', 'foo'); await withF([db.transaction()], async ([tran2]) => { - await tran2.put([], 'hello', 'bar'); + await tran2.put('hello', 'bar'); }); - expect(await tran1.get([], 'hello')).toBe('foo'); + expect(await tran1.get('hello')).toBe('foo'); }); // `tran2` write is lost because `tran1` committed last - expect(await db.get([], 'hello')).toBe('foo'); + expect(await db.get('hello')).toBe('foo'); }); test('iterator with same largest key', async () => { /* @@ -229,21 +227,20 @@ describe(DBTransaction.name, () => { | k | k = k | k = 11 | k = 11 | */ const results: Array<[string, string]> = []; - await db.put([], 'a', 'a'); - await db.put([], 'b', 'b'); - await db.put([], 'd', 'd'); - await db.put([], 'e', 'e'); - await db.put([], 'h', 'h'); - await db.put([], 'k', 'k'); + await db.put('a', 'a'); + await db.put('b', 'b'); + await db.put('d', 'd'); + await db.put('e', 'e'); + await db.put('h', 'h'); + await db.put('k', 'k'); await withF([db.transaction()], async ([tran]) => { - await tran.put([], 'a', '1'); - await tran.put([], 'c', '3'); - await tran.put([], 'e', '5'); - await tran.put([], 'f', '6'); - await tran.put([], 'j', '10'); - await tran.put([], 'k', '11'); - const iterator = await tran.iterator(); - for await (const [k, v] of iterator) { + await tran.put('a', '1'); + await tran.put('c', '3'); + await tran.put('e', '5'); + await tran.put('f', '6'); + await tran.put('j', '10'); + await tran.put('k', '11'); + for await (const [k, v] of tran.iterator()) { results.push([k.toString(), JSON.parse(v.toString())]); } }); @@ -276,21 +273,20 @@ describe(DBTransaction.name, () => { | k | k = k | k = 11 | k = 11 | */ const results: Array<[string, string]> = []; - await db.put([], 'a', 'a'); - await db.put([], 'b', 'b'); - await db.put([], 'd', 'd'); - await db.put([], 'e', 'e'); - await db.put([], 'h', 'h'); - await db.put([], 'k', 'k'); + await db.put('a', 'a'); + await db.put('b', 'b'); + await db.put('d', 'd'); + await db.put('e', 'e'); + await db.put('h', 'h'); + await db.put('k', 'k'); await withF([db.transaction()], async ([tran]) => { - await tran.put([], 'a', '1'); - await tran.put([], 'c', '3'); - await tran.put([], 'e', '5'); - await tran.put([], 'f', '6'); - await tran.put([], 'j', '10'); - await tran.put([], 'k', '11'); - const iterator = await tran.iterator({ reverse: true }); - for await (const [k, v] of iterator) { + await tran.put('a', '1'); + await tran.put('c', '3'); + await tran.put('e', '5'); + await tran.put('f', '6'); + await tran.put('j', '10'); + await tran.put('k', '11'); + for await (const [k, v] of tran.iterator({ reverse: true })) { results.push([k.toString(), JSON.parse(v.toString())]); } }); @@ -324,19 +320,18 @@ describe(DBTransaction.name, () => { | j | | j = 10 | j = 10 | */ const results: Array<[string, string]> = []; - await db.put([], 'a', 'a'); - await db.put([], 'b', 'b'); - await db.put([], 'd', 'd'); - await db.put([], 'e', 'e'); - await db.put([], 'h', 'h'); + await db.put('a', 'a'); + await db.put('b', 'b'); + await db.put('d', 'd'); + await db.put('e', 'e'); + await db.put('h', 'h'); await withF([db.transaction()], async ([tran]) => { - await tran.put([], 'a', '1'); - await tran.put([], 'c', '3'); - await tran.put([], 'e', '5'); - await tran.put([], 'f', '6'); - await tran.put([], 'j', '10'); - const iterator = await tran.iterator(); - for await (const [k, v] of iterator) { + await tran.put('a', '1'); + await tran.put('c', '3'); + await tran.put('e', '5'); + await tran.put('f', '6'); + await tran.put('j', '10'); + for await (const [k, v] of tran.iterator()) { results.push([k.toString(), JSON.parse(v.toString())]); } }); @@ -367,19 +362,18 @@ describe(DBTransaction.name, () => { | j | | j = 10 | j = 10 | */ const results: Array<[string, string]> = []; - await db.put([], 'a', 'a'); - await db.put([], 'b', 'b'); - await db.put([], 'd', 'd'); - await db.put([], 'e', 'e'); - await db.put([], 'h', 'h'); + await db.put('a', 'a'); + await db.put('b', 'b'); + await db.put('d', 'd'); + await db.put('e', 'e'); + await db.put('h', 'h'); await withF([db.transaction()], async ([tran]) => { - await tran.put([], 'a', '1'); - await tran.put([], 'c', '3'); - await tran.put([], 'e', '5'); - await tran.put([], 'f', '6'); - await tran.put([], 'j', '10'); - const iterator = await tran.iterator({ reverse: true }); - for await (const [k, v] of iterator) { + await tran.put('a', '1'); + await tran.put('c', '3'); + await tran.put('e', '5'); + await tran.put('f', '6'); + await tran.put('j', '10'); + for await (const [k, v] of tran.iterator({ reverse: true })) { results.push([k.toString(), JSON.parse(v.toString())]); } }); @@ -410,18 +404,17 @@ describe(DBTransaction.name, () => { | h | h = h | | h = h | */ const results: Array<[string, string]> = []; - await db.put([], 'a', 'a'); - await db.put([], 'b', 'b'); - await db.put([], 'd', 'd'); - await db.put([], 'e', 'e'); - await db.put([], 'h', 'h'); + await db.put('a', 'a'); + await db.put('b', 'b'); + await db.put('d', 'd'); + await db.put('e', 'e'); + await db.put('h', 'h'); await withF([db.transaction()], async ([tran]) => { - await tran.put([], 'a', '1'); - await tran.put([], 'c', '3'); - await tran.put([], 'e', '5'); - await tran.put([], 'f', '6'); - const iterator = await tran.iterator(); - for await (const [k, v] of iterator) { + await tran.put('a', '1'); + await tran.put('c', '3'); + await tran.put('e', '5'); + await tran.put('f', '6'); + for await (const [k, v] of tran.iterator()) { results.push([k.toString(), JSON.parse(v.toString())]); } }); @@ -449,18 +442,17 @@ describe(DBTransaction.name, () => { | h | h = h | | h = h | */ const results: Array<[string, string]> = []; - await db.put([], 'a', 'a'); - await db.put([], 'b', 'b'); - await db.put([], 'd', 'd'); - await db.put([], 'e', 'e'); - await db.put([], 'h', 'h'); + await db.put('a', 'a'); + await db.put('b', 'b'); + await db.put('d', 'd'); + await db.put('e', 'e'); + await db.put('h', 'h'); await withF([db.transaction()], async ([tran]) => { - await tran.put([], 'a', '1'); - await tran.put([], 'c', '3'); - await tran.put([], 'e', '5'); - await tran.put([], 'f', '6'); - const iterator = await tran.iterator({ reverse: true }); - for await (const [k, v] of iterator) { + await tran.put('a', '1'); + await tran.put('c', '3'); + await tran.put('e', '5'); + await tran.put('f', '6'); + for await (const [k, v] of tran.iterator({ reverse: true })) { results.push([k.toString(), JSON.parse(v.toString())]); } }); @@ -493,21 +485,20 @@ describe(DBTransaction.name, () => { | k | k = k | k = 11 | k = 11 | */ const results: Array<[string, undefined]> = []; - await db.put([], 'a', 'a'); - await db.put([], 'b', 'b'); - await db.put([], 'd', 'd'); - await db.put([], 'e', 'e'); - await db.put([], 'h', 'h'); - await db.put([], 'k', 'k'); + await db.put('a', 'a'); + await db.put('b', 'b'); + await db.put('d', 'd'); + await db.put('e', 'e'); + await db.put('h', 'h'); + await db.put('k', 'k'); await withF([db.transaction()], async ([tran]) => { - await tran.put([], 'a', '1'); - await tran.put([], 'c', '3'); - await tran.put([], 'e', '5'); - await tran.put([], 'f', '6'); - await tran.put([], 'j', '10'); - await tran.put([], 'k', '11'); - const iterator = await tran.iterator({ values: false }); - for await (const [k, v] of iterator) { + await tran.put('a', '1'); + await tran.put('c', '3'); + await tran.put('e', '5'); + await tran.put('f', '6'); + await tran.put('j', '10'); + await tran.put('k', '11'); + for await (const [k, v] of tran.iterator({ values: false })) { results.push([k.toString(), v]); } }); @@ -539,19 +530,19 @@ describe(DBTransaction.name, () => { | j | | j = 10 | j = 10 | | k | k = k | k = 11 | k = 11 | */ - await db.put([], 'a', 'a'); - await db.put([], 'b', 'b'); - await db.put([], 'd', 'd'); - await db.put([], 'e', 'e'); - await db.put([], 'h', 'h'); - await db.put([], 'k', 'k'); + await db.put('a', 'a'); + await db.put('b', 'b'); + await db.put('d', 'd'); + await db.put('e', 'e'); + await db.put('h', 'h'); + await db.put('k', 'k'); await withF([db.transaction()], async ([tran]) => { - await tran.put([], 'a', '1'); - await tran.put([], 'c', '3'); - await tran.put([], 'e', '5'); - await tran.put([], 'f', '6'); - await tran.put([], 'j', '10'); - await tran.put([], 'k', '11'); + await tran.put('a', '1'); + await tran.put('c', '3'); + await tran.put('e', '5'); + await tran.put('f', '6'); + await tran.put('j', '10'); + await tran.put('k', '11'); const iterator = await tran.iterator(); iterator.seek('a'); expect(await iterator.next()).toStrictEqual([ @@ -626,19 +617,19 @@ describe(DBTransaction.name, () => { expect(results).toStrictEqual([1, 2]); }); test('rollback on error', async () => { - await db.put([], '1', 'a'); - await db.put([], '2', 'b'); + await db.put('1', 'a'); + await db.put('2', 'b'); const mockFailure = jest.fn(); await expect( withF([db.transaction()], async ([tran]) => { - await tran.put([], '1', '1'); - await tran.put([], '2', '2'); + await tran.put('1', '1'); + await tran.put('2', '2'); tran.queueFailure(mockFailure); throw new Error('Oh no!'); }), ).rejects.toThrow('Oh no!'); expect(mockFailure).toBeCalled(); - expect(await db.get([], '1')).toBe('a'); - expect(await db.get([], '2')).toBe('b'); + expect(await db.get('1')).toBe('a'); + expect(await db.get('2')).toBe('b'); }); }); diff --git a/tests/utils.test.ts b/tests/utils.test.ts new file mode 100644 index 00000000..ba17a64b --- /dev/null +++ b/tests/utils.test.ts @@ -0,0 +1,23 @@ +import type { KeyPath } from '@/types'; +import * as utils from '@/utils'; + +describe('utils', () => { + test('parse key paths', () => { + // The key actual is allowed to contain the separator buffer + // However levels are not allowed for this + const keyPaths: Array = [ + ['foo', 'bar', Buffer.concat([utils.sep, Buffer.from('key'), utils.sep])], + [utils.sep], + [Buffer.concat([utils.sep, Buffer.from('foobar')])], + [Buffer.concat([Buffer.from('foobar'), utils.sep])], + [Buffer.concat([utils.sep, Buffer.from('foobar'), utils.sep])], + ]; + for (const keyPath of keyPaths) { + const key = utils.keyPathToKey(keyPath); + const keyPath_ = utils.parseKey(key); + expect(keyPath.map((b) => b.toString())).toStrictEqual( + keyPath_.map((b) => b.toString()), + ); + } + }); +});