From ee8f7b714226d913d15b0d30f2845fef391e6d0e Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 9 Aug 2022 12:10:41 +0200 Subject: [PATCH 1/6] just checking --- packages/gatsby/src/utils/__tests__/cache-lmdb.ts | 7 ++----- packages/gatsby/src/utils/cache-lmdb.ts | 11 +++++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/gatsby/src/utils/__tests__/cache-lmdb.ts b/packages/gatsby/src/utils/__tests__/cache-lmdb.ts index 0fc342121f057..9fd26660c86e2 100644 --- a/packages/gatsby/src/utils/__tests__/cache-lmdb.ts +++ b/packages/gatsby/src/utils/__tests__/cache-lmdb.ts @@ -13,12 +13,9 @@ describeWhenLMDB(`cache-lmdb`, () => { beforeAll(async () => { const { default: GatsbyCacheLmdb } = await import(`../cache-lmdb`) + cache = new GatsbyCacheLmdb({ name: `__test__` }).init() - const fileDir = path.join( - process.cwd(), - `.cache/caches-lmdb-${process.env.JEST_WORKER_ID}` - ) - await fs.emptyDir(fileDir) + await cache.reset() }) it(`it can be instantiated`, () => { diff --git a/packages/gatsby/src/utils/cache-lmdb.ts b/packages/gatsby/src/utils/cache-lmdb.ts index 98e94f45c1dc3..97ac7de5f6c79 100644 --- a/packages/gatsby/src/utils/cache-lmdb.ts +++ b/packages/gatsby/src/utils/cache-lmdb.ts @@ -65,6 +65,17 @@ export default class GatsbyCacheLmdb { return this.db } + public async reset(): Promise { + if (GatsbyCacheLmdb.store) { + await GatsbyCacheLmdb.store.close() + GatsbyCacheLmdb.store = undefined + } + + await fs.emptyDir(path.join(process.cwd(), `.cache/${cacheDbFile}`)) + + return this + } + async get(key): Promise { return this.getDb().get(key) } From 6b4eb899da42b1014b2c635d2b794f5b109b3c19 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 9 Aug 2022 14:14:18 +0200 Subject: [PATCH 2/6] how about now? --- .../gatsby/src/utils/__tests__/cache-lmdb.ts | 6 ++- packages/gatsby/src/utils/cache-lmdb.ts | 53 ++++++++++++------- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/packages/gatsby/src/utils/__tests__/cache-lmdb.ts b/packages/gatsby/src/utils/__tests__/cache-lmdb.ts index 9fd26660c86e2..9a45c3dc66303 100644 --- a/packages/gatsby/src/utils/__tests__/cache-lmdb.ts +++ b/packages/gatsby/src/utils/__tests__/cache-lmdb.ts @@ -12,10 +12,12 @@ describeWhenLMDB(`cache-lmdb`, () => { let cache beforeAll(async () => { - const { default: GatsbyCacheLmdb } = await import(`../cache-lmdb`) + const { default: GatsbyCacheLmdb, resetCache } = await import( + `../cache-lmdb` + ) + await resetCache() cache = new GatsbyCacheLmdb({ name: `__test__` }).init() - await cache.reset() }) it(`it can be instantiated`, () => { diff --git a/packages/gatsby/src/utils/cache-lmdb.ts b/packages/gatsby/src/utils/cache-lmdb.ts index 97ac7de5f6c79..e5faeb5e37b8a 100644 --- a/packages/gatsby/src/utils/cache-lmdb.ts +++ b/packages/gatsby/src/utils/cache-lmdb.ts @@ -13,8 +13,16 @@ const cacheDbFile = }` : `caches-lmdb` +const dbPath = path.join(process.cwd(), `.cache/${cacheDbFile}`) + +function getAlreadyOpenedStore(): RootDatabase | undefined { + if (!globalThis.__GATSBY_OPEN_ROOT_LMDBS) { + globalThis.__GATSBY_OPEN_ROOT_LMDBS = new Map() + } + return globalThis.__GATSBY_OPEN_ROOT_LMDBS.get(dbPath) +} + export default class GatsbyCacheLmdb { - private static store private db: Database | undefined private encoding: DatabaseOptions["encoding"] public readonly name: string @@ -44,15 +52,21 @@ export default class GatsbyCacheLmdb { } private static getStore(): RootDatabase { - if (!GatsbyCacheLmdb.store) { - GatsbyCacheLmdb.store = open({ - name: `root`, - path: path.join(process.cwd(), `.cache/${cacheDbFile}`), - compression: true, - maxDbs: 200, - }) + let rootDb = getAlreadyOpenedStore() + if (rootDb) { + return rootDb } - return GatsbyCacheLmdb.store + + rootDb = open({ + name: `root`, + path: path.join(dbPath), + compression: true, + maxDbs: 200, + }) + + globalThis.__GATSBY_OPEN_ROOT_LMDBS.set(dbPath, rootDb) + + return rootDb } private getDb(): Database { @@ -65,17 +79,6 @@ export default class GatsbyCacheLmdb { return this.db } - public async reset(): Promise { - if (GatsbyCacheLmdb.store) { - await GatsbyCacheLmdb.store.close() - GatsbyCacheLmdb.store = undefined - } - - await fs.emptyDir(path.join(process.cwd(), `.cache/${cacheDbFile}`)) - - return this - } - async get(key): Promise { return this.getDb().get(key) } @@ -89,3 +92,13 @@ export default class GatsbyCacheLmdb { return this.getDb().remove(key) as unknown as Promise } } + +export async function resetCache(): Promise { + const store = getAlreadyOpenedStore() + if (store) { + await store.close() + globalThis.__GATSBY_OPEN_ROOT_LMDBS.delete(dbPath) + } + + await fs.emptyDir(dbPath) +} From 285f050e8fdd45ec072f734647d9c6082409172e Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 9 Aug 2022 15:45:33 +0200 Subject: [PATCH 3/6] mdebug --- jest.config.js | 6 +----- packages/gatsby/src/utils/cache-lmdb.ts | 9 ++++++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/jest.config.js b/jest.config.js index 20102831a5ad8..31f2156e95bf3 100644 --- a/jest.config.js +++ b/jest.config.js @@ -55,11 +55,7 @@ module.exports = { }, snapshotSerializers: [`jest-serializer-path`], collectCoverageFrom: coverageDirs, - reporters: process.env.CI - ? [[`jest-silent-reporter`, { useDots: true, showPaths: true }]].concat( - useCoverage ? `jest-junit` : [] - ) - : [`default`].concat(useCoverage ? `jest-junit` : []), + reporters: [`default`].concat(useCoverage ? `jest-junit` : []), moduleFileExtensions: [`js`, `jsx`, `ts`, `tsx`, `json`], setupFiles: [`/.jestSetup.js`], setupFilesAfterEnv: [`jest-extended`], diff --git a/packages/gatsby/src/utils/cache-lmdb.ts b/packages/gatsby/src/utils/cache-lmdb.ts index e5faeb5e37b8a..841d2a1fad0dc 100644 --- a/packages/gatsby/src/utils/cache-lmdb.ts +++ b/packages/gatsby/src/utils/cache-lmdb.ts @@ -1,6 +1,6 @@ import { open, RootDatabase, Database, DatabaseOptions } from "lmdb" -import fs from "fs-extra" -import path from "path" +import * as fs from "fs-extra" +import * as path from "path" // Since the regular GatsbyCache saves to "caches" this should be "caches-lmdb" const cacheDbFile = @@ -57,9 +57,11 @@ export default class GatsbyCacheLmdb { return rootDb } + console.trace(`open ${dbPath}`) + rootDb = open({ name: `root`, - path: path.join(dbPath), + path: dbPath, compression: true, maxDbs: 200, }) @@ -96,6 +98,7 @@ export default class GatsbyCacheLmdb { export async function resetCache(): Promise { const store = getAlreadyOpenedStore() if (store) { + console.trace(`reset / close ${dbPath}`) await store.close() globalThis.__GATSBY_OPEN_ROOT_LMDBS.delete(dbPath) } From 8ffcf5aee569012aab715504200453be8ebeda73 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 9 Aug 2022 20:44:02 +0200 Subject: [PATCH 4/6] ugh --- jest.config.js | 1 + jest.environment.ts | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 jest.environment.ts diff --git a/jest.config.js b/jest.config.js index 31f2156e95bf3..cf1dac86ead75 100644 --- a/jest.config.js +++ b/jest.config.js @@ -59,4 +59,5 @@ module.exports = { moduleFileExtensions: [`js`, `jsx`, `ts`, `tsx`, `json`], setupFiles: [`/.jestSetup.js`], setupFilesAfterEnv: [`jest-extended`], + testEnvironment: `/jest.environment.ts`, } diff --git a/jest.environment.ts b/jest.environment.ts new file mode 100644 index 0000000000000..0b2fbde6f0a74 --- /dev/null +++ b/jest.environment.ts @@ -0,0 +1,33 @@ +/* + Jest is resetting `global` between test suites. This makes our global maps that track opened + dbs unusable. This custom environment is really just regular Node environment, but it does setup + global available in tests with SHARED maps within same process (different processes will use different + dbs, so that part is taken care of) +*/ +const NodeEnvironment = require(`jest-environment-node`) + +const GlobalsToPreserve = [`__GATSBY_OPEN_ROOT_LMDBS`, `__GATSBY_OPEN_LMDBS`] + +const preservedGlobals = new Map() + +class CustomEnvironment extends NodeEnvironment { + constructor(config, context) { + super(config, context) + } + + async setup(): Promise { + await super.setup() + for (const globalToCheck of GlobalsToPreserve) { + this.global[globalToCheck] = preservedGlobals.get(globalToCheck) + } + } + + async teardown(): Promise { + for (const globalToCheck of GlobalsToPreserve) { + preservedGlobals.set(globalToCheck, this.global[globalToCheck]) + } + await super.teardown() + } +} + +module.exports = CustomEnvironment From 8484d9b84594c881642181c150191f6558907cdf Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 10 Aug 2022 11:52:47 +0200 Subject: [PATCH 5/6] 2 --- jest.environment.ts | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/jest.environment.ts b/jest.environment.ts index 0b2fbde6f0a74..3746371ccf091 100644 --- a/jest.environment.ts +++ b/jest.environment.ts @@ -1,30 +1,20 @@ -/* - Jest is resetting `global` between test suites. This makes our global maps that track opened - dbs unusable. This custom environment is really just regular Node environment, but it does setup - global available in tests with SHARED maps within same process (different processes will use different - dbs, so that part is taken care of) -*/ const NodeEnvironment = require(`jest-environment-node`) -const GlobalsToPreserve = [`__GATSBY_OPEN_ROOT_LMDBS`, `__GATSBY_OPEN_LMDBS`] - -const preservedGlobals = new Map() - class CustomEnvironment extends NodeEnvironment { constructor(config, context) { super(config, context) } - async setup(): Promise { - await super.setup() - for (const globalToCheck of GlobalsToPreserve) { - this.global[globalToCheck] = preservedGlobals.get(globalToCheck) - } - } - async teardown(): Promise { - for (const globalToCheck of GlobalsToPreserve) { - preservedGlobals.set(globalToCheck, this.global[globalToCheck]) + // close open lmdbs after running test suite + // this prevent dangling open handles that sometimes cause problems + // particularly in windows tests (failures to move or delete a db file) + if (this.global.__GATSBY_OPEN_ROOT_LMDBS) { + for (const rootDb of this.global.__GATSBY_OPEN_ROOT_LMDBS.values()) { + await rootDb.clearAsync() + await rootDb.close() + } + this.global.__GATSBY_OPEN_ROOT_LMDBS = undefined } await super.teardown() } From b1af0a8445956d6ce21f23c36c41def90e025219 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 10 Aug 2022 12:20:43 +0200 Subject: [PATCH 6/6] 3 --- jest.config.js | 6 +++++- packages/gatsby/src/utils/cache-lmdb.ts | 3 --- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/jest.config.js b/jest.config.js index cf1dac86ead75..a516a2339dcdf 100644 --- a/jest.config.js +++ b/jest.config.js @@ -55,7 +55,11 @@ module.exports = { }, snapshotSerializers: [`jest-serializer-path`], collectCoverageFrom: coverageDirs, - reporters: [`default`].concat(useCoverage ? `jest-junit` : []), + reporters: process.env.CI + ? [[`jest-silent-reporter`, { useDots: true, showPaths: true }]].concat( + useCoverage ? `jest-junit` : [] + ) + : [`default`].concat(useCoverage ? `jest-junit` : []), moduleFileExtensions: [`js`, `jsx`, `ts`, `tsx`, `json`], setupFiles: [`/.jestSetup.js`], setupFilesAfterEnv: [`jest-extended`], diff --git a/packages/gatsby/src/utils/cache-lmdb.ts b/packages/gatsby/src/utils/cache-lmdb.ts index 841d2a1fad0dc..addac1cbc40b6 100644 --- a/packages/gatsby/src/utils/cache-lmdb.ts +++ b/packages/gatsby/src/utils/cache-lmdb.ts @@ -57,8 +57,6 @@ export default class GatsbyCacheLmdb { return rootDb } - console.trace(`open ${dbPath}`) - rootDb = open({ name: `root`, path: dbPath, @@ -98,7 +96,6 @@ export default class GatsbyCacheLmdb { export async function resetCache(): Promise { const store = getAlreadyOpenedStore() if (store) { - console.trace(`reset / close ${dbPath}`) await store.close() globalThis.__GATSBY_OPEN_ROOT_LMDBS.delete(dbPath) }