From ee9f774fe0f3c4e573d12ce346a0081f0d6e6886 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Fri, 15 Mar 2024 17:25:13 +0100 Subject: [PATCH 1/2] chore(cli-repl): add snapshot package list tests Add the ability to introspect the compiled mongosh executables for what is part of the snapshot and what is not, and then use that information in e2e tests to verify that it is accurate. --- packages/cli-repl/package.json | 4 +- packages/cli-repl/src/run.ts | 1 + .../cli-repl/src/webpack-self-inspection.ts | 47 ++++++ packages/cli-repl/webpack.config.js | 18 ++- packages/e2e-tests/test/e2e-snapshot.spec.ts | 143 ++++++++++++++++++ packages/e2e-tests/test/test-shell.ts | 29 ++-- ...ack-enable-reverse-module-lookup-plugin.js | 30 ++++ 7 files changed, 258 insertions(+), 14 deletions(-) create mode 100644 packages/cli-repl/src/webpack-self-inspection.ts create mode 100644 packages/e2e-tests/test/e2e-snapshot.spec.ts create mode 100644 scripts/webpack-enable-reverse-module-lookup-plugin.js diff --git a/packages/cli-repl/package.json b/packages/cli-repl/package.json index 38569fa87..3b47f8072 100644 --- a/packages/cli-repl/package.json +++ b/packages/cli-repl/package.json @@ -31,8 +31,8 @@ "check": "npm run lint && npm run depcheck", "depcheck": "depcheck", "prepublish": "npm run compile", - "webpack-build": "npm run compile && webpack --mode production", - "webpack-build-dev": "npm run compile && webpack --mode development", + "webpack-build": "npm run compile && webpack --mode production && cat dist/add-module-mapping.js >> dist/mongosh.js", + "webpack-build-dev": "npm run compile && webpack --mode development && cat dist/add-module-mapping.js >> dist/mongosh.js", "start-snapshot": "rm -f snapshot.blob && node --snapshot-blob snapshot.blob --build-snapshot dist/mongosh.js && node --snapshot-blob snapshot.blob dist/mongosh.js", "prettier": "prettier", "reformat": "npm run prettier -- --write . && npm run eslint --fix" diff --git a/packages/cli-repl/src/run.ts b/packages/cli-repl/src/run.ts index e5fb7cde1..9ef5fe304 100644 --- a/packages/cli-repl/src/run.ts +++ b/packages/cli-repl/src/run.ts @@ -30,6 +30,7 @@ import crypto from 'crypto'; import net from 'net'; import v8 from 'v8'; import { TimingCategories } from '@mongosh/types'; +import './webpack-self-inspection'; // TS does not yet have type definitions for v8.startupSnapshot if ((v8 as any)?.startupSnapshot?.isBuildingSnapshot?.()) { diff --git a/packages/cli-repl/src/webpack-self-inspection.ts b/packages/cli-repl/src/webpack-self-inspection.ts new file mode 100644 index 000000000..678ce0745 --- /dev/null +++ b/packages/cli-repl/src/webpack-self-inspection.ts @@ -0,0 +1,47 @@ +import v8 from 'v8'; + +// Allow us to inspect the set of loaded modules at a few interesting points in time, e.g. +// startup, snapshot, and after entering VM/REPL execution mode +declare const __webpack_module_cache__: Record | undefined; +declare const __webpack_modules__: Record | undefined; +declare const __webpack_reverse_module_lookup__: + | (() => Record) + | undefined; + +// Return all ids of modules loaded at the current time +function enumerateLoadedModules(): (string | number)[] | null { + if (typeof __webpack_module_cache__ !== 'undefined') { + return Object.keys(__webpack_module_cache__); + } + return null; +} +// Return all ids of modules that can be loaded/are known to webpack +function enumerateAllModules(): (string | number)[] | null { + if (typeof __webpack_modules__ !== 'undefined') { + return Object.keys(__webpack_modules__); + } + return null; +} +// Perform a reverse lookup to determine the "natural" name for a given +// module id (i.e. original filename, if available). +// Calling this the first time is potentially expensive. +function lookupNaturalModuleName(id: string | number): string | null { + let lookupTable = null; + if (typeof __webpack_reverse_module_lookup__ !== 'undefined') + lookupTable = __webpack_reverse_module_lookup__(); + return lookupTable?.[id] ?? null; +} +Object.defineProperty(process, '__mongosh_webpack_stats', { + value: { + enumerateLoadedModules, + enumerateAllModules, + lookupNaturalModuleName, + }, +}); +if ((v8 as any)?.startupSnapshot?.isBuildingSnapshot?.()) { + (v8 as any).startupSnapshot.addSerializeCallback(() => { + const atSnapshotTime = enumerateLoadedModules(); + (process as any).__mongosh_webpack_stats.enumerateSnapshotModules = () => + atSnapshotTime; + }); +} diff --git a/packages/cli-repl/webpack.config.js b/packages/cli-repl/webpack.config.js index f57999121..f5df4a1d8 100644 --- a/packages/cli-repl/webpack.config.js +++ b/packages/cli-repl/webpack.config.js @@ -5,6 +5,9 @@ const crypto = require('crypto'); const { merge } = require('webpack-merge'); const path = require('path'); const { WebpackDependenciesPlugin } = require('@mongodb-js/sbom-tools'); +const { + WebpackEnableReverseModuleLookupPlugin, +} = require('../../scripts/webpack-enable-reverse-module-lookup-plugin.js'); const baseWebpackConfig = require('../../config/webpack.base.config'); @@ -29,6 +32,11 @@ const webpackDependenciesPlugin = new WebpackDependenciesPlugin({ includeExternalProductionDependencies: true, }); +const enableReverseModuleLookupPlugin = + new WebpackEnableReverseModuleLookupPlugin({ + outputFilename: path.resolve(__dirname, 'dist', 'add-module-mapping.js'), + }); + /** @type import('webpack').Configuration */ const config = { output: { @@ -41,7 +49,7 @@ const config = { type: 'var', }, }, - plugins: [webpackDependenciesPlugin], + plugins: [webpackDependenciesPlugin, enableReverseModuleLookupPlugin], entry: './lib/run.js', resolve: { alias: { @@ -83,7 +91,13 @@ module.exports = merge(baseWebpackConfig, config); // startup that should depend on runtime state. function makeLazyForwardModule(pkg) { const S = JSON.stringify; - const tmpdir = path.resolve(__dirname, '..', 'tmp', 'lazy-webpack-modules'); + const tmpdir = path.resolve( + __dirname, + '..', + '..', + 'tmp', + 'lazy-webpack-modules' + ); fs.mkdirSync(tmpdir, { recursive: true }); const filename = path.join( tmpdir, diff --git a/packages/e2e-tests/test/e2e-snapshot.spec.ts b/packages/e2e-tests/test/e2e-snapshot.spec.ts new file mode 100644 index 000000000..578b6be23 --- /dev/null +++ b/packages/e2e-tests/test/e2e-snapshot.spec.ts @@ -0,0 +1,143 @@ +import { + skipIfApiStrict, + startSharedTestServer, +} from '../../../testing/integration-testing-hooks'; +import { TestShell } from './test-shell'; +import { expect } from 'chai'; + +const setDifference = (a: T[], b: T[]) => a.filter((e) => !b.includes(e)); +const expectIsSubset = (a: T[], b: T[]) => + expect(setDifference(a, b)).to.have.lengthOf(0); +const commonPrefix = (a: string, b: string): string => + a.startsWith(b) + ? b + : b.startsWith(a) + ? a + : b && commonPrefix(a, b.slice(0, -1)); + +describe('e2e startup banners', function () { + skipIfApiStrict(); + afterEach(TestShell.cleanup); + + const testServer = startSharedTestServer(); + + context('modules included in snapshots', function () { + it('includes the right modules at the right point in time', async function () { + if (!process.env.MONGOSH_TEST_EXECUTABLE_PATH) return this.skip(); + + const connectionString = await testServer.connectionString(); + const helperScript = ` + const S = process.__mongosh_webpack_stats; + const L = (list) => list.map(S.lookupNaturalModuleName).filter(name => name && !name.endsWith('.json')); + `; + const commonArgs = ['--quiet', '--json=relaxed', '--eval', helperScript]; + const argLists = [ + [...commonArgs, '--nodb', '--eval', 'L(S.enumerateAllModules())'], + [...commonArgs, '--nodb', '--eval', 'L(S.enumerateSnapshotModules())'], + [...commonArgs, '--nodb', '--eval', 'L(S.enumerateLoadedModules())'], + [ + ...commonArgs, + connectionString, + '--eval', + 'L(S.enumerateLoadedModules())', + ], + [ + ...commonArgs, + connectionString, + '--jsContext=repl', + '--eval', + 'L(S.enumerateLoadedModules())', + ], + ]; + const [ + all, + atSnapshotTime, + atNodbEvalTime, + atDbEvalTime, + atReplEvalTime, + ] = ( + await Promise.all( + argLists.map((args) => + TestShell.runAndGetOutputWithoutErrors({ args }) + ) + ) + ).map((output) => JSON.parse(output).sort() as string[]); + + // Ensure that: atSnapshotTime ⊆ atNodbEvalTime ⊆ atDbEvalTime ⊆ atReplEvalTime ⊆ all + expectIsSubset(atSnapshotTime, atNodbEvalTime); + expectIsSubset(atNodbEvalTime, atDbEvalTime); + expectIsSubset(atDbEvalTime, atReplEvalTime); + expectIsSubset(atReplEvalTime, all); + + const prefix = all.reduce(commonPrefix); + const stripPrefix = (s: string) => + s.startsWith(prefix) ? s.replace(prefix, '') : s; + + const categorized = [ + ...atSnapshotTime.map(stripPrefix).map((m) => [m, 'snapshot'] as const), + ...setDifference(atNodbEvalTime, atSnapshotTime) + .map(stripPrefix) + .map((m) => [m, 'nodb-eval'] as const), + ...setDifference(atDbEvalTime, atNodbEvalTime) + .map(stripPrefix) + .map((m) => [m, 'db-eval'] as const), + ...setDifference(atReplEvalTime, atDbEvalTime) + .map(stripPrefix) + .map((m) => [m, 'repl-eval'] as const), + ...setDifference(all, atReplEvalTime) + .map(stripPrefix) + .map((m) => [m, 'not-loaded'] as const), + ]; + + // This is very helpful for inspecting snapshotted contents manually: + // console.table(categorized.map(([m, c]) => [m.replace(prefix, ''), c])); + const verifyAllInCategoryMatch = ( + category: (typeof categorized)[number][1], + re: RegExp + ) => { + for (const [module, cat] of categorized) { + if (cat === category) { + expect(module).to.match( + re, + `Found unexpected '${module}' in category '${cat}'` + ); + } + } + }; + const verifyAllThatMatchAreInCategory = ( + category: (typeof categorized)[number][1], + re: RegExp + ) => { + for (const [module, cat] of categorized) { + if (re.test(module)) { + expect(cat).to.equal( + category, + `Expected '${module}' to be in category '${category}', actual category is '${cat}'` + ); + } + } + }; + + // The core test: Verify that in the categories beyond 'not loaded at all' + // and 'part of the snapshot', only a very specific set of modules is present, + // and that some modules are only in specific categories. + verifyAllInCategoryMatch('repl-eval', /^node_modules\/pretty-repl\//); + verifyAllInCategoryMatch( + 'db-eval', + /^node_modules\/(kerberos|os-dns-native|resolve-mongodb-srv)\// + ); + verifyAllInCategoryMatch( + 'nodb-eval', + /^node_modules\/(kerberos|mongodb-client-encryption)\// + ); + verifyAllThatMatchAreInCategory( + 'not-loaded', + /^node_modules\/(express|openid-client|qs|send|jose|execa|body-parser|@babel\/highlight|@babel\/code-frame)\// + ); + verifyAllThatMatchAreInCategory( + 'snapshot', + /^node_modules\/(@babel\/types|@babel\/traverse|@mongodb-js\/devtools-connect|mongodb)\/|^packages\// + ); + }); + }); +}); diff --git a/packages/e2e-tests/test/test-shell.ts b/packages/e2e-tests/test/test-shell.ts index 9cf384b88..70ebd8f90 100644 --- a/packages/e2e-tests/test/test-shell.ts +++ b/packages/e2e-tests/test/test-shell.ts @@ -30,22 +30,22 @@ function matches(str: string, pattern: string | RegExp): boolean { : pattern.test(str); } +export interface TestShellOptions { + args: string[]; + env?: Record; + removeSigintListeners?: boolean; + cwd?: string; + forceTerminal?: boolean; + consumeStdio?: boolean; +} + /** * Test shell helper class. */ export class TestShell { private static _openShells: TestShell[] = []; - static start( - options: { - args: string[]; - env?: Record; - removeSigintListeners?: boolean; - cwd?: string; - forceTerminal?: boolean; - consumeStdio?: boolean; - } = { args: [] } - ): TestShell { + static start(options: TestShellOptions = { args: [] }): TestShell { let shellProcess: ChildProcessWithoutNullStreams; let env = options.env || process.env; @@ -95,6 +95,15 @@ export class TestShell { return shell; } + static async runAndGetOutputWithoutErrors( + options: TestShellOptions + ): Promise { + const shell = this.start(options); + await shell.waitForExit(); + shell.assertNoErrors(); + return shell.output; + } + static async killall(): Promise { const exitPromises: Promise[] = []; while (TestShell._openShells.length) { diff --git a/scripts/webpack-enable-reverse-module-lookup-plugin.js b/scripts/webpack-enable-reverse-module-lookup-plugin.js new file mode 100644 index 000000000..4ca55b58e --- /dev/null +++ b/scripts/webpack-enable-reverse-module-lookup-plugin.js @@ -0,0 +1,30 @@ +'use strict'; +const path = require('path'); +const fs = require('fs'); +const zlib = require('zlib'); +const { promisify } = require('util'); + +class WebpackEnableReverseModuleLookupPlugin { + outputFilename; + constructor({ outputFilename }) { this.outputFilename = outputFilename; } + + apply(compiler) { + compiler.hooks.emit.tapPromise('EnableReverseModuleLookupPlugin', async(compilation) => { + const map = Object.create(null); + for (const module of compilation.modules) { + const id = compilation.chunkGraph.getModuleId(module); + if (id && module.resource) { + map[id] = module.resource; + } + } + const data = (await promisify(zlib.brotliCompress)(JSON.stringify(map))).toString('base64'); + await fs.promises.mkdir(path.dirname(this.outputFilename), { recursive: true }); + await fs.promises.writeFile(this.outputFilename, `function __webpack_reverse_module_lookup__() { + return __webpack_reverse_module_lookup__.data ??= JSON.parse( + require("zlib").brotliDecompressSync(Buffer.from(${JSON.stringify(data)}, 'base64'))); + }`); + }) + } +} + +module.exports = { WebpackEnableReverseModuleLookupPlugin }; From 71073d17a47b8531014a78013642cc7c9bfbc5f0 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 27 Mar 2024 17:29:11 +0100 Subject: [PATCH 2/2] fixup: normalize slashes to forward slashes for comparison --- packages/e2e-tests/test/e2e-snapshot.spec.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/e2e-tests/test/e2e-snapshot.spec.ts b/packages/e2e-tests/test/e2e-snapshot.spec.ts index 578b6be23..471e558ff 100644 --- a/packages/e2e-tests/test/e2e-snapshot.spec.ts +++ b/packages/e2e-tests/test/e2e-snapshot.spec.ts @@ -61,7 +61,11 @@ describe('e2e startup banners', function () { TestShell.runAndGetOutputWithoutErrors({ args }) ) ) - ).map((output) => JSON.parse(output).sort() as string[]); + ).map((output) => + (JSON.parse(output) as string[]) + .sort() + .map((pkg) => pkg.replace(/\\/g, '/')) + ); // Ensure that: atSnapshotTime ⊆ atNodbEvalTime ⊆ atDbEvalTime ⊆ atReplEvalTime ⊆ all expectIsSubset(atSnapshotTime, atNodbEvalTime);