test(system-tests): support npm for test projects #20664
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,310 @@ | ||
| import fs from 'fs-extra' | ||
| import path from 'path' | ||
| import cachedir from 'cachedir' | ||
| import execa from 'execa' | ||
| import { cyTmpDir, projectPath, projects, root } from '../fixtures' | ||
| import { getYarnCommand } from './yarn' | ||
| import { getNpmCommand } from './npm' | ||
|
|
||
| type Dependencies = Record<string, string> | ||
|
|
||
| /** | ||
| * Type for package.json files for system-tests example projects. | ||
| */ | ||
| type SystemTestPkgJson = { | ||
| /** | ||
| * By default, scaffolding will run install if there is a `package.json`. | ||
| * This option, if set, disables that. | ||
| */ | ||
| _cySkipDepInstall?: boolean | ||
| /** | ||
| * Run the yarn v3-style install command instead of yarn v1-style. | ||
| */ | ||
| _cyYarnV311?: boolean | ||
| /** | ||
| * By default, the automatic install will not run postinstall scripts. This | ||
| * option, if set, will cause postinstall scripts to run for this project. | ||
| */ | ||
| _cyRunScripts?: boolean | ||
| dependencies?: Dependencies | ||
| devDependencies?: Dependencies | ||
| optionalDependencies?: Dependencies | ||
| } | ||
|
|
||
| const log = (...args) => console.log('📦', ...args) | ||
|
There was a problem hiding this comment. Nitpicking here, but a fun TS trick to get the correct type of const log = (...args: Parameters<typeof console.log>) => console.log('📦', ...args)There was a problem hiding this comment. Nice, thanks for the tip. I think I'll leave this one as-is since it's so simple and we use this pattern elsewhere but I'll keep |
||
|
|
||
| /** | ||
| * Given a package name, returns the path to the module directory on disk. | ||
| */ | ||
| function pathToPackage (pkg: string): string { | ||
| return path.dirname(require.resolve(`${pkg}/package.json`)) | ||
| } | ||
|
|
||
| async function ensureCacheDir (cacheDir: string) { | ||
| try { | ||
| await fs.stat(cacheDir) | ||
| } catch (err) { | ||
| log(`Creating a new node_modules cache dir at ${cacheDir}`) | ||
| await fs.mkdirp(cacheDir) | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Symlink the cached `node_modules` directory to the temp project directory's `node_modules`. | ||
| */ | ||
| async function symlinkNodeModulesFromCache (tmpNodeModulesDir: string, cacheDir: string): Promise<void> { | ||
| await fs.symlink(cacheDir, tmpNodeModulesDir, 'junction') | ||
|
|
||
| log(`node_modules symlink created at ${tmpNodeModulesDir}`) | ||
| } | ||
|
|
||
| /** | ||
| * Copy the cached `node_modules` to the temp project directory's `node_modules`. | ||
| * | ||
| * @returns a callback that will copy changed `node_modules` back to the cached `node_modules`. | ||
| */ | ||
| async function copyNodeModulesFromCache (tmpNodeModulesDir: string, cacheDir: string): Promise<() => Promise<void>> { | ||
| await fs.copy(cacheDir, tmpNodeModulesDir, { dereference: true }) | ||
|
|
||
| log(`node_modules copied to ${tmpNodeModulesDir} from cache dir ${cacheDir}`) | ||
|
|
||
| return async () => { | ||
| try { | ||
| await fs.copy(tmpNodeModulesDir, cacheDir, { dereference: true }) | ||
| } catch (err) { | ||
| if (err.message === 'Source and destination must not be the same') return | ||
|
|
||
| throw err | ||
| } | ||
|
|
||
| log(`node_modules copied from ${tmpNodeModulesDir} to cache dir ${cacheDir}`) | ||
| } | ||
| } | ||
|
|
||
| async function getLockFilename (dir: string) { | ||
| const hasYarnLock = !!await fs.stat(path.join(dir, 'yarn.lock')).catch(() => false) | ||
| const hasNpmLock = !!await fs.stat(path.join(dir, 'package-lock.json')).catch(() => false) | ||
|
|
||
| if (hasYarnLock && hasNpmLock) throw new Error(`The example project at '${dir}' has conflicting lockfiles. Only use one package manager's lockfile per project.`) | ||
|
|
||
| if (hasNpmLock) return 'package-lock.json' | ||
|
|
||
| // default to yarn | ||
| return 'yarn.lock' | ||
| } | ||
|
|
||
| function getRelativePathToProjectDir (projectDir: string) { | ||
| return path.relative(projectDir, path.join(root, '..')) | ||
| } | ||
|
|
||
| async function restoreLockFileRelativePaths (opts: { projectDir: string, lockFilePath: string, relativePathToMonorepoRoot: string }) { | ||
| const relativePathToProjectDir = getRelativePathToProjectDir(opts.projectDir) | ||
| const lockFileContents = (await fs.readFile(opts.lockFilePath, 'utf8')) | ||
| .replaceAll(opts.relativePathToMonorepoRoot, relativePathToProjectDir) | ||
|
|
||
| await fs.writeFile(opts.lockFilePath, lockFileContents) | ||
| } | ||
|
|
||
| async function normalizeLockFileRelativePaths (opts: { project: string, projectDir: string, lockFilePath: string, lockFilename: string, relativePathToMonorepoRoot: string }) { | ||
| const relativePathToProjectDir = getRelativePathToProjectDir(opts.projectDir) | ||
| const lockFileContents = (await fs.readFile(opts.lockFilePath, 'utf8')) | ||
| .replaceAll(relativePathToProjectDir, opts.relativePathToMonorepoRoot) | ||
|
|
||
| // write back to the original project dir, not the tmp copy | ||
| await fs.writeFile(path.join(projects, opts.project, opts.lockFilename), lockFileContents) | ||
| } | ||
|
|
||
| /** | ||
| * Given a path to a `package.json`, convert any references to development | ||
| * versions of packages to absolute paths, so `yarn`/`npm` will not reach out to | ||
| * the Internet to obtain these packages once it runs in the temp dir. | ||
| * @returns a list of dependency names that were updated | ||
| */ | ||
| async function makeWorkspacePackagesAbsolute (pathToPkgJson: string): Promise<string[]> { | ||
| const pkgJson = await fs.readJson(pathToPkgJson) | ||
| const updatedDeps: string[] = [] | ||
|
|
||
| for (const deps of [pkgJson.dependencies, pkgJson.devDependencies, pkgJson.optionalDependencies]) { | ||
| for (const dep in deps) { | ||
| const version = deps[dep] | ||
|
|
||
| if (version.startsWith('file:')) { | ||
| const absPath = pathToPackage(dep) | ||
|
|
||
| log(`Setting absolute path in package.json for ${dep}: ${absPath}.`) | ||
|
|
||
| deps[dep] = `file:${absPath}` | ||
| updatedDeps.push(dep) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| await fs.writeJson(pathToPkgJson, pkgJson) | ||
|
|
||
| return updatedDeps | ||
| } | ||
|
|
||
| /** | ||
| * Given a `system-tests` project name, detect and install the `node_modules` | ||
| * specified in the project's `package.json`. No-op if no `package.json` is found. | ||
| * Will use `yarn` or `npm` based on the lockfile present. | ||
| */ | ||
| export async function scaffoldProjectNodeModules (project: string, updateLockFile: boolean = !!process.env.UPDATE_LOCK_FILE): Promise<void> { | ||
| const projectDir = projectPath(project) | ||
| const relativePathToMonorepoRoot = path.relative( | ||
| path.join(projects, project), | ||
| path.join(root, '..'), | ||
| ) | ||
| const projectPkgJsonPath = path.join(projectDir, 'package.json') | ||
|
|
||
| const runCmd = async (cmd) => { | ||
| log(`Running "${cmd}" in ${projectDir}`) | ||
| await execa(cmd, { cwd: projectDir, stdio: 'inherit', shell: true }) | ||
| } | ||
|
|
||
| const cacheNodeModulesDir = path.join(cachedir('cy-system-tests-node-modules'), project, 'node_modules') | ||
| const tmpNodeModulesDir = path.join(projectPath(project), 'node_modules') | ||
|
|
||
| async function removeWorkspacePackages (packages: string[]): Promise<void> { | ||
| for (const dep of packages) { | ||
| const depDir = path.join(tmpNodeModulesDir, dep) | ||
|
|
||
| log('Removing', depDir) | ||
| await fs.remove(depDir) | ||
| } | ||
| } | ||
|
|
||
| try { | ||
| // this will throw and exit early if the package.json does not exist | ||
| const pkgJson: SystemTestPkgJson = require(projectPkgJsonPath) | ||
|
|
||
| log(`Found package.json for project ${project}.`) | ||
|
|
||
| if (pkgJson._cySkipDepInstall) { | ||
| return log(`_cySkipDepInstall set in package.json, skipping dep-installer steps`) | ||
| } | ||
|
|
||
| if (!pkgJson.dependencies && !pkgJson.devDependencies && !pkgJson.optionalDependencies) { | ||
| return log(`No dependencies found, skipping dep-installer steps`) | ||
| } | ||
|
|
||
| const lockFilename = await getLockFilename(projectDir) | ||
| const hasYarnLock = lockFilename === 'yarn.lock' | ||
|
|
||
| // 1. Ensure there is a cache directory set up for this test project's `node_modules`. | ||
| await ensureCacheDir(cacheNodeModulesDir) | ||
|
|
||
| let persistCacheCb: () => Promise<void> | ||
|
|
||
| if (hasYarnLock) { | ||
| await symlinkNodeModulesFromCache(tmpNodeModulesDir, cacheNodeModulesDir) | ||
| } else { | ||
| // due to an issue in NPM, we cannot have `node_modules` be a symlink. fall back to copying. | ||
| // https://github.com/npm/npm/issues/10013 | ||
|
There was a problem hiding this comment. Seems the linked issue is 3-4 years old - I wonder if/when this will be fixed, or if it's a wontfix. 😢 There was a problem hiding this comment. Yeah, I'm not sure, I unfortunately wasn't able to find a corresponding issue in their un-archived repo. But it's definitely still an issue. |
||
| persistCacheCb = await copyNodeModulesFromCache(tmpNodeModulesDir, cacheNodeModulesDir) | ||
| } | ||
|
|
||
| // 2. Before running the package installer, resolve workspace deps to absolute paths. | ||
| // This is required to fix install for workspace-only packages. | ||
| const workspaceDeps = await makeWorkspacePackagesAbsolute(projectPkgJsonPath) | ||
|
|
||
| // 3. Delete cached workspace packages since the pkg manager will create a fresh symlink during install. | ||
| await removeWorkspacePackages(workspaceDeps) | ||
|
|
||
| // 4. Fix relative paths in temp dir's lockfile. | ||
| const lockFilePath = path.join(projectDir, lockFilename) | ||
|
|
||
| log(`Writing ${lockFilename} with fixed relative paths to temp dir`) | ||
| await restoreLockFileRelativePaths({ projectDir, lockFilePath, relativePathToMonorepoRoot }) | ||
|
|
||
| // 5. Run `yarn/npm install`. | ||
| const getCommandFn = hasYarnLock ? getYarnCommand : getNpmCommand | ||
| const cmd = getCommandFn({ | ||
| updateLockFile, | ||
| yarnV311: pkgJson._cyYarnV311, | ||
| isCI: !!process.env.CI, | ||
| runScripts: pkgJson._cyRunScripts, | ||
| }) | ||
|
|
||
| await runCmd(cmd) | ||
|
|
||
| // 6. Now that the lockfile is up to date, update workspace dependency paths in the lockfile with monorepo | ||
| // relative paths so it can be the same for all developers | ||
| log(`Copying ${lockFilename} and fixing relative paths for ${project}`) | ||
| await normalizeLockFileRelativePaths({ project, projectDir, lockFilePath, lockFilename, relativePathToMonorepoRoot }) | ||
|
|
||
| // 7. After install, we must now symlink *over* all workspace dependencies, or else | ||
| // `require` calls from installed workspace deps to peer deps will fail. | ||
| await removeWorkspacePackages(workspaceDeps) | ||
| for (const dep of workspaceDeps) { | ||
| const destDir = path.join(tmpNodeModulesDir, dep) | ||
| const targetDir = pathToPackage(dep) | ||
|
|
||
| log(`Symlinking workspace dependency: ${dep} (${destDir} -> ${targetDir})`) | ||
|
|
||
| await fs.mkdir(path.dirname(destDir), { recursive: true }) | ||
| await fs.symlink(targetDir, destDir, 'junction') | ||
| } | ||
|
|
||
| // 8. If necessary, ensure that the `node_modules` cache is updated by copying `node_modules` back. | ||
|
There was a problem hiding this comment. Big 👍 to the comments here |
||
| if (persistCacheCb) await persistCacheCb() | ||
| } catch (err) { | ||
| if (err.code === 'MODULE_NOT_FOUND') return | ||
|
|
||
| console.error(`⚠ An error occurred while installing the node_modules for ${project}.`) | ||
| console.error(err) | ||
| throw err | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Create symlinks to very commonly used (in example projects) `node_modules`. | ||
| * | ||
| * This is done because many `projects` use the same modules, like `lodash`, and it's not worth it | ||
| * to increase CI install times just to have it explicitly specified by `package.json`. A symlink | ||
| * is faster than a real `npm install`. | ||
| * | ||
| * Adding modules here *decreases the quality of test coverage* because it allows test projects | ||
| * to make assumptions about what modules are available that don't hold true in the real world. So | ||
| * *do not add a module here* unless you are really sure that it should be available in every | ||
| * single test project. | ||
| */ | ||
| export async function scaffoldCommonNodeModules () { | ||
|
There was a problem hiding this comment. Can you add a comment above this line describing what it does? |
||
| await Promise.all([ | ||
| '@cypress/code-coverage', | ||
| '@cypress/webpack-dev-server', | ||
| '@packages/socket', | ||
| '@packages/ts', | ||
| '@tooling/system-tests', | ||
| 'bluebird', | ||
| 'chai', | ||
| 'dayjs', | ||
| 'debug', | ||
| 'execa', | ||
| 'fs-extra', | ||
| 'https-proxy-agent', | ||
| 'jimp', | ||
| 'lazy-ass', | ||
| 'lodash', | ||
| 'proxyquire', | ||
| 'react', | ||
| 'semver', | ||
| 'systeminformation', | ||
| 'tslib', | ||
| 'typescript', | ||
| ].map(symlinkNodeModule)) | ||
| } | ||
|
|
||
| async function symlinkNodeModule (pkg) { | ||
| const from = path.join(cyTmpDir, 'node_modules', pkg) | ||
| const to = pathToPackage(pkg) | ||
|
|
||
| await fs.ensureDir(path.dirname(from)) | ||
| try { | ||
| await fs.symlink(to, from, 'junction') | ||
| } catch (err) { | ||
| if (err.code === 'EEXIST') return | ||
|
|
||
| throw err | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
not necessarily related to this PR, but I noticed sometimes vscode doesn't know what our tab size is (where does that even come from? eslint?) and will default to 4.