diff --git a/packages/metro-file-map/src/HasteFS.js b/packages/metro-file-map/src/HasteFS.js new file mode 100644 index 0000000000..8f743a8920 --- /dev/null +++ b/packages/metro-file-map/src/HasteFS.js @@ -0,0 +1,104 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @flow strict-local + */ + +import type {FileData, Path} from './flow-types'; + +import H from './constants'; +import * as fastPath from './lib/fast_path'; +// $FlowFixMe[untyped-import] - jest-util +import {globsToMatcher, replacePathSepForGlob} from 'jest-util'; + +// $FlowFixMe[unclear-type] - Check TS Config.Glob +type Glob = any; + +export default class HasteFS { + +_rootDir: Path; + +_files: FileData; + + constructor({rootDir, files}: {rootDir: Path, files: FileData}) { + this._rootDir = rootDir; + this._files = files; + } + + getModuleName(file: Path): ?string { + const fileMetadata = this._getFileData(file); + return (fileMetadata && fileMetadata[H.ID]) || null; + } + + getSize(file: Path): ?number { + const fileMetadata = this._getFileData(file); + return (fileMetadata && fileMetadata[H.SIZE]) || null; + } + + getDependencies(file: Path): ?Array { + const fileMetadata = this._getFileData(file); + + if (fileMetadata) { + return fileMetadata[H.DEPENDENCIES] + ? fileMetadata[H.DEPENDENCIES].split(H.DEPENDENCY_DELIM) + : []; + } else { + return null; + } + } + + getSha1(file: Path): ?string { + const fileMetadata = this._getFileData(file); + return (fileMetadata && fileMetadata[H.SHA1]) ?? null; + } + + exists(file: Path): boolean { + return this._getFileData(file) != null; + } + + getAllFiles(): Array { + return Array.from(this.getAbsoluteFileIterator()); + } + + getFileIterator(): Iterable { + return this._files.keys(); + } + + *getAbsoluteFileIterator(): Iterable { + for (const file of this.getFileIterator()) { + yield fastPath.resolve(this._rootDir, file); + } + } + + matchFiles(pattern: RegExp | string): Array { + const regexpPattern = + pattern instanceof RegExp ? pattern : new RegExp(pattern); + const files = []; + for (const file of this.getAbsoluteFileIterator()) { + if (regexpPattern.test(file)) { + files.push(file); + } + } + return files; + } + + matchFilesWithGlob(globs: $ReadOnlyArray, root: ?Path): Set { + const files = new Set(); + const matcher = globsToMatcher(globs); + + for (const file of this.getAbsoluteFileIterator()) { + const filePath = root != null ? fastPath.relative(root, file) : file; + if (matcher(replacePathSepForGlob(filePath))) { + files.add(file); + } + } + return files; + } + + _getFileData(file: Path) { + const relativePath = fastPath.relative(this._rootDir, file); + return this._files.get(relativePath); + } +} diff --git a/packages/metro-file-map/src/ModuleMap.js b/packages/metro-file-map/src/ModuleMap.js new file mode 100644 index 0000000000..cb035f3107 --- /dev/null +++ b/packages/metro-file-map/src/ModuleMap.js @@ -0,0 +1,269 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + */ + +import type { + DuplicatesSet, + HTypeValue, + IModuleMap, + ModuleMetaData, + Path, + RawModuleMap, + SerializableModuleMap, +} from './flow-types'; + +import H from './constants'; +import * as fastPath from './lib/fast_path'; + +const EMPTY_OBJ: {[string]: ModuleMetaData} = {}; +const EMPTY_MAP = new Map(); + +export default class ModuleMap implements IModuleMap { + static DuplicateHasteCandidatesError: typeof DuplicateHasteCandidatesError; + +_raw: RawModuleMap; + _json: ?SerializableModuleMap; + + // $FlowFixMe[unclear-type] - Refactor away this function + static _mapToArrayRecursive(map: Map): Array<[string, any]> { + let arr = Array.from(map); + if (arr[0] && arr[0][1] instanceof Map) { + arr = arr.map( + // $FlowFixMe[unclear-type] - Refactor away this function + el => ([el[0], this._mapToArrayRecursive(el[1])]: [string, any]), + ); + } + return arr; + } + + static _mapFromArrayRecursive( + // $FlowFixMe[unclear-type] - Refactor away this function + arr: $ReadOnlyArray<[string, any]>, + // $FlowFixMe[unclear-type] - Refactor away this function + ): Map { + if (arr[0] && Array.isArray(arr[1])) { + // $FlowFixMe[reassign-const] - Refactor away this function + arr = (arr.map(el => [ + el[0], + // $FlowFixMe[unclear-type] - Refactor away this function + this._mapFromArrayRecursive((el[1]: Array<[string, any]>)), + // $FlowFixMe[unclear-type] - Refactor away this function + ]): Array<[string, any]>); + } + return new Map(arr); + } + + constructor(raw: RawModuleMap) { + this._raw = raw; + } + + getModule( + name: string, + platform?: ?string, + supportsNativePlatform?: ?boolean, + type?: ?HTypeValue, + ): ?Path { + const module = this._getModuleMetadata( + name, + platform, + !!supportsNativePlatform, + ); + if (module && module[H.TYPE] === (type ?? H.MODULE)) { + const modulePath = module[H.PATH]; + return modulePath && fastPath.resolve(this._raw.rootDir, modulePath); + } + return null; + } + + getPackage( + name: string, + platform: ?string, + _supportsNativePlatform?: ?boolean, + ): ?Path { + return this.getModule(name, platform, null, H.PACKAGE); + } + + getMockModule(name: string): ?Path { + const mockPath = + this._raw.mocks.get(name) || this._raw.mocks.get(name + '/index'); + return mockPath != null + ? fastPath.resolve(this._raw.rootDir, mockPath) + : null; + } + + getRawModuleMap(): RawModuleMap { + return { + duplicates: this._raw.duplicates, + map: this._raw.map, + mocks: this._raw.mocks, + rootDir: this._raw.rootDir, + }; + } + + toJSON(): SerializableModuleMap { + if (!this._json) { + this._json = { + duplicates: (ModuleMap._mapToArrayRecursive( + this._raw.duplicates, + ): SerializableModuleMap['duplicates']), + map: Array.from(this._raw.map), + mocks: Array.from(this._raw.mocks), + rootDir: this._raw.rootDir, + }; + } + return this._json; + } + + static fromJSON(serializableModuleMap: SerializableModuleMap): ModuleMap { + return new ModuleMap({ + duplicates: (ModuleMap._mapFromArrayRecursive( + serializableModuleMap.duplicates, + ): RawModuleMap['duplicates']), + map: new Map(serializableModuleMap.map), + mocks: new Map(serializableModuleMap.mocks), + rootDir: serializableModuleMap.rootDir, + }); + } + + /** + * When looking up a module's data, we walk through each eligible platform for + * the query. For each platform, we want to check if there are known + * duplicates for that name+platform pair. The duplication logic normally + * removes elements from the `map` object, but we want to check upfront to be + * extra sure. If metadata exists both in the `duplicates` object and the + * `map`, this would be a bug. + */ + _getModuleMetadata( + name: string, + platform: ?string, + supportsNativePlatform: boolean, + ): ModuleMetaData | null { + const map = this._raw.map.get(name) || EMPTY_OBJ; + const dupMap = this._raw.duplicates.get(name) || EMPTY_MAP; + if (platform != null) { + this._assertNoDuplicates( + name, + platform, + supportsNativePlatform, + dupMap.get(platform), + ); + if (map[platform] != null) { + return map[platform]; + } + } + if (supportsNativePlatform) { + this._assertNoDuplicates( + name, + H.NATIVE_PLATFORM, + supportsNativePlatform, + dupMap.get(H.NATIVE_PLATFORM), + ); + if (map[H.NATIVE_PLATFORM]) { + return map[H.NATIVE_PLATFORM]; + } + } + this._assertNoDuplicates( + name, + H.GENERIC_PLATFORM, + supportsNativePlatform, + dupMap.get(H.GENERIC_PLATFORM), + ); + if (map[H.GENERIC_PLATFORM]) { + return map[H.GENERIC_PLATFORM]; + } + return null; + } + + _assertNoDuplicates( + name: string, + platform: string, + supportsNativePlatform: boolean, + relativePathSet: ?DuplicatesSet, + ) { + if (relativePathSet == null) { + return; + } + // Force flow refinement + const previousSet = relativePathSet; + const duplicates = new Map(); + + for (const [relativePath, type] of previousSet) { + const duplicatePath = fastPath.resolve(this._raw.rootDir, relativePath); + duplicates.set(duplicatePath, type); + } + + throw new DuplicateHasteCandidatesError( + name, + platform, + supportsNativePlatform, + duplicates, + ); + } + + static create(rootDir: Path): ModuleMap { + return new ModuleMap({ + duplicates: new Map(), + map: new Map(), + mocks: new Map(), + rootDir, + }); + } +} + +class DuplicateHasteCandidatesError extends Error { + hasteName: string; + platform: string | null; + supportsNativePlatform: boolean; + duplicatesSet: DuplicatesSet; + + constructor( + name: string, + platform: string, + supportsNativePlatform: boolean, + duplicatesSet: DuplicatesSet, + ) { + const platformMessage = getPlatformMessage(platform); + super( + `The name \`${name}\` was looked up in the Haste module map. It ` + + 'cannot be resolved, because there exists several different ' + + 'files, or packages, that provide a module for ' + + `that particular name and platform. ${platformMessage} You must ` + + 'delete or exclude files until there remains only one of these:\n\n' + + Array.from(duplicatesSet) + .map( + ([dupFilePath, dupFileType]) => + ` * \`${dupFilePath}\` (${getTypeMessage(dupFileType)})\n`, + ) + .sort() + .join(''), + ); + this.hasteName = name; + this.platform = platform; + this.supportsNativePlatform = supportsNativePlatform; + this.duplicatesSet = duplicatesSet; + } +} + +function getPlatformMessage(platform: string) { + if (platform === H.GENERIC_PLATFORM) { + return 'The platform is generic (no extension).'; + } + return `The platform extension is \`${platform}\`.`; +} + +function getTypeMessage(type: number) { + switch (type) { + case H.MODULE: + return 'module'; + case H.PACKAGE: + return 'package'; + } + return 'unknown'; +} + +ModuleMap.DuplicateHasteCandidatesError = DuplicateHasteCandidatesError; diff --git a/packages/metro-file-map/src/__tests__/__snapshots__/index-test.js.snap b/packages/metro-file-map/src/__tests__/__snapshots__/index-test.js.snap new file mode 100644 index 0000000000..8de2cfeed9 --- /dev/null +++ b/packages/metro-file-map/src/__tests__/__snapshots__/index-test.js.snap @@ -0,0 +1,39 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`HasteMap file system changes processing recovery from duplicate module IDs recovers when the most recent duplicate is fixed 1`] = ` +"The name \`Pear\` was looked up in the Haste module map. It cannot be resolved, because there exists several different files, or packages, that provide a module for that particular name and platform. The platform is generic (no extension). You must delete or exclude files until there remains only one of these: + + * \`/project/fruits/Pear.js\` (module) + * \`/project/fruits/another/Pear.js\` (module) +" +`; + +exports[`HasteMap file system changes processing recovery from duplicate module IDs recovers when the oldest version of the duplicates is fixed 1`] = ` +"The name \`Pear\` was looked up in the Haste module map. It cannot be resolved, because there exists several different files, or packages, that provide a module for that particular name and platform. The platform is generic (no extension). You must delete or exclude files until there remains only one of these: + + * \`/project/fruits/Pear.js\` (module) + * \`/project/fruits/another/Pear.js\` (module) +" +`; + +exports[`HasteMap tries to crawl using node as a fallback 1`] = ` +"jest-haste-map: Watchman crawl failed. Retrying once with node crawler. + Usually this happens when watchman isn't running. Create an empty \`.watchmanconfig\` file in your project's root folder or initialize a git or hg repository in your project. + Error: watchman error" +`; + +exports[`HasteMap warns on duplicate mock files 1`] = ` +"jest-haste-map: duplicate manual mock found: subdir/Blueberry + The following files share their name; please delete one of them: + * /fruits1/__mocks__/subdir/Blueberry.js + * /fruits2/__mocks__/subdir/Blueberry.js +" +`; + +exports[`HasteMap warns on duplicate module ids 1`] = ` +"jest-haste-map: Haste module naming collision: Strawberry + The following files share their name; please adjust your hasteImpl: + * /fruits/Strawberry.js + * /fruits/other/Strawberry.js +" +`; diff --git a/packages/metro-file-map/src/__tests__/dependencyExtractor.js b/packages/metro-file-map/src/__tests__/dependencyExtractor.js new file mode 100644 index 0000000000..a8030ec96a --- /dev/null +++ b/packages/metro-file-map/src/__tests__/dependencyExtractor.js @@ -0,0 +1,40 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +const blockCommentRe = /\/\*[^]*?\*\//g; +const lineCommentRe = /\/\/.*/g; +const LOAD_MODULE_RE = + /(?:^|[^.]\s*)(\bloadModule\s*?\(\s*?)([`'"])([^`'"]+)(\2\s*?\))/g; + +export function extract(code, filePath, defaultDependencyExtractor) { + const dependencies = defaultDependencyExtractor(code); + + const addDependency = (match, pre, quot, dep, post) => { + dependencies.add(dep); + return match; + }; + + code + .replace(blockCommentRe, '') + .replace(lineCommentRe, '') + .replace(LOAD_MODULE_RE, addDependency); + + return dependencies; +} + +let cacheKey; + +export function getCacheKey() { + return cacheKey; +} + +export function setCacheKey(key) { + cacheKey = key; +} diff --git a/packages/metro-file-map/src/__tests__/getMockName-test.js b/packages/metro-file-map/src/__tests__/getMockName-test.js new file mode 100644 index 0000000000..79322b985a --- /dev/null +++ b/packages/metro-file-map/src/__tests__/getMockName-test.js @@ -0,0 +1,22 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +import getMockName from '../getMockName'; +import path from 'path'; + +describe('getMockName', () => { + it('extracts mock name from file path', () => { + expect(getMockName(path.join('a', '__mocks__', 'c.js'))).toBe('c'); + + expect(getMockName(path.join('a', '__mocks__', 'c', 'd.js'))).toBe( + path.join('c', 'd').replace(/\\/g, '/'), + ); + }); +}); diff --git a/packages/metro-file-map/src/__tests__/haste_impl.js b/packages/metro-file-map/src/__tests__/haste_impl.js new file mode 100644 index 0000000000..c1dad25ff7 --- /dev/null +++ b/packages/metro-file-map/src/__tests__/haste_impl.js @@ -0,0 +1,39 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +'use strict'; + +const path = require('path'); +let cacheKey; + +module.exports = { + getCacheKey() { + return cacheKey; + }, + + getHasteName(filename) { + if ( + filename.includes('__mocks__') || + filename.includes('NoHaste') || + filename.includes(path.sep + 'module_dir' + path.sep) || + filename.includes(path.sep + 'sourcemaps' + path.sep) + ) { + return undefined; + } + + return filename + .substr(filename.lastIndexOf(path.sep) + 1) + .replace(/(\.(android|ios|native))?\.js$/, ''); + }, + + setCacheKey(key) { + cacheKey = key; + }, +}; diff --git a/packages/metro-file-map/src/__tests__/includes_dotfiles-test.js b/packages/metro-file-map/src/__tests__/includes_dotfiles-test.js new file mode 100644 index 0000000000..28f8aaece7 --- /dev/null +++ b/packages/metro-file-map/src/__tests__/includes_dotfiles-test.js @@ -0,0 +1,51 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +import HasteMap from '../index'; +import path from 'path'; + +const rootDir = path.join(__dirname, './test_dotfiles_root'); + +const commonOptions = { + extensions: ['js'], + maxWorkers: 1, + platforms: [], + resetCache: true, + retainAllFiles: true, + rootDir, + roots: [rootDir], +}; + +test('watchman crawler and node crawler both include dotfiles', async () => { + const hasteMapWithWatchman = new HasteMap({ + ...commonOptions, + name: 'withWatchman', + useWatchman: true, + }); + + const hasteMapWithNode = new HasteMap({ + ...commonOptions, + name: 'withNode', + useWatchman: false, + }); + + const [builtHasteMapWithWatchman, builtHasteMapWithNode] = await Promise.all([ + hasteMapWithWatchman.build(), + hasteMapWithNode.build(), + ]); + + expect( + builtHasteMapWithWatchman.hasteFS.matchFiles('.eslintrc.js'), + ).toHaveLength(1); + + expect(builtHasteMapWithWatchman.hasteFS.getAllFiles().sort()).toEqual( + builtHasteMapWithNode.hasteFS.getAllFiles().sort(), + ); +}); diff --git a/packages/metro-file-map/src/__tests__/index-test.js b/packages/metro-file-map/src/__tests__/index-test.js new file mode 100644 index 0000000000..98f2f3c94f --- /dev/null +++ b/packages/metro-file-map/src/__tests__/index-test.js @@ -0,0 +1,1746 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +import crypto from 'crypto'; +import * as path from 'path'; + +jest.useRealTimers(); + +function mockHashContents(contents) { + return crypto.createHash('sha1').update(contents).digest('hex'); +} + +jest.mock('child_process', () => ({ + // If this does not throw, we'll use the (mocked) watchman crawler + execSync() {}, +})); + +jest.mock('jest-worker', () => ({ + Worker: jest.fn(worker => { + mockWorker = jest.fn((...args) => require(worker).worker(...args)); + mockEnd = jest.fn(); + + return { + end: mockEnd, + worker: mockWorker, + }; + }), +})); + +jest.mock('../crawlers/node'); +jest.mock('../crawlers/watchman', () => + jest.fn(options => { + const path = require('path'); + + const {data, ignore, rootDir, roots, computeSha1} = options; + const list = mockChangedFiles || mockFs; + const removedFiles = new Map(); + + data.clocks = mockClocks; + + for (const file in list) { + if ( + new RegExp(roots.join('|').replace(/\\/g, '\\\\')).test(file) && + !ignore(file) + ) { + const relativeFilePath = path.relative(rootDir, file); + if (list[file]) { + const hash = computeSha1 ? mockHashContents(list[file]) : null; + + data.files.set(relativeFilePath, ['', 32, 42, 0, [], hash]); + } else { + const fileData = data.files.get(relativeFilePath); + if (fileData) { + removedFiles.set(relativeFilePath, fileData); + data.files.delete(relativeFilePath); + } + } + } + } + + return Promise.resolve({ + hasteMap: data, + removedFiles, + }); + }), +); + +const mockWatcherConstructor = jest.fn(root => { + const EventEmitter = require('events').EventEmitter; + mockEmitters[root] = new EventEmitter(); + mockEmitters[root].close = jest.fn(); + setTimeout(() => mockEmitters[root].emit('ready'), 0); + return mockEmitters[root]; +}); + +jest.mock('../watchers/NodeWatcher', () => mockWatcherConstructor); +jest.mock('../watchers/WatchmanWatcher', () => mockWatcherConstructor); + +let mockChangedFiles; +let mockFs; + +jest.mock('graceful-fs', () => ({ + existsSync: jest.fn(path => { + // A file change can be triggered by writing into the + // mockChangedFiles object. + if (mockChangedFiles && path in mockChangedFiles) { + return true; + } + + if (mockFs[path]) { + return true; + } + + return false; + }), + readFileSync: jest.fn((path, options) => { + // A file change can be triggered by writing into the + // mockChangedFiles object. + if (mockChangedFiles && path in mockChangedFiles) { + return mockChangedFiles[path]; + } + + if (mockFs[path]) { + return mockFs[path]; + } + + const error = new Error(`Cannot read path '${path}'.`); + error.code = 'ENOENT'; + throw error; + }), + writeFileSync: jest.fn((path, data, options) => { + expect(options).toBe(require('v8').serialize ? undefined : 'utf8'); + mockFs[path] = data; + }), +})); + +const cacheFilePath = '/cache-file'; +const object = data => Object.assign(Object.create(null), data); +const createMap = obj => new Map(Object.keys(obj).map(key => [key, obj[key]])); + +// Jest toEqual does not match Map instances from different contexts +// This normalizes them for the uses cases in this test +const deepNormalize = value => { + const stringTag = Object.prototype.toString.call(value); + switch (stringTag) { + case '[object Map]': + return new Map( + Array.from(value).map(([k, v]) => [deepNormalize(k), deepNormalize(v)]), + ); + case '[object Object]': + return Object.keys(value).reduce((obj, key) => { + obj[key] = deepNormalize(value[key]); + return obj; + }, {}); + default: + return value; + } +}; + +let consoleWarn; +let consoleError; +let defaultConfig; +let fs; +let H; +let HasteMap; +let mockClocks; +let mockEmitters; +let mockEnd; +let mockWorker; +let getCacheFilePath; + +describe('HasteMap', () => { + beforeEach(() => { + jest.resetModules(); + + mockEmitters = Object.create(null); + mockFs = object({ + [path.join('/', 'project', 'fruits', 'Banana.js')]: ` + const Strawberry = require("Strawberry"); + `, + [path.join('/', 'project', 'fruits', 'Pear.js')]: ` + const Banana = require("Banana"); + const Strawberry = require("Strawberry"); + `, + [path.join('/', 'project', 'fruits', 'Strawberry.js')]: ` + // Strawberry! + `, + [path.join('/', 'project', 'fruits', '__mocks__', 'Pear.js')]: ` + const Melon = require("Melon"); + `, + [path.join('/', 'project', 'vegetables', 'Melon.js')]: ` + // Melon! + `, + [path.join('/', 'project', 'video', 'video.mp4')]: Buffer.from([ + 0xfa, 0xce, 0xb0, 0x0c, + ]).toString(), + }); + mockClocks = createMap({ + fruits: 'c:fake-clock:1', + vegetables: 'c:fake-clock:2', + video: 'c:fake-clock:3', + }); + + mockChangedFiles = null; + + fs = require('graceful-fs'); + + consoleWarn = console.warn; + consoleError = console.error; + + console.warn = jest.fn(); + console.error = jest.fn(); + + HasteMap = require('../').default; + H = HasteMap.H; + + getCacheFilePath = HasteMap.getCacheFilePath; + HasteMap.getCacheFilePath = jest.fn(() => cacheFilePath); + + defaultConfig = { + extensions: ['js', 'json'], + hasteImplModulePath: require.resolve('./haste_impl.js'), + maxWorkers: 1, + name: 'haste-map-test', + platforms: ['ios', 'android'], + resetCache: false, + rootDir: path.join('/', 'project'), + roots: [ + path.join('/', 'project', 'fruits'), + path.join('/', 'project', 'vegetables'), + ], + useWatchman: true, + }; + }); + + afterEach(() => { + console.warn = consoleWarn; + console.error = consoleError; + }); + + it('exports constants', () => { + expect(HasteMap.H).toBe(require('../constants')); + }); + + it('creates valid cache file paths', () => { + jest.resetModules(); + HasteMap = require('../').default; + + expect( + HasteMap.getCacheFilePath('/', '@scoped/package', 'random-value'), + ).toMatch( + process.platform === 'win32' + ? /^\\-scoped-package-(.*)$/ + : /^\/-scoped-package-(.*)$/, + ); + }); + + it('creates different cache file paths for different roots', () => { + jest.resetModules(); + const HasteMap = require('../').default; + const hasteMap1 = new HasteMap({...defaultConfig, rootDir: '/root1'}); + const hasteMap2 = new HasteMap({...defaultConfig, rootDir: '/root2'}); + expect(hasteMap1.getCacheFilePath()).not.toBe(hasteMap2.getCacheFilePath()); + }); + + it('creates different cache file paths for different dependency extractor cache keys', () => { + jest.resetModules(); + const HasteMap = require('../').default; + const dependencyExtractor = require('./dependencyExtractor'); + const config = { + ...defaultConfig, + dependencyExtractor: require.resolve('./dependencyExtractor'), + }; + dependencyExtractor.setCacheKey('foo'); + const hasteMap1 = new HasteMap(config); + dependencyExtractor.setCacheKey('bar'); + const hasteMap2 = new HasteMap(config); + expect(hasteMap1.getCacheFilePath()).not.toBe(hasteMap2.getCacheFilePath()); + }); + + it('creates different cache file paths for different values of computeDependencies', () => { + jest.resetModules(); + const HasteMap = require('../').default; + const hasteMap1 = new HasteMap({ + ...defaultConfig, + computeDependencies: true, + }); + const hasteMap2 = new HasteMap({ + ...defaultConfig, + computeDependencies: false, + }); + expect(hasteMap1.getCacheFilePath()).not.toBe(hasteMap2.getCacheFilePath()); + }); + + it('creates different cache file paths for different hasteImplModulePath cache keys', () => { + jest.resetModules(); + const HasteMap = require('../').default; + const hasteImpl = require('./haste_impl'); + hasteImpl.setCacheKey('foo'); + const hasteMap1 = new HasteMap(defaultConfig); + hasteImpl.setCacheKey('bar'); + const hasteMap2 = new HasteMap(defaultConfig); + expect(hasteMap1.getCacheFilePath()).not.toBe(hasteMap2.getCacheFilePath()); + }); + + it('creates different cache file paths for different projects', () => { + jest.resetModules(); + const HasteMap = require('../').default; + const hasteMap1 = new HasteMap({...defaultConfig, name: '@scoped/package'}); + const hasteMap2 = new HasteMap({...defaultConfig, name: '-scoped-package'}); + expect(hasteMap1.getCacheFilePath()).not.toBe(hasteMap2.getCacheFilePath()); + }); + + it('matches files against a pattern', async () => { + const {hasteFS} = await new HasteMap(defaultConfig).build(); + expect( + hasteFS.matchFiles( + process.platform === 'win32' ? /project\\fruits/ : /project\/fruits/, + ), + ).toEqual([ + path.join('/', 'project', 'fruits', 'Banana.js'), + path.join('/', 'project', 'fruits', 'Pear.js'), + path.join('/', 'project', 'fruits', 'Strawberry.js'), + path.join('/', 'project', 'fruits', '__mocks__', 'Pear.js'), + ]); + + expect(hasteFS.matchFiles(/__mocks__/)).toEqual([ + path.join('/', 'project', 'fruits', '__mocks__', 'Pear.js'), + ]); + }); + + it('ignores files given a pattern', async () => { + const config = {...defaultConfig, ignorePattern: /Kiwi/}; + mockFs[path.join('/', 'project', 'fruits', 'Kiwi.js')] = ` + // Kiwi! + `; + const {hasteFS} = await new HasteMap(config).build(); + expect(hasteFS.matchFiles(/Kiwi/)).toEqual([]); + }); + + it('ignores vcs directories without ignore pattern', async () => { + mockFs[path.join('/', 'project', 'fruits', '.git', 'fruit-history.js')] = ` + // test + `; + const {hasteFS} = await new HasteMap(defaultConfig).build(); + expect(hasteFS.matchFiles('.git')).toEqual([]); + }); + + it('ignores vcs directories with ignore pattern regex', async () => { + const config = {...defaultConfig, ignorePattern: /Kiwi/}; + mockFs[path.join('/', 'project', 'fruits', 'Kiwi.js')] = ` + // Kiwi! + `; + + mockFs[path.join('/', 'project', 'fruits', '.git', 'fruit-history.js')] = ` + // test + `; + const {hasteFS} = await new HasteMap(config).build(); + expect(hasteFS.matchFiles(/Kiwi/)).toEqual([]); + expect(hasteFS.matchFiles('.git')).toEqual([]); + }); + + it('warn on ignore pattern except for regex', async () => { + const config = {ignorePattern: 'Kiwi', ...defaultConfig}; + mockFs['/project/fruits/Kiwi.js'] = ` + // Kiwi! + `; + + try { + await new HasteMap(config).build(); + } catch (err) { + expect(err.message).toBe( + 'jest-haste-map: the `ignorePattern` option must be a RegExp', + ); + } + }); + + it('builds a haste map on a fresh cache', async () => { + // Include these files in the map + mockFs[ + path.join('/', 'project', 'fruits', 'node_modules', 'react', 'React.js') + ] = ` + const Component = require("Component"); + `; + mockFs[ + path.join( + '/', + 'project', + 'fruits', + 'node_modules', + 'fbjs', + 'lib', + 'flatMap.js', + ) + ] = ` + // flatMap + `; + + // Ignore these + mockFs[ + path.join( + '/', + 'project', + 'fruits', + 'node_modules', + 'react', + 'node_modules', + 'fbjs', + 'lib', + 'mapObject.js', + ) + ] = ` + // mapObject + `; + mockFs[ + path.join( + '/', + 'project', + 'fruits', + 'node_modules', + 'react', + 'node_modules', + 'dummy', + 'merge.js', + ) + ] = ` + // merge + `; + mockFs[ + path.join( + '/', + 'project', + 'fruits', + 'node_modules', + 'react', + 'node_modules', + 'merge', + 'package.json', + ) + ] = ` + { + "name": "merge" + } + `; + mockFs[ + path.join('/', 'project', 'fruits', 'node_modules', 'jest', 'Jest.js') + ] = ` + const Test = require("Test"); + `; + mockFs[ + path.join('/', 'project', 'fruits', 'node_modules', 'fbjs2', 'fbjs2.js') + ] = ` + // fbjs2 + `; + + const hasteMap = new HasteMap({ + ...defaultConfig, + mocksPattern: '__mocks__', + }); + + const {__hasteMapForTest: data} = await hasteMap.build(); + + expect(data.clocks).toEqual(mockClocks); + + expect(data.files).toEqual( + createMap({ + [path.join('fruits', 'Banana.js')]: [ + 'Banana', + 32, + 42, + 1, + 'Strawberry', + null, + ], + [path.join('fruits', 'Pear.js')]: [ + 'Pear', + 32, + 42, + 1, + 'Banana\0Strawberry', + null, + ], + [path.join('fruits', 'Strawberry.js')]: [ + 'Strawberry', + 32, + 42, + 1, + '', + null, + ], + [path.join('fruits', '__mocks__', 'Pear.js')]: [ + '', + 32, + 42, + 1, + 'Melon', + null, + ], + [path.join('vegetables', 'Melon.js')]: ['Melon', 32, 42, 1, '', null], + }), + ); + + expect(data.map).toEqual( + createMap({ + Banana: { + [H.GENERIC_PLATFORM]: [path.join('fruits', 'Banana.js'), H.MODULE], + }, + Melon: { + [H.GENERIC_PLATFORM]: [path.join('vegetables', 'Melon.js'), H.MODULE], + }, + Pear: { + [H.GENERIC_PLATFORM]: [path.join('fruits', 'Pear.js'), H.MODULE], + }, + Strawberry: { + [H.GENERIC_PLATFORM]: [ + path.join('fruits', 'Strawberry.js'), + H.MODULE, + ], + }, + }), + ); + + expect(data.mocks).toEqual( + createMap({ + Pear: path.join('fruits', '__mocks__', 'Pear.js'), + }), + ); + + // The cache file must exactly mirror the data structure returned from a + // build + expect(deepNormalize(hasteMap.read())).toEqual(data); + }); + + it('throws if both symlinks and watchman is enabled', () => { + expect( + () => new HasteMap({...defaultConfig, enableSymlinks: true}), + ).toThrow( + 'Set either `enableSymlinks` to false or `useWatchman` to false.', + ); + expect( + () => + new HasteMap({ + ...defaultConfig, + enableSymlinks: true, + useWatchman: true, + }), + ).toThrow( + 'Set either `enableSymlinks` to false or `useWatchman` to false.', + ); + + expect( + () => + new HasteMap({ + ...defaultConfig, + enableSymlinks: false, + useWatchman: true, + }), + ).not.toThrow(); + + expect( + () => + new HasteMap({ + ...defaultConfig, + enableSymlinks: true, + useWatchman: false, + }), + ).not.toThrow(); + }); + + describe('builds a haste map on a fresh cache with SHA-1s', () => { + it.each([false, true])('uses watchman: %s', async useWatchman => { + const node = require('../crawlers/node'); + + node.mockImplementation(options => { + const {data} = options; + + // The node crawler returns "null" for the SHA-1. + data.files = createMap({ + [path.join('fruits', 'Banana.js')]: [ + 'Banana', + 32, + 42, + 0, + 'Strawberry', + null, + ], + [path.join('fruits', 'Pear.js')]: [ + 'Pear', + 32, + 42, + 0, + 'Banana\0Strawberry', + null, + ], + [path.join('fruits', 'Strawberry.js')]: [ + 'Strawberry', + 32, + 42, + 0, + '', + null, + ], + [path.join('fruits', '__mocks__', 'Pear.js')]: [ + '', + 32, + 42, + 0, + 'Melon', + null, + ], + [path.join('vegetables', 'Melon.js')]: ['Melon', 32, 42, 0, '', null], + }); + + return Promise.resolve({ + hasteMap: data, + removedFiles: new Map(), + }); + }); + + const hasteMap = new HasteMap({ + ...defaultConfig, + computeSha1: true, + maxWorkers: 1, + useWatchman, + }); + + const data = (await hasteMap.build()).__hasteMapForTest; + + expect(data.files).toEqual( + createMap({ + [path.join('fruits', 'Banana.js')]: [ + 'Banana', + 32, + 42, + 1, + 'Strawberry', + '7772b628e422e8cf59c526be4bb9f44c0898e3d1', + ], + [path.join('fruits', 'Pear.js')]: [ + 'Pear', + 32, + 42, + 1, + 'Banana\0Strawberry', + '89d0c2cc11dcc5e1df50b8af04ab1b597acfba2f', + ], + [path.join('fruits', 'Strawberry.js')]: [ + 'Strawberry', + 32, + 42, + 1, + '', + 'e8aa38e232b3795f062f1d777731d9240c0f8c25', + ], + [path.join('fruits', '__mocks__', 'Pear.js')]: [ + '', + 32, + 42, + 1, + 'Melon', + '8d40afbb6e2dc78e1ba383b6d02cafad35cceef2', + ], + [path.join('vegetables', 'Melon.js')]: [ + 'Melon', + 32, + 42, + 1, + '', + 'f16ccf6f2334ceff2ddb47628a2c5f2d748198ca', + ], + }), + ); + + expect(deepNormalize(hasteMap.read())).toEqual(data); + }); + }); + + it('does not crawl native files even if requested to do so', async () => { + mockFs[path.join('/', 'project', 'video', 'IRequireAVideo.js')] = ` + module.exports = require("./video.mp4"); + `; + + const hasteMap = new HasteMap({ + ...defaultConfig, + extensions: [...defaultConfig.extensions], + roots: [...defaultConfig.roots, path.join('/', 'project', 'video')], + }); + + const {__hasteMapForTest: data} = await hasteMap.build(); + + expect(data.map.get('IRequireAVideo')).toBeDefined(); + expect(data.files.get(path.join('video', 'video.mp4'))).toBeDefined(); + expect(fs.readFileSync).not.toBeCalledWith( + path.join('video', 'video.mp4'), + 'utf8', + ); + }); + + it('retains all files if `retainAllFiles` is specified', async () => { + mockFs[ + path.join('/', 'project', 'fruits', 'node_modules', 'fbjs', 'fbjs.js') + ] = ` + // fbjs! + `; + + const hasteMap = new HasteMap({ + ...defaultConfig, + mocksPattern: '__mocks__', + retainAllFiles: true, + }); + + const {__hasteMapForTest: data} = await hasteMap.build(); + // Expect the node module to be part of files but make sure it wasn't + // read. + expect( + data.files.get(path.join('fruits', 'node_modules', 'fbjs', 'fbjs.js')), + ).toEqual(['', 32, 42, 0, [], null]); + + expect(data.map.get('fbjs')).not.toBeDefined(); + + // cache file + 5 modules - the node_module + expect(fs.readFileSync.mock.calls.length).toBe(6); + }); + + it('warns on duplicate mock files', async () => { + expect.assertions(1); + + // Duplicate mock files for blueberry + mockFs[ + path.join( + '/', + 'project', + 'fruits1', + '__mocks__', + 'subdir', + 'Blueberry.js', + ) + ] = ` + // Blueberry + `; + mockFs[ + path.join( + '/', + 'project', + 'fruits2', + '__mocks__', + 'subdir', + 'Blueberry.js', + ) + ] = ` + // Blueberry too! + `; + + try { + await new HasteMap({ + mocksPattern: '__mocks__', + throwOnModuleCollision: true, + ...defaultConfig, + }).build(); + } catch { + expect( + console.error.mock.calls[0][0].replace(/\\/g, '/'), + ).toMatchSnapshot(); + } + }); + + it('warns on duplicate module ids', async () => { + mockFs[path.join('/', 'project', 'fruits', 'other', 'Strawberry.js')] = ` + const Banana = require("Banana"); + `; + + const {__hasteMapForTest: data} = await new HasteMap(defaultConfig).build(); + + // Duplicate modules are removed so that it doesn't cause + // non-determinism later on. + expect(data.map.get('Strawberry')[H.GENERIC_PLATFORM]).not.toBeDefined(); + + expect(console.warn.mock.calls[0][0].replace(/\\/g, '/')).toMatchSnapshot(); + }); + + it('warns on duplicate module ids only once', async () => { + mockFs[path.join('/', 'project', 'fruits', 'other', 'Strawberry.js')] = ` + const Banana = require("Banana"); + `; + + await new HasteMap(defaultConfig).build(); + expect(console.warn).toHaveBeenCalledTimes(1); + + await new HasteMap(defaultConfig).build(); + expect(console.warn).toHaveBeenCalledTimes(1); + }); + + it('throws on duplicate module ids if "throwOnModuleCollision" is set to true', async () => { + expect.assertions(1); + // Raspberry thinks it is a Strawberry + mockFs[path.join('/', 'project', 'fruits', 'another', 'Strawberry.js')] = ` + const Banana = require("Banana"); + `; + + try { + await new HasteMap({ + throwOnModuleCollision: true, + ...defaultConfig, + }).build(); + } catch (err) { + expect(err.message).toBe( + 'Duplicated files or mocks. Please check the console for more info', + ); + } + }); + + it('splits up modules by platform', async () => { + mockFs = Object.create(null); + mockFs[path.join('/', 'project', 'fruits', 'Strawberry.js')] = ` + const Banana = require("Banana"); + `; + + mockFs[path.join('/', 'project', 'fruits', 'Strawberry.ios.js')] = ` + const Raspberry = require("Raspberry"); + `; + + mockFs[path.join('/', 'project', 'fruits', 'Strawberry.android.js')] = ` + const Blackberry = require("Blackberry"); + `; + + const {__hasteMapForTest: data} = await new HasteMap(defaultConfig).build(); + + expect(data.files).toEqual( + createMap({ + [path.join('fruits', 'Strawberry.android.js')]: [ + 'Strawberry', + 32, + 42, + 1, + 'Blackberry', + null, + ], + [path.join('fruits', 'Strawberry.ios.js')]: [ + 'Strawberry', + 32, + 42, + 1, + 'Raspberry', + null, + ], + [path.join('fruits', 'Strawberry.js')]: [ + 'Strawberry', + 32, + 42, + 1, + 'Banana', + null, + ], + }), + ); + + expect(data.map).toEqual( + createMap({ + Strawberry: { + [H.GENERIC_PLATFORM]: [ + path.join('fruits', 'Strawberry.js'), + H.MODULE, + ], + android: [path.join('fruits', 'Strawberry.android.js'), H.MODULE], + ios: [path.join('fruits', 'Strawberry.ios.js'), H.MODULE], + }, + }), + ); + }); + + it('does not access the file system on a warm cache with no changes', async () => { + const {__hasteMapForTest: initialData} = await new HasteMap( + defaultConfig, + ).build(); + + // The first run should access the file system once for the (empty) + // cache file and five times for the files in the system. + expect(fs.readFileSync.mock.calls.length).toBe(6); + + fs.readFileSync.mockClear(); + + // Explicitly mock that no files have changed. + mockChangedFiles = Object.create(null); + + // Watchman would give us different clocks. + mockClocks = createMap({ + fruits: 'c:fake-clock:3', + vegetables: 'c:fake-clock:4', + }); + + const {__hasteMapForTest: data} = await new HasteMap(defaultConfig).build(); + expect(fs.readFileSync.mock.calls.length).toBe(1); + if (require('v8').deserialize) { + expect(fs.readFileSync).toBeCalledWith(cacheFilePath); + } else { + expect(fs.readFileSync).toBeCalledWith(cacheFilePath, 'utf8'); + } + expect(deepNormalize(data.clocks)).toEqual(mockClocks); + expect(deepNormalize(data.files)).toEqual(initialData.files); + expect(deepNormalize(data.map)).toEqual(initialData.map); + }); + + it('only does minimal file system access when files change', async () => { + const {__hasteMapForTest: initialData} = await new HasteMap( + defaultConfig, + ).build(); + fs.readFileSync.mockClear(); + + // Let's assume one JS file has changed. + mockChangedFiles = object({ + [path.join('/', 'project', 'fruits', 'Banana.js')]: ` + const Kiwi = require("Kiwi"); + `, + }); + + // Watchman would give us different clocks for `/project/fruits`. + mockClocks = createMap({ + fruits: 'c:fake-clock:3', + vegetables: 'c:fake-clock:2', + }); + + const {__hasteMapForTest: data} = await new HasteMap(defaultConfig).build(); + + expect(fs.readFileSync.mock.calls.length).toBe(2); + + if (require('v8').serialize) { + expect(fs.readFileSync).toBeCalledWith(cacheFilePath); + } else { + expect(fs.readFileSync).toBeCalledWith(cacheFilePath, 'utf8'); + } + expect(fs.readFileSync).toBeCalledWith( + path.join('/', 'project', 'fruits', 'Banana.js'), + 'utf8', + ); + + expect(deepNormalize(data.clocks)).toEqual(mockClocks); + + const files = new Map(initialData.files); + files.set(path.join('fruits', 'Banana.js'), [ + 'Banana', + 32, + 42, + 1, + 'Kiwi', + null, + ]); + + expect(deepNormalize(data.files)).toEqual(files); + + const map = new Map(initialData.map); + expect(deepNormalize(data.map)).toEqual(map); + }); + + it('correctly handles file deletions', async () => { + const {__hasteMapForTest: initialData} = await new HasteMap( + defaultConfig, + ).build(); + fs.readFileSync.mockClear(); + + // Let's assume one JS file was removed. + delete mockFs[path.join('/', 'project', 'fruits', 'Banana.js')]; + mockChangedFiles = object({ + [path.join('/', 'project', 'fruits', 'Banana.js')]: null, + }); + + // Watchman would give us different clocks for `/project/fruits`. + mockClocks = createMap({ + fruits: 'c:fake-clock:3', + vegetables: 'c:fake-clock:2', + }); + + const {__hasteMapForTest: data} = await new HasteMap(defaultConfig).build(); + + const files = new Map(initialData.files); + files.delete(path.join('fruits', 'Banana.js')); + expect(deepNormalize(data.files)).toEqual(files); + + const map = new Map(initialData.map); + map.delete('Banana'); + expect(deepNormalize(data.map)).toEqual(map); + }); + + it('correctly handles platform-specific file additions', async () => { + mockFs = Object.create(null); + mockFs[path.join('/', 'project', 'fruits', 'Strawberry.js')] = ` + const Banana = require("Banana"); + `; + let data; + ({__hasteMapForTest: data} = await new HasteMap(defaultConfig).build()); + expect(data.map.get('Strawberry')).toEqual({ + g: [path.join('fruits', 'Strawberry.js'), 0], + }); + + delete mockFs[path.join('/', 'project', 'fruits', 'Strawberry.ios.js')]; + mockChangedFiles = object({ + [path.join('/', 'project', 'fruits', 'Strawberry.ios.js')]: ` + const Raspberry = require("Raspberry"); + `, + }); + mockClocks = createMap({fruits: 'c:fake-clock:3'}); + ({__hasteMapForTest: data} = await new HasteMap(defaultConfig).build()); + expect(data.map.get('Strawberry')).toEqual({ + g: [path.join('fruits', 'Strawberry.js'), 0], + ios: [path.join('fruits', 'Strawberry.ios.js'), 0], + }); + }); + + it('correctly handles platform-specific file deletions', async () => { + mockFs = Object.create(null); + mockFs[path.join('/', 'project', 'fruits', 'Strawberry.js')] = ` + const Banana = require("Banana"); + `; + mockFs[path.join('/', 'project', 'fruits', 'Strawberry.ios.js')] = ` + const Raspberry = require("Raspberry"); + `; + let data; + ({__hasteMapForTest: data} = await new HasteMap(defaultConfig).build()); + expect(data.map.get('Strawberry')).toEqual({ + g: [path.join('fruits', 'Strawberry.js'), 0], + ios: [path.join('fruits', 'Strawberry.ios.js'), 0], + }); + + delete mockFs[path.join('/', 'project', 'fruits', 'Strawberry.ios.js')]; + mockChangedFiles = object({ + [path.join('/', 'project', 'fruits', 'Strawberry.ios.js')]: null, + }); + mockClocks = createMap({fruits: 'c:fake-clock:3'}); + ({__hasteMapForTest: data} = await new HasteMap(defaultConfig).build()); + expect(data.map.get('Strawberry')).toEqual({ + g: [path.join('fruits', 'Strawberry.js'), 0], + }); + }); + + it('correctly handles platform-specific file renames', async () => { + mockFs = Object.create(null); + mockFs[path.join('/', 'project', 'fruits', 'Strawberry.ios.js')] = ` + const Raspberry = require("Raspberry"); + `; + let data; + ({__hasteMapForTest: data} = await new HasteMap(defaultConfig).build()); + expect(data.map.get('Strawberry')).toEqual({ + ios: [path.join('fruits', 'Strawberry.ios.js'), 0], + }); + + delete mockFs[path.join('/', 'project', 'fruits', 'Strawberry.ios.js')]; + mockChangedFiles = object({ + [path.join('/', 'project', 'fruits', 'Strawberry.ios.js')]: null, + [path.join('/', 'project', 'fruits', 'Strawberry.js')]: ` + const Banana = require("Banana"); + `, + }); + mockClocks = createMap({fruits: 'c:fake-clock:3'}); + ({__hasteMapForTest: data} = await new HasteMap(defaultConfig).build()); + expect(data.map.get('Strawberry')).toEqual({ + g: [path.join('fruits', 'Strawberry.js'), 0], + }); + }); + + describe('duplicate modules', () => { + beforeEach(async () => { + mockFs[ + path.join('/', 'project', 'fruits', 'another', 'Strawberry.js') + ] = ` + const Blackberry = require("Blackberry"); + `; + + const {__hasteMapForTest: data} = await new HasteMap( + defaultConfig, + ).build(); + expect(deepNormalize(data.duplicates)).toEqual( + createMap({ + Strawberry: createMap({ + g: createMap({ + [path.join('fruits', 'Strawberry.js')]: H.MODULE, + [path.join('fruits', 'another', 'Strawberry.js')]: H.MODULE, + }), + }), + }), + ); + expect(data.map.get('Strawberry')).toEqual({}); + }); + + it('recovers when a duplicate file is deleted', async () => { + delete mockFs[ + path.join('/', 'project', 'fruits', 'another', 'Strawberry.js') + ]; + mockChangedFiles = object({ + [path.join('/', 'project', 'fruits', 'another', 'Strawberry.js')]: null, + }); + mockClocks = createMap({ + fruits: 'c:fake-clock:3', + vegetables: 'c:fake-clock:2', + }); + + const {__hasteMapForTest: data} = await new HasteMap( + defaultConfig, + ).build(); + expect(deepNormalize(data.duplicates)).toEqual(new Map()); + expect(data.map.get('Strawberry')).toEqual({ + g: [path.join('fruits', 'Strawberry.js'), H.MODULE], + }); + // Make sure the other files are not affected. + expect(data.map.get('Banana')).toEqual({ + g: [path.join('fruits', 'Banana.js'), H.MODULE], + }); + }); + + it('recovers with the correct type when a duplicate file is deleted', async () => { + mockFs[ + path.join('/', 'project', 'fruits', 'strawberryPackage', 'package.json') + ] = ` + {"name": "Strawberry"} + `; + + const {__hasteMapForTest: data} = await new HasteMap( + defaultConfig, + ).build(); + + expect(deepNormalize(data.duplicates)).toEqual( + createMap({ + Strawberry: createMap({ + g: createMap({ + [path.join('fruits', 'Strawberry.js')]: H.MODULE, + [path.join('fruits', 'another', 'Strawberry.js')]: H.MODULE, + [path.join('fruits', 'strawberryPackage', 'package.json')]: + H.PACKAGE, + }), + }), + }), + ); + + delete mockFs[ + path.join('/', 'project', 'fruits', 'another', 'Strawberry.js') + ]; + delete mockFs[ + path.join('/', 'project', 'fruits', 'strawberryPackage', 'package.json') + ]; + + mockChangedFiles = object({ + [path.join('/', 'project', 'fruits', 'another', 'Strawberry.js')]: null, + [path.join( + '/', + 'project', + 'fruits', + 'strawberryPackage', + 'package.json', + )]: null, + }); + mockClocks = createMap({ + fruits: 'c:fake-clock:4', + }); + + const {__hasteMapForTest: correctData} = await new HasteMap( + defaultConfig, + ).build(); + + expect(deepNormalize(correctData.duplicates)).toEqual(new Map()); + expect(correctData.map.get('Strawberry')).toEqual({ + g: [path.join('fruits', 'Strawberry.js'), H.MODULE], + }); + }); + + it('recovers when a duplicate module is renamed', async () => { + mockChangedFiles = object({ + [path.join('/', 'project', 'fruits', 'another', 'Pineapple.js')]: ` + const Blackberry = require("Blackberry"); + `, + [path.join('/', 'project', 'fruits', 'another', 'Strawberry.js')]: null, + }); + mockClocks = createMap({ + fruits: 'c:fake-clock:3', + vegetables: 'c:fake-clock:2', + }); + + const {__hasteMapForTest: data} = await new HasteMap( + defaultConfig, + ).build(); + expect(deepNormalize(data.duplicates)).toEqual(new Map()); + expect(data.map.get('Strawberry')).toEqual({ + g: [path.join('fruits', 'Strawberry.js'), H.MODULE], + }); + expect(data.map.get('Pineapple')).toEqual({ + g: [path.join('fruits', 'another', 'Pineapple.js'), H.MODULE], + }); + // Make sure the other files are not affected. + expect(data.map.get('Banana')).toEqual({ + g: [path.join('fruits', 'Banana.js'), H.MODULE], + }); + }); + }); + + it('discards the cache when configuration changes', async () => { + HasteMap.getCacheFilePath = getCacheFilePath; + await new HasteMap(defaultConfig).build(); + fs.readFileSync.mockClear(); + + // Explicitly mock that no files have changed. + mockChangedFiles = Object.create(null); + + // Watchman would give us different clocks. + mockClocks = createMap({ + fruits: 'c:fake-clock:3', + vegetables: 'c:fake-clock:4', + }); + + const config = {...defaultConfig, ignorePattern: /Kiwi|Pear/}; + const {moduleMap} = await new HasteMap(config).build(); + expect(moduleMap.getModule('Pear')).toBe(null); + }); + + it('ignores files that do not exist', async () => { + const watchman = require('../crawlers/watchman'); + const mockImpl = watchman.getMockImplementation(); + // Wrap the watchman mock and add an invalid file to the file list. + watchman.mockImplementation(options => + mockImpl(options).then(() => { + const {data} = options; + data.files.set(path.join('fruits', 'invalid', 'file.js'), [ + '', + 34, + 44, + 0, + [], + ]); + return {hasteMap: data, removedFiles: new Map()}; + }), + ); + + const {__hasteMapForTest: data} = await new HasteMap(defaultConfig).build(); + expect(data.files.size).toBe(5); + + // Ensure this file is not part of the file list. + expect(data.files.get(path.join('fruits', 'invalid', 'file.js'))).toBe( + undefined, + ); + }); + + it('distributes work across workers', async () => { + const jestWorker = require('jest-worker').Worker; + const path = require('path'); + const dependencyExtractor = path.join(__dirname, 'dependencyExtractor.js'); + await new HasteMap({ + ...defaultConfig, + dependencyExtractor, + hasteImplModulePath: undefined, + maxWorkers: 4, + }).build(); + + expect(jestWorker.mock.calls.length).toBe(1); + + expect(mockWorker.mock.calls.length).toBe(5); + + expect(mockWorker.mock.calls).toEqual([ + [ + { + computeDependencies: true, + computeSha1: false, + dependencyExtractor, + filePath: path.join('/', 'project', 'fruits', 'Banana.js'), + hasteImplModulePath: undefined, + rootDir: path.join('/', 'project'), + }, + ], + [ + { + computeDependencies: true, + computeSha1: false, + dependencyExtractor, + filePath: path.join('/', 'project', 'fruits', 'Pear.js'), + hasteImplModulePath: undefined, + rootDir: path.join('/', 'project'), + }, + ], + [ + { + computeDependencies: true, + computeSha1: false, + dependencyExtractor, + filePath: path.join('/', 'project', 'fruits', 'Strawberry.js'), + hasteImplModulePath: undefined, + rootDir: path.join('/', 'project'), + }, + ], + [ + { + computeDependencies: true, + computeSha1: false, + dependencyExtractor, + filePath: path.join('/', 'project', 'fruits', '__mocks__', 'Pear.js'), + hasteImplModulePath: undefined, + rootDir: path.join('/', 'project'), + }, + ], + [ + { + computeDependencies: true, + computeSha1: false, + dependencyExtractor, + filePath: path.join('/', 'project', 'vegetables', 'Melon.js'), + hasteImplModulePath: undefined, + rootDir: path.join('/', 'project'), + }, + ], + ]); + + expect(mockEnd).toBeCalled(); + }); + + it('tries to crawl using node as a fallback', async () => { + const watchman = require('../crawlers/watchman'); + const node = require('../crawlers/node'); + + watchman.mockImplementation(() => { + throw new Error('watchman error'); + }); + node.mockImplementation(options => { + const {data} = options; + data.files = createMap({ + [path.join('fruits', 'Banana.js')]: ['', 32, 42, 0, '', null], + }); + return Promise.resolve({ + hasteMap: data, + removedFiles: new Map(), + }); + }); + + const {__hasteMapForTest: data} = await new HasteMap(defaultConfig).build(); + expect(watchman).toBeCalled(); + expect(node).toBeCalled(); + + expect(data.files).toEqual( + createMap({ + [path.join('fruits', 'Banana.js')]: [ + 'Banana', + 32, + 42, + 1, + 'Strawberry', + null, + ], + }), + ); + + expect(console.warn.mock.calls[0][0]).toMatchSnapshot(); + }); + + it('tries to crawl using node as a fallback when promise fails once', async () => { + const watchman = require('../crawlers/watchman'); + const node = require('../crawlers/node'); + + watchman.mockImplementation(() => + Promise.reject(new Error('watchman error')), + ); + node.mockImplementation(options => { + const {data} = options; + data.files = createMap({ + [path.join('fruits', 'Banana.js')]: ['', 32, 42, 0, '', null], + }); + return Promise.resolve({ + hasteMap: data, + removedFiles: new Map(), + }); + }); + + const {__hasteMapForTest: data} = await new HasteMap(defaultConfig).build(); + + expect(watchman).toBeCalled(); + expect(node).toBeCalled(); + + expect(data.files).toEqual( + createMap({ + [path.join('fruits', 'Banana.js')]: [ + 'Banana', + 32, + 42, + 1, + 'Strawberry', + null, + ], + }), + ); + }); + + it('stops crawling when both crawlers fail', async () => { + expect.assertions(1); + const watchman = require('../crawlers/watchman'); + const node = require('../crawlers/node'); + + watchman.mockImplementation(() => + Promise.reject(new Error('watchman error')), + ); + + node.mockImplementation((roots, extensions, ignore, data) => + Promise.reject(new Error('node error')), + ); + + try { + await new HasteMap(defaultConfig).build(); + } catch (error) { + expect(error.message).toEqual( + 'Crawler retry failed:\n' + + ' Original error: watchman error\n' + + ' Retry error: node error\n', + ); + } + }); + + describe('file system changes processing', () => { + function waitForItToChange(hasteMap) { + return new Promise(resolve => { + hasteMap.once('change', resolve); + }); + } + + function mockDeleteFile(dirPath, filePath) { + const e = mockEmitters[dirPath]; + e.emit('all', 'delete', filePath, dirPath, undefined); + } + + function hm_it(title, fn, options) { + options = options || {}; + (options.only ? it.only : it)(title, async () => { + if (options.mockFs) { + mockFs = options.mockFs; + } + const watchConfig = {...defaultConfig, watch: true}; + const hm = new HasteMap(watchConfig); + await hm.build(); + try { + await fn(hm); + } finally { + hm.end(); + } + }); + } + + hm_it('provides a new set of hasteHS and moduleMap', async hm => { + const initialResult = await hm.build(); + const filePath = path.join('/', 'project', 'fruits', 'Banana.js'); + expect(initialResult.hasteFS.getModuleName(filePath)).toBeDefined(); + expect(initialResult.moduleMap.getModule('Banana')).toBe(filePath); + mockDeleteFile(path.join('/', 'project', 'fruits'), 'Banana.js'); + mockDeleteFile(path.join('/', 'project', 'fruits'), 'Banana.js'); + const {eventsQueue, hasteFS, moduleMap} = await waitForItToChange(hm); + expect(eventsQueue).toHaveLength(1); + const deletedBanana = {filePath, stat: undefined, type: 'delete'}; + expect(eventsQueue).toEqual([deletedBanana]); + // Verify we didn't change the original map. + expect(initialResult.hasteFS.getModuleName(filePath)).toBeDefined(); + expect(initialResult.moduleMap.getModule('Banana')).toBe(filePath); + expect(hasteFS.getModuleName(filePath)).toBeNull(); + expect(moduleMap.getModule('Banana')).toBeNull(); + }); + + const MOCK_STAT_FILE = { + isDirectory: () => false, + mtime: {getTime: () => 45}, + size: 55, + }; + + const MOCK_STAT_FOLDER = { + isDirectory: () => true, + mtime: {getTime: () => 45}, + size: 55, + }; + + hm_it('handles several change events at once', async hm => { + mockFs[path.join('/', 'project', 'fruits', 'Tomato.js')] = ` + // Tomato! + `; + mockFs[path.join('/', 'project', 'fruits', 'Pear.js')] = ` + // Pear! + `; + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emit( + 'all', + 'add', + 'Tomato.js', + path.join('/', 'project', 'fruits'), + MOCK_STAT_FILE, + ); + e.emit( + 'all', + 'change', + 'Pear.js', + path.join('/', 'project', 'fruits'), + MOCK_STAT_FILE, + ); + const {eventsQueue, hasteFS, moduleMap} = await waitForItToChange(hm); + expect(eventsQueue).toEqual([ + { + filePath: path.join('/', 'project', 'fruits', 'Tomato.js'), + stat: MOCK_STAT_FILE, + type: 'add', + }, + { + filePath: path.join('/', 'project', 'fruits', 'Pear.js'), + stat: MOCK_STAT_FILE, + type: 'change', + }, + ]); + expect( + hasteFS.getModuleName(path.join('/', 'project', 'fruits', 'Tomato.js')), + ).not.toBeNull(); + expect(moduleMap.getModule('Tomato')).toBeDefined(); + expect(moduleMap.getModule('Pear')).toBe( + path.join('/', 'project', 'fruits', 'Pear.js'), + ); + }); + + hm_it('does not emit duplicate change events', async hm => { + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emit( + 'all', + 'change', + 'tomato.js', + path.join('/', 'project', 'fruits'), + MOCK_STAT_FILE, + ); + e.emit( + 'all', + 'change', + 'tomato.js', + path.join('/', 'project', 'fruits'), + MOCK_STAT_FILE, + ); + const {eventsQueue} = await waitForItToChange(hm); + expect(eventsQueue).toHaveLength(1); + }); + + hm_it( + 'emits a change even if a file in node_modules has changed', + async hm => { + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emit( + 'all', + 'add', + 'apple.js', + path.join('/', 'project', 'fruits', 'node_modules', ''), + MOCK_STAT_FILE, + ); + const {eventsQueue, hasteFS} = await waitForItToChange(hm); + const filePath = path.join( + '/', + 'project', + 'fruits', + 'node_modules', + 'apple.js', + ); + expect(eventsQueue).toHaveLength(1); + expect(eventsQueue).toEqual([ + {filePath, stat: MOCK_STAT_FILE, type: 'add'}, + ]); + expect(hasteFS.getModuleName(filePath)).toBeDefined(); + }, + ); + + hm_it( + 'correctly tracks changes to both platform-specific versions of a single module name', + async hm => { + const {moduleMap: initMM} = await hm.build(); + expect(initMM.getModule('Orange', 'ios')).toBeTruthy(); + expect(initMM.getModule('Orange', 'android')).toBeTruthy(); + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emit( + 'all', + 'change', + 'Orange.ios.js', + path.join('/', 'project', 'fruits'), + MOCK_STAT_FILE, + ); + e.emit( + 'all', + 'change', + 'Orange.android.js', + path.join('/', 'project', 'fruits'), + MOCK_STAT_FILE, + ); + const {eventsQueue, hasteFS, moduleMap} = await waitForItToChange(hm); + expect(eventsQueue).toHaveLength(2); + expect(eventsQueue).toEqual([ + { + filePath: path.join('/', 'project', 'fruits', 'Orange.ios.js'), + stat: MOCK_STAT_FILE, + type: 'change', + }, + { + filePath: path.join('/', 'project', 'fruits', 'Orange.android.js'), + stat: MOCK_STAT_FILE, + type: 'change', + }, + ]); + expect( + hasteFS.getModuleName( + path.join('/', 'project', 'fruits', 'Orange.ios.js'), + ), + ).toBeTruthy(); + expect( + hasteFS.getModuleName( + path.join('/', 'project', 'fruits', 'Orange.android.js'), + ), + ).toBeTruthy(); + const iosVariant = moduleMap.getModule('Orange', 'ios'); + expect(iosVariant).toBe( + path.join('/', 'project', 'fruits', 'Orange.ios.js'), + ); + const androidVariant = moduleMap.getModule('Orange', 'android'); + expect(androidVariant).toBe( + path.join('/', 'project', 'fruits', 'Orange.android.js'), + ); + }, + { + mockFs: { + [path.join('/', 'project', 'fruits', 'Orange.android.js')]: ` + // Orange Android! + `, + [path.join('/', 'project', 'fruits', 'Orange.ios.js')]: ` + // Orange iOS! + `, + }, + }, + ); + + describe('recovery from duplicate module IDs', () => { + async function setupDuplicates(hm) { + mockFs[path.join('/', 'project', 'fruits', 'Pear.js')] = ` + // Pear! + `; + mockFs[path.join('/', 'project', 'fruits', 'another', 'Pear.js')] = ` + // Pear too! + `; + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emit( + 'all', + 'change', + 'Pear.js', + path.join('/', 'project', 'fruits'), + MOCK_STAT_FILE, + ); + e.emit( + 'all', + 'add', + 'Pear.js', + path.join('/', 'project', 'fruits', 'another'), + MOCK_STAT_FILE, + ); + const {hasteFS, moduleMap} = await waitForItToChange(hm); + expect( + hasteFS.exists( + path.join('/', 'project', 'fruits', 'another', 'Pear.js'), + ), + ).toBe(true); + try { + moduleMap.getModule('Pear'); + throw new Error('should be unreachable'); + } catch (error) { + const {DuplicateHasteCandidatesError} = + require('../ModuleMap').default; + expect(error).toBeInstanceOf(DuplicateHasteCandidatesError); + expect(error.hasteName).toBe('Pear'); + expect(error.platform).toBe('g'); + expect(error.supportsNativePlatform).toBe(false); + expect(error.duplicatesSet).toEqual( + createMap({ + [path.join('/', 'project', 'fruits', 'Pear.js')]: H.MODULE, + [path.join('/', 'project', 'fruits', 'another', 'Pear.js')]: + H.MODULE, + }), + ); + expect(error.message.replace(/\\/g, '/')).toMatchSnapshot(); + } + } + + hm_it( + 'recovers when the oldest version of the duplicates is fixed', + async hm => { + await setupDuplicates(hm); + mockFs[path.join('/', 'project', 'fruits', 'Pear.js')] = null; + mockFs[path.join('/', 'project', 'fruits', 'Pear2.js')] = ` + // Pear! + `; + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emit( + 'all', + 'delete', + 'Pear.js', + path.join('/', 'project', 'fruits'), + MOCK_STAT_FILE, + ); + e.emit( + 'all', + 'add', + 'Pear2.js', + path.join('/', 'project', 'fruits'), + MOCK_STAT_FILE, + ); + const {moduleMap} = await waitForItToChange(hm); + expect(moduleMap.getModule('Pear')).toBe( + path.join('/', 'project', 'fruits', 'another', 'Pear.js'), + ); + expect(moduleMap.getModule('Pear2')).toBe( + path.join('/', 'project', 'fruits', 'Pear2.js'), + ); + }, + ); + + hm_it('recovers when the most recent duplicate is fixed', async hm => { + await setupDuplicates(hm); + mockFs[path.join('/', 'project', 'fruits', 'another', 'Pear.js')] = + null; + mockFs[path.join('/', 'project', 'fruits', 'another', 'Pear2.js')] = ` + // Pear too! + `; + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emit( + 'all', + 'add', + 'Pear2.js', + path.join('/', 'project', 'fruits', 'another'), + MOCK_STAT_FILE, + ); + e.emit( + 'all', + 'delete', + 'Pear.js', + path.join('/', 'project', 'fruits', 'another'), + MOCK_STAT_FILE, + ); + const {moduleMap} = await waitForItToChange(hm); + expect(moduleMap.getModule('Pear')).toBe( + path.join('/', 'project', 'fruits', 'Pear.js'), + ); + expect(moduleMap.getModule('Pear2')).toBe( + path.join('/', 'project', 'fruits', 'another', 'Pear2.js'), + ); + }); + + hm_it('ignore directories', async hm => { + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emit( + 'all', + 'change', + 'tomato.js', + path.join('/', 'project', 'fruits'), + MOCK_STAT_FOLDER, + ); + e.emit( + 'all', + 'change', + 'tomato.js', + path.join('/', 'project', 'fruits', 'tomato.js', 'index.js'), + MOCK_STAT_FILE, + ); + const {eventsQueue} = await waitForItToChange(hm); + expect(eventsQueue).toHaveLength(1); + }); + }); + }); +}); diff --git a/packages/metro-file-map/src/__tests__/test_dotfiles_root/.eslintrc.js b/packages/metro-file-map/src/__tests__/test_dotfiles_root/.eslintrc.js new file mode 100644 index 0000000000..ff77557603 --- /dev/null +++ b/packages/metro-file-map/src/__tests__/test_dotfiles_root/.eslintrc.js @@ -0,0 +1,9 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ diff --git a/packages/metro-file-map/src/__tests__/test_dotfiles_root/index.js b/packages/metro-file-map/src/__tests__/test_dotfiles_root/index.js new file mode 100644 index 0000000000..ff77557603 --- /dev/null +++ b/packages/metro-file-map/src/__tests__/test_dotfiles_root/index.js @@ -0,0 +1,9 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ diff --git a/packages/metro-file-map/src/__tests__/worker-test.js b/packages/metro-file-map/src/__tests__/worker-test.js new file mode 100644 index 0000000000..12701d311d --- /dev/null +++ b/packages/metro-file-map/src/__tests__/worker-test.js @@ -0,0 +1,205 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +import H from '../constants'; +import {getSha1, worker} from '../worker'; +import * as fs from 'graceful-fs'; +import * as path from 'path'; + +jest.mock('graceful-fs', () => { + const path = require('path'); + const mockFs = { + [path.join('/project', 'fruits', 'Banana.js')]: ` + const Strawberry = require("Strawberry"); + `, + [path.join('/project', 'fruits', 'Pear.js')]: ` + const Banana = require("Banana"); + const Strawberry = require('Strawberry'); + const Lime = loadModule('Lime'); + `, + [path.join('/project', 'fruits', 'Strawberry.js')]: ` + // Strawberry! + `, + [path.join('/project', 'fruits', 'apple.png')]: Buffer.from([ + 137, 80, 78, 71, 13, 10, 26, 10, + ]), + [path.join('/project', 'package.json')]: ` + { + "name": "haste-package", + "main": "foo.js" + } + `, + }; + + return { + ...jest.createMockFromModule('graceful-fs'), + readFileSync: jest.fn((path, options) => { + if (mockFs[path]) { + return options === 'utf8' ? mockFs[path] : Buffer.from(mockFs[path]); + } + + throw new Error(`Cannot read path '${path}'.`); + }), + }; +}); + +const rootDir = '/project'; + +describe('worker', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('parses JavaScript files and extracts module information', async () => { + expect( + await worker({ + computeDependencies: true, + filePath: path.join('/project', 'fruits', 'Pear.js'), + rootDir, + }), + ).toEqual({ + dependencies: ['Banana', 'Strawberry'], + }); + + expect( + await worker({ + computeDependencies: true, + filePath: path.join('/project', 'fruits', 'Strawberry.js'), + rootDir, + }), + ).toEqual({ + dependencies: [], + }); + }); + + it('accepts a custom dependency extractor', async () => { + expect( + await worker({ + computeDependencies: true, + dependencyExtractor: path.join(__dirname, 'dependencyExtractor.js'), + filePath: path.join('/project', 'fruits', 'Pear.js'), + rootDir, + }), + ).toEqual({ + dependencies: ['Banana', 'Strawberry', 'Lime'], + }); + }); + + it('delegates to hasteImplModulePath for getting the id', async () => { + expect( + await worker({ + computeDependencies: true, + filePath: path.join('/project', 'fruits', 'Pear.js'), + hasteImplModulePath: require.resolve('./haste_impl.js'), + rootDir, + }), + ).toEqual({ + dependencies: ['Banana', 'Strawberry'], + id: 'Pear', + module: [path.join('fruits', 'Pear.js'), H.MODULE], + }); + + expect( + await worker({ + computeDependencies: true, + filePath: path.join('/project', 'fruits', 'Strawberry.js'), + rootDir, + }), + ).toEqual({ + dependencies: [], + id: 'Strawberry', + module: [path.join('fruits', 'Strawberry.js'), H.MODULE], + }); + }); + + it('parses package.json files as haste packages', async () => { + expect( + await worker({ + computeDependencies: true, + filePath: path.join('/project', 'package.json'), + rootDir, + }), + ).toEqual({ + dependencies: undefined, + id: 'haste-package', + module: ['package.json', H.PACKAGE], + }); + }); + + it('returns an error when a file cannot be accessed', async () => { + let error = null; + + try { + await worker({computeDependencies: true, filePath: '/kiwi.js', rootDir}); + } catch (err) { + error = err; + } + + expect(error.message).toEqual(`Cannot read path '/kiwi.js'.`); + }); + + it('simply computes SHA-1s when requested (works well with binary data)', async () => { + expect( + await getSha1({ + computeSha1: true, + filePath: path.join('/project', 'fruits', 'apple.png'), + rootDir, + }), + ).toEqual({sha1: '4caece539b039b16e16206ea2478f8c5ffb2ca05'}); + + expect( + await getSha1({ + computeSha1: false, + filePath: path.join('/project', 'fruits', 'Banana.js'), + rootDir, + }), + ).toEqual({sha1: null}); + + expect( + await getSha1({ + computeSha1: true, + filePath: path.join('/project', 'fruits', 'Banana.js'), + rootDir, + }), + ).toEqual({sha1: '7772b628e422e8cf59c526be4bb9f44c0898e3d1'}); + + expect( + await getSha1({ + computeSha1: true, + filePath: path.join('/project', 'fruits', 'Pear.js'), + rootDir, + }), + ).toEqual({sha1: 'c7a7a68a1c8aaf452669dd2ca52ac4a434d25552'}); + + await expect( + getSha1({computeSha1: true, filePath: '/i/dont/exist.js', rootDir}), + ).rejects.toThrow(); + }); + + it('avoids computing dependencies if not requested and Haste does not need it', async () => { + expect( + await worker({ + computeDependencies: false, + filePath: path.join('/project', 'fruits', 'Pear.js'), + hasteImplModulePath: path.resolve(__dirname, 'haste_impl.js'), + rootDir, + }), + ).toEqual({ + dependencies: undefined, + id: 'Pear', + module: [path.join('fruits', 'Pear.js'), H.MODULE], + sha1: undefined, + }); + + // Ensure not disk access happened. + expect(fs.readFileSync).not.toHaveBeenCalled(); + expect(fs.readFile).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/metro-file-map/src/constants.js b/packages/metro-file-map/src/constants.js new file mode 100644 index 0000000000..8d7ad87259 --- /dev/null +++ b/packages/metro-file-map/src/constants.js @@ -0,0 +1,51 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @noformat - Flow comment syntax + */ + +/* + * This file exports a set of constants that are used for Jest's haste map + * serialization. On very large repositories, the haste map cache becomes very + * large to the point where it is the largest overhead in starting up Jest. + * + * This constant key map allows to keep the map smaller without having to build + * a custom serialization library. + */ + +/*:: +import type {HType} from './flow-types'; +*/ + +'use strict'; + +const constants/*: HType */ = { + /* dependency serialization */ + DEPENDENCY_DELIM: '\0', + + /* file map attributes */ + ID: 0, + MTIME: 1, + SIZE: 2, + VISITED: 3, + DEPENDENCIES: 4, + SHA1: 5, + + /* module map attributes */ + PATH: 0, + TYPE: 1, + + /* module types */ + MODULE: 0, + PACKAGE: 1, + + /* platforms */ + GENERIC_PLATFORM: 'g', + NATIVE_PLATFORM: 'native', +}; + +module.exports = constants; diff --git a/packages/metro-file-map/src/crawlers/__tests__/node-test.js b/packages/metro-file-map/src/crawlers/__tests__/node-test.js new file mode 100644 index 0000000000..2ae3191d9e --- /dev/null +++ b/packages/metro-file-map/src/crawlers/__tests__/node-test.js @@ -0,0 +1,447 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +'use strict'; + +jest.useRealTimers(); + +jest.mock('child_process', () => ({ + spawn: jest.fn((cmd, args) => { + let closeCallback; + return { + on: jest.fn().mockImplementation((event, callback) => { + if (event === 'exit') { + callback(mockSpawnExit, null); + } + }), + stdout: { + on: jest.fn().mockImplementation((event, callback) => { + if (event === 'data') { + setTimeout(() => { + callback(mockResponse); + setTimeout(closeCallback, 0); + }, 0); + } else if (event === 'close') { + closeCallback = callback; + } + }), + setEncoding: jest.fn(), + }, + }; + }), +})); + +let mockHasReaddirWithFileTypesSupport = false; + +jest.mock('graceful-fs', () => { + const slash = require('slash'); + let mtime = 32; + const size = 42; + const stat = (path, callback) => { + setTimeout( + () => + callback(null, { + isDirectory() { + return slash(path).endsWith('/directory'); + }, + isSymbolicLink() { + return slash(path).endsWith('symlink'); + }, + mtime: { + getTime() { + return mtime++; + }, + }, + size, + }), + 0, + ); + }; + return { + lstat: jest.fn(stat), + readdir: jest.fn((dir, options, callback) => { + // readdir has an optional `options` arg that's in the middle of the args list. + // we always provide it in practice, but let's try to handle the case where it's not + // provided too + if (typeof callback === 'undefined') { + if (typeof options === 'function') { + callback = options; + } + throw new Error('readdir: callback is not a function!'); + } + + if (mockHasReaddirWithFileTypesSupport) { + if (slash(dir) === '/project/fruits') { + setTimeout( + () => + callback(null, [ + { + isDirectory: () => true, + isSymbolicLink: () => false, + name: 'directory', + }, + { + isDirectory: () => false, + isSymbolicLink: () => false, + name: 'tomato.js', + }, + { + isDirectory: () => false, + isSymbolicLink: () => true, + name: 'symlink', + }, + ]), + 0, + ); + } else if (slash(dir) === '/project/fruits/directory') { + setTimeout( + () => + callback(null, [ + { + isDirectory: () => false, + isSymbolicLink: () => false, + name: 'strawberry.js', + }, + ]), + 0, + ); + } else if (slash(dir) == '/error') { + setTimeout(() => callback({code: 'ENOTDIR'}, undefined), 0); + } + } else { + if (slash(dir) === '/project/fruits') { + setTimeout( + () => callback(null, ['directory', 'tomato.js', 'symlink']), + 0, + ); + } else if (slash(dir) === '/project/fruits/directory') { + setTimeout(() => callback(null, ['strawberry.js']), 0); + } else if (slash(dir) == '/error') { + setTimeout(() => callback({code: 'ENOTDIR'}, undefined), 0); + } + } + }), + stat: jest.fn(stat), + }; +}); + +const pearMatcher = path => /pear/.test(path); +const normalize = path => + process.platform === 'win32' ? path.replace(/\//g, '\\') : path; +const createMap = obj => + new Map(Object.keys(obj).map(key => [normalize(key), obj[key]])); + +const rootDir = '/project'; +let mockResponse; +let mockSpawnExit; +let nodeCrawl; +let childProcess; + +describe('node crawler', () => { + beforeEach(() => { + jest.resetModules(); + + mockResponse = [ + '/project/fruits/pear.js', + '/project/fruits/strawberry.js', + '/project/fruits/tomato.js', + ].join('\n'); + + mockSpawnExit = 0; + }); + + it('crawls for files based on patterns', async () => { + childProcess = require('child_process'); + nodeCrawl = require('../node'); + + mockResponse = [ + '/project/fruits/pear.js', + '/project/fruits/strawberry.js', + '/project/fruits/tomato.js', + '/project/vegetables/melon.json', + ].join('\n'); + + const {hasteMap, removedFiles} = await nodeCrawl({ + data: { + files: new Map(), + }, + extensions: ['js', 'json'], + ignore: pearMatcher, + rootDir, + roots: ['/project/fruits', '/project/vegtables'], + }); + + expect(childProcess.spawn).lastCalledWith('find', [ + '/project/fruits', + '/project/vegtables', + '-type', + 'f', + '(', + '-iname', + '*.js', + '-o', + '-iname', + '*.json', + ')', + ]); + + expect(hasteMap.files).not.toBe(null); + + expect(hasteMap.files).toEqual( + createMap({ + 'fruits/strawberry.js': ['', 32, 42, 0, '', null], + 'fruits/tomato.js': ['', 33, 42, 0, '', null], + 'vegetables/melon.json': ['', 34, 42, 0, '', null], + }), + ); + + expect(removedFiles).toEqual(new Map()); + }); + + it('updates only changed files', async () => { + nodeCrawl = require('../node'); + + // In this test sample, strawberry is changed and tomato is unchanged + const tomato = ['', 33, 42, 1, '', null]; + const files = createMap({ + 'fruits/strawberry.js': ['', 30, 40, 1, '', null], + 'fruits/tomato.js': tomato, + }); + + const {hasteMap, removedFiles} = await nodeCrawl({ + data: {files}, + extensions: ['js'], + ignore: pearMatcher, + rootDir, + roots: ['/project/fruits'], + }); + + expect(hasteMap.files).toEqual( + createMap({ + 'fruits/strawberry.js': ['', 32, 42, 0, '', null], + 'fruits/tomato.js': tomato, + }), + ); + + // Make sure it is the *same* unchanged object. + expect(hasteMap.files.get(normalize('fruits/tomato.js'))).toBe(tomato); + + expect(removedFiles).toEqual(new Map()); + }); + + it('returns removed files', async () => { + nodeCrawl = require('../node'); + + // In this test sample, previouslyExisted was present before and will not be + // when crawling this directory. + const files = createMap({ + 'fruits/previouslyExisted.js': ['', 30, 40, 1, '', null], + 'fruits/strawberry.js': ['', 33, 42, 0, '', null], + 'fruits/tomato.js': ['', 32, 42, 0, '', null], + }); + + const {hasteMap, removedFiles} = await nodeCrawl({ + data: {files}, + extensions: ['js'], + ignore: pearMatcher, + rootDir, + roots: ['/project/fruits'], + }); + + expect(hasteMap.files).toEqual( + createMap({ + 'fruits/strawberry.js': ['', 32, 42, 0, '', null], + 'fruits/tomato.js': ['', 33, 42, 0, '', null], + }), + ); + expect(removedFiles).toEqual( + createMap({ + 'fruits/previouslyExisted.js': ['', 30, 40, 1, '', null], + }), + ); + }); + + it('uses node fs APIs with incompatible find binary', async () => { + mockResponse = ''; + mockSpawnExit = 1; + childProcess = require('child_process'); + + nodeCrawl = require('../node'); + + const {hasteMap, removedFiles} = await nodeCrawl({ + data: { + files: new Map(), + }, + extensions: ['js'], + ignore: pearMatcher, + rootDir, + roots: ['/project/fruits'], + }); + + expect(childProcess.spawn).lastCalledWith( + 'find', + ['.', '-type', 'f', '(', '-iname', '*.ts', '-o', '-iname', '*.js', ')'], + {cwd: expect.any(String)}, + ); + expect(hasteMap.files).toEqual( + createMap({ + 'fruits/directory/strawberry.js': ['', 33, 42, 0, '', null], + 'fruits/tomato.js': ['', 32, 42, 0, '', null], + }), + ); + expect(removedFiles).toEqual(new Map()); + }); + + it('uses node fs APIs without find binary', async () => { + childProcess = require('child_process'); + childProcess.spawn.mockImplementationOnce(() => { + throw new Error(); + }); + nodeCrawl = require('../node'); + + const {hasteMap, removedFiles} = await nodeCrawl({ + data: { + files: new Map(), + }, + extensions: ['js'], + ignore: pearMatcher, + rootDir, + roots: ['/project/fruits'], + }); + + expect(hasteMap.files).toEqual( + createMap({ + 'fruits/directory/strawberry.js': ['', 33, 42, 0, '', null], + 'fruits/tomato.js': ['', 32, 42, 0, '', null], + }), + ); + expect(removedFiles).toEqual(new Map()); + }); + + it('uses node fs APIs if "forceNodeFilesystemAPI" is set to true, regardless of platform', async () => { + childProcess = require('child_process'); + nodeCrawl = require('../node'); + + const files = new Map(); + const {hasteMap, removedFiles} = await nodeCrawl({ + data: {files}, + extensions: ['js'], + forceNodeFilesystemAPI: true, + ignore: pearMatcher, + rootDir, + roots: ['/project/fruits'], + }); + + expect(childProcess.spawn).toHaveBeenCalledTimes(0); + expect(hasteMap.files).toEqual( + createMap({ + 'fruits/directory/strawberry.js': ['', 33, 42, 0, '', null], + 'fruits/tomato.js': ['', 32, 42, 0, '', null], + }), + ); + expect(removedFiles).toEqual(new Map()); + }); + + it('completes with empty roots', async () => { + nodeCrawl = require('../node'); + + const files = new Map(); + const {hasteMap, removedFiles} = await nodeCrawl({ + data: {files}, + extensions: ['js'], + forceNodeFilesystemAPI: true, + ignore: pearMatcher, + rootDir, + roots: [], + }); + + expect(hasteMap.files).toEqual(new Map()); + expect(removedFiles).toEqual(new Map()); + }); + + it('completes with fs.readdir throwing an error', async () => { + nodeCrawl = require('../node'); + + const files = new Map(); + const {hasteMap, removedFiles} = await nodeCrawl({ + data: {files}, + extensions: ['js'], + forceNodeFilesystemAPI: true, + ignore: pearMatcher, + rootDir, + roots: ['/error'], + }); + + expect(hasteMap.files).toEqual(new Map()); + expect(removedFiles).toEqual(new Map()); + }); + + describe('readdir withFileTypes support', () => { + it('calls lstat for directories and symlinks if readdir withFileTypes is not supported', async () => { + nodeCrawl = require('../node'); + const fs = require('graceful-fs'); + + const files = new Map(); + const {hasteMap, removedFiles} = await nodeCrawl({ + data: {files}, + extensions: ['js'], + forceNodeFilesystemAPI: true, + ignore: pearMatcher, + rootDir, + roots: ['/project/fruits'], + }); + + expect(hasteMap.files).toEqual( + createMap({ + 'fruits/directory/strawberry.js': ['', 33, 42, 0, '', null], + 'fruits/tomato.js': ['', 32, 42, 0, '', null], + }), + ); + expect(removedFiles).toEqual(new Map()); + // once for /project/fruits, once for /project/fruits/directory + expect(fs.readdir).toHaveBeenCalledTimes(2); + // once for each of: + // 1. /project/fruits/directory + // 2. /project/fruits/directory/strawberry.js + // 3. /project/fruits/tomato.js + // 4. /project/fruits/symlink + // (we never call lstat on the root /project/fruits, since we know it's a directory) + expect(fs.lstat).toHaveBeenCalledTimes(4); + }); + + it('avoids calling lstat for directories and symlinks if readdir withFileTypes is supported', async () => { + mockHasReaddirWithFileTypesSupport = true; + nodeCrawl = require('../node'); + const fs = require('graceful-fs'); + + const files = new Map(); + const {hasteMap, removedFiles} = await nodeCrawl({ + data: {files}, + extensions: ['js'], + forceNodeFilesystemAPI: true, + ignore: pearMatcher, + rootDir, + roots: ['/project/fruits'], + }); + + expect(hasteMap.files).toEqual( + createMap({ + 'fruits/directory/strawberry.js': ['', 33, 42, 0, '', null], + 'fruits/tomato.js': ['', 32, 42, 0, '', null], + }), + ); + expect(removedFiles).toEqual(new Map()); + // once for /project/fruits, once for /project/fruits/directory + expect(fs.readdir).toHaveBeenCalledTimes(2); + // once for strawberry.js, once for tomato.js + expect(fs.lstat).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js b/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js new file mode 100644 index 0000000000..9553dc2d24 --- /dev/null +++ b/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js @@ -0,0 +1,677 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +'use strict'; + +const path = require('path'); + +jest.mock('fb-watchman', () => { + const normalizePathSep = require('../../lib/normalizePathSep').default; + const Client = jest.fn(); + Client.prototype.capabilityCheck = jest.fn((args, callback) => + setImmediate(() => { + callback(null, { + capabilities: {'suffix-set': true}, + version: '2021.06.07.00', + }); + }), + ); + Client.prototype.command = jest.fn((args, callback) => + setImmediate(() => { + const path = args[1] ? normalizePathSep(args[1]) : undefined; + const response = mockResponse[args[0]][path]; + callback(null, response.next ? response.next().value : response); + }), + ); + Client.prototype.on = jest.fn(); + Client.prototype.end = jest.fn(); + return {Client}; +}); + +const forcePOSIXPaths = path => path.replace(/\\/g, '/'); +const pearMatcher = path => /pear/.test(path); + +let watchman; +let watchmanCrawl; +let mockResponse; +let mockFiles; + +const ROOT_MOCK = path.sep === '/' ? '/root-mock' : 'M:\\root-mock'; +const FRUITS_RELATIVE = 'fruits'; +const VEGETABLES_RELATIVE = 'vegetables'; +const FRUITS = path.resolve(ROOT_MOCK, FRUITS_RELATIVE); +const VEGETABLES = path.resolve(ROOT_MOCK, VEGETABLES_RELATIVE); +const ROOTS = [FRUITS, VEGETABLES]; +const BANANA_RELATIVE = path.join(FRUITS_RELATIVE, 'banana.js'); +const STRAWBERRY_RELATIVE = path.join(FRUITS_RELATIVE, 'strawberry.js'); +const KIWI_RELATIVE = path.join(FRUITS_RELATIVE, 'kiwi.js'); +const TOMATO_RELATIVE = path.join(FRUITS_RELATIVE, 'tomato.js'); +const MELON_RELATIVE = path.join(VEGETABLES_RELATIVE, 'melon.json'); + +const WATCH_PROJECT_MOCK = { + [FRUITS]: { + relative_path: 'fruits', + watch: forcePOSIXPaths(ROOT_MOCK), + }, + [VEGETABLES]: { + relative_path: 'vegetables', + watch: forcePOSIXPaths(ROOT_MOCK), + }, +}; + +const createMap = obj => new Map(Object.keys(obj).map(key => [key, obj[key]])); + +describe('watchman watch', () => { + beforeEach(() => { + watchmanCrawl = require('../watchman'); + + watchman = require('fb-watchman'); + + mockResponse = { + 'list-capabilities': { + [undefined]: { + capabilities: ['field-content.sha1hex'], + }, + }, + query: { + [ROOT_MOCK]: { + clock: 'c:fake-clock:1', + files: [ + { + exists: true, + mtime_ms: {toNumber: () => 30}, + name: 'fruits/strawberry.js', + size: 40, + }, + { + exists: true, + mtime_ms: {toNumber: () => 31}, + name: 'fruits/tomato.js', + size: 41, + }, + { + exists: true, + mtime_ms: {toNumber: () => 32}, + name: 'fruits/pear.js', + size: 42, + }, + { + exists: true, + mtime_ms: {toNumber: () => 33}, + name: 'vegetables/melon.json', + size: 43, + }, + ], + is_fresh_instance: true, + version: '4.5.0', + }, + }, + 'watch-project': WATCH_PROJECT_MOCK, + }; + + mockFiles = createMap({ + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], + [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null], + }); + }); + + afterEach(() => { + watchman.Client.mock.instances[0].command.mockClear(); + }); + + test('returns a list of all files when there are no clocks', async () => { + const {changedFiles, hasteMap, removedFiles} = await watchmanCrawl({ + data: { + clocks: new Map(), + files: new Map(), + }, + extensions: ['js', 'json'], + ignore: pearMatcher, + rootDir: ROOT_MOCK, + roots: ROOTS, + }); + const client = watchman.Client.mock.instances[0]; + const calls = client.command.mock.calls; + + expect(client.on).toBeCalled(); + expect(client.on).toBeCalledWith('error', expect.any(Function)); + + // Call 0 and 1 are for ['watch-project'] + expect(calls[0][0][0]).toEqual('watch-project'); + expect(calls[1][0][0]).toEqual('watch-project'); + + // Call 2 is the query + const query = calls[2][0]; + expect(query[0]).toEqual('query'); + + expect(query[2].expression).toEqual([ + 'allof', + ['type', 'f'], + ['suffix', ['js', 'json']], + ]); + + expect(query[2].fields).toEqual(['name', 'exists', 'mtime_ms', 'size']); + + expect(query[2].glob).toEqual(['fruits/**', 'vegetables/**']); + + expect(hasteMap.clocks).toEqual( + createMap({ + '': 'c:fake-clock:1', + }), + ); + + expect(changedFiles).toEqual(undefined); + + expect(hasteMap.files).toEqual(mockFiles); + + expect(removedFiles).toEqual(new Map()); + + expect(client.end).toBeCalled(); + }); + + test('updates file map and removedFiles when the clock is given', async () => { + mockResponse = { + 'list-capabilities': { + [undefined]: { + capabilities: ['field-content.sha1hex'], + }, + }, + query: { + [ROOT_MOCK]: { + clock: 'c:fake-clock:2', + files: [ + { + exists: true, + mtime_ms: {toNumber: () => 42}, + name: 'fruits/kiwi.js', + size: 40, + }, + { + exists: false, + mtime_ms: null, + name: 'fruits/tomato.js', + size: 0, + }, + ], + is_fresh_instance: false, + version: '4.5.0', + }, + }, + 'watch-project': WATCH_PROJECT_MOCK, + }; + + const clocks = createMap({ + '': 'c:fake-clock:1', + }); + + const {changedFiles, hasteMap, removedFiles} = await watchmanCrawl({ + data: { + clocks, + files: mockFiles, + }, + extensions: ['js', 'json'], + ignore: pearMatcher, + rootDir: ROOT_MOCK, + roots: ROOTS, + }); + + // The object was reused. + expect(hasteMap.files).toBe(mockFiles); + + expect(hasteMap.clocks).toEqual( + createMap({ + '': 'c:fake-clock:2', + }), + ); + + expect(changedFiles).toEqual( + createMap({ + [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], + }), + ); + + expect(hasteMap.files).toEqual( + createMap({ + [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], + }), + ); + + expect(removedFiles).toEqual( + createMap({ + [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null], + }), + ); + }); + + test('resets the file map and tracks removedFiles when watchman is fresh', async () => { + const mockTomatoSha1 = '321f6b7e8bf7f29aab89c5e41a555b1b0baa41a9'; + + mockResponse = { + 'list-capabilities': { + [undefined]: { + capabilities: ['field-content.sha1hex'], + }, + }, + query: { + [ROOT_MOCK]: { + clock: 'c:fake-clock:3', + files: [ + { + exists: true, + mtime_ms: {toNumber: () => 42}, + name: 'fruits/kiwi.js', + size: 52, + }, + { + exists: true, + mtime_ms: {toNumber: () => 41}, + name: 'fruits/banana.js', + size: 51, + }, + { + 'content.sha1hex': mockTomatoSha1, + exists: true, + mtime_ms: {toNumber: () => 76}, + name: 'fruits/tomato.js', + size: 41, + }, + ], + is_fresh_instance: true, + version: '4.5.0', + }, + }, + 'watch-project': WATCH_PROJECT_MOCK, + }; + + const mockBananaMetadata = ['Banana', 41, 51, 1, ['Raspberry'], null]; + mockFiles.set(BANANA_RELATIVE, mockBananaMetadata); + const mockTomatoMetadata = ['Tomato', 31, 41, 1, [], mockTomatoSha1]; + mockFiles.set(TOMATO_RELATIVE, mockTomatoMetadata); + + const clocks = createMap({ + '': 'c:fake-clock:1', + }); + + const {changedFiles, hasteMap, removedFiles} = await watchmanCrawl({ + data: { + clocks, + files: mockFiles, + }, + extensions: ['js', 'json'], + ignore: pearMatcher, + rootDir: ROOT_MOCK, + roots: ROOTS, + }); + + // The file object was *not* reused. + expect(hasteMap.files).not.toBe(mockFiles); + + expect(hasteMap.clocks).toEqual( + createMap({ + '': 'c:fake-clock:3', + }), + ); + + expect(changedFiles).toEqual(undefined); + + // strawberry and melon removed from the file list. + expect(hasteMap.files).toEqual( + createMap({ + [BANANA_RELATIVE]: mockBananaMetadata, + [KIWI_RELATIVE]: ['', 42, 52, 0, '', null], + [TOMATO_RELATIVE]: ['Tomato', 76, 41, 1, [], mockTomatoSha1], + }), + ); + + // Even though the file list was reset, old file objects are still reused + // if no changes have been made + expect(hasteMap.files.get(BANANA_RELATIVE)).toBe(mockBananaMetadata); + + // Old file objects are not reused if they have a different mtime + expect(hasteMap.files.get(TOMATO_RELATIVE)).not.toBe(mockTomatoMetadata); + + expect(removedFiles).toEqual( + createMap({ + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], + }), + ); + }); + + test('properly resets the file map when only one watcher is reset', async () => { + mockResponse = { + 'list-capabilities': { + [undefined]: { + capabilities: ['field-content.sha1hex'], + }, + }, + query: { + [FRUITS]: { + clock: 'c:fake-clock:3', + files: [ + { + exists: true, + mtime_ms: {toNumber: () => 42}, + name: 'kiwi.js', + size: 52, + }, + ], + is_fresh_instance: false, + version: '4.5.0', + }, + [VEGETABLES]: { + clock: 'c:fake-clock:4', + files: [ + { + exists: true, + mtime_ms: {toNumber: () => 33}, + name: 'melon.json', + size: 43, + }, + ], + is_fresh_instance: true, + version: '4.5.0', + }, + }, + 'watch-project': { + [FRUITS]: { + watch: forcePOSIXPaths(FRUITS), + }, + [VEGETABLES]: { + watch: forcePOSIXPaths(VEGETABLES), + }, + }, + }; + + const clocks = createMap({ + [FRUITS_RELATIVE]: 'c:fake-clock:1', + [VEGETABLES_RELATIVE]: 'c:fake-clock:2', + }); + + const {changedFiles, hasteMap, removedFiles} = await watchmanCrawl({ + data: { + clocks, + files: mockFiles, + }, + extensions: ['js', 'json'], + ignore: pearMatcher, + rootDir: ROOT_MOCK, + roots: ROOTS, + }); + + expect(hasteMap.clocks).toEqual( + createMap({ + [FRUITS_RELATIVE]: 'c:fake-clock:3', + [VEGETABLES_RELATIVE]: 'c:fake-clock:4', + }), + ); + + expect(changedFiles).toEqual(undefined); + + expect(hasteMap.files).toEqual( + createMap({ + [KIWI_RELATIVE]: ['', 42, 52, 0, '', null], + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], + }), + ); + + expect(removedFiles).toEqual( + createMap({ + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], + [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null], + }), + ); + }); + + test('does not add directory filters to query when watching a ROOT', async () => { + mockResponse = { + 'list-capabilities': { + [undefined]: { + capabilities: ['field-content.sha1hex'], + }, + }, + query: { + [ROOT_MOCK]: { + clock: 'c:fake-clock:1', + files: [], + is_fresh_instance: false, + version: '4.5.0', + }, + }, + 'watch-project': { + [FRUITS]: { + relative_path: 'fruits', + watch: forcePOSIXPaths(ROOT_MOCK), + }, + [ROOT_MOCK]: { + watch: forcePOSIXPaths(ROOT_MOCK), + }, + [VEGETABLES]: { + relative_path: 'vegetables', + watch: forcePOSIXPaths(ROOT_MOCK), + }, + }, + }; + + const {changedFiles, hasteMap, removedFiles} = await watchmanCrawl({ + data: { + clocks: new Map(), + files: new Map(), + }, + extensions: ['js', 'json'], + ignore: pearMatcher, + rootDir: ROOT_MOCK, + roots: [...ROOTS, ROOT_MOCK], + }); + + const client = watchman.Client.mock.instances[0]; + const calls = client.command.mock.calls; + + expect(client.on).toBeCalled(); + expect(client.on).toBeCalledWith('error', expect.any(Function)); + + // First 3 calls are for ['watch-project'] + expect(calls[0][0][0]).toEqual('watch-project'); + expect(calls[1][0][0]).toEqual('watch-project'); + expect(calls[2][0][0]).toEqual('watch-project'); + + // Call 4 is the query + const query = calls[3][0]; + expect(query[0]).toEqual('query'); + + expect(query[2].expression).toEqual(['allof', ['type', 'f']]); + + expect(query[2].fields).toEqual(['name', 'exists', 'mtime_ms', 'size']); + + expect(query[2].suffix).toEqual(['js', 'json']); + + expect(hasteMap.clocks).toEqual( + createMap({ + '': 'c:fake-clock:1', + }), + ); + + expect(changedFiles).toEqual(new Map()); + + expect(hasteMap.files).toEqual(new Map()); + + expect(removedFiles).toEqual(new Map()); + + expect(client.end).toBeCalled(); + }); + + test('SHA-1 requested and available', async () => { + mockResponse = { + 'list-capabilities': { + [undefined]: { + capabilities: ['field-content.sha1hex'], + }, + }, + query: { + [ROOT_MOCK]: { + clock: 'c:fake-clock:1', + files: [], + is_fresh_instance: false, + version: '4.5.0', + }, + }, + 'watch-project': { + [ROOT_MOCK]: { + watch: forcePOSIXPaths(ROOT_MOCK), + }, + }, + }; + + await watchmanCrawl({ + computeSha1: true, + data: { + clocks: new Map(), + files: new Map(), + }, + extensions: ['js', 'json'], + rootDir: ROOT_MOCK, + roots: [ROOT_MOCK], + }); + + const client = watchman.Client.mock.instances[0]; + const calls = client.command.mock.calls; + + expect(calls[0][0]).toEqual(['list-capabilities']); + expect(calls[2][0][2].fields).toContain('content.sha1hex'); + }); + + test('SHA-1 requested and NOT available', async () => { + mockResponse = { + 'list-capabilities': { + [undefined]: { + capabilities: [], + }, + }, + query: { + [ROOT_MOCK]: { + clock: 'c:fake-clock:1', + files: [], + is_fresh_instance: false, + version: '4.5.0', + }, + }, + 'watch-project': { + [ROOT_MOCK]: { + watch: forcePOSIXPaths(ROOT_MOCK), + }, + }, + }; + + await watchmanCrawl({ + computeSha1: true, + data: { + clocks: new Map(), + files: new Map(), + }, + extensions: ['js', 'json'], + rootDir: ROOT_MOCK, + roots: [ROOT_MOCK], + }); + + const client = watchman.Client.mock.instances[0]; + const calls = client.command.mock.calls; + + expect(calls[0][0]).toEqual(['list-capabilities']); + expect(calls[2][0][2].fields).not.toContain('content.sha1hex'); + }); + + test('source control query', async () => { + mockResponse = { + 'list-capabilities': { + [undefined]: { + capabilities: ['field-content.sha1hex'], + }, + }, + query: { + [ROOT_MOCK]: { + clock: { + clock: 'c:1608612057:79675:1:139410', + scm: { + mergebase: 'master', + 'mergebase-with': 'master', + }, + }, + files: [ + { + exists: true, + mtime_ms: {toNumber: () => 42}, + name: 'fruits/kiwi.js', + size: 40, + }, + { + exists: false, + mtime_ms: null, + name: 'fruits/tomato.js', + size: 0, + }, + ], + // Watchman is going to tell us that we have a fresh instance. + is_fresh_instance: true, + version: '4.5.0', + }, + }, + 'watch-project': WATCH_PROJECT_MOCK, + }; + + // Start with a source-control clock. + const clocks = createMap({ + '': {scm: {'mergebase-with': 'master'}}, + }); + + const {changedFiles, hasteMap, removedFiles} = await watchmanCrawl({ + data: { + clocks, + files: mockFiles, + }, + extensions: ['js', 'json'], + ignore: pearMatcher, + rootDir: ROOT_MOCK, + roots: ROOTS, + }); + + // The object was reused. + expect(hasteMap.files).toBe(mockFiles); + + // Transformed into a normal clock. + expect(hasteMap.clocks).toEqual( + createMap({ + '': 'c:1608612057:79675:1:139410', + }), + ); + + expect(changedFiles).toEqual( + createMap({ + [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], + }), + ); + + expect(hasteMap.files).toEqual( + createMap({ + [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], + }), + ); + + expect(removedFiles).toEqual( + createMap({ + [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null], + }), + ); + }); +}); diff --git a/packages/metro-file-map/src/crawlers/node.js b/packages/metro-file-map/src/crawlers/node.js new file mode 100644 index 0000000000..4ad97676ec --- /dev/null +++ b/packages/metro-file-map/src/crawlers/node.js @@ -0,0 +1,255 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + */ + +import type { + CrawlerOptions, + FileData, + IgnoreMatcher, + InternalHasteMap, +} from '../flow-types'; + +import H from '../constants'; +import * as fastPath from '../lib/fast_path'; +import {spawn} from 'child_process'; +import * as fs from 'graceful-fs'; +import * as path from 'path'; + +type Result = Array<[/* id */ string, /* mtime */ number, /* size */ number]>; + +type Callback = (result: Result) => void; + +async function hasNativeFindSupport( + forceNodeFilesystemAPI: boolean, +): Promise { + if (forceNodeFilesystemAPI) { + return false; + } + + try { + return await new Promise(resolve => { + // Check the find binary supports the non-POSIX -iname parameter wrapped in parens. + const args = [ + '.', + '-type', + 'f', + '(', + '-iname', + '*.ts', + '-o', + '-iname', + '*.js', + ')', + ]; + const child = spawn('find', args, {cwd: __dirname}); + child.on('error', () => { + resolve(false); + }); + child.on('exit', code => { + resolve(code === 0); + }); + }); + } catch { + return false; + } +} + +function find( + roots: $ReadOnlyArray, + extensions: $ReadOnlyArray, + ignore: IgnoreMatcher, + enableSymlinks: boolean, + callback: Callback, +): void { + const result: Result = []; + let activeCalls = 0; + + function search(directory: string): void { + activeCalls++; + fs.readdir(directory, {withFileTypes: true}, (err, entries) => { + activeCalls--; + if (err) { + callback(result); + return; + } + // node < v10.10 does not support the withFileTypes option, and + // entry will be a string. + entries.forEach((entry: string | fs.Dirent) => { + const file = path.join( + directory, + // $FlowFixMe[incompatible-call] - encoding is utf8 + typeof entry === 'string' ? entry : entry.name, + ); + + if (ignore(file)) { + return; + } + + if (typeof entry !== 'string') { + if (entry.isSymbolicLink()) { + return; + } + + if (entry.isDirectory()) { + search(file); + return; + } + } + + activeCalls++; + + const stat = enableSymlinks ? fs.stat : fs.lstat; + + stat(file, (err, stat) => { + activeCalls--; + + // This logic is unnecessary for node > v10.10, but leaving it in + // since we need it for backwards-compatibility still. + if (!err && stat && !stat.isSymbolicLink()) { + if (stat.isDirectory()) { + search(file); + } else { + const ext = path.extname(file).substr(1); + if (extensions.indexOf(ext) !== -1) { + result.push([file, stat.mtime.getTime(), stat.size]); + } + } + } + + if (activeCalls === 0) { + callback(result); + } + }); + }); + + if (activeCalls === 0) { + callback(result); + } + }); + } + + if (roots.length > 0) { + roots.forEach(search); + } else { + callback(result); + } +} + +function findNative( + roots: $ReadOnlyArray, + extensions: $ReadOnlyArray, + ignore: IgnoreMatcher, + enableSymlinks: boolean, + callback: Callback, +): void { + const args = Array.from(roots); + if (enableSymlinks) { + args.push('(', '-type', 'f', '-o', '-type', 'l', ')'); + } else { + args.push('-type', 'f'); + } + + if (extensions.length) { + args.push('('); + } + extensions.forEach((ext, index) => { + if (index) { + args.push('-o'); + } + args.push('-iname'); + args.push('*.' + ext); + }); + if (extensions.length) { + args.push(')'); + } + + const child = spawn('find', args); + let stdout = ''; + if (child.stdout == null) { + throw new Error( + 'stdout is null - this should never happen. Please open up an issue at https://github.com/facebook/metro', + ); + } + child.stdout.setEncoding('utf-8'); + child.stdout.on('data', data => (stdout += data)); + + child.stdout.on('close', () => { + const lines = stdout + .trim() + .split('\n') + .filter(x => !ignore(x)); + const result: Result = []; + let count = lines.length; + if (!count) { + callback([]); + } else { + lines.forEach(path => { + fs.stat(path, (err, stat) => { + // Filter out symlinks that describe directories + if (!err && stat && !stat.isDirectory()) { + result.push([path, stat.mtime.getTime(), stat.size]); + } + if (--count === 0) { + callback(result); + } + }); + }); + } + }); +} + +module.exports = async function nodeCrawl(options: CrawlerOptions): Promise<{ + removedFiles: FileData, + hasteMap: InternalHasteMap, +}> { + const { + data, + extensions, + forceNodeFilesystemAPI, + ignore, + rootDir, + enableSymlinks, + perfLogger, + roots, + } = options; + perfLogger?.markerPoint('nodeCrawl_start'); + const useNativeFind = await hasNativeFindSupport(forceNodeFilesystemAPI); + + return new Promise(resolve => { + const callback = (list: Result) => { + const files = new Map(); + const removedFiles = new Map(data.files); + list.forEach(fileData => { + const [filePath, mtime, size] = fileData; + const relativeFilePath = fastPath.relative(rootDir, filePath); + const existingFile = data.files.get(relativeFilePath); + if (existingFile && existingFile[H.MTIME] === mtime) { + files.set(relativeFilePath, existingFile); + } else { + // See ../constants.js; SHA-1 will always be null and fulfilled later. + files.set(relativeFilePath, ['', mtime, size, 0, '', null]); + } + removedFiles.delete(relativeFilePath); + }); + data.files = files; + + perfLogger?.markerPoint('nodeCrawl_end'); + resolve({ + hasteMap: data, + removedFiles, + }); + }; + + if (useNativeFind) { + findNative(roots, extensions, ignore, enableSymlinks, callback); + } else { + find(roots, extensions, ignore, enableSymlinks, callback); + } + }); +}; diff --git a/packages/metro-file-map/src/crawlers/watchman.js b/packages/metro-file-map/src/crawlers/watchman.js new file mode 100644 index 0000000000..1bee7f1383 --- /dev/null +++ b/packages/metro-file-map/src/crawlers/watchman.js @@ -0,0 +1,451 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + */ + +import type { + CrawlerOptions, + FileData, + FileMetaData, + InternalHasteMap, + Path, +} from '../flow-types'; + +import H from '../constants'; +import * as fastPath from '../lib/fast_path'; +import normalizePathSep from '../lib/normalizePathSep'; +import * as path from 'path'; + +const watchman = require('fb-watchman'); + +// $FlowFixMe[unclear-type] - Improve fb-watchman types to cover our uses +type WatchmanQuery = any; + +type WatchmanRoots = Map>; + +type WatchmanListCapabilitiesResponse = { + capabilities: Array, +}; + +type WatchmanCapabilityCheckResponse = { + // { 'suffix-set': true } + capabilities: $ReadOnly<{[string]: boolean}>, + // '2021.06.07.00' + version: string, +}; + +type WatchmanWatchProjectResponse = { + watch: string, + relative_path: string, +}; + +type WatchmanQueryResponse = { + warning?: string, + is_fresh_instance: boolean, + version: string, + clock: + | string + | { + scm: {'mergebase-with': string, mergebase: string}, + clock: string, + }, + files: Array<{ + name: string, + exists: boolean, + mtime_ms: number | {toNumber: () => number}, + size: number, + 'content.sha1hex'?: string, + }>, +}; + +const WATCHMAN_WARNING_INITIAL_DELAY_MILLISECONDS = 10000; +const WATCHMAN_WARNING_INTERVAL_MILLISECONDS = 20000; + +const watchmanURL = 'https://facebook.github.io/watchman/docs/troubleshooting'; + +function makeWatchmanError(error: Error): Error { + error.message = + `Watchman error: ${error.message.trim()}. Make sure watchman ` + + `is running for this project. See ${watchmanURL}.`; + return error; +} + +/** + * Wrap watchman capabilityCheck method as a promise. + * + * @param client watchman client + * @param caps capabilities to verify + * @returns a promise resolving to a list of verified capabilities + */ +async function capabilityCheck( + client: watchman.Client, + caps: $ReadOnly<{optional?: $ReadOnlyArray}>, +): Promise { + return new Promise((resolve, reject) => { + client.capabilityCheck( + // @ts-expect-error: incorrectly typed + caps, + (error, response) => { + if (error) { + reject(error); + } else { + resolve(response); + } + }, + ); + }); +} + +module.exports = async function watchmanCrawl( + options: CrawlerOptions, +): Promise<{ + changedFiles?: FileData, + removedFiles: FileData, + hasteMap: InternalHasteMap, +}> { + const fields = ['name', 'exists', 'mtime_ms', 'size']; + const {data, extensions, ignore, rootDir, roots, perfLogger} = options; + const clocks = data.clocks; + + perfLogger?.markerPoint('watchmanCrawl_start'); + const client = new watchman.Client(); + + perfLogger?.markerPoint('watchmanCrawl/negotiateCapabilities_start'); + // https://facebook.github.io/watchman/docs/capabilities.html + // Check adds about ~28ms + const capabilities = await capabilityCheck(client, { + // If a required capability is missing then an error will be thrown, + // we don't need this assertion, so using optional instead. + optional: ['suffix-set'], + }); + + const suffixExpression = capabilities?.capabilities['suffix-set'] + ? // If available, use the optimized `suffix-set` operation: + // https://facebook.github.io/watchman/docs/expr/suffix.html#suffix-set + ['suffix', extensions] + : ['anyof', ...extensions.map(extension => ['suffix', extension])]; + + let clientError; + // $FlowFixMe[prop-missing] - Client is not typed as an EventEmitter + client.on('error', error => (clientError = makeWatchmanError(error))); + + let didLogWatchmanWaitMessage = false; + + // $FlowFixMe[unclear-type] - Fix to use fb-watchman types + const cmd = async (command: string, ...args: Array): Promise => { + const logWatchmanWaitMessage = () => { + didLogWatchmanWaitMessage = true; + console.warn(`Waiting for Watchman (${command})...`); + }; + let intervalOrTimeoutId: TimeoutID | IntervalID = setTimeout(() => { + logWatchmanWaitMessage(); + intervalOrTimeoutId = setInterval( + logWatchmanWaitMessage, + WATCHMAN_WARNING_INTERVAL_MILLISECONDS, + ); + }, WATCHMAN_WARNING_INITIAL_DELAY_MILLISECONDS); + try { + return await new Promise((resolve, reject) => + // $FlowFixMe[incompatible-call] - dynamic call of command + client.command([command, ...args], (error, result) => + error ? reject(makeWatchmanError(error)) : resolve(result), + ), + ); + } finally { + // $FlowFixMe[incompatible-call] clearInterval / clearTimeout are interchangeable + clearInterval(intervalOrTimeoutId); + } + }; + + if (options.computeSha1) { + const {capabilities} = await cmd( + 'list-capabilities', + ); + + if (capabilities.indexOf('field-content.sha1hex') !== -1) { + fields.push('content.sha1hex'); + } + } + + perfLogger?.markerPoint('watchmanCrawl/negotiateCapabilities_end'); + + async function getWatchmanRoots( + roots: $ReadOnlyArray, + ): Promise { + perfLogger?.markerPoint('watchmanCrawl/getWatchmanRoots_start'); + const watchmanRoots = new Map(); + await Promise.all( + roots.map(async (root, index) => { + perfLogger?.markerPoint(`watchmanCrawl/watchProject_${index}_start`); + const response = await cmd( + 'watch-project', + root, + ); + perfLogger?.markerPoint(`watchmanCrawl/watchProject_${index}_end`); + const existing = watchmanRoots.get(response.watch); + // A root can only be filtered if it was never seen with a + // relative_path before. + const canBeFiltered = !existing || existing.length > 0; + + if (canBeFiltered) { + if (response.relative_path) { + watchmanRoots.set( + response.watch, + (existing || []).concat(response.relative_path), + ); + } else { + // Make the filter directories an empty array to signal that this + // root was already seen and needs to be watched for all files or + // directories. + watchmanRoots.set(response.watch, []); + } + } + }), + ); + perfLogger?.markerPoint('watchmanCrawl/getWatchmanRoots_end'); + return watchmanRoots; + } + + async function queryWatchmanForDirs(rootProjectDirMappings: WatchmanRoots) { + perfLogger?.markerPoint('watchmanCrawl/queryWatchmanForDirs_start'); + const results = new Map(); + let isFresh = false; + await Promise.all( + Array.from(rootProjectDirMappings).map( + async ([root, directoryFilters], index) => { + // Jest is only going to store one type of clock; a string that + // represents a local clock. However, the Watchman crawler supports + // a second type of clock that can be written by automation outside of + // Jest, called an "scm query", which fetches changed files based on + // source control mergebases. The reason this is necessary is because + // local clocks are not portable across systems, but scm queries are. + // By using scm queries, we can create the haste map on a different + // system and import it, transforming the clock into a local clock. + const since = clocks.get(fastPath.relative(rootDir, root)); + + perfLogger?.markerAnnotate({ + bool: { + [`watchmanCrawl/query_${index}_has_clock`]: since != null, + }, + }); + + const query: WatchmanQuery = { + fields, + expression: [ + 'allof', + // Match regular files only. Different Watchman generators treat + // symlinks differently, so this ensures consistent results. + ['type', 'f'], + ], + }; + + /** + * Watchman "query planner". + * + * Watchman file queries consist of 1 or more generators that feed + * files through the expression evaluator. + * + * Strategy: + * 1. Select the narrowest possible generator so that the expression + * evaluator has fewer candidates to process. + * 2. Evaluate expressions from narrowest to broadest. + * 3. Don't use an expression to recheck a condition that the + * generator already guarantees. + * 4. Compose expressions to avoid combinatorial explosions in the + * number of terms. + * + * The ordering of generators/filters, from narrow to broad, is: + * - since = O(changes) + * - glob / dirname = O(files in a subtree of the repo) + * - suffix = O(files in the repo) + * + * We assume that file extensions are ~uniformly distributed in the + * repo but Haste map projects are focused on a handful of + * directories. Therefore `glob` < `suffix`. + */ + let queryGenerator: ?( + | $TEMPORARY$string<'glob'> + | $TEMPORARY$string<'since'> + | $TEMPORARY$string<'suffix'> + ) = undefined; + if (since != null) { + // Use the `since` generator and filter by both path and extension. + query.since = since; + queryGenerator = 'since'; + query.expression.push( + ['anyof', ...directoryFilters.map(dir => ['dirname', dir])], + suffixExpression, + ); + } else if (directoryFilters.length > 0) { + // Use the `glob` generator and filter only by extension. + query.glob = directoryFilters.map(directory => `${directory}/**`); + query.glob_includedotfiles = true; + queryGenerator = 'glob'; + + query.expression.push(suffixExpression); + } else { + // Use the `suffix` generator with no path/extension filtering. + query.suffix = extensions; + queryGenerator = 'suffix'; + } + + perfLogger?.markerAnnotate({ + string: { + [`watchmanCrawl/query_${index}_generator`]: queryGenerator, + }, + }); + + perfLogger?.markerPoint(`watchmanCrawl/query_${index}_start`); + const response = await cmd( + 'query', + root, + query, + ); + perfLogger?.markerPoint(`watchmanCrawl/query_${index}_end`); + + if ('warning' in response) { + console.warn('watchman warning: ', response.warning); + } + + // When a source-control query is used, we ignore the "is fresh" + // response from Watchman because it will be true despite the query + // being incremental. + const isSourceControlQuery = + typeof since !== 'string' && since?.scm?.['mergebase-with'] != null; + if (!isSourceControlQuery) { + isFresh = isFresh || response.is_fresh_instance; + } + + results.set(root, response); + }, + ), + ); + + perfLogger?.markerPoint('watchmanCrawl/queryWatchmanForDirs_end'); + + return { + isFresh, + results, + }; + } + + let files = data.files; + let removedFiles = new Map(); + const changedFiles = new Map(); + let results: Map; + let isFresh = false; + try { + const watchmanRoots = await getWatchmanRoots(roots); + const watchmanFileResults = await queryWatchmanForDirs(watchmanRoots); + + // Reset the file map if watchman was restarted and sends us a list of + // files. + if (watchmanFileResults.isFresh) { + files = new Map(); + removedFiles = new Map(data.files); + isFresh = true; + } + + results = watchmanFileResults.results; + } finally { + client.end(); + } + + if (clientError) { + perfLogger?.markerPoint('watchmanCrawl_end'); + throw clientError; + } + + perfLogger?.markerPoint('watchmanCrawl/processResults_start'); + + for (const [watchRoot, response] of results) { + const fsRoot = normalizePathSep(watchRoot); + const relativeFsRoot = fastPath.relative(rootDir, fsRoot); + clocks.set( + relativeFsRoot, + // Ensure we persist only the local clock. + typeof response.clock === 'string' + ? response.clock + : response.clock.clock, + ); + + for (const fileData of response.files) { + const filePath = fsRoot + path.sep + normalizePathSep(fileData.name); + const relativeFilePath = fastPath.relative(rootDir, filePath); + const existingFileData = data.files.get(relativeFilePath); + + // If watchman is fresh, the removed files map starts with all files + // and we remove them as we verify they still exist. + if (isFresh && existingFileData && fileData.exists) { + removedFiles.delete(relativeFilePath); + } + + if (!fileData.exists) { + // No need to act on files that do not exist and were not tracked. + if (existingFileData) { + files.delete(relativeFilePath); + + // If watchman is not fresh, we will know what specific files were + // deleted since we last ran and can track only those files. + if (!isFresh) { + removedFiles.set(relativeFilePath, existingFileData); + } + } + } else if (!ignore(filePath)) { + const mtime = + typeof fileData.mtime_ms === 'number' + ? fileData.mtime_ms + : fileData.mtime_ms.toNumber(); + const size = fileData.size; + + let sha1hex = fileData['content.sha1hex']; + if (typeof sha1hex !== 'string' || sha1hex.length !== 40) { + sha1hex = undefined; + } + + let nextData: FileMetaData; + + if (existingFileData && existingFileData[H.MTIME] === mtime) { + nextData = existingFileData; + } else if ( + existingFileData && + sha1hex != null && + existingFileData[H.SHA1] === sha1hex + ) { + nextData = [ + existingFileData[0], + mtime, + existingFileData[2], + existingFileData[3], + existingFileData[4], + existingFileData[5], + ]; + } else { + // See ../constants.ts + nextData = ['', mtime, size, 0, '', sha1hex ?? null]; + } + + files.set(relativeFilePath, nextData); + changedFiles.set(relativeFilePath, nextData); + } + } + } + + data.files = files; + + perfLogger?.markerPoint('watchmanCrawl/processResults_end'); + perfLogger?.markerPoint('watchmanCrawl_end'); + if (didLogWatchmanWaitMessage) { + console.warn('Watchman query finished.'); + } + return { + changedFiles: isFresh ? undefined : changedFiles, + hasteMap: data, + removedFiles, + }; +}; diff --git a/packages/metro-file-map/src/flow-types.js b/packages/metro-file-map/src/flow-types.js new file mode 100644 index 0000000000..11f95a717c --- /dev/null +++ b/packages/metro-file-map/src/flow-types.js @@ -0,0 +1,178 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + */ + +'use strict'; + +import type HasteFS from './HasteFS'; +import type ModuleMap from './ModuleMap'; +import type {Stats} from 'graceful-fs'; + +export type ChangeEvent = { + eventsQueue: EventsQueue, + hasteFS: HasteFS, + moduleMap: ModuleMap, +}; + +export type Console = typeof global.console; + +export type CrawlerOptions = { + computeSha1: boolean, + enableSymlinks: boolean, + data: InternalHasteMap, + extensions: $ReadOnlyArray, + forceNodeFilesystemAPI: boolean, + ignore: IgnoreMatcher, + perfLogger?: ?PerfLogger, + rootDir: string, + roots: $ReadOnlyArray, +}; + +export type DuplicatesSet = Map; +export type DuplicatesIndex = Map>; + +export type EventsQueue = Array<{ + filePath: Path, + stat?: ?Stats, + type: string, +}>; + +export type HasteMap = { + hasteFS: HasteFS, + moduleMap: ModuleMap, + __hasteMapForTest?: ?InternalHasteMap, +}; + +export type HasteMapStatic = { + getCacheFilePath( + tmpdir: Path, + name: string, + ...extra: $ReadOnlyArray + ): string, + getModuleMapFromJSON(json: S): IModuleMap, +}; + +export type HasteRegExp = RegExp | ((str: string) => boolean); + +export type HType = { + ID: 0, + MTIME: 1, + SIZE: 2, + VISITED: 3, + DEPENDENCIES: 4, + SHA1: 5, + PATH: 0, + TYPE: 1, + MODULE: 0, + PACKAGE: 1, + GENERIC_PLATFORM: 'g', + NATIVE_PLATFORM: 'native', + DEPENDENCY_DELIM: '\0', +}; + +export type HTypeValue = $Values; + +export type IgnoreMatcher = (item: string) => boolean; + +export type InternalHasteMap = { + clocks: WatchmanClocks, + duplicates: DuplicatesIndex, + files: FileData, + map: ModuleMapData, + mocks: MockData, +}; + +export type FileData = Map; + +export type FileMetaData = [ + /* id */ string, + /* mtime */ number, + /* size */ number, + /* visited */ 0 | 1, + /* dependencies */ string, + /* sha1 */ ?string, +]; + +export interface IModuleMap { + getModule( + name: string, + platform?: ?string, + supportsNativePlatform?: ?boolean, + type?: ?HTypeValue, + ): ?Path; + + getPackage( + name: string, + platform: ?string, + _supportsNativePlatform: ?boolean, + ): ?Path; + + getMockModule(name: string): ?Path; + + getRawModuleMap(): RawModuleMap; + + toJSON(): S; +} + +export type MockData = Map; +export type ModuleMapData = Map; + +type ModuleMapItem = {[platform: string]: ModuleMetaData}; +export type ModuleMetaData = [/* path */ string, /* type */ number]; + +export type Path = string; + +export interface PerfLogger { + markerPoint(name: string): void; + markerAnnotate(annotations: PerfAnnotations): void; +} + +export type PerfAnnotations = $Shape<{ + string: {[key: string]: string}, + int: {[key: string]: number}, + double: {[key: string]: number}, + bool: {[key: string]: boolean}, + string_array: {[key: string]: Array}, + int_array: {[key: string]: Array}, + double_array: {[key: string]: Array}, + bool_array: {[key: string]: Array}, +}>; + +export type RawModuleMap = { + rootDir: Path, + duplicates: DuplicatesIndex, + map: ModuleMapData, + mocks: MockData, +}; + +export type SerializableModuleMap = { + duplicates: $ReadOnlyArray<[string, [string, [string, [string, number]]]]>, + map: $ReadOnlyArray<[string, ModuleMapItem]>, + mocks: $ReadOnlyArray<[string, Path]>, + rootDir: Path, +}; + +export type WatchmanClockSpec = string | {scm: {'mergebase-with': string}}; +export type WatchmanClocks = Map; + +export type WorkerMessage = $ReadOnly<{ + computeDependencies: boolean, + computeSha1: boolean, + dependencyExtractor?: ?string, + rootDir: string, + filePath: string, + hasteImplModulePath?: ?string, +}>; + +export type WorkerMetadata = $ReadOnly<{ + dependencies?: ?$ReadOnlyArray, + id?: ?string, + module?: ?ModuleMetaData, + sha1?: ?string, +}>; diff --git a/packages/metro-file-map/src/getMockName.js b/packages/metro-file-map/src/getMockName.js new file mode 100644 index 0000000000..93fba01bba --- /dev/null +++ b/packages/metro-file-map/src/getMockName.js @@ -0,0 +1,22 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @flow strict + */ + +import * as path from 'path'; + +const MOCKS_PATTERN = path.sep + '__mocks__' + path.sep; + +const getMockName = (filePath: string): string => { + const mockPath = filePath.split(MOCKS_PATTERN)[1]; + return mockPath + .substring(0, mockPath.lastIndexOf(path.extname(mockPath))) + .replace(/\\/g, '/'); +}; + +export default getMockName; diff --git a/packages/metro-file-map/src/index.js b/packages/metro-file-map/src/index.js index 47ea60dc8b..305fdfb64a 100644 --- a/packages/metro-file-map/src/index.js +++ b/packages/metro-file-map/src/index.js @@ -4,10 +4,1183 @@ * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. * + * @flow strict-local * @format - * @flow strict */ -'use strict'; +import type { + ChangeEvent, + Console, + CrawlerOptions, + EventsQueue, + FileData, + FileMetaData, + HasteMap as InternalHasteMapObject, + HasteMapStatic, + HasteRegExp, + HType, + InternalHasteMap, + MockData, + ModuleMapData, + ModuleMetaData, + Path, + PerfLogger, + SerializableModuleMap, + WorkerMetadata, +} from './flow-types'; +import type {Stats} from 'graceful-fs'; -throw new Error('Coming soon...'); +import H from './constants'; +import getMockName from './getMockName'; +import HasteFS from './HasteFS'; +import * as fastPath from './lib/fast_path'; +import getPlatformExtension from './lib/getPlatformExtension'; +import normalizePathSep from './lib/normalizePathSep'; +import HasteModuleMap from './ModuleMap'; +import FSEventsWatcher from './watchers/FSEventsWatcher'; +// $FlowFixMe[untyped-import] - it's a fork: https://github.com/facebook/jest/pull/10919 +import NodeWatcher from './watchers/NodeWatcher'; +// $FlowFixMe[untyped-import] - WatchmanWatcher +import WatchmanWatcher from './watchers/WatchmanWatcher'; +import {getSha1, worker} from './worker'; +import {execSync} from 'child_process'; +import {createHash} from 'crypto'; +import EventEmitter from 'events'; +import invariant from 'invariant'; +// $FlowFixMe[untyped-import] - jest-regex-util +import {escapePathForRegex} from 'jest-regex-util'; +// $FlowFixMe[untyped-import] - jest-serializer untyped (in OSS only) +import serializer from 'jest-serializer'; +// $FlowFixMe[untyped-import] - jest-worker +import {Worker} from 'jest-worker'; +import {tmpdir} from 'os'; +import * as path from 'path'; + +// $FlowFixMe[untyped-import] - requiring JSON +const {version: VERSION} = require('../package.json'); +const nodeCrawl = require('./crawlers/node'); +const watchmanCrawl = require('./crawlers/watchman'); + +type Options = { + cacheDirectory?: ?string, + computeDependencies?: ?boolean, + computeSha1?: ?boolean, + console?: ?Console, + dependencyExtractor?: ?string, + enableSymlinks?: ?boolean, + extensions: $ReadOnlyArray, + forceNodeFilesystemAPI?: boolean, + hasteImplModulePath?: ?string, + hasteMapModulePath?: ?string, + ignorePattern?: HasteRegExp, + maxWorkers: number, + mocksPattern?: ?string, + name: string, + perfLogger?: ?PerfLogger, + platforms: $ReadOnlyArray, + resetCache?: ?boolean, + retainAllFiles: boolean, + rootDir: string, + roots: $ReadOnlyArray, + skipPackageJson?: ?boolean, + throwOnModuleCollision?: ?boolean, + useWatchman?: ?boolean, + watch?: ?boolean, +}; + +type InternalOptions = { + cacheDirectory: string, + computeDependencies: boolean, + computeSha1: boolean, + dependencyExtractor: ?string, + enableSymlinks: boolean, + extensions: $ReadOnlyArray, + forceNodeFilesystemAPI: boolean, + hasteImplModulePath: ?string, + ignorePattern: HasteRegExp, + maxWorkers: number, + mocksPattern: ?RegExp, + name: string, + perfLogger: ?PerfLogger, + platforms: $ReadOnlyArray, + resetCache: ?boolean, + retainAllFiles: boolean, + rootDir: string, + roots: $ReadOnlyArray, + skipPackageJson: boolean, + throwOnModuleCollision: boolean, + useWatchman: boolean, + watch: boolean, +}; + +interface Watcher { + close(): Promise; +} + +type WorkerInterface = {worker: typeof worker, getSha1: typeof getSha1}; + +export {default as ModuleMap} from './ModuleMap'; +export type {SerializableModuleMap} from './flow-types'; +export type {IModuleMap} from './flow-types'; +export type {default as FS} from './HasteFS'; +export type {ChangeEvent, HasteMap as HasteMapObject} from './flow-types'; + +const CHANGE_INTERVAL = 30; +const MAX_WAIT_TIME = 240000; +const NODE_MODULES = path.sep + 'node_modules' + path.sep; +const PACKAGE_JSON = path.sep + 'package.json'; +const VCS_DIRECTORIES = ['.git', '.hg'] + .map(vcs => escapePathForRegex(path.sep + vcs + path.sep)) + .join('|'); + +const canUseWatchman = ((): boolean => { + try { + execSync('watchman --version', {stdio: ['ignore']}); + return true; + } catch {} + return false; +})(); + +/** + * HasteMap is a JavaScript implementation of Facebook's haste module system. + * + * This implementation is inspired by https://github.com/facebook/node-haste + * and was built with for high-performance in large code repositories with + * hundreds of thousands of files. This implementation is scalable and provides + * predictable performance. + * + * Because the haste map creation and synchronization is critical to startup + * performance and most tasks are blocked by I/O this class makes heavy use of + * synchronous operations. It uses worker processes for parallelizing file + * access and metadata extraction. + * + * The data structures created by `jest-haste-map` can be used directly from the + * cache without further processing. The metadata objects in the `files` and + * `map` objects contain cross-references: a metadata object from one can look + * up the corresponding metadata object in the other map. Note that in most + * projects, the number of files will be greater than the number of haste + * modules one module can refer to many files based on platform extensions. + * + * type HasteMap = { + * clocks: WatchmanClocks, + * files: {[filepath: string]: FileMetaData}, + * map: {[id: string]: ModuleMapItem}, + * mocks: {[id: string]: string}, + * } + * + * // Watchman clocks are used for query synchronization and file system deltas. + * type WatchmanClocks = {[filepath: string]: string}; + * + * type FileMetaData = { + * id: ?string, // used to look up module metadata objects in `map`. + * mtime: number, // check for outdated files. + * size: number, // size of the file in bytes. + * visited: boolean, // whether the file has been parsed or not. + * dependencies: Array, // all relative dependencies of this file. + * sha1: ?string, // SHA-1 of the file, if requested via options. + * }; + * + * // Modules can be targeted to a specific platform based on the file name. + * // Example: platform.ios.js and Platform.android.js will both map to the same + * // `Platform` module. The platform should be specified during resolution. + * type ModuleMapItem = {[platform: string]: ModuleMetaData}; + * + * // + * type ModuleMetaData = { + * path: string, // the path to look up the file object in `files`. + * type: string, // the module type (either `package` or `module`). + * }; + * + * Note that the data structures described above are conceptual only. The actual + * implementation uses arrays and constant keys for metadata storage. Instead of + * `{id: 'flatMap', mtime: 3421, size: 42, visited: true, dependencies: []}` the real + * representation is similar to `['flatMap', 3421, 42, 1, []]` to save storage space + * and reduce parse and write time of a big JSON blob. + * + * The HasteMap is created as follows: + * 1. read data from the cache or create an empty structure. + * + * 2. crawl the file system. + * * empty cache: crawl the entire file system. + * * cache available: + * * if watchman is available: get file system delta changes. + * * if watchman is unavailable: crawl the entire file system. + * * build metadata objects for every file. This builds the `files` part of + * the `HasteMap`. + * + * 3. parse and extract metadata from changed files. + * * this is done in parallel over worker processes to improve performance. + * * the worst case is to parse all files. + * * the best case is no file system access and retrieving all data from + * the cache. + * * the average case is a small number of changed files. + * + * 4. serialize the new `HasteMap` in a cache file. + * Worker processes can directly access the cache through `HasteMap.read()`. + * + */ +export default class HasteMap extends EventEmitter { + _buildPromise: ?Promise; + _cachePath: Path; + _changeInterval: ?IntervalID; + _console: Console; + _options: InternalOptions; + _watchers: Array; + _worker: ?WorkerInterface; + + static getStatic( + config: $ReadOnly<{haste: $ReadOnly<{hasteMapModulePath?: string}>}>, + ): HasteMapStatic<> { + if (config.haste.hasteMapModulePath != null) { + // $FlowFixMe[unsupported-syntax] - Dynamic require + return require(config.haste.hasteMapModulePath); + } + // $FlowFixMe[prop-missing] - returning static class + // $FlowFixMe[method-unbinding] - returning static class + return HasteMap; + } + + static create(options: Options): HasteMap { + if (options.hasteMapModulePath != null) { + // $FlowFixMe[unsupported-syntax] - Dynamic require + const CustomHasteMap = require(options.hasteMapModulePath); + return new CustomHasteMap(options); + } + return new HasteMap(options); + } + + constructor(options: Options) { + if (options.perfLogger) { + options.perfLogger?.markerPoint('constructor_start'); + } + super(); + + // $FlowFixMe[prop-missing] - ignorePattern is added later + this._options = { + cacheDirectory: options.cacheDirectory ?? tmpdir(), + computeDependencies: + options.computeDependencies == null + ? true + : options.computeDependencies, + computeSha1: options.computeSha1 || false, + dependencyExtractor: options.dependencyExtractor ?? null, + enableSymlinks: options.enableSymlinks || false, + extensions: options.extensions, + forceNodeFilesystemAPI: !!options.forceNodeFilesystemAPI, + hasteImplModulePath: options.hasteImplModulePath, + maxWorkers: options.maxWorkers, + mocksPattern: + options.mocksPattern != null && options.mocksPattern !== '' + ? new RegExp(options.mocksPattern) + : null, + name: options.name, + perfLogger: options.perfLogger, + platforms: options.platforms, + resetCache: options.resetCache, + retainAllFiles: options.retainAllFiles, + rootDir: options.rootDir, + roots: Array.from(new Set(options.roots)), + skipPackageJson: !!options.skipPackageJson, + throwOnModuleCollision: !!options.throwOnModuleCollision, + useWatchman: options.useWatchman == null ? true : options.useWatchman, + watch: !!options.watch, + }; + this._console = options.console || global.console; + + if (options.ignorePattern) { + const inputIgnorePattern = options.ignorePattern; + if (inputIgnorePattern instanceof RegExp) { + this._options.ignorePattern = new RegExp( + inputIgnorePattern.source.concat('|' + VCS_DIRECTORIES), + inputIgnorePattern.flags, + ); + } else { + throw new Error( + 'jest-haste-map: the `ignorePattern` option must be a RegExp', + ); + } + } else { + this._options.ignorePattern = new RegExp(VCS_DIRECTORIES); + } + + if (this._options.enableSymlinks && this._options.useWatchman) { + throw new Error( + 'jest-haste-map: enableSymlinks config option was set, but ' + + 'is incompatible with watchman.\n' + + 'Set either `enableSymlinks` to false or `useWatchman` to false.', + ); + } + + const rootDirHash = createHash('md5').update(options.rootDir).digest('hex'); + let hasteImplHash = ''; + let dependencyExtractorHash = ''; + + if (options.hasteImplModulePath != null) { + // $FlowFixMe[unsupported-syntax] - Dynamic require + const hasteImpl = require(options.hasteImplModulePath); + if (hasteImpl.getCacheKey) { + hasteImplHash = String(hasteImpl.getCacheKey()); + } + } + + if (options.dependencyExtractor != null) { + // $FlowFixMe[unsupported-syntax] - Dynamic require + const dependencyExtractor = require(options.dependencyExtractor); + if (dependencyExtractor.getCacheKey) { + dependencyExtractorHash = String(dependencyExtractor.getCacheKey()); + } + } + + this._cachePath = HasteMap.getCacheFilePath( + this._options.cacheDirectory, + `haste-map-${this._options.name}-${rootDirHash}`, + VERSION, + this._options.name, + this._options.roots + .map(root => fastPath.relative(options.rootDir, root)) + .join(':'), + this._options.extensions.join(':'), + this._options.platforms.join(':'), + this._options.computeSha1.toString(), + options.mocksPattern || '', + (options.ignorePattern || '').toString(), + hasteImplHash, + dependencyExtractorHash, + this._options.computeDependencies.toString(), + ); + this._buildPromise = null; + this._watchers = []; + this._worker = null; + this._options.perfLogger?.markerPoint('constructor_end'); + } + + static getCacheFilePath( + tmpdir: Path, + name: string, + ...extra: Array + ): string { + const hash = createHash('md5').update(extra.join('')); + return path.join( + tmpdir, + name.replace(/\W/g, '-') + '-' + hash.digest('hex'), + ); + } + + static getModuleMapFromJSON(json: SerializableModuleMap): HasteModuleMap { + return HasteModuleMap.fromJSON(json); + } + + getCacheFilePath(): string { + return this._cachePath; + } + + build(): Promise { + this._options.perfLogger?.markerPoint('build_start'); + if (!this._buildPromise) { + this._buildPromise = (async () => { + const data = await this._buildFileMap(); + + // Persist when we don't know if files changed (changedFiles undefined) + // or when we know a file was changed or deleted. + let hasteMap: InternalHasteMap; + if ( + data.changedFiles == null || + data.changedFiles.size > 0 || + data.removedFiles.size > 0 + ) { + hasteMap = await this._buildHasteMap(data); + this._persist(hasteMap); + } else { + hasteMap = data.hasteMap; + } + + const rootDir = this._options.rootDir; + const hasteFS = new HasteFS({ + files: hasteMap.files, + rootDir, + }); + const moduleMap = new HasteModuleMap({ + duplicates: hasteMap.duplicates, + map: hasteMap.map, + mocks: hasteMap.mocks, + rootDir, + }); + const __hasteMapForTest = + (process.env.NODE_ENV === 'test' && hasteMap) || null; + await this._watch(hasteMap); + return { + __hasteMapForTest, + hasteFS, + moduleMap, + }; + })(); + } + return this._buildPromise.then(result => { + this._options.perfLogger?.markerPoint('build_end'); + return result; + }); + } + + /** + * 1. read data from the cache or create an empty structure. + */ + read(): InternalHasteMap { + let hasteMap: InternalHasteMap; + + this._options.perfLogger?.markerPoint('read_start'); + try { + // $FlowFixMe[incompatible-cast] - readFileSync returns mixed + hasteMap = (serializer.readFileSync(this._cachePath): InternalHasteMap); + } catch { + hasteMap = this._createEmptyMap(); + } + this._options.perfLogger?.markerPoint('read_end'); + + return hasteMap; + } + + readModuleMap(): HasteModuleMap { + const data = this.read(); + return new HasteModuleMap({ + duplicates: data.duplicates, + map: data.map, + mocks: data.mocks, + rootDir: this._options.rootDir, + }); + } + + /** + * 2. crawl the file system. + */ + async _buildFileMap(): Promise<{ + removedFiles: FileData, + changedFiles?: FileData, + hasteMap: InternalHasteMap, + }> { + let hasteMap: InternalHasteMap; + this._options.perfLogger?.markerPoint('buildFileMap_start'); + try { + hasteMap = + this._options.resetCache === true + ? this._createEmptyMap() + : await this.read(); + } catch { + hasteMap = this._createEmptyMap(); + } + return this._crawl(hasteMap).then(result => { + this._options.perfLogger?.markerPoint('buildFileMap_end'); + return result; + }); + } + + /** + * 3. parse and extract metadata from changed files. + */ + _processFile( + hasteMap: InternalHasteMap, + map: ModuleMapData, + mocks: MockData, + filePath: Path, + workerOptions?: {forceInBand: boolean}, + ): ?Promise { + const rootDir = this._options.rootDir; + + const setModule = (id: string, module: ModuleMetaData) => { + let moduleMap = map.get(id); + if (!moduleMap) { + // $FlowFixMe[unclear-type] - Add type coverage + moduleMap = (Object.create(null): any); + map.set(id, moduleMap); + } + const platform = + getPlatformExtension(module[H.PATH], this._options.platforms) || + H.GENERIC_PLATFORM; + + const existingModule = moduleMap[platform]; + + if (existingModule && existingModule[H.PATH] !== module[H.PATH]) { + const method = this._options.throwOnModuleCollision ? 'error' : 'warn'; + + this._console[method]( + [ + 'jest-haste-map: Haste module naming collision: ' + id, + ' The following files share their name; please adjust your hasteImpl:', + ' * ' + path.sep + existingModule[H.PATH], + ' * ' + path.sep + module[H.PATH], + '', + ].join('\n'), + ); + + if (this._options.throwOnModuleCollision) { + throw new DuplicateError(existingModule[H.PATH], module[H.PATH]); + } + + // We do NOT want consumers to use a module that is ambiguous. + delete moduleMap[platform]; + + if (Object.keys(moduleMap).length === 1) { + map.delete(id); + } + + let dupsByPlatform = hasteMap.duplicates.get(id); + if (dupsByPlatform == null) { + dupsByPlatform = new Map(); + hasteMap.duplicates.set(id, dupsByPlatform); + } + + const dups = new Map([ + [module[H.PATH], module[H.TYPE]], + [existingModule[H.PATH], existingModule[H.TYPE]], + ]); + dupsByPlatform.set(platform, dups); + + return; + } + + const dupsByPlatform = hasteMap.duplicates.get(id); + if (dupsByPlatform != null) { + const dups = dupsByPlatform.get(platform); + if (dups != null) { + dups.set(module[H.PATH], module[H.TYPE]); + } + return; + } + + moduleMap[platform] = module; + }; + + const relativeFilePath = fastPath.relative(rootDir, filePath); + const fileMetadata = hasteMap.files.get(relativeFilePath); + if (!fileMetadata) { + throw new Error( + 'jest-haste-map: File to process was not found in the haste map.', + ); + } + + const moduleMetadata = hasteMap.map.get(fileMetadata[H.ID]); + const computeSha1 = this._options.computeSha1 && !fileMetadata[H.SHA1]; + + // Callback called when the response from the worker is successful. + const workerReply = (metadata: WorkerMetadata) => { + // `1` for truthy values instead of `true` to save cache space. + fileMetadata[H.VISITED] = 1; + + const metadataId = metadata.id; + const metadataModule = metadata.module; + + if (metadataId != null && metadataModule) { + fileMetadata[H.ID] = metadataId; + setModule(metadataId, metadataModule); + } + + fileMetadata[H.DEPENDENCIES] = metadata.dependencies + ? metadata.dependencies.join(H.DEPENDENCY_DELIM) + : ''; + + if (computeSha1) { + fileMetadata[H.SHA1] = metadata.sha1; + } + }; + + // Callback called when the response from the worker is an error. + const workerError = (error: mixed) => { + if ( + error == null || + typeof error !== 'object' || + error.message == null || + error.stack == null + ) { + // $FlowFixMe[reassign-const] - Refactor this + error = new Error(error); + // $FlowFixMe[incompatible-use] - error is mixed + error.stack = ''; // Remove stack for stack-less errors. + } + + // $FlowFixMe[incompatible-use] - error is mixed + if (!['ENOENT', 'EACCES'].includes(error.code)) { + throw error; + } + + // If a file cannot be read we remove it from the file list and + // ignore the failure silently. + hasteMap.files.delete(relativeFilePath); + }; + + // If we retain all files in the virtual HasteFS representation, we avoid + // reading them if they aren't important (node_modules). + if (this._options.retainAllFiles && filePath.includes(NODE_MODULES)) { + if (computeSha1) { + return this._getWorker(workerOptions) + .getSha1({ + computeDependencies: this._options.computeDependencies, + computeSha1, + dependencyExtractor: this._options.dependencyExtractor, + filePath, + hasteImplModulePath: this._options.hasteImplModulePath, + rootDir, + }) + .then(workerReply, workerError); + } + + return null; + } + + if ( + this._options.mocksPattern && + this._options.mocksPattern.test(filePath) + ) { + const mockPath = getMockName(filePath); + const existingMockPath = mocks.get(mockPath); + + if (existingMockPath != null) { + const secondMockPath = fastPath.relative(rootDir, filePath); + if (existingMockPath !== secondMockPath) { + const method = this._options.throwOnModuleCollision + ? 'error' + : 'warn'; + + this._console[method]( + [ + 'jest-haste-map: duplicate manual mock found: ' + mockPath, + ' The following files share their name; please delete one of them:', + ' * ' + path.sep + existingMockPath, + ' * ' + path.sep + secondMockPath, + '', + ].join('\n'), + ); + + if (this._options.throwOnModuleCollision) { + throw new DuplicateError(existingMockPath, secondMockPath); + } + } + } + + mocks.set(mockPath, relativeFilePath); + } + + if (fileMetadata[H.VISITED]) { + if (!fileMetadata[H.ID]) { + return null; + } + + if (moduleMetadata != null) { + const platform = + getPlatformExtension(filePath, this._options.platforms) || + H.GENERIC_PLATFORM; + + const module = moduleMetadata[platform]; + + if (module == null) { + return null; + } + + const moduleId = fileMetadata[H.ID]; + let modulesByPlatform = map.get(moduleId); + if (!modulesByPlatform) { + // $FlowFixMe[unclear-type] - ModuleMapItem? + modulesByPlatform = (Object.create(null): any); + map.set(moduleId, modulesByPlatform); + } + modulesByPlatform[platform] = module; + + return null; + } + } + + return this._getWorker(workerOptions) + .worker({ + computeDependencies: this._options.computeDependencies, + computeSha1, + dependencyExtractor: this._options.dependencyExtractor, + filePath, + hasteImplModulePath: this._options.hasteImplModulePath, + rootDir, + }) + .then(workerReply, workerError); + } + + _buildHasteMap(data: { + removedFiles: FileData, + changedFiles?: FileData, + hasteMap: InternalHasteMap, + }): Promise { + this._options.perfLogger?.markerPoint('buildHasteMap_start'); + const {removedFiles, changedFiles, hasteMap} = data; + + // If any files were removed or we did not track what files changed, process + // every file looking for changes. Otherwise, process only changed files. + let map: ModuleMapData; + let mocks: MockData; + let filesToProcess: FileData; + if (changedFiles == null || removedFiles.size) { + map = new Map(); + mocks = new Map(); + filesToProcess = hasteMap.files; + } else { + map = hasteMap.map; + mocks = hasteMap.mocks; + filesToProcess = changedFiles; + } + + for (const [relativeFilePath, fileMetadata] of removedFiles) { + this._recoverDuplicates(hasteMap, relativeFilePath, fileMetadata[H.ID]); + } + + const promises = []; + for (const relativeFilePath of filesToProcess.keys()) { + if ( + this._options.skipPackageJson && + relativeFilePath.endsWith(PACKAGE_JSON) + ) { + continue; + } + // SHA-1, if requested, should already be present thanks to the crawler. + const filePath = fastPath.resolve( + this._options.rootDir, + relativeFilePath, + ); + const promise = this._processFile(hasteMap, map, mocks, filePath); + if (promise) { + promises.push(promise); + } + } + + return Promise.all(promises).then( + () => { + this._cleanup(); + hasteMap.map = map; + hasteMap.mocks = mocks; + this._options.perfLogger?.markerPoint('buildHasteMap_end'); + return hasteMap; + }, + error => { + this._cleanup(); + throw error; + }, + ); + } + + _cleanup() { + const worker = this._worker; + + // $FlowFixMe[prop-missing] - end is not on WorkerInterface + if (worker && typeof worker.end === 'function') { + worker.end(); + } + + this._worker = null; + } + + /** + * 4. serialize the new `HasteMap` in a cache file. + */ + _persist(hasteMap: InternalHasteMap) { + this._options.perfLogger?.markerPoint('persist_start'); + serializer.writeFileSync(this._cachePath, hasteMap); + this._options.perfLogger?.markerPoint('persist_end'); + } + + /** + * Creates workers or parses files and extracts metadata in-process. + */ + _getWorker(options?: {forceInBand: boolean}): WorkerInterface { + if (!this._worker) { + if ((options && options.forceInBand) || this._options.maxWorkers <= 1) { + this._worker = {getSha1, worker}; + } else { + this._worker = new Worker(require.resolve('./worker'), { + exposedMethods: ['getSha1', 'worker'], + maxRetries: 3, + numWorkers: this._options.maxWorkers, + }); + } + } + + return this._worker; + } + + _crawl(hasteMap: InternalHasteMap) { + this._options.perfLogger?.markerPoint('crawl_start'); + const options = this._options; + const ignore = filePath => this._ignore(filePath); + const crawl = + canUseWatchman && this._options.useWatchman ? watchmanCrawl : nodeCrawl; + const crawlerOptions: CrawlerOptions = { + computeSha1: options.computeSha1, + data: hasteMap, + enableSymlinks: options.enableSymlinks, + extensions: options.extensions, + forceNodeFilesystemAPI: options.forceNodeFilesystemAPI, + ignore, + perfLogger: options.perfLogger, + rootDir: options.rootDir, + roots: options.roots, + }; + + const retry = (error: Error) => { + if (crawl === watchmanCrawl) { + this._console.warn( + 'jest-haste-map: Watchman crawl failed. Retrying once with node ' + + 'crawler.\n' + + " Usually this happens when watchman isn't running. Create an " + + "empty `.watchmanconfig` file in your project's root folder or " + + 'initialize a git or hg repository in your project.\n' + + ' ' + + error.toString(), + ); + return nodeCrawl(crawlerOptions).catch(e => { + throw new Error( + 'Crawler retry failed:\n' + + ` Original error: ${error.message}\n` + + ` Retry error: ${e.message}\n`, + ); + }); + } + + throw error; + }; + + const logEnd = (result: T): T => { + this._options.perfLogger?.markerPoint('crawl_end'); + return result; + }; + + try { + return crawl(crawlerOptions).catch(retry).then(logEnd); + } catch (error) { + return retry(error).then(logEnd); + } + } + + /** + * Watch mode + */ + _watch(hasteMap: InternalHasteMap): Promise { + this._options.perfLogger?.markerPoint('watch_start'); + if (!this._options.watch) { + this._options.perfLogger?.markerPoint('watch_end'); + return Promise.resolve(); + } + + // In watch mode, we'll only warn about module collisions and we'll retain + // all files, even changes to node_modules. + this._options.throwOnModuleCollision = false; + this._options.retainAllFiles = true; + + // WatchmanWatcher > FSEventsWatcher > sane.NodeWatcher + const WatcherImpl = + canUseWatchman && this._options.useWatchman + ? WatchmanWatcher + : FSEventsWatcher.isSupported() + ? FSEventsWatcher + : NodeWatcher; + + const extensions = this._options.extensions; + const ignorePattern = this._options.ignorePattern; + const rootDir = this._options.rootDir; + + let changeQueue: Promise = Promise.resolve(); + let eventsQueue: EventsQueue = []; + // We only need to copy the entire haste map once on every "frame". + let mustCopy = true; + + const createWatcher = (root: Path): Promise => { + const watcher = new WatcherImpl(root, { + dot: true, + glob: extensions.map(extension => '**/*.' + extension), + ignored: ignorePattern, + }); + + return new Promise((resolve, reject) => { + const rejectTimeout = setTimeout( + () => reject(new Error('Failed to start watch mode.')), + MAX_WAIT_TIME, + ); + + watcher.once('ready', () => { + clearTimeout(rejectTimeout); + watcher.on('all', onChange); + resolve(watcher); + }); + }); + }; + + const emitChange = () => { + if (eventsQueue.length) { + mustCopy = true; + const changeEvent: ChangeEvent = { + eventsQueue, + hasteFS: new HasteFS({files: hasteMap.files, rootDir}), + moduleMap: new HasteModuleMap({ + duplicates: hasteMap.duplicates, + map: hasteMap.map, + mocks: hasteMap.mocks, + rootDir, + }), + }; + this.emit('change', changeEvent); + eventsQueue = []; + } + }; + + const onChange = ( + type: string, + filePath: Path, + root: Path, + stat?: Stats, + ) => { + const absoluteFilePath = path.join(root, normalizePathSep(filePath)); + if ( + (stat && stat.isDirectory()) || + this._ignore(absoluteFilePath) || + !extensions.some(extension => absoluteFilePath.endsWith(extension)) + ) { + return; + } + + const relativeFilePath = fastPath.relative(rootDir, absoluteFilePath); + const fileMetadata = hasteMap.files.get(relativeFilePath); + + // The file has been accessed, not modified + if ( + type === 'change' && + fileMetadata && + stat && + fileMetadata[H.MTIME] === stat.mtime.getTime() + ) { + return; + } + + changeQueue = changeQueue + .then(() => { + // If we get duplicate events for the same file, ignore them. + if ( + eventsQueue.find( + event => + event.type === type && + event.filePath === absoluteFilePath && + ((!event.stat && !stat) || + (!!event.stat && + !!stat && + event.stat.mtime.getTime() === stat.mtime.getTime())), + ) + ) { + return null; + } + + if (mustCopy) { + mustCopy = false; + // $FlowFixMe[reassign-const] - Refactor this + hasteMap = { + clocks: new Map(hasteMap.clocks), + duplicates: new Map(hasteMap.duplicates), + files: new Map(hasteMap.files), + map: new Map(hasteMap.map), + mocks: new Map(hasteMap.mocks), + }; + } + + const add = () => { + eventsQueue.push({filePath: absoluteFilePath, stat, type}); + return null; + }; + + const fileMetadata = hasteMap.files.get(relativeFilePath); + + // If it's not an addition, delete the file and all its metadata + if (fileMetadata != null) { + const moduleName = fileMetadata[H.ID]; + const platform = + getPlatformExtension(absoluteFilePath, this._options.platforms) || + H.GENERIC_PLATFORM; + hasteMap.files.delete(relativeFilePath); + + let moduleMap = hasteMap.map.get(moduleName); + if (moduleMap != null) { + // We are forced to copy the object because jest-haste-map exposes + // the map as an immutable entity. + moduleMap = Object.assign(Object.create(null), moduleMap); + delete moduleMap[platform]; + if (Object.keys(moduleMap).length === 0) { + hasteMap.map.delete(moduleName); + } else { + hasteMap.map.set(moduleName, moduleMap); + } + } + + if ( + this._options.mocksPattern && + this._options.mocksPattern.test(absoluteFilePath) + ) { + const mockName = getMockName(absoluteFilePath); + hasteMap.mocks.delete(mockName); + } + + this._recoverDuplicates(hasteMap, relativeFilePath, moduleName); + } + + // If the file was added or changed, + // parse it and update the haste map. + if (type === 'add' || type === 'change') { + invariant( + stat, + 'since the file exists or changed, it should have stats', + ); + const fileMetadata: FileMetaData = [ + '', + stat.mtime.getTime(), + stat.size, + 0, + '', + null, + ]; + hasteMap.files.set(relativeFilePath, fileMetadata); + const promise = this._processFile( + hasteMap, + hasteMap.map, + hasteMap.mocks, + absoluteFilePath, + {forceInBand: true}, + ); + // Cleanup + this._cleanup(); + if (promise) { + return promise.then(add); + } else { + // If a file in node_modules has changed, + // emit an event regardless. + add(); + } + } else { + add(); + } + return null; + }) + .catch((error: Error) => { + this._console.error( + `jest-haste-map: watch error:\n ${error.stack}\n`, + ); + }); + }; + + this._changeInterval = setInterval(emitChange, CHANGE_INTERVAL); + + return Promise.all(this._options.roots.map(createWatcher)).then( + watchers => { + this._watchers = watchers; + this._options.perfLogger?.markerPoint('watch_end'); + }, + ); + } + + /** + * This function should be called when the file under `filePath` is removed + * or changed. When that happens, we want to figure out if that file was + * part of a group of files that had the same ID. If it was, we want to + * remove it from the group. Furthermore, if there is only one file + * remaining in the group, then we want to restore that single file as the + * correct resolution for its ID, and cleanup the duplicates index. + */ + _recoverDuplicates( + hasteMap: InternalHasteMap, + relativeFilePath: string, + moduleName: string, + ) { + let dupsByPlatform = hasteMap.duplicates.get(moduleName); + if (dupsByPlatform == null) { + return; + } + + const platform = + getPlatformExtension(relativeFilePath, this._options.platforms) || + H.GENERIC_PLATFORM; + let dups = dupsByPlatform.get(platform); + if (dups == null) { + return; + } + + dupsByPlatform = new Map(dupsByPlatform); + hasteMap.duplicates.set(moduleName, dupsByPlatform); + + dups = new Map(dups); + dupsByPlatform.set(platform, dups); + dups.delete(relativeFilePath); + + if (dups.size !== 1) { + return; + } + + const uniqueModule = dups.entries().next().value; + + if (!uniqueModule) { + return; + } + + let dedupMap = hasteMap.map.get(moduleName); + + if (dedupMap == null) { + // $FlowFixMe[unclear-type] - ModuleMapItem? + dedupMap = (Object.create(null): any); + hasteMap.map.set(moduleName, dedupMap); + } + dedupMap[platform] = uniqueModule; + dupsByPlatform.delete(platform); + if (dupsByPlatform.size === 0) { + hasteMap.duplicates.delete(moduleName); + } + } + + async end(): Promise { + if (this._changeInterval) { + clearInterval(this._changeInterval); + } + + if (!this._watchers.length) { + return; + } + + await Promise.all(this._watchers.map(watcher => watcher.close())); + + this._watchers = []; + } + + /** + * Helpers + */ + _ignore(filePath: Path): boolean { + const ignorePattern = this._options.ignorePattern; + const ignoreMatched = + ignorePattern instanceof RegExp + ? ignorePattern.test(filePath) + : ignorePattern && ignorePattern(filePath); + + return ( + ignoreMatched || + (!this._options.retainAllFiles && filePath.includes(NODE_MODULES)) + ); + } + + _createEmptyMap(): InternalHasteMap { + return { + clocks: new Map(), + duplicates: new Map(), + files: new Map(), + map: new Map(), + mocks: new Map(), + }; + } + + static H: HType = H; +} + +export class DuplicateError extends Error { + mockPath1: string; + mockPath2: string; + + constructor(mockPath1: string, mockPath2: string) { + super('Duplicated files or mocks. Please check the console for more info'); + + this.mockPath1 = mockPath1; + this.mockPath2 = mockPath2; + } +} diff --git a/packages/metro-file-map/src/lib/__tests__/dependencyExtractor-test.js b/packages/metro-file-map/src/lib/__tests__/dependencyExtractor-test.js new file mode 100644 index 0000000000..5b103cb947 --- /dev/null +++ b/packages/metro-file-map/src/lib/__tests__/dependencyExtractor-test.js @@ -0,0 +1,266 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +import {extract} from '../dependencyExtractor'; + +describe('dependencyExtractor', () => { + it('should not extract dependencies inside comments', () => { + const code = ` + // import a from 'ignore-line-comment'; + // import 'ignore-line-comment'; + // import './ignore-line-comment'; + // require('ignore-line-comment'); + /* + * import a from 'ignore-block-comment'; + * import './ignore-block-comment'; + * import 'ignore-block-comment'; + * require('ignore-block-comment'); + */ + `; + expect(extract(code)).toEqual(new Set()); + }); + + it('should not extract dependencies inside comments (windows line endings)', () => { + const code = [ + '// const module1 = require("module1");', + '/**', + ' * const module2 = require("module2");', + ' */', + ].join('\r\n'); + + expect(extract(code)).toEqual(new Set([])); + }); + + it('should not extract dependencies inside comments (unicode line endings)', () => { + const code = [ + '// const module1 = require("module1");\u2028', + '// const module1 = require("module2");\u2029', + '/*\u2028', + 'const module2 = require("module3");\u2029', + ' */', + ].join(''); + + expect(extract(code)).toEqual(new Set([])); + }); + + it('should extract dependencies from `import` statements', () => { + const code = ` + // Good + import * as depNS from 'dep1'; + import { + a as aliased_a, + b, + } from 'dep2'; + import depDefault from 'dep3'; + import * as depNS, { + a as aliased_a, + b, + }, depDefault from 'dep4'; + + // Bad + foo . import ('inv1'); + foo . export ('inv2'); + `; + expect(extract(code)).toEqual(new Set(['dep1', 'dep2', 'dep3', 'dep4'])); + }); + + // https://github.com/facebook/jest/issues/8547 + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#Import_a_module_for_its_side_effects_only + it('should extract dependencies from side-effect only `import` statements', () => { + const code = ` + // Good + import './side-effect-dep1'; + import 'side-effect-dep2'; + + // Bad + import ./inv1; + import inv2 + `; + expect(extract(code)).toEqual( + new Set(['./side-effect-dep1', 'side-effect-dep2']), + ); + }); + + it('should not extract dependencies from `import type/typeof` statements', () => { + const code = ` + // Bad + import typeof {foo} from 'inv1'; + import type {foo} from 'inv2'; + `; + expect(extract(code)).toEqual(new Set([])); + }); + + it('should extract dependencies from `export` statements', () => { + const code = ` + // Good + export * as depNS from 'dep1'; + export { + a as aliased_a, + b, + } from 'dep2'; + export depDefault from 'dep3'; + export * as depNS, { + a as aliased_a, + b, + }, depDefault from 'dep4'; + + // Bad + foo . export ('inv1'); + foo . export ('inv2'); + `; + expect(extract(code)).toEqual(new Set(['dep1', 'dep2', 'dep3', 'dep4'])); + }); + + it('should extract dependencies from `export-from` statements', () => { + const code = ` + // Good + export * as depNS from 'dep1'; + export { + a as aliased_a, + b, + } from 'dep2'; + export depDefault from 'dep3'; + export * as depNS, { + a as aliased_a, + b, + }, depDefault from 'dep4'; + + // Bad + foo . export ('inv1'); + foo . export ('inv2'); + `; + expect(extract(code)).toEqual(new Set(['dep1', 'dep2', 'dep3', 'dep4'])); + }); + + it('should not extract dependencies from `export type/typeof` statements', () => { + const code = ` + // Bad + export typeof {foo} from 'inv1'; + export type {foo} from 'inv2'; + `; + expect(extract(code)).toEqual(new Set([])); + }); + + it('should extract dependencies from dynamic `import` calls', () => { + const code = ` + // Good + import('dep1').then(); + const dep2 = await import( + "dep2", + ); + if (await import(\`dep3\`)) {} + + // Bad + await foo . import('inv1') + await ximport('inv2'); + importx('inv3'); + import('inv4', 'inv5'); + `; + expect(extract(code)).toEqual(new Set(['dep1', 'dep2', 'dep3'])); + }); + + it('should extract dependencies from `require` calls', () => { + const code = ` + // Good + require('dep1'); + const dep2 = require( + "dep2", + ); + if (require(\`dep3\`).cond) {} + + // Bad + foo . require('inv1') + xrequire('inv2'); + requirex('inv3'); + require('inv4', 'inv5'); + `; + expect(extract(code)).toEqual(new Set(['dep1', 'dep2', 'dep3'])); + }); + + it('should extract dependencies from `jest.requireActual` calls', () => { + const code = ` + // Good + jest.requireActual('dep1'); + const dep2 = jest.requireActual( + "dep2", + ); + if (jest.requireActual(\`dep3\`).cond) {} + jest + .requireActual('dep4'); + + // Bad + foo . jest.requireActual('inv1') + xjest.requireActual('inv2'); + jest.requireActualx('inv3'); + jest.requireActual('inv4', 'inv5'); + `; + expect(extract(code)).toEqual(new Set(['dep1', 'dep2', 'dep3', 'dep4'])); + }); + + it('should extract dependencies from `jest.requireMock` calls', () => { + const code = ` + // Good + jest.requireMock('dep1'); + const dep2 = jest.requireMock( + "dep2", + ); + if (jest.requireMock(\`dep3\`).cond) {} + jest + .requireMock('dep4'); + + // Bad + foo . jest.requireMock('inv1') + xjest.requireMock('inv2'); + jest.requireMockx('inv3'); + jest.requireMock('inv4', 'inv5'); + `; + expect(extract(code)).toEqual(new Set(['dep1', 'dep2', 'dep3', 'dep4'])); + }); + + it('should extract dependencies from `jest.genMockFromModule` calls', () => { + const code = ` + // Good + jest.genMockFromModule('dep1'); + const dep2 = jest.genMockFromModule( + "dep2", + ); + if (jest.genMockFromModule(\`dep3\`).cond) {} + jest + .requireMock('dep4'); + + // Bad + foo . jest.genMockFromModule('inv1') + xjest.genMockFromModule('inv2'); + jest.genMockFromModulex('inv3'); + jest.genMockFromModule('inv4', 'inv5'); + `; + expect(extract(code)).toEqual(new Set(['dep1', 'dep2', 'dep3', 'dep4'])); + }); + + it('should extract dependencies from `jest.createMockFromModule` calls', () => { + const code = ` + // Good + jest.createMockFromModule('dep1'); + const dep2 = jest.createMockFromModule( + "dep2", + ); + if (jest.createMockFromModule(\`dep3\`).cond) {} + jest + .requireMock('dep4'); + + // Bad + foo . jest.createMockFromModule('inv1') + xjest.createMockFromModule('inv2'); + jest.createMockFromModulex('inv3'); + jest.createMockFromModule('inv4', 'inv5'); + `; + expect(extract(code)).toEqual(new Set(['dep1', 'dep2', 'dep3', 'dep4'])); + }); +}); diff --git a/packages/metro-file-map/src/lib/__tests__/fast_path-test.js b/packages/metro-file-map/src/lib/__tests__/fast_path-test.js new file mode 100644 index 0000000000..66ab6ee568 --- /dev/null +++ b/packages/metro-file-map/src/lib/__tests__/fast_path-test.js @@ -0,0 +1,51 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +import {relative, resolve} from '../fast_path'; +import path from 'path'; + +describe('fastPath.relative', () => { + it('should get relative paths inside the root', () => { + const root = path.join(__dirname, 'foo', 'bar'); + const filename = path.join(__dirname, 'foo', 'bar', 'baz', 'foobar'); + const relativeFilename = path.join('baz', 'foobar'); + expect(relative(root, filename)).toBe(relativeFilename); + }); + + it('should get relative paths outside the root', () => { + const root = path.join(__dirname, 'foo', 'bar'); + const filename = path.join(__dirname, 'foo', 'baz', 'foobar'); + const relativeFilename = path.join('..', 'baz', 'foobar'); + expect(relative(root, filename)).toBe(relativeFilename); + }); + + it('should get relative paths outside the root when start with same word', () => { + const root = path.join(__dirname, 'foo', 'bar'); + const filename = path.join(__dirname, 'foo', 'barbaz', 'foobar'); + const relativeFilename = path.join('..', 'barbaz', 'foobar'); + expect(relative(root, filename)).toBe(relativeFilename); + }); +}); + +describe('fastPath.resolve', () => { + it('should get the absolute path for paths inside the root', () => { + const root = path.join(__dirname, 'foo', 'bar'); + const relativeFilename = path.join('baz', 'foobar'); + const filename = path.join(__dirname, 'foo', 'bar', 'baz', 'foobar'); + expect(resolve(root, relativeFilename)).toBe(filename); + }); + + it('should get the absolute path for paths outside the root', () => { + const root = path.join(__dirname, 'foo', 'bar'); + const relativeFilename = path.join('..', 'baz', 'foobar'); + const filename = path.join(__dirname, 'foo', 'baz', 'foobar'); + expect(resolve(root, relativeFilename)).toBe(filename); + }); +}); diff --git a/packages/metro-file-map/src/lib/__tests__/getPlatformExtension-test.js b/packages/metro-file-map/src/lib/__tests__/getPlatformExtension-test.js new file mode 100644 index 0000000000..4475c746f5 --- /dev/null +++ b/packages/metro-file-map/src/lib/__tests__/getPlatformExtension-test.js @@ -0,0 +1,23 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +import getPlatformExtension from '../getPlatformExtension'; + +describe('getPlatformExtension', () => { + it('should get platform ext', () => { + expect(getPlatformExtension('a.ios.js')).toBe('ios'); + expect(getPlatformExtension('a.android.js')).toBe('android'); + expect(getPlatformExtension('/b/c/a.ios.js')).toBe('ios'); + expect(getPlatformExtension('/b/c.android/a.ios.js')).toBe('ios'); + expect(getPlatformExtension('/b/c/a@1.5x.ios.png')).toBe('ios'); + expect(getPlatformExtension('/b/c/a@1.5x.lol.png')).toBe(null); + expect(getPlatformExtension('/b/c/a.lol.png')).toBe(null); + }); +}); diff --git a/packages/metro-file-map/src/lib/__tests__/normalizePathSep-test.js b/packages/metro-file-map/src/lib/__tests__/normalizePathSep-test.js new file mode 100644 index 0000000000..66fad866ce --- /dev/null +++ b/packages/metro-file-map/src/lib/__tests__/normalizePathSep-test.js @@ -0,0 +1,27 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @emails oncall+react_native + * @format + */ + +'use strict'; + +describe('normalizePathSep', () => { + it('does nothing on posix', () => { + jest.resetModules(); + jest.mock('path', () => jest.requireActual('path').posix); + const normalizePathSep = require('../normalizePathSep').default; + expect(normalizePathSep('foo/bar/baz.js')).toEqual('foo/bar/baz.js'); + }); + + it('replace slashes on windows', () => { + jest.resetModules(); + jest.mock('path', () => jest.requireActual('path').win32); + const normalizePathSep = require('../normalizePathSep').default; + expect(normalizePathSep('foo/bar/baz.js')).toEqual('foo\\bar\\baz.js'); + }); +}); diff --git a/packages/metro-file-map/src/lib/dependencyExtractor.js b/packages/metro-file-map/src/lib/dependencyExtractor.js new file mode 100644 index 0000000000..1f86799c27 --- /dev/null +++ b/packages/metro-file-map/src/lib/dependencyExtractor.js @@ -0,0 +1,101 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @flow strict + */ + +'use strict'; + +const NOT_A_DOT = '(? + `([\`'"])([^'"\`]*?)(?:\\${pos})`; +const WORD_SEPARATOR = '\\b'; +const LEFT_PARENTHESIS = '\\('; +const RIGHT_PARENTHESIS = '\\)'; +const WHITESPACE = '\\s*'; +const OPTIONAL_COMMA = '(:?,\\s*)?'; + +function createRegExp( + parts /*: $ReadOnlyArray */, + flags /*: string */, +) { + return new RegExp(parts.join(''), flags); +} + +function alternatives(...parts /*: $ReadOnlyArray */) { + return `(?:${parts.join('|')})`; +} + +function functionCallStart(...names /*: $ReadOnlyArray */) { + return [ + NOT_A_DOT, + WORD_SEPARATOR, + alternatives(...names), + WHITESPACE, + LEFT_PARENTHESIS, + WHITESPACE, + ]; +} + +const BLOCK_COMMENT_RE = /\/\*[^]*?\*\//g; +const LINE_COMMENT_RE = /\/\/.*/g; + +const REQUIRE_OR_DYNAMIC_IMPORT_RE = createRegExp( + [ + ...functionCallStart('require', 'import'), + CAPTURE_STRING_LITERAL(1), + WHITESPACE, + OPTIONAL_COMMA, + RIGHT_PARENTHESIS, + ], + 'g', +); + +const IMPORT_OR_EXPORT_RE = createRegExp( + [ + '\\b(?:import|export)\\s+(?!type(?:of)?\\s+)(?:[^\'"]+\\s+from\\s+)?', + CAPTURE_STRING_LITERAL(1), + ], + 'g', +); + +const JEST_EXTENSIONS_RE = createRegExp( + [ + ...functionCallStart( + 'jest\\s*\\.\\s*(?:requireActual|requireMock|genMockFromModule|createMockFromModule)', + ), + CAPTURE_STRING_LITERAL(1), + WHITESPACE, + OPTIONAL_COMMA, + RIGHT_PARENTHESIS, + ], + 'g', +); + +function extract(code /*: string */) /*: $ReadOnlySet */ { + const dependencies /*: Set */ = new Set(); + + const addDependency = ( + match /*: string */, + _ /*: string */, + dep /*: string */, + ) => { + dependencies.add(dep); + return match; + }; + + code + .replace(BLOCK_COMMENT_RE, '') + .replace(LINE_COMMENT_RE, '') + .replace(IMPORT_OR_EXPORT_RE, addDependency) + .replace(REQUIRE_OR_DYNAMIC_IMPORT_RE, addDependency) + .replace(JEST_EXTENSIONS_RE, addDependency); + + return dependencies; +} + +module.exports = {extract}; diff --git a/packages/metro-file-map/src/lib/fast_path.js b/packages/metro-file-map/src/lib/fast_path.js new file mode 100644 index 0000000000..f72f291e65 --- /dev/null +++ b/packages/metro-file-map/src/lib/fast_path.js @@ -0,0 +1,28 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @flow strict + */ + +import * as path from 'path'; + +// rootDir and filename must be absolute paths (resolved) +export function relative(rootDir: string, filename: string): string { + return filename.indexOf(rootDir + path.sep) === 0 + ? filename.substr(rootDir.length + 1) + : path.relative(rootDir, filename); +} + +const INDIRECTION_FRAGMENT = '..' + path.sep; + +// rootDir must be an absolute path and relativeFilename must be simple +// (e.g.: foo/bar or ../foo/bar, but never ./foo or foo/../bar) +export function resolve(rootDir: string, relativeFilename: string): string { + return relativeFilename.indexOf(INDIRECTION_FRAGMENT) === 0 + ? path.resolve(rootDir, relativeFilename) + : rootDir + path.sep + relativeFilename; +} diff --git a/packages/metro-file-map/src/lib/getPlatformExtension.js b/packages/metro-file-map/src/lib/getPlatformExtension.js new file mode 100644 index 0000000000..c52bd3aca0 --- /dev/null +++ b/packages/metro-file-map/src/lib/getPlatformExtension.js @@ -0,0 +1,30 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @flow strict + */ + +const SUPPORTED_PLATFORM_EXTS = new Set(['android', 'ios', 'native', 'web']); + +// Extract platform extension: index.ios.js -> ios +export default function getPlatformExtension( + file: string, + platforms?: $ReadOnlyArray, +): ?string { + const last = file.lastIndexOf('.'); + const secondToLast = file.lastIndexOf('.', last - 1); + if (secondToLast === -1) { + return null; + } + const platform = file.substring(secondToLast + 1, last); + // If an overriding platform array is passed, check that first + + if (platforms && platforms.indexOf(platform) !== -1) { + return platform; + } + return SUPPORTED_PLATFORM_EXTS.has(platform) ? platform : null; +} diff --git a/packages/metro-file-map/src/lib/normalizePathSep.js b/packages/metro-file-map/src/lib/normalizePathSep.js new file mode 100644 index 0000000000..7fc03e4000 --- /dev/null +++ b/packages/metro-file-map/src/lib/normalizePathSep.js @@ -0,0 +1,21 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @flow strict + */ + +import * as path from 'path'; + +let normalizePathSep: (string: string) => string; +if (path.sep === '/') { + normalizePathSep = (filePath: string): string => filePath; +} else { + normalizePathSep = (filePath: string): string => + filePath.replace(/\//g, path.sep); +} + +export default normalizePathSep; diff --git a/packages/metro-file-map/src/watchers/FSEventsWatcher.js b/packages/metro-file-map/src/watchers/FSEventsWatcher.js new file mode 100644 index 0000000000..52bdd98269 --- /dev/null +++ b/packages/metro-file-map/src/watchers/FSEventsWatcher.js @@ -0,0 +1,199 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @flow strict-local + */ + +// $FlowFixMe[untyped-import] - anymatch +import anymatch from 'anymatch'; +import EventEmitter from 'events'; +import * as fs from 'graceful-fs'; +import * as path from 'path'; +// $FlowFixMe[untyped-import] - walker +import walker from 'walker'; + +// $FlowFixMe[untyped-import] - micromatch +const micromatch = require('micromatch'); + +type Matcher = typeof anymatch.Matcher; + +// $FlowFixMe[unclear-type] - fsevents +let fsevents: any = null; +try { + // $FlowFixMe[cannot-resolve-module] - Optional, Darwin only + fsevents = require('fsevents'); +} catch { + // Optional dependency, only supported on Darwin. +} + +const CHANGE_EVENT = 'change'; +const DELETE_EVENT = 'delete'; +const ADD_EVENT = 'add'; +const ALL_EVENT = 'all'; + +type FsEventsWatcherEvent = + | typeof CHANGE_EVENT + | typeof DELETE_EVENT + | typeof ADD_EVENT + | typeof ALL_EVENT; + +/** + * Export `FSEventsWatcher` class. + * Watches `dir`. + */ +export default class FSEventsWatcher extends EventEmitter { + +root: string; + +ignored: ?Matcher; + +glob: $ReadOnlyArray; + +dot: boolean; + +hasIgnore: boolean; + +doIgnore: (path: string) => boolean; + +fsEventsWatchStopper: () => Promise; + _tracked: Set; + + static isSupported(): boolean { + return fsevents != null; + } + + static _normalizeProxy( + callback: (normalizedPath: string, stats: fs.Stats) => void, + ) { + return (filepath: string, stats: fs.Stats): void => + callback(path.normalize(filepath), stats); + } + + static _recReaddir( + dir: string, + dirCallback: (normalizedPath: string, stats: fs.Stats) => void, + fileCallback: (normalizedPath: string, stats: fs.Stats) => void, + // $FlowFixMe[unclear-type] Add types for callback + endCallback: Function, + // $FlowFixMe[unclear-type] Add types for callback + errorCallback: Function, + ignored?: Matcher, + ) { + walker(dir) + .filterDir( + (currentDir: string) => !ignored || !anymatch(ignored, currentDir), + ) + .on('dir', FSEventsWatcher._normalizeProxy(dirCallback)) + .on('file', FSEventsWatcher._normalizeProxy(fileCallback)) + .on('error', errorCallback) + .on('end', () => { + endCallback(); + }); + } + + constructor( + dir: string, + opts: $ReadOnly<{ + ignored?: Matcher, + glob: string | $ReadOnlyArray, + dot: boolean, + }>, + ) { + if (!fsevents) { + throw new Error( + '`fsevents` unavailable (this watcher can only be used on Darwin)', + ); + } + + super(); + + this.dot = opts.dot || false; + this.ignored = opts.ignored; + this.glob = Array.isArray(opts.glob) ? opts.glob : [opts.glob]; + + this.hasIgnore = + Boolean(opts.ignored) && !(Array.isArray(opts) && opts.length > 0); + this.doIgnore = opts.ignored ? anymatch(opts.ignored) : () => false; + + this.root = path.resolve(dir); + this.fsEventsWatchStopper = fsevents.watch( + this.root, + // $FlowFixMe[method-unbinding] - Refactor + this._handleEvent.bind(this), + ); + + this._tracked = new Set(); + FSEventsWatcher._recReaddir( + this.root, + (filepath: string) => { + this._tracked.add(filepath); + }, + (filepath: string) => { + this._tracked.add(filepath); + }, + // $FlowFixMe[method-unbinding] - Refactor + this.emit.bind(this, 'ready'), + // $FlowFixMe[method-unbinding] - Refactor + this.emit.bind(this, 'error'), + this.ignored, + ); + } + + /** + * End watching. + */ + async close(callback?: () => void): Promise { + await this.fsEventsWatchStopper(); + this.removeAllListeners(); + if (typeof callback === 'function') { + // $FlowFixMe[extra-arg] - Is this a Node-style callback or as typed? + process.nextTick(callback.bind(null, null, true)); + } + } + + _isFileIncluded(relativePath: string) { + if (this.doIgnore(relativePath)) { + return false; + } + return this.glob.length + ? micromatch([relativePath], this.glob, {dot: this.dot}).length > 0 + : this.dot || micromatch([relativePath], '**/*').length > 0; + } + + _handleEvent(filepath: string) { + const relativePath = path.relative(this.root, filepath); + if (!this._isFileIncluded(relativePath)) { + return; + } + + fs.lstat(filepath, (error, stat) => { + if (error && error.code !== 'ENOENT') { + this.emit('error', error); + return; + } + + if (error) { + // Ignore files that aren't tracked and don't exist. + if (!this._tracked.has(filepath)) { + return; + } + + this._emit(DELETE_EVENT, relativePath); + this._tracked.delete(filepath); + return; + } + + if (this._tracked.has(filepath)) { + this._emit(CHANGE_EVENT, relativePath, stat); + } else { + this._tracked.add(filepath); + this._emit(ADD_EVENT, relativePath, stat); + } + }); + } + + /** + * Emit events. + */ + _emit(type: FsEventsWatcherEvent, file: string, stat?: fs.Stats) { + this.emit(type, file, this.root, stat); + this.emit(ALL_EVENT, type, file, this.root, stat); + } +} diff --git a/packages/metro-file-map/src/watchers/NodeWatcher.js b/packages/metro-file-map/src/watchers/NodeWatcher.js new file mode 100644 index 0000000000..3d03dceae8 --- /dev/null +++ b/packages/metro-file-map/src/watchers/NodeWatcher.js @@ -0,0 +1,380 @@ +/** + * vendored from https://github.com/amasad/sane/blob/64ff3a870c42e84f744086884bf55a4f9c22d376/src/node_watcher.js + * @format + */ +'use strict'; + +const common = require('./common'); +const EventEmitter = require('events').EventEmitter; +const fs = require('fs'); +const platform = require('os').platform(); +const path = require('path'); + +/** + * Constants + */ + +const DEFAULT_DELAY = common.DEFAULT_DELAY; +const CHANGE_EVENT = common.CHANGE_EVENT; +const DELETE_EVENT = common.DELETE_EVENT; +const ADD_EVENT = common.ADD_EVENT; +const ALL_EVENT = common.ALL_EVENT; + +/** + * Export `NodeWatcher` class. + * Watches `dir`. + * + * @class NodeWatcher + * @param {String} dir + * @param {Object} opts + * @public + */ + +module.exports = class NodeWatcher extends EventEmitter { + constructor(dir, opts) { + super(); + + common.assignOptions(this, opts); + + this.watched = Object.create(null); + this.changeTimers = Object.create(null); + this.dirRegistery = Object.create(null); + this.root = path.resolve(dir); + this.watchdir = this.watchdir.bind(this); + this.register = this.register.bind(this); + this.checkedEmitError = this.checkedEmitError.bind(this); + + this.watchdir(this.root); + common.recReaddir( + this.root, + this.watchdir, + this.register, + this.emit.bind(this, 'ready'), + this.checkedEmitError, + this.ignored, + ); + } + + /** + * Register files that matches our globs to know what to type of event to + * emit in the future. + * + * Registery looks like the following: + * + * dirRegister => Map { + * dirpath => Map { + * filename => true + * } + * } + * + * @param {string} filepath + * @return {boolean} whether or not we have registered the file. + * @private + */ + + register(filepath) { + const relativePath = path.relative(this.root, filepath); + if ( + !common.isFileIncluded(this.globs, this.dot, this.doIgnore, relativePath) + ) { + return false; + } + + const dir = path.dirname(filepath); + if (!this.dirRegistery[dir]) { + this.dirRegistery[dir] = Object.create(null); + } + + const filename = path.basename(filepath); + this.dirRegistery[dir][filename] = true; + + return true; + } + + /** + * Removes a file from the registery. + * + * @param {string} filepath + * @private + */ + + unregister(filepath) { + const dir = path.dirname(filepath); + if (this.dirRegistery[dir]) { + const filename = path.basename(filepath); + delete this.dirRegistery[dir][filename]; + } + } + + /** + * Removes a dir from the registery. + * + * @param {string} dirpath + * @private + */ + + unregisterDir(dirpath) { + if (this.dirRegistery[dirpath]) { + delete this.dirRegistery[dirpath]; + } + } + + /** + * Checks if a file or directory exists in the registery. + * + * @param {string} fullpath + * @return {boolean} + * @private + */ + + registered(fullpath) { + const dir = path.dirname(fullpath); + return ( + this.dirRegistery[fullpath] || + (this.dirRegistery[dir] && + this.dirRegistery[dir][path.basename(fullpath)]) + ); + } + + /** + * Emit "error" event if it's not an ignorable event + * + * @param error + * @private + */ + checkedEmitError(error) { + if (!isIgnorableFileError(error)) { + this.emit('error', error); + } + } + + /** + * Watch a directory. + * + * @param {string} dir + * @private + */ + + watchdir(dir) { + if (this.watched[dir]) { + return; + } + + const watcher = fs.watch( + dir, + {persistent: true}, + this.normalizeChange.bind(this, dir), + ); + this.watched[dir] = watcher; + + watcher.on('error', this.checkedEmitError); + + if (this.root !== dir) { + this.register(dir); + } + } + + /** + * Stop watching a directory. + * + * @param {string} dir + * @private + */ + + stopWatching(dir) { + if (this.watched[dir]) { + this.watched[dir].close(); + delete this.watched[dir]; + } + } + + /** + * End watching. + * + * @public + */ + + close() { + Object.keys(this.watched).forEach(this.stopWatching, this); + this.removeAllListeners(); + + return Promise.resolve(); + } + + /** + * On some platforms, as pointed out on the fs docs (most likely just win32) + * the file argument might be missing from the fs event. Try to detect what + * change by detecting if something was deleted or the most recent file change. + * + * @param {string} dir + * @param {string} event + * @param {string} file + * @public + */ + + detectChangedFile(dir, event, callback) { + if (!this.dirRegistery[dir]) { + return; + } + + let found = false; + let closest = {mtime: 0}; + let c = 0; + Object.keys(this.dirRegistery[dir]).forEach(function (file, i, arr) { + fs.lstat(path.join(dir, file), (error, stat) => { + if (found) { + return; + } + + if (error) { + if (isIgnorableFileError(error)) { + found = true; + callback(file); + } else { + this.emit('error', error); + } + } else { + if (stat.mtime > closest.mtime) { + stat.file = file; + closest = stat; + } + if (arr.length === ++c) { + callback(closest.file); + } + } + }); + }, this); + } + + /** + * Normalize fs events and pass it on to be processed. + * + * @param {string} dir + * @param {string} event + * @param {string} file + * @public + */ + + normalizeChange(dir, event, file) { + if (!file) { + this.detectChangedFile(dir, event, actualFile => { + if (actualFile) { + this.processChange(dir, event, actualFile); + } + }); + } else { + this.processChange(dir, event, path.normalize(file)); + } + } + + /** + * Process changes. + * + * @param {string} dir + * @param {string} event + * @param {string} file + * @public + */ + + processChange(dir, event, file) { + const fullPath = path.join(dir, file); + const relativePath = path.join(path.relative(this.root, dir), file); + + fs.lstat(fullPath, (error, stat) => { + if (error && error.code !== 'ENOENT') { + this.emit('error', error); + } else if (!error && stat.isDirectory()) { + // win32 emits usless change events on dirs. + if (event !== 'change') { + this.watchdir(fullPath); + if ( + common.isFileIncluded( + this.globs, + this.dot, + this.doIgnore, + relativePath, + ) + ) { + this.emitEvent(ADD_EVENT, relativePath, stat); + } + } + } else { + const registered = this.registered(fullPath); + if (error && error.code === 'ENOENT') { + this.unregister(fullPath); + this.stopWatching(fullPath); + this.unregisterDir(fullPath); + if (registered) { + this.emitEvent(DELETE_EVENT, relativePath); + } + } else if (registered) { + this.emitEvent(CHANGE_EVENT, relativePath, stat); + } else { + if (this.register(fullPath)) { + this.emitEvent(ADD_EVENT, relativePath, stat); + } + } + } + }); + } + + /** + * Triggers a 'change' event after debounding it to take care of duplicate + * events on os x. + * + * @private + */ + + emitEvent(type, file, stat) { + const key = type + '-' + file; + const addKey = ADD_EVENT + '-' + file; + if (type === CHANGE_EVENT && this.changeTimers[addKey]) { + // Ignore the change event that is immediately fired after an add event. + // (This happens on Linux). + return; + } + clearTimeout(this.changeTimers[key]); + this.changeTimers[key] = setTimeout(() => { + delete this.changeTimers[key]; + if (type === ADD_EVENT && stat.isDirectory()) { + // Recursively emit add events and watch for sub-files/folders + common.recReaddir( + path.resolve(this.root, file), + function emitAddDir(dir, stats) { + this.watchdir(dir); + this.rawEmitEvent(ADD_EVENT, path.relative(this.root, dir), stats); + }.bind(this), + function emitAddFile(file, stats) { + this.register(file); + this.rawEmitEvent(ADD_EVENT, path.relative(this.root, file), stats); + }.bind(this), + function endCallback() {}, + this.checkedEmitError, + this.ignored, + ); + } else { + this.rawEmitEvent(type, file, stat); + } + }, DEFAULT_DELAY); + } + + /** + * Actually emit the events + */ + rawEmitEvent(type, file, stat) { + this.emit(type, file, this.root, stat); + this.emit(ALL_EVENT, type, file, this.root, stat); + } +}; +/** + * Determine if a given FS error can be ignored + * + * @private + */ +function isIgnorableFileError(error) { + return ( + error.code === 'ENOENT' || + // Workaround Windows node issue #4337. + (error.code === 'EPERM' && platform === 'win32') + ); +} diff --git a/packages/metro-file-map/src/watchers/RecrawlWarning.js b/packages/metro-file-map/src/watchers/RecrawlWarning.js new file mode 100644 index 0000000000..3c8e4d1e83 --- /dev/null +++ b/packages/metro-file-map/src/watchers/RecrawlWarning.js @@ -0,0 +1,59 @@ +/** + * vendored from https://github.com/amasad/sane/blob/64ff3a870c42e84f744086884bf55a4f9c22d376/src/utils/recrawl-warning-dedupe.js + * @format + */ + +'use strict'; + +class RecrawlWarning { + constructor(root, count) { + this.root = root; + this.count = count; + } + + static findByRoot(root) { + for (let i = 0; i < this.RECRAWL_WARNINGS.length; i++) { + const warning = this.RECRAWL_WARNINGS[i]; + if (warning.root === root) { + return warning; + } + } + + return undefined; + } + + static isRecrawlWarningDupe(warningMessage) { + if (typeof warningMessage !== 'string') { + return false; + } + const match = warningMessage.match(this.REGEXP); + if (!match) { + return false; + } + const count = Number(match[1]); + const root = match[2]; + + const warning = this.findByRoot(root); + + if (warning) { + // only keep the highest count, assume count to either stay the same or + // increase. + if (warning.count >= count) { + return true; + } else { + // update the existing warning to the latest (highest) count + warning.count = count; + return false; + } + } else { + this.RECRAWL_WARNINGS.push(new RecrawlWarning(root, count)); + return false; + } + } +} + +RecrawlWarning.RECRAWL_WARNINGS = []; +RecrawlWarning.REGEXP = + /Recrawled this watch (\d+) times, most recently because:\n([^:]+)/; + +module.exports = RecrawlWarning; diff --git a/packages/metro-file-map/src/watchers/WatchmanWatcher.js b/packages/metro-file-map/src/watchers/WatchmanWatcher.js new file mode 100644 index 0000000000..9a207c567e --- /dev/null +++ b/packages/metro-file-map/src/watchers/WatchmanWatcher.js @@ -0,0 +1,325 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +import common from './common'; +import RecrawlWarning from './RecrawlWarning'; +import assert from 'assert'; +import {EventEmitter} from 'events'; +import watchman from 'fb-watchman'; +import * as fs from 'graceful-fs'; +import path from 'path'; + +const CHANGE_EVENT = common.CHANGE_EVENT; +const DELETE_EVENT = common.DELETE_EVENT; +const ADD_EVENT = common.ADD_EVENT; +const ALL_EVENT = common.ALL_EVENT; +const SUB_NAME = 'sane-sub'; + +/** + * Watches `dir`. + * + * @class PollWatcher + * @param String dir + * @param {Object} opts + * @public + */ + +export default function WatchmanWatcher(dir, opts) { + common.assignOptions(this, opts); + this.root = path.resolve(dir); + this.init(); +} + +// eslint-disable-next-line no-proto +WatchmanWatcher.prototype.__proto__ = EventEmitter.prototype; + +/** + * Run the watchman `watch` command on the root and subscribe to changes. + * + * @private + */ + +WatchmanWatcher.prototype.init = function () { + if (this.client) { + this.client.removeAllListeners(); + } + + const self = this; + this.client = new watchman.Client(); + this.client.on('error', error => { + self.emit('error', error); + }); + this.client.on('subscription', this.handleChangeEvent.bind(this)); + this.client.on('end', () => { + console.warn('[sane] Warning: Lost connection to watchman, reconnecting..'); + self.init(); + }); + + this.watchProjectInfo = null; + + function getWatchRoot() { + return self.watchProjectInfo ? self.watchProjectInfo.root : self.root; + } + + function onCapability(error, resp) { + if (handleError(self, error)) { + // The Watchman watcher is unusable on this system, we cannot continue + return; + } + + handleWarning(resp); + + self.capabilities = resp.capabilities; + + if (self.capabilities.relative_root) { + self.client.command(['watch-project', getWatchRoot()], onWatchProject); + } else { + self.client.command(['watch', getWatchRoot()], onWatch); + } + } + + function onWatchProject(error, resp) { + if (handleError(self, error)) { + return; + } + + handleWarning(resp); + + self.watchProjectInfo = { + relativePath: resp.relative_path ? resp.relative_path : '', + root: resp.watch, + }; + + self.client.command(['clock', getWatchRoot()], onClock); + } + + function onWatch(error, resp) { + if (handleError(self, error)) { + return; + } + + handleWarning(resp); + + self.client.command(['clock', getWatchRoot()], onClock); + } + + function onClock(error, resp) { + if (handleError(self, error)) { + return; + } + + handleWarning(resp); + + const options = { + fields: ['name', 'exists', 'new'], + since: resp.clock, + }; + + // If the server has the wildmatch capability available it supports + // the recursive **/*.foo style match and we can offload our globs + // to the watchman server. This saves both on data size to be + // communicated back to us and compute for evaluating the globs + // in our node process. + if (self.capabilities.wildmatch) { + if (self.globs.length === 0) { + if (!self.dot) { + // Make sure we honor the dot option if even we're not using globs. + options.expression = [ + 'match', + '**', + 'wholename', + { + includedotfiles: false, + }, + ]; + } + } else { + options.expression = ['anyof']; + for (const i in self.globs) { + options.expression.push([ + 'match', + self.globs[i], + 'wholename', + { + includedotfiles: self.dot, + }, + ]); + } + } + } + + if (self.capabilities.relative_root) { + options.relative_root = self.watchProjectInfo.relativePath; + } + + self.client.command( + ['subscribe', getWatchRoot(), SUB_NAME, options], + onSubscribe, + ); + } + + function onSubscribe(error, resp) { + if (handleError(self, error)) { + return; + } + + handleWarning(resp); + + self.emit('ready'); + } + + self.client.capabilityCheck( + { + optional: ['wildmatch', 'relative_root'], + }, + onCapability, + ); +}; + +/** + * Handles a change event coming from the subscription. + * + * @param {Object} resp + * @private + */ + +WatchmanWatcher.prototype.handleChangeEvent = function (resp) { + assert.equal(resp.subscription, SUB_NAME, 'Invalid subscription event.'); + if (resp.is_fresh_instance) { + this.emit('fresh_instance'); + } + if (resp.is_fresh_instance) { + this.emit('fresh_instance'); + } + if (Array.isArray(resp.files)) { + resp.files.forEach(this.handleFileChange, this); + } +}; + +/** + * Handles a single change event record. + * + * @param {Object} changeDescriptor + * @private + */ + +WatchmanWatcher.prototype.handleFileChange = function (changeDescriptor) { + const self = this; + let absPath; + let relativePath; + + if (this.capabilities.relative_root) { + relativePath = changeDescriptor.name; + absPath = path.join( + this.watchProjectInfo.root, + this.watchProjectInfo.relativePath, + relativePath, + ); + } else { + absPath = path.join(this.root, changeDescriptor.name); + relativePath = changeDescriptor.name; + } + + if ( + !(self.capabilities.wildmatch && !this.hasIgnore) && + !common.isFileIncluded(this.globs, this.dot, this.doIgnore, relativePath) + ) { + return; + } + + if (!changeDescriptor.exists) { + self.emitEvent(DELETE_EVENT, relativePath, self.root); + } else { + fs.lstat(absPath, (error, stat) => { + // Files can be deleted between the event and the lstat call + // the most reliable thing to do here is to ignore the event. + if (error && error.code === 'ENOENT') { + return; + } + + if (handleError(self, error)) { + return; + } + + const eventType = changeDescriptor.new ? ADD_EVENT : CHANGE_EVENT; + + // Change event on dirs are mostly useless. + if (!(eventType === CHANGE_EVENT && stat.isDirectory())) { + self.emitEvent(eventType, relativePath, self.root, stat); + } + }); + } +}; + +/** + * Dispatches the event. + * + * @param {string} eventType + * @param {string} filepath + * @param {string} root + * @param {fs.Stat} stat + * @private + */ + +WatchmanWatcher.prototype.emitEvent = function ( + eventType, + filepath, + root, + stat, +) { + this.emit(eventType, filepath, root, stat); + this.emit(ALL_EVENT, eventType, filepath, root, stat); +}; + +/** + * Closes the watcher. + * + */ + +WatchmanWatcher.prototype.close = function () { + this.client.removeAllListeners(); + this.client.end(); + return Promise.resolve(); +}; + +/** + * Handles an error and returns true if exists. + * + * @param {WatchmanWatcher} self + * @param {Error} error + * @private + */ + +function handleError(self, error) { + if (error != null) { + self.emit('error', error); + return true; + } else { + return false; + } +} + +/** + * Handles a warning in the watchman resp object. + * + * @param {object} resp + * @private + */ + +function handleWarning(resp) { + if ('warning' in resp) { + if (RecrawlWarning.isRecrawlWarningDupe(resp.warning)) { + return true; + } + console.warn(resp.warning); + return true; + } else { + return false; + } +} diff --git a/packages/metro-file-map/src/watchers/common.js b/packages/metro-file-map/src/watchers/common.js new file mode 100644 index 0000000000..1d49fb6dd5 --- /dev/null +++ b/packages/metro-file-map/src/watchers/common.js @@ -0,0 +1,115 @@ +/** + * vendored from https://github.com/amasad/sane/blob/64ff3a870c42e84f744086884bf55a4f9c22d376/src/common.js + * @format + */ + +'use strict'; + +const anymatch = require('anymatch'); +const micromatch = require('micromatch'); +const platform = require('os').platform(); +const path = require('path'); +const walker = require('walker'); + +/** + * Constants + */ + +exports.DEFAULT_DELAY = 100; +exports.CHANGE_EVENT = 'change'; +exports.DELETE_EVENT = 'delete'; +exports.ADD_EVENT = 'add'; +exports.ALL_EVENT = 'all'; + +/** + * Assigns options to the watcher. + * + * @param {NodeWatcher|PollWatcher|WatchmanWatcher} watcher + * @param {?object} opts + * @return {boolean} + * @public + */ + +exports.assignOptions = function (watcher, opts) { + opts = opts || {}; + watcher.globs = opts.glob || []; + watcher.dot = opts.dot || false; + watcher.ignored = opts.ignored || false; + + if (!Array.isArray(watcher.globs)) { + watcher.globs = [watcher.globs]; + } + watcher.hasIgnore = + Boolean(opts.ignored) && !(Array.isArray(opts) && opts.length > 0); + watcher.doIgnore = opts.ignored ? anymatch(opts.ignored) : () => false; + + if (opts.watchman && opts.watchmanPath) { + watcher.watchmanPath = opts.watchmanPath; + } + + return opts; +}; + +/** + * Checks a file relative path against the globs array. + * + * @param {array} globs + * @param {string} relativePath + * @return {boolean} + * @public + */ + +exports.isFileIncluded = function (globs, dot, doIgnore, relativePath) { + if (doIgnore(relativePath)) { + return false; + } + return globs.length + ? micromatch.some(relativePath, globs, {dot}) + : dot || micromatch.some(relativePath, '**/*'); +}; + +/** + * Traverse a directory recursively calling `callback` on every directory. + * + * @param {string} dir + * @param {function} dirCallback + * @param {function} fileCallback + * @param {function} endCallback + * @param {*} ignored + * @public + */ + +exports.recReaddir = function ( + dir, + dirCallback, + fileCallback, + endCallback, + errorCallback, + ignored, +) { + walker(dir) + .filterDir(currentDir => !anymatch(ignored, currentDir)) + .on('dir', normalizeProxy(dirCallback)) + .on('file', normalizeProxy(fileCallback)) + .on('error', errorCallback) + .on('end', () => { + if (platform === 'win32') { + setTimeout(endCallback, 1000); + } else { + endCallback(); + } + }); +}; + +/** + * Returns a callback that when called will normalize a path and call the + * original callback + * + * @param {function} callback + * @return {function} + * @private + */ + +function normalizeProxy(callback) { + return (filepath, stats) => callback(path.normalize(filepath), stats); +} diff --git a/packages/metro-file-map/src/worker.js b/packages/metro-file-map/src/worker.js new file mode 100644 index 0000000000..86f27261c0 --- /dev/null +++ b/packages/metro-file-map/src/worker.js @@ -0,0 +1,128 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @noformat + */ + +/*:: +import type {WorkerMessage, WorkerMetadata} from './flow-types'; +*/ + +'use strict'; + +const H = require('./constants'); +const dependencyExtractor = require('./lib/dependencyExtractor'); +const exclusionList = require('./workerExclusionList'); +const {createHash} = require('crypto'); +const fs = require('graceful-fs'); +const path = require('path'); + +const PACKAGE_JSON = path.sep + 'package.json'; + +let hasteImpl /*: ?{getHasteName: string => string} */ = null; +let hasteImplModulePath /*: ?string */ = null; + +function sha1hex(content /*: string | Buffer */) /*: string */ { + return createHash('sha1').update(content).digest('hex'); +} + +async function worker( + data /*: WorkerMessage */, +) /*: Promise */ { + if ( + data.hasteImplModulePath != null && + data.hasteImplModulePath !== hasteImplModulePath + ) { + if (hasteImpl) { + throw new Error('jest-haste-map: hasteImplModulePath changed'); + } + hasteImplModulePath = data.hasteImplModulePath; + // $FlowFixMe[unsupported-syntax] - dynamic require + hasteImpl = require(hasteImplModulePath); + } + + let content/*: ?string */; + let dependencies/*: WorkerMetadata['dependencies'] */; + let id/*: WorkerMetadata['id'] */; + let module/*: WorkerMetadata['module'] */; + let sha1/*: WorkerMetadata['sha1'] */; + + const {computeDependencies, computeSha1, rootDir, filePath} = data; + + const getContent = () /*: string */ => { + if (content == null) { + content = fs.readFileSync(filePath, 'utf8'); + } + + return content; + }; + + if (filePath.endsWith(PACKAGE_JSON)) { + // Process a package.json that is returned as a PACKAGE type with its name. + try { + const fileData = JSON.parse(getContent()); + + if (fileData.name) { + const relativeFilePath = path.relative(rootDir, filePath); + id = fileData.name; + module = [relativeFilePath, H.PACKAGE]; + } + } catch (err) { + throw new Error(`Cannot parse ${filePath} as JSON: ${err.message}`); + } + } else if (!exclusionList.has(filePath.substr(filePath.lastIndexOf('.')))) { + // Process a random file that is returned as a MODULE. + if (hasteImpl) { + id = hasteImpl.getHasteName(filePath); + } + + if (computeDependencies) { + dependencies = Array.from( + data.dependencyExtractor != null + // $FlowFixMe[unsupported-syntax] - dynamic require + ? require(data.dependencyExtractor).extract( + getContent(), + filePath, + dependencyExtractor.extract, + ) + : dependencyExtractor.extract(getContent()), + ); + } + + if (id != null) { + const relativeFilePath = path.relative(rootDir, filePath); + module = [relativeFilePath, H.MODULE]; + } + } + + // If a SHA-1 is requested on update, compute it. + if (computeSha1) { + sha1 = sha1hex(getContent() || fs.readFileSync(filePath)); + } + + return {dependencies, id, module, sha1}; +} + +async function getSha1( + data /*: WorkerMessage */, +) /*: Promise */ { + const sha1 = data.computeSha1 + ? sha1hex(fs.readFileSync(data.filePath)) + : null; + + return { + dependencies: undefined, + id: undefined, + module: undefined, + sha1, + }; +} + +module.exports = { + worker, + getSha1, +}; diff --git a/packages/metro-file-map/src/workerExclusionList.js b/packages/metro-file-map/src/workerExclusionList.js new file mode 100644 index 0000000000..050a44a39b --- /dev/null +++ b/packages/metro-file-map/src/workerExclusionList.js @@ -0,0 +1,67 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @flow strict + */ + +// This list is compiled after the MDN list of the most common MIME types (see +// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/ +// Complete_list_of_MIME_types). +// +// Only MIME types starting with "image/", "video/", "audio/" and "font/" are +// reflected in the list. Adding "application/" is too risky since some text +// file formats (like ".js" and ".json") have an "application/" MIME type. +// +// Feel free to add any extensions that cannot be a Haste module. + +'use strict'; + +const extensions /*: $ReadOnlySet */ = new Set([ + // JSONs are never haste modules, except for "package.json", which is handled. + '.json', + + // Image extensions. + '.bmp', + '.gif', + '.ico', + '.jpeg', + '.jpg', + '.png', + '.svg', + '.tiff', + '.tif', + '.webp', + + // Video extensions. + '.avi', + '.mp4', + '.mpeg', + '.mpg', + '.ogv', + '.webm', + '.3gp', + '.3g2', + + // Audio extensions. + '.aac', + '.midi', + '.mid', + '.mp3', + '.oga', + '.wav', + '.3gp', + '.3g2', + + // Font extensions. + '.eot', + '.otf', + '.ttf', + '.woff', + '.woff2', +]); + +module.exports = extensions;