From f88ae4bee0886abc48370b6dd672af84e7bbd2de Mon Sep 17 00:00:00 2001 From: scotthovestadt Date: Tue, 2 Apr 2019 12:42:21 -0700 Subject: [PATCH] Fix fs watcher EMFILE error. (#8258) * Fix fs watcher EMFILE error. * Improve types. * Resolve eslint error. * Resolve eslint error. * Update CHANGELOG.md * Eslint on CI. * Update CHANGELOG.md * Copyright header. --- CHANGELOG.md | 2 + packages/jest-haste-map/package.json | 9 +- packages/jest-haste-map/src/index.ts | 5 + .../jest-haste-map/src/lib/FSEventsWatcher.ts | 187 ++++++++++++++++++ yarn.lock | 14 +- 5 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 packages/jest-haste-map/src/lib/FSEventsWatcher.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6efadbb3da66..bb99b1239a92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ ### Fixes +- `[jest-haste-map]` Resolve fs watcher EMFILE error ([#8258](https://github.com/facebook/jest/pull/8258)) + ### Chore & Maintenance - `[expect]` Remove repetition of matcherName and options in matchers ([#8224](https://github.com/facebook/jest/pull/8224)) diff --git a/packages/jest-haste-map/package.json b/packages/jest-haste-map/package.json index 07e831226343..3de6f7ae0521 100644 --- a/packages/jest-haste-map/package.json +++ b/packages/jest-haste-map/package.json @@ -11,6 +11,7 @@ "types": "build/index.d.ts", "dependencies": { "@jest/types": "^24.6.0", + "anymatch": "^2.0.0", "fb-watchman": "^2.0.0", "graceful-fs": "^4.1.15", "invariant": "^2.2.4", @@ -18,15 +19,21 @@ "jest-util": "^24.6.0", "jest-worker": "^24.6.0", "micromatch": "^3.1.10", - "sane": "^4.0.3" + "sane": "^4.0.3", + "walker": "^1.0.7" }, "devDependencies": { + "@types/anymatch": "^1.3.1", "@types/fb-watchman": "^2.0.0", + "@types/fsevents": "^1.1.0", "@types/graceful-fs": "^4.1.2", "@types/invariant": "^2.2.29", "@types/micromatch": "^3.1.0", "@types/sane": "^2.0.0" }, + "optionalDependencies": { + "fsevents": "^1.2.7" + }, "engines": { "node": ">= 6" }, diff --git a/packages/jest-haste-map/src/index.ts b/packages/jest-haste-map/src/index.ts index d39be76252cf..5f14909645c7 100644 --- a/packages/jest-haste-map/src/index.ts +++ b/packages/jest-haste-map/src/index.ts @@ -29,6 +29,7 @@ import normalizePathSep from './lib/normalizePathSep'; import watchmanCrawl from './crawlers/watchman'; // @ts-ignore: not converted to TypeScript - it's a fork: https://github.com/facebook/jest/pull/5387 import WatchmanWatcher from './lib/WatchmanWatcher'; +import FSEventsWatcher from './lib/FSEventsWatcher'; import * as fastPath from './lib/fast_path'; import { ChangeEvent, @@ -795,10 +796,14 @@ class HasteMap extends EventEmitter { this._options.throwOnModuleCollision = false; this._options.retainAllFiles = true; + // WatchmanWatcher > FSEventsWatcher > sane.NodeWatcher const Watcher: sane.Watcher = canUseWatchman && this._options.useWatchman ? WatchmanWatcher + : FSEventsWatcher.isSupported() + ? FSEventsWatcher : sane.NodeWatcher; + const extensions = this._options.extensions; const ignorePattern = this._options.ignorePattern; const rootDir = this._options.rootDir; diff --git a/packages/jest-haste-map/src/lib/FSEventsWatcher.ts b/packages/jest-haste-map/src/lib/FSEventsWatcher.ts new file mode 100644 index 000000000000..b0ebad2867dd --- /dev/null +++ b/packages/jest-haste-map/src/lib/FSEventsWatcher.ts @@ -0,0 +1,187 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import fs from 'fs'; +import path from 'path'; +import {EventEmitter} from 'events'; +import anymatch from 'anymatch'; +import micromatch from 'micromatch'; +// eslint-disable-next-line +import {Watcher} from 'fsevents'; +// @ts-ignore no types +import walker from 'walker'; + +let fsevents: (path: string) => Watcher; +try { + fsevents = require('fsevents'); +} catch (e) { + // Optional dependency, only supported on Darwin. +} + +const CHANGE_EVENT = 'change'; +const DELETE_EVENT = 'delete'; +const ADD_EVENT = 'add'; +const ALL_EVENT = 'all'; + +type FsEventsWatcherEvent = + | typeof CHANGE_EVENT + | typeof DELETE_EVENT + | typeof ADD_EVENT + | typeof ALL_EVENT; + +/** + * Export `FSEventsWatcher` class. + * Watches `dir`. + */ +class FSEventsWatcher extends EventEmitter { + public readonly root: string; + public readonly ignored?: anymatch.Matcher; + public readonly glob: Array; + public readonly dot: boolean; + public readonly hasIgnore: boolean; + public readonly doIgnore: (path: string) => boolean; + public readonly watcher: Watcher; + private _tracked: Set; + + static isSupported() { + return fsevents !== undefined; + } + + private static normalizeProxy( + callback: (normalizedPath: string, stats: fs.Stats) => void, + ) { + return (filepath: string, stats: fs.Stats) => + callback(path.normalize(filepath), stats); + } + + private static recReaddir( + dir: string, + dirCallback: (normalizedPath: string, stats: fs.Stats) => void, + fileCallback: (normalizedPath: string, stats: fs.Stats) => void, + endCallback: Function, + errorCallback: Function, + ignored?: anymatch.Matcher, + ) { + walker(dir) + .filterDir( + (currentDir: string) => !ignored || !anymatch(ignored, currentDir), + ) + .on('dir', FSEventsWatcher.normalizeProxy(dirCallback)) + .on('file', FSEventsWatcher.normalizeProxy(fileCallback)) + .on('error', errorCallback) + .on('end', () => { + endCallback(); + }); + } + + constructor( + dir: string, + opts: { + root: string; + ignored?: anymatch.Matcher; + glob: string | Array; + dot: boolean; + }, + ) { + if (!fsevents) { + throw new Error( + '`fsevents` unavailable (this watcher can only be used on Darwin)', + ); + } + + super(); + + this.dot = opts.dot || false; + this.ignored = opts.ignored; + this.glob = Array.isArray(opts.glob) ? opts.glob : [opts.glob]; + + this.hasIgnore = + Boolean(opts.ignored) && !(Array.isArray(opts) && opts.length > 0); + this.doIgnore = opts.ignored ? anymatch(opts.ignored) : () => false; + + this.root = path.resolve(dir); + this.watcher = fsevents(this.root); + + this.watcher.start().on('change', this.handleEvent.bind(this)); + this._tracked = new Set(); + FSEventsWatcher.recReaddir( + this.root, + (filepath: string) => { + this._tracked.add(filepath); + }, + (filepath: string) => { + this._tracked.add(filepath); + }, + this.emit.bind(this, 'ready'), + this.emit.bind(this, 'error'), + this.ignored, + ); + } + + /** + * End watching. + */ + close(callback?: () => void) { + this.watcher.stop(); + this.removeAllListeners(); + if (typeof callback === 'function') { + process.nextTick(callback.bind(null, null, true)); + } + } + + private isFileIncluded(relativePath: string) { + if (this.doIgnore(relativePath)) { + return false; + } + return this.glob.length + ? micromatch.some(relativePath, this.glob, {dot: this.dot}) + : this.dot || micromatch.some(relativePath, '**/*'); + } + + private handleEvent(filepath: string) { + const relativePath = path.relative(this.root, filepath); + if (!this.isFileIncluded(relativePath)) { + return; + } + + fs.lstat(filepath, (error, stat) => { + if (error && error.code !== 'ENOENT') { + this.emit('error', error); + return; + } + + if (error) { + // Ignore files that aren't tracked and don't exist. + if (!this._tracked.has(filepath)) { + return; + } + + this._emit(DELETE_EVENT, relativePath); + this._tracked.delete(filepath); + return; + } + + if (this._tracked.has(filepath)) { + this._emit(CHANGE_EVENT, relativePath, stat); + } else { + this._tracked.add(filepath); + this._emit(ADD_EVENT, relativePath, stat); + } + }); + } + + /** + * Emit events. + */ + private _emit(type: FsEventsWatcherEvent, file: string, stat?: fs.Stats) { + this.emit(type, file, this.root, stat); + this.emit(ALL_EVENT, type, file, this.root, stat); + } +} + +export = FSEventsWatcher; diff --git a/yarn.lock b/yarn.lock index 4a4fd3a84fd2..478514037524 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1583,6 +1583,11 @@ dependencies: "@types/color-name" "*" +"@types/anymatch@^1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@types/anymatch/-/anymatch-1.3.1.tgz#336badc1beecb9dacc38bea2cf32adf627a8421a" + integrity sha512-/+CRPXpBDpo2RK9C68N3b2cOvO0Cf5B9aPijHsoDQTHivnGSObdOF2BRQOYjojWTDy6nQvMjmqRXIxH55VjxxA== + "@types/babel-types@*": version "7.0.6" resolved "https://registry.yarnpkg.com/@types/babel-types/-/babel-types-7.0.6.tgz#a7cfaaeee96e90c4c54da0e580aaff3f4cffacac" @@ -1697,6 +1702,13 @@ dependencies: "@types/events" "*" +"@types/fsevents@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@types/fsevents/-/fsevents-1.1.0.tgz#11964ddbfed08bd45b037f37fbb53e5012609323" + integrity sha512-H29jEeDqjbolciuREYzd/f7xCc8l2QZmjfSHy+IQZoBHtl88/y4ZLOdz22SUbXSXsc/qI9bTOcDxwgdbY/RQ9A== + dependencies: + "@types/node" "*" + "@types/glob@*", "@types/glob@^7.1.1": version "7.1.1" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575" @@ -13236,7 +13248,7 @@ wait-for-expect@^1.1.0: resolved "https://registry.yarnpkg.com/wait-for-expect/-/wait-for-expect-1.1.0.tgz#6607375c3f79d32add35cd2c87ce13f351a3d453" integrity sha512-vQDokqxyMyknfX3luCDn16bSaRcOyH6gGuUXMIbxBLeTo6nWuEWYqMTT9a+44FmW8c2m6TRWBdNvBBjA1hwEKg== -walker@~1.0.5: +walker@^1.0.7, walker@~1.0.5: version "1.0.7" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.7.tgz#2f7f9b8fd10d677262b18a884e28d19618e028fb" integrity sha1-L3+bj9ENZ3JisYqITijRlhjgKPs=