From b49c4ad8fa9519bc0754ad30f2ad7fd241c4f6c1 Mon Sep 17 00:00:00 2001 From: Scott Hovestadt Date: Mon, 21 Dec 2020 22:22:59 -0800 Subject: [PATCH 1/4] feat(jest-haste-map): handle injected scm clocks --- .../src/crawlers/__tests__/watchman.test.js | 85 ++++++++++++++++ .../jest-haste-map/src/crawlers/watchman.ts | 96 ++++++++++++++----- packages/jest-haste-map/src/types.ts | 3 +- 3 files changed, 161 insertions(+), 23 deletions(-) diff --git a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js index b86340059106..fe800dfefe2d 100644 --- a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js +++ b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js @@ -589,4 +589,89 @@ describe('watchman watch', () => { expect(calls[0][0]).toEqual(['list-capabilities']); expect(calls[2][0][2].fields).not.toContain('content.sha1hex'); }); + + test('source control query', async () => { + mockResponse = { + 'list-capabilities': { + [undefined]: { + capabilities: ['field-content.sha1hex'], + }, + }, + query: { + [ROOT_MOCK]: { + clock: { + scm: { + 'mergebase-with': 'master', + mergebase: 'master', + }, + clock: 'c:1608612057:79675:1:139410', + }, + files: [ + { + exists: true, + mtime_ms: {toNumber: () => 42}, + name: 'fruits/kiwi.js', + size: 40, + }, + { + exists: false, + mtime_ms: null, + name: 'fruits/tomato.js', + size: 0, + }, + ], + // Watchman is going to tell us that we have a fresh instance. + is_fresh_instance: true, + version: '4.5.0', + }, + }, + 'watch-project': WATCH_PROJECT_MOCK, + }; + + // Start with a source-control clock. + const clocks = createMap({ + '': {scm: {'mergebase-with': 'master'}}, + }); + + return watchmanCrawl({ + data: { + clocks, + files: mockFiles, + }, + extensions: ['js', 'json'], + ignore: pearMatcher, + rootDir: ROOT_MOCK, + roots: ROOTS, + }).then(({changedFiles, hasteMap, removedFiles}) => { + // The object was reused. + expect(hasteMap.files).toBe(mockFiles); + + // Transformed into a normal clock. + expect(hasteMap.clocks).toEqual( + createMap({ + '': 'c:1608612057:79675:1:139410', + }), + ); + + expect(changedFiles).toEqual( + createMap({ + [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], + }), + ); + + expect(hasteMap.files).toEqual( + createMap({ + [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], + }), + ); + + expect(removedFiles).toEqual( + createMap({ + [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null], + }), + ); + }); + }); }); diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index e5647370813b..442bff500f8e 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -20,6 +20,34 @@ import type { type WatchmanRoots = Map>; +type WatchmanListCapabilitiesResponse = { + capabilities: Array; +}; + +type WatchmanWatchProjectResponse = { + watch: string; + relative_path: string; +}; + +type WatchmanQueryResponse = { + warning?: string; + is_fresh_instance: boolean; + version: string; + clock: + | string + | { + scm: {'mergebase-with': string; mergebase: string}; + clock: string; + }; + files: Array<{ + name: string; + exists: boolean; + mtime_ms: number | {toNumber: () => number}; + size: number; + 'content.sha1hex'?: string; + }>; +}; + const watchmanURL = 'https://facebook.github.io/watchman/docs/troubleshooting'; function WatchmanError(error: Error): Error { @@ -49,8 +77,7 @@ export = async function watchmanCrawl( let clientError; client.on('error', error => (clientError = WatchmanError(error))); - // TODO: type better than `any` - const cmd = (...args: Array): Promise => + const cmd = (...args: Array): Promise => new Promise((resolve, reject) => client.command(args, (error, result) => error ? reject(WatchmanError(error)) : resolve(result), @@ -58,7 +85,9 @@ export = async function watchmanCrawl( ); if (options.computeSha1) { - const {capabilities} = await cmd('list-capabilities'); + const {capabilities} = await cmd( + 'list-capabilities', + ); if (capabilities.indexOf('field-content.sha1hex') !== -1) { fields.push('content.sha1hex'); @@ -71,7 +100,10 @@ export = async function watchmanCrawl( const watchmanRoots = new Map(); await Promise.all( roots.map(async root => { - const response = await cmd('watch-project', root); + const response = await cmd( + 'watch-project', + root, + ); const existing = watchmanRoots.get(response.watch); // A root can only be filtered if it was never seen with a // relative_path before. @@ -96,7 +128,7 @@ export = async function watchmanCrawl( } async function queryWatchmanForDirs(rootProjectDirMappings: WatchmanRoots) { - const files = new Map(); + const results = new Map(); let isFresh = false; await Promise.all( Array.from(rootProjectDirMappings).map( @@ -122,26 +154,41 @@ export = async function watchmanCrawl( } const relativeRoot = fastPath.relative(rootDir, root); - const query = clocks.has(relativeRoot) - ? // Use the `since` generator if we have a clock available - {expression, fields, since: clocks.get(relativeRoot)} - : // Otherwise use the `glob` filter - {expression, fields, glob, glob_includedotfiles: true}; - - const response = await cmd('query', root, query); + const since = clocks.get(relativeRoot); + const query = + since !== undefined + ? // Use the `since` generator if we have a clock available + {expression, fields, since} + : // Otherwise use the `glob` filter + {expression, fields, glob, glob_includedotfiles: true}; + + const response = await cmd( + 'query', + root, + query, + ); if ('warning' in response) { console.warn('watchman warning: ', response.warning); } - isFresh = isFresh || response.is_fresh_instance; - files.set(root, response); + // When a source-control query is used, we ignore the "is fresh" + // response from Watchman because it will be true despite the query + // being incremental. + const isSourceControlQuery = + typeof since !== 'string' && + since?.scm?.['mergebase-with'] !== undefined; + if (!isSourceControlQuery) { + isFresh = isFresh || response.is_fresh_instance; + } + + results.set(root, response); }, ), ); return { - files, + results, isFresh, }; } @@ -149,7 +196,7 @@ export = async function watchmanCrawl( let files = data.files; let removedFiles = new Map(); const changedFiles = new Map(); - let watchmanFiles: Map; + let results: Map; let isFresh = false; try { const watchmanRoots = await getWatchmanRoots(roots); @@ -163,7 +210,7 @@ export = async function watchmanCrawl( isFresh = true; } - watchmanFiles = watchmanFileResults.files; + results = watchmanFileResults.results; } finally { client.end(); } @@ -172,11 +219,16 @@ export = async function watchmanCrawl( throw clientError; } - // TODO: remove non-null - for (const [watchRoot, response] of watchmanFiles!) { + for (const [watchRoot, response] of results) { const fsRoot = normalizePathSep(watchRoot); const relativeFsRoot = fastPath.relative(rootDir, fsRoot); - clocks.set(relativeFsRoot, response.clock); + clocks.set( + relativeFsRoot, + // Ensure we persist only the local clock. + typeof response.clock === 'string' + ? response.clock + : response.clock.clock, + ); for (const fileData of response.files) { const filePath = fsRoot + path.sep + normalizePathSep(fileData.name); @@ -209,7 +261,7 @@ export = async function watchmanCrawl( let sha1hex = fileData['content.sha1hex']; if (typeof sha1hex !== 'string' || sha1hex.length !== 40) { - sha1hex = null; + sha1hex = undefined; } let nextData: FileMetaData; @@ -231,7 +283,7 @@ export = async function watchmanCrawl( ]; } else { // See ../constants.ts - nextData = ['', mtime, size, 0, '', sha1hex]; + nextData = ['', mtime, size, 0, '', sha1hex ?? null]; } files.set(relativeFilePath, nextData); diff --git a/packages/jest-haste-map/src/types.ts b/packages/jest-haste-map/src/types.ts index 14a08bac856e..61dee043c751 100644 --- a/packages/jest-haste-map/src/types.ts +++ b/packages/jest-haste-map/src/types.ts @@ -55,7 +55,8 @@ export type FileMetaData = [ export type MockData = Map; export type ModuleMapData = Map; -export type WatchmanClocks = Map; +export type WatchmanClockSpec = string | {scm: {'mergebase-with': string}}; +export type WatchmanClocks = Map; export type HasteRegExp = RegExp | ((str: string) => boolean); export type DuplicatesSet = Map; From 2ceb30344d53744fda8bb09168e3b6e29410a92b Mon Sep 17 00:00:00 2001 From: Scott Hovestadt Date: Mon, 21 Dec 2020 22:39:27 -0800 Subject: [PATCH 2/4] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57c0f6692aa5..fbae23d6afc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - `[jest-runner]` [**BREAKING**] Run transforms over `testRunnner` ([#8823](https://github.com/facebook/jest/pull/8823)) - `[jest-runtime, jest-transform]` share `cacheFS` between runtime and transformer ([#10901](https://github.com/facebook/jest/pull/10901)) - `[jest-transform]` Pass config options defined in Jest's config to transformer's `process` and `getCacheKey` functions ([#10926](https://github.com/facebook/jest/pull/10926)) +- `[jest-haste-map]` Handle injected scm clocks ([#10966](https://github.com/facebook/jest/pull/10966)) ### Fixes From f557fd07158743a76919cca7e570c0c0d4b27b26 Mon Sep 17 00:00:00 2001 From: Scott Hovestadt Date: Mon, 21 Dec 2020 22:47:48 -0800 Subject: [PATCH 3/4] Fix lint errors. --- .../jest-haste-map/src/crawlers/__tests__/watchman.test.js | 4 ++-- packages/jest-haste-map/src/crawlers/watchman.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js index fe800dfefe2d..91ea8ee70ca8 100644 --- a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js +++ b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js @@ -600,11 +600,11 @@ describe('watchman watch', () => { query: { [ROOT_MOCK]: { clock: { + clock: 'c:1608612057:79675:1:139410', scm: { - 'mergebase-with': 'master', mergebase: 'master', + 'mergebase-with': 'master', }, - clock: 'c:1608612057:79675:1:139410', }, files: [ { diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index 442bff500f8e..250b5a7bb9f2 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -188,8 +188,8 @@ export = async function watchmanCrawl( ); return { - results, isFresh, + results, }; } From 583dce7da6fccb4e58bee75b1297c24cfb81c56b Mon Sep 17 00:00:00 2001 From: Scott Hovestadt Date: Tue, 22 Dec 2020 10:12:06 -0800 Subject: [PATCH 4/4] Apply feedback. --- CHANGELOG.md | 2 +- .../src/crawlers/__tests__/watchman.test.js | 62 +++++++++---------- .../jest-haste-map/src/crawlers/watchman.ts | 12 +++- 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbae23d6afc9..2eba4d741411 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - `[jest-config, jest-runtime]` Support ESM for files other than `.js` and `.mjs` ([#10823](https://github.com/facebook/jest/pull/10823)) - `[jest-config, jest-runtime]` [**BREAKING**] Use "modern" implementation as default for fake timers ([#10874](https://github.com/facebook/jest/pull/10874)) - `[jest-core]` make `TestWatcher` extend `emittery` ([#10324](https://github.com/facebook/jest/pull/10324)) +- `[jest-haste-map]` Handle injected scm clocks ([#10966](https://github.com/facebook/jest/pull/10966)) - `[jest-repl, jest-runner]` [**BREAKING**] Run transforms over environment ([#8751](https://github.com/facebook/jest/pull/8751)) - `[jest-runner]` [**BREAKING**] set exit code to 1 if test logs after teardown ([#10728](https://github.com/facebook/jest/pull/10728)) - `[jest-snapshot]` [**BREAKING**] Make prettier optional for inline snapshots - fall back to string replacement ([#7792](https://github.com/facebook/jest/pull/7792)) @@ -15,7 +16,6 @@ - `[jest-runner]` [**BREAKING**] Run transforms over `testRunnner` ([#8823](https://github.com/facebook/jest/pull/8823)) - `[jest-runtime, jest-transform]` share `cacheFS` between runtime and transformer ([#10901](https://github.com/facebook/jest/pull/10901)) - `[jest-transform]` Pass config options defined in Jest's config to transformer's `process` and `getCacheKey` functions ([#10926](https://github.com/facebook/jest/pull/10926)) -- `[jest-haste-map]` Handle injected scm clocks ([#10966](https://github.com/facebook/jest/pull/10966)) ### Fixes diff --git a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js index 91ea8ee70ca8..cce27c7e9487 100644 --- a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js +++ b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js @@ -633,7 +633,7 @@ describe('watchman watch', () => { '': {scm: {'mergebase-with': 'master'}}, }); - return watchmanCrawl({ + const {changedFiles, hasteMap, removedFiles} = await watchmanCrawl({ data: { clocks, files: mockFiles, @@ -642,36 +642,36 @@ describe('watchman watch', () => { ignore: pearMatcher, rootDir: ROOT_MOCK, roots: ROOTS, - }).then(({changedFiles, hasteMap, removedFiles}) => { - // The object was reused. - expect(hasteMap.files).toBe(mockFiles); - - // Transformed into a normal clock. - expect(hasteMap.clocks).toEqual( - createMap({ - '': 'c:1608612057:79675:1:139410', - }), - ); - - expect(changedFiles).toEqual( - createMap({ - [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], - }), - ); - - expect(hasteMap.files).toEqual( - createMap({ - [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], - [MELON_RELATIVE]: ['', 33, 43, 0, '', null], - [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], - }), - ); - - expect(removedFiles).toEqual( - createMap({ - [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null], - }), - ); }); + + // The object was reused. + expect(hasteMap.files).toBe(mockFiles); + + // Transformed into a normal clock. + expect(hasteMap.clocks).toEqual( + createMap({ + '': 'c:1608612057:79675:1:139410', + }), + ); + + expect(changedFiles).toEqual( + createMap({ + [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], + }), + ); + + expect(hasteMap.files).toEqual( + createMap({ + [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], + }), + ); + + expect(removedFiles).toEqual( + createMap({ + [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null], + }), + ); }); }); diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index 250b5a7bb9f2..372f76550f6d 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -153,8 +153,16 @@ export = async function watchmanCrawl( } } - const relativeRoot = fastPath.relative(rootDir, root); - const since = clocks.get(relativeRoot); + // Jest is only going to store one type of clock; a string that + // represents a local clock. However, the Watchman crawler supports + // a second type of clock that can be written by automation outside of + // Jest, called an "scm query", which fetches changed files based on + // source control mergebases. The reason this is necessary is because + // local clocks are not portable across systems, but scm queries are. + // By using scm queries, we can create the haste map on a different + // system and import it, transforming the clock into a local clock. + const since = clocks.get(fastPath.relative(rootDir, root)); + const query = since !== undefined ? // Use the `since` generator if we have a clock available