Add node worker-thread support to jest-worker (#7408)

* Restructure workers (create a base class and inherit from it)

Clean up types, create missing interfaes and make Flow and Jest happy

* Move child.js to workers folder

* Restructure base classes and make a working POC

* Remove BaseWorker

* Remove MessageChannel and cleanup super() calls

* Use worker threads implementation if possible

* Restructure queues

* Support experimental modules in jest-resolver

* Rename child.js to processChild.js

* Remove private properties from WorkerPoolInterface

* Move common line outside of if-else

* Unify interface (use workerId) and remove recursion

* Remove opt-out option for worker_threads in node 10.5+

* Alphabetical import sorting

* Unlock worker after onEnd

* Cache queue head in the getNextJob loop

* Elegant while loop

* Remove redundand .binds

* Clean up interfaces and responsibilites

* Update jest-worker

* Add changelog and update jest-worker readme

* Fix lint lol

* Fixes from review

* Update Changelog

* rm function

* rm []

* Make imports alphabetical 🤮

* Go back to any

* Fix lint

* \n

* Fix formatting

* Add docs

* Revert canUseWorkerThreads

* Fix lint

* Fix pathing on windows

.gitignore

@@ -1,3 +1,4 @@
+.idea
 .DS_STORE
 .eslintcache
 *.swp

CHANGELOG.md

@@ -33,6 +33,7 @@
 - `[jest-haste-map]` Accept a `getCacheKey` method in `hasteImplModulePath` modules to reset the cache when the logic changes ([#7350](https://github.com/facebook/jest/pull/7350))
 - `[jest-config]` Add `haste.computeSha1` option to compute the sha-1 of the files in the haste map ([#7345](https://github.com/facebook/jest/pull/7345))
 - `[expect]` `expect(Infinity).toBeCloseTo(Infinity)` Treats `Infinity` as equal in toBeCloseTo matcher ([#7405](https://github.com/facebook/jest/pull/7405))
+- `[jest-worker]` Add node worker-thread support to jest-worker ([#7408](https://github.com/facebook/jest/pull/7408))
 
 ### Fixes
 

packages/jest-resolve/src/isBuiltinModule.js

@@ -15,11 +15,13 @@ declare var process: {
   binding(type: string): {},
 };
 
+const EXPERIMENTAL_MODULES = ['worker_threads'];
+
 const BUILTIN_MODULES =
-  builtinModules ||
-  Object.keys(process.binding('natives')).filter(
-    (module: string) => !/^internal\//.test(module),
-  );
+  builtinModules.concat(EXPERIMENTAL_MODULES) ||
+  Object.keys(process.binding('natives'))
+    .filter((module: string) => !/^internal\//.test(module))
+    .concat(EXPERIMENTAL_MODULES);
 
 export default function isBuiltinModule(module: string): boolean {
   return BUILTIN_MODULES.indexOf(module) !== -1;

packages/jest-worker/README.md

@@ -39,6 +39,12 @@ export function hello(param) {
 }
 ```
 
+## Experimental worker
+
+Node 10 shipped with [worker-threads](https://nodejs.org/api/worker_threads.html), a "threading API" that uses SharedArrayBuffers to communicate between the main process and its child threads. This experimental Node feature can significantly improve the communication time between parent and child processes in `jest-worker`.
+
+We will use worker threads where available. To enable in Node 10+, run the Node process with the `--experimental-worker` flag.
+
 ## API
 
 The only exposed method is a constructor (`Worker`) that is initialized by passing the worker path, plus an options object.
@@ -77,6 +83,12 @@ By default, no process is bound to any worker.
 
 The arguments that will be passed to the `setup` method during initialization.
 
+#### `workerPool: (workerPath: string, options?: WorkerPoolOptions) => WorkerPoolInterface` (optional)
+
+Provide a custom worker pool to be used for spawning child processes. By default, Jest will use a node thread pool if available and fall back to child process threads.
+
+The arguments that will be passed to the `setup` method during initialization.
+
 ## Worker
 
 The returned `Worker` instance has all the exposed methods, plus some additional ones to interact with the workers itself:

packages/jest-worker/src/Farm.js

@@ -0,0 +1,160 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ */
+
+'use strict';
+
+import type {
+  ChildMessage,
+  QueueChildMessage,
+  WorkerInterface,
+  OnStart,
+  OnEnd,
+} from './types';
+import {CHILD_MESSAGE_CALL} from './types';
+
+export default class Farm {
+  _computeWorkerKey: (string, ...Array<any>) => ?string;
+  _cacheKeys: {[string]: WorkerInterface, __proto__: null};
+  _callback: Function;
+  _last: Array<QueueChildMessage>;
+  _locks: Array<boolean>;
+  _numOfWorkers: number;
+  _offset: number;
+  _queue: Array<?QueueChildMessage>;
+
+  constructor(
+    numOfWorkers: number,
+    callback: Function,
+    computeWorkerKey?: (string, ...Array<any>) => ?string,
+  ) {
+    this._callback = callback;
+    this._numOfWorkers = numOfWorkers;
+    this._cacheKeys = Object.create(null);
+    this._queue = [];
+    this._last = [];
+    this._locks = [];
+    this._offset = 0;
+    if (computeWorkerKey) {
+      this._computeWorkerKey = computeWorkerKey;
+    }
+  }
+
+  doWork(method: string, ...args: Array<any>): Promise<mixed> {
+    return new Promise((resolve, reject) => {
+      const computeWorkerKey = this._computeWorkerKey;
+      const request: ChildMessage = [CHILD_MESSAGE_CALL, false, method, args];
+
+      let worker: ?WorkerInterface = null;
+      let hash: ?string = null;
+
+      if (computeWorkerKey) {
+        hash = computeWorkerKey.apply(this, [method].concat(args));
+        worker = hash == null ? null : this._cacheKeys[hash];
+      }
+
+      const onStart: OnStart = (worker: WorkerInterface) => {
+        if (hash != null) {
+          this._cacheKeys[hash] = worker;
+        }
+      };
+
+      const onEnd: OnEnd = (error: ?Error, result: ?mixed) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve(result);
+        }
+      };
+
+      const task = {onEnd, onStart, request};
+      if (worker) {
+        this._enqueue(task, worker.getWorkerId());
+      } else {
+        this._push(task);
+      }
+    });
+  }
+
+  _getNextJob(workerId: number): ?QueueChildMessage {
+    let queueHead = this._queue[workerId];
+
+    while (queueHead && queueHead.request[1]) {
+      queueHead = queueHead.next;
+    }
+
+    this._queue[workerId] = queueHead;
+
+    return queueHead;
+  }
+
+  _process(workerId: number): Farm {
+    if (this.isLocked(workerId)) {
+      return this;
+    }
+
+    const job = this._getNextJob(workerId);
+
+    if (!job) {
+      return this;
+    }
+
+    const onEnd = (error: ?Error, result: mixed) => {
+      job.onEnd(error, result);
+      this.unlock(workerId);
+      this._process(workerId);
+    };
+
+    this.lock(workerId);
+
+    this._callback(workerId, job.request, job.onStart, onEnd);
+
+    job.request[1] = true;
+
+    return this;
+  }
+
+  _enqueue(task: QueueChildMessage, workerId: number): Farm {
+    if (task.request[1]) {
+      return this;
+    }
+
+    if (this._queue[workerId]) {
+      this._last[workerId].next = task;
+    } else {
+      this._queue[workerId] = task;
+    }
+
+    this._last[workerId] = task;
+    this._process(workerId);
+
+    return this;
+  }
+
+  _push(task: QueueChildMessage): Farm {
+    for (let i = 0; i < this._numOfWorkers; i++) {
+      const workerIdx = (this._offset + i) % this._numOfWorkers;
+      this._enqueue(task, workerIdx);
+    }
+    this._offset++;
+
+    return this;
+  }
+
+  lock(workerId: number): void {
+    this._locks[workerId] = true;
+  }
+
+  unlock(workerId: number): void {
+    this._locks[workerId] = false;
+  }
+
+  isLocked(workerId: number): boolean {
+    return this._locks[workerId];
+  }
+}

packages/jest-worker/src/WorkerPool.js

@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ */
+
+'use strict';
+
+import BaseWorkerPool from './base/BaseWorkerPool';
+
+import type {
+  ChildMessage,
+  WorkerOptions,
+  OnStart,
+  OnEnd,
+  WorkerPoolInterface,
+  WorkerInterface,
+} from './types';
+
+const canUseWorkerThreads = () => {
+  try {
+    // $FlowFixMe: Flow doesn't know about experimental APIs
+    require('worker_threads');
+    return true;
+  } catch (_) {
+    return false;
+  }
+};
+
+class WorkerPool extends BaseWorkerPool implements WorkerPoolInterface {
+  send(
+    workerId: number,
+    request: ChildMessage,
+    onStart: OnStart,
+    onEnd: OnEnd,
+  ): void {
+    this.getWorkerById(workerId).send(request, onStart, onEnd);
+  }
+
+  createWorker(workerOptions: WorkerOptions): WorkerInterface {
+    let Worker;
+    if (canUseWorkerThreads()) {
+      Worker = require('./workers/NodeThreadsWorker').default;
+    } else {
+      Worker = require('./workers/ChildProcessWorker').default;
+    }
+
+    return new Worker(workerOptions);
+  }
+}
+
+export default WorkerPool;

packages/jest-worker/src/__performance_tests__/test.js

@@ -1,4 +1,9 @@
-// Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
 
 'use strict';
 

packages/jest-worker/src/__performance_tests__/workers/jest_worker.js

@@ -1,4 +1,9 @@
-// Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
 
 'use strict';
 

packages/jest-worker/src/__performance_tests__/workers/pi.js

@@ -1,4 +1,9 @@
-// Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
 
 'use strict';
 

packages/jest-worker/src/__performance_tests__/workers/worker_farm.js

@@ -1,4 +1,9 @@
-// Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
 
 'use strict';