[kbn/optimizer] fix windows compatibility (elastic#69304)

Co-authored-by: spalger <spalger@users.noreply.github.com>
spalger · Jun 16, 2020 · db1df7b · db1df7b
1 parent efbb4cc
commit db1df7b
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 26 deletions.
diff --git a/packages/kbn-optimizer/src/optimizer/observe_worker.ts b/packages/kbn-optimizer/src/optimizer/observe_worker.ts
@@ -22,7 +22,7 @@ import { inspect } from 'util';
 
 import execa from 'execa';
 import * as Rx from 'rxjs';
-import { map, takeUntil } from 'rxjs/operators';
+import { map, takeUntil, first, ignoreElements } from 'rxjs/operators';
 
 import { isWorkerMsg, WorkerConfig, WorkerMsg, Bundle, BundleRefs } from '../common';
 
@@ -68,19 +68,11 @@ if (inspectFlagIndex !== -1) {
 
 function usingWorkerProc<T>(
   config: OptimizerConfig,
-  workerConfig: WorkerConfig,
-  bundles: Bundle[],
   fn: (proc: execa.ExecaChildProcess) => Rx.Observable<T>
 ) {
   return Rx.using(
     (): ProcResource => {
-      const args = [
-        JSON.stringify(workerConfig),
-        JSON.stringify(bundles.map((b) => b.toSpec())),
-        BundleRefs.fromBundles(config.bundles).toSpecJson(),
-      ];
-
-      const proc = execa.node(require.resolve('../worker/run_worker'), args, {
+      const proc = execa.node(require.resolve('../worker/run_worker'), [], {
         nodeOptions: [
           ...(inspectFlag && config.inspectWorkers
             ? [`${inspectFlag}=${inspectPortCounter++}`]
@@ -129,6 +121,51 @@ function observeStdio$(stream: Readable, name: WorkerStdio['stream']) {
   );
 }
 
+/**
+ * We used to pass configuration to the worker as JSON encoded arguments, but they
+ * grew too large for argv, especially on Windows, so we had to move to an async init
+ * where we send the args over IPC. To keep the logic simple we basically mock the
+ * argv behavior and don't use complicated messages or anything so that state can
+ * be initialized in the worker before most of the code is run.
+ */
+function initWorker(
+  proc: execa.ExecaChildProcess,
+  config: OptimizerConfig,
+  workerConfig: WorkerConfig,
+  bundles: Bundle[]
+) {
+  const msg$ = Rx.fromEvent<[unknown]>(proc, 'message').pipe(
+    // validate the initialization messages from the process
+    map(([msg]) => {
+      if (typeof msg === 'string') {
+        switch (msg) {
+          case 'init':
+            return 'init' as const;
+          case 'ready':
+            return 'ready' as const;
+        }
+      }
+
+      throw new Error(`unexpected message from worker while initializing: [${inspect(msg)}]`);
+    })
+  );
+
+  return Rx.concat(
+    msg$.pipe(first((msg) => msg === 'init')),
+    Rx.defer(() => {
+      proc.send({
+        args: [
+          JSON.stringify(workerConfig),
+          JSON.stringify(bundles.map((b) => b.toSpec())),
+          BundleRefs.fromBundles(config.bundles).toSpecJson(),
+        ],
+      });
+      return [];
+    }),
+    msg$.pipe(first((msg) => msg === 'ready'))
+  ).pipe(ignoreElements());
+}
+
 /**
  * Start a worker process with the specified `workerConfig` and
  * `bundles` and return an observable of the events related to
@@ -140,10 +177,11 @@ export function observeWorker(
   workerConfig: WorkerConfig,
   bundles: Bundle[]
 ): Rx.Observable<WorkerMsg | WorkerStatus> {
-  return usingWorkerProc(config, workerConfig, bundles, (proc) => {
-    let lastMsg: WorkerMsg;
+  return usingWorkerProc(config, (proc) => {
+    const init$ = initWorker(proc, config, workerConfig, bundles);
 
-    return Rx.merge(
+    let lastMsg: WorkerMsg;
+    const worker$: Rx.Observable<WorkerMsg | WorkerStatus> = Rx.merge(
       Rx.of({
         type: 'worker started',
         bundles,
@@ -201,5 +239,7 @@ export function observeWorker(
           )
         )
     );
+
+    return Rx.concat(init$, worker$);
   });
 }
diff --git a/packages/kbn-optimizer/src/worker/entry_point_creator.ts b/packages/kbn-optimizer/src/worker/entry_point_creator.ts
@@ -17,9 +17,13 @@
  * under the License.
  */
 
-module.exports = function ({ entries }: { entries: Array<{ importId: string; relPath: string }> }) {
-  const lines = entries.map(({ importId, relPath }) => [
-    `__kbnBundles__.define('${importId}', __webpack_require__, require.resolve('./${relPath}'))`,
+module.exports = function ({
+  entries,
+}: {
+  entries: Array<{ importId: string; requirePath: string }>;
+}) {
+  const lines = entries.map(({ importId, requirePath }) => [
+    `__kbnBundles__.define('${importId}', __webpack_require__, require.resolve('${requirePath}'))`,
   ]);
 
   return {

diff --git a/packages/kbn-optimizer/src/worker/run_worker.ts b/packages/kbn-optimizer/src/worker/run_worker.ts
@@ -17,7 +17,10 @@
  * under the License.
  */
 
+import { inspect } from 'util';
+
 import * as Rx from 'rxjs';
+import { take, mergeMap } from 'rxjs/operators';
 
 import {
   parseBundles,
@@ -80,15 +83,38 @@ setInterval(() => {
   }
 }, 1000).unref();
 
+function assertInitMsg(msg: unknown): asserts msg is { args: string[] } {
+  if (typeof msg !== 'object' || !msg) {
+    throw new Error(`expected init message to be an object: ${inspect(msg)}`);
+  }
+
+  const { args } = msg as Record<string, unknown>;
+  if (!args || !Array.isArray(args) || !args.every((a) => typeof a === 'string')) {
+    throw new Error(
+      `expected init message to have an 'args' property that's an array of strings: ${inspect(msg)}`
+    );
+  }
+}
+
 Rx.defer(() => {
-  const workerConfig = parseWorkerConfig(process.argv[2]);
-  const bundles = parseBundles(process.argv[3]);
-  const bundleRefs = BundleRefs.parseSpec(process.argv[4]);
+  process.send!('init');
+
+  return Rx.fromEvent<[unknown]>(process as any, 'message').pipe(
+    take(1),
+    mergeMap(([msg]) => {
+      assertInitMsg(msg);
+      process.send!('ready');
+
+      const workerConfig = parseWorkerConfig(msg.args[0]);
+      const bundles = parseBundles(msg.args[1]);
+      const bundleRefs = BundleRefs.parseSpec(msg.args[2]);
 
-  // set BROWSERSLIST_ENV so that style/babel loaders see it before running compilers
-  process.env.BROWSERSLIST_ENV = workerConfig.browserslistEnv;
+      // set BROWSERSLIST_ENV so that style/babel loaders see it before running compilers
+      process.env.BROWSERSLIST_ENV = workerConfig.browserslistEnv;
 
-  return runCompilers(workerConfig, bundles, bundleRefs);
+      return runCompilers(workerConfig, bundles, bundleRefs);
+    })
+  );
 }).subscribe(
   (msg) => {
     send(msg);

diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts
@@ -100,10 +100,16 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker:
                 entries: bundle.publicDirNames.map((name) => {
                   const absolute = Path.resolve(bundle.contextDir, name);
                   const newContext = Path.dirname(ENTRY_CREATOR);
-                  return {
-                    importId: `${bundle.type}/${bundle.id}/${name}`,
-                    relPath: Path.relative(newContext, absolute),
-                  };
+                  const importId = `${bundle.type}/${bundle.id}/${name}`;
+
+                  // relative path from context of the ENTRY_CREATOR, with linux path separators
+                  let requirePath = Path.relative(newContext, absolute).split('\\').join('/');
+                  if (!requirePath.startsWith('.')) {
+                    // ensure requirePath is identified by node as relative
+                    requirePath = `./${requirePath}`;
+                  }
+
+                  return { importId, requirePath };
                 }),
               },
             },