[src/dev/build] build Kibana Platform bundles from source (elastic#73591

) Co-authored-by: spalger <spalger@users.noreply.github.com>
spalger · Aug 4, 2020 · d2829da · d2829da
1 parent be5c24d
commit d2829da
Show file tree

Hide file tree

Showing 9 changed files with 99 additions and 37 deletions.
diff --git a/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.ts b/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.ts
@@ -19,9 +19,12 @@
 
 import { REPO_ROOT } from '../repo_root';
 
-export function createAbsolutePathSerializer(rootPath: string = REPO_ROOT) {
+export function createAbsolutePathSerializer(
+  rootPath: string = REPO_ROOT,
+  replacement = '<absolute path>'
+) {
   return {
     test: (value: any) => typeof value === 'string' && value.startsWith(rootPath),
-    serialize: (value: string) => value.replace(rootPath, '<absolute path>').replace(/\\/g, '/'),
+    serialize: (value: string) => value.replace(rootPath, replacement).replace(/\\/g, '/'),
   };
 }
diff --git a/packages/kbn-optimizer/src/common/bundle_cache.ts b/packages/kbn-optimizer/src/common/bundle_cache.ts
@@ -104,4 +104,18 @@ export class BundleCache {
   public getOptimizerCacheKey() {
     return this.get().optimizerCacheKey;
   }
+
+  public clear() {
+    this.state = undefined;
+
+    if (this.path) {
+      try {
+        Fs.unlinkSync(this.path);
+      } catch (error) {
+        if (error.code !== 'ENOENT') {
+          throw error;
+        }
+      }
+    }
+  }
 }
diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts
@@ -21,7 +21,9 @@ import { createAbsolutePathSerializer } from '@kbn/dev-utils';
 
 import { getPluginBundles } from './get_plugin_bundles';
 
-expect.addSnapshotSerializer(createAbsolutePathSerializer('/repo'));
+expect.addSnapshotSerializer(createAbsolutePathSerializer('/repo', '<repoRoot>'));
+expect.addSnapshotSerializer(createAbsolutePathSerializer('/output', '<outputRoot>'));
+expect.addSnapshotSerializer(createAbsolutePathSerializer('/outside/of/repo', '<outsideOfRepo>'));
 
 it('returns a bundle for core and each plugin', () => {
   expect(
@@ -56,46 +58,47 @@ it('returns a bundle for core and each plugin', () => {
           manifestPath: '/repo/x-pack/plugins/box/kibana.json',
         },
       ],
-      '/repo'
+      '/repo',
+      '/output'
     ).map((b) => b.toSpec())
   ).toMatchInlineSnapshot(`
     Array [
       Object {
         "banner": undefined,
-        "contextDir": <absolute path>/plugins/foo,
+        "contextDir": <repoRoot>/plugins/foo,
         "id": "foo",
-        "manifestPath": <absolute path>/plugins/foo/kibana.json,
-        "outputDir": <absolute path>/plugins/foo/target/public,
+        "manifestPath": <repoRoot>/plugins/foo/kibana.json,
+        "outputDir": <outputRoot>/plugins/foo/target/public,
         "publicDirNames": Array [
           "public",
         ],
-        "sourceRoot": <absolute path>,
+        "sourceRoot": <repoRoot>,
         "type": "plugin",
       },
       Object {
         "banner": undefined,
-        "contextDir": "/outside/of/repo/plugins/baz",
+        "contextDir": <outsideOfRepo>/plugins/baz,
         "id": "baz",
-        "manifestPath": "/outside/of/repo/plugins/baz/kibana.json",
-        "outputDir": "/outside/of/repo/plugins/baz/target/public",
+        "manifestPath": <outsideOfRepo>/plugins/baz/kibana.json,
+        "outputDir": <outsideOfRepo>/plugins/baz/target/public,
         "publicDirNames": Array [
           "public",
         ],
-        "sourceRoot": <absolute path>,
+        "sourceRoot": <repoRoot>,
         "type": "plugin",
       },
       Object {
         "banner": "/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements.
      * Licensed under the Elastic License; you may not use this file except in compliance with the Elastic License. */
     ",
-        "contextDir": <absolute path>/x-pack/plugins/box,
+        "contextDir": <repoRoot>/x-pack/plugins/box,
         "id": "box",
-        "manifestPath": <absolute path>/x-pack/plugins/box/kibana.json,
-        "outputDir": <absolute path>/x-pack/plugins/box/target/public,
+        "manifestPath": <repoRoot>/x-pack/plugins/box/kibana.json,
+        "outputDir": <outputRoot>/x-pack/plugins/box/target/public,
         "publicDirNames": Array [
           "public",
         ],
-        "sourceRoot": <absolute path>,
+        "sourceRoot": <repoRoot>,
         "type": "plugin",
       },
     ]

diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts
@@ -23,7 +23,11 @@ import { Bundle } from '../common';
 
 import { KibanaPlatformPlugin } from './kibana_platform_plugins';
 
-export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: string) {
+export function getPluginBundles(
+  plugins: KibanaPlatformPlugin[],
+  repoRoot: string,
+  outputRoot: string
+) {
   const xpackDirSlash = Path.resolve(repoRoot, 'x-pack') + Path.sep;
 
   return plugins
@@ -36,7 +40,11 @@ export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: stri
           publicDirNames: ['public', ...p.extraPublicDirs],
           sourceRoot: repoRoot,
           contextDir: p.directory,
-          outputDir: Path.resolve(p.directory, 'target/public'),
+          outputDir: Path.resolve(
+            outputRoot,
+            Path.relative(repoRoot, p.directory),
+            'target/public'
+          ),
           manifestPath: p.manifestPath,
           banner: p.directory.startsWith(xpackDirSlash)
             ? `/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements.\n` +

diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts
@@ -23,16 +23,20 @@ jest.mock('./get_plugin_bundles.ts');
 jest.mock('../common/theme_tags.ts');
 jest.mock('./filter_by_id.ts');
 
-import Path from 'path';
-import Os from 'os';
+jest.mock('os', () => {
+  const realOs = jest.requireActual('os');
+  jest.spyOn(realOs, 'cpus').mockImplementation(() => {
+    return ['foo'] as any;
+  });
+  return realOs;
+});
 
+import Path from 'path';
 import { REPO_ROOT, createAbsolutePathSerializer } from '@kbn/dev-utils';
 
-import { OptimizerConfig } from './optimizer_config';
+import { OptimizerConfig, ParsedOptions } from './optimizer_config';
 import { parseThemeTags } from '../common';
 
-jest.spyOn(Os, 'cpus').mockReturnValue(['foo'] as any);
-
 expect.addSnapshotSerializer(createAbsolutePathSerializer());
 
 beforeEach(() => {
@@ -118,6 +122,7 @@ describe('OptimizerConfig::parseOptions()', () => {
         "includeCoreBundle": false,
         "inspectWorkers": false,
         "maxWorkerCount": 2,
+        "outputRoot": <absolute path>,
         "pluginPaths": Array [],
         "pluginScanDirs": Array [
           <absolute path>/src/plugins,
@@ -145,6 +150,7 @@ describe('OptimizerConfig::parseOptions()', () => {
         "includeCoreBundle": false,
         "inspectWorkers": false,
         "maxWorkerCount": 2,
+        "outputRoot": <absolute path>,
         "pluginPaths": Array [],
         "pluginScanDirs": Array [
           <absolute path>/src/plugins,
@@ -172,6 +178,7 @@ describe('OptimizerConfig::parseOptions()', () => {
         "includeCoreBundle": false,
         "inspectWorkers": false,
         "maxWorkerCount": 2,
+        "outputRoot": <absolute path>,
         "pluginPaths": Array [],
         "pluginScanDirs": Array [
           <absolute path>/src/plugins,
@@ -201,6 +208,7 @@ describe('OptimizerConfig::parseOptions()', () => {
         "includeCoreBundle": false,
         "inspectWorkers": false,
         "maxWorkerCount": 2,
+        "outputRoot": <absolute path>,
         "pluginPaths": Array [],
         "pluginScanDirs": Array [
           <absolute path>/src/plugins,
@@ -227,6 +235,7 @@ describe('OptimizerConfig::parseOptions()', () => {
         "includeCoreBundle": false,
         "inspectWorkers": false,
         "maxWorkerCount": 2,
+        "outputRoot": <absolute path>,
         "pluginPaths": Array [],
         "pluginScanDirs": Array [
           <absolute path>/x/y/z,
@@ -253,6 +262,7 @@ describe('OptimizerConfig::parseOptions()', () => {
         "includeCoreBundle": false,
         "inspectWorkers": false,
         "maxWorkerCount": 100,
+        "outputRoot": <absolute path>,
         "pluginPaths": Array [],
         "pluginScanDirs": Array [],
         "profileWebpack": false,
@@ -276,6 +286,7 @@ describe('OptimizerConfig::parseOptions()', () => {
         "includeCoreBundle": false,
         "inspectWorkers": false,
         "maxWorkerCount": 100,
+        "outputRoot": <absolute path>,
         "pluginPaths": Array [],
         "pluginScanDirs": Array [],
         "profileWebpack": false,
@@ -299,6 +310,7 @@ describe('OptimizerConfig::parseOptions()', () => {
         "includeCoreBundle": false,
         "inspectWorkers": false,
         "maxWorkerCount": 100,
+        "outputRoot": <absolute path>,
         "pluginPaths": Array [],
         "pluginScanDirs": Array [],
         "profileWebpack": false,
@@ -323,6 +335,7 @@ describe('OptimizerConfig::parseOptions()', () => {
         "includeCoreBundle": false,
         "inspectWorkers": false,
         "maxWorkerCount": 100,
+        "outputRoot": <absolute path>,
         "pluginPaths": Array [],
         "pluginScanDirs": Array [],
         "profileWebpack": false,
@@ -347,6 +360,7 @@ describe('OptimizerConfig::parseOptions()', () => {
         "includeCoreBundle": false,
         "inspectWorkers": false,
         "maxWorkerCount": 100,
+        "outputRoot": <absolute path>,
         "pluginPaths": Array [],
         "pluginScanDirs": Array [],
         "profileWebpack": false,
@@ -384,18 +398,22 @@ describe('OptimizerConfig::create()', () => {
     getPluginBundles.mockReturnValue([Symbol('bundle1'), Symbol('bundle2')]);
     filterById.mockReturnValue(Symbol('filtered bundles'));
 
-    jest.spyOn(OptimizerConfig, 'parseOptions').mockImplementation((): any => ({
+    jest.spyOn(OptimizerConfig, 'parseOptions').mockImplementation((): {
+      [key in keyof ParsedOptions]: any;
+    } => ({
       cache: Symbol('parsed cache'),
       dist: Symbol('parsed dist'),
       maxWorkerCount: Symbol('parsed max worker count'),
       pluginPaths: Symbol('parsed plugin paths'),
       pluginScanDirs: Symbol('parsed plugin scan dirs'),
       repoRoot: Symbol('parsed repo root'),
+      outputRoot: Symbol('parsed output root'),
       watch: Symbol('parsed watch'),
       themeTags: Symbol('theme tags'),
       inspectWorkers: Symbol('parsed inspect workers'),
       profileWebpack: Symbol('parsed profile webpack'),
       filters: [],
+      includeCoreBundle: false,
     }));
   });
 
@@ -474,6 +492,7 @@ describe('OptimizerConfig::create()', () => {
           Array [
             Symbol(new platform plugins),
             Symbol(parsed repo root),
+            Symbol(parsed output root),
           ],
         ],
         "instances": Array [

diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts
@@ -55,6 +55,13 @@ function omit<T, K extends keyof T>(obj: T, keys: K[]): Omit<T, K> {
 interface Options {
   /** absolute path to root of the repo/build */
   repoRoot: string;
+  /**
+   * absolute path to the root directory where output should be written to. This
+   * defaults to the repoRoot but can be customized to write output somewhere else.
+   *
+   * This is how we write output to the build directory in the Kibana build tasks.
+   */
+  outputRoot?: string;
   /** enable to run the optimizer in watch mode */
   watch?: boolean;
   /** the maximum number of workers that will be created */
@@ -107,8 +114,9 @@ interface Options {
   themes?: ThemeTag | '*' | ThemeTag[];
 }
 
-interface ParsedOptions {
+export interface ParsedOptions {
   repoRoot: string;
+  outputRoot: string;
   watch: boolean;
   maxWorkerCount: number;
   profileWebpack: boolean;
@@ -139,6 +147,11 @@ export class OptimizerConfig {
       throw new TypeError('repoRoot must be an absolute path');
     }
 
+    const outputRoot = options.outputRoot ?? repoRoot;
+    if (!Path.isAbsolute(outputRoot)) {
+      throw new TypeError('outputRoot must be an absolute path');
+    }
+
     /**
      * BEWARE: this needs to stay roughly synchronized with
      * `src/core/server/config/env.ts` which determines which paths
@@ -182,6 +195,7 @@ export class OptimizerConfig {
       watch,
       dist,
       repoRoot,
+      outputRoot,
       maxWorkerCount,
       profileWebpack,
       cache,
@@ -206,11 +220,11 @@ export class OptimizerConfig {
               publicDirNames: ['public', 'public/utils'],
               sourceRoot: options.repoRoot,
               contextDir: Path.resolve(options.repoRoot, 'src/core'),
-              outputDir: Path.resolve(options.repoRoot, 'src/core/target/public'),
+              outputDir: Path.resolve(options.outputRoot, 'src/core/target/public'),
             }),
           ]
         : []),
-      ...getPluginBundles(plugins, options.repoRoot),
+      ...getPluginBundles(plugins, options.repoRoot, options.outputRoot),
     ];
 
     return new OptimizerConfig(

diff --git a/src/dev/build/tasks/build_kibana_platform_plugins.ts b/src/dev/build/tasks/build_kibana_platform_plugins.ts
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { CiStatsReporter } from '@kbn/dev-utils';
+import { CiStatsReporter, REPO_ROOT } from '@kbn/dev-utils';
 import {
   runOptimizer,
   OptimizerConfig,
@@ -29,9 +29,10 @@ import { Task } from '../lib';
 
 export const BuildKibanaPlatformPlugins: Task = {
   description: 'Building distributable versions of Kibana platform plugins',
-  async run(config, log, build) {
-    const optimizerConfig = OptimizerConfig.create({
-      repoRoot: build.resolvePath(),
+  async run(_, log, build) {
+    const config = OptimizerConfig.create({
+      repoRoot: REPO_ROOT,
+      outputRoot: build.resolvePath(),
       cache: false,
       oss: build.isOss(),
       examples: false,
@@ -42,11 +43,10 @@ export const BuildKibanaPlatformPlugins: Task = {
 
     const reporter = CiStatsReporter.fromEnv(log);
 
-    await runOptimizer(optimizerConfig)
-      .pipe(
-        reportOptimizerStats(reporter, optimizerConfig, log),
-        logOptimizerState(log, optimizerConfig)
-      )
+    await runOptimizer(config)
+      .pipe(reportOptimizerStats(reporter, config, log), logOptimizerState(log, config))
       .toPromise();
+
+    await Promise.all(config.bundles.map((b) => b.cache.clear()));
   },
 };
diff --git a/src/dev/build/tasks/copy_source_task.ts b/src/dev/build/tasks/copy_source_task.ts
@@ -30,7 +30,7 @@ export const CopySource: Task = {
         'src/**',
         '!src/**/*.{test,test.mocks,mock}.{js,ts,tsx}',
         '!src/**/mocks.ts', // special file who imports .mock files
-        '!src/**/{__tests__,__snapshots__,__mocks__}/**',
+        '!src/**/{target,__tests__,__snapshots__,__mocks__}/**',
         '!src/test_utils/**',
         '!src/fixtures/**',
         '!src/legacy/core_plugins/console/public/tests/**',

diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy
@@ -108,6 +108,7 @@ def uploadGcsArtifact(uploadPrefix, pattern) {
 def withGcsArtifactUpload(workerName, closure) {
   def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}"
   def ARTIFACT_PATTERNS = [
+    '**/target/public/.kbn-optimizer-cache',
     'target/kibana-*',
     'target/kibana-security-solution/**/*.png',
     'target/junit/**/*',