Merge branch 'master' into ml-links

elastic · Dec 30, 2020 · 61e7d6f · 61e7d6f
2 parents b8345a5 + 0c9a573
commit 61e7d6f
Show file tree

Hide file tree

Showing 62 changed files with 1,298 additions and 227 deletions.
diff --git a/docs/user/ml/images/ml-data-visualizer-sample.jpg b/docs/user/ml/images/ml-data-visualizer-sample.jpg
diff --git a/packages/kbn-test/src/functional_test_runner/cli.ts b/packages/kbn-test/src/functional_test_runner/cli.ts
@@ -141,22 +141,27 @@ export function runFtrCli() {
           config: 'test/functional/config.js',
         },
         help: `
-        --config=path      path to a config file
-        --bail             stop tests after the first failure
-        --grep <pattern>   pattern used to select which tests to run
-        --invert           invert grep to exclude tests
-        --include=file     a test file to be included, pass multiple times for multiple files
-        --exclude=file     a test file to be excluded, pass multiple times for multiple files
-        --include-tag=tag  a tag to be included, pass multiple times for multiple tags
-        --exclude-tag=tag  a tag to be excluded, pass multiple times for multiple tags
-        --test-stats       print the number of tests (included and excluded) to STDERR
-        --updateBaselines  replace baseline screenshots with whatever is generated from the test
-        --updateSnapshots  replace inline and file snapshots with whatever is generated from the test
-        -u                 replace both baseline screenshots and snapshots
-        --kibana-install-dir  directory where the Kibana install being tested resides
-        --throttle         enable network throttling in Chrome browser
-        --headless         run browser in headless mode
-      `,
+          --config=path      path to a config file
+          --bail             stop tests after the first failure
+          --grep <pattern>   pattern used to select which tests to run
+          --invert           invert grep to exclude tests
+          --include=file     a test file to be included, pass multiple times for multiple files
+          --exclude=file     a test file to be excluded, pass multiple times for multiple files
+          --include-tag=tag  a tag to be included, pass multiple times for multiple tags. Only
+                               suites which have one of the passed include-tag tags will be executed.
+                               When combined with the --exclude-tag flag both conditions must be met
+                               for a suite to run.
+          --exclude-tag=tag  a tag to be excluded, pass multiple times for multiple tags. Any suite
+                               which has any of the exclude-tags will be excluded. When combined with
+                               the --include-tag flag both conditions must be met for a suite to run.
+          --test-stats       print the number of tests (included and excluded) to STDERR
+          --updateBaselines  replace baseline screenshots with whatever is generated from the test
+          --updateSnapshots  replace inline and file snapshots with whatever is generated from the test
+          -u                 replace both baseline screenshots and snapshots
+          --kibana-install-dir  directory where the Kibana install being tested resides
+          --throttle         enable network throttling in Chrome browser
+          --headless         run browser in headless mode
+        `,
       },
     }
   );

diff --git a/src/core/server/logging/layouts/json_layout.ts b/src/core/server/logging/layouts/json_layout.ts
@@ -18,7 +18,7 @@
  */
 
 import moment from 'moment-timezone';
-import { merge } from 'lodash';
+import { merge } from '@kbn/std';
 import { schema } from '@kbn/config-schema';
 import { LogRecord, Layout } from '@kbn/logging';
 
@@ -53,22 +53,19 @@ export class JsonLayout implements Layout {
   }
 
   public format(record: LogRecord): string {
-    return JSON.stringify(
-      merge(
-        {
-          '@timestamp': moment(record.timestamp).format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
-          message: record.message,
-          error: JsonLayout.errorToSerializableObject(record.error),
-          log: {
-            level: record.level.id.toUpperCase(),
-            logger: record.context,
-          },
-          process: {
-            pid: record.pid,
-          },
-        },
-        record.meta
-      )
-    );
+    const log = {
+      '@timestamp': moment(record.timestamp).format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
+      message: record.message,
+      error: JsonLayout.errorToSerializableObject(record.error),
+      log: {
+        level: record.level.id.toUpperCase(),
+        logger: record.context,
+      },
+      process: {
+        pid: record.pid,
+      },
+    };
+    const output = record.meta ? merge(log, record.meta) : log;
+    return JSON.stringify(output);
   }
 }
diff --git a/vars/tasks.groovy b/vars/tasks.groovy
@@ -115,14 +115,16 @@ def functionalXpack(Map params = [:]) {
       task(kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'))
     }
 
-     whenChanged([
-       'x-pack/plugins/security_solution/',
-       'x-pack/test/security_solution_cypress/',
-       'x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/',
-       'x-pack/plugins/triggers_actions_ui/public/application/context/actions_connectors_context.tsx',
-     ]) {
-       task(kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh'))
-     }
+    whenChanged([
+      'x-pack/plugins/security_solution/',
+      'x-pack/test/security_solution_cypress/',
+      'x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/',
+      'x-pack/plugins/triggers_actions_ui/public/application/context/actions_connectors_context.tsx',
+    ]) {
+      if (githubPr.isPr()) {
+        task(kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh'))
+      }
+    }
   }
 }
 

diff --git a/x-pack/build_chromium/README.md b/x-pack/build_chromium/README.md
@@ -1,35 +1,105 @@
 # Chromium build
 
-We ship our own headless build of Chromium which is significantly smaller than the standard binaries shipped by Google. The scripts in this folder can be used to initialize the build environments and run the build on Mac, Windows, and Linux.
+We ship our own headless build of Chromium which is significantly smaller than
+the standard binaries shipped by Google. The scripts in this folder can be used
+to accept a commit hash from the Chromium repository, and initialize the build
+environments and run the build on Mac, Windows, and Linux.
 
-The official Chromium build process is poorly documented, and seems to have breaking changes fairly regularly. The build pre-requisites, and the build flags change over time, so it is likely that the scripts in this directory will be out of date by the time we have to do another Chromium build.
-
-This document is an attempt to note all of the gotchas we've come across while building, so that the next time we have to tinker here, we'll have a good starting point.
-
-# Before you begin
-You'll need access to our GCP account, which is where we have two machines provisioned for the Linux and Windows builds. Mac builds can be achieved locally, and are a great place to start to gain familiarity.
+## Before you begin
+If you wish to use a remote VM to build, you'll need access to our GCP account,
+which is where we have two machines provisioned for the Linux and Windows
+builds. Mac builds can be achieved locally, and are a great place to start to
+gain familiarity.
 
 1. Login to our GCP instance [here using your okta credentials](https://console.cloud.google.com/).
 2. Click the "Compute Engine" tab.
 3. Ensure that `chromium-build-linux` and `chromium-build-windows-12-beefy` are there.
 4. If #3 fails, you'll have to spin up new instances. Generally, these need `n1-standard-8` types or 8 vCPUs/30 GB memory.
 5. Ensure that there's enough room left on the disk. `ncdu` is a good linux util to verify what's claming space.
 
+## Usage
+
+```
+# Create a dedicated working directory for this directory of Python scripts.
+mkdir ~/chromium && cd ~/chromium
+# Copy the scripts from the Kibana repo to use them conveniently in the working directory
+cp -r ~/path/to/kibana/x-pack/build_chromium .
+# Install the OS packages, configure the environment, download the chromium source
+python ./build_chromium/init.sh [arch_name]
+
+# Run the build script with the path to the chromium src directory, the git commit id
+python ./build_chromium/build.py <commit_id>
+
+# You can add an architecture flag for ARM
+python ./build_chromium/build.py <commit_id> arm64
+```
+
+## Getting the Commit ID
+Getting `<commit_id>` can be tricky. The best technique seems to be:
+1. Create a temporary working directory and intialize yarn
+2. `yarn add puppeteer # install latest puppeter`
+3. Look through puppeteer's node module files to find the "chromium revision" (a custom versioning convention for Chromium).
+4. Use `https://crrev.com` and look up the revision and find the git commit info.
+
+The official Chromium build process is poorly documented, and seems to have
+breaking changes fairly regularly. The build pre-requisites, and the build
+flags change over time, so it is likely that the scripts in this directory will
+be out of date by the time we have to do another Chromium build.
+
+This document is an attempt to note all of the gotchas we've come across while
+building, so that the next time we have to tinker here, we'll have a good
+starting point.
+
 ## Build args
 
-Chromium is built via a build tool called "ninja". The build can be configured by specifying build flags either in an "args.gn" file or via commandline args. We have an "args.gn" file per platform:
+A good how-to on building Chromium from source is
+[here](https://chromium.googlesource.com/chromium/src/+/master/docs/get_the_code.md).
+
+There are documents for each OS that will explain how to customize arguments
+for the build using the `gn` tool. Those instructions do not apply for the
+Kibana Chromium build. Our `build.py` script ensure the correct `args.gn`
+file gets used for build arguments.
 
-- mac: darwin/args.gn
-- linux 64bit: linux-x64/args.gn
+We have an `args.gn` file per platform:
+
+- mac: `darwin/args.gn`
+- linux 64bit: `linux-x64/args.gn`
+- windows: `windows/args.gn`
 - ARM 64bit: linux-aarch64/args.gn
-- windows: windows/args.gn
 
-The various build flags are not well documented. Some are documented [here](https://www.chromium.org/developers/gn-build-configuration). Some, such as `enable_basic_printing = false`, I only found by poking through 3rd party build scripts.
+To get a list of the build arguments that are enabled, install `depot_tools` and run
+`gn args out/headless --list`. It prints out all of the flags and their
+settings, including the defaults.
+
+The various build flags are not well documented. Some are documented
+[here](https://www.chromium.org/developers/gn-build-configuration). 
 
-As of this writing, there is an officially supported headless Chromium build args file for Linux: `build/args/headless.gn`. This does not work on Windows or Mac, so we have taken that as our starting point, and modified it until the Windows / Mac builds succeeded.
+As of this writing, there is an officially supported headless Chromium build
+args file for Linux: `build/args/headless.gn`. This does not work on Windows or
+Mac, so we have taken that as our starting point, and modified it until the
+Windows / Mac builds succeeded.
 
 **NOTE:** Please, make sure you consult @elastic/kibana-security before you change, remove or add any of the build flags.
 
+## Building locally
+
+You can skip the step of running `<os_name>/init.sh` for your OS if you already
+have your environment set up, and the chromium source cloned.
+
+To get the Chromium code, refer to the [documentation](https://chromium.googlesource.com/chromium/src/+/master/docs/get_the_code.md).
+Install `depot_tools` as suggested, since it comes with useful scripts. Use the
+`fetch` command to clone the chromium repository. To set up and run the build,
+use the Kibana `build.py` script (in this directory).
+
+It's recommended that you create a working directory for the chromium source
+code and all the build tools, and run the commands from there:
+```
+mkdir ~/chromium && cd ~/chromium
+cp -r ~/path/to/kibana/x-pack/build_chromium .
+python ./build_chromium/init.sh [arch_name]
+python ./build_chromium/build.py <commit_id>
+```
+
 ## VMs
 
 I ran Linux and Windows VMs in GCP with the following specs:
@@ -57,7 +127,8 @@ The more cores the better, as the build makes effective use of each. For Linux,
 
 ## Initializing each VM / environment
 
-You only need to initialize each environment once. NOTE: on Mac OS you'll need to install XCode and accept the license agreement.
+In a VM, you'll want to use the init scripts to to initialize each environment.
+On Mac OS you'll need to install XCode and accept the license agreement.
 
 Create the build folder:
 
@@ -86,16 +157,6 @@ In windows, at least, you will need to do a number of extra steps:
 
 ## Building
 
-Find the sha of the Chromium commit you wish to build. Most likely, you want to build the Chromium revision that is tied to the version of puppeteer that we're using.
-
-Find the Chromium revision (run in kibana's working directory):
-
-- `cat node_modules/puppeteer-core/package.json | grep chromium_revision`
-- Take the revision number from that, and tack it to the end of this URL: https://crrev.com
-  - (For example, puppeteer@1.19.0 has rev (674921): https://crrev.com/674921)
-- Grab the SHA from there
-  - (For example, rev 674921 has sha 312d84c8ce62810976feda0d3457108a6dfff9e6)
-
 Note: In Linux, you should run the build command in tmux so that if your ssh session disconnects, the build can keep going. To do this, just type `tmux` into your terminal to hop into a tmux session. If you get disconnected, you can hop back in like so:
 
 - SSH into the server