Merge branch 'master' into endpoint_telemetry_cloned_vms

.ci/Dockerfile

@@ -1,7 +1,7 @@
 # NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
 # If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
 
-ARG NODE_VERSION=10.22.1
+ARG NODE_VERSION=12.19.1
 
 FROM node:${NODE_VERSION} AS base
 

.ci/build_docker.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euo pipefail
+
+cd "$(dirname "${0}")"
+
+cp /usr/local/bin/runbld ./
+cp /usr/local/bin/bash_standard_lib.sh ./
+
+if which docker >/dev/null; then
+    docker build -t kibana-ci -f ./Dockerfile .
+else
+    echo "Docker binary is not available. Skipping the docker build this time."
+fi

.ci/es-snapshots/Jenkinsfile_verify_es

@@ -55,6 +55,7 @@ kibanaPipeline(timeoutMinutes: 150) {
             'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
             'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
             'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
+            'xpack-ciGroup11': kibanaPipeline.xpackCiGroupProcess(11),
           ]),
         ])
       }

.ci/jobs.yml

@@ -31,6 +31,7 @@ JOB:
   - x-pack-ciGroup8
   - x-pack-ciGroup9
   - x-pack-ciGroup10
+  - x-pack-ciGroup11
   - x-pack-accessibility
   - x-pack-visualRegression
 

.ci/packer_cache_for_branch.sh

@@ -18,7 +18,7 @@ node scripts/es snapshot --download-only;
 node scripts/es snapshot --license=oss --download-only;
 
 # download reporting browsers
-(cd "x-pack" && yarn gulp downloadChromium);
+(cd "x-pack" && node ../node_modules/.bin/gulp downloadChromium);
 
 # cache the chromedriver archive
 chromedriverDistVersion="$(node -e "console.log(require('chromedriver').version)")"
@@ -49,14 +49,13 @@ tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \
   .chromium \
   .es \
   .chromedriver \
-  .geckodriver;
-
-echo "Adding node_modules"
-# Find all of the node_modules directories that aren't test fixtures, and aren't inside other node_modules directories, and append them to the tar
-find . -type d -name node_modules -not -path '*__fixtures__*' -prune -print0 | xargs -0I % tar -rf "$HOME/.kibana/bootstrap_cache/$branch.tar" "%"
+  .geckodriver \
+  .yarn-local-mirror;
 
 echo "created $HOME/.kibana/bootstrap_cache/$branch.tar"
 
+.ci/build_docker.sh
+
 if [[ "$branch" != "master" ]]; then
   rm --preserve-root -rf "$checkoutDir"
 fi

.ci/teamcity/bootstrap.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/util.sh"
+
+tc_start_block "Bootstrap"
+
+tc_start_block "yarn install and kbn bootstrap"
+verify_no_git_changes yarn kbn bootstrap --prefer-offline
+tc_end_block "yarn install and kbn bootstrap"
+
+tc_start_block "build kbn-pm"
+verify_no_git_changes yarn kbn run build -i @kbn/pm
+tc_end_block "build kbn-pm"
+
+tc_start_block "build plugin list docs"
+verify_no_git_changes node scripts/build_plugin_list_docs
+tc_end_block "build plugin list docs"
+
+tc_end_block "Bootstrap"

.ci/teamcity/checks/bundle_limits.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+node scripts/build_kibana_platform_plugins --validate-limits

.ci/teamcity/checks/doc_api_changes.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:checkDocApiChanges

.ci/teamcity/checks/file_casing.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:checkFileCasing

.ci/teamcity/checks/i18n.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:i18nCheck

.ci/teamcity/checks/licenses.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:licenses

.ci/teamcity/checks/telemetry.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:telemetryCheck

.ci/teamcity/checks/test_hardening.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:test_hardening

.ci/teamcity/checks/ts_projects.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:checkTsProjects

.ci/teamcity/checks/type_check.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:typeCheck

.ci/teamcity/checks/verify_dependency_versions.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:verifyDependencyVersions

.ci/teamcity/checks/verify_notice.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:verifyNotice

.ci/teamcity/ci_stats.js

@@ -0,0 +1,59 @@
+const https = require('https');
+const token = process.env.CI_STATS_TOKEN;
+const host = process.env.CI_STATS_HOST;
+
+const request = (url, options, data = null) => {
+  const httpOptions = {
+    ...options,
+    headers: {
+      ...(options.headers || {}),
+      Authorization: `token ${token}`,
+    },
+  };
+
+  return new Promise((resolve, reject) => {
+    console.log(`Calling https://${host}${url}`);
+
+    const req = https.request(`https://${host}${url}`, httpOptions, (res) => {
+      if (res.statusCode < 200 || res.statusCode >= 300) {
+        return reject(new Error(`Status Code: ${res.statusCode}`));
+      }
+
+      const data = [];
+      res.on('data', (d) => {
+        data.push(d);
+      })
+
+      res.on('end', () => {
+        try {
+          let resp = Buffer.concat(data).toString();
+
+          try {
+            if (resp.trim()) {
+              resp = JSON.parse(resp);
+            }
+          } catch (ex) {
+            console.error(ex);
+          }
+
+          resolve(resp);
+        } catch (ex) {
+          reject(ex);
+        }
+      });
+    })
+
+    req.on('error', reject);
+
+    if (data) {
+      req.write(JSON.stringify(data));
+    }
+
+    req.end();
+  });
+}
+
+module.exports = {
+  get: (url) => request(url, { method: 'GET' }),
+  post: (url, data) => request(url, { method: 'POST' }, data),
+}

.ci/teamcity/ci_stats_complete.js

@@ -0,0 +1,18 @@
+const ciStats = require('./ci_stats');
+
+// This might be better as an API call in the future.
+// Instead, it relies on a separate step setting the BUILD_STATUS env var. BUILD_STATUS is not something provided by TeamCity.
+const BUILD_STATUS = process.env.BUILD_STATUS === 'SUCCESS' ? 'SUCCESS' : 'FAILURE';
+
+(async () => {
+  try {
+    if (process.env.CI_STATS_BUILD_ID) {
+      await ciStats.post(`/v1/build/_complete?id=${process.env.CI_STATS_BUILD_ID}`, {
+        result: BUILD_STATUS,
+      });
+    }
+  } catch (ex) {
+    console.error(ex);
+    process.exit(1);
+  }
+})();

.ci/teamcity/default/accessibility.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-accessibility
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "X-Pack accessibility tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --config test/accessibility/config.ts

.ci/teamcity/default/build.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+tc_start_block "Build Platform Plugins"
+node scripts/build_kibana_platform_plugins \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
+  --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
+  --verbose
+tc_end_block "Build Platform Plugins"
+
+export KBN_NP_PLUGINS_BUILT=true
+
+tc_start_block "Build Default Distribution"
+
+cd "$KIBANA_DIR"
+node scripts/build --debug --no-oss
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+installDir="$KIBANA_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+tc_end_block "Build Default Distribution"

.ci/teamcity/default/build_plugins.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+tc_start_block "Build Platform Plugins"
+node scripts/build_kibana_platform_plugins \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
+  --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
+  --verbose
+tc_end_block "Build Platform Plugins"
+
+tc_set_env KBN_NP_PLUGINS_BUILT true

.ci/teamcity/default/ci_group.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export CI_GROUP="$1"
+export JOB=kibana-default-ciGroup${CI_GROUP}
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Default Distro Chrome Functional tests / Group ${CI_GROUP}" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --include-tag "ciGroup$CI_GROUP"

.ci/teamcity/default/firefox.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-firefoxSmoke
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "X-Pack firefox smoke test" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --include-tag "includeFirefox" \
+    --config test/functional/config.firefox.js \
+    --config test/functional_embedded/config.firefox.ts

.ci/teamcity/default/saved_object_field_metrics.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-savedObjectFieldMetrics
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Capture Kibana Saved Objects field count metrics" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --config test/saved_objects_field_count/config.ts

.ci/teamcity/default/security_solution.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-securitySolution
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Security Solution Cypress Tests" \
+ node scripts/functional_tests \
+   --debug --bail \
+   --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+   --config test/security_solution_cypress/cli_config.ts