Merge remote-tracking branch 'upstream/master' into kibana-grouped-nav

docs/settings/monitoring-settings.asciidoc

@@ -7,10 +7,10 @@
 
 By default, the Monitoring application is enabled, but data collection
 is disabled.  When you first start {kib} monitoring, you are prompted to
-enable data collection. If you are using {security}, you must be 
+enable data collection. If you are using {security}, you must be
 signed in as a user with the `cluster:manage` privilege to enable
 data collection. The built-in `superuser` role has this privilege and the
-built-in `elastic` user has this role. 
+built-in `elastic` user has this role.
 
 You can adjust how monitoring data is
 collected from {kib} and displayed in {kib} by configuring settings in the
@@ -134,3 +134,11 @@ For {es} clusters that are running in containers, this setting changes the
 statistics. It also adds the calculated Cgroup CPU utilization to the
 *Node Overview* page instead of the overall operating system's CPU
 utilization. Defaults to `false`.
+
+`xpack.monitoring.ui.container.logstash.enabled`::
+
+For {ls} nodes that are running in containers, this setting
+changes the {ls} *Node Listing* to display the CPU utilization
+based on the reported Cgroup statistics. It also adds the
+calculated Cgroup CPU utilization to the {ls} node detail
+pages instead of the overall operating system’s CPU utilization. Defaults to `false`.

docs/user/security/reporting.asciidoc

@@ -125,23 +125,33 @@ the {reporting} endpoints to authorized users. This requires that you:
 
 . Enable {security} on your {es} cluster. For more information,
 see {ref}/security-getting-started.html[Getting Started with Security].
-. Configure an SSL certificate for Kibana. For more information, see
-<<using-kibana-with-security>>.
-. Configure {watcher} to trust the Kibana server's certificate by adding it to
-the {watcher} truststore on each node:
-.. Import the {kib} server certificate into the {watcher} truststore using
-Java Keytool:
+. Configure TLS/SSL encryption for the {kib} server. For more information, see
+<<configuring-tls>>.
+. Specify the {kib} server's CA certificate chain in `elasticsearch.yml`:
 +
-[source,shell]
----------------------------------------------------------
-keytool -importcert -keystore watcher-truststore.jks -file server.crt
----------------------------------------------------------
-+
-NOTE: If the truststore doesn't already exist, it is created.
+--
+If you are using your own CA to sign the {kib} server certificate, then you need
+to specify the CA certificate chain in {es} to properly establish trust in TLS
+connections between {watcher} and {kib}. If your CA certificate chain is
+contained in a PKCS #12 trust store, specify it like so:
+
+[source,yaml]
+--------------------------------------------------------------------------------
+xpack.http.ssl.truststore.path: "/path/to/your/truststore.p12"
+xpack.http.ssl.truststore.type: "PKCS12"
+xpack.http.ssl.truststore.password: "optional decryption password"
+--------------------------------------------------------------------------------
+
+Otherwise, if your CA certificate chain is in PEM format, specify it like so:
+
+[source,yaml]
+--------------------------------------------------------------------------------
+xpack.http.ssl.certificate_authorities: ["/path/to/your/cacert1.pem", "/path/to/your/cacert2.pem"]
+--------------------------------------------------------------------------------
+
+For more information, see {ref}/notification-settings.html#ssl-notification-settings[the {watcher} HTTP TLS/SSL Settings].
+--
 
-.. Make sure the `xpack.http.ssl.truststore.path` setting in
-`elasticsearch.yml` specifies the location of the {watcher}
-truststore.
 . Add one or more users who have the permissions
 necessary to use {kib} and {reporting}. For more information, see
 <<secure-reporting>>.

package.json

@@ -117,8 +117,8 @@
     "@elastic/apm-rum": "^4.6.0",
     "@elastic/charts": "^16.1.0",
     "@elastic/datemath": "5.0.2",
-    "@elastic/ems-client": "1.0.5",
-    "@elastic/eui": "17.3.1",
+    "@elastic/ems-client": "7.6.0",
+    "@elastic/eui": "18.0.0",
     "@elastic/filesaver": "1.1.2",
     "@elastic/good": "8.1.1-kibana2",
     "@elastic/numeral": "2.3.3",
@@ -163,7 +163,6 @@
     "compare-versions": "3.5.1",
     "core-js": "^3.2.1",
     "css-loader": "2.1.1",
-    "custom-event-polyfill": "^0.3.0",
     "d3": "3.5.17",
     "d3-cloud": "1.2.5",
     "deep-freeze-strict": "^1.1.1",
@@ -254,6 +253,7 @@
     "rison-node": "1.0.2",
     "rxjs": "^6.5.3",
     "script-loader": "0.7.2",
+    "seedrandom": "^3.0.5",
     "semver": "^5.5.0",
     "style-it": "^2.1.3",
     "style-loader": "0.23.1",
@@ -365,7 +365,7 @@
     "@types/semver": "^5.5.0",
     "@types/sinon": "^7.0.13",
     "@types/strip-ansi": "^3.0.0",
-    "@types/styled-components": "^4.4.1",
+    "@types/styled-components": "^4.4.2",
     "@types/supertest": "^2.0.5",
     "@types/supertest-as-promised": "^2.0.38",
     "@types/testing-library__react": "^9.1.2",

packages/kbn-spec-to-console/lib/convert.js

@@ -24,10 +24,16 @@ const convertParts = require('./convert/parts');
 
 module.exports = spec => {
   const result = {};
-  // TODO:
-  // Since https://github.com/elastic/elasticsearch/pull/42346 has been merged into ES master
-  // the JSON doc specification has been updated. We need to update this script to take advantage
-  // of the added information but it will also require updating console's editor autocomplete.
+  /**
+   * TODO:
+   * Since https://github.com/elastic/elasticsearch/pull/42346 has been merged into ES master
+   * the JSON doc specification has been updated. We need to update this script to take advantage
+   * of the added information but it will also require updating console editor autocomplete.
+   *
+   * Note: for now we exclude all deprecated patterns from the generated spec to prevent them
+   * from being used in autocompletion. It would be really nice if we could use this information
+   * instead of just not including it.
+   */
   Object.keys(spec).forEach(api => {
     const source = spec[api];
     if (!source.url) {
@@ -46,8 +52,10 @@ module.exports = spec => {
     const urlComponents = {};
 
     if (source.url.paths) {
-      patterns = convertPaths(source.url.paths);
-      source.url.paths.forEach(pathsObject => {
+      // We filter out all deprecated url patterns here.
+      const paths = source.url.paths.filter(path => !path.deprecated);
+      patterns = convertPaths(paths);
+      paths.forEach(pathsObject => {
         pathsObject.methods.forEach(method => methodSet.add(method));
         if (pathsObject.parts) {
           for (const partName of Object.keys(pathsObject.parts)) {

packages/kbn-ui-shared-deps/entry.js

@@ -17,6 +17,9 @@
  * under the License.
  */
 
+// import global polyfills before everything else
+require('./polyfills');
+
 // must load before angular
 export const Jquery = require('jquery');
 window.$ = window.jQuery = Jquery;

packages/kbn-ui-shared-deps/package.json

@@ -9,12 +9,15 @@
     "kbn:watch": "node scripts/build --watch"
   },
   "devDependencies": {
-    "@elastic/eui": "17.3.1",
+    "@elastic/eui": "18.0.0",
     "@elastic/charts": "^16.1.0",
     "@kbn/dev-utils": "1.0.0",
     "@yarnpkg/lockfile": "^1.1.0",
+    "abortcontroller-polyfill": "^1.3.0",
     "angular": "^1.7.9",
+    "core-js": "^3.2.1",
     "css-loader": "^2.1.1",
+    "custom-event-polyfill": "^0.3.0",
     "del": "^5.1.0",
     "jquery": "^3.4.1",
     "mini-css-extract-plugin": "0.8.0",
@@ -24,6 +27,9 @@
     "react-intl": "^2.8.0",
     "react": "^16.12.0",
     "read-pkg": "^5.2.0",
-    "webpack": "4.41.0"
+    "regenerator-runtime": "^0.13.3",
+    "symbol-observable": "^1.2.0",
+    "webpack": "4.41.0",
+    "whatwg-fetch": "^3.0.0"
   }
 }

packages/kbn-ui-shared-deps/polyfills.js

@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+require('core-js/stable');
+require('regenerator-runtime/runtime');
+require('custom-event-polyfill');
+require('whatwg-fetch');
+require('abortcontroller-polyfill/dist/polyfill-patch-fetch');
+require('./vendor/childnode_remove_polyfill');
+require('symbol-observable');

webpackShims/childnode-remove-polyfill.js → packages/kbn-ui-shared-deps/vendor/childnode_remove_polyfill.js

@@ -1,21 +1,4 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
+/* eslint-disable @kbn/eslint/require-license-header */
 
 /* @notice
  * This product bundles childnode-remove which is available under a

src/core/server/config/deprecation/core_deprecations.ts

@@ -91,12 +91,25 @@ const cspRulesDeprecation: ConfigDeprecation = (settings, fromPath, log) => {
   return settings;
 };
 
+const mapManifestServiceUrlDeprecation: ConfigDeprecation = (settings, fromPath, log) => {
+  if (has(settings, 'map.manifestServiceUrl')) {
+    log(
+      'You should no longer use the map.manifestServiceUrl setting in kibana.yml to configure the location ' +
+        'of the Elastic Maps Service settings. These settings have moved to the "map.emsTileApiUrl" and ' +
+        '"map.emsFileApiUrl" settings instead. These settings are for development use only and should not be ' +
+        'modified for use in production environments.'
+    );
+  }
+  return settings;
+};
+
 export const coreDeprecationProvider: ConfigDeprecationProvider = ({
   unusedFromRoot,
   renameFromRoot,
 }) => [
   unusedFromRoot('savedObjects.indexCheckTimeout'),
   unusedFromRoot('server.xsrf.token'),
+  unusedFromRoot('maps.manifestServiceUrl'),
   renameFromRoot('optimize.lazy', 'optimize.watch'),
   renameFromRoot('optimize.lazyPort', 'optimize.watchPort'),
   renameFromRoot('optimize.lazyHost', 'optimize.watchHost'),
@@ -110,4 +123,5 @@ export const coreDeprecationProvider: ConfigDeprecationProvider = ({
   dataPathDeprecation,
   rewriteBasePathDeprecation,
   cspRulesDeprecation,
+  mapManifestServiceUrlDeprecation,
 ];

src/dev/build/README.md

@@ -44,7 +44,7 @@ The majority of this logic is extracted from the grunt build that has existed fo
 
 We have introduced in our bundle a webpack dll for the client vendor modules in order to improve
 the optimization time both in dev and in production. As for those modules we already have the 
-code into the vendors.bundle.dll.js we have decided to delete those bundled modules from the 
+code into the vendors_${chunk_number}.bundle.dll.js we have decided to delete those bundled modules from the 
 distributable node_modules folder. However, in order to accomplish this, we need to exclude 
 every node_module used in the server side code. This logic is performed 
 under `nodejs_modules/clean_client_modules_on_dll_task.js`. In case we need to add any new cli

src/dev/build/tasks/nodejs_modules/clean_client_modules_on_dll_task.js

@@ -98,12 +98,16 @@ export const CleanClientModulesOnDLLTask = {
     // Consider this as our whiteList for the modules we can't delete
     const whiteListedModules = [...serverDependencies, ...kbnWebpackLoaders, ...manualExceptions];
 
-    // Resolve the client vendors dll manifest path
-    const dllManifestPath = `${baseDir}/built_assets/dlls/vendors.manifest.dll.json`;
+    // Resolve the client vendors dll manifest paths
+    // excluding the runtime one
+    const dllManifestPaths = await globby([
+      `${baseDir}/built_assets/dlls/vendors_*.manifest.dll.json`,
+      `!${baseDir}/built_assets/dlls/vendors_runtime.manifest.dll.json`,
+    ]);
 
     // Get dll entries filtering out the ones
     // from any whitelisted module
-    const dllEntries = await getDllEntries(dllManifestPath, whiteListedModules, baseDir);
+    const dllEntries = await getDllEntries(dllManifestPaths, whiteListedModules, baseDir);
 
     for (const relativeEntryPath of dllEntries) {
       const entryPath = `${baseDir}/${relativeEntryPath}`;

src/dev/build/tasks/nodejs_modules/webpack_dll.js

@@ -28,27 +28,37 @@ function checkDllEntryAccess(entry, baseDir = '') {
   return isFileAccessible(resolvedPath);
 }
 
-export async function getDllEntries(manifestPath, whiteListedModules, baseDir = '') {
-  const manifest = JSON.parse(await read(manifestPath));
-
-  if (!manifest || !manifest.content) {
-    // It should fails because if we don't have the manifest file
-    // or it is malformed something wrong is happening and we
-    // should stop
-    throw new Error(`The following dll manifest doesn't exists: ${manifestPath}`);
-  }
+export async function getDllEntries(manifestPaths, whiteListedModules, baseDir = '') {
+  // Read and parse all manifests
+  const manifests = await Promise.all(
+    manifestPaths.map(async manifestPath => JSON.parse(await read(manifestPath)))
+  );
 
-  const modules = Object.keys(manifest.content);
-  if (!modules.length) {
-    // It should fails because if we don't have any
-    // module inside the client vendors dll something
-    // wrong is happening and we should stop too
-    throw new Error(`The following dll manifest is reporting an empty dll: ${manifestPath}`);
-  }
+  // Process and group modules from all manifests
+  const manifestsModules = manifests.flatMap((manifest, idx) => {
+    if (!manifest || !manifest.content) {
+      // It should fails because if we don't have the manifest file
+      // or it is malformed something wrong is happening and we
+      // should stop
+      throw new Error(`The following dll manifest doesn't exists: ${manifestPaths[idx]}`);
+    }
+
+    const modules = Object.keys(manifest.content);
+    if (!modules.length) {
+      // It should fails because if we don't have any
+      // module inside the client vendors dll something
+      // wrong is happening and we should stop too
+      throw new Error(
+        `The following dll manifest is reporting an empty dll: ${manifestPaths[idx]}`
+      );
+    }
+
+    return modules;
+  });
 
   // Only includes modules who are not in the white list of modules
   // and that are node_modules
-  return modules.filter(entry => {
+  return manifestsModules.filter(entry => {
     const isWhiteListed = whiteListedModules.some(nonEntry =>
       normalizePosixPath(entry).includes(`node_modules/${nonEntry}`)
     );

src/dev/build/tasks/nodejs_modules/webpack_dll.test.js

@@ -52,7 +52,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
 
     isFileAccessible.mockImplementation(() => true);
 
-    const mockManifestPath = '/mock/mock_dll_manifest.json';
+    const mockManifestPath = ['/mock/mock_dll_manifest.json'];
     const mockModulesWhitelist = ['dep1'];
     const dllEntries = await getDllEntries(mockManifestPath, mockModulesWhitelist);
 
@@ -66,7 +66,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
 
     isFileAccessible.mockImplementation(() => false);
 
-    const mockManifestPath = '/mock/mock_dll_manifest.json';
+    const mockManifestPath = ['/mock/mock_dll_manifest.json'];
     const mockModulesWhitelist = ['dep1'];
     const dllEntries = await getDllEntries(mockManifestPath, mockModulesWhitelist);
 
@@ -78,7 +78,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
   it('should throw an error for no manifest file', async () => {
     read.mockImplementationOnce(async () => noManifestMock);
 
-    const mockManifestPath = '/mock/mock_dll_manifest.json';
+    const mockManifestPath = ['/mock/mock_dll_manifest.json'];
 
     try {
       await getDllEntries(mockManifestPath, []);
@@ -92,7 +92,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
   it('should throw an error for no manifest content field', async () => {
     read.mockImplementation(async () => noContentFieldManifestMock);
 
-    const mockManifestPath = '/mock/mock_dll_manifest.json';
+    const mockManifestPath = ['/mock/mock_dll_manifest.json'];
 
     try {
       await getDllEntries(mockManifestPath, []);
@@ -106,7 +106,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
   it('should throw an error for manifest file without any content', async () => {
     read.mockImplementation(async () => emptyManifestContentMock);
 
-    const mockManifestPath = '/mock/mock_dll_manifest.json';
+    const mockManifestPath = ['/mock/mock_dll_manifest.json'];
 
     try {
       await getDllEntries(mockManifestPath, []);