Merge branch '7.17' of github.com:elastic/kibana into issue-126508-op…

…timization-for-metric-threshold-rule-7-17

.buildkite/pipelines/bazel_cache.yml

@@ -1,5 +1,7 @@
 steps:
   - label: ':pipeline: Create pipeline with priority'
+    agents:
+      queue: kibana-default
     concurrency_group: bazel_macos
     concurrency: 1
     concurrency_method: eager

.buildkite/pipelines/es_snapshots/promote.yml

@@ -10,3 +10,5 @@ steps:
         required: true
   - label: Promote Snapshot
     command: .buildkite/scripts/steps/es_snapshots/promote.sh
+    agents:
+      queue: kibana-default

.buildkite/pipelines/es_snapshots/verify.yml

@@ -14,6 +14,8 @@ steps:
   - command: .buildkite/scripts/lifecycle/pre_build.sh
     label: Pre-Build
     timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
 
   - wait
 
@@ -85,6 +87,8 @@ steps:
   - command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh
     label: Trigger promotion
     timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
     depends_on:
       - default-cigroup
       - default-cigroup-docker
@@ -98,3 +102,5 @@ steps:
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build
     timeout_in_minutes: 10
+    agents:
+      queue: kibana-default

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -51,6 +51,9 @@ const pipeline = {
     {
       command: '.buildkite/pipelines/flaky_tests/runner.sh',
       label: 'Create pipeline',
+      agents: {
+        queue: 'kibana-default',
+      },
     },
   ],
 };

.buildkite/pipelines/hourly.yml

@@ -2,6 +2,8 @@ steps:
   - command: .buildkite/scripts/lifecycle/pre_build.sh
     label: Pre-Build
     timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
 
   - wait
 
@@ -174,3 +176,5 @@ steps:
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build
     timeout_in_minutes: 10
+    agents:
+      queue: kibana-default

.buildkite/pipelines/on_merge.yml

@@ -5,6 +5,8 @@ steps:
   - command: .buildkite/scripts/lifecycle/pre_build.sh
     label: Pre-Build
     timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
 
   - wait
 
@@ -34,3 +36,5 @@ steps:
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build
     timeout_in_minutes: 10
+    agents:
+      queue: kibana-default

.buildkite/pipelines/performance/daily.yml

@@ -1,25 +1,27 @@
 steps:
-  - block: ":gear: Performance Tests Configuration"
-    prompt: "Fill out the details for performance test"
+  - block: ':gear: Performance Tests Configuration'
+    prompt: 'Fill out the details for performance test'
     fields:
-      - text: ":arrows_counterclockwise: Iterations"
-        key: "performance-test-iteration-count"
-        hint: "How many times you want to run tests? "
+      - text: ':arrows_counterclockwise: Iterations'
+        key: 'performance-test-iteration-count'
+        hint: 'How many times you want to run tests? '
         required: true
     if: build.env('PERF_TEST_COUNT') == null
 
-  - label: ":male-mechanic::skin-tone-2: Pre-Build"
+  - label: ':male-mechanic::skin-tone-2: Pre-Build'
     command: .buildkite/scripts/lifecycle/pre_build.sh
+    agents:
+      queue: kibana-default
 
   - wait
 
-  - label: ":factory_worker: Build Kibana Distribution and Plugins"
+  - label: ':factory_worker: Build Kibana Distribution and Plugins'
     command: .buildkite/scripts/steps/build_kibana.sh
     agents:
       queue: c2-16
     key: build
 
-  - label: ":muscle: Performance Tests with Playwright config"
+  - label: ':muscle: Performance Tests with Playwright config'
     command: .buildkite/scripts/steps/functional/performance_playwright.sh
     agents:
       queue: c2-16
@@ -28,6 +30,7 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - label: ":male_superhero::skin-tone-2: Post-Build"
+  - label: ':male_superhero::skin-tone-2: Post-Build'
     command: .buildkite/scripts/lifecycle/post_build.sh
-
+    agents:
+      queue: kibana-default

.buildkite/pipelines/pull_request/base.yml

@@ -2,6 +2,8 @@ steps:
   - command: .buildkite/scripts/lifecycle/pre_build.sh
     label: Pre-Build
     timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
 
   - wait
 

.buildkite/pipelines/pull_request/post_build.yml

@@ -4,3 +4,5 @@ steps:
 
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build
+    agents:
+      queue: kibana-default

.buildkite/pipelines/update_demo_env.yml

@@ -2,6 +2,8 @@ steps:
   - command: .buildkite/scripts/steps/demo_env/es_and_init.sh
     label: Initialize Environment and Deploy ES
     timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
 
   - command: .buildkite/scripts/steps/demo_env/kibana.sh
     label: Build and Deploy Kibana

.buildkite/scripts/common/env.sh

@@ -95,12 +95,3 @@ fi
 
 export BUILD_TS_REFS_DISABLE=true
 export DISABLE_BOOTSTRAP_VALIDATION=true
-
-export TEST_KIBANA_HOST=localhost
-export TEST_KIBANA_PORT=6101
-export TEST_KIBANA_URL="http://elastic:changeme@localhost:6101"
-export TEST_ES_URL="http://elastic:changeme@localhost:6102"
-export TEST_ES_TRANSPORT_PORT=6301-6309
-export TEST_CORS_SERVER_PORT=6106
-export ALERTING_PROXY_PORT=6105
-export TEST_PROXY_SERVER_PORT=6107

docs/developer/advanced/development-es-snapshots.asciidoc

@@ -13,6 +13,7 @@ https://ci.kibana.dev/es-snapshots[A dashboard] is available that shows the curr
 2. Each snapshot is uploaded to a public Google Cloud Storage bucket, `kibana-ci-es-snapshots-daily`.
 ** At this point, the snapshot is not automatically used in CI or local development. It needs to be tested/verified first.
 3. Each snapshot is tested with the latest commit of the corresponding {kib} branch, using the full CI suite.
+3a. If a test fails during snapshot verification the Kibana Operations team will skip it and create an issue for the team to fix the test, or work with the Elasticsearch team to get a fix implemented there. Once the fix is ready a Kibana PR can be opened to unskip the test.
 4. After CI
 ** If the snapshot passes, it is promoted and automatically used in CI and local development.
 ** If the snapshot fails, the issue must be investigated and resolved. A new incompatibility may exist between {es} and {kib}.

docs/osquery/osquery.asciidoc

@@ -273,12 +273,18 @@ for an agent policy through Fleet.
 This integration supports x64 architecture on Windows, MacOS, and Linux platforms, 
 and ARM64 architecture on Linux.
 
-NOTE: The original {filebeat-ref}/filebeat-module-osquery.html[Filebeat Osquery module]
+[NOTE]
+=========================
+
+* The original {filebeat-ref}/filebeat-module-osquery.html[Filebeat Osquery module]
 and the https://docs.elastic.co/en/integrations/osquery[Osquery]
 integration collect logs from self-managed Osquery deployments.
 The *Osquery Manager* integration manages Osquery deployments
 and supports running and scheduling queries from {kib}.
 
+* *Osquery Manager* cannot be integrated with an Elastic Agent in standalone mode.
+=========================
+
 [float]
 === Customize Osquery sub-feature privileges
 

docs/setup/configuring-reporting.asciidoc

@@ -30,6 +30,7 @@ If you are using Ubuntu/Debian systems, install the following packages:
 
 * `fonts-liberation`
 * `libfontconfig1`
+* `libnss3`
 
 If the system is missing dependencies, *Reporting* fails in a non-deterministic way. {kib} runs a self-test at server startup, and
 if it encounters errors, logs them in the Console. The error message does not include

docs/user/security/authentication/index.asciidoc

@@ -5,7 +5,7 @@
 <titleabbrev>Authentication</titleabbrev>
 ++++
 :keywords: administrator, concept, security, authentication
-:description: A list of the supported authentication mechanisms in {kib}. 
+:description: A list of the supported authentication mechanisms in {kib}.
 
 {kib} supports the following authentication mechanisms:
 
@@ -483,4 +483,4 @@ To make this iframe leverage anonymous access automatically, you will need to mo
 
 NOTE: `auth_provider_hint` query string parameter goes *before* the hash URL fragment.
 
-For more information on how to embed, refer to <<embedding, Embed {kib} content in a web page>>.
+For more information, refer to <<embed-code, Embed code>>.

packages/kbn-es/src/index.ts

@@ -10,3 +10,4 @@
 export { run } from './cli';
 // @ts-expect-error not typed yet
 export { Cluster } from './cluster';
+export { SYSTEM_INDICES_SUPERUSER } from './utils';

packages/kbn-es/src/integration_tests/__fixtures__/es_bin.js

@@ -69,6 +69,16 @@ const { ES_KEY_PATH, ES_CERT_PATH } = require('@kbn/dev-utils');
         });
       }
 
+      if (url.pathname === '/_cluster/health') {
+        return send(
+          200,
+          {
+            status: 'green',
+          },
+          { 'x-elastic-product': 'Elasticsearch' }
+        );
+      }
+
       return send(404, {
         error: {
           reason: 'not found',

packages/kbn-es/src/utils/index.ts

@@ -14,6 +14,6 @@ export { findMostRecentlyChanged } from './find_most_recently_changed';
 // @ts-expect-error not typed yet
 export { extractConfigFiles } from './extract_config_files';
 // @ts-expect-error not typed yet
-export { NativeRealm } from './native_realm';
+export { NativeRealm, SYSTEM_INDICES_SUPERUSER } from './native_realm';
 export { buildSnapshot } from './build_snapshot';
 export { archiveForPlatform } from './build_snapshot';

packages/kbn-es/src/utils/native_realm.js

@@ -11,6 +11,9 @@ const chalk = require('chalk');
 
 const { log: defaultLog } = require('./log');
 
+export const SYSTEM_INDICES_SUPERUSER =
+  process.env.TEST_ES_SYSTEM_INDICES_USER || 'system_indices_superuser';
+
 exports.NativeRealm = class NativeRealm {
   constructor({ elasticPassword, port, log = defaultLog, ssl = false, caCert }) {
     this._client = new Client({
@@ -53,18 +56,33 @@ exports.NativeRealm = class NativeRealm {
     });
   }
 
+  async clusterReady() {
+    return await this._autoRetry({ maxAttempts: 10 }, async () => {
+      const {
+        body: { status: status },
+      } = await this._client.cluster.health({ wait_for_status: 'yellow' });
+
+      if (status === 'red') {
+        throw new Error(`not ready, cluster health is ${status}`);
+      }
+    });
+  }
+
   async setPasswords(options) {
+    await this.clusterReady();
+
     if (!(await this.isSecurityEnabled())) {
       this._log.info('security is not enabled, unable to set native realm passwords');
       return;
     }
 
     const reservedUsers = await this.getReservedUsers();
-    await Promise.all(
-      reservedUsers.map(async (user) => {
+    await Promise.all([
+      ...reservedUsers.map(async (user) => {
         await this.setPassword(user, options[`password.${user}`]);
-      })
-    );
+      }),
+      this._createSystemIndicesUser(),
+    ]);
   }
 
   async getReservedUsers(retryOpts = {}) {
@@ -100,7 +118,7 @@ exports.NativeRealm = class NativeRealm {
   }
 
   async _autoRetry(opts, fn) {
-    const { attempt = 1, maxAttempts = 3 } = opts;
+    const { attempt = 1, maxAttempts = 3, sleep = 1000 } = opts;
 
     try {
       return await fn(attempt);
@@ -111,7 +129,7 @@ exports.NativeRealm = class NativeRealm {
 
       const sec = 1.5 * attempt;
       this._log.warning(`assuming ES isn't initialized completely, trying again in ${sec} seconds`);
-      await new Promise((resolve) => setTimeout(resolve, sec * 1000));
+      await new Promise((resolve) => setTimeout(resolve, sleep));
 
       const nextOpts = {
         ...opts,
@@ -120,4 +138,43 @@ exports.NativeRealm = class NativeRealm {
       return await this._autoRetry(nextOpts, fn);
     }
   }
+
+  async _createSystemIndicesUser() {
+    if (!(await this.isSecurityEnabled())) {
+      this._log.info('security is not enabled, unable to create role and user');
+      return;
+    }
+
+    await this._client.security.putRole({
+      name: SYSTEM_INDICES_SUPERUSER,
+      refresh: 'wait_for',
+      body: {
+        cluster: ['all'],
+        indices: [
+          {
+            names: ['*'],
+            privileges: ['all'],
+            allow_restricted_indices: true,
+          },
+        ],
+        applications: [
+          {
+            application: '*',
+            privileges: ['*'],
+            resources: ['*'],
+          },
+        ],
+        run_as: ['*'],
+      },
+    });
+
+    await this._client.security.putUser({
+      username: SYSTEM_INDICES_SUPERUSER,
+      refresh: 'wait_for',
+      body: {
+        password: this._elasticPassword,
+        roles: [SYSTEM_INDICES_SUPERUSER],
+      },
+    });
+  }
 };