Merge branch 'main' into relementy_agg_based_remove

.backportrc.json

@@ -45,5 +45,6 @@
   },
   "autoMerge": true,
   "autoMergeMethod": "squash",
-  "backportBinary": "node scripts/backport"
+  "backportBinary": "node scripts/backport",
+  "prDescription": "{defaultPrDescription}\n\n<!--BACKPORT {commits} BACKPORT-->"
 }

.bazelignore

@@ -2,16 +2,26 @@
 # Issues are opened for to include that feature but not available yet
 # https://github.com/bazelbuild/bazel/issues/7093
 # https://github.com/bazelbuild/bazel/issues/8106
+.buildkite
 .ci
+.chromium
 .git
 .github
 .idea
+.native_modules
 .teamcity
 .yarn-local-mirror
+.vscode
+api_docs
 bazel-bin
 bazel-kibana
 bazel-out
 bazel-testlogs
 build
+data
+dev_docs
+docs
+legacy_rfcs
 node_modules
 target
+vars

.buildkite/ftr_configs.yml

@@ -44,8 +44,6 @@ disabled:
   - x-pack/plugins/observability/e2e/synthetics_run.ts
 
   # Configs that exist but weren't running in CI when this file was introduced
-  - test/visual_regression/config.ts
-  - x-pack/test/visual_regression/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/config.ts
   - x-pack/test/alerting_api_integration/spaces_only_legacy/config.ts
@@ -55,6 +53,7 @@ disabled:
   - x-pack/test/load/config.ts
   - x-pack/test/plugin_api_perf/config.js
   - x-pack/test/screenshot_creation/config.ts
+  - x-pack/test/fleet_packages/config.ts
 
 defaultQueue: 'n2-4-spot'
 enabled:

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -332,34 +332,36 @@ export async function pickTestGroupRunOrder() {
     { title: string; expectedDurationMin: number; names: string[] }
   > = {};
 
-  for (const { groups, queue } of getRunGroups(bk, types, FUNCTIONAL_TYPE)) {
-    for (const group of groups) {
-      if (!group.names.length) {
-        continue;
+  if (ftrConfigsByQueue.size) {
+    for (const { groups, queue } of getRunGroups(bk, types, FUNCTIONAL_TYPE)) {
+      for (const group of groups) {
+        if (!group.names.length) {
+          continue;
+        }
+
+        const key = `ftr_configs_${configCounter++}`;
+        let sortBy;
+        let title;
+        if (group.names.length === 1) {
+          title = group.names[0];
+          sortBy = title;
+        } else {
+          sortBy = ++groupCounter;
+          title = `FTR Configs #${sortBy}`;
+        }
+
+        functionalGroups.push({
+          title,
+          key,
+          sortBy,
+          queue: queue ?? defaultQueue,
+        });
+        ftrRunOrder[key] = {
+          title,
+          expectedDurationMin: group.durationMin,
+          names: group.names,
+        };
       }
-
-      const key = `ftr_configs_${configCounter++}`;
-      let sortBy;
-      let title;
-      if (group.names.length === 1) {
-        title = group.names[0];
-        sortBy = title;
-      } else {
-        sortBy = ++groupCounter;
-        title = `FTR Configs #${sortBy}`;
-      }
-
-      functionalGroups.push({
-        title,
-        key,
-        sortBy,
-        queue: queue ?? defaultQueue,
-      });
-      ftrRunOrder[key] = {
-        title,
-        expectedDurationMin: group.durationMin,
-        names: group.names,
-      };
     }
   }
 

.buildkite/pipelines/artifacts.yml

@@ -51,7 +51,17 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
+  - command: KIBANA_DOCKER_CONTEXT=ubi8 .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: KIBANA_DOCKER_CONTEXT=ubi9 .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
       queue: n2-2

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -133,7 +133,7 @@ for (const testSuite of testSuites) {
       env: {
         FTR_CONFIG: testSuite.ftrConfig,
       },
-      label: `FTR Config: ${testSuite.ftrConfig}`,
+      label: `${testSuite.ftrConfig}`,
       parallelism: testSuite.count,
       concurrency,
       concurrency_group: process.env.UUID,

.buildkite/pipelines/performance/daily.yml

@@ -19,11 +19,11 @@ steps:
     depends_on: build
     key: tests
 
-  - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
-    command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
-    agents:
-      queue: n2-2
-    depends_on: tests
+  # - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
+  #   command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
+  #   agents:
+  #     queue: n2-2
+  #   depends_on: tests
 
   - label: ':chart_with_upwards_trend: Report performance metrics to ci-stats'
     command: .buildkite/scripts/steps/functional/report_performance_metrics.sh

.buildkite/pipelines/pull_request/base.yml

@@ -75,10 +75,3 @@ steps:
       automatic:
         - exit_status: '-1'
           limit: 3
-
-  - command: .buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh
-    label: 'Build Webpack Bundle Analyzer reports'
-    agents:
-      queue: c2-4
-    key: webpack_bundle_analyzer
-    timeout_in_minutes: 60

.buildkite/pipelines/pull_request/deploy_cloud.yml

@@ -2,6 +2,10 @@ steps:
   - command: .buildkite/scripts/steps/cloud/build_and_deploy.sh
     label: 'Build and Deploy to Cloud'
     agents:
-      queue: n2-2
+      queue: n2-2-spot
     depends_on: build
     timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/pipelines/pull_request/webpack_bundle_analyzer.yml

@@ -0,0 +1,7 @@
+steps:
+  - command: .buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh
+    label: 'Build Webpack Bundle Analyzer reports'
+    agents:
+      queue: n2-4-spot
+    key: webpack_bundle_analyzer
+    timeout_in_minutes: 60

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -126,6 +126,10 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_cloud.yml'));
     }
 
+    if (GITHUB_PR_LABELS.includes('ci:build-webpack-bundle-analyzer')) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/webpack_bundle_analyzer.yml'));
+    }
+
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
     uploadPipeline(pipeline.join('\n'));

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -19,8 +19,10 @@ if [[ "$KIBANA_DOCKER_CONTEXT" == "default" ]]; then
   DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
 elif [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
   DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
-elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi" ]]; then
+elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi8" ]]; then
   DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
+elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi9" ]]; then
+  DOCKER_CONTEXT_FILE="kibana-ubi9-$FULL_VERSION-docker-build-context.tar.gz"
 fi
 
 tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"

.buildkite/scripts/steps/fleet/install_all_packages.sh

@@ -2,10 +2,11 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
-
-.buildkite/scripts/bootstrap.sh
+source .buildkite/scripts/steps/functional/common.sh
 
 echo '--- Installing all packages'
-cd x-pack/plugins/fleet
-node scripts/install_all_packages
+
+checks-reporter-with-killswitch "Fleet packages Tests" \
+ node scripts/functional_tests \
+   --debug --bail \
+   --config x-pack/test/fleet_packages/config.ts

.buildkite/scripts/steps/functional/performance_playwright.sh

@@ -33,7 +33,11 @@ unset ELASTIC_APM_GLOBAL_LABELS
 export TEST_ES_URL=http://elastic:changeme@localhost:9200
 export TEST_ES_DISABLE_STARTUP=true
 
-sleep 120
+# Pings the es server every seconds 2 mins until it is status is green
+curl --retry 120 \
+  --retry-delay 1 \
+  --retry-all-errors \
+  -I -XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow"
 
 journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover")
 

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -4,9 +4,9 @@ set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
 
-
+BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-}
 FTR_CONFIG_GROUP_KEY=${FTR_CONFIG_GROUP_KEY:-}
-if [ "$FTR_CONFIG_GROUP_KEY" == "" ]; then
+if [ "$FTR_CONFIG_GROUP_KEY" == "" ] && [ "$BUILDKITE_PARALLEL_JOB" == "" ]; then
   echo "Missing FTR_CONFIG_GROUP_KEY env var"
   exit 1
 fi
@@ -30,12 +30,17 @@ if [[ ! "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
   fi
 fi
 
-if [[ "$configs" == "" ]]; then
+if [ "$configs" == "" ] && [ "$FTR_CONFIG_GROUP_KEY" != "" ]; then
   echo "--- downloading ftr test run order"
   buildkite-agent artifact download ftr_run_order.json .
   configs=$(jq -r '.[env.FTR_CONFIG_GROUP_KEY].names[]' ftr_run_order.json)
 fi
 
+if [ "$configs" == "" ]; then
+  echo "unable to determine configs to run"
+  exit 1
+fi
+
 failedConfigs=""
 results=()
 

.buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh

@@ -4,6 +4,8 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
+export NODE_OPTIONS="--max-old-space-size=8192"
+
 node scripts/build_kibana_platform_plugins.js --dist --profile
 
 mkdir -p built_assets/webpack_bundle_analyzer

.ci/Dockerfile

@@ -1,7 +1,7 @@
 # NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
 # If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
 
-ARG NODE_VERSION=16.14.2
+ARG NODE_VERSION=16.16.0
 
 FROM node:${NODE_VERSION} AS base