Merge branch 'main' into new-o11y-rules-page-functional-tests

.buildkite/ftr_configs.yml

@@ -32,6 +32,7 @@ disabled:
   - x-pack/test/security_solution_cypress/upgrade_config.ts
   - x-pack/test/security_solution_cypress/visual_config.ts
   - x-pack/test/functional_enterprise_search/with_host_configured.config.ts
+  - x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
 

.buildkite/pipelines/artifacts.yml

@@ -52,6 +52,17 @@ steps:
           limit: 1
 
   - command: KIBANA_DOCKER_CONTEXT=cloud .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
       queue: n2-2

.buildkite/scripts/common/env.sh

@@ -38,6 +38,7 @@ export TEST_BROWSER_HEADLESS=1
 export ELASTIC_APM_ENVIRONMENT=ci
 export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
 export ELASTIC_APM_SERVER_URL=https://kibana-ci-apm.apm.us-central1.gcp.cloud.es.io
+# Not really a secret, if APM supported public auth we would use it and APM requires that we use this name
 export ELASTIC_APM_SECRET_TOKEN=7YKhoXsO4MzjhXjx2c
 
 if is_pr; then

.buildkite/scripts/lifecycle/post_command.sh

@@ -26,6 +26,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'x-pack/test/functional/apps/reporting/reports/session/*.pdf'
   buildkite-agent artifact upload 'x-pack/test/functional/failure_debug/html/*.html'
   buildkite-agent artifact upload '.es/**/*.hprof'
+  buildkite-agent artifact upload 'data/es_debug_*.tar.gz'
 
   echo "--- Run Failed Test Reporter"
   node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'

.buildkite/scripts/steps/artifacts/cloud.sh

@@ -43,39 +43,39 @@ jq '
   ' .buildkite/scripts/steps/cloud/deploy.json > "$DEPLOYMENT_SPEC"
 
 ecctl deployment create --track --output json --file "$DEPLOYMENT_SPEC" &> "$LOGS"
-CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS")
-CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS")
+CLOUD_DEPLOYMENT_USERNAME=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS")
+CLOUD_DEPLOYMENT_PASSWORD=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS")
 CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS")
 CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$LOGS")
 
 CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
 CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
 
-# NOTE: disabled pending log sanitization
-# echo "--- Setup FTR"
-# export TEST_KIBANA_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol)")
-# export TEST_KIBANA_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
-# export TEST_KIBANA_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
-# export TEST_KIBANA_USERNAME=$CLOUD_DEPLOYMENT_USERNAME"
-# export TEST_KIBANA_PASS=$CLOUD_DEPLOYMENT_PASSWORD"
+echo "Kibana: $CLOUD_DEPLOYMENT_KIBANA_URL"
+echo "ES: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL"
 
-# export TEST_ES_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol)")
-# export TEST_ES_HOSTNAME==$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
-# export TEST_ES_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
-# export TEST_ES_USER="$CLOUD_DEPLOYMENT_USERNAME"
-# export TEST_ES_PASS="$CLOUD_DEPLOYMENT_PASSWORD"
+function shutdown {
+  echo "--- Shutdown deployment"
+  ecctl deployment shutdown "$CLOUD_DEPLOYMENT_ID" --force --track --output json &> "$LOGS"
+}
+trap "shutdown" EXIT
 
-# export TEST_BROWSER_HEADLESS=1
+export TEST_KIBANA_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol.replace(':', ''))")
+export TEST_KIBANA_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
+export TEST_KIBANA_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
+export TEST_KIBANA_USERNAME="$CLOUD_DEPLOYMENT_USERNAME"
+export TEST_KIBANA_PASSWORD="$CLOUD_DEPLOYMENT_PASSWORD"
 
-# Error: attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ConnectionError: self signed certificate in certificate chain
-# export NODE_TLS_REJECT_UNAUTHORIZED=0
+export TEST_ES_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_ELASTICSEARCH_URL').protocol.replace(':', ''))")
+export TEST_ES_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_ELASTICSEARCH_URL').hostname)")
+export TEST_ES_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_ELASTICSEARCH_URL').port)")
+export TEST_ES_USERNAME="$CLOUD_DEPLOYMENT_USERNAME"
+export TEST_ES_PASSWORD="$CLOUD_DEPLOYMENT_PASSWORD"
 
-# echo "--- Run default functional tests"
-# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud
+export TEST_BROWSER_HEADLESS=1
 
-# echo "--- Run x-pack functional tests"
-# cd x-pack
-# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud 
+# Error: attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ConnectionError: self signed certificate in certificate chain
+export NODE_TLS_REJECT_UNAUTHORIZED=0
 
-echo "--- Shutdown deployment"
-ecctl deployment shutdown "$CLOUD_DEPLOYMENT_ID" --force --track --output json &> "$LOGS"
+echo "--- FTR - Reporting"
+node --no-warnings scripts/functional_test_runner.js --config x-pack/test/functional/apps/visualize/config.ts --include-tag=smoke --quiet

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -19,6 +19,8 @@ if [[ "$KIBANA_DOCKER_CONTEXT" == "default" ]]; then
   DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
 elif [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
   DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
+elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi" ]]; then
+  DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
 fi
 
 tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"

.eslintignore

@@ -30,7 +30,7 @@ snapshots.js
 /x-pack/plugins/reporting/server/export_types/printable_pdf_v2/server/lib/pdf/assets/**
 
 # package overrides
-/packages/elastic-eslint-config-kibana
+/packages/kbn-eslint-config
 /packages/kbn-plugin-generator/template
 /packages/kbn-generate/templates
 /packages/kbn-pm/dist

.eslintrc.js

@@ -230,7 +230,7 @@ const RESTRICTED_IMPORTS = [
 module.exports = {
   root: true,
 
-  extends: ['@elastic/eslint-config-kibana', 'plugin:@elastic/eui/recommended'],
+  extends: ['plugin:@elastic/eui/recommended', '@kbn/eslint-config'],
 
   overrides: [
     /**
@@ -304,7 +304,7 @@ module.exports = {
      */
     {
       files: [
-        'packages/elastic-eslint-config-kibana/**/*.{js,mjs,ts,tsx}',
+        'packages/kbn-eslint-config/**/*.{js,mjs,ts,tsx}',
         'packages/kbn-datemath/**/*.{js,mjs,ts,tsx}',
       ],
       rules: {

dev_docs/operations/ci_stats.mdx

@@ -0,0 +1,15 @@
+---
+id: kibDevDocsOpsCiStats
+slug: /kibana-dev-docs/ops/ci-stats
+title: "Kibana CI Stats"
+description: A service that we run to track little bits of data about CI runs.
+tags: ['kibana', 'dev', 'contributor', 'operations', 'ci']
+---
+
+Kibana CI Stats (sometimes written "ci-stats") is a service run at https://ci-stats.kibana.dev. This service runs a series of APIs in front of an Elasticsearch Cluster which allows us to record data about each run of CI. This data includes metrics, times, test results, and more. We are working on storing performance data in there too, and eventually will spend time to write good UIs on top of this data.
+
+The service implementation is available at https://github.com/elastic/kibana-ci-stats/ (private because there doesn't seem to be a good reason to make it public).
+
+The service is run on Google Cloud Run, which allows us to build a container, push it to GCR, define a memory limit, vCPU count, and concurrent request limit per container, and Google will automatically scale the container for us. It works pretty well and hides a lot of the complexity of running the service. The repo uses Buildkite CI to build and deploy the container when pushing to the main branch. All changes to the main branch must come from PRs, but at this time we don't require review for PRs.
+
+The website at https://ci-stats.kibana.dev uses EUI and Elastic Charts, and currently has users powered by Github OAuth. When someone authenticates with ci-stats they are first redirected to Github for authentication, then their membership in the Elastic org is checked. Users in the Elastic org will be able to do things that other users can't, like [trigger flaky test runner jobs](https://ci-stats.kibana.dev/trigger_flaky_test_runner).

dev_docs/operations/operations_landing.mdx

@@ -0,0 +1,47 @@
+---
+id: kibDevDocsOpsOverview
+slug: /kibana-dev-docs/ops
+title: Kibana Operations
+description: Links to all the documentation maintained by the Kibana Operations team
+tags: ['kibana', 'dev', 'contributor', 'operations']
+layout: landing
+---
+
+<DocLandingHero
+  title='Welcome to the ops docs!'
+  description="👋 Here you will find documentation about all the things the Kibana Operations team manages. Feel free to look around and ask questions!"
+/>
+
+<DocRelatedArticles
+  sectionTitle='CI'
+  items={[
+    { pageId: "kibDevDocsOpsCiStats" },
+  ]}
+/>
+
+<DocRelatedArticles
+  sectionTitle='Build tooling'
+  items={[
+    { pageId: "kibDevDocsOpsOptimizer" },
+    { pageId: "kibDevDocsOpsBabelPreset" },
+    { pageId: "kibDevDocsOpsTypeSummarizer" },
+  ]}
+/>
+
+<DocRelatedArticles
+  sectionTitle='Linting & Validation'
+  items={[
+    { pageId: "kibDevDocsOpsEslintConfig" },
+    { pageId: "kibDevDocsOpsEslintWithTypes" },
+    { pageId: "kibDevDocsOpsEslintPluginImports" },
+  ]}
+/>
+
+<DocRelatedArticles
+  sectionTitle='Utilities'
+  items={[
+    { pageId: "kibDevDocsToolingLog" },
+    { pageId: "kibDevDocsOpsJestSerializers"},
+    { pageId: "kibDevDocsOpsExpect" },
+  ]}
+/>

docs/api/cases/cases-api-add-comment.asciidoc

@@ -119,6 +119,7 @@ The API returns details about the case and its comments. For example:
     "syncAlerts":false
   },
   "owner": "cases",
+  "duration": null,
   "closed_at": null,
   "closed_by": null,
   "created_at": "2022-03-24T00:37:03.906Z",

docs/api/cases/cases-api-create.asciidoc

@@ -205,6 +205,7 @@ the case identifier, version, and creation time. For example:
   },
   "owner": "securitySolution",
   "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants. Operation bubblegum is active. Repeat - operation bubblegum is now active",
+  "duration": null,
   "closed_at": null,
   "closed_by": null,
   "created_at": "2022-05-13T09:16:17.416Z",

docs/api/cases/cases-api-find-cases.asciidoc

@@ -125,6 +125,7 @@ The API returns a JSON object listing the retrieved cases. For example:
         "syncAlerts": true
       },
       "owner": "securitySolution",
+      "duration": null,
       "closed_at": null,
       "closed_by": null,
       "created_at": "2022-03-29T13:03:23.533Z",
@@ -164,6 +165,7 @@ The API returns a JSON object listing the retrieved cases. For example:
         "syncAlerts": false
       },
       "owner": "cases",
+      "duration": null,
       "closed_at": null,
       "closed_by": null,
       "created_at": "2022-03-29T11:30:02.658Z",

docs/api/cases/cases-api-get-case.asciidoc

@@ -59,6 +59,7 @@ The API returns a JSON object with the retrieved case. For example:
   "version": "Wzk4LDFd",
   "comments": [],
   "totalComment": 0,
+  "totalAlerts": 0,
   "closed_at": null,
   "closed_by": null,
   "created_at": "2020-03-29T11:30:02.658Z",
@@ -90,6 +91,7 @@ The API returns a JSON object with the retrieved case. For example:
     "syncAlerts": true
   },
   "owner": "securitySolution",
+  "duration": null,
   "tags": [
     "phishing",
     "social engineering",

docs/api/cases/cases-api-push.asciidoc

@@ -68,6 +68,7 @@ The API returns a JSON object representing the pushed case. For example:
     "syncAlerts": true
   },
   "owner": "securitySolution",
+  "duration": null,
   "closed_at": null,
   "closed_by": null,
   "created_at": "2022-03-29T11:30:02.658Z",

docs/api/cases/cases-api-update-comment.asciidoc

@@ -106,35 +106,36 @@ The API returns details about the case and its comments. For example:
   "comments":[{
     "id": "8af6ac20-74f6-11ea-b83a-553aecdb28b6",
     "version": "WzIwNjM3LDFd",
-    "comment":"That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives. Even worse, he likes baked beans.",
-    "type":"user",
-    "owner":"cases",
-    "created_at":"2022-03-24T00:37:10.832Z",
+    "comment": "That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives. Even worse, he likes baked beans.",
+    "type": "user",
+    "owner": "cases",
+    "created_at": "2022-03-24T00:37:10.832Z",
     "created_by": {
         "email": "moneypenny@hms.gov.uk",
         "full_name": "Ms Moneypenny",
         "username": "moneypenny"
     },
-    "pushed_at":null,
-    "pushed_by":null,
-    "updated_at":"2022-03-24T01:27:06.210Z",
+    "pushed_at": null,
+    "pushed_by": null,
+    "updated_at": "2022-03-24T01:27:06.210Z",
     "updated_by": {
         "email": "jbond@hms.gov.uk",
         "full_name": "James Bond",
         "username": "_007"
       }
     }
   ],
-  "totalAlerts":0,
+  "totalAlerts": 0,
   "id": "293f1bc0-74f6-11ea-b83a-553aecdb28b6",
   "version": "WzIwNjM2LDFd",
   "totalComment": 1,
   "title": "This case will self-destruct in 5 seconds",
-   "tags": ["phishing","social engineering"],
+  "tags": ["phishing","social engineering"],
   "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants.",
   "settings": {"syncAlerts":false},
-  "owner": "cases","
-  closed_at": null,
+  "owner": "cases",
+  "duration": null,
+  "closed_at": null,
   "closed_by": null,
   "created_at": "2022-03-24T00:37:03.906Z",
   "created_by": {

docs/api/cases/cases-api-update.asciidoc

@@ -226,6 +226,7 @@ The API returns the updated case with a new `version` value. For example:
     },
     "owner": "securitySolution",
     "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants. Operation bubblegum is active. Repeat - operation bubblegum is now active!",
+    "duration": null,
     "closed_at": null,
     "closed_by": null,
     "created_at": "2022-05-13T09:16:17.416Z",

docs/apm/services.asciidoc

@@ -12,3 +12,41 @@ and requires anomaly detection to be enabled.
 
 [role="screenshot"]
 image::apm/images/apm-services-overview.png[Example view of services table the APM app in Kibana]
+
+[float]
+[[service-groups]]
+==== Service groups
+
+preview::[]
+
+Group services together to build meaningful views that remove noise and simplify investigations across services.
+Service groups are {kib} space-specific and available for any users with appropriate access.
+
+[role="screenshot"]
+image::apm/images/apm-service-group.png[Example view of service group in the APM app in Kibana]
+
+To enable Service groups, open {kib} and navigate to **Stack Management** > **Advanced Settings** > **Observability**,
+and enable the **Service groups feature**.
+
+To create a service group, navigate to **Observability** > **APM** > **Services** and select **Create group**.
+Specify a name, color, and description.
+Then, using the <<kuery-query, Kibana Query Language (KQL)>>, specify a query to select services for the group.
+Services that match the query within the last 24 hours will be assigned to the group.
+
+[NOTE]
+====
+Once a service group has been saved, this list of services within it is static.
+If a newly added service matches the KQL query, it will not be automatically added to the service group.
+Similarly, if a service stops matching the KQL query, it will not be removed from the group.
+
+To update the list of services within a group,
+edit the service group, click **Refresh** next to the KQL query, and click **Save group**.
+====
+
+**Examples**
+
+Not sure where to get started? Here are some sample queries you can build from:
+
+* Group services by environment--in this example, "production": `service.environment : "production"`
+* Group services by name--this example groups those that end in "beat": `service.name : *beat` (matches services named "Auditbeat", "Heartbeat", "Filebeat", etc.)
+* Group services with a high transaction duration in the last 24 hours: `transaction.duration.us >= 50000000`

docs/developer/getting-started/running-kibana-advanced.asciidoc

@@ -30,7 +30,7 @@ If you’re installing dependencies and seeing an error that looks
 something like
 
 ....
-Unsupported URL Type: link:packages/elastic-eslint-config-kibana
+Unsupported URL Type: link:packages/kbn-eslint-config
 ....
 
 you’re likely running `npm`. To install dependencies in {kib} you

docs/development/core/server/kibana-plugin-core-server.savedobjectsupdateoptions.md

@@ -18,6 +18,7 @@ export interface SavedObjectsUpdateOptions<Attributes = unknown> extends SavedOb
 |  --- | --- | --- |
 |  [references?](./kibana-plugin-core-server.savedobjectsupdateoptions.references.md) | SavedObjectReference\[\] | <i>(Optional)</i> A reference to another saved object. |
 |  [refresh?](./kibana-plugin-core-server.savedobjectsupdateoptions.refresh.md) | MutatingOperationRefreshSetting | <i>(Optional)</i> The Elasticsearch Refresh setting for this operation |
+|  [retryOnConflict?](./kibana-plugin-core-server.savedobjectsupdateoptions.retryonconflict.md) | number | <i>(Optional)</i> The Elasticsearch <code>retry_on_conflict</code> setting for this operation. Defaults to <code>0</code> when <code>version</code> is provided, <code>3</code> otherwise. |
 |  [upsert?](./kibana-plugin-core-server.savedobjectsupdateoptions.upsert.md) | Attributes | <i>(Optional)</i> If specified, will be used to perform an upsert if the document doesn't exist |
 |  [version?](./kibana-plugin-core-server.savedobjectsupdateoptions.version.md) | string | <i>(Optional)</i> An opaque version number which changes on each successful write operation. Can be used for implementing optimistic concurrency control. |