From 242cb6f1d5cf97766fe8a5697488877e2390befe Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Tue, 28 Nov 2023 14:35:17 +0100 Subject: [PATCH 01/30] [Security Solution] Specific Cypress executions for `Rule Management` team (#171868) Co-authored-by: Georgii Gorbachev --- .../verify_es_serverless_image.yml | 26 ++++++++++ .buildkite/pipelines/on_merge.yml | 48 +++++++++++++++++++ .buildkite/pipelines/pull_request/base.yml | 48 +++++++++++++++++++ .../security_solution_cypress.yml | 24 ++++++++++ .../security_serverless_rule_management.sh | 16 +++++++ ...rverless_rule_management_prebuilt_rules.sh | 16 +++++++ .../security_solution_rule_management.sh | 16 +++++++ ...solution_rule_management_prebuilt_rules.sh | 16 +++++++ .github/CODEOWNERS | 2 - .../cypress/README.md | 23 ++++++--- .../install_update_authorization.cy.ts | 12 ++--- .../install_update_error_handling.cy.ts | 14 +++--- .../prebuilt_rules/install_via_fleet.cy.ts | 14 +++--- .../prebuilt_rules/install_workflow.cy.ts | 20 ++++---- .../prebuilt_rules/management.cy.ts | 21 ++++---- .../prebuilt_rules/notifications.cy.ts | 19 ++++---- .../prebuilt_rules_preview.cy.ts | 24 +++++----- .../prebuilt_rules/update_workflow.ts | 18 +++---- .../rule_details/common_flows.cy.ts | 26 +++++----- .../rule_details/esql_rule.cy.ts | 16 +++---- .../security_solution_cypress/package.json | 14 ++++-- 21 files changed, 332 insertions(+), 101 deletions(-) create mode 100644 .buildkite/scripts/steps/functional/security_serverless_rule_management.sh create mode 100644 .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh create mode 100644 .buildkite/scripts/steps/functional/security_solution_rule_management.sh create mode 100644 .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/install_update_authorization.cy.ts (94%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/install_update_error_handling.cy.ts (94%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/install_via_fleet.cy.ts (90%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/install_workflow.cy.ts (85%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/management.cy.ts (91%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/notifications.cy.ts (92%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/prebuilt_rules_preview.cy.ts (97%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/update_workflow.ts (88%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/rule_details/common_flows.cy.ts (88%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/rule_details/esql_rule.cy.ts (69%) diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml index 8e64513b14900..8d1b778b67983 100644 --- a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml +++ b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml @@ -95,6 +95,32 @@ steps: - exit_status: '*' limit: 1 + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh + label: 'Serverless Rule Management - Security Solution Cypress Tests' + if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'" + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh + label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'" + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh label: 'Defend Workflows Cypress Tests on Serverless' if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'" diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index 8b00db428a713..8256eb2395633 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -115,6 +115,54 @@ steps: - exit_status: '*' limit: 1 + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh + label: 'Serverless Rule Management - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh + label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh + label: 'Rule Management - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh + label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + - command: .buildkite/scripts/steps/functional/security_solution.sh label: 'Security Solution Cypress Tests' agents: diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index 49215bbd00f11..8238afbee4fd2 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -93,6 +93,30 @@ steps: - exit_status: '*' limit: 1 + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh + label: 'Serverless Rule Management - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh + label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + - command: .buildkite/scripts/steps/functional/security_solution.sh label: 'Security Solution Cypress Tests' agents: @@ -117,6 +141,30 @@ steps: - exit_status: '*' limit: 1 + - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh + label: 'Rule Management - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh + label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh label: 'Investigations - Security Solution Cypress Tests' agents: diff --git a/.buildkite/pipelines/security_solution/security_solution_cypress.yml b/.buildkite/pipelines/security_solution/security_solution_cypress.yml index 247505ef1c85a..77e7fea574352 100644 --- a/.buildkite/pipelines/security_solution/security_solution_cypress.yml +++ b/.buildkite/pipelines/security_solution/security_solution_cypress.yml @@ -30,6 +30,30 @@ steps: # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. timeout_in_minutes: 300 parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management + label: 'Serverless MKI QA Rule Management - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. + timeout_in_minutes: 300 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management:prebuilt_rules + label: 'Serverless MKI QA Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. + timeout_in_minutes: 300 + parallelism: 6 retry: automatic: - exit_status: '*' diff --git a/.buildkite/scripts/steps/functional/security_serverless_rule_management.sh b/.buildkite/scripts/steps/functional/security_serverless_rule_management.sh new file mode 100644 index 0000000000000..5d360e0db4f29 --- /dev/null +++ b/.buildkite/scripts/steps/functional/security_serverless_rule_management.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/steps/functional/common.sh +source .buildkite/scripts/steps/functional/common_cypress.sh + +export JOB=kibana-security-solution-chrome +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} + +echo "--- Rule Management Cypress Tests on Serverless" + +cd x-pack/test/security_solution_cypress + +set +e +yarn cypress:rule_management:run:serverless; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh b/.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh new file mode 100644 index 0000000000000..bc7dc3269d8cb --- /dev/null +++ b/.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/steps/functional/common.sh +source .buildkite/scripts/steps/functional/common_cypress.sh + +export JOB=kibana-security-solution-chrome +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} + +echo "--- Rule Management - Prebuilt Rules - Cypress Tests on Serverless" + +cd x-pack/test/security_solution_cypress + +set +e +yarn cypress:rule_management:prebuilt_rules:run:serverless; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/steps/functional/security_solution_rule_management.sh b/.buildkite/scripts/steps/functional/security_solution_rule_management.sh new file mode 100644 index 0000000000000..847cb42896cf1 --- /dev/null +++ b/.buildkite/scripts/steps/functional/security_solution_rule_management.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/steps/functional/common.sh +source .buildkite/scripts/steps/functional/common_cypress.sh + +export JOB=kibana-security-solution-chrome +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} + +echo "--- Rule Management - Security Solution Cypress Tests" + +cd x-pack/test/security_solution_cypress + +set +e +yarn cypress:rule_management:run:ess; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh b/.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh new file mode 100644 index 0000000000000..d8b19ad3363b5 --- /dev/null +++ b/.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/steps/functional/common.sh +source .buildkite/scripts/steps/functional/common_cypress.sh + +export JOB=kibana-security-solution-chrome +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} + +echo "--- Rule Management - Prebuilt Rules - Security Solution Cypress Tests" + +cd x-pack/test/security_solution_cypress + +set +e +yarn cypress:rule_management:prebuilt_rules:run:ess; status=$?; yarn junit:merge || :; exit $status diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 89e152a2fe40f..7d075295240c9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1322,9 +1322,7 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout @elastic/ /x-pack/plugins/security_solution/common/api/detection_engine/rule_monitoring @elastic/security-detection-rule-management /x-pack/plugins/security_solution/common/detection_engine/rule_management @elastic/security-detection-rule-management -/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules @elastic/security-detection-rule-management /x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management @elastic/security-detection-rule-management -/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details @elastic/security-detection-rule-management /x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules @elastic/security-detection-rule-management /x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/rule_management @elastic/security-detection-rule-management /x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules @elastic/security-detection-rule-management diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md index 8940d6c86e73e..88786aed7ff56 100644 --- a/x-pack/test/security_solution_cypress/cypress/README.md +++ b/x-pack/test/security_solution_cypress/cypress/README.md @@ -62,19 +62,25 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio | cypress | Runs the default Cypress command | | cypress:open:ess | Opens the Cypress UI with all tests in the `e2e` directory. This also runs a local kibana and ES instance. The kibana instance will reload when you make code changes. This is the recommended way to debug and develop tests. | | cypress:open:serverless | Opens the Cypress UI with all tests in the `e2e` directory. This also runs a mocked serverless environment. The kibana instance will reload when you make code changes. This is the recommended way to debug and develop tests. | -| cypress:run:ess | Runs all tests tagged as ESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode | +| cypress:run:ess | Runs all tests tagged as ESS placed in the `e2e` directory excluding `investigations`,`explore` and `detection_response/rule_management` directories in headless mode | | cypress:run:cases:ess | Runs all tests under `explore/cases` in the `e2e` directory related to the Cases area team in headless mode | | cypress:ess | Runs all ESS tests with the specified configuration in headless mode and produces a report using `cypress-multi-reporters` | +| cypress:rule_management:run:ess | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | +| cypress:rule_management:prebuilt_rules:run:ess | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | | cypress:run:respops:ess | Runs all tests related to the Response Ops area team, specifically tests in `detection_alerts`, `detection_rules`, and `exceptions` directories in headless mode | -| cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations` and `explore` directories in headless mode | -| cypress:investigations:run:ess | Runs all tests tagged as ESS in the `e2e/investigations` directory in headless mode | +| cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations`, `explore` and `rule_management` directories in headless mode | +| cypress:rule_management:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | +| cypress:rule_management:prebuilt_rules:run:serverless | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | +| cypress:investigations:run:ess | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode | | cypress:explore:run:ess | Runs all tests tagged as ESS in the `e2e/explore` directory in headless mode | | cypress:investigations:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode | | cypress:explore:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode | | cypress:open:qa:serverless | Opens the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment. The kibana instance will reload when you make code changes. This is the recommended way to debug tests in QA. Follow the readme in order to learn about the known limitations. | -| cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode using the QA environment and real MKI projects.| +| cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations`, `explore` and `rule_management` directories in headless mode using the QA environment and real MKI projects.| | cypress:run:qa:serverless:explore | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode using the QA environment and real MKI prorjects. | | cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. | +| cypress:run:qa:serverless:rule_management | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` directory, excluding `e2e/detection_response/rule_management/prebuilt_rules` in headless mode using the QA environment and reak MKI projects. | +| cypress:run:qa:serverless:rule_management:prebuilt_rules | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode using the QA environment and reak MKI projects. | | junit:merge | Merges individual test reports into a single report and moves the report to the `junit` directory | Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. @@ -94,7 +100,7 @@ Below you can find the folder structure used on our Cypress tests. Cypress convention starting version 10 (previously known as integration). Contains the specs that are going to be executed. -### e2e/explore and e2e/investigations +### Area teams folders These directories contain tests which are run in their own Buildkite pipeline. @@ -103,7 +109,8 @@ If you belong to one of the teams listed in the table, please add new e2e specs | Directory | Area team | | -- | -- | | `e2e/explore` | Threat Hunting Explore | -| `e2e/investigations | Threat Hunting Investigations | +| `e2e/investigations` | Threat Hunting Investigations | +| `e2e/detection_response/rule_management` | Detection Rule Management | ### fixtures/ @@ -203,6 +210,8 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio | cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations` and `explore` directories in headless mode | | cypress:investigations:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode | | cypress:explore:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode | +| cypress:rule_management:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | +| cypress:rule_management:prebuilt_rules:run:serverless | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. @@ -248,6 +257,8 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio | cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode using the QA environment and real MKI projects.| | cypress:run:qa:serverless:explore | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode using the QA environment and real MKI prorjects. | | cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. | +| cypress:run:qa:serverless:rule_management | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` directory, excluding `e2e/detection_response/rule_management/prebuilt_rules` in headless mode using the QA environment and reak MKI projects. | +| cypress:run:qa:serverless:rule_management:prebuilt_rules | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode using the QA environment and reak MKI projects. | Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts similarity index 94% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts index e0078dd54e7ea..29e650dd4de66 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts @@ -12,14 +12,14 @@ import { } from '@kbn/security-solution-plugin/common/constants'; import { ROLES } from '@kbn/security-solution-plugin/common/test'; -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { createAndInstallMockedPrebuiltRules, installPrebuiltRuleAssets, preventPrebuiltRulesPackageInstallation, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { visit } from '../../../tasks/navigation'; -import { RULES_MANAGEMENT_URL } from '../../../urls/rules_management'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { visit } from '../../../../tasks/navigation'; +import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; import { ADD_ELASTIC_RULES_BTN, getInstallSingleRuleButtonByRuleId, @@ -31,8 +31,8 @@ import { RULES_UPDATES_TAB, RULE_CHECKBOX, UPGRADE_ALL_RULES_BUTTON, -} from '../../../screens/alerts_detection_rules'; -import { login } from '../../../tasks/login'; +} from '../../../../screens/alerts_detection_rules'; +import { login } from '../../../../tasks/login'; // Rule to test update const RULE_1_ID = 'rule_1'; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts similarity index 94% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts index 7e288910ccb60..db84d92e4ddb6 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { getInstallSingleRuleButtonByRuleId, getUpgradeSingleRuleButtonByRuleId, @@ -14,14 +14,14 @@ import { SELECT_ALL_RULES_ON_PAGE_CHECKBOX, UPGRADE_ALL_RULES_BUTTON, UPGRADE_SELECTED_RULES_BUTTON, -} from '../../../screens/alerts_detection_rules'; -import { selectRulesByName } from '../../../tasks/alerts_detection_rules'; +} from '../../../../screens/alerts_detection_rules'; +import { selectRulesByName } from '../../../../tasks/alerts_detection_rules'; import { installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, preventPrebuiltRulesPackageInstallation, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { login } from '../../../tasks/login'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { login } from '../../../../tasks/login'; import { clickAddElasticRulesButton, assertInstallationRequestIsComplete, @@ -33,8 +33,8 @@ import { assertRulesPresentInAddPrebuiltRulesTable, assertRuleUpgradeFailureToastShown, assertRulesPresentInRuleUpdatesTable, -} from '../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; +} from '../../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; describe( 'Detection rules, Prebuilt Rules Installation and Update - Error handling', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_via_fleet.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_via_fleet.cy.ts similarity index 90% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_via_fleet.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_via_fleet.cy.ts index 6da3d58c0530d..762e79bb27003 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_via_fleet.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_via_fleet.cy.ts @@ -8,13 +8,13 @@ import type { BulkInstallPackageInfo } from '@kbn/fleet-plugin/common'; import type { Rule } from '@kbn/security-solution-plugin/public/detection_engine/rule_management/logic/types'; -import { resetRulesTableState } from '../../../tasks/common'; -import { INSTALL_ALL_RULES_BUTTON, TOASTER } from '../../../screens/alerts_detection_rules'; -import { getRuleAssets } from '../../../tasks/api_calls/prebuilt_rules'; -import { login } from '../../../tasks/login'; -import { clickAddElasticRulesButton } from '../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; -import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +import { resetRulesTableState } from '../../../../tasks/common'; +import { INSTALL_ALL_RULES_BUTTON, TOASTER } from '../../../../screens/alerts_detection_rules'; +import { getRuleAssets } from '../../../../tasks/api_calls/prebuilt_rules'; +import { login } from '../../../../tasks/login'; +import { clickAddElasticRulesButton } from '../../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts similarity index 85% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts index ec4615bcf59e4..523d0ec0ad4e0 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { resetRulesTableState } from '../../../tasks/common'; -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { resetRulesTableState } from '../../../../tasks/common'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { getInstallSingleRuleButtonByRuleId, GO_BACK_TO_RULES_TABLE_BUTTON, @@ -16,19 +16,19 @@ import { RULE_CHECKBOX, SELECT_ALL_RULES_ON_PAGE_CHECKBOX, TOASTER, -} from '../../../screens/alerts_detection_rules'; -import { selectRulesByName } from '../../../tasks/alerts_detection_rules'; -import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../screens/breadcrumbs'; -import { installPrebuiltRuleAssets } from '../../../tasks/api_calls/prebuilt_rules'; -import { login } from '../../../tasks/login'; +} from '../../../../screens/alerts_detection_rules'; +import { selectRulesByName } from '../../../../tasks/alerts_detection_rules'; +import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../../screens/breadcrumbs'; +import { installPrebuiltRuleAssets } from '../../../../tasks/api_calls/prebuilt_rules'; +import { login } from '../../../../tasks/login'; import { assertInstallationRequestIsComplete, assertRuleInstallationSuccessToastShown, assertRulesPresentInInstalledRulesTable, clickAddElasticRulesButton, -} from '../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; -import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +} from '../../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts similarity index 91% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts index f3101f513915f..15e020b5e0663 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { COLLAPSED_ACTION_BTN, ELASTIC_RULES_BTN, @@ -15,7 +15,7 @@ import { RULE_SWITCH, SELECT_ALL_RULES_ON_PAGE_CHECKBOX, INSTALL_ALL_RULES_BUTTON, -} from '../../../screens/alerts_detection_rules'; +} from '../../../../screens/alerts_detection_rules'; import { deleteFirstRule, disableAutoRefresh, @@ -24,21 +24,24 @@ import { selectRulesByName, waitForPrebuiltDetectionRulesToBeLoaded, waitForRuleToUpdate, -} from '../../../tasks/alerts_detection_rules'; +} from '../../../../tasks/alerts_detection_rules'; import { deleteSelectedRules, disableSelectedRules, enableSelectedRules, -} from '../../../tasks/rules_bulk_actions'; +} from '../../../../tasks/rules_bulk_actions'; import { createAndInstallMockedPrebuiltRules, getAvailablePrebuiltRulesCount, preventPrebuiltRulesPackageInstallation, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { deleteAlertsAndRules, deletePrebuiltRulesAssets } from '../../../tasks/api_calls/common'; -import { login } from '../../../tasks/login'; -import { visit } from '../../../tasks/navigation'; -import { RULES_MANAGEMENT_URL } from '../../../urls/rules_management'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { + deleteAlertsAndRules, + deletePrebuiltRulesAssets, +} from '../../../../tasks/api_calls/common'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; +import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; const rules = Array.from(Array(5)).map((_, i) => { return createRuleAssetSavedObject({ diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts similarity index 92% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts index 92bf9e7f1471c..4812efc740ae2 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts @@ -5,22 +5,25 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { ADD_ELASTIC_RULES_BTN, ADD_ELASTIC_RULES_EMPTY_PROMPT_BTN, RULES_UPDATES_TAB, -} from '../../../screens/alerts_detection_rules'; -import { deleteFirstRule } from '../../../tasks/alerts_detection_rules'; -import { deleteAlertsAndRules, deletePrebuiltRulesAssets } from '../../../tasks/api_calls/common'; +} from '../../../../screens/alerts_detection_rules'; +import { deleteFirstRule } from '../../../../tasks/alerts_detection_rules'; +import { + deleteAlertsAndRules, + deletePrebuiltRulesAssets, +} from '../../../../tasks/api_calls/common'; import { installAllPrebuiltRulesRequest, installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { resetRulesTableState } from '../../../tasks/common'; -import { login } from '../../../tasks/login'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { resetRulesTableState } from '../../../../tasks/common'; +import { login } from '../../../../tasks/login'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; const RULE_1 = createRuleAssetSavedObject({ name: 'Test rule 1', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts similarity index 97% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_preview.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts index 6deeb6f5202c0..81f37b7760df2 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts @@ -12,22 +12,22 @@ import type { PrebuiltRuleAsset } from '@kbn/security-solution-plugin/server/lib import type { Threshold } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; import { AlertSuppression } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { INSTALL_PREBUILT_RULE_BUTTON, INSTALL_PREBUILT_RULE_PREVIEW, UPDATE_PREBUILT_RULE_PREVIEW, UPDATE_PREBUILT_RULE_BUTTON, -} from '../../../screens/alerts_detection_rules'; -import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../screens/breadcrumbs'; +} from '../../../../screens/alerts_detection_rules'; +import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../../screens/breadcrumbs'; import { installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { createSavedQuery, deleteSavedQueries } from '../../../tasks/api_calls/saved_queries'; -import { fetchMachineLearningModules } from '../../../tasks/api_calls/machine_learning'; -import { resetRulesTableState } from '../../../tasks/common'; -import { login } from '../../../tasks/login'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { createSavedQuery, deleteSavedQueries } from '../../../../tasks/api_calls/saved_queries'; +import { fetchMachineLearningModules } from '../../../../tasks/api_calls/machine_learning'; +import { resetRulesTableState } from '../../../../tasks/common'; +import { login } from '../../../../tasks/login'; import { assertRuleInstallationSuccessToastShown, assertRulesNotPresentInAddPrebuiltRulesTable, @@ -36,7 +36,7 @@ import { assertRuleUpgradeSuccessToastShown, clickAddElasticRulesButton, clickRuleUpdatesTab, -} from '../../../tasks/prebuilt_rules'; +} from '../../../../tasks/prebuilt_rules'; import { assertAlertSuppressionPropertiesShown, assertCommonPropertiesShown, @@ -55,13 +55,13 @@ import { closeRulePreview, openRuleInstallPreview, openRuleUpdatePreview, -} from '../../../tasks/prebuilt_rules_preview'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; +} from '../../../../tasks/prebuilt_rules_preview'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; import { deleteAlertsAndRules, deleteDataView, postDataView, -} from '../../../tasks/api_calls/common'; +} from '../../../../tasks/api_calls/common'; const TEST_ENV_TAGS = ['@ess', '@serverless']; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/update_workflow.ts similarity index 88% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/update_workflow.ts index edeb8ac98623b..d858280dd5294 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/update_workflow.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { getUpgradeSingleRuleButtonByRuleId, NO_RULES_AVAILABLE_FOR_UPGRADE_MESSAGE, @@ -13,22 +13,22 @@ import { SELECT_ALL_RULES_ON_PAGE_CHECKBOX, UPGRADE_ALL_RULES_BUTTON, UPGRADE_SELECTED_RULES_BUTTON, -} from '../../../screens/alerts_detection_rules'; -import { selectRulesByName } from '../../../tasks/alerts_detection_rules'; -import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +} from '../../../../screens/alerts_detection_rules'; +import { selectRulesByName } from '../../../../tasks/alerts_detection_rules'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; import { installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { resetRulesTableState } from '../../../tasks/common'; -import { login } from '../../../tasks/login'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { resetRulesTableState } from '../../../../tasks/common'; +import { login } from '../../../../tasks/login'; import { assertRulesNotPresentInRuleUpdatesTable, assertRuleUpgradeSuccessToastShown, assertUpgradeRequestIsComplete, clickRuleUpdatesTab, -} from '../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; +} from '../../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/common_flows.cy.ts similarity index 88% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/common_flows.cy.ts index f5704122d9e33..0610786fc1b89 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/common_flows.cy.ts @@ -5,17 +5,17 @@ * 2.0. */ -import { deleteRuleFromDetailsPage } from '../../../tasks/alerts_detection_rules'; +import { deleteRuleFromDetailsPage } from '../../../../tasks/alerts_detection_rules'; import { CUSTOM_RULES_BTN, RULES_MANAGEMENT_TABLE, RULES_ROW, -} from '../../../screens/alerts_detection_rules'; -import { createRule } from '../../../tasks/api_calls/rules'; -import { getDetails } from '../../../tasks/rule_details'; -import { ruleFields } from '../../../data/detection_engine'; -import { getTimeline } from '../../../objects/timeline'; -import { getExistingRule, getNewRule } from '../../../objects/rule'; +} from '../../../../screens/alerts_detection_rules'; +import { createRule } from '../../../../tasks/api_calls/rules'; +import { getDetails } from '../../../../tasks/rule_details'; +import { ruleFields } from '../../../../data/detection_engine'; +import { getTimeline } from '../../../../objects/timeline'; +import { getExistingRule, getNewRule } from '../../../../objects/rule'; import { ABOUT_DETAILS, @@ -42,13 +42,13 @@ import { THREAT_TACTIC, THREAT_TECHNIQUE, TIMELINE_TEMPLATE_DETAILS, -} from '../../../screens/rule_details'; +} from '../../../../screens/rule_details'; -import { createTimeline } from '../../../tasks/api_calls/timelines'; -import { deleteAlertsAndRules, deleteConnectors } from '../../../tasks/api_calls/common'; -import { login } from '../../../tasks/login'; -import { visit } from '../../../tasks/navigation'; -import { ruleDetailsUrl } from '../../../urls/rule_details'; +import { createTimeline } from '../../../../tasks/api_calls/timelines'; +import { deleteAlertsAndRules, deleteConnectors } from '../../../../tasks/api_calls/common'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; +import { ruleDetailsUrl } from '../../../../urls/rule_details'; // This test is meant to test all common aspects of the rule details page that should function // the same regardless of rule type. For any rule type specific functionalities, please include diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/esql_rule.cy.ts similarity index 69% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/esql_rule.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/esql_rule.cy.ts index 7d1419e911e33..c59b7db55c743 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/esql_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/esql_rule.cy.ts @@ -5,24 +5,24 @@ * 2.0. */ -import { getEsqlRule } from '../../../objects/rule'; +import { getEsqlRule } from '../../../../objects/rule'; import { ESQL_QUERY_DETAILS, DEFINITION_DETAILS, RULE_NAME_HEADER, RULE_TYPE_DETAILS, -} from '../../../screens/rule_details'; +} from '../../../../screens/rule_details'; -import { createRule } from '../../../tasks/api_calls/rules'; +import { createRule } from '../../../../tasks/api_calls/rules'; -import { getDetails } from '../../../tasks/rule_details'; -import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +import { getDetails } from '../../../../tasks/rule_details'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; -import { login } from '../../../tasks/login'; -import { visit } from '../../../tasks/navigation'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; -import { ruleDetailsUrl } from '../../../urls/rule_details'; +import { ruleDetailsUrl } from '../../../../urls/rule_details'; describe('Detection ES|QL rules, details view', { tags: ['@ess'] }, () => { const rule = getEsqlRule(); diff --git a/x-pack/test/security_solution_cypress/package.json b/x-pack/test/security_solution_cypress/package.json index e43f32a447575..e1f552fdba9de 100644 --- a/x-pack/test/security_solution_cypress/package.json +++ b/x-pack/test/security_solution_cypress/package.json @@ -7,9 +7,11 @@ "scripts": { "cypress": "NODE_OPTIONS=--openssl-legacy-provider ../../../node_modules/.bin/cypress", "cypress:open:ess": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel open --spec './cypress/e2e/**/*.cy.ts' --config-file ../../test/security_solution_cypress/cypress/cypress.config.ts --ftr-config-file ../../test/security_solution_cypress/cli_config", - "cypress:run:ess": "yarn cypress:ess --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'", + "cypress:run:ess": "yarn cypress:ess --spec './cypress/e2e/!(investigations|explore|detection_response/rule_management)/**/*.cy.ts'", "cypress:run:cases:ess": "yarn cypress:ess --spec './cypress/e2e/explore/cases/*.cy.ts'", "cypress:ess": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel run --config-file ../../test/security_solution_cypress/cypress/cypress_ci.config.ts --ftr-config-file ../../test/security_solution_cypress/cli_config", + "cypress:rule_management:run:ess":"yarn cypress:ess --spec './cypress/e2e/detection_response/rule_management/!(prebuilt_rules)/**/*.cy.ts'", + "cypress:rule_management:prebuilt_rules:run:ess": "yarn cypress:ess --spec './cypress/e2e/detection_response/rule_management/prebuilt_rules/**/*.cy.ts'", "cypress:run:respops:ess": "yarn cypress:ess --spec './cypress/e2e/(detection_response|exceptions)/**/*.cy.ts'", "cypress:investigations:run:ess": "yarn cypress:ess --spec './cypress/e2e/investigations/**/*.cy.ts'", "cypress:explore:run:ess": "yarn cypress:ess --spec './cypress/e2e/explore/**/*.cy.ts'", @@ -21,16 +23,20 @@ "cypress:cloud:serverless": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider NODE_TLS_REJECT_UNAUTHORIZED=0 ../../../node_modules/.bin/cypress", "cypress:open:cloud:serverless": "yarn cypress:cloud:serverless open --config-file ./cypress/cypress_serverless.config.ts --env CLOUD_SERVERLESS=true", "cypress:open:serverless": "yarn cypress:serverless open --config-file ../../test/security_solution_cypress/cypress/cypress_serverless.config.ts --spec './cypress/e2e/**/*.cy.ts'", - "cypress:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'", + "cypress:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/!(investigations|explore|detection_response/rule_management)/**/*.cy.ts'", "cypress:run:cloud:serverless": "yarn cypress:cloud:serverless run --config-file ./cypress/cypress_ci_serverless.config.ts --env CLOUD_SERVERLESS=true", + "cypress:rule_management:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/detection_response/rule_management/!(prebuilt_rules)/**/*.cy.ts'", + "cypress:rule_management:prebuilt_rules:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/detection_response/rule_management/prebuilt_rules/**/*.cy.ts'", "cypress:investigations:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'", "cypress:explore:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/explore/**/*.cy.ts'", "cypress:changed-specs-only:serverless": "yarn cypress:serverless --changed-specs-only --env burn=5", "cypress:burn:serverless": "yarn cypress:serverless --env burn=2", "cypress:qa:serverless": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel_serverless --config-file ../../test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts", "cypress:open:qa:serverless": "yarn cypress:qa:serverless open", - "cypress:run:qa:serverless": "yarn cypress:qa:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'", + "cypress:run:qa:serverless": "yarn cypress:qa:serverless --spec './cypress/e2e/!(investigations|explore|detection_response/rule_management)/**/*.cy.ts'", "cypress:run:qa:serverless:investigations": "yarn cypress:qa:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'", - "cypress:run:qa:serverless:explore": "yarn cypress:qa:serverless --spec './cypress/e2e/explore/**/*.cy.ts'" + "cypress:run:qa:serverless:explore": "yarn cypress:qa:serverless --spec './cypress/e2e/explore/**/*.cy.ts'", + "cypress:run:qa:serverless:rule_management": "yarn cypress:qa:serverless --spec './cypress/e2e/detection_response/rule_management/!(prebuilt_rules)/**/*.cy.ts'", + "cypress:run:qa:serverless:rule_management:prebuilt_rules": "yarn cypress:qa:serverless --spec './cypress/e2e/detection_response/rule_management/prebuilt_rules/**/*.cy.ts'" } } \ No newline at end of file From 36c86fc532c1fec5fdd8583805862b35349ed280 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Tue, 28 Nov 2023 14:38:50 +0100 Subject: [PATCH 02/30] [cloud_security_posture_functional] fix functional tests (#171736) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Some async functions miss `await` that makes CI to fail https://buildkite.com/elastic/kibana-pull-request/builds/177811#018bf56a-125a-448d-b3bb-e9da9b2c512a ``` 2023-11-22 06:36:50 CEST | 44 passing (9.0m) -- | --   | 2023-11-22 06:36:50 CEST | 1 pending   | 2023-11-22 06:36:50 CEST |     | 2023-11-22 06:36:50 CEST | warn browser[SEVERE] ERROR FETCHING BROWSR LOGS: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.   | 2023-11-22 06:36:50 CEST | Unhandled Promise rejection detected:   | 2023-11-22 06:36:50 CEST |     | 2023-11-22 06:36:50 CEST | NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.   | 2023-11-22 06:36:50 CEST | at /var/lib/buildkite-agent/builds/kb-n2-4-spot-da643dd7dfd9a4eb/elastic/kibana-pull-request/kibana/node_modules/selenium-webdriver/lib/webdriver.js:776:9   | 2023-11-22 06:36:50 CEST | at Object.thenFinally [as finally] (/var/lib/buildkite-agent/builds/kb-n2-4-spot-da643dd7dfd9a4eb/elastic/kibana-pull-request/kibana/node_modules/selenium-webdriver/lib/promise.js:101:12)   | 2023-11-22 06:36:50 CEST | at processTicksAndRejections (node:internal/process/task_queues:95:5)   | 2023-11-22 06:36:50 CEST | at remote.ts:101:7   | 2023-11-22 06:36:50 CEST | at tryWebDriverCall (remote.ts:34:7)   | 2023-11-22 06:36:50 CEST | at remote.ts:100:5   | 2023-11-22 06:36:50 CEST | at lifecycle_phase.ts:76:11   | 2023-11-22 06:36:50 CEST | at async Promise.all (index 2)   | 2023-11-22 06:36:50 CEST | at LifecyclePhase.trigger (lifecycle_phase.ts:73:5)   | 2023-11-22 06:36:50 CEST | at FunctionalTestRunner.runHarness (functional_test_runner.ts:258:9)   | 2023-11-22 06:36:50 CEST | at FunctionalTestRunner.run (functional_test_runner.ts:48:12)   | 2023-11-22 06:36:50 CEST | at runFtr (run_ftr.ts:21:24)   | 2023-11-22 06:36:50 CEST | at run_tests.ts:116:11   | 2023-11-22 06:36:50 CEST | at withProcRunner (with_proc_runner.ts:29:5)   | 2023-11-22 06:36:50 CEST | at run_tests.ts:87:7   | 2023-11-22 06:36:50 CEST | at tooling_log.ts:84:18   | 2023-11-22 06:36:50 CEST | at runTests (run_tests.ts:64:5)   | 2023-11-22 06:36:50 CEST | at description (cli.ts:24:7)   | 2023-11-22 06:36:50 CEST | at run.ts:73:10   | 2023-11-22 06:36:50 CEST | at withProcRunner (with_proc_runner.ts:29:5)   | 2023-11-22 06:36:50 CEST | at run (run.ts:71:5) {   | 2023-11-22 06:36:50 CEST | remoteStacktrace: '' ``` --- .../pages/findings_old_data.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts b/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts index a8cda10482e2e..1269c85c1ecda 100644 --- a/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts +++ b/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts @@ -10,7 +10,7 @@ import Chance from 'chance'; import type { FtrProviderContext } from '../ftr_provider_context'; // eslint-disable-next-line import/no-default-export -export default function ({ getPageObjects, getService }: FtrProviderContext) { +export default function ({ getPageObjects }: FtrProviderContext) { const pageObjects = getPageObjects(['common', 'findings', 'header']); const chance = new Chance(); const hoursToMillisecond = (hours: number) => hours * 60 * 60 * 1000; @@ -77,7 +77,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { await findings.index.add(dataOldKspm); await findings.navigateToLatestFindingsPage(); - pageObjects.header.waitUntilLoadingHasFinished(); + await pageObjects.header.waitUntilLoadingHasFinished(); expect(await findings.isLatestFindingsTableThere()).to.be(false); }); it('returns no Findings CSPM', async () => { @@ -86,7 +86,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { await findings.index.add(dataOldCspm); await findings.navigateToLatestFindingsPage(); - pageObjects.header.waitUntilLoadingHasFinished(); + await pageObjects.header.waitUntilLoadingHasFinished(); expect(await findings.isLatestFindingsTableThere()).to.be(false); }); }); From 177dbd1da5900ceb8c3fbd9efa27362361963fff Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 28 Nov 2023 08:00:28 -0600 Subject: [PATCH 03/30] Revert "[Security Solution] Specific Cypress executions for `Rule Management` team (#171868)" This reverts commit 242cb6f1d5cf97766fe8a5697488877e2390befe. --- .../verify_es_serverless_image.yml | 26 ---------- .buildkite/pipelines/on_merge.yml | 48 ------------------- .buildkite/pipelines/pull_request/base.yml | 48 ------------------- .../security_solution_cypress.yml | 24 ---------- .../security_serverless_rule_management.sh | 16 ------- ...rverless_rule_management_prebuilt_rules.sh | 16 ------- .../security_solution_rule_management.sh | 16 ------- ...solution_rule_management_prebuilt_rules.sh | 16 ------- .github/CODEOWNERS | 2 + .../cypress/README.md | 23 +++------ .../install_update_authorization.cy.ts | 12 ++--- .../install_update_error_handling.cy.ts | 14 +++--- .../prebuilt_rules/install_via_fleet.cy.ts | 14 +++--- .../prebuilt_rules/install_workflow.cy.ts | 20 ++++---- .../prebuilt_rules/management.cy.ts | 21 ++++---- .../prebuilt_rules/notifications.cy.ts | 19 ++++---- .../prebuilt_rules_preview.cy.ts | 24 +++++----- .../prebuilt_rules/update_workflow.ts | 18 +++---- .../rule_details/common_flows.cy.ts | 26 +++++----- .../rule_details/esql_rule.cy.ts | 16 +++---- .../security_solution_cypress/package.json | 14 ++---- 21 files changed, 101 insertions(+), 332 deletions(-) delete mode 100644 .buildkite/scripts/steps/functional/security_serverless_rule_management.sh delete mode 100644 .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh delete mode 100644 .buildkite/scripts/steps/functional/security_solution_rule_management.sh delete mode 100644 .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{rule_management => }/prebuilt_rules/install_update_authorization.cy.ts (94%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{rule_management => }/prebuilt_rules/install_update_error_handling.cy.ts (94%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{rule_management => }/prebuilt_rules/install_via_fleet.cy.ts (90%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{rule_management => }/prebuilt_rules/install_workflow.cy.ts (85%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{rule_management => }/prebuilt_rules/management.cy.ts (91%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{rule_management => }/prebuilt_rules/notifications.cy.ts (92%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{rule_management => }/prebuilt_rules/prebuilt_rules_preview.cy.ts (97%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{rule_management => }/prebuilt_rules/update_workflow.ts (88%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{rule_management => }/rule_details/common_flows.cy.ts (88%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{rule_management => }/rule_details/esql_rule.cy.ts (69%) diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml index 8d1b778b67983..8e64513b14900 100644 --- a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml +++ b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml @@ -95,32 +95,6 @@ steps: - exit_status: '*' limit: 1 - - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh - label: 'Serverless Rule Management - Security Solution Cypress Tests' - if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'" - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 60 - parallelism: 8 - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh - label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests' - if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'" - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 60 - parallelism: 6 - retry: - automatic: - - exit_status: '*' - limit: 1 - - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh label: 'Defend Workflows Cypress Tests on Serverless' if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'" diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index 8256eb2395633..8b00db428a713 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -115,54 +115,6 @@ steps: - exit_status: '*' limit: 1 - - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh - label: 'Serverless Rule Management - Security Solution Cypress Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 60 - parallelism: 8 - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh - label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 60 - parallelism: 6 - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh - label: 'Rule Management - Security Solution Cypress Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 60 - parallelism: 8 - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh - label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 60 - parallelism: 6 - retry: - automatic: - - exit_status: '*' - limit: 1 - - command: .buildkite/scripts/steps/functional/security_solution.sh label: 'Security Solution Cypress Tests' agents: diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index 8238afbee4fd2..49215bbd00f11 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -93,30 +93,6 @@ steps: - exit_status: '*' limit: 1 - - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh - label: 'Serverless Rule Management - Security Solution Cypress Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 60 - parallelism: 8 - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh - label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 60 - parallelism: 6 - retry: - automatic: - - exit_status: '*' - limit: 1 - - command: .buildkite/scripts/steps/functional/security_solution.sh label: 'Security Solution Cypress Tests' agents: @@ -141,30 +117,6 @@ steps: - exit_status: '*' limit: 1 - - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh - label: 'Rule Management - Security Solution Cypress Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 60 - parallelism: 8 - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh - label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 60 - parallelism: 6 - retry: - automatic: - - exit_status: '*' - limit: 1 - - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh label: 'Investigations - Security Solution Cypress Tests' agents: diff --git a/.buildkite/pipelines/security_solution/security_solution_cypress.yml b/.buildkite/pipelines/security_solution/security_solution_cypress.yml index 77e7fea574352..247505ef1c85a 100644 --- a/.buildkite/pipelines/security_solution/security_solution_cypress.yml +++ b/.buildkite/pipelines/security_solution/security_solution_cypress.yml @@ -30,30 +30,6 @@ steps: # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. timeout_in_minutes: 300 parallelism: 8 - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management - label: 'Serverless MKI QA Rule Management - Security Solution Cypress Tests' - agents: - queue: n2-4-spot - # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. - timeout_in_minutes: 300 - parallelism: 8 - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management:prebuilt_rules - label: 'Serverless MKI QA Rule Management - Prebuilt Rules - Security Solution Cypress Tests' - agents: - queue: n2-4-spot - # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. - timeout_in_minutes: 300 - parallelism: 6 retry: automatic: - exit_status: '*' diff --git a/.buildkite/scripts/steps/functional/security_serverless_rule_management.sh b/.buildkite/scripts/steps/functional/security_serverless_rule_management.sh deleted file mode 100644 index 5d360e0db4f29..0000000000000 --- a/.buildkite/scripts/steps/functional/security_serverless_rule_management.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/scripts/steps/functional/common.sh -source .buildkite/scripts/steps/functional/common_cypress.sh - -export JOB=kibana-security-solution-chrome -export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} - -echo "--- Rule Management Cypress Tests on Serverless" - -cd x-pack/test/security_solution_cypress - -set +e -yarn cypress:rule_management:run:serverless; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh b/.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh deleted file mode 100644 index bc7dc3269d8cb..0000000000000 --- a/.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/scripts/steps/functional/common.sh -source .buildkite/scripts/steps/functional/common_cypress.sh - -export JOB=kibana-security-solution-chrome -export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} - -echo "--- Rule Management - Prebuilt Rules - Cypress Tests on Serverless" - -cd x-pack/test/security_solution_cypress - -set +e -yarn cypress:rule_management:prebuilt_rules:run:serverless; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/steps/functional/security_solution_rule_management.sh b/.buildkite/scripts/steps/functional/security_solution_rule_management.sh deleted file mode 100644 index 847cb42896cf1..0000000000000 --- a/.buildkite/scripts/steps/functional/security_solution_rule_management.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/scripts/steps/functional/common.sh -source .buildkite/scripts/steps/functional/common_cypress.sh - -export JOB=kibana-security-solution-chrome -export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} - -echo "--- Rule Management - Security Solution Cypress Tests" - -cd x-pack/test/security_solution_cypress - -set +e -yarn cypress:rule_management:run:ess; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh b/.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh deleted file mode 100644 index d8b19ad3363b5..0000000000000 --- a/.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/scripts/steps/functional/common.sh -source .buildkite/scripts/steps/functional/common_cypress.sh - -export JOB=kibana-security-solution-chrome -export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} - -echo "--- Rule Management - Prebuilt Rules - Security Solution Cypress Tests" - -cd x-pack/test/security_solution_cypress - -set +e -yarn cypress:rule_management:prebuilt_rules:run:ess; status=$?; yarn junit:merge || :; exit $status diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7d075295240c9..89e152a2fe40f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1322,7 +1322,9 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout @elastic/ /x-pack/plugins/security_solution/common/api/detection_engine/rule_monitoring @elastic/security-detection-rule-management /x-pack/plugins/security_solution/common/detection_engine/rule_management @elastic/security-detection-rule-management +/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules @elastic/security-detection-rule-management /x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management @elastic/security-detection-rule-management +/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details @elastic/security-detection-rule-management /x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules @elastic/security-detection-rule-management /x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/rule_management @elastic/security-detection-rule-management /x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules @elastic/security-detection-rule-management diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md index 88786aed7ff56..8940d6c86e73e 100644 --- a/x-pack/test/security_solution_cypress/cypress/README.md +++ b/x-pack/test/security_solution_cypress/cypress/README.md @@ -62,25 +62,19 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio | cypress | Runs the default Cypress command | | cypress:open:ess | Opens the Cypress UI with all tests in the `e2e` directory. This also runs a local kibana and ES instance. The kibana instance will reload when you make code changes. This is the recommended way to debug and develop tests. | | cypress:open:serverless | Opens the Cypress UI with all tests in the `e2e` directory. This also runs a mocked serverless environment. The kibana instance will reload when you make code changes. This is the recommended way to debug and develop tests. | -| cypress:run:ess | Runs all tests tagged as ESS placed in the `e2e` directory excluding `investigations`,`explore` and `detection_response/rule_management` directories in headless mode | +| cypress:run:ess | Runs all tests tagged as ESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode | | cypress:run:cases:ess | Runs all tests under `explore/cases` in the `e2e` directory related to the Cases area team in headless mode | | cypress:ess | Runs all ESS tests with the specified configuration in headless mode and produces a report using `cypress-multi-reporters` | -| cypress:rule_management:run:ess | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | -| cypress:rule_management:prebuilt_rules:run:ess | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | | cypress:run:respops:ess | Runs all tests related to the Response Ops area team, specifically tests in `detection_alerts`, `detection_rules`, and `exceptions` directories in headless mode | -| cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations`, `explore` and `rule_management` directories in headless mode | -| cypress:rule_management:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | -| cypress:rule_management:prebuilt_rules:run:serverless | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | -| cypress:investigations:run:ess | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode | +| cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations` and `explore` directories in headless mode | +| cypress:investigations:run:ess | Runs all tests tagged as ESS in the `e2e/investigations` directory in headless mode | | cypress:explore:run:ess | Runs all tests tagged as ESS in the `e2e/explore` directory in headless mode | | cypress:investigations:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode | | cypress:explore:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode | | cypress:open:qa:serverless | Opens the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment. The kibana instance will reload when you make code changes. This is the recommended way to debug tests in QA. Follow the readme in order to learn about the known limitations. | -| cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations`, `explore` and `rule_management` directories in headless mode using the QA environment and real MKI projects.| +| cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode using the QA environment and real MKI projects.| | cypress:run:qa:serverless:explore | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode using the QA environment and real MKI prorjects. | | cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. | -| cypress:run:qa:serverless:rule_management | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` directory, excluding `e2e/detection_response/rule_management/prebuilt_rules` in headless mode using the QA environment and reak MKI projects. | -| cypress:run:qa:serverless:rule_management:prebuilt_rules | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode using the QA environment and reak MKI projects. | | junit:merge | Merges individual test reports into a single report and moves the report to the `junit` directory | Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. @@ -100,7 +94,7 @@ Below you can find the folder structure used on our Cypress tests. Cypress convention starting version 10 (previously known as integration). Contains the specs that are going to be executed. -### Area teams folders +### e2e/explore and e2e/investigations These directories contain tests which are run in their own Buildkite pipeline. @@ -109,8 +103,7 @@ If you belong to one of the teams listed in the table, please add new e2e specs | Directory | Area team | | -- | -- | | `e2e/explore` | Threat Hunting Explore | -| `e2e/investigations` | Threat Hunting Investigations | -| `e2e/detection_response/rule_management` | Detection Rule Management | +| `e2e/investigations | Threat Hunting Investigations | ### fixtures/ @@ -210,8 +203,6 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio | cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations` and `explore` directories in headless mode | | cypress:investigations:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode | | cypress:explore:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode | -| cypress:rule_management:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | -| cypress:rule_management:prebuilt_rules:run:serverless | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. @@ -257,8 +248,6 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio | cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode using the QA environment and real MKI projects.| | cypress:run:qa:serverless:explore | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode using the QA environment and real MKI prorjects. | | cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. | -| cypress:run:qa:serverless:rule_management | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` directory, excluding `e2e/detection_response/rule_management/prebuilt_rules` in headless mode using the QA environment and reak MKI projects. | -| cypress:run:qa:serverless:rule_management:prebuilt_rules | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode using the QA environment and reak MKI projects. | Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts similarity index 94% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts index 29e650dd4de66..e0078dd54e7ea 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts @@ -12,14 +12,14 @@ import { } from '@kbn/security-solution-plugin/common/constants'; import { ROLES } from '@kbn/security-solution-plugin/common/test'; -import { createRuleAssetSavedObject } from '../../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../helpers/rules'; import { createAndInstallMockedPrebuiltRules, installPrebuiltRuleAssets, preventPrebuiltRulesPackageInstallation, -} from '../../../../tasks/api_calls/prebuilt_rules'; -import { visit } from '../../../../tasks/navigation'; -import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; +} from '../../../tasks/api_calls/prebuilt_rules'; +import { visit } from '../../../tasks/navigation'; +import { RULES_MANAGEMENT_URL } from '../../../urls/rules_management'; import { ADD_ELASTIC_RULES_BTN, getInstallSingleRuleButtonByRuleId, @@ -31,8 +31,8 @@ import { RULES_UPDATES_TAB, RULE_CHECKBOX, UPGRADE_ALL_RULES_BUTTON, -} from '../../../../screens/alerts_detection_rules'; -import { login } from '../../../../tasks/login'; +} from '../../../screens/alerts_detection_rules'; +import { login } from '../../../tasks/login'; // Rule to test update const RULE_1_ID = 'rule_1'; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts similarity index 94% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts index db84d92e4ddb6..7e288910ccb60 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../helpers/rules'; import { getInstallSingleRuleButtonByRuleId, getUpgradeSingleRuleButtonByRuleId, @@ -14,14 +14,14 @@ import { SELECT_ALL_RULES_ON_PAGE_CHECKBOX, UPGRADE_ALL_RULES_BUTTON, UPGRADE_SELECTED_RULES_BUTTON, -} from '../../../../screens/alerts_detection_rules'; -import { selectRulesByName } from '../../../../tasks/alerts_detection_rules'; +} from '../../../screens/alerts_detection_rules'; +import { selectRulesByName } from '../../../tasks/alerts_detection_rules'; import { installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, preventPrebuiltRulesPackageInstallation, -} from '../../../../tasks/api_calls/prebuilt_rules'; -import { login } from '../../../../tasks/login'; +} from '../../../tasks/api_calls/prebuilt_rules'; +import { login } from '../../../tasks/login'; import { clickAddElasticRulesButton, assertInstallationRequestIsComplete, @@ -33,8 +33,8 @@ import { assertRulesPresentInAddPrebuiltRulesTable, assertRuleUpgradeFailureToastShown, assertRulesPresentInRuleUpdatesTable, -} from '../../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../../tasks/rules_management'; +} from '../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../tasks/rules_management'; describe( 'Detection rules, Prebuilt Rules Installation and Update - Error handling', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_via_fleet.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_via_fleet.cy.ts similarity index 90% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_via_fleet.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_via_fleet.cy.ts index 762e79bb27003..6da3d58c0530d 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_via_fleet.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_via_fleet.cy.ts @@ -8,13 +8,13 @@ import type { BulkInstallPackageInfo } from '@kbn/fleet-plugin/common'; import type { Rule } from '@kbn/security-solution-plugin/public/detection_engine/rule_management/logic/types'; -import { resetRulesTableState } from '../../../../tasks/common'; -import { INSTALL_ALL_RULES_BUTTON, TOASTER } from '../../../../screens/alerts_detection_rules'; -import { getRuleAssets } from '../../../../tasks/api_calls/prebuilt_rules'; -import { login } from '../../../../tasks/login'; -import { clickAddElasticRulesButton } from '../../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../../tasks/rules_management'; -import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; +import { resetRulesTableState } from '../../../tasks/common'; +import { INSTALL_ALL_RULES_BUTTON, TOASTER } from '../../../screens/alerts_detection_rules'; +import { getRuleAssets } from '../../../tasks/api_calls/prebuilt_rules'; +import { login } from '../../../tasks/login'; +import { clickAddElasticRulesButton } from '../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../tasks/rules_management'; +import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts similarity index 85% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts index 523d0ec0ad4e0..ec4615bcf59e4 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { resetRulesTableState } from '../../../../tasks/common'; -import { createRuleAssetSavedObject } from '../../../../helpers/rules'; +import { resetRulesTableState } from '../../../tasks/common'; +import { createRuleAssetSavedObject } from '../../../helpers/rules'; import { getInstallSingleRuleButtonByRuleId, GO_BACK_TO_RULES_TABLE_BUTTON, @@ -16,19 +16,19 @@ import { RULE_CHECKBOX, SELECT_ALL_RULES_ON_PAGE_CHECKBOX, TOASTER, -} from '../../../../screens/alerts_detection_rules'; -import { selectRulesByName } from '../../../../tasks/alerts_detection_rules'; -import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../../screens/breadcrumbs'; -import { installPrebuiltRuleAssets } from '../../../../tasks/api_calls/prebuilt_rules'; -import { login } from '../../../../tasks/login'; +} from '../../../screens/alerts_detection_rules'; +import { selectRulesByName } from '../../../tasks/alerts_detection_rules'; +import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../screens/breadcrumbs'; +import { installPrebuiltRuleAssets } from '../../../tasks/api_calls/prebuilt_rules'; +import { login } from '../../../tasks/login'; import { assertInstallationRequestIsComplete, assertRuleInstallationSuccessToastShown, assertRulesPresentInInstalledRulesTable, clickAddElasticRulesButton, -} from '../../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../../tasks/rules_management'; -import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; +} from '../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../tasks/rules_management'; +import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts similarity index 91% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts index 15e020b5e0663..f3101f513915f 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../helpers/rules'; import { COLLAPSED_ACTION_BTN, ELASTIC_RULES_BTN, @@ -15,7 +15,7 @@ import { RULE_SWITCH, SELECT_ALL_RULES_ON_PAGE_CHECKBOX, INSTALL_ALL_RULES_BUTTON, -} from '../../../../screens/alerts_detection_rules'; +} from '../../../screens/alerts_detection_rules'; import { deleteFirstRule, disableAutoRefresh, @@ -24,24 +24,21 @@ import { selectRulesByName, waitForPrebuiltDetectionRulesToBeLoaded, waitForRuleToUpdate, -} from '../../../../tasks/alerts_detection_rules'; +} from '../../../tasks/alerts_detection_rules'; import { deleteSelectedRules, disableSelectedRules, enableSelectedRules, -} from '../../../../tasks/rules_bulk_actions'; +} from '../../../tasks/rules_bulk_actions'; import { createAndInstallMockedPrebuiltRules, getAvailablePrebuiltRulesCount, preventPrebuiltRulesPackageInstallation, -} from '../../../../tasks/api_calls/prebuilt_rules'; -import { - deleteAlertsAndRules, - deletePrebuiltRulesAssets, -} from '../../../../tasks/api_calls/common'; -import { login } from '../../../../tasks/login'; -import { visit } from '../../../../tasks/navigation'; -import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; +} from '../../../tasks/api_calls/prebuilt_rules'; +import { deleteAlertsAndRules, deletePrebuiltRulesAssets } from '../../../tasks/api_calls/common'; +import { login } from '../../../tasks/login'; +import { visit } from '../../../tasks/navigation'; +import { RULES_MANAGEMENT_URL } from '../../../urls/rules_management'; const rules = Array.from(Array(5)).map((_, i) => { return createRuleAssetSavedObject({ diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts similarity index 92% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts index 4812efc740ae2..92bf9e7f1471c 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts @@ -5,25 +5,22 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../helpers/rules'; import { ADD_ELASTIC_RULES_BTN, ADD_ELASTIC_RULES_EMPTY_PROMPT_BTN, RULES_UPDATES_TAB, -} from '../../../../screens/alerts_detection_rules'; -import { deleteFirstRule } from '../../../../tasks/alerts_detection_rules'; -import { - deleteAlertsAndRules, - deletePrebuiltRulesAssets, -} from '../../../../tasks/api_calls/common'; +} from '../../../screens/alerts_detection_rules'; +import { deleteFirstRule } from '../../../tasks/alerts_detection_rules'; +import { deleteAlertsAndRules, deletePrebuiltRulesAssets } from '../../../tasks/api_calls/common'; import { installAllPrebuiltRulesRequest, installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, -} from '../../../../tasks/api_calls/prebuilt_rules'; -import { resetRulesTableState } from '../../../../tasks/common'; -import { login } from '../../../../tasks/login'; -import { visitRulesManagementTable } from '../../../../tasks/rules_management'; +} from '../../../tasks/api_calls/prebuilt_rules'; +import { resetRulesTableState } from '../../../tasks/common'; +import { login } from '../../../tasks/login'; +import { visitRulesManagementTable } from '../../../tasks/rules_management'; const RULE_1 = createRuleAssetSavedObject({ name: 'Test rule 1', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_preview.cy.ts similarity index 97% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_preview.cy.ts index 81f37b7760df2..6deeb6f5202c0 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_preview.cy.ts @@ -12,22 +12,22 @@ import type { PrebuiltRuleAsset } from '@kbn/security-solution-plugin/server/lib import type { Threshold } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; import { AlertSuppression } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; -import { createRuleAssetSavedObject } from '../../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../helpers/rules'; import { INSTALL_PREBUILT_RULE_BUTTON, INSTALL_PREBUILT_RULE_PREVIEW, UPDATE_PREBUILT_RULE_PREVIEW, UPDATE_PREBUILT_RULE_BUTTON, -} from '../../../../screens/alerts_detection_rules'; -import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../../screens/breadcrumbs'; +} from '../../../screens/alerts_detection_rules'; +import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../screens/breadcrumbs'; import { installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, -} from '../../../../tasks/api_calls/prebuilt_rules'; -import { createSavedQuery, deleteSavedQueries } from '../../../../tasks/api_calls/saved_queries'; -import { fetchMachineLearningModules } from '../../../../tasks/api_calls/machine_learning'; -import { resetRulesTableState } from '../../../../tasks/common'; -import { login } from '../../../../tasks/login'; +} from '../../../tasks/api_calls/prebuilt_rules'; +import { createSavedQuery, deleteSavedQueries } from '../../../tasks/api_calls/saved_queries'; +import { fetchMachineLearningModules } from '../../../tasks/api_calls/machine_learning'; +import { resetRulesTableState } from '../../../tasks/common'; +import { login } from '../../../tasks/login'; import { assertRuleInstallationSuccessToastShown, assertRulesNotPresentInAddPrebuiltRulesTable, @@ -36,7 +36,7 @@ import { assertRuleUpgradeSuccessToastShown, clickAddElasticRulesButton, clickRuleUpdatesTab, -} from '../../../../tasks/prebuilt_rules'; +} from '../../../tasks/prebuilt_rules'; import { assertAlertSuppressionPropertiesShown, assertCommonPropertiesShown, @@ -55,13 +55,13 @@ import { closeRulePreview, openRuleInstallPreview, openRuleUpdatePreview, -} from '../../../../tasks/prebuilt_rules_preview'; -import { visitRulesManagementTable } from '../../../../tasks/rules_management'; +} from '../../../tasks/prebuilt_rules_preview'; +import { visitRulesManagementTable } from '../../../tasks/rules_management'; import { deleteAlertsAndRules, deleteDataView, postDataView, -} from '../../../../tasks/api_calls/common'; +} from '../../../tasks/api_calls/common'; const TEST_ENV_TAGS = ['@ess', '@serverless']; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/update_workflow.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts similarity index 88% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/update_workflow.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts index d858280dd5294..edeb8ac98623b 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/update_workflow.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../helpers/rules'; import { getUpgradeSingleRuleButtonByRuleId, NO_RULES_AVAILABLE_FOR_UPGRADE_MESSAGE, @@ -13,22 +13,22 @@ import { SELECT_ALL_RULES_ON_PAGE_CHECKBOX, UPGRADE_ALL_RULES_BUTTON, UPGRADE_SELECTED_RULES_BUTTON, -} from '../../../../screens/alerts_detection_rules'; -import { selectRulesByName } from '../../../../tasks/alerts_detection_rules'; -import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; +} from '../../../screens/alerts_detection_rules'; +import { selectRulesByName } from '../../../tasks/alerts_detection_rules'; +import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; import { installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, -} from '../../../../tasks/api_calls/prebuilt_rules'; -import { resetRulesTableState } from '../../../../tasks/common'; -import { login } from '../../../../tasks/login'; +} from '../../../tasks/api_calls/prebuilt_rules'; +import { resetRulesTableState } from '../../../tasks/common'; +import { login } from '../../../tasks/login'; import { assertRulesNotPresentInRuleUpdatesTable, assertRuleUpgradeSuccessToastShown, assertUpgradeRequestIsComplete, clickRuleUpdatesTab, -} from '../../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../../tasks/rules_management'; +} from '../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../tasks/rules_management'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/common_flows.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts similarity index 88% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/common_flows.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts index 0610786fc1b89..f5704122d9e33 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/common_flows.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts @@ -5,17 +5,17 @@ * 2.0. */ -import { deleteRuleFromDetailsPage } from '../../../../tasks/alerts_detection_rules'; +import { deleteRuleFromDetailsPage } from '../../../tasks/alerts_detection_rules'; import { CUSTOM_RULES_BTN, RULES_MANAGEMENT_TABLE, RULES_ROW, -} from '../../../../screens/alerts_detection_rules'; -import { createRule } from '../../../../tasks/api_calls/rules'; -import { getDetails } from '../../../../tasks/rule_details'; -import { ruleFields } from '../../../../data/detection_engine'; -import { getTimeline } from '../../../../objects/timeline'; -import { getExistingRule, getNewRule } from '../../../../objects/rule'; +} from '../../../screens/alerts_detection_rules'; +import { createRule } from '../../../tasks/api_calls/rules'; +import { getDetails } from '../../../tasks/rule_details'; +import { ruleFields } from '../../../data/detection_engine'; +import { getTimeline } from '../../../objects/timeline'; +import { getExistingRule, getNewRule } from '../../../objects/rule'; import { ABOUT_DETAILS, @@ -42,13 +42,13 @@ import { THREAT_TACTIC, THREAT_TECHNIQUE, TIMELINE_TEMPLATE_DETAILS, -} from '../../../../screens/rule_details'; +} from '../../../screens/rule_details'; -import { createTimeline } from '../../../../tasks/api_calls/timelines'; -import { deleteAlertsAndRules, deleteConnectors } from '../../../../tasks/api_calls/common'; -import { login } from '../../../../tasks/login'; -import { visit } from '../../../../tasks/navigation'; -import { ruleDetailsUrl } from '../../../../urls/rule_details'; +import { createTimeline } from '../../../tasks/api_calls/timelines'; +import { deleteAlertsAndRules, deleteConnectors } from '../../../tasks/api_calls/common'; +import { login } from '../../../tasks/login'; +import { visit } from '../../../tasks/navigation'; +import { ruleDetailsUrl } from '../../../urls/rule_details'; // This test is meant to test all common aspects of the rule details page that should function // the same regardless of rule type. For any rule type specific functionalities, please include diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/esql_rule.cy.ts similarity index 69% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/esql_rule.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/esql_rule.cy.ts index c59b7db55c743..7d1419e911e33 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/esql_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/esql_rule.cy.ts @@ -5,24 +5,24 @@ * 2.0. */ -import { getEsqlRule } from '../../../../objects/rule'; +import { getEsqlRule } from '../../../objects/rule'; import { ESQL_QUERY_DETAILS, DEFINITION_DETAILS, RULE_NAME_HEADER, RULE_TYPE_DETAILS, -} from '../../../../screens/rule_details'; +} from '../../../screens/rule_details'; -import { createRule } from '../../../../tasks/api_calls/rules'; +import { createRule } from '../../../tasks/api_calls/rules'; -import { getDetails } from '../../../../tasks/rule_details'; -import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; +import { getDetails } from '../../../tasks/rule_details'; +import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; -import { login } from '../../../../tasks/login'; -import { visit } from '../../../../tasks/navigation'; +import { login } from '../../../tasks/login'; +import { visit } from '../../../tasks/navigation'; -import { ruleDetailsUrl } from '../../../../urls/rule_details'; +import { ruleDetailsUrl } from '../../../urls/rule_details'; describe('Detection ES|QL rules, details view', { tags: ['@ess'] }, () => { const rule = getEsqlRule(); diff --git a/x-pack/test/security_solution_cypress/package.json b/x-pack/test/security_solution_cypress/package.json index e1f552fdba9de..e43f32a447575 100644 --- a/x-pack/test/security_solution_cypress/package.json +++ b/x-pack/test/security_solution_cypress/package.json @@ -7,11 +7,9 @@ "scripts": { "cypress": "NODE_OPTIONS=--openssl-legacy-provider ../../../node_modules/.bin/cypress", "cypress:open:ess": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel open --spec './cypress/e2e/**/*.cy.ts' --config-file ../../test/security_solution_cypress/cypress/cypress.config.ts --ftr-config-file ../../test/security_solution_cypress/cli_config", - "cypress:run:ess": "yarn cypress:ess --spec './cypress/e2e/!(investigations|explore|detection_response/rule_management)/**/*.cy.ts'", + "cypress:run:ess": "yarn cypress:ess --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'", "cypress:run:cases:ess": "yarn cypress:ess --spec './cypress/e2e/explore/cases/*.cy.ts'", "cypress:ess": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel run --config-file ../../test/security_solution_cypress/cypress/cypress_ci.config.ts --ftr-config-file ../../test/security_solution_cypress/cli_config", - "cypress:rule_management:run:ess":"yarn cypress:ess --spec './cypress/e2e/detection_response/rule_management/!(prebuilt_rules)/**/*.cy.ts'", - "cypress:rule_management:prebuilt_rules:run:ess": "yarn cypress:ess --spec './cypress/e2e/detection_response/rule_management/prebuilt_rules/**/*.cy.ts'", "cypress:run:respops:ess": "yarn cypress:ess --spec './cypress/e2e/(detection_response|exceptions)/**/*.cy.ts'", "cypress:investigations:run:ess": "yarn cypress:ess --spec './cypress/e2e/investigations/**/*.cy.ts'", "cypress:explore:run:ess": "yarn cypress:ess --spec './cypress/e2e/explore/**/*.cy.ts'", @@ -23,20 +21,16 @@ "cypress:cloud:serverless": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider NODE_TLS_REJECT_UNAUTHORIZED=0 ../../../node_modules/.bin/cypress", "cypress:open:cloud:serverless": "yarn cypress:cloud:serverless open --config-file ./cypress/cypress_serverless.config.ts --env CLOUD_SERVERLESS=true", "cypress:open:serverless": "yarn cypress:serverless open --config-file ../../test/security_solution_cypress/cypress/cypress_serverless.config.ts --spec './cypress/e2e/**/*.cy.ts'", - "cypress:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/!(investigations|explore|detection_response/rule_management)/**/*.cy.ts'", + "cypress:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'", "cypress:run:cloud:serverless": "yarn cypress:cloud:serverless run --config-file ./cypress/cypress_ci_serverless.config.ts --env CLOUD_SERVERLESS=true", - "cypress:rule_management:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/detection_response/rule_management/!(prebuilt_rules)/**/*.cy.ts'", - "cypress:rule_management:prebuilt_rules:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/detection_response/rule_management/prebuilt_rules/**/*.cy.ts'", "cypress:investigations:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'", "cypress:explore:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/explore/**/*.cy.ts'", "cypress:changed-specs-only:serverless": "yarn cypress:serverless --changed-specs-only --env burn=5", "cypress:burn:serverless": "yarn cypress:serverless --env burn=2", "cypress:qa:serverless": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel_serverless --config-file ../../test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts", "cypress:open:qa:serverless": "yarn cypress:qa:serverless open", - "cypress:run:qa:serverless": "yarn cypress:qa:serverless --spec './cypress/e2e/!(investigations|explore|detection_response/rule_management)/**/*.cy.ts'", + "cypress:run:qa:serverless": "yarn cypress:qa:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'", "cypress:run:qa:serverless:investigations": "yarn cypress:qa:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'", - "cypress:run:qa:serverless:explore": "yarn cypress:qa:serverless --spec './cypress/e2e/explore/**/*.cy.ts'", - "cypress:run:qa:serverless:rule_management": "yarn cypress:qa:serverless --spec './cypress/e2e/detection_response/rule_management/!(prebuilt_rules)/**/*.cy.ts'", - "cypress:run:qa:serverless:rule_management:prebuilt_rules": "yarn cypress:qa:serverless --spec './cypress/e2e/detection_response/rule_management/prebuilt_rules/**/*.cy.ts'" + "cypress:run:qa:serverless:explore": "yarn cypress:qa:serverless --spec './cypress/e2e/explore/**/*.cy.ts'" } } \ No newline at end of file From a6582337e1c8b7f8f335694b92f791cb274dc519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20=C3=81brah=C3=A1m?= Date: Tue, 28 Nov 2023 15:11:24 +0100 Subject: [PATCH 04/30] [Defend Workflows][8.12 port] Unblock fleet setup when cannot decrypt uninstall tokens (#172058) ## Summary This PR is the `8.12` port of: - #171998 The original PR was opened to `8.11` to make it faster to include it in `8.12.2`. Now this PR is meant to port the changes to `main`, so: - we can build upon it, - and can easily backport any further changes to `8.11.x` > [!Important] > The changes cannot be tested on `main` because they are hidden by other behaviours (namely the retry logic for reading Message SIgning key) that weren't part of `8.11`. Those behaviours will be also adapted in follow up PRs. --- x-pack/plugins/fleet/server/mocks/index.ts | 2 + .../server/services/agent_policy.test.ts | 43 ++++++++-- .../fleet/server/services/agent_policy.ts | 15 ++++ .../uninstall_token_service/index.test.ts | 75 +++++++++++++++++ .../security/uninstall_token_service/index.ts | 80 +++++++++++-------- x-pack/plugins/fleet/server/services/setup.ts | 16 +++- 6 files changed, 189 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 2716fd82b6811..ec8ada164623d 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -201,5 +201,7 @@ export function createUninstallTokenServiceMock(): UninstallTokenServiceInterfac generateTokensForPolicyIds: jest.fn(), generateTokensForAllPolicies: jest.fn(), encryptTokens: jest.fn(), + checkTokenValidityForAllPolicies: jest.fn(), + checkTokenValidityForPolicy: jest.fn(), }; } diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index 710ac46b94592..b6950ba672817 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -32,6 +32,7 @@ import { getFullAgentPolicy } from './agent_policies'; import * as outputsHelpers from './agent_policies/outputs_helpers'; import { auditLoggingService } from './audit_logging'; import { licenseService } from './license'; +import type { UninstallTokenServiceInterface } from './security/uninstall_token_service'; function getSavedObjectMock(agentPolicyAttributes: any) { const mock = savedObjectsClientMock.create(); @@ -182,13 +183,13 @@ describe('agent policy', () => { }); }); - it('should throw FleetUnauthorizedError if is_protected=true with insufficient license', () => { + it('should throw FleetUnauthorizedError if is_protected=true with insufficient license', async () => { jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); const soClient = getAgentPolicyCreateMock(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - expect( + await expect( agentPolicyService.create(soClient, esClient, { name: 'test', namespace: 'default', @@ -199,13 +200,13 @@ describe('agent policy', () => { ); }); - it('should not throw FleetUnauthorizedError if is_protected=false with insufficient license', () => { + it('should not throw FleetUnauthorizedError if is_protected=false with insufficient license', async () => { jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); const soClient = getAgentPolicyCreateMock(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - expect( + await expect( agentPolicyService.create(soClient, esClient, { name: 'test', namespace: 'default', @@ -619,7 +620,7 @@ describe('agent policy', () => { }); }); - it('should throw FleetUnauthorizedError if is_protected=true with insufficient license', () => { + it('should throw FleetUnauthorizedError if is_protected=true with insufficient license', async () => { jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); const soClient = getAgentPolicyCreateMock(); @@ -632,7 +633,7 @@ describe('agent policy', () => { references: [], }); - expect( + await expect( agentPolicyService.update(soClient, esClient, 'test-id', { name: 'test', namespace: 'default', @@ -643,7 +644,7 @@ describe('agent policy', () => { ); }); - it('should not throw FleetUnauthorizedError if is_protected=false with insufficient license', () => { + it('should not throw FleetUnauthorizedError if is_protected=false with insufficient license', async () => { jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); const soClient = getAgentPolicyCreateMock(); @@ -656,7 +657,7 @@ describe('agent policy', () => { references: [], }); - expect( + await expect( agentPolicyService.update(soClient, esClient, 'test-id', { name: 'test', namespace: 'default', @@ -665,6 +666,32 @@ describe('agent policy', () => { new FleetUnauthorizedError('Tamper protection requires Platinum license') ); }); + + it('should throw Error if is_protected=true with invalid uninstall token', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + + mockedAppContextService.getUninstallTokenService.mockReturnValueOnce({ + checkTokenValidityForPolicy: jest.fn().mockRejectedValueOnce(new Error('reason')), + } as unknown as UninstallTokenServiceInterface); + + const soClient = getAgentPolicyCreateMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + soClient.get.mockResolvedValue({ + attributes: {}, + id: 'test-id', + type: 'mocked', + references: [], + }); + + await expect( + agentPolicyService.update(soClient, esClient, 'test-id', { + name: 'test', + namespace: 'default', + is_protected: true, + }) + ).rejects.toThrowError(new Error('Cannot enable Agent Tamper Protection: reason')); + }); }); describe('deployPolicy', () => { diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 8673bd6ad91a9..5e8c897d5611a 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -512,6 +512,7 @@ class AgentPolicyService { } this.checkTamperProtectionLicense(agentPolicy); + await this.checkForValidUninstallToken(agentPolicy, id); const logger = appContextService.getLogger(); @@ -1212,6 +1213,20 @@ class AgentPolicyService { throw new FleetUnauthorizedError('Tamper protection requires Platinum license'); } } + private async checkForValidUninstallToken( + agentPolicy: { is_protected?: boolean }, + policyId: string + ): Promise { + if (agentPolicy?.is_protected) { + const uninstallTokenService = appContextService.getUninstallTokenService(); + + try { + await uninstallTokenService?.checkTokenValidityForPolicy(policyId); + } catch (e) { + throw new Error(`Cannot enable Agent Tamper Protection: ${e.message}`); + } + } + } } export const agentPolicyService = new AgentPolicyService(); diff --git a/x-pack/plugins/fleet/server/services/security/uninstall_token_service/index.test.ts b/x-pack/plugins/fleet/server/services/security/uninstall_token_service/index.test.ts index 4cf657b7255c5..4b3be1de81632 100644 --- a/x-pack/plugins/fleet/server/services/security/uninstall_token_service/index.test.ts +++ b/x-pack/plugins/fleet/server/services/security/uninstall_token_service/index.test.ts @@ -499,5 +499,80 @@ describe('UninstallTokenService', () => { }); }); }); + + describe('check validity of tokens', () => { + const okaySO = getDefaultSO(canEncrypt); + + const errorWithDecryptionSO2 = { + ...getDefaultSO2(canEncrypt), + error: new Error('error reason'), + }; + const missingTokenSO2 = { + ...getDefaultSO2(canEncrypt), + attributes: { + ...getDefaultSO2(canEncrypt).attributes, + token: undefined, + token_plain: undefined, + }, + }; + + describe('checkTokenValidityForAllPolicies', () => { + it('resolves if all of the tokens are available', async () => { + mockCreatePointInTimeFinderAsInternalUser(); + + await expect( + uninstallTokenService.checkTokenValidityForAllPolicies() + ).resolves.not.toThrowError(); + }); + + it('rejects if any of the tokens is missing', async () => { + mockCreatePointInTimeFinderAsInternalUser([okaySO, missingTokenSO2]); + + await expect( + uninstallTokenService.checkTokenValidityForAllPolicies() + ).rejects.toThrowError( + 'Invalid uninstall token: Saved object is missing the `token` attribute.' + ); + }); + + it('rejects if token decryption gives error', async () => { + mockCreatePointInTimeFinderAsInternalUser([okaySO, errorWithDecryptionSO2]); + + await expect( + uninstallTokenService.checkTokenValidityForAllPolicies() + ).rejects.toThrowError('Error when reading Uninstall Token: error reason'); + }); + }); + + describe('checkTokenValidityForPolicy', () => { + it('resolves if token is available', async () => { + mockCreatePointInTimeFinderAsInternalUser(); + + await expect( + uninstallTokenService.checkTokenValidityForPolicy(okaySO.attributes.policy_id) + ).resolves.not.toThrowError(); + }); + + it('rejects if token is missing', async () => { + mockCreatePointInTimeFinderAsInternalUser([okaySO, missingTokenSO2]); + + await expect( + uninstallTokenService.checkTokenValidityForPolicy(missingTokenSO2.attributes.policy_id) + ).rejects.toThrowError( + 'Invalid uninstall token: Saved object is missing the `token` attribute.' + ); + }); + + it('rejects if token decryption gives error', async () => { + mockCreatePointInTimeFinderAsInternalUser([okaySO, errorWithDecryptionSO2]); + + await expect( + uninstallTokenService.checkTokenValidityForPolicy( + errorWithDecryptionSO2.attributes.policy_id + ) + ).rejects.toThrowError('Error when reading Uninstall Token: error reason'); + }); + }); + }); }); }); diff --git a/x-pack/plugins/fleet/server/services/security/uninstall_token_service/index.ts b/x-pack/plugins/fleet/server/services/security/uninstall_token_service/index.ts index 8309910be6f53..9e03e7869c584 100644 --- a/x-pack/plugins/fleet/server/services/security/uninstall_token_service/index.ts +++ b/x-pack/plugins/fleet/server/services/security/uninstall_token_service/index.ts @@ -109,7 +109,7 @@ export interface UninstallTokenServiceInterface { * @param force generate a new token even if one already exists * @returns hashedToken */ - generateTokenForPolicyId(policyId: string, force?: boolean): Promise; + generateTokenForPolicyId(policyId: string, force?: boolean): Promise; /** * Generate uninstall tokens for given policy ids @@ -119,7 +119,7 @@ export interface UninstallTokenServiceInterface { * @param force generate a new token even if one already exists * @returns Record */ - generateTokensForPolicyIds(policyIds: string[], force?: boolean): Promise>; + generateTokensForPolicyIds(policyIds: string[], force?: boolean): Promise; /** * Generate uninstall tokens all policies @@ -128,12 +128,26 @@ export interface UninstallTokenServiceInterface { * @param force generate a new token even if one already exists * @returns Record */ - generateTokensForAllPolicies(force?: boolean): Promise>; + generateTokensForAllPolicies(force?: boolean): Promise; /** * If encryption is available, checks for any plain text uninstall tokens and encrypts them */ encryptTokens(): Promise; + + /** + * Check whether the selected policy has a valid uninstall token. Rejects returning promise if not. + * + * @param policyId policy Id to check + */ + checkTokenValidityForPolicy(policyId: string): Promise; + + /** + * Check whether all policies have a valid uninstall token. Rejects returning promise if not. + * + * @param policyId policy Id to check + */ + checkTokenValidityForAllPolicies(): Promise; } export class UninstallTokenService implements UninstallTokenServiceInterface { @@ -210,7 +224,11 @@ export class UninstallTokenService implements UninstallTokenServiceInterface { tokensFinder.close(); const uninstallTokens: UninstallToken[] = tokenObject.map( - ({ id: _id, attributes, created_at: createdAt }) => { + ({ id: _id, attributes, created_at: createdAt, error }) => { + if (error) { + throw new UninstallTokenError(`Error when reading Uninstall Token: ${error.message}`); + } + this.assertPolicyId(attributes); this.assertToken(attributes); this.assertCreatedAt(createdAt); @@ -304,32 +322,30 @@ export class UninstallTokenService implements UninstallTokenServiceInterface { return this.getHashedTokensForPolicyIds(policyIds); } - public async generateTokenForPolicyId(policyId: string, force: boolean = false): Promise { - return (await this.generateTokensForPolicyIds([policyId], force))[policyId]; + public generateTokenForPolicyId(policyId: string, force: boolean = false): Promise { + return this.generateTokensForPolicyIds([policyId], force); } public async generateTokensForPolicyIds( policyIds: string[], force: boolean = false - ): Promise> { + ): Promise { const { agentTamperProtectionEnabled } = appContextService.getExperimentalFeatures(); if (!agentTamperProtectionEnabled || !policyIds.length) { - return {}; + return; } - const existingTokens = force - ? {} - : (await this.getDecryptedTokensForPolicyIds(policyIds)).reduce( - (acc, { policy_id: policyId, token }) => { - acc[policyId] = token; - return acc; - }, - {} as Record - ); + const existingTokens = new Set(); + + if (!force) { + (await this.getTokenObjectsByIncludeFilter(policyIds)).forEach((tokenObject) => { + existingTokens.add(tokenObject._source[UNINSTALL_TOKENS_SAVED_OBJECT_TYPE].policy_id); + }); + } const missingTokenPolicyIds = force ? policyIds - : policyIds.filter((policyId) => !existingTokens[policyId]); + : policyIds.filter((policyId) => !existingTokens.has(policyId)); const newTokensMap = missingTokenPolicyIds.reduce((acc, policyId) => { const token = this.generateToken(); @@ -338,7 +354,6 @@ export class UninstallTokenService implements UninstallTokenServiceInterface { [policyId]: token, }; }, {} as Record); - await this.persistTokens(missingTokenPolicyIds, newTokensMap); if (force) { const config = appContextService.getConfig(); @@ -349,21 +364,9 @@ export class UninstallTokenService implements UninstallTokenServiceInterface { await agentPolicyService.deployPolicies(this.soClient, policyIdsBatch) ); } - - const tokensMap = { - ...existingTokens, - ...newTokensMap, - }; - - return Object.entries(tokensMap).reduce((acc, [policyId, token]) => { - acc[policyId] = this.hashToken(token); - return acc; - }, {} as Record); } - public async generateTokensForAllPolicies( - force: boolean = false - ): Promise> { + public async generateTokensForAllPolicies(force: boolean = false): Promise { const policyIds = await this.getAllPolicyIds(); return this.generateTokensForPolicyIds(policyIds, force); } @@ -486,6 +489,15 @@ export class UninstallTokenService implements UninstallTokenServiceInterface { return this._soClient; } + public async checkTokenValidityForPolicy(policyId: string): Promise { + await this.getDecryptedTokensForPolicyIds([policyId]); + } + + public async checkTokenValidityForAllPolicies(): Promise { + const policyIds = await this.getAllPolicyIds(); + await this.getDecryptedTokensForPolicyIds(policyIds); + } + private get isEncryptionAvailable(): boolean { return appContextService.getEncryptedSavedObjectsSetup()?.canEncrypt ?? false; } @@ -498,7 +510,9 @@ export class UninstallTokenService implements UninstallTokenServiceInterface { private assertToken(attributes: UninstallTokenSOAttributes | undefined) { if (!attributes?.token && !attributes?.token_plain) { - throw new UninstallTokenError('Uninstall Token is missing the token.'); + throw new UninstallTokenError( + 'Invalid uninstall token: Saved object is missing the `token` attribute.' + ); } } diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 178499011bc61..60ce6460d0ac2 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -6,6 +6,7 @@ */ import fs from 'fs/promises'; + import apm from 'elastic-apm-node'; import { compact } from 'lodash'; @@ -13,6 +14,8 @@ import pMap from 'p-map'; import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; +import type { UninstallTokenError } from '../../common/errors'; + import { AUTO_UPDATE_PACKAGES } from '../../common/constants'; import type { PreconfigurationError } from '../../common/constants'; import type { DefaultPackagesInstallationError } from '../../common/types'; @@ -54,7 +57,10 @@ import { cleanUpOldFileIndices } from './setup/clean_old_fleet_indices'; export interface SetupStatus { isInitialized: boolean; nonFatalErrors: Array< - PreconfigurationError | DefaultPackagesInstallationError | UpgradeManagedPackagePoliciesResult + | PreconfigurationError + | DefaultPackagesInstallationError + | UpgradeManagedPackagePoliciesResult + | { error: UninstallTokenError } >; } @@ -196,9 +202,17 @@ async function createSetupSideEffects( logger.debug('Checking for and encrypting plain text uninstall tokens'); await appContextService.getUninstallTokenService()?.encryptTokens(); } + + logger.debug('Checking validity of Uninstall Tokens'); + try { + await appContextService.getUninstallTokenService()?.checkTokenValidityForAllPolicies(); + } catch (error) { + nonFatalErrors.push({ error }); + } stepSpan?.end(); stepSpan = apm.startSpan('Upgrade agent policy schema', 'preconfiguration'); + logger.debug('Upgrade Agent policy schema version'); await upgradeAgentPolicySchemaVersion(soClient); stepSpan?.end(); From 503123105fa532baa9e74ae690d734298cd1a840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yulia=20=C4=8Cech?= <6585477+yuliacech@users.noreply.github.com> Date: Tue, 28 Nov 2023 15:18:52 +0100 Subject: [PATCH 05/30] [Index Management] Add data streams tests for serverless (#171926) ## Summary This PR adds data streams api integration tests for serverless. The helpers code was extracted to a service for re-use in stateful and serverless tests. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../index_management/data_streams.ts | 84 +---- .../lib/datastreams.helpers.ts | 96 ++++++ .../services/index_management.ts | 4 + .../common/index_management/datastreams.ts | 325 ++++++++++++++++++ .../common/index_management/index.ts | 1 + x-pack/test_serverless/tsconfig.json | 1 + 6 files changed, 437 insertions(+), 74 deletions(-) create mode 100644 x-pack/test/api_integration/apis/management/index_management/lib/datastreams.helpers.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts diff --git a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts index 6ed1ec9ed1c4c..791e23149aff1 100644 --- a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts +++ b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts @@ -9,85 +9,21 @@ import expect from '@kbn/expect'; import { DataStream } from '@kbn/index-management-plugin/common'; import { FtrProviderContext } from '../../../ftr_provider_context'; -// @ts-ignore import { API_BASE_PATH } from './constants'; +import { datastreamsHelpers } from './lib/datastreams.helpers'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); - const es = getService('es'); - - const createDataStream = async (name: string) => { - // A data stream requires an index template before it can be created. - await es.indices.putIndexTemplate({ - name, - body: { - // We need to match the names of backing indices with this template. - index_patterns: [name + '*'], - template: { - mappings: { - properties: { - '@timestamp': { - type: 'date', - }, - }, - }, - lifecycle: { - // @ts-expect-error @elastic/elasticsearch enabled prop is not typed yet - enabled: true, - }, - }, - data_stream: {}, - }, - }); - await es.indices.createDataStream({ name }); - }; - - const updateIndexTemplateMappings = async (name: string, mappings: any) => { - await es.indices.putIndexTemplate({ - name, - body: { - // We need to match the names of backing indices with this template. - index_patterns: [name + '*'], - template: { - mappings, - }, - data_stream: {}, - }, - }); - }; - - const getDatastream = async (name: string) => { - const { - data_streams: [datastream], - } = await es.indices.getDataStream({ name }); - return datastream; - }; - - const getMapping = async (name: string) => { - const res = await es.indices.getMapping({ index: name }); - - return Object.values(res)[0]!.mappings; - }; - - const deleteComposableIndexTemplate = async (name: string) => { - await es.indices.deleteIndexTemplate({ name }); - }; - - const deleteDataStream = async (name: string) => { - await es.indices.deleteDataStream({ name }); - await deleteComposableIndexTemplate(name); - }; - - const assertDataStreamStorageSizeExists = (storageSize: string, storageSizeBytes: number) => { - // Storage size of a document doesn't look like it would be deterministic (could vary depending - // on how ES, Lucene, and the file system interact), so we'll just assert its presence and - // type. - expect(storageSize).to.be.ok(); - expect(typeof storageSize).to.be('string'); - expect(storageSizeBytes).to.be.ok(); - expect(typeof storageSizeBytes).to.be('number'); - }; + const { + createDataStream, + deleteDataStream, + assertDataStreamStorageSizeExists, + deleteComposableIndexTemplate, + updateIndexTemplateMappings, + getMapping, + getDatastream, + } = datastreamsHelpers(getService); describe('Data streams', function () { describe('Get', () => { diff --git a/x-pack/test/api_integration/apis/management/index_management/lib/datastreams.helpers.ts b/x-pack/test/api_integration/apis/management/index_management/lib/datastreams.helpers.ts new file mode 100644 index 0000000000000..65e2d733dd696 --- /dev/null +++ b/x-pack/test/api_integration/apis/management/index_management/lib/datastreams.helpers.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export function datastreamsHelpers(getService: FtrProviderContext['getService']) { + const es = getService('es'); + + const createDataStream = async (name: string) => { + // A data stream requires an index template before it can be created. + await es.indices.putIndexTemplate({ + name, + body: { + // We need to match the names of backing indices with this template. + index_patterns: [name + '*'], + template: { + mappings: { + properties: { + '@timestamp': { + type: 'date', + }, + }, + }, + lifecycle: { + // @ts-expect-error @elastic/elasticsearch enabled prop is not typed yet + enabled: true, + }, + }, + data_stream: {}, + }, + }); + + await es.indices.createDataStream({ name }); + }; + + const updateIndexTemplateMappings = async (name: string, mappings: any) => { + await es.indices.putIndexTemplate({ + name, + body: { + // We need to match the names of backing indices with this template. + index_patterns: [name + '*'], + template: { + mappings, + }, + data_stream: {}, + }, + }); + }; + + const getDatastream = async (name: string) => { + const { + data_streams: [datastream], + } = await es.indices.getDataStream({ name }); + return datastream; + }; + + const getMapping = async (name: string) => { + const res = await es.indices.getMapping({ index: name }); + + return Object.values(res)[0]!.mappings; + }; + + const deleteComposableIndexTemplate = async (name: string) => { + await es.indices.deleteIndexTemplate({ name }); + }; + + const deleteDataStream = async (name: string) => { + await es.indices.deleteDataStream({ name }); + await deleteComposableIndexTemplate(name); + }; + + const assertDataStreamStorageSizeExists = (storageSize: string, storageSizeBytes: number) => { + // Storage size of a document doesn't look like it would be deterministic (could vary depending + // on how ES, Lucene, and the file system interact), so we'll just assert its presence and + // type. + expect(storageSize).to.be.ok(); + expect(typeof storageSize).to.be('string'); + expect(storageSizeBytes).to.be.ok(); + expect(typeof storageSizeBytes).to.be('number'); + }; + + return { + createDataStream, + updateIndexTemplateMappings, + getDatastream, + getMapping, + deleteComposableIndexTemplate, + deleteDataStream, + assertDataStreamStorageSizeExists, + }; +} diff --git a/x-pack/test/api_integration/services/index_management.ts b/x-pack/test/api_integration/services/index_management.ts index 44d33752e4147..f5a57a9b74259 100644 --- a/x-pack/test/api_integration/services/index_management.ts +++ b/x-pack/test/api_integration/services/index_management.ts @@ -9,6 +9,7 @@ import { FtrProviderContext } from '../ftr_provider_context'; import { indicesApi } from '../apis/management/index_management/lib/indices.api'; import { mappingsApi } from '../apis/management/index_management/lib/mappings.api'; import { indicesHelpers } from '../apis/management/index_management/lib/indices.helpers'; +import { datastreamsHelpers } from '../apis/management/index_management/lib/datastreams.helpers'; export function IndexManagementProvider({ getService }: FtrProviderContext) { return { @@ -16,6 +17,9 @@ export function IndexManagementProvider({ getService }: FtrProviderContext) { api: indicesApi(getService), helpers: indicesHelpers(getService), }, + datastreams: { + helpers: datastreamsHelpers(getService), + }, mappings: { api: mappingsApi(getService), }, diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts new file mode 100644 index 0000000000000..adc84ffecb638 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts @@ -0,0 +1,325 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { DataStream } from '@kbn/index-management-plugin/common'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +const API_BASE_PATH = '/api/index_management'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const indexManagementService = getService('indexManagement'); + let helpers: typeof indexManagementService['datastreams']['helpers']; + let createDataStream: typeof helpers['createDataStream']; + let deleteDataStream: typeof helpers['deleteDataStream']; + let deleteComposableIndexTemplate: typeof helpers['deleteComposableIndexTemplate']; + let updateIndexTemplateMappings: typeof helpers['updateIndexTemplateMappings']; + let getMapping: typeof helpers['getMapping']; + let getDatastream: typeof helpers['getDatastream']; + + describe('Data streams', function () { + before(async () => { + ({ + datastreams: { helpers }, + } = indexManagementService); + ({ + createDataStream, + deleteDataStream, + deleteComposableIndexTemplate, + updateIndexTemplateMappings, + getMapping, + getDatastream, + } = helpers); + }); + describe('Get', () => { + const testDataStreamName = 'test-data-stream'; + + before(async () => await createDataStream(testDataStreamName)); + after(async () => await deleteDataStream(testDataStreamName)); + + it('returns an array of data streams', async () => { + const { body: dataStreams } = await supertest + .get(`${API_BASE_PATH}/data_streams`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + expect(dataStreams).to.be.an('array'); + + // returned array can contain automatically created data streams + const testDataStream = dataStreams.find( + (dataStream: DataStream) => dataStream.name === testDataStreamName + ); + + expect(testDataStream).to.be.ok(); + + // ES determines these values so we'll just echo them back. + const { name: indexName, uuid } = testDataStream!.indices[0]; + + expect(testDataStream).to.eql({ + name: testDataStreamName, + lifecycle: { + enabled: true, + }, + privileges: { + delete_index: true, + manage_data_stream_lifecycle: true, + }, + timeStampField: { name: '@timestamp' }, + indices: [ + { + name: indexName, + uuid, + preferILM: true, + managedBy: 'Data stream lifecycle', + }, + ], + nextGenerationManagedBy: 'Data stream lifecycle', + generation: 1, + health: 'green', + indexTemplateName: testDataStreamName, + hidden: false, + }); + }); + + it('includes stats when provided the includeStats query parameter', async () => { + const { body: dataStreams } = await supertest + .get(`${API_BASE_PATH}/data_streams?includeStats=true`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + expect(dataStreams).to.be.an('array'); + + // returned array can contain automatically created data streams + const testDataStream = dataStreams.find( + (dataStream: DataStream) => dataStream.name === testDataStreamName + ); + + expect(testDataStream).to.be.ok(); + + // ES determines these values so we'll just echo them back. + const { name: indexName, uuid } = testDataStream!.indices[0]; + const { storageSize, storageSizeBytes, ...dataStreamWithoutStorageSize } = testDataStream!; + + expect(dataStreamWithoutStorageSize).to.eql({ + name: testDataStreamName, + privileges: { + delete_index: true, + manage_data_stream_lifecycle: true, + }, + timeStampField: { name: '@timestamp' }, + indices: [ + { + name: indexName, + managedBy: 'Data stream lifecycle', + preferILM: true, + uuid, + }, + ], + generation: 1, + health: 'green', + indexTemplateName: testDataStreamName, + nextGenerationManagedBy: 'Data stream lifecycle', + maxTimeStamp: 0, + hidden: false, + lifecycle: { + enabled: true, + }, + }); + }); + + it('returns a single data stream by ID', async () => { + const { body: dataStream } = await supertest + .get(`${API_BASE_PATH}/data_streams/${testDataStreamName}`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + // ES determines these values so we'll just echo them back. + const { name: indexName, uuid } = dataStream.indices[0]; + const { storageSize, storageSizeBytes, ...dataStreamWithoutStorageSize } = dataStream; + + expect(dataStreamWithoutStorageSize).to.eql({ + name: testDataStreamName, + privileges: { + delete_index: true, + manage_data_stream_lifecycle: true, + }, + timeStampField: { name: '@timestamp' }, + indices: [ + { + name: indexName, + managedBy: 'Data stream lifecycle', + preferILM: true, + uuid, + }, + ], + generation: 1, + health: 'green', + indexTemplateName: testDataStreamName, + nextGenerationManagedBy: 'Data stream lifecycle', + maxTimeStamp: 0, + hidden: false, + lifecycle: { + enabled: true, + }, + }); + }); + }); + + describe('Update', () => { + const testDataStreamName = 'test-data-stream'; + + before(async () => await createDataStream(testDataStreamName)); + after(async () => await deleteDataStream(testDataStreamName)); + + it('updates the data retention of a DS', async () => { + const { body } = await supertest + .put(`${API_BASE_PATH}/data_streams/${testDataStreamName}/data_retention`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .send({ + dataRetention: '7d', + }) + .expect(200); + + expect(body).to.eql({ success: true }); + }); + + it('sets data retention to infinite', async () => { + const { body } = await supertest + .put(`${API_BASE_PATH}/data_streams/${testDataStreamName}/data_retention`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .send({}) + .expect(200); + + expect(body).to.eql({ success: true }); + }); + + it('can disable lifecycle for a given policy', async () => { + const { body } = await supertest + .put(`${API_BASE_PATH}/data_streams/${testDataStreamName}/data_retention`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .send({ enabled: false }) + .expect(200); + + expect(body).to.eql({ success: true }); + + const datastream = await getDatastream(testDataStreamName); + expect(datastream.lifecycle).to.be(undefined); + }); + }); + + describe('Delete', () => { + const testDataStreamName1 = 'test-data-stream1'; + const testDataStreamName2 = 'test-data-stream2'; + + before(async () => { + await Promise.all([ + createDataStream(testDataStreamName1), + createDataStream(testDataStreamName2), + ]); + }); + + after(async () => { + // The Delete API only deletes the data streams, so we still need to manually delete their + // related index patterns to clean up. + await Promise.all([ + deleteComposableIndexTemplate(testDataStreamName1), + deleteComposableIndexTemplate(testDataStreamName2), + ]); + }); + + it('deletes multiple data streams', async () => { + await supertest + .post(`${API_BASE_PATH}/delete_data_streams`) + .set('x-elastic-internal-origin', 'xxx') + .set('kbn-xsrf', 'xxx') + .send({ + dataStreams: [testDataStreamName1, testDataStreamName2], + }) + .expect(200); + + await supertest + .get(`${API_BASE_PATH}/data_streams/${testDataStreamName1}`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(404); + + await supertest + .get(`${API_BASE_PATH}/data_streams/${testDataStreamName2}`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(404); + }); + }); + + describe('Mappings from template', () => { + const testDataStreamName1 = 'test-data-stream-mappings-1'; + + before(async () => { + await createDataStream(testDataStreamName1); + }); + + after(async () => { + await deleteDataStream(testDataStreamName1); + }); + + it('Apply mapping from index template', async () => { + const beforeMapping = await getMapping(testDataStreamName1); + expect(beforeMapping.properties).eql({ + '@timestamp': { type: 'date' }, + }); + await updateIndexTemplateMappings(testDataStreamName1, { + properties: { + test: { type: 'integer' }, + }, + }); + await supertest + .post(`${API_BASE_PATH}/data_streams/${testDataStreamName1}/mappings_from_template`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + const afterMapping = await getMapping(testDataStreamName1); + expect(afterMapping.properties).eql({ + '@timestamp': { type: 'date' }, + test: { type: 'integer' }, + }); + }); + }); + + describe('Rollover', () => { + const testDataStreamName1 = 'test-data-stream-rollover-1'; + + before(async () => { + await createDataStream(testDataStreamName1); + }); + + after(async () => { + await deleteDataStream(testDataStreamName1); + }); + + it('Rollover datastreams', async () => { + await supertest + .post(`${API_BASE_PATH}/data_streams/${testDataStreamName1}/rollover`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + const datastream = await getDatastream(testDataStreamName1); + + expect(datastream.generation).equal(2); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts index e06aaf9225cfa..07f5da6aba981 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts @@ -14,6 +14,7 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./index_templates')); loadTestFile(require.resolve('./indices')); loadTestFile(require.resolve('./create_enrich_policies')); + loadTestFile(require.resolve('./datastreams')); loadTestFile(require.resolve('./mappings')); }); } diff --git a/x-pack/test_serverless/tsconfig.json b/x-pack/test_serverless/tsconfig.json index 6f843b935d63f..af6f6e4b25e99 100644 --- a/x-pack/test_serverless/tsconfig.json +++ b/x-pack/test_serverless/tsconfig.json @@ -68,5 +68,6 @@ "@kbn/apm-synthtrace-client", "@kbn/reporting-export-types-csv-common", "@kbn/mock-idp-plugin", + "@kbn/index-management-plugin", ] } From 517c815c4808df7d16403bac8f2be4f40c240e44 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Tue, 28 Nov 2023 15:20:36 +0100 Subject: [PATCH 06/30] [Fleet] making service_token an output secret (#171875) ## Summary Related to https://github.com/elastic/kibana/issues/104986 Making remote ES output's service_token a secret. fleet-server change here: https://github.com/elastic/fleet-server/pull/3051#discussion_r1406183654 Steps to verify: - Enable remote ES output and output secrets in `kibana.dev.yml` locally: ``` xpack.fleet.enableExperimental: ['remoteESOutput', 'outputSecretsStorage'] ``` - Start es, kibana, fleet-server locally and start a second es locally - see detailed steps here: https://github.com/elastic/fleet-server/pull/3051 - Create a remote ES output, verify that the service_token is stored as a secret reference ``` GET .kibana_ingest/_search?q=type:ingest-outputs ``` - Verify that the enrolled agent sends data to the remote ES successfully image image ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../current_mappings.json | 8 +++ .../check_registered_types.test.ts | 2 +- .../plugins/fleet/common/openapi/bundled.json | 50 +++++++++++++- .../plugins/fleet/common/openapi/bundled.yaml | 31 +++++++++ .../schemas/output_create_request.yaml | 2 + ...t_create_request_remote_elasticsearch.yaml | 27 ++++++++ .../fleet/common/types/models/output.ts | 26 +++++--- .../edit_output_flyout/index.test.tsx | 6 +- .../components/edit_output_flyout/index.tsx | 8 ++- .../output_form_remote_es.tsx | 66 +++++++++++++------ .../output_form_validators.tsx | 2 + .../edit_output_flyout/use_output_form.tsx | 26 +++++++- .../server/routes/output/handler.test.ts | 37 +++++++++++ .../fleet/server/routes/output/handler.ts | 13 +++- .../fleet/server/saved_objects/index.ts | 6 ++ .../fleet/server/services/fleet_proxies.ts | 2 +- .../services/preconfiguration/outputs.ts | 2 +- .../plugins/fleet/server/services/secrets.ts | 46 +++++++++++-- .../fleet/server/types/models/output.ts | 12 +++- .../fleet/server/types/so_attributes.ts | 4 +- .../apis/outputs/crud.ts | 17 +++++ 21 files changed, 347 insertions(+), 46 deletions(-) create mode 100644 x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 603f0efd54a87..91246b667626c 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1844,6 +1844,14 @@ } } } + }, + "service_token": { + "dynamic": false, + "properties": { + "id": { + "type": "keyword" + } + } } } } diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 2c7d540132139..fb5010cff0280 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () => "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4", "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", - "ingest-outputs": "8546f1123ec30dcbd6f238f72729c5f1656a4d9b", + "ingest-outputs": "4dd3cb38a91c848df95336a24a5abde2c8560fd1", "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43", "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index e0b754430521d..2f583f9b32140 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -8257,6 +8257,50 @@ "type" ] }, + "output_create_request_remote_elasticsearch": { + "title": "remote_elasticsearch", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "is_default": { + "type": "boolean" + }, + "is_default_monitoring": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "remote_elasticsearch" + ] + }, + "hosts": { + "type": "array", + "items": { + "type": "string" + } + }, + "service_token": { + "type": "string" + }, + "secrets": { + "type": "object", + "properties": { + "service_token": { + "type": "string" + } + } + } + }, + "required": [ + "name" + ] + }, "output_create_request": { "title": "Output", "oneOf": [ @@ -8268,6 +8312,9 @@ }, { "$ref": "#/components/schemas/output_create_request_logstash" + }, + { + "$ref": "#/components/schemas/output_create_request_remote_elasticsearch" } ], "discriminator": { @@ -8275,7 +8322,8 @@ "mapping": { "elasticsearch": "#/components/schemas/output_create_request_elasticsearch", "kafka": "#/components/schemas/output_create_request_kafka", - "logstash": "#/components/schemas/output_create_request_logstash" + "logstash": "#/components/schemas/output_create_request_logstash", + "remote_elasticsearch": "#/components/schemas/output_create_request_remote_elasticsearch" } } }, diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index d977af4f9c2b5..d103c2f5e2d9a 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -5331,18 +5331,49 @@ components: - name - hosts - type + output_create_request_remote_elasticsearch: + title: remote_elasticsearch + type: object + properties: + id: + type: string + is_default: + type: boolean + is_default_monitoring: + type: boolean + name: + type: string + type: + type: string + enum: + - remote_elasticsearch + hosts: + type: array + items: + type: string + service_token: + type: string + secrets: + type: object + properties: + service_token: + type: string + required: + - name output_create_request: title: Output oneOf: - $ref: '#/components/schemas/output_create_request_elasticsearch' - $ref: '#/components/schemas/output_create_request_kafka' - $ref: '#/components/schemas/output_create_request_logstash' + - $ref: '#/components/schemas/output_create_request_remote_elasticsearch' discriminator: propertyName: type mapping: elasticsearch: '#/components/schemas/output_create_request_elasticsearch' kafka: '#/components/schemas/output_create_request_kafka' logstash: '#/components/schemas/output_create_request_logstash' + remote_elasticsearch: '#/components/schemas/output_create_request_remote_elasticsearch' output_update_request_elasticsearch: title: elasticsearch type: object diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request.yaml index 21506825cfd61..9fc0ad6d24590 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request.yaml @@ -3,9 +3,11 @@ oneOf: - $ref: './output_create_request_elasticsearch.yaml' - $ref: './output_create_request_kafka.yaml' - $ref: './output_create_request_logstash.yaml' + - $ref: './output_create_request_remote_elasticsearch.yaml' discriminator: propertyName: type mapping: elasticsearch: './output_create_request_elasticsearch.yaml' kafka: './output_create_request_kafka.yaml' logstash: './output_create_request_logstash.yaml' + remote_elasticsearch: './output_create_request_remote_elasticsearch.yaml' diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml new file mode 100644 index 0000000000000..844b92df39b84 --- /dev/null +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml @@ -0,0 +1,27 @@ +title: remote_elasticsearch +type: object +properties: + id: + type: string + is_default: + type: boolean + is_default_monitoring: + type: boolean + name: + type: string + type: + type: string + enum: ['remote_elasticsearch'] + hosts: + type: array + items: + type: string + service_token: + type: string + secrets: + type: object + properties: + service_token: + type: string +required: + - name diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 3283f4d01e540..f726c88bf77cd 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -43,15 +43,7 @@ interface NewBaseOutput { proxy_id?: string | null; shipper?: ShipperOutput | null; allow_edit?: string[]; - secrets?: { - ssl?: { - key?: - | string - | { - id: string; - }; - }; - }; + secrets?: {}; } export interface NewElasticsearchOutput extends NewBaseOutput { @@ -61,10 +53,26 @@ export interface NewElasticsearchOutput extends NewBaseOutput { export interface NewRemoteElasticsearchOutput extends NewBaseOutput { type: OutputType['RemoteElasticsearch']; service_token?: string; + secrets?: { + service_token?: + | string + | { + id: string; + }; + }; } export interface NewLogstashOutput extends NewBaseOutput { type: OutputType['Logstash']; + secrets?: { + ssl?: { + key?: + | string + | { + id: string; + }; + }; + }; } export type NewOutput = diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx index 2308cc824db6d..2126a275818e1 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx @@ -191,7 +191,9 @@ describe('EditOutputFlyout', () => { }); it('should render the flyout if the output provided is a remote ES output', async () => { - jest.spyOn(ExperimentalFeaturesService, 'get').mockReturnValue({ remoteESOutput: true }); + jest + .spyOn(ExperimentalFeaturesService, 'get') + .mockReturnValue({ remoteESOutput: true, outputSecretsStorage: true }); const { utils } = renderFlyout({ type: 'remote_elasticsearch', name: 'remote es output', @@ -208,6 +210,8 @@ describe('EditOutputFlyout', () => { expect(utils.queryByTestId('settingsOutputsFlyout.typeInput')?.textContent).toContain( 'Remote Elasticsearch' ); + + expect(utils.queryByTestId('serviceTokenSecretInput')).not.toBeNull(); }); it('should not display remote ES output in type lists if serverless', async () => { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx index 62732c12d189c..0e742876af82d 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx @@ -272,7 +272,13 @@ export const EditOutputFlyout: React.FunctionComponent = const renderRemoteElasticsearchSection = () => { if (isRemoteESOutputEnabled) { - return ; + return ( + + ); } return null; }; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx index 9b4c8f085af9a..9e5fe1bc519fb 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx @@ -13,13 +13,16 @@ import { i18n } from '@kbn/i18n'; import { MultiRowInput } from '../multi_row_input'; import type { OutputFormInputsType } from './use_output_form'; +import { SecretFormRow } from './output_form_secret_form_row'; interface Props { inputs: OutputFormInputsType; + useSecretsStorage: boolean; + onUsePlainText: () => void; } export const OutputFormRemoteEsSection: React.FunctionComponent = (props) => { - const { inputs } = props; + const { inputs, useSecretsStorage, onUsePlainText } = props; return ( <> @@ -38,27 +41,50 @@ export const OutputFormRemoteEsSection: React.FunctionComponent = (props) isUrl /> - + } + {...inputs.serviceTokenInput.formRowProps} + > + - } - {...inputs.serviceTokenInput.formRowProps} - > - + ) : ( + - + title={i18n.translate('xpack.fleet.settings.editOutputFlyout.serviceTokenLabel', { + defaultMessage: 'Service Token', + })} + {...inputs.serviceTokenSecretInput.formRowProps} + onUsePlainText={onUsePlainText} + > + + + )} ; caTrustedFingerprintInput: ReturnType; serviceTokenInput: ReturnType; + serviceTokenSecretInput: ReturnType; sslCertificateInput: ReturnType; sslKeyInput: ReturnType; sslKeySecretInput: ReturnType; @@ -215,6 +217,12 @@ export function useOutputForm(onSucess: () => void, output?: Output) { validateServiceToken, isDisabled('service_token') ); + + const serviceTokenSecretInput = useSecretInput( + (output as NewRemoteElasticsearchOutput)?.secrets?.service_token ?? '', + validateServiceTokenSecret, + isDisabled('service_token') + ); /* Shipper feature flag - currently depends on the content of the yaml # Enables the shipper: @@ -293,7 +301,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { const sslKeyInput = useInput(output?.ssl?.key ?? '', validateSSLKey, isSSLEditable); const sslKeySecretInput = useSecretInput( - output?.secrets?.ssl?.key, + (output as NewLogstashOutput)?.secrets?.ssl?.key, validateSSLKeySecret, isSSLEditable ); @@ -503,6 +511,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { defaultMonitoringOutputInput, caTrustedFingerprintInput, serviceTokenInput, + serviceTokenSecretInput, sslCertificateInput, sslKeyInput, sslKeySecretInput, @@ -562,6 +571,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { const additionalYamlConfigValid = additionalYamlConfigInput.validate(); const caTrustedFingerprintValid = caTrustedFingerprintInput.validate(); const serviceTokenValid = serviceTokenInput.validate(); + const serviceTokenSecretValid = serviceTokenSecretInput.validate(); const sslCertificateValid = sslCertificateInput.validate(); const sslKeyValid = sslKeyInput.validate(); const sslKeySecretValid = sslKeySecretInput.validate(); @@ -607,7 +617,11 @@ export function useOutputForm(onSucess: () => void, output?: Output) { } if (isRemoteElasticsearch) { return ( - elasticsearchUrlsValid && additionalYamlConfigValid && nameInputValid && serviceTokenValid + elasticsearchUrlsValid && + additionalYamlConfigValid && + nameInputValid && + ((serviceTokenInput.value && serviceTokenValid) || + (serviceTokenSecretInput.value && serviceTokenSecretValid)) ); } else { // validate ES @@ -637,6 +651,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { additionalYamlConfigInput, caTrustedFingerprintInput, serviceTokenInput, + serviceTokenSecretInput, sslCertificateInput, sslKeyInput, sslKeySecretInput, @@ -852,6 +867,12 @@ export function useOutputForm(onSucess: () => void, output?: Output) { is_default_monitoring: defaultMonitoringOutputInput.value, config_yaml: additionalYamlConfigInput.value, service_token: serviceTokenInput.value, + ...(!serviceTokenInput.value && + serviceTokenSecretInput.value && { + secrets: { + service_token: serviceTokenSecretInput.value, + }, + }), proxy_id: proxyIdValue, ...shipperParams, } as NewRemoteElasticsearchOutput; @@ -958,6 +979,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) { elasticsearchUrlInput.value, caTrustedFingerprintInput.value, serviceTokenInput.value, + serviceTokenSecretInput.value, confirm, notifications.toasts, ]); diff --git a/x-pack/plugins/fleet/server/routes/output/handler.test.ts b/x-pack/plugins/fleet/server/routes/output/handler.test.ts index 84443b3ad7196..5cf3b544e1553 100644 --- a/x-pack/plugins/fleet/server/routes/output/handler.test.ts +++ b/x-pack/plugins/fleet/server/routes/output/handler.test.ts @@ -88,4 +88,41 @@ describe('output handler', () => { expect(res).toEqual({ body: { item: { id: 'output1' } } }); }); + + it('should return error if both service_token and secrets.service_token is provided for remote_elasticsearch output', async () => { + jest + .spyOn(appContextService, 'getCloud') + .mockReturnValue({ isServerlessEnabled: false } as any); + + const res = await postOutputHandler( + mockContext, + { + body: { + type: 'remote_elasticsearch', + service_token: 'token1', + secrets: { service_token: 'token2' }, + }, + } as any, + mockResponse as any + ); + + expect(res).toEqual({ + body: { message: 'Cannot specify both service_token and secrets.service_token' }, + statusCode: 400, + }); + }); + + it('should return ok if one of service_token and secrets.service_token is provided for remote_elasticsearch output', async () => { + jest + .spyOn(appContextService, 'getCloud') + .mockReturnValue({ isServerlessEnabled: false } as any); + + const res = await postOutputHandler( + mockContext, + { body: { type: 'remote_elasticsearch', secrets: { service_token: 'token2' } } } as any, + mockResponse as any + ); + + expect(res).toEqual({ body: { item: { id: 'output1' } } }); + }); }); diff --git a/x-pack/plugins/fleet/server/routes/output/handler.ts b/x-pack/plugins/fleet/server/routes/output/handler.ts index e100d9fd67e47..dc6682fe82727 100644 --- a/x-pack/plugins/fleet/server/routes/output/handler.ts +++ b/x-pack/plugins/fleet/server/routes/output/handler.ts @@ -37,9 +37,20 @@ function ensureNoDuplicateSecrets(output: Partial) { if (output.type === outputType.Kafka && output?.password && output?.secrets?.password) { throw Boom.badRequest('Cannot specify both password and secrets.password'); } - if (output.ssl?.key && output.secrets?.ssl?.key) { + if ( + (output.type === outputType.Kafka || output.type === outputType.Logstash) && + output.ssl?.key && + output.secrets?.ssl?.key + ) { throw Boom.badRequest('Cannot specify both ssl.key and secrets.ssl.key'); } + if ( + output.type === outputType.RemoteElasticsearch && + output.service_token && + output.secrets?.service_token + ) { + throw Boom.badRequest('Cannot specify both service_token and secrets.service_token'); + } } export const getOutputsHandler: RequestHandler = async (context, request, response) => { diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index b87310e850c81..7df074f418fb9 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -270,6 +270,12 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, }, }, + service_token: { + dynamic: false, + properties: { + id: { type: 'keyword' }, + }, + }, }, }, }, diff --git a/x-pack/plugins/fleet/server/services/fleet_proxies.ts b/x-pack/plugins/fleet/server/services/fleet_proxies.ts index 604276e6d5880..cf45b90804c22 100644 --- a/x-pack/plugins/fleet/server/services/fleet_proxies.ts +++ b/x-pack/plugins/fleet/server/services/fleet_proxies.ts @@ -205,7 +205,7 @@ async function updateRelatedSavedObject( outputService.update(soClient, esClient, output.id, { ...omit(output, 'id'), proxy_id: null, - }); + } as Partial); }, { concurrency: 20 } ); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts index 5bc7c452b481e..8872e20eb4fb2 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts @@ -82,7 +82,7 @@ export async function createOrUpdatePreconfiguredOutputs( ca_sha256: outputData.ca_sha256 ?? null, ca_trusted_fingerprint: outputData.ca_trusted_fingerprint ?? null, ssl: outputData.ssl ?? null, - }; + } as NewOutput; if (!data.hosts || data.hosts.length === 0) { data.hosts = outputService.getDefaultESHosts(); diff --git a/x-pack/plugins/fleet/server/services/secrets.ts b/x-pack/plugins/fleet/server/services/secrets.ts index 36a88b4a7a4c1..46e166ea4ecf1 100644 --- a/x-pack/plugins/fleet/server/services/secrets.ts +++ b/x-pack/plugins/fleet/server/services/secrets.ts @@ -10,7 +10,13 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/ import { keyBy } from 'lodash'; import { set } from '@kbn/safer-lodash-set'; -import type { KafkaOutput, Output, OutputSecretPath } from '../../common/types'; +import type { + KafkaOutput, + NewLogstashOutput, + NewRemoteElasticsearchOutput, + Output, + OutputSecretPath, +} from '../../common/types'; import { packageHasNoPolicyTemplates } from '../../common/services/policy_template'; @@ -280,11 +286,14 @@ function getOutputSecretPaths( ): OutputSecretPath[] { const outputSecretPaths: OutputSecretPath[] = []; - if ((outputType === 'kafka' || outputType === 'logstash') && output.secrets?.ssl?.key) { - outputSecretPaths.push({ - path: 'secrets.ssl.key', - value: output.secrets.ssl.key, - }); + if (outputType === 'logstash') { + const logstashOutput = output as NewLogstashOutput; + if (logstashOutput?.secrets?.ssl?.key) { + outputSecretPaths.push({ + path: 'secrets.ssl.key', + value: logstashOutput.secrets.ssl.key, + }); + } } if (outputType === 'kafka') { @@ -295,6 +304,22 @@ function getOutputSecretPaths( value: kafkaOutput.secrets.password, }); } + if (kafkaOutput?.secrets?.ssl?.key) { + outputSecretPaths.push({ + path: 'secrets.ssl.key', + value: kafkaOutput.secrets.ssl.key, + }); + } + } + + if (outputType === 'remote_elasticsearch') { + const remoteESOutput = output as NewRemoteElasticsearchOutput; + if (remoteESOutput.secrets?.service_token) { + outputSecretPaths.push({ + path: 'secrets.service_token', + value: remoteESOutput.secrets.service_token, + }); + } } return outputSecretPaths; @@ -340,6 +365,15 @@ export function getOutputSecretReferences(output: Output): PolicySecretReference }); } + if ( + output.type === 'remote_elasticsearch' && + typeof output?.secrets?.service_token === 'object' + ) { + outputSecretPaths.push({ + id: output.secrets.service_token.id, + }); + } + return outputSecretPaths; } diff --git a/x-pack/plugins/fleet/server/types/models/output.ts b/x-pack/plugins/fleet/server/types/models/output.ts index d77663b65e784..4c391e8383a58 100644 --- a/x-pack/plugins/fleet/server/types/models/output.ts +++ b/x-pack/plugins/fleet/server/types/models/output.ts @@ -130,13 +130,23 @@ const ElasticSearchUpdateSchema = { export const RemoteElasticSearchSchema = { ...ElasticSearchSchema, type: schema.literal(outputType.RemoteElasticsearch), - service_token: schema.string(), + service_token: schema.maybe(schema.string()), + secrets: schema.maybe( + schema.object({ + service_token: schema.maybe(secretRefSchema), + }) + ), }; const RemoteElasticSearchUpdateSchema = { ...ElasticSearchUpdateSchema, type: schema.maybe(schema.literal(outputType.RemoteElasticsearch)), service_token: schema.maybe(schema.string()), + secrets: schema.maybe( + schema.object({ + service_token: schema.maybe(secretRefSchema), + }) + ), }; /** diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index 66974744d4adb..df98f0b00a424 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -154,7 +154,9 @@ interface OutputSoElasticsearchAttributes extends OutputSoBaseAttributes { export interface OutputSoRemoteElasticsearchAttributes extends OutputSoBaseAttributes { type: OutputType['RemoteElasticsearch']; service_token?: string; - secrets?: {}; + secrets?: { + service_token?: { id: string }; + }; } interface OutputSoLogstashAttributes extends OutputSoBaseAttributes { diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index 5f6558df992ca..bd44a6be9427e 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -1132,6 +1132,23 @@ export default function (providerContext: FtrProviderContext) { // @ts-ignore _source unknown type expect(secret._source.value).to.equal('pass'); }); + + it('should create service_token secret correctly', async function () { + const res = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Remote Elasticsearch With Service Token Secret', + type: 'remote_elasticsearch', + hosts: ['https://remote-es:9200'], + secrets: { service_token: 'token' }, + }); + + const secretId = res.body.item.secrets.service_token.id; + const secret = await getSecretById(secretId); + // @ts-ignore _source unknown type + expect(secret._source.value).to.equal('token'); + }); }); describe('DELETE /outputs/{outputId}', () => { From 0a72738e4c223e7eebaf0dae688187152c1f1331 Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Tue, 28 Nov 2023 09:36:16 -0500 Subject: [PATCH 07/30] [Security Solution][Endpoint] server-side standard interface for response actions clients (#171755) ## Summary PR introduces a standard interface for Response Actions clients - currently only Endpoint, but in the near future, other clients will be introduced like SentinelOne. This PR is in preperation for that feature in a post v8.12 release. Changes include: - Introduction of `EndpointActionsClient` class (first Actions client using new standard interface) - Changed Response Actions API handler to: - use new `EndpointActionsClient` for processing response actions - added support for handling file `upload` response action (previously a separate handler) - now handles all errors using the common HTTP error handler - Deleted `upload` specific API HTTP handler - no longer needed as common handler will now also process `upload` response actions **NOTE:** No changes in functionality as a result of this PR. Just preparation work needed to support Bi-Directional Response Actions. --- .../actions/common/response_actions.ts | 5 + .../endpoint/actions/get_processes_route.ts | 2 + .../api/endpoint/actions/isolate_route.ts | 2 + .../response_actions/base_actions_provider.ts | 86 +++++++ .../endpoint/lib/response_actions/types.ts | 65 ++++++ .../actions/file_upload_handler.test.ts | 201 ---------------- .../routes/actions/file_upload_handler.ts | 158 ------------- .../routes/actions/response_actions.test.ts | 178 +++++++++++++-- .../routes/actions/response_actions.ts | 127 ++++++++--- .../clients/endpoint_actions_client.ts | 215 ++++++++++++++++++ .../services/actions/clients/index.ts | 8 + .../endpoint/services/actions/create/types.ts | 3 +- 12 files changed, 642 insertions(+), 408 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/response_actions/base_actions_provider.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/response_actions/types.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/actions/file_upload_handler.test.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/actions/file_upload_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/actions/clients/endpoint_actions_client.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/actions/clients/index.ts diff --git a/x-pack/plugins/security_solution/common/api/endpoint/actions/common/response_actions.ts b/x-pack/plugins/security_solution/common/api/endpoint/actions/common/response_actions.ts index c332f23f27025..269f041a25a10 100644 --- a/x-pack/plugins/security_solution/common/api/endpoint/actions/common/response_actions.ts +++ b/x-pack/plugins/security_solution/common/api/endpoint/actions/common/response_actions.ts @@ -5,7 +5,9 @@ * 2.0. */ +import type { TypeOf } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; +import { UploadActionRequestSchema } from '../..'; import { ExecuteActionRequestSchema } from '../execute_route'; import { EndpointActionGetFileSchema } from '../get_file_route'; import { KillOrSuspendProcessRequestSchema, NoParametersRequestSchema } from './base'; @@ -15,4 +17,7 @@ export const ResponseActionBodySchema = schema.oneOf([ KillOrSuspendProcessRequestSchema.body, EndpointActionGetFileSchema.body, ExecuteActionRequestSchema.body, + UploadActionRequestSchema.body, ]); + +export type ResponseActionsRequestBody = TypeOf; diff --git a/x-pack/plugins/security_solution/common/api/endpoint/actions/get_processes_route.ts b/x-pack/plugins/security_solution/common/api/endpoint/actions/get_processes_route.ts index e68194411748f..a9e56e52ba292 100644 --- a/x-pack/plugins/security_solution/common/api/endpoint/actions/get_processes_route.ts +++ b/x-pack/plugins/security_solution/common/api/endpoint/actions/get_processes_route.ts @@ -5,6 +5,8 @@ * 2.0. */ +import type { TypeOf } from '@kbn/config-schema'; import { NoParametersRequestSchema } from './common/base'; export const GetProcessesRouteRequestSchema = NoParametersRequestSchema; +export type GetProcessesRequestBody = TypeOf; diff --git a/x-pack/plugins/security_solution/common/api/endpoint/actions/isolate_route.ts b/x-pack/plugins/security_solution/common/api/endpoint/actions/isolate_route.ts index a58364c2be243..0df0d8d913457 100644 --- a/x-pack/plugins/security_solution/common/api/endpoint/actions/isolate_route.ts +++ b/x-pack/plugins/security_solution/common/api/endpoint/actions/isolate_route.ts @@ -5,6 +5,8 @@ * 2.0. */ +import type { TypeOf } from '@kbn/config-schema'; import { NoParametersRequestSchema } from './common/base'; export const IsolateRouteRequestSchema = NoParametersRequestSchema; +export type IsolationRouteRequestBody = TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/response_actions/base_actions_provider.ts b/x-pack/plugins/security_solution/server/endpoint/lib/response_actions/base_actions_provider.ts new file mode 100644 index 0000000000000..906402b877e0e --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/response_actions/base_actions_provider.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { CasesClient } from '@kbn/cases-plugin/server'; +import type { Logger } from '@kbn/logging'; +import type { EndpointAppContext } from '../../types'; +import type { ResponseActionsProvider } from './types'; +import type { + ActionDetails, + GetProcessesActionOutputContent, + KillOrSuspendProcessRequestBody, + KillProcessActionOutputContent, + ResponseActionExecuteOutputContent, + ResponseActionGetFileOutputContent, + ResponseActionGetFileParameters, + ResponseActionParametersWithPidOrEntityId, + ResponseActionsExecuteParameters, + ResponseActionUploadOutputContent, + ResponseActionUploadParameters, + SuspendProcessActionOutputContent, +} from '../../../../common/endpoint/types'; +import type { + IsolationRouteRequestBody, + ExecuteActionRequestBody, + GetProcessesRequestBody, + ResponseActionGetFileRequestBody, + UploadActionApiRequestBody, +} from '../../../../common/api/endpoint'; + +export interface BaseActionsProviderOptions { + endpointContext: EndpointAppContext; + esClient: ElasticsearchClient; + casesClient?: CasesClient; + /** Username that will be stored along with the action's ES documents */ + username: string; +} + +export abstract class BaseResponseActionsClient implements ResponseActionsProvider { + protected readonly log: Logger; + + constructor(protected readonly options: BaseActionsProviderOptions) { + this.log = options.endpointContext.logFactory.get(this.constructor.name ?? 'ActionsProvider'); + } + + // TODO:PT implement a generic way to update cases without relying on the Attachments being endpoint agents + // protected async updateCases(): Promise { + // throw new Error('Method not yet implemented'); + // } + + public abstract isolate(options: IsolationRouteRequestBody): Promise; + + public abstract release(options: IsolationRouteRequestBody): Promise; + + public abstract killProcess( + options: KillOrSuspendProcessRequestBody + ): Promise< + ActionDetails + >; + + public abstract suspendProcess( + options: KillOrSuspendProcessRequestBody + ): Promise< + ActionDetails + >; + + public abstract runningProcesses( + options: GetProcessesRequestBody + ): Promise>; + + public abstract getFile( + options: ResponseActionGetFileRequestBody + ): Promise>; + + public abstract execute( + options: ExecuteActionRequestBody + ): Promise>; + + public abstract upload( + options: UploadActionApiRequestBody + ): Promise>; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/response_actions/types.ts b/x-pack/plugins/security_solution/server/endpoint/lib/response_actions/types.ts new file mode 100644 index 0000000000000..5578128cf4248 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/response_actions/types.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + ActionDetails, + KillOrSuspendProcessRequestBody, + KillProcessActionOutputContent, + ResponseActionParametersWithPidOrEntityId, + SuspendProcessActionOutputContent, + GetProcessesActionOutputContent, + ResponseActionGetFileOutputContent, + ResponseActionGetFileParameters, + ResponseActionExecuteOutputContent, + ResponseActionsExecuteParameters, + ResponseActionUploadOutputContent, + ResponseActionUploadParameters, +} from '../../../../common/endpoint/types'; +import type { + IsolationRouteRequestBody, + GetProcessesRequestBody, + ResponseActionGetFileRequestBody, + ExecuteActionRequestBody, + UploadActionApiRequestBody, +} from '../../../../common/api/endpoint'; + +/** + * The interface required for a Response Actions provider + */ +export interface ResponseActionsProvider { + isolate: (options: IsolationRouteRequestBody) => Promise; + + release: (options: IsolationRouteRequestBody) => Promise; + + killProcess: ( + options: KillOrSuspendProcessRequestBody + ) => Promise< + ActionDetails + >; + + suspendProcess: ( + options: KillOrSuspendProcessRequestBody + ) => Promise< + ActionDetails + >; + + runningProcesses: ( + options: GetProcessesRequestBody + ) => Promise>; + + getFile: ( + options: ResponseActionGetFileRequestBody + ) => Promise>; + + execute: ( + options: ExecuteActionRequestBody + ) => Promise>; + + upload: ( + options: UploadActionApiRequestBody + ) => Promise>; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/file_upload_handler.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/file_upload_handler.test.ts deleted file mode 100644 index d8cda7554a089..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/file_upload_handler.test.ts +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { HttpApiTestSetupMock } from '../../mocks'; -import { createHttpApiTestSetupMock } from '../../mocks'; -import type { UploadActionApiRequestBody } from '../../../../common/api/endpoint'; -import type { getActionFileUploadHandler } from './file_upload_handler'; -import { registerActionFileUploadRoute } from './file_upload_handler'; -import { UPLOAD_ROUTE } from '../../../../common/endpoint/constants'; -import { getEndpointAuthzInitialStateMock } from '../../../../common/endpoint/service/authz/mocks'; -import { EndpointAuthorizationError } from '../../errors'; -import type { HapiReadableStream } from '../../../types'; -import { createHapiReadableStreamMock } from '../../services/actions/mocks'; -import { EndpointActionGenerator } from '../../../../common/endpoint/data_generators/endpoint_action_generator'; -import { CustomHttpRequestError } from '../../../utils/custom_http_request_error'; -import type { ActionDetails } from '../../../../common/endpoint/types'; -import { omit } from 'lodash'; -import type { FleetToHostFileClientInterface } from '@kbn/fleet-plugin/server'; - -describe('Upload response action create API handler', () => { - type UploadHttpApiTestSetupMock = HttpApiTestSetupMock; - - let testSetup: UploadHttpApiTestSetupMock; - let httpRequestMock: ReturnType; - let httpHandlerContextMock: UploadHttpApiTestSetupMock['httpHandlerContextMock']; - let httpResponseMock: UploadHttpApiTestSetupMock['httpResponseMock']; - - let fleetFilesClientMock: jest.Mocked; - - beforeEach(async () => { - testSetup = createHttpApiTestSetupMock(); - - ({ httpHandlerContextMock, httpResponseMock } = testSetup); - httpRequestMock = testSetup.createRequestMock(); - - fleetFilesClientMock = - (await testSetup.endpointAppContextMock.service.getFleetToHostFilesClient()) as jest.Mocked; - }); - - describe('registerActionFileUploadRoute()', () => { - it('should register the route', () => { - registerActionFileUploadRoute(testSetup.routerMock, testSetup.endpointAppContextMock); - - expect( - testSetup.getRegisteredVersionedRoute('post', UPLOAD_ROUTE, '2023-10-31') - ).toBeDefined(); - }); - - it('should NOT register route if feature flag is false', () => { - // @ts-expect-error - testSetup.endpointAppContextMock.experimentalFeatures.responseActionUploadEnabled = false; - registerActionFileUploadRoute(testSetup.routerMock, testSetup.endpointAppContextMock); - - expect(() => - testSetup.getRegisteredVersionedRoute('post', UPLOAD_ROUTE, '2023-10-31') - ).toThrow('No routes registered for [POST /api/endpoint/action/upload]'); - }); - - it('should use maxUploadResponseActionFileBytes config value', () => { - // @ts-expect-error - testSetup.endpointAppContextMock.serverConfig.maxUploadResponseActionFileBytes = 999; - registerActionFileUploadRoute(testSetup.routerMock, testSetup.endpointAppContextMock); - - expect( - testSetup.getRegisteredVersionedRoute('post', UPLOAD_ROUTE, '2023-10-31').routeConfig - ?.options?.body - ).toEqual({ - accepts: ['multipart/form-data'], - maxBytes: 999, - output: 'stream', - }); - }); - - it('should error if user has no authz to api', async () => { - ( - (await httpHandlerContextMock.securitySolution).getEndpointAuthz as jest.Mock - ).mockResolvedValue(getEndpointAuthzInitialStateMock({ canWriteFileOperations: false })); - registerActionFileUploadRoute(testSetup.routerMock, testSetup.endpointAppContextMock); - await testSetup - .getRegisteredVersionedRoute('post', UPLOAD_ROUTE, '2023-10-31') - .routeHandler(httpHandlerContextMock, httpRequestMock, httpResponseMock); - - expect(httpResponseMock.forbidden).toHaveBeenCalledWith({ - body: expect.any(EndpointAuthorizationError), - }); - }); - }); - - describe('route request handler', () => { - let callHandler: () => ReturnType>; - let fileContent: HapiReadableStream; - let createdUploadAction: ActionDetails; - - beforeEach(() => { - fileContent = createHapiReadableStreamMock(); - - const reqBody: UploadActionApiRequestBody = { - file: fileContent, - endpoint_ids: ['123-456'], - parameters: { - overwrite: true, - }, - }; - - httpRequestMock = testSetup.createRequestMock({ body: reqBody }); - registerActionFileUploadRoute(testSetup.routerMock, testSetup.endpointAppContextMock); - - createdUploadAction = new EndpointActionGenerator('seed').generateActionDetails({ - command: 'upload', - }); - - ( - testSetup.endpointAppContextMock.service.getActionCreateService().createAction as jest.Mock - ).mockResolvedValue(createdUploadAction); - - (testSetup.endpointAppContextMock.service.getEndpointMetadataService as jest.Mock) = jest - .fn() - .mockReturnValue({ - getMetadataForEndpoints: jest.fn().mockResolvedValue([ - { - elastic: { - agent: { - id: '123-456', - }, - }, - }, - ]), - }); - - const handler: ReturnType = - testSetup.getRegisteredVersionedRoute('post', UPLOAD_ROUTE, '2023-10-31').routeHandler; - - callHandler = () => handler(httpHandlerContextMock, httpRequestMock, httpResponseMock); - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - it('should create a file', async () => { - await callHandler(); - - expect(fleetFilesClientMock.create).toHaveBeenCalledWith(fileContent, ['123-456']); - }); - - it('should create the action using parameters with stored file info', async () => { - await callHandler(); - - const createActionMock = testSetup.endpointAppContextMock.service.getActionCreateService() - .createAction as jest.Mock; - - expect(createActionMock).toHaveBeenCalledWith( - { - command: 'upload', - endpoint_ids: ['123-456'], - parameters: { - file_id: '123-456-789', - file_name: 'foo.txt', - file_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - file_size: 45632, - overwrite: true, - }, - user: undefined, - }, - ['123-456'] - ); - }); - - it('should delete file if creation of Action fails', async () => { - const createActionMock = testSetup.endpointAppContextMock.service.getActionCreateService() - .createAction as jest.Mock; - createActionMock.mockImplementation(async () => { - throw new CustomHttpRequestError('oh oh'); - }); - await callHandler(); - - expect(fleetFilesClientMock.delete).toHaveBeenCalledWith('123-456-789'); - }); - - it('should update file with action id', async () => { - await callHandler(); - - expect(fleetFilesClientMock.update).toHaveBeenCalledWith('123-456-789', { actionId: '123' }); - }); - - it('should return expected response on success', async () => { - await callHandler(); - - expect(httpResponseMock.ok).toHaveBeenCalledWith({ - body: { - action: createdUploadAction.action, - data: omit(createdUploadAction, 'action'), - }, - }); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/file_upload_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/file_upload_handler.ts deleted file mode 100644 index 3b51bb48a0104..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/file_upload_handler.ts +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { RequestHandler } from '@kbn/core/server'; -import type { UploadActionApiRequestBody } from '../../../../common/api/endpoint'; -import { UploadActionRequestSchema } from '../../../../common/api/endpoint'; -import type { ResponseActionsApiCommandNames } from '../../../../common/endpoint/service/response_actions/constants'; -import type { - ResponseActionUploadParameters, - ResponseActionUploadOutputContent, - HostMetadata, -} from '../../../../common/endpoint/types'; -import { UPLOAD_ROUTE } from '../../../../common/endpoint/constants'; -import { withEndpointAuthz } from '../with_endpoint_authz'; -import type { - SecuritySolutionPluginRouter, - SecuritySolutionRequestHandlerContext, - HapiReadableStream, -} from '../../../types'; -import type { EndpointAppContext } from '../../types'; -import { errorHandler } from '../error_handler'; -import { updateCases } from '../../services/actions/create/update_cases'; - -export const registerActionFileUploadRoute = ( - router: SecuritySolutionPluginRouter, - endpointContext: EndpointAppContext -) => { - if (!endpointContext.experimentalFeatures.responseActionUploadEnabled) { - return; - } - - const logger = endpointContext.logFactory.get('uploadAction'); - - router.versioned - .post({ - access: 'public', - path: UPLOAD_ROUTE, - options: { - authRequired: true, - tags: ['access:securitySolution'], - body: { - accepts: ['multipart/form-data'], - output: 'stream', - maxBytes: endpointContext.serverConfig.maxUploadResponseActionFileBytes, - }, - }, - }) - .addVersion( - { - version: '2023-10-31', - validate: { - request: UploadActionRequestSchema, - }, - }, - withEndpointAuthz( - { all: ['canWriteFileOperations'] }, - logger, - getActionFileUploadHandler(endpointContext) - ) - ); -}; - -export const getActionFileUploadHandler = ( - endpointContext: EndpointAppContext -): RequestHandler< - never, - never, - UploadActionApiRequestBody, - SecuritySolutionRequestHandlerContext -> => { - const logger = endpointContext.logFactory.get('uploadAction'); - - return async (context, req, res) => { - const fleetFiles = await endpointContext.service.getFleetToHostFilesClient(); - const user = endpointContext.service.security?.authc.getCurrentUser(req); - const fileStream = req.body.file as HapiReadableStream; - const { file: _, parameters: userParams, ...actionPayload } = req.body; - const uploadParameters: ResponseActionUploadParameters = { - ...userParams, - file_id: '', - file_name: '', - file_sha256: '', - file_size: 0, - }; - - try { - const createdFile = await fleetFiles.create(fileStream, actionPayload.endpoint_ids); - - uploadParameters.file_id = createdFile.id; - uploadParameters.file_name = createdFile.name; - uploadParameters.file_sha256 = createdFile.sha256; - uploadParameters.file_size = createdFile.size; - } catch (err) { - return errorHandler(logger, res, err); - } - - const createActionPayload = { - ...actionPayload, - parameters: uploadParameters, - command: 'upload' as ResponseActionsApiCommandNames, - user, - }; - - const esClient = (await context.core).elasticsearch.client.asInternalUser; - const endpointData = await endpointContext.service - .getEndpointMetadataService() - .getMetadataForEndpoints(esClient, [...new Set(createActionPayload.endpoint_ids)]); - const agentIds = endpointData.map((endpoint: HostMetadata) => endpoint.elastic.agent.id); - - try { - const casesClient = await endpointContext.service.getCasesClient(req); - const { action: actionId, ...data } = await endpointContext.service - .getActionCreateService() - .createAction( - createActionPayload, - agentIds - ); - - // Update the file meta to include the action id, and if any errors (unlikely), - // then just log them and still allow api to return success since the action has - // already been created and potentially dispatched to Endpoint. Action ID is not - // needed by the Endpoint or fleet-server's API, so no need to fail here - try { - await fleetFiles.update(uploadParameters.file_id, { actionId: data.id }); - } catch (e) { - logger.warn(`Attempt to update File meta with Action ID failed: ${e.message}`, e); - } - - // update cases - await updateCases({ casesClient, createActionPayload, endpointData }); - - return res.ok({ - body: { - action: actionId, - data, - }, - }); - } catch (err) { - if (uploadParameters.file_id) { - // Try to delete the created file since creating the action threw an error - try { - await fleetFiles.delete(uploadParameters.file_id); - } catch (e) { - logger.error( - `Attempt to clean up file (after action creation was unsuccessful) failed; ${e.message}`, - e - ); - } - } - - return errorHandler(logger, res, err); - } - }; -}; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/response_actions.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/response_actions.test.ts index dd10093cc343c..8f7682c83daad 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/response_actions.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/response_actions.test.ts @@ -34,6 +34,7 @@ import { UNISOLATE_HOST_ROUTE, GET_FILE_ROUTE, EXECUTE_ROUTE, + UPLOAD_ROUTE, } from '../../../../common/endpoint/constants'; import type { ActionDetails, @@ -46,7 +47,9 @@ import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data' import type { EndpointAuthz } from '../../../../common/endpoint/types/authz'; import type { SecuritySolutionRequestHandlerContextMock } from '../../../lib/detection_engine/routes/__mocks__/request_context'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; +import type { HttpApiTestSetupMock } from '../../mocks'; import { + createHttpApiTestSetupMock, createMockEndpointAppContext, createMockEndpointAppContextServiceSetupContract, createMockEndpointAppContextServiceStartContract, @@ -59,6 +62,13 @@ import * as ActionDetailsService from '../../services/actions/action_details_by_ import { CaseStatuses } from '@kbn/cases-components'; import { getEndpointAuthzInitialStateMock } from '../../../../common/endpoint/service/authz/mocks'; import { actionCreateService } from '../../services/actions'; +import type { UploadActionApiRequestBody } from '../../../../common/api/endpoint'; +import type { FleetToHostFileClientInterface } from '@kbn/fleet-plugin/server'; +import type { HapiReadableStream, SecuritySolutionRequestHandlerContext } from '../../../types'; +import { createHapiReadableStreamMock } from '../../services/actions/mocks'; +import { EndpointActionGenerator } from '../../../../common/endpoint/data_generators/endpoint_action_generator'; +import { CustomHttpRequestError } from '../../../utils/custom_http_request_error'; +import { omit } from 'lodash'; interface CallRouteInterface { body?: ResponseActionRequestBody; @@ -827,26 +837,26 @@ describe('Response actions', () => { }); it('handles errors', async () => { - const errMessage = 'Uh oh!'; - await callRoute( - UNISOLATE_HOST_ROUTE_V2, - { - body: { endpoint_ids: ['XYZ'] }, - version: '2023-10-31', - indexErrorResponse: { - statusCode: 500, - body: { - result: errMessage, + const expectedError = new Error('Uh oh!'); + + await expect( + callRoute( + UNISOLATE_HOST_ROUTE_V2, + { + body: { endpoint_ids: ['XYZ'] }, + version: '2023-10-31', + indexErrorResponse: { + statusCode: 500, + body: { + result: expectedError.message, + }, }, }, - }, - { endpointDsExists: true } - ); + { endpointDsExists: true } + ) + ).rejects.toEqual(expectedError); expect(mockResponse.ok).not.toBeCalled(); - const response = mockResponse.customError.mock.calls[0][0]; - expect(response.statusCode).toEqual(500); - expect((response.body as Error).message).toEqual(errMessage); }); }); @@ -1001,4 +1011,140 @@ describe('Response actions', () => { }); }); }); + + describe('Upload response action handler', () => { + type UploadHttpApiTestSetupMock = HttpApiTestSetupMock< + never, + never, + UploadActionApiRequestBody + >; + type UploadRequestHandler = RequestHandler< + never, + never, + UploadActionApiRequestBody, + SecuritySolutionRequestHandlerContext + >; + + let testSetup: UploadHttpApiTestSetupMock; + let httpRequestMock: ReturnType; + let httpHandlerContextMock: UploadHttpApiTestSetupMock['httpHandlerContextMock']; + let httpResponseMock: UploadHttpApiTestSetupMock['httpResponseMock']; + let fleetFilesClientMock: jest.Mocked; + let callHandler: () => ReturnType; + let fileContent: HapiReadableStream; + let createdUploadAction: ActionDetails; + + beforeEach(async () => { + testSetup = createHttpApiTestSetupMock(); + + ({ httpHandlerContextMock, httpResponseMock } = testSetup); + httpRequestMock = testSetup.createRequestMock(); + + fleetFilesClientMock = + (await testSetup.endpointAppContextMock.service.getFleetToHostFilesClient()) as jest.Mocked; + + fileContent = createHapiReadableStreamMock(); + + const reqBody: UploadActionApiRequestBody = { + file: fileContent, + endpoint_ids: ['123-456'], + parameters: { + overwrite: true, + }, + }; + + httpRequestMock = testSetup.createRequestMock({ body: reqBody }); + registerResponseActionRoutes(testSetup.routerMock, testSetup.endpointAppContextMock); + + createdUploadAction = new EndpointActionGenerator('seed').generateActionDetails({ + command: 'upload', + }); + + ( + testSetup.endpointAppContextMock.service.getActionCreateService().createAction as jest.Mock + ).mockResolvedValue(createdUploadAction); + + (testSetup.endpointAppContextMock.service.getEndpointMetadataService as jest.Mock) = jest + .fn() + .mockReturnValue({ + getMetadataForEndpoints: jest.fn().mockResolvedValue([ + { + elastic: { + agent: { + id: '123-456', + }, + }, + }, + ]), + }); + + const handler = testSetup.getRegisteredVersionedRoute('post', UPLOAD_ROUTE, '2023-10-31') + .routeHandler as UploadRequestHandler; + + callHandler = () => handler(httpHandlerContextMock, httpRequestMock, httpResponseMock); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should create a file', async () => { + await callHandler(); + + expect(fleetFilesClientMock.create).toHaveBeenCalledWith(fileContent, ['123-456']); + }); + + it('should create the action using parameters with stored file info', async () => { + await callHandler(); + + const createActionMock = testSetup.endpointAppContextMock.service.getActionCreateService() + .createAction as jest.Mock; + + expect(createActionMock).toHaveBeenCalledWith( + { + command: 'upload', + endpoint_ids: ['123-456'], + parameters: { + file_id: '123-456-789', + file_name: 'foo.txt', + file_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + file_size: 45632, + overwrite: true, + }, + user: { username: 'unknown' }, + }, + ['123-456'] + ); + }); + + it('should delete file if creation of Action fails', async () => { + const createActionMock = testSetup.endpointAppContextMock.service.getActionCreateService() + .createAction as jest.Mock; + createActionMock.mockImplementation(async () => { + throw new CustomHttpRequestError('oh oh'); + }); + await callHandler(); + + expect(fleetFilesClientMock.delete).toHaveBeenCalledWith('123-456-789'); + }); + + it('should update file with action id', async () => { + await callHandler(); + + expect(fleetFilesClientMock.update).toHaveBeenCalledWith('123-456-789', { + actionId: '123', + }); + }); + + it('should return expected response on success', async () => { + await callHandler(); + + expect(httpResponseMock.ok).toHaveBeenCalledWith({ + body: { + action: createdUploadAction.action, + data: omit(createdUploadAction, 'action'), + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/response_actions.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/response_actions.ts index 3b98efa12cd98..c8669869833fe 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/response_actions.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/response_actions.ts @@ -7,9 +7,14 @@ import type { RequestHandler } from '@kbn/core/server'; import type { TypeOf } from '@kbn/config-schema'; +import { CustomHttpRequestError } from '../../../utils/custom_http_request_error'; +import { EndpointActionsClient } from '../../services/actions/clients'; import type { - ResponseActionBodySchema, NoParametersRequestSchema, + ResponseActionsRequestBody, + ExecuteActionRequestBody, + ResponseActionGetFileRequestBody, + UploadActionApiRequestBody, } from '../../../../common/api/endpoint'; import { ExecuteActionRequestSchema, @@ -19,6 +24,7 @@ import { SuspendProcessRouteRequestSchema, UnisolateRouteRequestSchema, GetProcessesRouteRequestSchema, + UploadActionRequestSchema, } from '../../../../common/api/endpoint'; import { @@ -31,13 +37,14 @@ import { UNISOLATE_HOST_ROUTE, GET_FILE_ROUTE, EXECUTE_ROUTE, + UPLOAD_ROUTE, } from '../../../../common/endpoint/constants'; import type { EndpointActionDataParameterTypes, ResponseActionParametersWithPidOrEntityId, ResponseActionsExecuteParameters, ActionDetails, - HostMetadata, + KillOrSuspendProcessRequestBody, } from '../../../../common/endpoint/types'; import type { ResponseActionsApiCommandNames } from '../../../../common/endpoint/service/response_actions/constants'; import type { @@ -46,8 +53,7 @@ import type { } from '../../../types'; import type { EndpointAppContext } from '../../types'; import { withEndpointAuthz } from '../with_endpoint_authz'; -import { registerActionFileUploadRoute } from './file_upload_handler'; -import { updateCases } from '../../services/actions/create/update_cases'; +import { errorHandler } from '../error_handler'; export function registerResponseActionRoutes( router: SecuritySolutionPluginRouter, @@ -243,7 +249,33 @@ export function registerResponseActionRoutes( ) ); - registerActionFileUploadRoute(router, endpointContext); + router.versioned + .post({ + access: 'public', + path: UPLOAD_ROUTE, + options: { + authRequired: true, + tags: ['access:securitySolution'], + body: { + accepts: ['multipart/form-data'], + output: 'stream', + maxBytes: endpointContext.serverConfig.maxUploadResponseActionFileBytes, + }, + }, + }) + .addVersion( + { + version: '2023-10-31', + validate: { + request: UploadActionRequestSchema, + }, + }, + withEndpointAuthz( + { all: ['canWriteFileOperations'] }, + logger, + responseActionRequestHandler(endpointContext, 'upload') + ) + ); } function responseActionRequestHandler( @@ -252,43 +284,76 @@ function responseActionRequestHandler, + ResponseActionsRequestBody, SecuritySolutionRequestHandlerContext > { + const logger = endpointContext.logFactory.get('responseActionsHandler'); + return async (context, req, res) => { const user = endpointContext.service.security?.authc.getCurrentUser(req); const esClient = (await context.core).elasticsearch.client.asInternalUser; - - let action: ActionDetails; + const casesClient = await endpointContext.service.getCasesClient(req); + const actionsClient = new EndpointActionsClient({ + esClient, + casesClient, + endpointContext, + username: user?.username ?? 'unknown', + }); try { - const createActionPayload = { ...req.body, command, user }; - const endpointData = await endpointContext.service - .getEndpointMetadataService() - .getMetadataForEndpoints(esClient, [...new Set(createActionPayload.endpoint_ids)]); - const agentIds = endpointData.map((endpoint: HostMetadata) => endpoint.elastic.agent.id); + let action: ActionDetails; - action = await endpointContext.service - .getActionCreateService() - .createAction(createActionPayload, agentIds); + switch (command) { + case 'isolate': + action = await actionsClient.isolate(req.body); + break; - // update cases - const casesClient = await endpointContext.service.getCasesClient(req); - await updateCases({ casesClient, createActionPayload, endpointData }); - } catch (err) { - return res.customError({ - statusCode: 500, - body: err, + case 'unisolate': + action = await actionsClient.release(req.body); + break; + + case 'running-processes': + action = await actionsClient.runningProcesses(req.body); + break; + + case 'execute': + action = await actionsClient.execute(req.body as ExecuteActionRequestBody); + break; + + case 'suspend-process': + action = await actionsClient.suspendProcess(req.body as KillOrSuspendProcessRequestBody); + break; + + case 'kill-process': + action = await actionsClient.killProcess(req.body as KillOrSuspendProcessRequestBody); + break; + + case 'get-file': + action = await actionsClient.getFile(req.body as ResponseActionGetFileRequestBody); + break; + + case 'upload': + action = await actionsClient.upload(req.body as UploadActionApiRequestBody); + break; + + default: + throw new CustomHttpRequestError( + `No handler found for response action command: [${command}]`, + 501 + ); + } + + const { action: actionId, ...data } = action; + + return res.ok({ + body: { + action: actionId, + data, + }, }); + } catch (err) { + return errorHandler(logger, res, err); } - - const { action: actionId, ...data } = action; - return res.ok({ - body: { - action: actionId, - data, - }, - }); }; } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/endpoint_actions_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/endpoint_actions_client.ts new file mode 100644 index 0000000000000..2b5813fa6c909 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/endpoint_actions_client.ts @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { HapiReadableStream } from '../../../../types'; +import type { ResponseActionsApiCommandNames } from '../../../../../common/endpoint/service/response_actions/constants'; +import { updateCases } from '../create/update_cases'; +import type { CreateActionPayload } from '../create/types'; +import type { + ExecuteActionRequestBody, + GetProcessesRequestBody, + IsolationRouteRequestBody, + ResponseActionGetFileRequestBody, + UploadActionApiRequestBody, + ResponseActionsRequestBody, +} from '../../../../../common/api/endpoint'; +import { BaseResponseActionsClient } from '../../../lib/response_actions/base_actions_provider'; +import type { + ActionDetails, + HostMetadata, + GetProcessesActionOutputContent, + KillOrSuspendProcessRequestBody, + KillProcessActionOutputContent, + ResponseActionExecuteOutputContent, + ResponseActionGetFileOutputContent, + ResponseActionGetFileParameters, + ResponseActionParametersWithPidOrEntityId, + ResponseActionsExecuteParameters, + ResponseActionUploadOutputContent, + ResponseActionUploadParameters, + SuspendProcessActionOutputContent, + HostMetadataInterface, + ImmutableObject, +} from '../../../../../common/endpoint/types'; + +export class EndpointActionsClient extends BaseResponseActionsClient { + private async checkAgentIds(ids: string[]): Promise<{ + valid: string[]; + invalid: string[]; + allValid: boolean; + hosts: HostMetadata[]; + }> { + const foundEndpointHosts = await this.options.endpointContext.service + .getEndpointMetadataService() + .getMetadataForEndpoints(this.options.esClient, [...new Set(ids)]); + const validIds = foundEndpointHosts.map((endpoint: HostMetadata) => endpoint.elastic.agent.id); + const invalidIds = ids.filter((id) => !validIds.includes(id)); + + if (invalidIds.length) { + this.log.debug(`The following agent ids are not valid: ${JSON.stringify(invalidIds)}`); + } + + return { + valid: validIds, + invalid: invalidIds, + allValid: invalidIds.length === 0, + hosts: foundEndpointHosts, + }; + } + + private async handleResponseAction< + TOptions extends ResponseActionsRequestBody = ResponseActionsRequestBody, + TResponse extends ActionDetails = ActionDetails + >(command: ResponseActionsApiCommandNames, options: TOptions): Promise { + const agentIds = await this.checkAgentIds(options.endpoint_ids); + const createPayload: CreateActionPayload = { + ...options, + command, + user: { username: this.options.username }, + }; + + const response = await this.options.endpointContext.service + .getActionCreateService() + .createAction(createPayload, agentIds.valid); + + await this.updateCases(createPayload, agentIds.hosts); + + return response as TResponse; + } + + protected async updateCases( + createActionPayload: CreateActionPayload, + endpointData: Array> + ): Promise { + return updateCases({ + casesClient: this.options.casesClient, + createActionPayload, + endpointData, + }); + } + + async isolate(options: IsolationRouteRequestBody): Promise { + return this.handleResponseAction('isolate', options); + } + + async release(options: IsolationRouteRequestBody): Promise { + return this.handleResponseAction( + 'unisolate', + options + ); + } + + async killProcess( + options: KillOrSuspendProcessRequestBody + ): Promise< + ActionDetails + > { + return this.handleResponseAction< + KillOrSuspendProcessRequestBody, + ActionDetails + >('kill-process', options); + } + + async suspendProcess( + options: KillOrSuspendProcessRequestBody + ): Promise< + ActionDetails + > { + return this.handleResponseAction< + KillOrSuspendProcessRequestBody, + ActionDetails + >('suspend-process', options); + } + + async runningProcesses( + options: GetProcessesRequestBody + ): Promise> { + return this.handleResponseAction< + GetProcessesRequestBody, + ActionDetails + >('running-processes', options); + } + + async getFile( + options: ResponseActionGetFileRequestBody + ): Promise> { + return this.handleResponseAction< + ResponseActionGetFileRequestBody, + ActionDetails + >('get-file', options); + } + + async execute( + options: ExecuteActionRequestBody + ): Promise> { + return this.handleResponseAction< + ExecuteActionRequestBody, + ActionDetails + >('execute', options); + } + + async upload( + options: UploadActionApiRequestBody + ): Promise> { + const fleetFiles = await this.options.endpointContext.service.getFleetToHostFilesClient(); + const fileStream = options.file as HapiReadableStream; + const { file: _, parameters: userParams, ...actionPayload } = options; + const uploadParameters: ResponseActionUploadParameters = { + ...userParams, + file_id: '', + file_name: '', + file_sha256: '', + file_size: 0, + }; + + const createdFile = await fleetFiles.create(fileStream, actionPayload.endpoint_ids); + + uploadParameters.file_id = createdFile.id; + uploadParameters.file_name = createdFile.name; + uploadParameters.file_sha256 = createdFile.sha256; + uploadParameters.file_size = createdFile.size; + + const createFileActionOptions = { + ...actionPayload, + parameters: uploadParameters, + command: 'upload' as ResponseActionsApiCommandNames, + }; + + try { + const createdAction = await this.handleResponseAction< + typeof createFileActionOptions, + ActionDetails + >('upload', createFileActionOptions); + + // Update the file meta to include the action id, and if any errors (unlikely), + // then just log them and still allow api to return success since the action has + // already been created and potentially dispatched to Endpoint. Action ID is not + // needed by the Endpoint or fleet-server's API, so no need to fail here + try { + await fleetFiles.update(uploadParameters.file_id, { actionId: createdAction.id }); + } catch (e) { + this.log.warn(`Attempt to update File meta with Action ID failed: ${e.message}`, e); + } + + return createdAction; + } catch (err) { + if (uploadParameters.file_id) { + // Try to delete the created file since creating the action threw an error + try { + await fleetFiles.delete(uploadParameters.file_id); + } catch (e) { + this.log.error( + `Attempt to clean up file id [${uploadParameters.file_id}] (after action creation was unsuccessful) failed; ${e.message}`, + e + ); + } + } + + throw err; + } + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/index.ts new file mode 100644 index 0000000000000..83a11352dfe95 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { EndpointActionsClient } from './endpoint_actions_client'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/create/types.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/create/types.ts index aef5082bbb4a5..f8a10b18d594d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/create/types.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/create/types.ts @@ -5,7 +5,6 @@ * 2.0. */ -import type { AuthenticationServiceStart } from '@kbn/security-plugin/server'; import type { LicenseType } from '@kbn/licensing-plugin/server'; import type { TypeOf } from '@kbn/config-schema'; import type { ResponseActionBodySchema } from '../../../../../common/api/endpoint'; @@ -17,7 +16,7 @@ import type { ResponseActionsApiCommandNames } from '../../../../../common/endpo export type CreateActionPayload = TypeOf & { command: ResponseActionsApiCommandNames; - user?: ReturnType; + user?: { username: string } | null | undefined; rule_id?: string; rule_name?: string; error?: string; From 8413078c31f814257f3fab0e10c4d47f0eec8edc Mon Sep 17 00:00:00 2001 From: Navarone Feekery <13634519+navarone-feekery@users.noreply.github.com> Date: Tue, 28 Nov 2023 15:38:16 +0100 Subject: [PATCH 08/30] [Search] Update Connector ACL name pattern (#172057) ## Summary This PR changes the ACL index pattern logic. Previously an index named `search-foo` would create an ACL index named `.search-acl-filter-foo`. If a user wants to also use an index named just `foo`, the ACL index generated would be identical. These changes simplify the index name creation. Now, indices will look like: - `search-foo` -> `.search-acl-filter-search-foo` - `foo` -> `.search-acl-filter-foo` Migrations for this have been added already to Enterprise Search. --- .../components/search_index/documents.tsx | 3 +-- .../server/lib/connectors/start_sync.test.ts | 2 +- .../server/lib/connectors/start_sync.ts | 4 +--- .../lib/indices/delete_access_control_index.test.ts | 6 +++--- .../server/lib/indices/delete_access_control_index.ts | 7 ++++--- .../server/lib/indices/generate_api_key.test.ts | 2 +- .../server/lib/indices/generate_api_key.ts | 9 ++++++--- 7 files changed, 17 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx index b2932c9547e27..999bd333a0ffb 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx @@ -26,7 +26,6 @@ import { ENTERPRISE_SEARCH_DOCUMENTS_DEFAULT_DOC_COUNT, } from '../../../../../common/constants'; import { Status } from '../../../../../common/types/api'; -import { stripSearchPrefix } from '../../../../../common/utils/strip_search_prefix'; import { DEFAULT_META } from '../../../shared/constants'; import { KibanaLogic } from '../../../shared/kibana'; @@ -69,7 +68,7 @@ export const SearchIndexDocuments: React.FC = () => { const indexToShow = selectedIndexType === 'content-index' ? indexName - : stripSearchPrefix(indexName, CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX); + : `${CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX}${indexName}`; const mappingLogic = mappingsWithPropsApiLogic(indexToShow); const documentLogic = searchDocumentsApiLogic(indexToShow); diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.test.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.test.ts index 978896289763f..6c5d44b00628d 100644 --- a/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.test.ts @@ -345,7 +345,7 @@ describe('startSync lib function', () => { }, filtering: null, id: 'connectorId', - index_name: `${CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX}index_name`, + index_name: `${CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX}search-index_name`, language: null, pipeline: null, service_type: null, diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.ts index ab58fd1417b73..634d7e97de5ec 100644 --- a/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.ts +++ b/x-pack/plugins/enterprise_search/server/lib/connectors/start_sync.ts @@ -23,7 +23,6 @@ import { } from '../../../common/constants'; import { ErrorCode } from '../../../common/types/error_codes'; -import { stripSearchPrefix } from '../../../common/utils/strip_search_prefix'; export const startSync = async ( client: IScopedClusterClient, @@ -70,10 +69,9 @@ export const startSync = async ( }); } - const indexNameWithoutSearchPrefix = index_name ? stripSearchPrefix(index_name) : ''; const targetIndexName = jobType === SyncJobType.ACCESS_CONTROL - ? `${CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX}${indexNameWithoutSearchPrefix}` + ? `${CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX}${index_name}` : index_name ?? undefined; return await startConnectorSync(client.asCurrentUser, { diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/delete_access_control_index.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/delete_access_control_index.test.ts index 0439c12c203db..fc14896e23c58 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/delete_access_control_index.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/delete_access_control_index.test.ts @@ -32,7 +32,7 @@ describe('deleteAccessControlIndex lib function', () => { deleteAccessControlIndex(mockClient as unknown as IScopedClusterClient, 'indexName') ).resolves.toEqual(true); expect(mockClient.asCurrentUser.indices.delete).toHaveBeenCalledWith({ - index: 'indexName', + index: '.search-acl-filter-indexName', }); }); }); @@ -58,7 +58,7 @@ describe('deleteAccessControlIndex lib function', () => { deleteAccessControlIndex(mockClient as unknown as IScopedClusterClient, 'indexName') ).resolves.not.toThrowError(); expect(mockClient.asCurrentUser.indices.delete).toHaveBeenCalledWith({ - index: 'indexName', + index: '.search-acl-filter-indexName', }); }); }); @@ -84,7 +84,7 @@ describe('deleteAccessControlIndex lib function', () => { deleteAccessControlIndex(mockClient as unknown as IScopedClusterClient, 'indexName') ).rejects.toEqual(mockErrorRejection); expect(mockClient.asCurrentUser.indices.delete).toHaveBeenCalledWith({ - index: 'indexName', + index: '.search-acl-filter-indexName', }); }); }); diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/delete_access_control_index.ts b/x-pack/plugins/enterprise_search/server/lib/indices/delete_access_control_index.ts index 1937df11b912c..19769f09fa244 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/delete_access_control_index.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/delete_access_control_index.ts @@ -8,14 +8,15 @@ import { IScopedClusterClient } from '@kbn/core/server'; import { CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX } from '../../../common/constants'; -import { stripSearchPrefix } from '../../../common/utils/strip_search_prefix'; import { isIndexNotFoundException } from '../../utils/identify_exceptions'; -export const deleteAccessControlIndex = async (client: IScopedClusterClient, index: string) => { +export const deleteAccessControlIndex = async (client: IScopedClusterClient, indexName: string) => { + const aclIndexName = `${CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX}${indexName}`; + try { return await client.asCurrentUser.indices.delete({ - index: stripSearchPrefix(index, CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX), + index: aclIndexName, }); } catch (e) { // Gracefully exit if index not found. This is a valid case. diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts index a28450108290a..92c87b354a470 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.test.ts @@ -94,7 +94,7 @@ describe('generateApiKey lib function', () => { cluster: ['monitor'], index: [ { - names: ['search-test', '.search-acl-filter-test', `${CONNECTORS_INDEX}*`], + names: ['search-test', '.search-acl-filter-search-test', `${CONNECTORS_INDEX}*`], privileges: ['all'], }, ], diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts index d256bc6a91d88..fb2ddbaad9f9d 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/generate_api_key.ts @@ -7,13 +7,16 @@ import { IScopedClusterClient } from '@kbn/core/server'; -import { ConnectorDocument, CONNECTORS_INDEX } from '@kbn/search-connectors'; +import { + ConnectorDocument, + CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX, + CONNECTORS_INDEX, +} from '@kbn/search-connectors'; import { toAlphanumeric } from '../../../common/utils/to_alphanumeric'; export const generateApiKey = async (client: IScopedClusterClient, indexName: string) => { - // removes the "search-" prefix if present, and applies the new prefix - const aclIndexName = indexName.replace(/^(?:search-)?(.*)$/, '.search-acl-filter-$1'); + const aclIndexName = `${CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX}${indexName}`; const apiKeyResult = await client.asCurrentUser.security.createApiKey({ name: `${indexName}-connector`, From 3b8b829581fdeb8da2dc1cd7282b4275bb0481d2 Mon Sep 17 00:00:00 2001 From: Julien Lind Date: Tue, 28 Nov 2023 15:41:36 +0100 Subject: [PATCH 09/30] Update translations.ts to fix typo (#172036) Closes https://github.com/elastic/kibana/issues/172034 --- .../detections/components/user_privileges/translations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/user_privileges/translations.ts b/x-pack/plugins/security_solution/public/detections/components/user_privileges/translations.ts index c2ae389a0c571..2771a8e48a04f 100644 --- a/x-pack/plugins/security_solution/public/detections/components/user_privileges/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/user_privileges/translations.ts @@ -17,6 +17,6 @@ export const LISTS_PRIVILEGES_FETCH_FAILURE = i18n.translate( export const DETECTION_ENGINE_PRIVILEGES_FETCH_FAILURE = i18n.translate( 'xpack.securitySolution.containers.detectionEngine.alerts.detectionEnginePrivileges.errorFetching', { - defaultMessage: 'Failed to retreive detection engine privileges', + defaultMessage: 'Failed to retrieve detection engine privileges', } ); From 348ef4e39cf2e68e0589263df58af1e4c24946ae Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 28 Nov 2023 10:37:42 -0500 Subject: [PATCH 10/30] [Fleet] Persist package upgrade errors (#171797) --- .../current_mappings.json | 4 ++ .../check_registered_types.test.ts | 2 +- .../plugins/fleet/common/openapi/bundled.json | 32 ++++++++- .../plugins/fleet/common/openapi/bundled.yaml | 20 ++++++ .../components/schemas/installation_info.yaml | 20 ++++++ .../plugins/fleet/common/types/models/epm.ts | 11 +++ .../fleet/server/routes/epm/handlers.ts | 2 + .../fleet/server/saved_objects/index.ts | 13 ++++ .../services/epm/packages/_install_package.ts | 9 +++ .../services/epm/packages/install.test.ts | 1 + .../server/services/epm/packages/install.ts | 65 +++++++++++------ .../packages/install_errors_helpers.test.ts | 71 +++++++++++++++++++ .../epm/packages/install_errors_helpers.ts | 44 ++++++++++++ x-pack/plugins/fleet/server/types/index.tsx | 1 + .../apis/epm/install_error_rollback.ts | 39 +++++++--- .../apis/epm/install_remove_assets.ts | 1 + .../apis/epm/update_assets.ts | 1 + .../error_handling/0.3.0/docs/README.md | 3 + .../0.3.0/img/logo_overrides_64_color.svg | 7 ++ .../visualization/sample_visualization.json | 14 ++++ .../error_handling/0.3.0/manifest.yml | 22 ++++++ 21 files changed, 350 insertions(+), 32 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_errors_helpers.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_errors_helpers.ts create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/docs/README.md create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/img/logo_overrides_64_color.svg create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/kibana/visualization/sample_visualization.json create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/manifest.yml diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 91246b667626c..fcc8422c45865 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1967,6 +1967,10 @@ } } }, + "latest_install_failed_attempts": { + "type": "object", + "enabled": false + }, "installed_kibana": { "dynamic": false, "properties": {} diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index fb5010cff0280..6f0ae86e4e026 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -84,7 +84,7 @@ describe('checking migration metadata changes on all registered SO types', () => "dashboard": "0611794ce10d25a36da0770c91376c575e92e8f2", "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b", "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d", - "epm-packages": "2449bb565f987eff70b1b39578bb17e90c404c6e", + "epm-packages": "c23d3d00c051a08817335dba26f542b64b18a56a", "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1", "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582", "event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88", diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index 2f583f9b32140..102a80ad003fd 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -6069,6 +6069,35 @@ "install_format_schema_version": { "type": "string" }, + "latest_install_failed_attempts": { + "description": "Latest failed install errors", + "type": "array", + "items": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "target_version": { + "type": "string" + }, + "error": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stack": { + "type": "string" + } + } + } + } + } + }, "verification_status": { "type": "string", "enum": [ @@ -6120,7 +6149,8 @@ "install_version", "install_started_at", "install_source", - "verification_status" + "verification_status", + "latest_install_failed_attempts" ] }, "search_result": { diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index d103c2f5e2d9a..2e47aaf003062 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -3824,6 +3824,25 @@ components: type: string install_format_schema_version: type: string + latest_install_failed_attempts: + description: Latest failed install errors + type: array + items: + type: object + properties: + created_at: + type: string + target_version: + type: string + error: + type: object + properties: + name: + type: string + message: + type: string + stack: + type: string verification_status: type: string enum: @@ -3863,6 +3882,7 @@ components: - install_started_at - install_source - verification_status + - latest_install_failed_attempts search_result: title: Search result type: object diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml index 4d13611a6afbd..c5db5f12d4cc3 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml @@ -47,6 +47,25 @@ properties: type: string install_format_schema_version: type: string + latest_install_failed_attempts: + description: Latest failed install errors + type: array + items: + type: object + properties: + created_at: + type: string + target_version: + type: string + error: + type: object + properties: + name: + type: string + message: + type: string + stack: + type: string verification_status: type: string enum: @@ -86,3 +105,4 @@ required: - install_started_at - install_source - verification_status + - latest_install_failed_attempts diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index c4d4e9c763766..24bde5b5f5f5a 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -530,6 +530,16 @@ export interface ExperimentalDataStreamFeature { features: Partial>; } +export interface InstallFailedAttempt { + created_at: string; + target_version: string; + error: { + name: string; + message: string; + stack?: string; + }; +} + export interface Installation { installed_kibana: KibanaAssetReference[]; installed_es: EsAssetReference[]; @@ -549,6 +559,7 @@ export interface Installation { experimental_data_stream_features?: ExperimentalDataStreamFeature[]; internal?: boolean; removable?: boolean; + latest_install_failed_attempts?: InstallFailedAttempt[]; } export interface PackageUsageStats { diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 8ce1bf7006f6b..c03272119dd16 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -690,7 +690,9 @@ const soToInstallationInfo = (pkg: PackageListItem | PackageInfo) => { verification_status: attributes.verification_status, verification_key_id: attributes.verification_key_id, experimental_data_stream_features: attributes.experimental_data_stream_features, + latest_install_failed_attempts: attributes.latest_install_failed_attempts, }; + return { // When savedObject gets removed, replace `pkg` with `...omit(pkg, 'savedObject')` ...pkg, diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 7df074f418fb9..85cfae347e03e 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -457,6 +457,7 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ deferred: { type: 'boolean' }, }, }, + latest_install_failed_attempts: { type: 'object', enabled: false }, installed_kibana: { dynamic: false, properties: {}, @@ -487,6 +488,18 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, }, }, + modelVersions: { + '1': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + latest_install_failed_attempts: { type: 'object', enabled: false }, + }, + }, + ], + }, + }, migrations: { '7.14.0': migrateInstallationToV7140, '7.14.1': migrateInstallationToV7140, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 105331f54d2cf..4eefdc5875539 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -57,6 +57,7 @@ import { installIndexTemplatesAndPipelines, } from './install'; import { withPackageSpan } from './utils'; +import { clearLatestFailedAttempts } from './install_errors_helpers'; // this is only exported for testing // use a leading underscore to indicate it's not the supported path @@ -103,6 +104,10 @@ export async function _installPackage({ try { // if some installation already exists if (installedPkg) { + if (installType === 'update' && pkgVersion === '1.17.0') { + // throw new Error('Test error '); + } + const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; const hasExceededTimeout = Date.now() - Date.parse(installedPkg.attributes.install_started_at) < @@ -333,6 +338,10 @@ export async function _installPackage({ install_status: 'installed', package_assets: packageAssetRefs, install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, + latest_install_failed_attempts: clearLatestFailedAttempts( + pkgVersion, + installedPkg?.attributes.latest_install_failed_attempts ?? [] + ), }) ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index f3e464d7a9f67..bc3e138b5619c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -243,6 +243,7 @@ describe('install', () => { it('should send telemetry on install failure, async error', async () => { jest.mocked(install._installPackage).mockRejectedValue(new Error('error')); jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ spaceId: DEFAULT_SPACE_ID, installSource: 'registry', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index c7a3d4638057e..2dbf1662d4364 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -106,6 +106,7 @@ import { cacheAssets } from './custom_integrations/assets/cache'; import { generateDatastreamEntries } from './custom_integrations/assets/dataset/utils'; import { checkForNamingCollision } from './custom_integrations/validation/check_naming_collision'; import { checkDatasetsNameFormat } from './custom_integrations/validation/check_dataset_name_format'; +import { addErrorToLatestFailedAttempts } from './install_errors_helpers'; export async function isPackageInstalled(options: { savedObjectsClient: SavedObjectsClientContract; @@ -236,6 +237,13 @@ export async function handleInstallPackageFailure({ version: pkgVersion, }); + const latestInstallFailedAttempts = addErrorToLatestFailedAttempts({ + error, + targetVersion: pkgVersion, + createdAt: new Date().toISOString(), + latestAttempts: installedPkg?.attributes.latest_install_failed_attempts, + }); + // if there is an unknown server error, uninstall any package assets or reinstall the previous version if update try { const installType = getInstallType({ pkgVersion, installedPkg }); @@ -245,18 +253,18 @@ export async function handleInstallPackageFailure({ return; } + await updateInstallStatusToFailed({ + logger, + savedObjectsClient, + pkgName, + status: 'install_failed', + latestInstallFailedAttempts, + }); + if (installType === 'reinstall') { logger.error(`Failed to reinstall ${pkgkey}: [${error.toString()}]`, { error }); } - await updateInstallStatus({ savedObjectsClient, pkgName, status: 'install_failed' }).catch( - (err) => { - if (!SavedObjectsErrorHelpers.isNotFoundError(err)) { - logger.error(`failed to update package status to: install_failed ${err}`); - } - } - ); - if (installType === 'update') { if (!installedPkg) { logger.error( @@ -278,13 +286,20 @@ export async function handleInstallPackageFailure({ } } catch (e) { // If an error happens while removing the integration or while doing a rollback update the status to failed - await updateInstallStatus({ savedObjectsClient, pkgName, status: 'install_failed' }).catch( - (err) => { - if (!SavedObjectsErrorHelpers.isNotFoundError(err)) { - logger.error(`failed to update package status to: install_failed ${err}`); - } - } - ); + await updateInstallStatusToFailed({ + logger, + savedObjectsClient, + pkgName, + status: 'install_failed', + latestInstallFailedAttempts: installedPkg + ? addErrorToLatestFailedAttempts({ + error: e, + targetVersion: installedPkg.attributes.version, + createdAt: installedPkg.attributes.install_started_at, + latestAttempts: latestInstallFailedAttempts, + }) + : [], + }); logger.error(`failed to uninstall or rollback package after installation error ${e}`); } } @@ -883,24 +898,34 @@ export const updateVersion = async ( }); }; -export const updateInstallStatus = async ({ +export const updateInstallStatusToFailed = async ({ + logger, savedObjectsClient, pkgName, status, + latestInstallFailedAttempts, }: { + logger: Logger; savedObjectsClient: SavedObjectsClientContract; pkgName: string; status: EpmPackageInstallStatus; + latestInstallFailedAttempts: any; }) => { auditLoggingService.writeCustomSoAuditLog({ action: 'update', id: pkgName, savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, }); - - return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { - install_status: status, - }); + try { + return await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { + install_status: status, + latest_install_failed_attempts: latestInstallFailedAttempts, + }); + } catch (err) { + if (!SavedObjectsErrorHelpers.isNotFoundError(err)) { + logger.error(`failed to update package status to: install_failed ${err}`); + } + } }; export async function restartInstallation(options: { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_errors_helpers.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_errors_helpers.test.ts new file mode 100644 index 0000000000000..264ce1cbd5503 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_errors_helpers.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InstallFailedAttempt } from '../../../types'; + +import { + clearLatestFailedAttempts, + addErrorToLatestFailedAttempts, +} from './install_errors_helpers'; + +const generateFailedAttempt = (version: string) => ({ + target_version: version, + created_at: new Date().toISOString(), + error: { + name: 'test', + message: 'test', + }, +}); + +const mapFailledAttempsToTargetVersion = (attemps: InstallFailedAttempt[]) => + attemps.map((attempt) => attempt.target_version); + +describe('Install error helpers', () => { + describe('clearLatestFailedAttempts', () => { + const previousFailedAttemps: InstallFailedAttempt[] = [ + generateFailedAttempt('0.1.0'), + generateFailedAttempt('0.2.0'), + ]; + it('should clear previous error on succesfull upgrade', () => { + const currentFailledAttemps = clearLatestFailedAttempts('0.2.0', previousFailedAttemps); + + expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual([]); + }); + + it('should not clear previous upgrade error on succesfull rollback', () => { + const currentFailledAttemps = clearLatestFailedAttempts('0.1.0', previousFailedAttemps); + + expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual(['0.2.0']); + }); + }); + + describe('addErrorToLatestFailedAttempts', () => { + it('should only keep 5 errors', () => { + const previousFailedAttemps: InstallFailedAttempt[] = [ + generateFailedAttempt('0.2.5'), + generateFailedAttempt('0.2.4'), + generateFailedAttempt('0.2.3'), + generateFailedAttempt('0.2.2'), + generateFailedAttempt('0.2.1'), + ]; + const currentFailledAttemps = addErrorToLatestFailedAttempts({ + targetVersion: '0.2.6', + createdAt: new Date().toISOString(), + error: new Error('new test'), + latestAttempts: previousFailedAttemps, + }); + + expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual([ + '0.2.6', + '0.2.5', + '0.2.4', + '0.2.3', + '0.2.2', + ]); + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_errors_helpers.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_errors_helpers.ts new file mode 100644 index 0000000000000..1642acd7349b4 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_errors_helpers.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { lt } from 'semver'; + +import type { InstallFailedAttempt } from '../../../types'; + +const MAX_ATTEMPTS_TO_KEEP = 5; + +export function clearLatestFailedAttempts( + installedVersion: string, + latestAttempts: InstallFailedAttempt[] = [] +) { + return latestAttempts.filter((attempt) => lt(installedVersion, attempt.target_version)); +} + +export function addErrorToLatestFailedAttempts({ + error, + createdAt, + targetVersion, + latestAttempts = [], +}: { + createdAt: string; + targetVersion: string; + error: Error; + latestAttempts?: InstallFailedAttempt[]; +}): InstallFailedAttempt[] { + return [ + { + created_at: createdAt, + target_version: targetVersion, + error: { + name: error.name, + message: error.message, + stack: error.stack, + }, + }, + ...latestAttempts, + ].slice(0, MAX_ATTEMPTS_TO_KEEP); +} diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index b9dc7528651eb..d56c92eb7aa8b 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -41,6 +41,7 @@ export type { Installation, EpmPackageInstallStatus, InstallationStatus, + InstallFailedAttempt, PackageInfo, ArchivePackage, RegistryVarsEntry, diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts b/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts index 7649237f0ab4d..5192b8a4e914b 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts @@ -16,10 +16,11 @@ export default function (providerContext: FtrProviderContext) { const pkgName = 'error_handling'; const goodPackageVersion = '0.1.0'; const badPackageVersion = '0.2.0'; + const goodUpgradePackageVersion = '0.3.0'; const kibanaServer = getService('kibanaServer'); - const installPackage = async (pkg: string, version: string) => { - await supertest + const installPackage = (pkg: string, version: string) => { + return supertest .post(`/api/fleet/epm/packages/${pkg}/${version}`) .set('kbn-xsrf', 'xxxx') .send({ force: true }); @@ -45,28 +46,46 @@ export default function (providerContext: FtrProviderContext) { afterEach(async () => { await kibanaServer.savedObjects.cleanStandardList(); await uninstallPackage(pkgName, goodPackageVersion); + await uninstallPackage(pkgName, goodUpgradePackageVersion); }); it('on a fresh install, it should uninstall a broken package during rollback', async function () { - await supertest - .post(`/api/fleet/epm/packages/${pkgName}/${badPackageVersion}`) - .set('kbn-xsrf', 'xxxx') - .expect(422); // the broken package contains a broken visualization triggering a 422 from Kibana + // the broken package contains a broken visualization triggering a 422 from Kibana + await installPackage(pkgName, badPackageVersion).expect(422); const pkgInfoResponse = await getPackageInfo(pkgName, badPackageVersion); expect(JSON.parse(pkgInfoResponse.text).item.status).to.be('not_installed'); + expect(pkgInfoResponse.body.item.savedObject).to.be(undefined); }); it('on an upgrade, it should fall back to the previous good version during rollback', async function () { await installPackage(pkgName, goodPackageVersion); - await supertest - .post(`/api/fleet/epm/packages/${pkgName}/${badPackageVersion}`) - .set('kbn-xsrf', 'xxxx') - .expect(422); // the broken package contains a broken visualization triggering a 422 from Kibana + // the broken package contains a broken visualization triggering a 422 from Kibana + await installPackage(pkgName, badPackageVersion).expect(422); const goodPkgInfoResponse = await getPackageInfo(pkgName, goodPackageVersion); expect(JSON.parse(goodPkgInfoResponse.text).item.status).to.be('installed'); expect(JSON.parse(goodPkgInfoResponse.text).item.version).to.be('0.1.0'); + const latestInstallFailedAttempts = + goodPkgInfoResponse.body.item.savedObject.attributes.latest_install_failed_attempts; + expect(latestInstallFailedAttempts).to.have.length(1); + expect(latestInstallFailedAttempts[0].target_version).to.be('0.2.0'); + expect(latestInstallFailedAttempts[0].error.message).to.contain( + 'Document "sample_visualization" belongs to a more recent version of Kibana [12.7.0]' + ); + }); + + it('on a succesfull upgrade, it should clear previous upgrade errors', async function () { + await installPackage(pkgName, goodPackageVersion); + await installPackage(pkgName, badPackageVersion).expect(422); + await installPackage(pkgName, goodUpgradePackageVersion).expect(200); + + const goodPkgInfoResponse = await getPackageInfo(pkgName, goodUpgradePackageVersion); + expect(JSON.parse(goodPkgInfoResponse.text).item.status).to.be('installed'); + expect(JSON.parse(goodPkgInfoResponse.text).item.version).to.be('0.3.0'); + const latestInstallFailedAttempts = + goodPkgInfoResponse.body.item.savedObject.attributes.latest_install_failed_attempts; + expect(latestInstallFailedAttempts).to.have.length(0); }); }); } diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index aaf31e54798db..55d85aead4771 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -786,6 +786,7 @@ const expectAssetsInstalled = ({ install_status: 'installed', install_started_at: res.attributes.install_started_at, install_source: 'registry', + latest_install_failed_attempts: [], install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, verification_status: 'unknown', verification_key_id: null, diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 75040c8400d04..eaea702cc6c8c 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -516,6 +516,7 @@ export default function (providerContext: FtrProviderContext) { install_started_at: res.attributes.install_started_at, install_source: 'registry', install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, + latest_install_failed_attempts: [], verification_status: 'unknown', verification_key_id: null, }); diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/docs/README.md b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/docs/README.md new file mode 100644 index 0000000000000..260499f4b0078 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/docs/README.md @@ -0,0 +1,3 @@ +This package should install without errors. + +Version 0.2.0 of this package should fail during installation. We need this good version to test rollback. \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/img/logo_overrides_64_color.svg b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/img/logo_overrides_64_color.svg new file mode 100644 index 0000000000000..b03007a76ffcc --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/img/logo_overrides_64_color.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/kibana/visualization/sample_visualization.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/kibana/visualization/sample_visualization.json new file mode 100644 index 0000000000000..01afe600853ef --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/kibana/visualization/sample_visualization.json @@ -0,0 +1,14 @@ +{ + "attributes": { + "description": "sample visualization", + "title": "sample vis title", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Log Level\",\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per day\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"Log levels over time [Logs Kafka] ECS\",\"type\":\"histogram\"}" + }, + "id": "sample_visualization", + "type": "visualization", + "migrationVersion": { + "visualization": "7.7.0" + } +} diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/manifest.yml new file mode 100644 index 0000000000000..883e4234886e7 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/manifest.yml @@ -0,0 +1,22 @@ +format_version: 1.0.0 +name: error_handling +title: Error handling +description: tests error handling and rollback +version: 0.3.0 +categories: [] +release: beta +type: integration +license: basic +owner: + github: elastic/fleet + +requirement: + elasticsearch: + versions: '>7.7.0' + kibana: + versions: '>7.7.0' + +icons: + - src: '/img/logo_overrides_64_color.svg' + size: '16x16' + type: 'image/svg+xml' From a4ed14bec840da00d3a7edc87db142c79f7ca55b Mon Sep 17 00:00:00 2001 From: Wafaa Nasr Date: Tue, 28 Nov 2023 16:40:09 +0100 Subject: [PATCH 11/30] [Security Solution][API testing] Move and restructures Lists APIS (#171992) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Following the initial work in this https://github.com/elastic/kibana/pull/166755 - Addresses part of https://github.com/elastic/kibana/issues/151902 for List APIs tests - Added a new folder under the `security_solution_api_integration` called `lists_and_exception_lists` to hold the lists and exception lists tests, and split the `List` APIs into two groups since the execution time in Serverless was close to 30 mins - Modified the [x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts](https://github.com/elastic/kibana/pull/171992/files#diff-4e3545fdeb8c8d9467cfa1c4aa88194e189193a92fa6f1cf5f859b1ef1beb45c), [x-pack/plugins/lists/common/schemas/response/exception_list_schema.mock.ts](https://github.com/elastic/kibana/pull/171992/files#diff-c3cc18faf07aab86e307185d41599c3596a3f8b360d3e4829591afa148283238) , [x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts](https://github.com/elastic/kibana/pull/171992/files#diff-608579ca5e65da74f41319a58d81ab12cc3d79d389b087806c7b74949fbc6cc3), [x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts](https://github.com/elastic/kibana/pull/171992/files#diff-efc64eb35937a8da28fc982c527253c0923650ae4163d4bbc203d3ebc2949835) to accept `elastic user` input because it changes in ESS and Serverless - Deleted the `x-pack/test/lists_api_integration` folder - Moved the utility files associated with Basic tests to the new directory `security_solution_api_integration`. Files not actively used in the previous folder were moved, while duplicate files remained in their original positions. - Updated the below files imports from the old `lists_api_integration` folder to the new `lists_and_exception_lists` ``` lists_api_integration/ - exceptions/operators_data_types/date_numeric_types/date.ts - exceptions/operators_data_types/date_numeric_types/double.ts - exceptions/operators_data_types/date_numeric_types/float.ts - exceptions/operators_data_types/date_numeric_types/integer.ts - exceptions/operators_data_types/ips/ip.ts - exceptions/operators_data_types/ips/ip_array.ts - exceptions/operators_data_types/keyword/keyword.ts - exceptions/operators_data_types/keyword/keyword_array.ts - exceptions/operators_data_types/long/long.ts - exceptions/operators_data_types/text/text.ts - exceptions/operators_data_types/text/text_array.ts - exceptions/workflows/create_endpoint_exceptions.ts - exceptions/workflows/create_rule_exceptions.ts - exceptions/workflows/find_rule_exception_references.ts - exceptions/workflows/role_based_add_edit_comments.ts - exceptions/workflows/role_based_rule_exceptions_workflows.ts - exceptions/workflows/rule_exception_synchronizations.ts - rule_execution_logic/execution_logic/esql.ts - rule_execution_logic/execution_logic/machine_learning.ts - rule_execution_logic/execution_logic/new_terms.ts - rule_execution_logic/execution_logic/query.ts - telemetry/task_based/all_types.ts - telemetry/task_based/detection_rules.ts - telemetry/task_based/security_lists.ts ``` ``` These files should be moved too soon to the new `lists_api_integration` detection_engine_api_integration/security_and_spaces/group10 - import_export_rules.ts - import_rules.ts - perform_bulk_actions ``` - Updated the below files imports to the `ftr_provider_context_with_spaces.d.ts` ` - risk_engine/risk_scoring_task/task_execution_nondefault_spaces.ts` - The QA phase concluded with all tests passing successfully. 🟢 - Updated the CodeOwner file for the newly moved tests - Add a new util file to `deleteAllExceptions` under the old `detection_engine_api` folder since the Rule management related-tests are still need to be moved over to the new folder - Old/new group details, decisions, and execution time are mentioned in this [document](https://docs.google.com/document/d/1CRFfDWMzw3ob03euWIvT4-IoiLXjoiPWI8mTBqP4Zks/edit) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .buildkite/ftr_configs.yml | 6 +- .github/CODEOWNERS | 14 +-- .../exception_list_item_schema.mock.ts | 31 +++--- .../response/exception_list_schema.mock.ts | 29 +++--- .../schemas/response/list_item_schema.mock.ts | 8 +- .../schemas/response/list_schema.mock.ts | 8 +- .../group10/import_export_rules.ts | 2 +- .../group10/import_rules.ts | 2 +- .../group10/perform_bulk_action.ts | 2 +- .../security_and_spaces/tests/import_rules.ts | 2 +- .../utils/delete_all_exceptions.ts | 67 +++++++++++++ .../utils/index.ts | 1 + .../lists_api_integration/common/config.ts | 66 ------------- .../common/ftr_provider_context.d.ts | 12 --- .../lists_api_integration/common/services.ts | 8 -- .../security_and_spaces/tests/index.ts | 46 --------- .../ftr_provider_context_with_spaces.d.ts | 4 +- .../package.json | 14 ++- .../actions/configs/ess.config.ts | 2 +- .../actions/configs/serverless.config.ts | 2 +- .../date_numeric_types/date.ts | 2 +- .../date_numeric_types/double.ts | 2 +- .../date_numeric_types/float.ts | 2 +- .../date_numeric_types/integer.ts | 2 +- .../exceptions/operators_data_types/ips/ip.ts | 2 +- .../operators_data_types/ips/ip_array.ts | 2 +- .../operators_data_types/keyword/keyword.ts | 2 +- .../keyword/keyword_array.ts | 2 +- .../operators_data_types/long/long.ts | 2 +- .../operators_data_types/text/text.ts | 2 +- .../operators_data_types/text/text_array.ts | 2 +- .../workflows/create_endpoint_exceptions.ts | 2 +- .../workflows/create_rule_exceptions.ts | 2 +- .../find_rule_exception_references.ts | 2 +- .../workflows/role_based_add_edit_comments.ts | 2 +- .../role_based_rule_exceptions_workflows.ts | 2 +- .../rule_exception_synchronizations.ts | 2 +- .../execution_logic/esql.ts | 2 +- .../execution_logic/machine_learning.ts | 2 +- .../execution_logic/new_terms.ts | 2 +- .../execution_logic/query.ts | 2 +- .../telemetry/task_based/all_types.ts | 2 +- .../telemetry/task_based/detection_rules.ts | 2 +- .../telemetry/task_based/security_lists.ts | 2 +- .../task_execution_nondefault_spaces.ts | 2 +- .../configs/ess.config.ts | 22 +++++ .../configs/serverless.config.ts | 15 +++ .../exception_lists_items/index.ts | 14 +++ .../items}/create_exception_list_items.ts | 22 +++-- .../items}/delete_exception_list_items.ts | 19 ++-- .../items}/find_exception_list_items.ts | 21 ++-- .../exception_lists_items/items/index.ts | 17 ++++ .../items}/read_exception_list_items.ts | 24 +++-- .../items}/update_exception_list_items.ts | 13 +-- .../lists}/create_exception_lists.ts | 15 +-- .../lists}/delete_exception_lists.ts | 16 ++-- .../lists}/duplicate_exception_list.ts | 14 +-- .../lists}/export_exception_list.ts | 7 +- .../lists}/find_exception_lists.ts | 11 ++- .../lists}/get_exception_filter.ts | 19 ++-- .../lists}/import_exceptions.ts | 8 +- .../exception_lists_items/lists/index.ts | 23 +++++ .../lists}/read_exception_lists.ts | 20 ++-- .../lists}/summary_exception_lists.ts | 8 +- .../lists}/update_exception_lists.ts | 16 ++-- .../lists_items/configs/ess.config.ts | 22 +++++ .../lists_items/configs/serverless.config.ts} | 12 +-- .../default_license/lists_items/index.ts | 14 +++ .../lists_items/items}/create_list_items.ts | 19 ++-- .../lists_items/items}/delete_list_items.ts | 17 ++-- .../lists_items/items}/export_list_items.ts | 8 +- .../lists_items/items}/find_list_items.ts | 12 ++- .../lists_items/items}/import_list_items.ts | 45 ++------- .../items/import_list_items_migrations.ts | 61 ++++++++++++ .../lists_items/items/index.ts | 23 +++++ .../lists_items/items}/patch_list_items.ts | 76 ++------------- .../items/patch_list_items_migrations.ts | 95 +++++++++++++++++++ .../lists_items/items}/read_list_items.ts | 18 ++-- .../lists_items/items}/update_list_items.ts | 76 ++------------- .../items/update_list_items_migrations.ts | 95 +++++++++++++++++++ .../lists_items/lists}/create_lists.ts | 18 ++-- .../lists_items/lists/create_lists_index.ts | 47 +++++++++ .../lists/create_lists_index_migrations.ts} | 26 +---- .../lists_items/lists}/delete_lists.ts | 25 +++-- .../lists_items/lists}/find_lists.ts | 12 ++- .../lists_items/lists}/find_lists_by_size.ts | 25 +++-- .../lists_items/lists/index.ts | 23 +++++ .../lists_items/lists}/patch_lists.ts | 74 ++------------- .../lists/patch_lists_migrations.ts | 90 ++++++++++++++++++ .../lists}/read_list_privileges.ts | 7 +- .../lists_items/lists}/read_lists.ts | 17 ++-- .../lists_items/lists}/update_lists.ts | 76 ++------------- .../lists/update_lists_migrations.ts | 94 ++++++++++++++++++ .../lists_and_exception_lists}/utils.ts | 2 +- .../tsconfig.json | 1 + x-pack/test/tsconfig.json | 1 - 96 files changed, 1128 insertions(+), 703 deletions(-) create mode 100644 x-pack/test/detection_engine_api_integration/utils/delete_all_exceptions.ts delete mode 100644 x-pack/test/lists_api_integration/common/config.ts delete mode 100644 x-pack/test/lists_api_integration/common/ftr_provider_context.d.ts delete mode 100644 x-pack/test/lists_api_integration/common/services.ts delete mode 100644 x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts rename x-pack/test/security_solution_api_integration/{test_suites/entity_analytics/default_license/risk_engine/risk_scoring_task => }/ftr_provider_context_with_spaces.d.ts (71%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/ess.config.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/serverless.config.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/index.ts rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items}/create_exception_list_items.ts (92%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items}/delete_exception_list_items.ts (88%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items}/find_exception_list_items.ts (90%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/index.ts rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items}/read_exception_list_items.ts (89%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items}/update_exception_list_items.ts (97%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists}/create_exception_lists.ts (86%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists}/delete_exception_lists.ts (89%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists}/duplicate_exception_list.ts (93%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists}/export_exception_list.ts (97%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists}/find_exception_lists.ts (87%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists}/get_exception_filter.ts (92%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists}/import_exceptions.ts (99%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/index.ts rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists}/read_exception_lists.ts (87%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists}/summary_exception_lists.ts (95%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists}/update_exception_lists.ts (94%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/ess.config.ts rename x-pack/test/{lists_api_integration/security_and_spaces/config.ts => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/serverless.config.ts} (50%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/index.ts rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items}/create_list_items.ts (86%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items}/delete_list_items.ts (85%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items}/export_list_items.ts (95%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items}/find_list_items.ts (92%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items}/import_list_items.ts (74%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/import_list_items_migrations.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/index.ts rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items}/patch_list_items.ts (73%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/patch_list_items_migrations.ts rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items}/read_list_items.ts (85%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items}/update_list_items.ts (80%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/update_list_items_migrations.ts rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists}/create_lists.ts (81%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists_index.ts rename x-pack/test/{lists_api_integration/security_and_spaces/tests/create_lists_index.ts => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists_index_migrations.ts} (77%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists}/delete_lists.ts (92%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists}/find_lists.ts (86%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists}/find_lists_by_size.ts (79%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/index.ts rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists}/patch_lists.ts (73%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/patch_lists_migrations.ts rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists}/read_list_privileges.ts (91%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists}/read_lists.ts (83%) rename x-pack/test/{lists_api_integration/security_and_spaces/tests => security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists}/update_lists.ts (79%) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/update_lists_migrations.ts rename x-pack/test/{lists_api_integration => security_solution_api_integration/test_suites/lists_and_exception_lists}/utils.ts (99%) diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index e51fc01430b8c..003dbe3fd22f6 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -333,7 +333,6 @@ enabled: - x-pack/test/kubernetes_security/basic/config.ts - x-pack/test/licensing_plugin/config.public.ts - x-pack/test/licensing_plugin/config.ts - - x-pack/test/lists_api_integration/security_and_spaces/config.ts - x-pack/test/monitoring_api_integration/config.ts - x-pack/test/observability_api_integration/basic/config.ts - x-pack/test/observability_api_integration/trial/config.ts @@ -492,4 +491,9 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/detection_engine/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/detection_engine/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/ess.config.ts + diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 89e152a2fe40f..9944de64b186d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1223,7 +1223,6 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib /x-pack/test/functional/es_archives/endpoint/ @elastic/security-solution /x-pack/test/plugin_functional/test_suites/resolver/ @elastic/security-solution /x-pack/test/detection_engine_api_integration @elastic/security-solution -/x-pack/test/lists_api_integration @elastic/security-solution /x-pack/test/api_integration/apis/security_solution @elastic/security-solution #CC# /x-pack/plugins/security_solution/ @elastic/security-solution @@ -1394,13 +1393,14 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout @elastic/ /x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics @elastic/security-detection-engine /x-pack/test/security_solution_cypress/cypress/e2e/exceptions @elastic/security-detection-engine /x-pack/test/security_solution_cypress/cypress/e2e/overview @elastic/security-detection-engine -x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions @elastic/security-detection-engine -x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation @elastic/security-detection-engine -x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions @elastic/security-detection-engine -x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts @elastic/security-detection-engine -x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/user_roles @elastic/security-detection-engine -x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/detection_engine @elastic/security-detection-engine +/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions @elastic/security-detection-engine +/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation @elastic/security-detection-engine +/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions @elastic/security-detection-engine +/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts @elastic/security-detection-engine +/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/user_roles @elastic/security-detection-engine +/x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/detection_engine @elastic/security-detection-engine /x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users @elastic/security-detection-engine +/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists @elastic/security-detection-engine ## Security Threat Intelligence - Under Security Platform /x-pack/plugins/security_solution/public/common/components/threat_match @elastic/security-detection-engine diff --git a/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts index 4ebfcd0fec910..529e1bac24aff 100644 --- a/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts @@ -53,18 +53,19 @@ export const getExceptionListItemSchemaMock = ( * This is useful for end to end tests where we remove the auto generated parts for comparisons * such as created_at, updated_at, and id. */ -export const getExceptionListItemResponseMockWithoutAutoGeneratedValues = - (): Partial => ({ - comments: [], - created_by: ELASTIC_USER, - description: DESCRIPTION, - entries: ENTRIES, - item_id: ITEM_ID, - list_id: LIST_ID, - name: NAME, - namespace_type: 'single', - os_types: OS_TYPES, - tags: [], - type: ITEM_TYPE, - updated_by: ELASTIC_USER, - }); +export const getExceptionListItemResponseMockWithoutAutoGeneratedValues = ( + elasticUser: string = ELASTIC_USER +): Partial => ({ + comments: [], + created_by: elasticUser, + description: DESCRIPTION, + entries: ENTRIES, + item_id: ITEM_ID, + list_id: LIST_ID, + name: NAME, + namespace_type: 'single', + os_types: OS_TYPES, + tags: [], + type: ITEM_TYPE, + updated_by: elasticUser, +}); diff --git a/x-pack/plugins/lists/common/schemas/response/exception_list_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/exception_list_schema.mock.ts index eca17b4c835d6..cdf86c2cc720d 100644 --- a/x-pack/plugins/lists/common/schemas/response/exception_list_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/exception_list_schema.mock.ts @@ -82,17 +82,18 @@ export const getTrustedAppsListSchemaMock = (): ExceptionListSchema => { * This is useful for end to end tests where we remove the auto generated parts for comparisons * such as created_at, updated_at, and id. */ -export const getExceptionResponseMockWithoutAutoGeneratedValues = - (): Partial => ({ - created_by: ELASTIC_USER, - description: DESCRIPTION, - immutable: IMMUTABLE, - list_id: LIST_ID, - name: NAME, - namespace_type: 'single', - os_types: [], - tags: [], - type: ENDPOINT_TYPE, - updated_by: ELASTIC_USER, - version: VERSION, - }); +export const getExceptionResponseMockWithoutAutoGeneratedValues = ( + elasticUser: string = ELASTIC_USER +): Partial => ({ + created_by: elasticUser, + description: DESCRIPTION, + immutable: IMMUTABLE, + list_id: LIST_ID, + name: NAME, + namespace_type: 'single', + os_types: [], + tags: [], + type: ENDPOINT_TYPE, + updated_by: elasticUser, + version: VERSION, +}); diff --git a/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts index cce497f87335c..4d773fa63306f 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts @@ -40,10 +40,12 @@ export const getListItemResponseMock = (): ListItemSchema => ({ * This is useful for end to end tests where we remove the auto generated parts for comparisons * such as created_at, updated_at, and id. */ -export const getListItemResponseMockWithoutAutoGeneratedValues = (): Partial => ({ - created_by: ELASTIC_USER, +export const getListItemResponseMockWithoutAutoGeneratedValues = ( + elasticUser: string = ELASTIC_USER +): Partial => ({ + created_by: elasticUser, list_id: LIST_ID, type: TYPE, - updated_by: ELASTIC_USER, + updated_by: elasticUser, value: VALUE, }); diff --git a/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts index abc52634e0232..7e6f1647ceb33 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts @@ -44,12 +44,14 @@ export const getListResponseMock = (): ListSchema => ({ * This is useful for end to end tests where we remove the auto generated parts for comparisons * such as created_at, updated_at, and id. */ -export const getListResponseMockWithoutAutoGeneratedValues = (): Partial => ({ - created_by: ELASTIC_USER, +export const getListResponseMockWithoutAutoGeneratedValues = ( + elasticUser: string = ELASTIC_USER +): Partial => ({ + created_by: elasticUser, description: DESCRIPTION, immutable: IMMUTABLE, name: NAME, type: TYPE, - updated_by: ELASTIC_USER, + updated_by: elasticUser, version: VERSION, }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_export_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_export_rules.ts index 8943a4b67c99b..aee267db951d5 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_export_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_export_rules.ts @@ -26,8 +26,8 @@ import { deleteAllRules, deleteAllAlerts, getSimpleRule, + deleteAllExceptions, } from '../../utils'; -import { deleteAllExceptions } from '../../../lists_api_integration/utils'; import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution'; // This test was meant to be more full flow, ensuring that diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts index c92591e8f3f74..f8dbcffad3ba4 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts @@ -38,8 +38,8 @@ import { createRule, getRule, getRuleSOById, + deleteAllExceptions, } from '../../utils'; -import { deleteAllExceptions } from '../../../lists_api_integration/utils'; import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution'; const getImportRuleBuffer = (connectorId: string) => { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts index 2529e794089a9..d14d82edbba51 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts @@ -23,7 +23,6 @@ import { getCreateExceptionListDetectionSchemaMock } from '@kbn/lists-plugin/com import { EXCEPTION_LIST_ITEM_URL, EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants'; import { getCreateExceptionListItemMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_item_schema.mock'; import { WebhookAuthType } from '@kbn/stack-connectors-plugin/common/webhook/constants'; -import { deleteAllExceptions } from '../../../lists_api_integration/utils'; import { binaryToString, createLegacyRuleAction, @@ -44,6 +43,7 @@ import { createRuleThroughAlertingEndpoint, getRuleSavedObjectWithLegacyInvestigationFields, getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, + deleteAllExceptions, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/import_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/import_rules.ts index 002bf3ddda8a4..f3fb8671906c6 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/import_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/import_rules.ts @@ -27,8 +27,8 @@ import { getWebHookAction, removeServerGeneratedProperties, ruleToNdjson, + deleteAllExceptions, } from '../../utils'; -import { deleteAllExceptions } from '../../../lists_api_integration/utils'; import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution'; const getImportRuleBuffer = (connectorId: string) => { diff --git a/x-pack/test/detection_engine_api_integration/utils/delete_all_exceptions.ts b/x-pack/test/detection_engine_api_integration/utils/delete_all_exceptions.ts new file mode 100644 index 0000000000000..aed98bc61561a --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/delete_all_exceptions.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// Should be deleted once all all the remaining tests in this folder get moved to the new /security_solution_api_integration folder + +import type SuperTest from 'supertest'; + +import type { ExceptionList, NamespaceType } from '@kbn/securitysolution-io-ts-list-types'; +import { EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants'; + +import { ToolingLog } from '@kbn/tooling-log'; +import { countDownTest } from './count_down_test'; + +/** + * Remove all exceptions from both the "single" and "agnostic" spaces. + * This will retry 50 times before giving up and hopefully still not interfere with other tests + * @param supertest The supertest handle + */ +export const deleteAllExceptions = async ( + supertest: SuperTest.SuperTest, + log: ToolingLog +): Promise => { + await deleteAllExceptionsByType(supertest, log, 'single'); + await deleteAllExceptionsByType(supertest, log, 'agnostic'); +}; + +/** + * Remove all exceptions by a given type such as "agnostic" or "single". + * This will retry 50 times before giving up and hopefully still not interfere with other tests + * @param supertest The supertest handle + */ +export const deleteAllExceptionsByType = async ( + supertest: SuperTest.SuperTest, + log: ToolingLog, + type: NamespaceType +): Promise => { + await countDownTest( + async () => { + const { body } = await supertest + .get(`${EXCEPTION_LIST_URL}/_find?per_page=9999&namespace_type=${type}`) + .set('kbn-xsrf', 'true') + .send(); + const ids: string[] = body.data.map((exception: ExceptionList) => exception.id); + for await (const id of ids) { + await supertest + .delete(`${EXCEPTION_LIST_URL}?id=${id}&namespace_type=${type}`) + .set('kbn-xsrf', 'true') + .send(); + } + const { body: finalCheck } = await supertest + .get(`${EXCEPTION_LIST_URL}/_find?namespace_type=${type}`) + .set('kbn-xsrf', 'true') + .send(); + return { + passed: finalCheck.data.length === 0, + }; + }, + `deleteAllExceptions by type: "${type}"`, + log, + 50, + 1000 + ); +}; diff --git a/x-pack/test/detection_engine_api_integration/utils/index.ts b/x-pack/test/detection_engine_api_integration/utils/index.ts index d3fd910681a3e..1b18044d95449 100644 --- a/x-pack/test/detection_engine_api_integration/utils/index.ts +++ b/x-pack/test/detection_engine_api_integration/utils/index.ts @@ -86,3 +86,4 @@ export * from './prebuilt_rules/delete_all_prebuilt_rule_assets'; export * from './prebuilt_rules/install_mock_prebuilt_rules'; export * from './prebuilt_rules/install_prebuilt_rules_and_timelines'; export * from './get_legacy_action_so'; +export * from './delete_all_exceptions'; diff --git a/x-pack/test/lists_api_integration/common/config.ts b/x-pack/test/lists_api_integration/common/config.ts deleted file mode 100644 index 62663ae3915b0..0000000000000 --- a/x-pack/test/lists_api_integration/common/config.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { CA_CERT_PATH } from '@kbn/dev-utils'; -import { FtrConfigProviderContext } from '@kbn/test'; -import { services } from './services'; - -interface CreateTestConfigOptions { - license: string; - disabledPlugins?: string[]; - ssl?: boolean; -} - -export function createTestConfig(name: string, options: CreateTestConfigOptions) { - const { license = 'trial', disabledPlugins = [], ssl = false } = options; - - return async ({ readConfigFile }: FtrConfigProviderContext) => { - const xPackApiIntegrationTestsConfig = await readConfigFile( - require.resolve('../../api_integration/config.ts') - ); - const servers = { - ...xPackApiIntegrationTestsConfig.get('servers'), - elasticsearch: { - ...xPackApiIntegrationTestsConfig.get('servers.elasticsearch'), - protocol: ssl ? 'https' : 'http', - }, - }; - - return { - testFiles: [require.resolve(`../${name}/tests/`)], - servers, - services, - junit: { - reportName: 'X-Pack Lists Integration Tests', - }, - esTestCluster: { - ...xPackApiIntegrationTestsConfig.get('esTestCluster'), - license, - ssl, - serverArgs: [ - `xpack.license.self_generated.type=${license}`, - `xpack.security.enabled=${!disabledPlugins.includes('security')}`, - ], - }, - kbnTestServer: { - ...xPackApiIntegrationTestsConfig.get('kbnTestServer'), - serverArgs: [ - ...xPackApiIntegrationTestsConfig.get('kbnTestServer.serverArgs'), - ...disabledPlugins - .filter((k) => k !== 'security') - .map((key) => `--xpack.${key}.enabled=false`), - ...(ssl - ? [ - `--elasticsearch.hosts=${servers.elasticsearch.protocol}://${servers.elasticsearch.hostname}:${servers.elasticsearch.port}`, - `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`, - ] - : []), - ], - }, - }; - }; -} diff --git a/x-pack/test/lists_api_integration/common/ftr_provider_context.d.ts b/x-pack/test/lists_api_integration/common/ftr_provider_context.d.ts deleted file mode 100644 index aa56557c09df8..0000000000000 --- a/x-pack/test/lists_api_integration/common/ftr_provider_context.d.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { GenericFtrProviderContext } from '@kbn/test'; - -import { services } from './services'; - -export type FtrProviderContext = GenericFtrProviderContext; diff --git a/x-pack/test/lists_api_integration/common/services.ts b/x-pack/test/lists_api_integration/common/services.ts deleted file mode 100644 index 7e415338c405f..0000000000000 --- a/x-pack/test/lists_api_integration/common/services.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { services } from '../../api_integration/services'; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts deleted file mode 100644 index 79217043b36bb..0000000000000 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../common/ftr_provider_context'; - -// eslint-disable-next-line import/no-default-export -export default ({ loadTestFile }: FtrProviderContext): void => { - describe('lists api security and spaces enabled', function () { - loadTestFile(require.resolve('./create_lists')); - loadTestFile(require.resolve('./create_lists_index')); - loadTestFile(require.resolve('./create_list_items')); - loadTestFile(require.resolve('./patch_lists')); - loadTestFile(require.resolve('./patch_list_items')); - loadTestFile(require.resolve('./read_lists')); - loadTestFile(require.resolve('./read_list_items')); - loadTestFile(require.resolve('./update_lists')); - loadTestFile(require.resolve('./update_list_items')); - loadTestFile(require.resolve('./delete_lists')); - loadTestFile(require.resolve('./delete_list_items')); - loadTestFile(require.resolve('./duplicate_exception_list')); - loadTestFile(require.resolve('./find_lists')); - loadTestFile(require.resolve('./find_list_items')); - loadTestFile(require.resolve('./find_lists_by_size')); - loadTestFile(require.resolve('./get_exception_filter')); - loadTestFile(require.resolve('./import_exceptions')); - loadTestFile(require.resolve('./import_list_items')); - loadTestFile(require.resolve('./export_list_items')); - loadTestFile(require.resolve('./export_exception_list')); - loadTestFile(require.resolve('./create_exception_lists')); - loadTestFile(require.resolve('./create_exception_list_items')); - loadTestFile(require.resolve('./read_exception_lists')); - loadTestFile(require.resolve('./read_exception_list_items')); - loadTestFile(require.resolve('./update_exception_lists')); - loadTestFile(require.resolve('./update_exception_list_items')); - loadTestFile(require.resolve('./delete_exception_lists')); - loadTestFile(require.resolve('./delete_exception_list_items')); - loadTestFile(require.resolve('./find_exception_lists')); - loadTestFile(require.resolve('./find_exception_list_items')); - loadTestFile(require.resolve('./read_list_privileges')); - loadTestFile(require.resolve('./summary_exception_lists')); - }); -}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_scoring_task/ftr_provider_context_with_spaces.d.ts b/x-pack/test/security_solution_api_integration/ftr_provider_context_with_spaces.d.ts similarity index 71% rename from x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_scoring_task/ftr_provider_context_with_spaces.d.ts rename to x-pack/test/security_solution_api_integration/ftr_provider_context_with_spaces.d.ts index 922a5d9d25b71..f6f4e00f0b4a3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_scoring_task/ftr_provider_context_with_spaces.d.ts +++ b/x-pack/test/security_solution_api_integration/ftr_provider_context_with_spaces.d.ts @@ -6,8 +6,8 @@ */ import { GenericFtrProviderContext } from '@kbn/test'; -import { SpacesServiceProvider } from '../../../../../../common/services/spaces'; -import { services as serverlessServices } from '../../../../../../../test_serverless/api_integration/services'; +import { SpacesServiceProvider } from '../common/services/spaces'; +import { services as serverlessServices } from '../../test_serverless/api_integration/services'; const services = { ...serverlessServices, diff --git a/x-pack/test/security_solution_api_integration/package.json b/x-pack/test/security_solution_api_integration/package.json index 96d592a5c9f7e..05299d0db6b96 100644 --- a/x-pack/test/security_solution_api_integration/package.json +++ b/x-pack/test/security_solution_api_integration/package.json @@ -11,6 +11,8 @@ "run-tests:dr:basicEssentials": "node ./scripts/index.js runner detections_response basic_essentials_license", "initialize-server:ea:default": "node ./scripts/index.js server entity_analytics default_license", "run-tests:ea:default": "node ./scripts/index.js runner entity_analytics default_license", + "initialize-server:lists:default": "node ./scripts/index.js server lists_and_exception_lists default_license", + "run-tests:lists:default": "node ./scripts/index.js runner lists_and_exception_lists default_license", "exception_workflows:server:serverless": "npm run initialize-server:dr:default exceptions/workflows serverless", "exception_workflows:runner:serverless": "npm run run-tests:dr:default exceptions/workflows serverless serverlessEnv", "exception_workflows:qa:serverless": "npm run run-tests:dr:default exceptions/workflows serverless qaEnv", @@ -105,6 +107,16 @@ "detection_engine_basicessentionals:runner:serverless": "npm run run-tests:dr:basicEssentials detection_engine serverless serverlessEnv", "detection_engine_basicessentionals:qa:serverless": "npm run run-tests:dr:basicEssentials detection_engine serverless qaEnv", "detection_engine_basicessentionals:server:ess": "npm run initialize-server:dr:basicEssentials detection_engine ess", - "detection_engine_basicessentionals:runner:ess": "npm run run-tests:dr:basicEssentials detection_engine ess essEnv" + "detection_engine_basicessentionals:runner:ess": "npm run run-tests:dr:basicEssentials detection_engine ess essEnv", + "exception_lists_items:server:serverless": "npm run initialize-server:lists:default exception_lists_items serverless", + "exception_lists_items:runner:serverless": "npm run run-tests:lists:default exception_lists_items serverless serverlessEnv", + "exception_lists_items:qa:serverless": "npm run run-tests:lists:default exception_lists_items serverless qaEnv", + "exception_lists_items:server:ess": "npm run initialize-server:lists:default exception_lists_items ess", + "exception_lists_items:runner:ess": "npm run run-tests:lists:default exception_lists_items ess essEnv", + "lists_items:server:serverless": "npm run initialize-server:lists:default lists_items serverless", + "lists_items:runner:serverless": "npm run run-tests:lists:default lists_items serverless serverlessEnv", + "lists_items:qa:serverless": "npm run run-tests:lists:default lists_items serverless qaEnv", + "lists_items:server:ess": "npm run initialize-server:lists:default lists_items ess", + "lists_items:runner:ess": "npm run run-tests:lists:default lists_items ess essEnv" } } diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts index cec8d1cca41b5..e508918b0538d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts @@ -16,7 +16,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { ...functionalConfig.getAll(), testFiles: [require.resolve('..')], junit: { - reportName: 'Detection Engine ESS/Actions API Integration Tests', + reportName: 'Detection Engine ESS - Actions API Integration Tests', }, }; } diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts index 66edc0eef7f30..ea876833ea839 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts @@ -10,6 +10,6 @@ import { createTestConfig } from '../../../../../config/serverless/config.base'; export default createTestConfig({ testFiles: [require.resolve('..')], junit: { - reportName: 'Detection Engine Serverless/Actions API Integration Tests', + reportName: 'Detection Engine Serverless - Actions API Integration Tests', }, }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/date.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/date.ts index 3ccad9a60e943..c3cedb6daf88f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/date.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/date.ts @@ -12,7 +12,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, createRuleWithExceptionEntries, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/double.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/double.ts index 20efdb98b631e..9f2673b8542a3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/double.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/double.ts @@ -12,7 +12,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, createRuleWithExceptionEntries, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/float.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/float.ts index 25d7d2e83c77f..850276622cc6e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/float.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/float.ts @@ -12,7 +12,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, createRuleWithExceptionEntries, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/integer.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/integer.ts index 5df6119486113..fe26c4a2d729e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/integer.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/integer.ts @@ -12,7 +12,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, createRuleWithExceptionEntries, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips/ip.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips/ip.ts index 9e73771d11f09..ca09070e46763 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips/ip.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips/ip.ts @@ -12,7 +12,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, createRuleWithExceptionEntries, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips/ip_array.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips/ip_array.ts index 12c4eb6d55368..0c2808ee252a4 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips/ip_array.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips/ip_array.ts @@ -12,7 +12,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, createRuleWithExceptionEntries, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword/keyword.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword/keyword.ts index 11289a31b1242..9d4b1bfd80a19 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword/keyword.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword/keyword.ts @@ -12,7 +12,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, createRuleWithExceptionEntries, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword/keyword_array.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword/keyword_array.ts index 58f41321e7a86..284d20adfc3ee 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword/keyword_array.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword/keyword_array.ts @@ -12,7 +12,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/long/long.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/long/long.ts index 69803854e9306..497f24ec217a8 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/long/long.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/long/long.ts @@ -12,7 +12,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, createRuleWithExceptionEntries, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/text/text.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/text/text.ts index df7a42dc88de2..8713cc8ce859c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/text/text.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/text/text.ts @@ -13,7 +13,7 @@ import { deleteListsIndex, importFile, importTextFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, createRuleWithExceptionEntries, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/text/text_array.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/text/text_array.ts index 915d353281f66..8c4a265a182bd 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/text/text_array.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/text/text_array.ts @@ -12,7 +12,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../../lists_api_integration/utils'; +} from '../../../../../lists_and_exception_lists/utils'; import { createRule, createRuleWithExceptionEntries, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_endpoint_exceptions.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_endpoint_exceptions.ts index 1c647fe52810c..0d908b7449126 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_endpoint_exceptions.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_endpoint_exceptions.ts @@ -24,7 +24,7 @@ import { createListsIndex, deleteAllExceptions, deleteListsIndex, -} from '../../../../../../lists_api_integration/utils'; +} from '../../../../lists_and_exception_lists/utils'; import { FtrProviderContext } from '../../../../../ftr_provider_context'; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_rule_exceptions.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_rule_exceptions.ts index 3607cba64151e..1f1e4d91d4a09 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_rule_exceptions.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_rule_exceptions.ts @@ -31,7 +31,7 @@ import { import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties, -} from '../../../../../../lists_api_integration/utils'; +} from '../../../../lists_and_exception_lists/utils'; import { FtrProviderContext } from '../../../../../ftr_provider_context'; const getRuleExceptionItemMock = (): CreateRuleExceptionListItemSchema => ({ diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/find_rule_exception_references.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/find_rule_exception_references.ts index a2f996539f199..87f9c4a17914b 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/find_rule_exception_references.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/find_rule_exception_references.ts @@ -30,7 +30,7 @@ import { deleteAllAlerts, createAlertsIndex, } from '../../../utils'; -import { deleteAllExceptions } from '../../../../../../lists_api_integration/utils'; +import { deleteAllExceptions } from '../../../../lists_and_exception_lists/utils'; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_add_edit_comments.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_add_edit_comments.ts index 3ce0aa0bed874..8e36816213cff 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_add_edit_comments.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_add_edit_comments.ts @@ -18,7 +18,7 @@ import { getCreateExceptionListItemMinimalSchemaMock } from '@kbn/lists-plugin/c import { ROLES } from '@kbn/security-solution-plugin/common/test'; import { getUpdateMinimalExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_exception_list_item_schema.mock'; import { UpdateExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; -import { deleteAllExceptions } from '../../../../../../lists_api_integration/utils'; +import { deleteAllExceptions } from '../../../../lists_and_exception_lists/utils'; import { createUserAndRole, deleteUserAndRole, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts index f62501b026c20..870df90d3e475 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts @@ -60,7 +60,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../lists_api_integration/utils'; +} from '../../../../lists_and_exception_lists/utils'; import { createUserAndRole, deleteUserAndRole, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/rule_exception_synchronizations.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/rule_exception_synchronizations.ts index d89055f698ce6..7bbfcf5659420 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/rule_exception_synchronizations.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/rule_exception_synchronizations.ts @@ -30,7 +30,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../lists_api_integration/utils'; +} from '../../../../lists_and_exception_lists/utils'; import { FtrProviderContext } from '../../../../../ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/esql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/esql.ts index e869854f0f44a..ede46e40254fd 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/esql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/esql.ts @@ -26,7 +26,7 @@ import { removeRandomValuedPropertiesFromAlert, patchRule, } from '../../../utils'; -import { deleteAllExceptions } from '../../../../../../lists_api_integration/utils'; +import { deleteAllExceptions } from '../../../../lists_and_exception_lists/utils'; import { FtrProviderContext } from '../../../../../ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/machine_learning.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/machine_learning.ts index 81523755f4eea..6426738b61427 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/machine_learning.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/machine_learning.ts @@ -31,7 +31,7 @@ import { deleteAllExceptions, deleteListsIndex, importFile, -} from '../../../../../../lists_api_integration/utils'; +} from '../../../../lists_and_exception_lists/utils'; import { createRule, deleteAllRules, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/new_terms.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/new_terms.ts index 0aedb19748cf4..5e7f919520058 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/new_terms.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/new_terms.ts @@ -24,7 +24,7 @@ import { previewRuleWithExceptionEntries, removeRandomValuedPropertiesFromAlert, } from '../../../utils'; -import { deleteAllExceptions } from '../../../../../../lists_api_integration/utils'; +import { deleteAllExceptions } from '../../../../lists_and_exception_lists/utils'; import { FtrProviderContext } from '../../../../../ftr_provider_context'; import { EsArchivePathBuilder } from '../../../../../es_archive_path_builder'; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/query.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/query.ts index a1241c60f5ccf..19c02fe389fe4 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/query.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/execution_logic/query.ts @@ -46,7 +46,7 @@ import { } from '@kbn/security-solution-plugin/common/constants'; import { getMaxSignalsWarning as getMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils'; import moment from 'moment'; -import { deleteAllExceptions } from '../../../../../../lists_api_integration/utils'; +import { deleteAllExceptions } from '../../../../lists_and_exception_lists/utils'; import { createExceptionList, createExceptionListItem, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/all_types.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/all_types.ts index 26ac4627a0260..59da2429f01e8 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/all_types.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/all_types.ts @@ -14,7 +14,7 @@ import { getSecurityTelemetryStats, removeTimeFieldsFromTelemetryStats, } from '../../../utils'; -import { deleteAllExceptions } from '../../../../../../lists_api_integration/utils'; +import { deleteAllExceptions } from '../../../../lists_and_exception_lists/utils'; import { FtrProviderContext } from '../../../../../ftr_provider_context'; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/detection_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/detection_rules.ts index bed9457c34707..1298e9aeb9eaa 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/detection_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/detection_rules.ts @@ -23,7 +23,7 @@ import { createExceptionListItem, removeTimeFieldsFromTelemetryStats, } from '../../../utils'; -import { deleteAllExceptions } from '../../../../../../lists_api_integration/utils'; +import { deleteAllExceptions } from '../../../../lists_and_exception_lists/utils'; import { FtrProviderContext } from '../../../../../ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/security_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/security_lists.ts index d654cba35902f..3a4e8cb4c3ea3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/security_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/task_based/security_lists.ts @@ -20,7 +20,7 @@ import { createExceptionList, removeTimeFieldsFromTelemetryStats, } from '../../../utils'; -import { deleteAllExceptions } from '../../../../../../lists_api_integration/utils'; +import { deleteAllExceptions } from '../../../../lists_and_exception_lists/utils'; import { FtrProviderContext } from '../../../../../ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_scoring_task/task_execution_nondefault_spaces.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_scoring_task/task_execution_nondefault_spaces.ts index cf9bf98f88d9d..d869fae2f3f95 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_scoring_task/task_execution_nondefault_spaces.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_scoring_task/task_execution_nondefault_spaces.ts @@ -23,7 +23,7 @@ import { deleteRiskScoreIndices, } from '../../../utils'; -import { FtrProviderContextWithSpaces } from './ftr_provider_context_with_spaces'; +import { FtrProviderContextWithSpaces } from '../../../../../ftr_provider_context_with_spaces'; export default ({ getService }: FtrProviderContextWithSpaces): void => { const supertest = getService('supertest'); diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/ess.config.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/ess.config.ts new file mode 100644 index 0000000000000..366e0b956e370 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/ess.config.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile( + require.resolve('../../../../../config/ess/config.base.trial') + ); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('..')], + junit: { + reportName: 'Detection Engine ESS - Execption Lists and Items Integration Tests APIS', + }, + }; +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/serverless.config.ts new file mode 100644 index 0000000000000..bb1410030e0db --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/serverless.config.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTestConfig } from '../../../../../config/serverless/config.base'; + +export default createTestConfig({ + testFiles: [require.resolve('..')], + junit: { + reportName: 'Detection Engine Serverless - Execption Lists and Items Integration Tests APIS', + }, +}); diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/index.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/index.ts new file mode 100644 index 0000000000000..b490143ff28bd --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../ftr_provider_context'; +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Exception Lists and Items API', function () { + loadTestFile(require.resolve('./items')); + loadTestFile(require.resolve('./lists')); + }); +} diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/create_exception_list_items.ts similarity index 92% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/create_exception_list_items.ts index ab4eb4d802fc9..ad7c413a3c6a7 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/create_exception_list_items.ts @@ -15,21 +15,21 @@ import { getCreateExceptionListItemMinimalSchemaMock, getCreateExceptionListItemMinimalSchemaMockWithoutId, } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_item_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { removeListItemServerGeneratedProperties, removeExceptionListItemServerGeneratedProperties, -} from '../../utils'; + deleteAllExceptions, +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -import { deleteAllExceptions } from '../../utils'; - -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('create_exception_list_items', () => { + describe('@ess @serverless create_exception_list_items', () => { describe('validation errors', () => { it('should give a 404 error that the exception list must exist first before being able to add a list item to the exception list', async () => { const { body } = await supertest @@ -64,7 +64,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeExceptionListItemServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getExceptionListItemResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should create a match any exception item with multiple case sensitive values', async () => { @@ -93,7 +95,7 @@ export default ({ getService }: FtrProviderContext) => { const bodyToCompare = removeExceptionListItemServerGeneratedProperties(body); expect(bodyToCompare).to.eql({ - ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(), + ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), entries, }); }); @@ -113,7 +115,7 @@ export default ({ getService }: FtrProviderContext) => { const bodyToCompare = removeListItemServerGeneratedProperties(body); const outputList: Partial = { - ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(), + ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), item_id: body.item_id, }; expect(bodyToCompare).to.eql(outputList); @@ -163,7 +165,7 @@ export default ({ getService }: FtrProviderContext) => { const bodyToCompare = removeExceptionListItemServerGeneratedProperties(body); expect(bodyToCompare).to.eql({ - ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(), + ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), expire_time: datetime, }); }); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/delete_exception_list_items.ts similarity index 88% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/delete_exception_list_items.ts index 1a97ac6db4cb0..4d3b2be7de27f 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/delete_exception_list_items.ts @@ -15,16 +15,21 @@ import { getCreateExceptionListItemMinimalSchemaMockWithoutId, } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_item_schema.mock'; import { getCreateExceptionListMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties } from '../../utils'; +import { + deleteAllExceptions, + removeExceptionListItemServerGeneratedProperties, +} from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('delete_exception_list_items', () => { + describe('@ess @serverless delete_exception_list_items', () => { describe('delete exception list items', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); @@ -56,7 +61,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeExceptionListItemServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getExceptionListItemResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should delete a single exception list item using an auto generated id', async () => { @@ -80,7 +87,7 @@ export default ({ getService }: FtrProviderContext) => { .set('kbn-xsrf', 'true') .expect(200); const outputtedList: Partial = { - ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(), + ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), item_id: body.item_id, }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/find_exception_list_items.ts similarity index 90% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/find_exception_list_items.ts index 909930d713473..3fcbd476b7547 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/find_exception_list_items.ts @@ -17,16 +17,21 @@ import { getCreateExceptionListMinimalSchemaMock, getCreateExceptionListDetectionSchemaMock, } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties } from '../../utils'; +import { + deleteAllExceptions, + removeExceptionListItemServerGeneratedProperties, +} from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('find_exception_list_items', () => { + describe('@ess @serverless find_exception_list_items', () => { describe('find exception list items', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); @@ -104,7 +109,7 @@ export default ({ getService }: FtrProviderContext): void => { data: [ { comments: [], - created_by: 'elastic', + created_by: ELASTICSEARCH_USERNAME, description: 'some description', entries: [ { @@ -121,7 +126,7 @@ export default ({ getService }: FtrProviderContext): void => { os_types: ['windows'], tags: [], type: 'simple', - updated_by: 'elastic', + updated_by: ELASTICSEARCH_USERNAME, }, ], page: 1, @@ -171,7 +176,9 @@ export default ({ getService }: FtrProviderContext): void => { body.data = [removeExceptionListItemServerGeneratedProperties(body.data[0])]; expect(body).to.eql({ - data: [getExceptionListItemResponseMockWithoutAutoGeneratedValues()], + data: [ + getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), + ], page: 1, per_page: 20, total: 1, diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/index.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/index.ts new file mode 100644 index 0000000000000..df84a7a9ee3d5 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Lists API', function () { + loadTestFile(require.resolve('./create_exception_list_items')); + loadTestFile(require.resolve('./read_exception_list_items')); + loadTestFile(require.resolve('./update_exception_list_items')); + loadTestFile(require.resolve('./delete_exception_list_items')); + loadTestFile(require.resolve('./find_exception_list_items')); + }); +} diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/read_exception_list_items.ts similarity index 89% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/read_exception_list_items.ts index 42783a461ce7e..a2cffa490194c 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/read_exception_list_items.ts @@ -15,16 +15,20 @@ import { getCreateExceptionListItemMinimalSchemaMockWithoutId, } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_item_schema.mock'; import { getCreateExceptionListMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties } from '../../utils'; +import { + deleteAllExceptions, + removeExceptionListItemServerGeneratedProperties, +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('read_exception_list_items', () => { + describe('@ess @serverless read_exception_list_items', () => { describe('reading exception list items', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); @@ -45,7 +49,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeExceptionListItemServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getExceptionListItemResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should be able to read a single exception list item using id', async () => { @@ -69,7 +75,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeExceptionListItemServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getExceptionListItemResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should be able to read a single list item with an auto-generated id', async () => { @@ -92,7 +100,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputtedList: Partial = { - ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(), + ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), item_id: body.item_id, }; @@ -120,7 +128,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputtedList: Partial = { - ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(), + ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), item_id: body.item_id, }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/update_exception_list_items.ts similarity index 97% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/update_exception_list_items.ts index cfb01154be854..6e0a1daa024e1 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/items/update_exception_list_items.ts @@ -19,20 +19,21 @@ import { } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_item_schema.mock'; import { getCreateExceptionListMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; import { getUpdateMinimalExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_exception_list_item_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties, removeExceptionListServerGeneratedProperties, -} from '../../utils'; +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('update_exception_list_items', () => { + describe('@ess @serverless update_exception_list_items', () => { describe('update exception list items', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); @@ -252,7 +253,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputList: Partial = { - ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(), + ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', }; @@ -292,7 +293,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputList: Partial = { - ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(), + ...getExceptionListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', item_id: body.item_id, }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/create_exception_lists.ts similarity index 86% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/create_exception_lists.ts index a3c71a8be2fc1..da066b078b0c4 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/create_exception_lists.ts @@ -14,16 +14,17 @@ import { getCreateExceptionListMinimalSchemaMock, getCreateExceptionListMinimalSchemaMockWithoutId, } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../utils'; +import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../../utils'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('create_exception_lists', () => { + describe('@ess @serverless create_exception_lists', () => { describe('creating exception lists', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); @@ -37,7 +38,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeExceptionListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getExceptionResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should create a simple exception list without a list_id', async () => { @@ -49,7 +52,7 @@ export default ({ getService }: FtrProviderContext) => { const bodyToCompare = removeExceptionListServerGeneratedProperties(body); const outputtedList: Partial = { - ...getExceptionResponseMockWithoutAutoGeneratedValues(), + ...getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), list_id: bodyToCompare.list_id, }; expect(bodyToCompare).to.eql(outputtedList); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/delete_exception_lists.ts similarity index 89% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/delete_exception_lists.ts index 5361bcb82f6a2..ae109897fb138 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/delete_exception_lists.ts @@ -14,16 +14,18 @@ import { getCreateExceptionListMinimalSchemaMock, getCreateExceptionListMinimalSchemaMockWithoutId, } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../utils'; +import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('delete_exception_lists', () => { + describe('@ess @serverless delete_exception_lists', () => { describe('delete exception lists', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); @@ -46,7 +48,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeExceptionListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getExceptionResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should delete a single exception list using an auto generated id', async () => { @@ -64,7 +68,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputtedList: Partial = { - ...getExceptionResponseMockWithoutAutoGeneratedValues(), + ...getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), list_id: body.list_id, }; const bodyToCompare = removeExceptionListServerGeneratedProperties(body); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/duplicate_exception_list.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/duplicate_exception_list.ts similarity index 93% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/duplicate_exception_list.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/duplicate_exception_list.ts index 54e7a89042b02..14a4dcfb65ec7 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/duplicate_exception_list.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/duplicate_exception_list.ts @@ -15,16 +15,18 @@ import { import { getExceptionResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/exception_list_schema.mock'; import { getCreateExceptionListDetectionSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; import { getCreateExceptionListItemMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_item_schema.mock'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; -import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../utils'; +import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('duplicate_exception_lists', () => { + describe('@ess @serverless duplicate_exception_lists', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); }); @@ -48,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => { const bodyToCompare = removeExceptionListServerGeneratedProperties(body); expect(bodyToCompare).to.eql({ - ...getExceptionResponseMockWithoutAutoGeneratedValues(), + ...getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), type: 'detection', list_id: body.list_id, name: `${getCreateExceptionListDetectionSchemaMock().name} [Duplicate]`, @@ -89,7 +91,7 @@ export default ({ getService }: FtrProviderContext) => { const listBodyToCompare = removeExceptionListServerGeneratedProperties(listBody); expect(listBodyToCompare).to.eql({ - ...getExceptionResponseMockWithoutAutoGeneratedValues(), + ...getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), type: 'detection', list_id: listBody.list_id, name: `${getCreateExceptionListDetectionSchemaMock().name} [Duplicate]`, diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/export_exception_list.ts similarity index 97% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/export_exception_list.ts index bfde054eb8f1e..726ec6269508f 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/export_exception_list.ts @@ -11,20 +11,19 @@ import { EXCEPTION_LIST_URL, EXCEPTION_LIST_ITEM_URL } from '@kbn/securitysoluti import { getCreateExceptionListMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; import { getCreateExceptionListItemMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_item_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { removeExceptionListServerGeneratedProperties, removeExceptionListItemServerGeneratedProperties, binaryToString, deleteAllExceptions, -} from '../../utils'; +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); - describe('export_exception_list_route', () => { + describe('@ess @serverless export_exception_list_route', () => { describe('exporting exception lists', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/find_exception_lists.ts similarity index 87% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/find_exception_lists.ts index 349a44c506949..cb405b7b7d642 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/find_exception_lists.ts @@ -10,16 +10,17 @@ import expect from '@kbn/expect'; import { EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants'; import { getExceptionResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/exception_list_schema.mock'; import { getCreateExceptionListMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../../utils'; -import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('find_exception_lists', () => { + describe('@ess @serverless find_exception_lists', () => { describe('find exception lists', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); @@ -57,7 +58,7 @@ export default ({ getService }: FtrProviderContext): void => { body.data = [removeExceptionListServerGeneratedProperties(body.data[0])]; expect(body).to.eql({ - data: [getExceptionResponseMockWithoutAutoGeneratedValues()], + data: [getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME)], page: 1, per_page: 20, total: 1, diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/get_exception_filter.ts similarity index 92% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/get_exception_filter.ts index 913ce14db1f00..633262ca1baa4 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/get_exception_filter.ts @@ -6,7 +6,10 @@ */ import expect from '@kbn/expect'; - +import { + ELASTIC_HTTP_VERSION_HEADER, + X_ELASTIC_INTERNAL_ORIGIN_REQUEST, +} from '@kbn/core-http-common'; import { INTERNAL_EXCEPTION_FILTER, EXCEPTION_LIST_URL, @@ -18,16 +21,16 @@ import { } from '@kbn/lists-plugin/common/schemas/request/get_exception_filter_schema.mock'; import { getCreateExceptionListItemMinimalSchemaMockWithoutId } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_item_schema.mock'; import { getCreateExceptionListDetectionSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { createListsIndex, deleteListsIndex } from '../../utils'; +import { createListsIndex, deleteListsIndex } from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); - describe('get_exception_filter', () => { + describe('@ess @serverless get_exception_filter', () => { describe('get exception filter', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -41,7 +44,8 @@ export default ({ getService }: FtrProviderContext): void => { const { body } = await supertest .post(`${INTERNAL_EXCEPTION_FILTER}`) .set('kbn-xsrf', 'true') - .set('elastic-api-version', '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') .send(getExceptionFilterFromExceptionItemsSchemaMock()) .expect(200); @@ -122,7 +126,8 @@ export default ({ getService }: FtrProviderContext): void => { const { body } = await supertest .post(`${INTERNAL_EXCEPTION_FILTER}`) .set('kbn-xsrf', 'true') - .set('elastic-api-version', '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') .send(getExceptionFilterFromExceptionIdsSchemaMock()) .expect(200); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/import_exceptions.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/import_exceptions.ts similarity index 99% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/import_exceptions.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/import_exceptions.ts index 8736ad75eeb6f..bc191618914d2 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/import_exceptions.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/import_exceptions.ts @@ -14,15 +14,15 @@ import { getImportExceptionsListItemSchemaMock, getImportExceptionsListSchemaMock, } from '@kbn/lists-plugin/common/schemas/request/import_exceptions_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { deleteAllExceptions } from '../../utils'; +import { deleteAllExceptions } from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); - describe('import_exceptions', () => { + describe('@ess @serverless import_exceptions', () => { beforeEach(async () => { await deleteAllExceptions(supertest, log); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/index.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/index.ts new file mode 100644 index 0000000000000..0e27b93b93b6f --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Lists API', function () { + loadTestFile(require.resolve('./duplicate_exception_list')); + loadTestFile(require.resolve('./get_exception_filter')); + loadTestFile(require.resolve('./import_exceptions')); + loadTestFile(require.resolve('./export_exception_list')); + loadTestFile(require.resolve('./create_exception_lists')); + loadTestFile(require.resolve('./read_exception_lists')); + loadTestFile(require.resolve('./update_exception_lists')); + loadTestFile(require.resolve('./delete_exception_lists')); + loadTestFile(require.resolve('./find_exception_lists')); + loadTestFile(require.resolve('./summary_exception_lists')); + }); +} diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/read_exception_lists.ts similarity index 87% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/read_exception_lists.ts index 31293e97cb078..b2061928e1759 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/read_exception_lists.ts @@ -14,16 +14,18 @@ import { getCreateExceptionListMinimalSchemaMock, getCreateExceptionListMinimalSchemaMockWithoutId, } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../utils'; +import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('read_exception_lists', () => { + describe('@ess @serverless read_exception_lists', () => { describe('reading exception lists', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); @@ -43,7 +45,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeExceptionListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getExceptionResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should be able to read a single exception list using id', async () => { @@ -60,7 +64,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeExceptionListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getExceptionResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should be able to read a single list with an auto-generated list_id', async () => { @@ -77,7 +83,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputtedList: Partial = { - ...getExceptionResponseMockWithoutAutoGeneratedValues(), + ...getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), list_id: body.list_id, }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/summary_exception_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/summary_exception_lists.ts similarity index 95% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/summary_exception_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/summary_exception_lists.ts index d5848e7ee44be..2c0c9780714f7 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/summary_exception_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/summary_exception_lists.ts @@ -12,18 +12,18 @@ import { EXCEPTION_LIST_URL, EXCEPTION_LIST_ITEM_URL } from '@kbn/securitysoluti import { LIST_ID } from '@kbn/lists-plugin/common/constants.mock'; import { getCreateExceptionListMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; import { getCreateExceptionListItemMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_item_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { createListsIndex, deleteListsIndex, deleteAllExceptions } from '../../utils'; +import { createListsIndex, deleteListsIndex, deleteAllExceptions } from '../../../utils'; interface SummaryResponseType { body: ExceptionListSummarySchema; } -// eslint-disable-next-line import/no-default-export +import { FtrProviderContext } from '../../../../../ftr_provider_context'; + export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); - describe('summary_exception_lists', () => { + describe('@ess @serverless summary_exception_lists', () => { describe('summary exception lists', () => { beforeEach(async () => { await createListsIndex(supertest, log); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/update_exception_lists.ts similarity index 94% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/update_exception_lists.ts index d4c5cd422e772..1a5aed16b128c 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/lists/update_exception_lists.ts @@ -15,16 +15,18 @@ import { EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants'; import { getExceptionResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/exception_list_schema.mock'; import { getCreateExceptionListMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; import { getUpdateMinimalExceptionListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_exception_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../utils'; +import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('update_exception_lists', () => { + describe('@ess @serverless update_exception_lists', () => { describe('update exception lists', () => { afterEach(async () => { await deleteAllExceptions(supertest, log); @@ -51,7 +53,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputList: Partial = { - ...getExceptionResponseMockWithoutAutoGeneratedValues(), + ...getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', version: 2, }; @@ -84,7 +86,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputList: Partial = { - ...getExceptionResponseMockWithoutAutoGeneratedValues(), + ...getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', list_id: body.list_id, version: 2, @@ -116,7 +118,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputList: Partial = { - ...getExceptionResponseMockWithoutAutoGeneratedValues(), + ...getExceptionResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', description: 'some other description', version: 2, diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/ess.config.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/ess.config.ts new file mode 100644 index 0000000000000..522c44b41d85a --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/ess.config.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile( + require.resolve('../../../../../config/ess/config.base.trial') + ); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('..')], + junit: { + reportName: 'Detection Engine ESS - Lists and Items Integration Tests APIS', + }, + }; +} diff --git a/x-pack/test/lists_api_integration/security_and_spaces/config.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/serverless.config.ts similarity index 50% rename from x-pack/test/lists_api_integration/security_and_spaces/config.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/serverless.config.ts index 8b7e43945c8a2..7e324d6e29836 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/serverless.config.ts @@ -5,11 +5,11 @@ * 2.0. */ -import { createTestConfig } from '../common/config'; +import { createTestConfig } from '../../../../../config/serverless/config.base'; -// eslint-disable-next-line import/no-default-export -export default createTestConfig('security_and_spaces', { - disabledPlugins: [], - license: 'trial', - ssl: true, +export default createTestConfig({ + testFiles: [require.resolve('..')], + junit: { + reportName: 'Detection Engine Serverless - Lists and Items Integration Tests APIS', + }, }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/index.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/index.ts new file mode 100644 index 0000000000000..9422d4b2315f9 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../ftr_provider_context'; +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Lists and Items API', function () { + loadTestFile(require.resolve('./items')); + loadTestFile(require.resolve('./lists')); + }); +} diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/create_list_items.ts similarity index 86% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/create_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/create_list_items.ts index 40be89af0284a..f174fdff3f774 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/create_list_items.ts @@ -16,20 +16,21 @@ import { getCreateMinimalListItemSchemaMockWithoutId, } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; - import { createListsIndex, deleteListsIndex, removeListItemServerGeneratedProperties, -} from '../../utils'; +} from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('create_list_items', () => { + describe('@ess @serverless create_list_items', () => { describe('validation errors', () => { it('should give a 404 error that the list must exist first before being able to add a list item', async () => { const { body } = await supertest @@ -68,7 +69,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListItemServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListItemResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should create a simple list item without an id', async () => { @@ -85,7 +88,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListItemServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListItemResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should cause a 409 conflict if we attempt to create the same list item twice', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/delete_list_items.ts similarity index 85% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/delete_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/delete_list_items.ts index d317e647174f7..f17d950a10dc1 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/delete_list_items.ts @@ -11,20 +11,21 @@ import { LIST_URL, LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createListsIndex, deleteListsIndex, removeListItemServerGeneratedProperties, -} from '../../utils'; +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('delete_list_items', () => { + describe('@ess @serverless delete_list_items', () => { describe('deleting list items', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -56,7 +57,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListItemServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListItemResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should delete a single list using an auto generated id', async () => { @@ -81,7 +84,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListItemServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListItemResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should return an error if the id does not exist when trying to delete it', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/export_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/export_list_items.ts similarity index 95% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/export_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/export_list_items.ts index fe471e2794b43..b9c99a49af931 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/export_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/export_list_items.ts @@ -11,16 +11,16 @@ import { LIST_ITEM_URL, LIST_URL } from '@kbn/securitysolution-list-constants'; import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { LIST_ID, NAME } from '@kbn/lists-plugin/common/constants.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { createListsIndex, deleteListsIndex, binaryToString } from '../../utils'; +import { createListsIndex, deleteListsIndex, binaryToString } from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); - describe('export_list_items', () => { + describe('@ess @serverless export_list_items', () => { describe('exporting lists', () => { beforeEach(async () => { await createListsIndex(supertest, log); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/find_list_items.ts similarity index 92% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/find_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/find_list_items.ts index c4ecce7d4d0e7..0c8aa8aa50fa8 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/find_list_items.ts @@ -12,20 +12,22 @@ import { LIST_ITEM_ID, LIST_ID } from '@kbn/lists-plugin/common/constants.mock'; import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createListsIndex, deleteListsIndex, removeListItemServerGeneratedProperties, -} from '../../utils'; +} from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('find_list_items', () => { + describe('@ess @serverless find_list_items', () => { describe('find list items', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -107,7 +109,7 @@ export default ({ getService }: FtrProviderContext): void => { // cursor is a constant changing value so we have to delete it as well. delete body.cursor; expect(body).to.eql({ - data: [getListItemResponseMockWithoutAutoGeneratedValues()], + data: [getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME)], page: 1, per_page: 20, total: 1, diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/import_list_items.ts similarity index 74% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/import_list_items.ts index 89ae216adc865..99a96ae9052b1 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/import_list_items.ts @@ -11,7 +11,6 @@ import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; import { getImportListItemAsBuffer } from '@kbn/lists-plugin/common/schemas/request/import_list_item_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createListsIndex, @@ -19,16 +18,16 @@ import { removeListServerGeneratedProperties, removeListItemServerGeneratedProperties, waitFor, - createListsIndices, -} from '../../utils'; +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); - const es = getService('es'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('import_list_items', () => { + describe('@ess @serverless import_list_items', () => { describe('importing list items without an index', () => { it('should not import a list item if the index does not exist yet', async () => { const { body } = await supertest @@ -74,7 +73,7 @@ export default ({ getService }: FtrProviderContext): void => { const bodyToCompare = removeListServerGeneratedProperties(body); const outputtedList: Partial = { - ...getListResponseMockWithoutAutoGeneratedValues(), + ...getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'list_items.txt', description: 'File uploaded from file system of list_items.txt', }; @@ -107,41 +106,11 @@ export default ({ getService }: FtrProviderContext): void => { const bodyToCompare = removeListItemServerGeneratedProperties(body[0]); const outputtedList: Partial = { - ...getListItemResponseMockWithoutAutoGeneratedValues(), + ...getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), list_id: 'list_items.txt', }; expect(bodyToCompare).to.eql(outputtedList); }); - - describe('legacy index (before migration to data streams)', () => { - beforeEach(async () => { - await deleteListsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteListsIndex(supertest, log); - }); - - it('should import list to legacy index and migrate it', async () => { - // create legacy indices - await createListsIndices(es); - - const { body } = await supertest - .post(`${LIST_ITEM_URL}/_import?type=ip`) - .set('kbn-xsrf', 'true') - .attach('file', getImportListItemAsBuffer(['127.0.0.1', '127.0.0.2']), 'list_items.txt') - .expect('Content-Type', 'application/json; charset=utf-8') - .expect(200); - - const bodyToCompare = removeListServerGeneratedProperties(body); - const outputtedList: Partial = { - ...getListResponseMockWithoutAutoGeneratedValues(), - name: 'list_items.txt', - description: 'File uploaded from file system of list_items.txt', - }; - expect(bodyToCompare).to.eql(outputtedList); - }); - }); }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/import_list_items_migrations.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/import_list_items_migrations.ts new file mode 100644 index 0000000000000..cd614bd07e359 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/import_list_items_migrations.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import type { ListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; +import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; +import { getImportListItemAsBuffer } from '@kbn/lists-plugin/common/schemas/request/import_list_item_schema.mock'; + +import { + deleteListsIndex, + removeListServerGeneratedProperties, + createListsIndices, +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + const log = getService('log'); + const es = getService('es'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); + + describe('@ess import_list_items_migrations', () => { + describe('import list to legacy index and migrate it', () => { + describe('legacy index (before migration to data streams)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + it('should import list to legacy index and migrate it', async () => { + // create legacy indices + await createListsIndices(es); + + const { body } = await supertest + .post(`${LIST_ITEM_URL}/_import?type=ip`) + .set('kbn-xsrf', 'true') + .attach('file', getImportListItemAsBuffer(['127.0.0.1', '127.0.0.2']), 'list_items.txt') + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(200); + + const bodyToCompare = removeListServerGeneratedProperties(body); + const outputtedList: Partial = { + ...getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), + name: 'list_items.txt', + description: 'File uploaded from file system of list_items.txt', + }; + expect(bodyToCompare).to.eql(outputtedList); + }); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/index.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/index.ts new file mode 100644 index 0000000000000..d89f7ca5566df --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Lists API', function () { + loadTestFile(require.resolve('./create_list_items')); + loadTestFile(require.resolve('./patch_list_items')); + loadTestFile(require.resolve('./patch_list_items_migrations')); + loadTestFile(require.resolve('./read_list_items')); + loadTestFile(require.resolve('./update_list_items')); + loadTestFile(require.resolve('./update_list_items_migrations')); + loadTestFile(require.resolve('./delete_list_items')); + loadTestFile(require.resolve('./find_list_items')); + loadTestFile(require.resolve('./import_list_items')); + loadTestFile(require.resolve('./import_list_items_migrations')); + loadTestFile(require.resolve('./export_list_items')); + }); +} diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/patch_list_items.ts similarity index 73% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/patch_list_items.ts index 0827b5813b6c4..01a9d332ba355 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/patch_list_items.ts @@ -12,30 +12,27 @@ import type { CreateListItemSchema, ListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_URL, LIST_ITEM_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; +import { LIST_URL, LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getUpdateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_list_item_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createListsIndex, deleteListsIndex, removeListItemServerGeneratedProperties, - createListsIndices, - createListBypassingChecks, - createListItemBypassingChecks, -} from '../../utils'; +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); const retry = getService('retry'); - const es = getService('es'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('patch_list_items', () => { + describe('@ess @serverless patch_list_items', () => { describe('patch list items', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -73,7 +70,7 @@ export default ({ getService }: FtrProviderContext) => { .send(patchListItemPayload); const outputListItem: Partial = { - ...getListItemResponseMockWithoutAutoGeneratedValues(), + ...getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), value: '192.168.0.2', }; const bodyToCompare = removeListItemServerGeneratedProperties(body); @@ -123,7 +120,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputListItem: Partial = { - ...getListItemResponseMockWithoutAutoGeneratedValues(), + ...getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), value: '192.168.0.2', }; const bodyToCompare = { @@ -219,63 +216,6 @@ export default ({ getService }: FtrProviderContext) => { message: 'list item id: "some-other-id" not found', }); }); - - describe('legacy list items index (list created before migration to data stream)', () => { - beforeEach(async () => { - await deleteListsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteListsIndex(supertest, log); - }); - it('should patch list item that was created in legacy index and migrated through LIST_INDEX request', async () => { - const listId = 'random-list'; - const listItemId = 'random-list-item'; - // create legacy indices - await createListsIndices(es); - // create a simple list - await createListBypassingChecks({ es, id: listId }); - await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); - // migrates old indices to data streams - await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); - - const patchPayload: PatchListItemSchema = { - id: listItemId, - value: 'new one', - }; - - const { body } = await supertest - .patch(LIST_ITEM_URL) - .set('kbn-xsrf', 'true') - .send(patchPayload) - .expect(200); - - expect(body.value).to.be('new one'); - }); - - it('should patch list item that was created in legacy index and not yet migrated', async () => { - const listId = 'random-list'; - const listItemId = 'random-list-item'; - // create legacy indices - await createListsIndices(es); - // create a simple list - await createListBypassingChecks({ es, id: listId }); - await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); - - const patchPayload: PatchListItemSchema = { - id: listItemId, - value: 'new one', - }; - - const { body } = await supertest - .patch(LIST_ITEM_URL) - .set('kbn-xsrf', 'true') - .send(patchPayload) - .expect(200); - - expect(body.value).to.be('new one'); - }); - }); }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/patch_list_items_migrations.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/patch_list_items_migrations.ts new file mode 100644 index 0000000000000..8e0ee81414faa --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/patch_list_items_migrations.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import type { PatchListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { LIST_ITEM_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; + +import { + createListsIndex, + deleteListsIndex, + createListsIndices, + createListBypassingChecks, + createListItemBypassingChecks, +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const log = getService('log'); + const es = getService('es'); + + describe('@ess patch_list_items_migrations', () => { + describe('patch list items', () => { + beforeEach(async () => { + await createListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + describe('legacy list items index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should patch list item that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + const patchPayload: PatchListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.value).to.be('new one'); + }); + + it('should patch list item that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + + const patchPayload: PatchListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.value).to.be('new one'); + }); + }); + }); + }); +}; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/read_list_items.ts similarity index 85% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/read_list_items.ts index dc99e9d4d180a..b0005ccb3fc0b 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/read_list_items.ts @@ -11,20 +11,22 @@ import { LIST_URL, LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createListsIndex, deleteListsIndex, removeListItemServerGeneratedProperties, -} from '../../utils'; +} from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('read_list_items', () => { + describe('@ess @serverless read_list_items', () => { describe('reading list items', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -53,7 +55,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListItemServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListItemResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should be able to read a single list item with an auto-generated list id', async () => { @@ -75,7 +79,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListItemServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListItemResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should return 404 if given a fake id', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/update_list_items.ts similarity index 80% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/update_list_items.ts index e2bcddeb24841..90d246d141866 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/update_list_items.ts @@ -12,30 +12,27 @@ import type { CreateListItemSchema, ListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_URL, LIST_ITEM_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; +import { LIST_URL, LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getUpdateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_list_item_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createListsIndex, deleteListsIndex, removeListItemServerGeneratedProperties, - createListsIndices, - createListBypassingChecks, - createListItemBypassingChecks, -} from '../../utils'; +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); const retry = getService('retry'); - const es = getService('es'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('update_list_items', () => { + describe('@ess @serverless update_list_items', () => { describe('update list items', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -72,7 +69,7 @@ export default ({ getService }: FtrProviderContext) => { .send(updatedListItem); const outputListItem: Partial = { - ...getListItemResponseMockWithoutAutoGeneratedValues(), + ...getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), value: '192.168.0.2', }; const bodyToCompare = removeListItemServerGeneratedProperties(body); @@ -122,7 +119,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputListItem: Partial = { - ...getListItemResponseMockWithoutAutoGeneratedValues(), + ...getListItemResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), value: '192.168.0.2', }; const bodyToCompare = { @@ -305,63 +302,6 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); }); }); - - describe('legacy list items index (list created before migration to data stream)', () => { - beforeEach(async () => { - await deleteListsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteListsIndex(supertest, log); - }); - it('should update list item that was created in legacy index and migrated through LIST_INDEX request', async () => { - const listId = 'random-list'; - const listItemId = 'random-list-item'; - // create legacy indices - await createListsIndices(es); - // create a simple list - await createListBypassingChecks({ es, id: listId }); - await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); - // migrates old indices to data streams - await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); - - const updatedListItem: UpdateListItemSchema = { - id: listItemId, - value: 'new one', - }; - - const { body } = await supertest - .put(LIST_ITEM_URL) - .set('kbn-xsrf', 'true') - .send(updatedListItem) - .expect(200); - - expect(body.value).to.be('new one'); - }); - - it('should update list item that was created in legacy index and not yet migrated', async () => { - const listId = 'random-list'; - const listItemId = 'random-list-item'; - // create legacy indices - await createListsIndices(es); - // create a simple list - await createListBypassingChecks({ es, id: listId }); - await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); - - const updatedListItem: UpdateListItemSchema = { - id: listItemId, - value: 'new one', - }; - - const { body } = await supertest - .put(LIST_ITEM_URL) - .set('kbn-xsrf', 'true') - .send(updatedListItem) - .expect(200); - - expect(body.value).to.be('new one'); - }); - }); }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/update_list_items_migrations.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/update_list_items_migrations.ts new file mode 100644 index 0000000000000..06296ed60589e --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/items/update_list_items_migrations.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import type { UpdateListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { LIST_ITEM_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; + +import { + createListsIndex, + deleteListsIndex, + createListsIndices, + createListBypassingChecks, + createListItemBypassingChecks, +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const log = getService('log'); + const es = getService('es'); + + describe('@ess update_list_items_migrations', () => { + describe('update list items', () => { + beforeEach(async () => { + await createListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + describe('legacy list items index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should update list item that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + const updatedListItem: UpdateListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(200); + + expect(body.value).to.be('new one'); + }); + + it('should update list item that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + + const updatedListItem: UpdateListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(200); + + expect(body.value).to.be('new one'); + }); + }); + }); + }); +}; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists.ts similarity index 81% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists.ts index b64f9d4206db3..8cd7517c9efe8 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists.ts @@ -13,20 +13,22 @@ import { getCreateMinimalListSchemaMockWithoutId, } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createListsIndex, deleteListsIndex, removeListServerGeneratedProperties, -} from '../../utils'; +} from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('create_lists', () => { + describe('@ess @serverless create_lists', () => { describe('creating lists', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -44,7 +46,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should create a simple list without a list_id', async () => { @@ -55,7 +59,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should cause a 409 conflict if we attempt to create the same list_id twice', async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists_index.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists_index.ts new file mode 100644 index 0000000000000..bb9167f4ba8a2 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists_index.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; + +import { deleteListsIndex } from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const log = getService('log'); + + describe('@ess @serverless create_list_index_route', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + it('should create lists data streams', async () => { + const { body: fetchedIndices } = await supertest + .get(LIST_INDEX) + .set('kbn-xsrf', 'true') + .expect(404); + + expect(fetchedIndices).to.eql({ + message: 'data stream .lists-default and data stream .items-default does not exist', + status_code: 404, + }); + + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true').expect(200); + + const { body } = await supertest.get(LIST_INDEX).set('kbn-xsrf', 'true').expect(200); + + expect(body).to.eql({ list_index: true, list_item_index: true }); + }); + }); +}; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists_index_migrations.ts similarity index 77% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists_index_migrations.ts index ab549f27c4d2d..4cd8b1afd74a0 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/create_lists_index_migrations.ts @@ -9,17 +9,17 @@ import expect from '@kbn/expect'; import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; import { getTemplateExists, getIndexTemplateExists } from '@kbn/securitysolution-es-utils'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { createLegacyListsIndices, deleteListsIndex } from '../../utils'; +import { createLegacyListsIndices, deleteListsIndex } from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); const es = getService('es'); - describe('create_list_index_route', () => { + describe('@ess create_list_index_route_migrations', () => { beforeEach(async () => { await deleteListsIndex(supertest, log); }); @@ -28,24 +28,6 @@ export default ({ getService }: FtrProviderContext) => { await deleteListsIndex(supertest, log); }); - it('should create lists data streams', async () => { - const { body: fetchedIndices } = await supertest - .get(LIST_INDEX) - .set('kbn-xsrf', 'true') - .expect(404); - - expect(fetchedIndices).to.eql({ - message: 'data stream .lists-default and data stream .items-default does not exist', - status_code: 404, - }); - - await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true').expect(200); - - const { body } = await supertest.get(LIST_INDEX).set('kbn-xsrf', 'true').expect(200); - - expect(body).to.eql({ list_index: true, list_item_index: true }); - }); - it('should migrate lists indices to data streams and remove old legacy templates', async () => { // create legacy indices await createLegacyListsIndices(es); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/delete_lists.ts similarity index 92% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/delete_lists.ts index 1e9b4911c3092..87b54d9a2e99a 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/delete_lists.ts @@ -27,15 +27,16 @@ import { deleteAllExceptions, deleteListsIndex, removeListServerGeneratedProperties, -} from '../../utils'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('delete_lists', () => { + describe('@ess @serverless delete_lists', () => { describe('deleting lists', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -60,7 +61,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should delete a single list with a list id containing non-alphanumeric characters', async () => { @@ -82,7 +85,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should delete a single list using an auto generated id', async () => { @@ -100,7 +105,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should return an error if the id does not exist when trying to delete it', async () => { @@ -252,7 +259,9 @@ export default ({ getService }: FtrProviderContext) => { .set('kbn-xsrf', 'true'); const bodyToCompare = removeListServerGeneratedProperties(deleteListBody.body); - expect(bodyToCompare).to.eql(getListResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); await supertest .get(`${LIST_ITEM_URL}/_find?list_id=${LIST_ID}`) diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/find_lists.ts similarity index 86% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/find_lists.ts index ffb13b66c8db9..3c47ad92eb824 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/find_lists.ts @@ -10,20 +10,22 @@ import expect from '@kbn/expect'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createListsIndex, deleteListsIndex, removeListServerGeneratedProperties, -} from '../../utils'; +} from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('find_lists', () => { + describe('@ess @serverless find_lists', () => { describe('find lists', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -68,7 +70,7 @@ export default ({ getService }: FtrProviderContext): void => { // cursor is a constant changing value so we have to delete it as well. delete body.cursor; expect(body).to.eql({ - data: [getListResponseMockWithoutAutoGeneratedValues()], + data: [getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME)], page: 1, per_page: 20, total: 1, diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/find_lists_by_size.ts similarity index 79% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/find_lists_by_size.ts index 0d348f5c63424..813293ed1e7cc 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/find_lists_by_size.ts @@ -6,24 +6,29 @@ */ import expect from '@kbn/expect'; - +import { + ELASTIC_HTTP_VERSION_HEADER, + X_ELASTIC_INTERNAL_ORIGIN_REQUEST, +} from '@kbn/core-http-common'; import { LIST_URL, INTERNAL_FIND_LISTS_BY_SIZE } from '@kbn/securitysolution-list-constants'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createListsIndex, deleteListsIndex, removeListServerGeneratedProperties, -} from '../../utils'; +} from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('find_lists_by_size', () => { + describe('@ess @serverless find_lists_by_size', () => { describe('find lists by size', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -37,7 +42,8 @@ export default ({ getService }: FtrProviderContext): void => { const { body } = await supertest .get(`${INTERNAL_FIND_LISTS_BY_SIZE}`) .set('kbn-xsrf', 'true') - .set('elastic-api-version', '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') .send() .expect(200); @@ -66,7 +72,8 @@ export default ({ getService }: FtrProviderContext): void => { const { body } = await supertest .get(`${INTERNAL_FIND_LISTS_BY_SIZE}`) .set('kbn-xsrf', 'true') - .set('elastic-api-version', '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') .send() .expect(200); @@ -75,10 +82,10 @@ export default ({ getService }: FtrProviderContext): void => { // cursor is a constant changing value so we have to delete it as well. delete body.cursor; expect(body).to.eql({ - smallLists: [getListResponseMockWithoutAutoGeneratedValues()], + smallLists: [getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME)], largeLists: [ { - ...getListResponseMockWithoutAutoGeneratedValues(), + ...getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), type: 'text', }, ], diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/index.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/index.ts new file mode 100644 index 0000000000000..c661171dfedbe --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Lists API', function () { + loadTestFile(require.resolve('./create_lists')); + loadTestFile(require.resolve('./create_lists_index')); + loadTestFile(require.resolve('./create_lists_index_migrations')); + loadTestFile(require.resolve('./patch_lists')); + loadTestFile(require.resolve('./patch_lists_migrations')); + loadTestFile(require.resolve('./read_lists')); + loadTestFile(require.resolve('./update_lists')); + loadTestFile(require.resolve('./update_lists_migrations')); + loadTestFile(require.resolve('./delete_lists')); + loadTestFile(require.resolve('./find_lists')); + loadTestFile(require.resolve('./find_lists_by_size')); + }); +} diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/patch_lists.ts similarity index 73% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/patch_lists.ts index 87076851bd34c..2586dcb23ab4f 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/patch_lists.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import type { PatchListSchema, ListSchema } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; +import { LIST_URL } from '@kbn/securitysolution-list-constants'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; @@ -17,19 +17,17 @@ import { createListsIndex, deleteListsIndex, removeListServerGeneratedProperties, - createListsIndices, - createListBypassingChecks, -} from '../../utils'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); const retry = getService('retry'); - const es = getService('es'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('patch_lists', () => { + describe('@ess @serverless patch_lists', () => { describe('patch lists', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -61,7 +59,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputList: Partial = { - ...getListResponseMockWithoutAutoGeneratedValues(), + ...getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', version: 2, }; @@ -102,7 +100,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputList: Partial = { - ...getListResponseMockWithoutAutoGeneratedValues(), + ...getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', version: 2, }; @@ -142,7 +140,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputList: Partial = { - ...getListResponseMockWithoutAutoGeneratedValues(), + ...getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', version: 2, }; @@ -212,60 +210,6 @@ export default ({ getService }: FtrProviderContext) => { message: 'list id: "5096dec6-b6b9-4d8d-8f93-6c2602079d9d" not found', }); }); - - describe('legacy list index (list created before migration to data stream)', () => { - beforeEach(async () => { - await deleteListsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteListsIndex(supertest, log); - }); - it('should update list container that was created in legacy index and migrated through LIST_INDEX request', async () => { - const listId = 'random-list'; - // create legacy indices - await createListsIndices(es); - // create a simple list - await createListBypassingChecks({ es, id: listId }); - - // migrates old indices to data streams - await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); - - // patch a simple list's name - const patchPayload: PatchListSchema = { - id: listId, - name: 'some other name', - }; - const { body } = await supertest - .patch(LIST_URL) - .set('kbn-xsrf', 'true') - .send(patchPayload) - .expect(200); - - expect(body.name).to.be('some other name'); - }); - - it('should update list container that was created in legacy index and not yet migrated', async () => { - const listId = 'random-list'; - // create legacy indices - await createListsIndices(es); - // create a simple list - await createListBypassingChecks({ es, id: listId }); - - // patch a simple list's name - const patchPayload: PatchListSchema = { - id: listId, - name: 'some other name', - }; - const { body } = await supertest - .patch(LIST_URL) - .set('kbn-xsrf', 'true') - .send(patchPayload) - .expect(200); - - expect(body.name).to.be('some other name'); - }); - }); }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/patch_lists_migrations.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/patch_lists_migrations.ts new file mode 100644 index 0000000000000..d131dc4ba05bd --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/patch_lists_migrations.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import type { PatchListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { LIST_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; + +import { + createListsIndex, + deleteListsIndex, + createListsIndices, + createListBypassingChecks, +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const log = getService('log'); + const es = getService('es'); + + describe('@ess patch_lists_migrations', () => { + describe('patch lists', () => { + beforeEach(async () => { + await createListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + describe('legacy list index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should update list container that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + // patch a simple list's name + const patchPayload: PatchListSchema = { + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.name).to.be('some other name'); + }); + + it('should update list container that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // patch a simple list's name + const patchPayload: PatchListSchema = { + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.name).to.be('some other name'); + }); + }); + }); + }); +}; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/read_list_privileges.ts similarity index 91% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/read_list_privileges.ts index 9af6143b2152f..85e4309d048a2 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/read_list_privileges.ts @@ -9,16 +9,15 @@ import expect from '@kbn/expect'; import { LIST_PRIVILEGES_URL } from '@kbn/securitysolution-list-constants'; import { getReadPrivilegeMock } from '@kbn/lists-plugin/server/routes/list_privileges/read_list_privileges_route.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { FtrProviderContextWithSpaces } from '../../../../../ftr_provider_context_with_spaces'; -// eslint-disable-next-line import/no-default-export -export default ({ getService }: FtrProviderContext) => { +export default ({ getService }: FtrProviderContextWithSpaces) => { const supertest = getService('supertest'); const security = getService('security'); const spacesService = getService('spaces'); const supertestWithoutAuth = getService('supertestWithoutAuth'); - describe('read_list_privileges', () => { + describe('@ess @serverless read_list_privileges', () => { const space1Id = 'space_1'; const user1 = { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/read_lists.ts similarity index 83% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/read_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/read_lists.ts index 162b57501c479..025725fe01575 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/read_lists.ts @@ -13,20 +13,21 @@ import { getCreateMinimalListSchemaMockWithoutId, } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createListsIndex, deleteListsIndex, removeListServerGeneratedProperties, -} from '../../utils'; +} from '../../../utils'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('read_lists', () => { + describe('@ess @serverless read_lists', () => { describe('reading lists', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -50,7 +51,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should be able to read a single list with an auto-generated list id', async () => { @@ -67,7 +70,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const bodyToCompare = removeListServerGeneratedProperties(body); - expect(bodyToCompare).to.eql(getListResponseMockWithoutAutoGeneratedValues()); + expect(bodyToCompare).to.eql( + getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME) + ); }); it('should return 404 if given a fake id', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/update_lists.ts similarity index 79% rename from x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/update_lists.ts index d9fc0bbe38bd3..28084f54d2abe 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/update_lists.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import type { UpdateListSchema, ListSchema } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; +import { LIST_URL } from '@kbn/securitysolution-list-constants'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; @@ -17,19 +17,18 @@ import { createListsIndex, deleteListsIndex, removeListServerGeneratedProperties, - createListsIndices, - createListBypassingChecks, -} from '../../utils'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +} from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); const retry = getService('retry'); - const es = getService('es'); + const config = getService('config'); + const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); - describe('update_lists', () => { + describe('@ess @serverless update_lists', () => { describe('update lists', () => { beforeEach(async () => { await createListsIndex(supertest, log); @@ -60,7 +59,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputList: Partial = { - ...getListResponseMockWithoutAutoGeneratedValues(), + ...getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', version: 2, }; @@ -90,7 +89,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); const outputList: Partial = { - ...getListResponseMockWithoutAutoGeneratedValues(), + ...getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', version: 2, }; @@ -189,7 +188,7 @@ export default ({ getService }: FtrProviderContext) => { const { body } = await supertest.put(LIST_URL).set('kbn-xsrf', 'true').send(updatedList); const outputList: Partial = { - ...getListResponseMockWithoutAutoGeneratedValues(), + ...getListResponseMockWithoutAutoGeneratedValues(ELASTICSEARCH_USERNAME), name: 'some other name', description: 'some other description', version: 2, @@ -280,61 +279,6 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); }); }); - - describe('legacy list index (list created before migration to data stream)', () => { - beforeEach(async () => { - await deleteListsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteListsIndex(supertest, log); - }); - it('should update list container that was created in legacy index and migrated through LIST_INDEX request', async () => { - const listId = 'random-list'; - // create legacy indices - await createListsIndices(es); - // create a simple list - await createListBypassingChecks({ es, id: listId }); - - // migrates old indices to data streams - await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); - - // update a simple list's name - const updatedList: UpdateListSchema = { - ...getUpdateMinimalListSchemaMock(), - id: listId, - name: 'some other name', - }; - const { body } = await supertest - .put(LIST_URL) - .set('kbn-xsrf', 'true') - .send(updatedList) - .expect(200); - - expect(body.name).to.be('some other name'); - }); - - it('should update list container that was created in legacy index and not yet migrated', async () => { - const listId = 'random-list'; - // create legacy indices - await createListsIndices(es); - // create a simple list - await createListBypassingChecks({ es, id: listId }); - - // update a simple list's name - const updatedList: UpdateListSchema = { - ...getUpdateMinimalListSchemaMock(), - id: listId, - name: 'some other name', - }; - const { body } = await supertest - .put(LIST_URL) - .set('kbn-xsrf', 'true') - .send(updatedList) - .expect(200); - expect(body.name).to.be('some other name'); - }); - }); }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/update_lists_migrations.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/update_lists_migrations.ts new file mode 100644 index 0000000000000..3acffe061f2cd --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/lists/update_lists_migrations.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import type { UpdateListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { LIST_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; + +import { getUpdateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_list_schema.mock'; +import { + createListsIndex, + deleteListsIndex, + createListsIndices, + createListBypassingChecks, +} from '../../../utils'; + +import { FtrProviderContext } from '../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const log = getService('log'); + const es = getService('es'); + + describe('@ess update_lists_migrations', () => { + describe('update lists', () => { + beforeEach(async () => { + await createListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + describe('legacy list index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should update list container that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + // update a simple list's name + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList) + .expect(200); + + expect(body.name).to.be('some other name'); + }); + + it('should update list container that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // update a simple list's name + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList) + .expect(200); + expect(body.name).to.be('some other name'); + }); + }); + }); + }); +}; diff --git a/x-pack/test/lists_api_integration/utils.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/utils.ts similarity index 99% rename from x-pack/test/lists_api_integration/utils.ts rename to x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/utils.ts index 780042a293dcc..be05bb5a47518 100644 --- a/x-pack/test/lists_api_integration/utils.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/utils.ts @@ -33,7 +33,7 @@ import { ToolingLog } from '@kbn/tooling-log'; import { getImportListItemAsBuffer } from '@kbn/lists-plugin/common/schemas/request/import_list_item_schema.mock'; import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; -import { countDownTest } from '../detection_engine_api_integration/utils'; +import { countDownTest } from '../detections_response/utils'; /** * Creates the lists and lists items index for use inside of beforeEach blocks of tests diff --git a/x-pack/test/security_solution_api_integration/tsconfig.json b/x-pack/test/security_solution_api_integration/tsconfig.json index 782bc4fd46b32..6b1ca69f6aed1 100644 --- a/x-pack/test/security_solution_api_integration/tsconfig.json +++ b/x-pack/test/security_solution_api_integration/tsconfig.json @@ -36,5 +36,6 @@ "@kbn/securitysolution-ecs", "@kbn/fleet-plugin", "@kbn/repo-info", + "@kbn/securitysolution-es-utils", ] } diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 025dff494c5fe..0c763cbf20c5e 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -67,7 +67,6 @@ "@kbn/ftr-common-functional-services", "@kbn/securitysolution-io-ts-list-types", "@kbn/securitysolution-list-constants", - "@kbn/securitysolution-es-utils", "@kbn/expect", "@kbn/dev-cli-errors", "@kbn/ci-stats-reporter", From 1aea9c366188d204d6f154836d984cf254d992c0 Mon Sep 17 00:00:00 2001 From: Sander Philipse <94373878+sphilipse@users.noreply.github.com> Date: Tue, 28 Nov 2023 16:50:02 +0100 Subject: [PATCH 12/30] [Search] Add Search Hub as overview page (#172011) ## Summary This amends the Getting Started page in Search and turns it into the Search Hub. Screenshot 2023-11-27 at 19 14 05 Screenshot 2023-11-27 at 19 14 13 --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- packages/kbn-search-api-panels/index.tsx | 8 +- .../product_selector/ingestion_selector.tsx | 294 +++++++++---- .../product_selector/product_selector.scss | 4 - .../product_selector/product_selector.tsx | 6 +- .../product_selector/welcome_banner.tsx | 79 ++++ .../applications/shared/layout/nav.test.tsx | 321 +++++--------- .../public/applications/shared/layout/nav.tsx | 21 +- .../public/assets/images/connector.svg | 11 + .../public/assets/images/crawler.svg | 4 + .../public/assets/images/file_upload_logo.svg | 403 +++++++++++++++++ .../public/assets/images/sample_data_logo.svg | 407 ++++++++++++++++++ .../assets/images/search_language_clients.svg | 299 +++++++++++++ .../translations/translations/fr-FR.json | 1 - .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - 15 files changed, 1546 insertions(+), 314 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/welcome_banner.tsx create mode 100644 x-pack/plugins/enterprise_search/public/assets/images/connector.svg create mode 100644 x-pack/plugins/enterprise_search/public/assets/images/crawler.svg create mode 100644 x-pack/plugins/enterprise_search/public/assets/images/file_upload_logo.svg create mode 100644 x-pack/plugins/enterprise_search/public/assets/images/sample_data_logo.svg create mode 100644 x-pack/plugins/enterprise_search/public/assets/images/search_language_clients.svg diff --git a/packages/kbn-search-api-panels/index.tsx b/packages/kbn-search-api-panels/index.tsx index 7372e381c576b..513cc0fda807c 100644 --- a/packages/kbn-search-api-panels/index.tsx +++ b/packages/kbn-search-api-panels/index.tsx @@ -54,18 +54,18 @@ export const WelcomeBanner: React.FC = ({ {Boolean(user) && ( - +

{user ? i18n.translate('searchApiPanels.welcomeBanner.header.greeting.customTitle', { - defaultMessage: 'Hi {name}!', + defaultMessage: '👋 Hi {name}!', values: { name: user.full_name || user.username }, }) : i18n.translate('searchApiPanels.welcomeBanner.header.greeting.defaultTitle', { - defaultMessage: 'Hi!', + defaultMessage: '👋 Hi', })}

- + )} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/ingestion_selector.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/ingestion_selector.tsx index abb569361e7c3..26d826d7b8fb0 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/ingestion_selector.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/ingestion_selector.tsx @@ -11,18 +11,33 @@ import { generatePath } from 'react-router-dom'; import { useValues } from 'kea'; -import { EuiButton, EuiCard, EuiFlexGroup, EuiFlexItem, EuiIcon } from '@elastic/eui'; +import { + EuiButton, + EuiCard, + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiSpacer, + EuiText, + IconType, +} from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { ENTERPRISE_SEARCH_CONTENT_PLUGIN, + ENTERPRISE_SEARCH_ELASTICSEARCH_URL, INGESTION_METHOD_IDS, } from '../../../../../common/constants'; import apiLogo from '../../../../assets/images/api_cloud.svg'; +import connectorIcon from '../../../../assets/images/connector.svg'; +import crawlerIcon from '../../../../assets/images/crawler.svg'; +import fileUploadLogo from '../../../../assets/images/file_upload_logo.svg'; +import sampleDataLogo from '../../../../assets/images/sample_data_logo.svg'; import connectorLogo from '../../../../assets/images/search_connector.svg'; import crawlerLogo from '../../../../assets/images/search_crawler.svg'; +import languageClientsLogo from '../../../../assets/images/search_language_clients.svg'; import { NEW_API_PATH, @@ -31,104 +46,229 @@ import { } from '../../../enterprise_search_content/routes'; import { HttpLogic } from '../../../shared/http/http_logic'; import { KibanaLogic } from '../../../shared/kibana'; -import { EuiButtonTo, EuiLinkTo } from '../../../shared/react_router_helpers'; - -const START_LABEL = i18n.translate('xpack.enterpriseSearch.ingestSelector.startButton', { - defaultMessage: 'Start', -}); +import { EuiLinkTo } from '../../../shared/react_router_helpers'; export const IngestionSelector: React.FC = () => { - const { config, productFeatures } = useValues(KibanaLogic); + const { + application: { navigateToApp }, + config, + productFeatures, + } = useValues(KibanaLogic); const { errorConnectingMessage } = useValues(HttpLogic); const crawlerDisabled = Boolean(errorConnectingMessage || !config.host); return ( - - - } - textAlign="left" - title={i18n.translate('xpack.enterpriseSearch.ingestSelector.method.api', { - defaultMessage: 'API', - })} - description={i18n.translate( - 'xpack.enterpriseSearch.ingestSelector.method.api.description', - { - defaultMessage: - 'Add documents programmatically by connecting with the API using your preferred language client.', - } - )} - footer={ - - {START_LABEL} - - } - /> - - {productFeatures.hasConnectors && ( + <> + - } - textAlign="left" - title={i18n.translate('xpack.enterpriseSearch.ingestSelector.method.connectors', { - defaultMessage: 'Connectors', + - {START_LABEL} - - } /> - )} - {productFeatures.hasWebCrawler && ( + {productFeatures.hasConnectors && ( + + + + )} + {productFeatures.hasWebCrawler && ( + + + + )} + + + - } - textAlign="left" - title={i18n.translate('xpack.enterpriseSearch.ingestSelector.method.crawler', { - defaultMessage: 'Web Crawler', + - {START_LABEL} - - } + buttonIcon="visVega" + buttonLabel={i18n.translate( + 'xpack.enterpriseSearch.ingestSelector.method.browseClientsLabel', + { + defaultMessage: 'Browse clients', + } + )} + href={generatePath(ENTERPRISE_SEARCH_ELASTICSEARCH_URL)} /> - )} - + + navigateToApp('home', { path: '#/tutorial_directory/fileDataViz' })} + /> + + + navigateToApp('home', { path: '#/tutorial_directory/sampleData' })} + /> + + + + ); +}; + +interface IngestionCardProps { + buttonIcon: IconType; + buttonLabel: string; + description: string; + href?: string; + isDisabled?: boolean; + logo: IconType; + onClick?: () => void; + title: string; +} + +const IngestionCard: React.FC = ({ + buttonIcon, + buttonLabel, + description, + href, + isDisabled, + logo, + onClick, + title, +}) => { + return ( + + + + + + {title} + + + + } + description={ + + {description} + + } + footer={ + onClick ? ( + + {buttonLabel} + + ) : ( + + + {buttonLabel} + + + ) + } + /> ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.scss b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.scss index 8d02868008375..b21b889138b16 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.scss +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.scss @@ -1,7 +1,3 @@ -.entSearchProductSelectorHeader { - background-color: $euiColorPrimary; -} - .entSearchProductSelectorHeader > div { padding-bottom: 0; padding-top: 0; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.tsx index 13b84c9c6e40b..cd869c5444b37 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/product_selector.tsx @@ -18,7 +18,6 @@ import { EuiTitle, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { WelcomeBanner } from '@kbn/search-api-panels'; import { AuthenticatedUser } from '@kbn/security-plugin/common'; @@ -39,6 +38,7 @@ import { EnterpriseSearchProductCard } from './enterprise_search_product_card'; import { IngestionSelector } from './ingestion_selector'; import './product_selector.scss'; +import { WelcomeBanner } from './welcome_banner'; interface ProductSelectorProps { access: { @@ -81,9 +81,7 @@ export const ProductSelector: React.FC = ({ - - - + diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/welcome_banner.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/welcome_banner.tsx new file mode 100644 index 0000000000000..ea14d118d4c6e --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/components/product_selector/welcome_banner.tsx @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { EuiSpacer, EuiFlexGroup, EuiFlexItem, EuiTitle, EuiText, EuiImage } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { AuthenticatedUser } from '@kbn/security-plugin/public'; + +export interface WelcomeBannerProps { + assetBasePath?: string; + image?: string; + user?: AuthenticatedUser; +} + +export const WelcomeBanner: React.FC = ({ user, assetBasePath, image }) => ( + <> + + + + {/* Reversing column direction here so screenreaders keep h1 as the first element */} + + + +

+ +

+ + + + {i18n.translate('xpack.enterpriseSearch.welcomeBanner.header.titleDescription', { + defaultMessage: + "There's endless ways to ingest and explore data with Elasticsearch, but here's a few of the most popular", + })} + + + {Boolean(user) && ( + + +

+ {user + ? i18n.translate( + 'xpack.enterpriseSearch.welcomeBanner.header.greeting.customTitle', + { + defaultMessage: '👋 Hi {name}!', + values: { name: user.full_name || user.username }, + } + ) + : i18n.translate( + 'xpack.enterpriseSearch.welcomeBanner.header.greeting.defaultTitle', + { + defaultMessage: '👋 Hi', + } + )} +

+ + + )} + + + + + + + + +); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx index f68a2eee1c5af..6d064a1863b1a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx @@ -26,6 +26,92 @@ const DEFAULT_PRODUCT_ACCESS: ProductAccess = { hasAppSearchAccess: true, hasWorkplaceSearchAccess: true, }; +const baseNavItems = [ + { + href: '/app/enterprise_search/overview', + id: 'overview', + items: undefined, + name: 'Overview', + }, + { + id: 'content', + items: [ + { + href: '/app/enterprise_search/content/search_indices', + id: 'search_indices', + items: undefined, + name: 'Indices', + }, + { + href: '/app/enterprise_search/content/settings', + id: 'settings', + items: undefined, + name: 'Settings', + }, + ], + name: 'Content', + }, + { + id: 'applications', + items: [ + { + href: '/app/enterprise_search/applications/search_applications', + id: 'searchApplications', + items: undefined, + name: 'Search Applications', + }, + { + href: '/app/enterprise_search/analytics', + id: 'analyticsCollections', + items: undefined, + name: 'Behavioral Analytics', + }, + ], + name: 'Applications', + }, + { + id: 'es_getting_started', + items: [ + { + href: '/app/enterprise_search/elasticsearch', + id: 'elasticsearch', + items: undefined, + name: 'Elasticsearch', + }, + { + href: '/app/enterprise_search/vector_search', + id: 'vectorSearch', + items: undefined, + name: 'Vector Search', + }, + { + href: '/app/enterprise_search/ai_search', + id: 'aiSearch', + items: undefined, + name: 'AI Search', + }, + ], + name: 'Getting started', + }, + { + id: 'enterpriseSearch', + items: [ + { + href: '/app/enterprise_search/app_search', + id: 'app_search', + items: undefined, + name: 'App Search', + }, + { + href: '/app/enterprise_search/workplace_search', + id: 'workplace_search', + items: undefined, + name: 'Workplace Search', + }, + ], + name: 'Enterprise Search', + }, +]; describe('useEnterpriseSearchContentNav', () => { beforeEach(() => { @@ -41,79 +127,7 @@ describe('useEnterpriseSearchContentNav', () => { productFeatures: DEFAULT_PRODUCT_FEATURES, }); - expect(useEnterpriseSearchNav()).toEqual([ - { - id: 'content', - items: [ - { - href: '/app/enterprise_search/content/search_indices', - id: 'search_indices', - name: 'Indices', - }, - { - href: '/app/enterprise_search/content/settings', - id: 'settings', - items: undefined, - name: 'Settings', - }, - ], - name: 'Content', - }, - { - id: 'applications', - items: [ - { - href: '/app/enterprise_search/applications/search_applications', - id: 'searchApplications', - name: 'Search Applications', - }, - { - href: '/app/enterprise_search/analytics', - id: 'analyticsCollections', - name: 'Behavioral Analytics', - }, - ], - name: 'Applications', - }, - { - href: '/app/enterprise_search/overview', - id: 'es_getting_started', - items: [ - { - href: '/app/enterprise_search/elasticsearch', - id: 'elasticsearch', - name: 'Elasticsearch', - }, - { - href: '/app/enterprise_search/vector_search', - id: 'vectorSearch', - name: 'Vector Search', - }, - { - href: '/app/enterprise_search/ai_search', - id: 'aiSearch', - name: 'AI Search', - }, - ], - name: 'Getting started', - }, - { - id: 'enterpriseSearch', - items: [ - { - href: '/app/enterprise_search/app_search', - id: 'app_search', - name: 'App Search', - }, - { - href: '/app/enterprise_search/workplace_search', - id: 'workplace_search', - name: 'Workplace Search', - }, - ], - name: 'Enterprise Search', - }, - ]); + expect(useEnterpriseSearchNav()).toEqual(baseNavItems); }); it('excludes legacy products when the user has no access to them', () => { @@ -205,84 +219,14 @@ describe('useEnterpriseSearchApplicationNav', () => { }); it('returns an array of top-level Enterprise Search nav items', () => { - expect(useEnterpriseSearchApplicationNav()).toEqual([ - { - id: 'content', - items: [ - { - href: '/app/enterprise_search/content/search_indices', - id: 'search_indices', - name: 'Indices', - }, - { - href: '/app/enterprise_search/content/settings', - id: 'settings', - name: 'Settings', - }, - ], - name: 'Content', - }, - { - id: 'applications', - items: [ - { - href: '/app/enterprise_search/applications/search_applications', - id: 'searchApplications', - name: 'Search Applications', - }, - { - href: '/app/enterprise_search/analytics', - id: 'analyticsCollections', - name: 'Behavioral Analytics', - }, - ], - name: 'Applications', - }, - { - href: '/app/enterprise_search/overview', - id: 'es_getting_started', - items: [ - { - href: '/app/enterprise_search/elasticsearch', - id: 'elasticsearch', - name: 'Elasticsearch', - }, - { - href: '/app/enterprise_search/vector_search', - id: 'vectorSearch', - name: 'Vector Search', - }, - { - href: '/app/enterprise_search/ai_search', - id: 'aiSearch', - name: 'AI Search', - }, - ], - name: 'Getting started', - }, - { - id: 'enterpriseSearch', - items: [ - { - href: '/app/enterprise_search/app_search', - id: 'app_search', - name: 'App Search', - }, - { - href: '/app/enterprise_search/workplace_search', - id: 'workplace_search', - name: 'Workplace Search', - }, - ], - name: 'Enterprise Search', - }, - ]); + expect(useEnterpriseSearchApplicationNav()).toEqual(baseNavItems); }); it('returns selected engine sub nav items', () => { const engineName = 'my-test-engine'; const navItems = useEnterpriseSearchApplicationNav(engineName); expect(navItems?.map((ni) => ni.name)).toEqual([ + 'Overview', 'Content', 'Applications', 'Getting started', @@ -339,6 +283,7 @@ describe('useEnterpriseSearchApplicationNav', () => { const engineName = 'my-test-engine'; const navItems = useEnterpriseSearchApplicationNav(engineName, true); expect(navItems?.map((ni) => ni.name)).toEqual([ + 'Overview', 'Content', 'Applications', 'Getting started', @@ -397,74 +342,6 @@ describe('useEnterpriseSearchApplicationNav', () => { }); describe('useEnterpriseSearchAnalyticsNav', () => { - const baseNavs = [ - { - id: 'content', - items: [ - { - href: '/app/enterprise_search/content/search_indices', - id: 'search_indices', - name: 'Indices', - }, - ], - name: 'Content', - }, - { - id: 'applications', - items: [ - { - href: '/app/enterprise_search/applications/search_applications', - id: 'searchApplications', - name: 'Search Applications', - }, - { - href: '/app/enterprise_search/analytics', - id: 'analyticsCollections', - name: 'Behavioral Analytics', - }, - ], - name: 'Applications', - }, - { - href: '/app/enterprise_search/overview', - id: 'es_getting_started', - items: [ - { - href: '/app/enterprise_search/elasticsearch', - id: 'elasticsearch', - name: 'Elasticsearch', - }, - { - href: '/app/enterprise_search/vector_search', - id: 'vectorSearch', - name: 'Vector Search', - }, - { - href: '/app/enterprise_search/ai_search', - id: 'aiSearch', - name: 'AI Search', - }, - ], - name: 'Getting started', - }, - { - id: 'enterpriseSearch', - items: [ - { - href: '/app/enterprise_search/app_search', - id: 'app_search', - name: 'App Search', - }, - { - href: '/app/enterprise_search/workplace_search', - id: 'workplace_search', - name: 'Workplace Search', - }, - ], - name: 'Enterprise Search', - }, - ]; - beforeEach(() => { jest.clearAllMocks(); setMockValues({ @@ -474,12 +351,26 @@ describe('useEnterpriseSearchAnalyticsNav', () => { it('returns basic nav all params are empty', () => { const navItems = useEnterpriseSearchAnalyticsNav(); - expect(navItems).toEqual(baseNavs); + // filter out settings item because we're setting hasDefaultIngestPipeline to false + expect(navItems).toEqual( + baseNavItems.map((item) => + item.id === 'content' + ? { ...item, items: item.items?.filter((contentItem) => contentItem.id !== 'settings') } + : item + ) + ); }); it('returns basic nav if only name provided', () => { + // filter out settings item because we're setting hasDefaultIngestPipeline to false const navItems = useEnterpriseSearchAnalyticsNav('my-test-collection'); - expect(navItems).toEqual(baseNavs); + expect(navItems).toEqual( + baseNavItems.map((item) => + item.id === 'content' + ? { ...item, items: item.items?.filter((contentItem) => contentItem.id !== 'settings') } + : item + ) + ); }); it('returns nav with sub items when name and paths provided', () => { diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx index 8fe9c36766c60..4dff6c90635e4 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx @@ -34,6 +34,17 @@ export const useEnterpriseSearchNav = () => { if (!isSidebarEnabled) return undefined; const navItems: Array> = [ + { + id: 'overview', + name: i18n.translate('xpack.enterpriseSearch.nav.overviewTitle', { + defaultMessage: 'Overview', + }), + ...generateNavLink({ + shouldNotCreateHref: true, + shouldShowActiveForSubroutes: true, + to: ENTERPRISE_SEARCH_OVERVIEW_PLUGIN.URL, + }), + }, { id: 'content', items: [ @@ -98,13 +109,6 @@ export const useEnterpriseSearchNav = () => { }, { id: 'es_getting_started', - name: i18n.translate('xpack.enterpriseSearch.nav.enterpriseSearchOverviewTitle', { - defaultMessage: 'Getting started', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: ENTERPRISE_SEARCH_OVERVIEW_PLUGIN.URL, - }), items: [ { id: 'elasticsearch', @@ -135,6 +139,9 @@ export const useEnterpriseSearchNav = () => { }), }, ], + name: i18n.translate('xpack.enterpriseSearch.nav.enterpriseSearchOverviewTitle', { + defaultMessage: 'Getting started', + }), }, ...(productAccess.hasAppSearchAccess || productAccess.hasWorkplaceSearchAccess ? [ diff --git a/x-pack/plugins/enterprise_search/public/assets/images/connector.svg b/x-pack/plugins/enterprise_search/public/assets/images/connector.svg new file mode 100644 index 0000000000000..3b37b963f435f --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/assets/images/connector.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/x-pack/plugins/enterprise_search/public/assets/images/crawler.svg b/x-pack/plugins/enterprise_search/public/assets/images/crawler.svg new file mode 100644 index 0000000000000..94aafafddf68b --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/assets/images/crawler.svg @@ -0,0 +1,4 @@ + + + + diff --git a/x-pack/plugins/enterprise_search/public/assets/images/file_upload_logo.svg b/x-pack/plugins/enterprise_search/public/assets/images/file_upload_logo.svg new file mode 100644 index 0000000000000..90bbc987154ac --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/assets/images/file_upload_logo.svg @@ -0,0 +1,403 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/x-pack/plugins/enterprise_search/public/assets/images/sample_data_logo.svg b/x-pack/plugins/enterprise_search/public/assets/images/sample_data_logo.svg new file mode 100644 index 0000000000000..07e0d8fe5818a --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/assets/images/sample_data_logo.svg @@ -0,0 +1,407 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/x-pack/plugins/enterprise_search/public/assets/images/search_language_clients.svg b/x-pack/plugins/enterprise_search/public/assets/images/search_language_clients.svg new file mode 100644 index 0000000000000..c4ff851bc70ea --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/assets/images/search_language_clients.svg @@ -0,0 +1,299 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 8f1a076968b64..e72ad4482d77c 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -15295,7 +15295,6 @@ "xpack.enterpriseSearch.ingestSelector.method.connectors.description": "Extraire, transformer, indexer et synchroniser des données issues d'une source de données tiers.", "xpack.enterpriseSearch.ingestSelector.method.crawler": "Robot d'indexation", "xpack.enterpriseSearch.ingestSelector.method.crawler.description": "Découvrir, extraire et indexer du contenu interrogeable provenant de sites web et de bases de connaissances.", - "xpack.enterpriseSearch.ingestSelector.startButton": "Début", "xpack.enterpriseSearch.inlineEditableTable.newRowButtonLabel": "Nouvelle ligne", "xpack.enterpriseSearch.integrations.apiDescription": "Ajouter la recherche à votre application avec les API robustes d'Elasticsearch.", "xpack.enterpriseSearch.integrations.apiName": "API", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 32199558eb576..4fb5f53604319 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -15308,7 +15308,6 @@ "xpack.enterpriseSearch.ingestSelector.method.connectors.description": "サードパーティのデータソースからデータを抽出、変換、インデックス化、同期します。", "xpack.enterpriseSearch.ingestSelector.method.crawler": "Webクローラー", "xpack.enterpriseSearch.ingestSelector.method.crawler.description": "Webサイトやナレッジベースから検索可能なコンテンツを検出、抽出、インデックス化します。", - "xpack.enterpriseSearch.ingestSelector.startButton": "開始", "xpack.enterpriseSearch.inlineEditableTable.newRowButtonLabel": "新しい行", "xpack.enterpriseSearch.integrations.apiDescription": "Elasticsearchの堅牢なAPIを使用して、検索をアプリケーションに追加します。", "xpack.enterpriseSearch.integrations.apiName": "API", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 7881711f59fc5..bf8ca19d55dc6 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -15308,7 +15308,6 @@ "xpack.enterpriseSearch.ingestSelector.method.connectors.description": "提取、转换、索引和同步来自第三方数据源的数据。", "xpack.enterpriseSearch.ingestSelector.method.crawler": "网络爬虫", "xpack.enterpriseSearch.ingestSelector.method.crawler.description": "发现、提取和索引网站和知识库中的可搜索内容。", - "xpack.enterpriseSearch.ingestSelector.startButton": "启动", "xpack.enterpriseSearch.inlineEditableTable.newRowButtonLabel": "新行", "xpack.enterpriseSearch.integrations.apiDescription": "通过 Elasticsearch 稳健的 API 将搜索功能添加到您的应用程序。", "xpack.enterpriseSearch.integrations.apiName": "API", From 87e192ff23bf10efa0f0297bf9239569b8699729 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Tue, 28 Nov 2023 17:11:31 +0100 Subject: [PATCH 13/30] [Security Solution] Indent fix (#172070) Co-authored-by: Georgii Gorbachev --- .../verify_es_serverless_image.yml | 26 ++++++++++ .buildkite/pipelines/on_merge.yml | 48 +++++++++++++++++++ .buildkite/pipelines/pull_request/base.yml | 48 +++++++++++++++++++ .../security_solution_cypress.yml | 24 ++++++++++ .../security_serverless_rule_management.sh | 16 +++++++ ...rverless_rule_management_prebuilt_rules.sh | 16 +++++++ .../security_solution_rule_management.sh | 16 +++++++ ...solution_rule_management_prebuilt_rules.sh | 16 +++++++ .github/CODEOWNERS | 2 - .../cypress/README.md | 23 ++++++--- .../install_update_authorization.cy.ts | 12 ++--- .../install_update_error_handling.cy.ts | 14 +++--- .../prebuilt_rules/install_via_fleet.cy.ts | 14 +++--- .../prebuilt_rules/install_workflow.cy.ts | 20 ++++---- .../prebuilt_rules/management.cy.ts | 21 ++++---- .../prebuilt_rules/notifications.cy.ts | 19 ++++---- .../prebuilt_rules_preview.cy.ts | 24 +++++----- .../prebuilt_rules/update_workflow.ts | 18 +++---- .../rule_details/common_flows.cy.ts | 26 +++++----- .../rule_details/esql_rule.cy.ts | 16 +++---- .../security_solution_cypress/package.json | 14 ++++-- 21 files changed, 332 insertions(+), 101 deletions(-) create mode 100644 .buildkite/scripts/steps/functional/security_serverless_rule_management.sh create mode 100644 .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh create mode 100644 .buildkite/scripts/steps/functional/security_solution_rule_management.sh create mode 100644 .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/install_update_authorization.cy.ts (94%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/install_update_error_handling.cy.ts (94%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/install_via_fleet.cy.ts (90%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/install_workflow.cy.ts (85%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/management.cy.ts (91%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/notifications.cy.ts (92%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/prebuilt_rules_preview.cy.ts (97%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/prebuilt_rules/update_workflow.ts (88%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/rule_details/common_flows.cy.ts (88%) rename x-pack/test/security_solution_cypress/cypress/e2e/detection_response/{ => rule_management}/rule_details/esql_rule.cy.ts (69%) diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml index 8e64513b14900..8d1b778b67983 100644 --- a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml +++ b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml @@ -95,6 +95,32 @@ steps: - exit_status: '*' limit: 1 + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh + label: 'Serverless Rule Management - Security Solution Cypress Tests' + if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'" + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh + label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'" + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh label: 'Defend Workflows Cypress Tests on Serverless' if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'" diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index 8b00db428a713..f92089099cbc5 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -115,6 +115,54 @@ steps: - exit_status: '*' limit: 1 + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh + label: 'Serverless Rule Management - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh + label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh + label: 'Rule Management - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh + label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + - command: .buildkite/scripts/steps/functional/security_solution.sh label: 'Security Solution Cypress Tests' agents: diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index 49215bbd00f11..8238afbee4fd2 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -93,6 +93,30 @@ steps: - exit_status: '*' limit: 1 + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh + label: 'Serverless Rule Management - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh + label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + - command: .buildkite/scripts/steps/functional/security_solution.sh label: 'Security Solution Cypress Tests' agents: @@ -117,6 +141,30 @@ steps: - exit_status: '*' limit: 1 + - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh + label: 'Rule Management - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh + label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 60 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 + - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh label: 'Investigations - Security Solution Cypress Tests' agents: diff --git a/.buildkite/pipelines/security_solution/security_solution_cypress.yml b/.buildkite/pipelines/security_solution/security_solution_cypress.yml index 247505ef1c85a..77e7fea574352 100644 --- a/.buildkite/pipelines/security_solution/security_solution_cypress.yml +++ b/.buildkite/pipelines/security_solution/security_solution_cypress.yml @@ -30,6 +30,30 @@ steps: # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. timeout_in_minutes: 300 parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management + label: 'Serverless MKI QA Rule Management - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. + timeout_in_minutes: 300 + parallelism: 8 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management:prebuilt_rules + label: 'Serverless MKI QA Rule Management - Prebuilt Rules - Security Solution Cypress Tests' + agents: + queue: n2-4-spot + # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate. + timeout_in_minutes: 300 + parallelism: 6 retry: automatic: - exit_status: '*' diff --git a/.buildkite/scripts/steps/functional/security_serverless_rule_management.sh b/.buildkite/scripts/steps/functional/security_serverless_rule_management.sh new file mode 100644 index 0000000000000..5d360e0db4f29 --- /dev/null +++ b/.buildkite/scripts/steps/functional/security_serverless_rule_management.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/steps/functional/common.sh +source .buildkite/scripts/steps/functional/common_cypress.sh + +export JOB=kibana-security-solution-chrome +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} + +echo "--- Rule Management Cypress Tests on Serverless" + +cd x-pack/test/security_solution_cypress + +set +e +yarn cypress:rule_management:run:serverless; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh b/.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh new file mode 100644 index 0000000000000..bc7dc3269d8cb --- /dev/null +++ b/.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/steps/functional/common.sh +source .buildkite/scripts/steps/functional/common_cypress.sh + +export JOB=kibana-security-solution-chrome +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} + +echo "--- Rule Management - Prebuilt Rules - Cypress Tests on Serverless" + +cd x-pack/test/security_solution_cypress + +set +e +yarn cypress:rule_management:prebuilt_rules:run:serverless; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/steps/functional/security_solution_rule_management.sh b/.buildkite/scripts/steps/functional/security_solution_rule_management.sh new file mode 100644 index 0000000000000..847cb42896cf1 --- /dev/null +++ b/.buildkite/scripts/steps/functional/security_solution_rule_management.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/steps/functional/common.sh +source .buildkite/scripts/steps/functional/common_cypress.sh + +export JOB=kibana-security-solution-chrome +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} + +echo "--- Rule Management - Security Solution Cypress Tests" + +cd x-pack/test/security_solution_cypress + +set +e +yarn cypress:rule_management:run:ess; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh b/.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh new file mode 100644 index 0000000000000..d8b19ad3363b5 --- /dev/null +++ b/.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/steps/functional/common.sh +source .buildkite/scripts/steps/functional/common_cypress.sh + +export JOB=kibana-security-solution-chrome +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} + +echo "--- Rule Management - Prebuilt Rules - Security Solution Cypress Tests" + +cd x-pack/test/security_solution_cypress + +set +e +yarn cypress:rule_management:prebuilt_rules:run:ess; status=$?; yarn junit:merge || :; exit $status diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9944de64b186d..a31103b1f35bc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1321,9 +1321,7 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout @elastic/ /x-pack/plugins/security_solution/common/api/detection_engine/rule_monitoring @elastic/security-detection-rule-management /x-pack/plugins/security_solution/common/detection_engine/rule_management @elastic/security-detection-rule-management -/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules @elastic/security-detection-rule-management /x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management @elastic/security-detection-rule-management -/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details @elastic/security-detection-rule-management /x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules @elastic/security-detection-rule-management /x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/rule_management @elastic/security-detection-rule-management /x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules @elastic/security-detection-rule-management diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md index 8940d6c86e73e..88786aed7ff56 100644 --- a/x-pack/test/security_solution_cypress/cypress/README.md +++ b/x-pack/test/security_solution_cypress/cypress/README.md @@ -62,19 +62,25 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio | cypress | Runs the default Cypress command | | cypress:open:ess | Opens the Cypress UI with all tests in the `e2e` directory. This also runs a local kibana and ES instance. The kibana instance will reload when you make code changes. This is the recommended way to debug and develop tests. | | cypress:open:serverless | Opens the Cypress UI with all tests in the `e2e` directory. This also runs a mocked serverless environment. The kibana instance will reload when you make code changes. This is the recommended way to debug and develop tests. | -| cypress:run:ess | Runs all tests tagged as ESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode | +| cypress:run:ess | Runs all tests tagged as ESS placed in the `e2e` directory excluding `investigations`,`explore` and `detection_response/rule_management` directories in headless mode | | cypress:run:cases:ess | Runs all tests under `explore/cases` in the `e2e` directory related to the Cases area team in headless mode | | cypress:ess | Runs all ESS tests with the specified configuration in headless mode and produces a report using `cypress-multi-reporters` | +| cypress:rule_management:run:ess | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | +| cypress:rule_management:prebuilt_rules:run:ess | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | | cypress:run:respops:ess | Runs all tests related to the Response Ops area team, specifically tests in `detection_alerts`, `detection_rules`, and `exceptions` directories in headless mode | -| cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations` and `explore` directories in headless mode | -| cypress:investigations:run:ess | Runs all tests tagged as ESS in the `e2e/investigations` directory in headless mode | +| cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations`, `explore` and `rule_management` directories in headless mode | +| cypress:rule_management:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | +| cypress:rule_management:prebuilt_rules:run:serverless | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | +| cypress:investigations:run:ess | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode | | cypress:explore:run:ess | Runs all tests tagged as ESS in the `e2e/explore` directory in headless mode | | cypress:investigations:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode | | cypress:explore:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode | | cypress:open:qa:serverless | Opens the Cypress UI with all tests in the `e2e` directory tagged as SERVERLESS. This also creates an MKI project in console.qa enviornment. The kibana instance will reload when you make code changes. This is the recommended way to debug tests in QA. Follow the readme in order to learn about the known limitations. | -| cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode using the QA environment and real MKI projects.| +| cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations`, `explore` and `rule_management` directories in headless mode using the QA environment and real MKI projects.| | cypress:run:qa:serverless:explore | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode using the QA environment and real MKI prorjects. | | cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. | +| cypress:run:qa:serverless:rule_management | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` directory, excluding `e2e/detection_response/rule_management/prebuilt_rules` in headless mode using the QA environment and reak MKI projects. | +| cypress:run:qa:serverless:rule_management:prebuilt_rules | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode using the QA environment and reak MKI projects. | | junit:merge | Merges individual test reports into a single report and moves the report to the `junit` directory | Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. @@ -94,7 +100,7 @@ Below you can find the folder structure used on our Cypress tests. Cypress convention starting version 10 (previously known as integration). Contains the specs that are going to be executed. -### e2e/explore and e2e/investigations +### Area teams folders These directories contain tests which are run in their own Buildkite pipeline. @@ -103,7 +109,8 @@ If you belong to one of the teams listed in the table, please add new e2e specs | Directory | Area team | | -- | -- | | `e2e/explore` | Threat Hunting Explore | -| `e2e/investigations | Threat Hunting Investigations | +| `e2e/investigations` | Threat Hunting Investigations | +| `e2e/detection_response/rule_management` | Detection Rule Management | ### fixtures/ @@ -203,6 +210,8 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio | cypress:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e` directory excluding `investigations` and `explore` directories in headless mode | | cypress:investigations:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode | | cypress:explore:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode | +| cypress:rule_management:run:serverless | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` excluding `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | +| cypress:rule_management:prebuilt_rules:run:serverless | Runs all tests tagged as ESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode | Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. @@ -248,6 +257,8 @@ Run the tests with the following yarn scripts from `x-pack/test/security_solutio | cypress:run:qa:serverless | Runs all tests tagged as SERVERLESS placed in the `e2e` directory excluding `investigations` and `explore` directories in headless mode using the QA environment and real MKI projects.| | cypress:run:qa:serverless:explore | Runs all tests tagged as SERVERLESS in the `e2e/explore` directory in headless mode using the QA environment and real MKI prorjects. | | cypress:run:qa:serverless:investigations | Runs all tests tagged as SERVERLESS in the `e2e/investigations` directory in headless mode using the QA environment and reak MKI projects. | +| cypress:run:qa:serverless:rule_management | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management` directory, excluding `e2e/detection_response/rule_management/prebuilt_rules` in headless mode using the QA environment and reak MKI projects. | +| cypress:run:qa:serverless:rule_management:prebuilt_rules | Runs all tests tagged as SERVERLESS in the `e2e/detection_response/rule_management/prebuilt_rules` directory in headless mode using the QA environment and reak MKI projects. | Please note that all the headless mode commands do not open the Cypress UI and are typically used in CI/CD environments. The scripts that open the Cypress UI are useful for development and debugging. diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts similarity index 94% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts index e0078dd54e7ea..29e650dd4de66 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_authorization.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts @@ -12,14 +12,14 @@ import { } from '@kbn/security-solution-plugin/common/constants'; import { ROLES } from '@kbn/security-solution-plugin/common/test'; -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { createAndInstallMockedPrebuiltRules, installPrebuiltRuleAssets, preventPrebuiltRulesPackageInstallation, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { visit } from '../../../tasks/navigation'; -import { RULES_MANAGEMENT_URL } from '../../../urls/rules_management'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { visit } from '../../../../tasks/navigation'; +import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; import { ADD_ELASTIC_RULES_BTN, getInstallSingleRuleButtonByRuleId, @@ -31,8 +31,8 @@ import { RULES_UPDATES_TAB, RULE_CHECKBOX, UPGRADE_ALL_RULES_BUTTON, -} from '../../../screens/alerts_detection_rules'; -import { login } from '../../../tasks/login'; +} from '../../../../screens/alerts_detection_rules'; +import { login } from '../../../../tasks/login'; // Rule to test update const RULE_1_ID = 'rule_1'; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts similarity index 94% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts index 7e288910ccb60..db84d92e4ddb6 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_update_error_handling.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { getInstallSingleRuleButtonByRuleId, getUpgradeSingleRuleButtonByRuleId, @@ -14,14 +14,14 @@ import { SELECT_ALL_RULES_ON_PAGE_CHECKBOX, UPGRADE_ALL_RULES_BUTTON, UPGRADE_SELECTED_RULES_BUTTON, -} from '../../../screens/alerts_detection_rules'; -import { selectRulesByName } from '../../../tasks/alerts_detection_rules'; +} from '../../../../screens/alerts_detection_rules'; +import { selectRulesByName } from '../../../../tasks/alerts_detection_rules'; import { installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, preventPrebuiltRulesPackageInstallation, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { login } from '../../../tasks/login'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { login } from '../../../../tasks/login'; import { clickAddElasticRulesButton, assertInstallationRequestIsComplete, @@ -33,8 +33,8 @@ import { assertRulesPresentInAddPrebuiltRulesTable, assertRuleUpgradeFailureToastShown, assertRulesPresentInRuleUpdatesTable, -} from '../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; +} from '../../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; describe( 'Detection rules, Prebuilt Rules Installation and Update - Error handling', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_via_fleet.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_via_fleet.cy.ts similarity index 90% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_via_fleet.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_via_fleet.cy.ts index 6da3d58c0530d..762e79bb27003 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_via_fleet.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_via_fleet.cy.ts @@ -8,13 +8,13 @@ import type { BulkInstallPackageInfo } from '@kbn/fleet-plugin/common'; import type { Rule } from '@kbn/security-solution-plugin/public/detection_engine/rule_management/logic/types'; -import { resetRulesTableState } from '../../../tasks/common'; -import { INSTALL_ALL_RULES_BUTTON, TOASTER } from '../../../screens/alerts_detection_rules'; -import { getRuleAssets } from '../../../tasks/api_calls/prebuilt_rules'; -import { login } from '../../../tasks/login'; -import { clickAddElasticRulesButton } from '../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; -import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +import { resetRulesTableState } from '../../../../tasks/common'; +import { INSTALL_ALL_RULES_BUTTON, TOASTER } from '../../../../screens/alerts_detection_rules'; +import { getRuleAssets } from '../../../../tasks/api_calls/prebuilt_rules'; +import { login } from '../../../../tasks/login'; +import { clickAddElasticRulesButton } from '../../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts similarity index 85% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts index ec4615bcf59e4..523d0ec0ad4e0 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/install_workflow.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { resetRulesTableState } from '../../../tasks/common'; -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { resetRulesTableState } from '../../../../tasks/common'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { getInstallSingleRuleButtonByRuleId, GO_BACK_TO_RULES_TABLE_BUTTON, @@ -16,19 +16,19 @@ import { RULE_CHECKBOX, SELECT_ALL_RULES_ON_PAGE_CHECKBOX, TOASTER, -} from '../../../screens/alerts_detection_rules'; -import { selectRulesByName } from '../../../tasks/alerts_detection_rules'; -import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../screens/breadcrumbs'; -import { installPrebuiltRuleAssets } from '../../../tasks/api_calls/prebuilt_rules'; -import { login } from '../../../tasks/login'; +} from '../../../../screens/alerts_detection_rules'; +import { selectRulesByName } from '../../../../tasks/alerts_detection_rules'; +import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../../screens/breadcrumbs'; +import { installPrebuiltRuleAssets } from '../../../../tasks/api_calls/prebuilt_rules'; +import { login } from '../../../../tasks/login'; import { assertInstallationRequestIsComplete, assertRuleInstallationSuccessToastShown, assertRulesPresentInInstalledRulesTable, clickAddElasticRulesButton, -} from '../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; -import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +} from '../../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts similarity index 91% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts index f3101f513915f..15e020b5e0663 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/management.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { COLLAPSED_ACTION_BTN, ELASTIC_RULES_BTN, @@ -15,7 +15,7 @@ import { RULE_SWITCH, SELECT_ALL_RULES_ON_PAGE_CHECKBOX, INSTALL_ALL_RULES_BUTTON, -} from '../../../screens/alerts_detection_rules'; +} from '../../../../screens/alerts_detection_rules'; import { deleteFirstRule, disableAutoRefresh, @@ -24,21 +24,24 @@ import { selectRulesByName, waitForPrebuiltDetectionRulesToBeLoaded, waitForRuleToUpdate, -} from '../../../tasks/alerts_detection_rules'; +} from '../../../../tasks/alerts_detection_rules'; import { deleteSelectedRules, disableSelectedRules, enableSelectedRules, -} from '../../../tasks/rules_bulk_actions'; +} from '../../../../tasks/rules_bulk_actions'; import { createAndInstallMockedPrebuiltRules, getAvailablePrebuiltRulesCount, preventPrebuiltRulesPackageInstallation, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { deleteAlertsAndRules, deletePrebuiltRulesAssets } from '../../../tasks/api_calls/common'; -import { login } from '../../../tasks/login'; -import { visit } from '../../../tasks/navigation'; -import { RULES_MANAGEMENT_URL } from '../../../urls/rules_management'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { + deleteAlertsAndRules, + deletePrebuiltRulesAssets, +} from '../../../../tasks/api_calls/common'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; +import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; const rules = Array.from(Array(5)).map((_, i) => { return createRuleAssetSavedObject({ diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts similarity index 92% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts index 92bf9e7f1471c..4812efc740ae2 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/notifications.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts @@ -5,22 +5,25 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { ADD_ELASTIC_RULES_BTN, ADD_ELASTIC_RULES_EMPTY_PROMPT_BTN, RULES_UPDATES_TAB, -} from '../../../screens/alerts_detection_rules'; -import { deleteFirstRule } from '../../../tasks/alerts_detection_rules'; -import { deleteAlertsAndRules, deletePrebuiltRulesAssets } from '../../../tasks/api_calls/common'; +} from '../../../../screens/alerts_detection_rules'; +import { deleteFirstRule } from '../../../../tasks/alerts_detection_rules'; +import { + deleteAlertsAndRules, + deletePrebuiltRulesAssets, +} from '../../../../tasks/api_calls/common'; import { installAllPrebuiltRulesRequest, installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { resetRulesTableState } from '../../../tasks/common'; -import { login } from '../../../tasks/login'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { resetRulesTableState } from '../../../../tasks/common'; +import { login } from '../../../../tasks/login'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; const RULE_1 = createRuleAssetSavedObject({ name: 'Test rule 1', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts similarity index 97% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_preview.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts index 6deeb6f5202c0..81f37b7760df2 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/prebuilt_rules_preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts @@ -12,22 +12,22 @@ import type { PrebuiltRuleAsset } from '@kbn/security-solution-plugin/server/lib import type { Threshold } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; import { AlertSuppression } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { INSTALL_PREBUILT_RULE_BUTTON, INSTALL_PREBUILT_RULE_PREVIEW, UPDATE_PREBUILT_RULE_PREVIEW, UPDATE_PREBUILT_RULE_BUTTON, -} from '../../../screens/alerts_detection_rules'; -import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../screens/breadcrumbs'; +} from '../../../../screens/alerts_detection_rules'; +import { RULE_MANAGEMENT_PAGE_BREADCRUMB } from '../../../../screens/breadcrumbs'; import { installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { createSavedQuery, deleteSavedQueries } from '../../../tasks/api_calls/saved_queries'; -import { fetchMachineLearningModules } from '../../../tasks/api_calls/machine_learning'; -import { resetRulesTableState } from '../../../tasks/common'; -import { login } from '../../../tasks/login'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { createSavedQuery, deleteSavedQueries } from '../../../../tasks/api_calls/saved_queries'; +import { fetchMachineLearningModules } from '../../../../tasks/api_calls/machine_learning'; +import { resetRulesTableState } from '../../../../tasks/common'; +import { login } from '../../../../tasks/login'; import { assertRuleInstallationSuccessToastShown, assertRulesNotPresentInAddPrebuiltRulesTable, @@ -36,7 +36,7 @@ import { assertRuleUpgradeSuccessToastShown, clickAddElasticRulesButton, clickRuleUpdatesTab, -} from '../../../tasks/prebuilt_rules'; +} from '../../../../tasks/prebuilt_rules'; import { assertAlertSuppressionPropertiesShown, assertCommonPropertiesShown, @@ -55,13 +55,13 @@ import { closeRulePreview, openRuleInstallPreview, openRuleUpdatePreview, -} from '../../../tasks/prebuilt_rules_preview'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; +} from '../../../../tasks/prebuilt_rules_preview'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; import { deleteAlertsAndRules, deleteDataView, postDataView, -} from '../../../tasks/api_calls/common'; +} from '../../../../tasks/api_calls/common'; const TEST_ENV_TAGS = ['@ess', '@serverless']; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/update_workflow.ts similarity index 88% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/update_workflow.ts index edeb8ac98623b..d858280dd5294 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/prebuilt_rules/update_workflow.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/update_workflow.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createRuleAssetSavedObject } from '../../../helpers/rules'; +import { createRuleAssetSavedObject } from '../../../../helpers/rules'; import { getUpgradeSingleRuleButtonByRuleId, NO_RULES_AVAILABLE_FOR_UPGRADE_MESSAGE, @@ -13,22 +13,22 @@ import { SELECT_ALL_RULES_ON_PAGE_CHECKBOX, UPGRADE_ALL_RULES_BUTTON, UPGRADE_SELECTED_RULES_BUTTON, -} from '../../../screens/alerts_detection_rules'; -import { selectRulesByName } from '../../../tasks/alerts_detection_rules'; -import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +} from '../../../../screens/alerts_detection_rules'; +import { selectRulesByName } from '../../../../tasks/alerts_detection_rules'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; import { installPrebuiltRuleAssets, createAndInstallMockedPrebuiltRules, -} from '../../../tasks/api_calls/prebuilt_rules'; -import { resetRulesTableState } from '../../../tasks/common'; -import { login } from '../../../tasks/login'; +} from '../../../../tasks/api_calls/prebuilt_rules'; +import { resetRulesTableState } from '../../../../tasks/common'; +import { login } from '../../../../tasks/login'; import { assertRulesNotPresentInRuleUpdatesTable, assertRuleUpgradeSuccessToastShown, assertUpgradeRequestIsComplete, clickRuleUpdatesTab, -} from '../../../tasks/prebuilt_rules'; -import { visitRulesManagementTable } from '../../../tasks/rules_management'; +} from '../../../../tasks/prebuilt_rules'; +import { visitRulesManagementTable } from '../../../../tasks/rules_management'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/common_flows.cy.ts similarity index 88% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/common_flows.cy.ts index f5704122d9e33..0610786fc1b89 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/common_flows.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/common_flows.cy.ts @@ -5,17 +5,17 @@ * 2.0. */ -import { deleteRuleFromDetailsPage } from '../../../tasks/alerts_detection_rules'; +import { deleteRuleFromDetailsPage } from '../../../../tasks/alerts_detection_rules'; import { CUSTOM_RULES_BTN, RULES_MANAGEMENT_TABLE, RULES_ROW, -} from '../../../screens/alerts_detection_rules'; -import { createRule } from '../../../tasks/api_calls/rules'; -import { getDetails } from '../../../tasks/rule_details'; -import { ruleFields } from '../../../data/detection_engine'; -import { getTimeline } from '../../../objects/timeline'; -import { getExistingRule, getNewRule } from '../../../objects/rule'; +} from '../../../../screens/alerts_detection_rules'; +import { createRule } from '../../../../tasks/api_calls/rules'; +import { getDetails } from '../../../../tasks/rule_details'; +import { ruleFields } from '../../../../data/detection_engine'; +import { getTimeline } from '../../../../objects/timeline'; +import { getExistingRule, getNewRule } from '../../../../objects/rule'; import { ABOUT_DETAILS, @@ -42,13 +42,13 @@ import { THREAT_TACTIC, THREAT_TECHNIQUE, TIMELINE_TEMPLATE_DETAILS, -} from '../../../screens/rule_details'; +} from '../../../../screens/rule_details'; -import { createTimeline } from '../../../tasks/api_calls/timelines'; -import { deleteAlertsAndRules, deleteConnectors } from '../../../tasks/api_calls/common'; -import { login } from '../../../tasks/login'; -import { visit } from '../../../tasks/navigation'; -import { ruleDetailsUrl } from '../../../urls/rule_details'; +import { createTimeline } from '../../../../tasks/api_calls/timelines'; +import { deleteAlertsAndRules, deleteConnectors } from '../../../../tasks/api_calls/common'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; +import { ruleDetailsUrl } from '../../../../urls/rule_details'; // This test is meant to test all common aspects of the rule details page that should function // the same regardless of rule type. For any rule type specific functionalities, please include diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/esql_rule.cy.ts similarity index 69% rename from x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/esql_rule.cy.ts rename to x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/esql_rule.cy.ts index 7d1419e911e33..c59b7db55c743 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details/esql_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/esql_rule.cy.ts @@ -5,24 +5,24 @@ * 2.0. */ -import { getEsqlRule } from '../../../objects/rule'; +import { getEsqlRule } from '../../../../objects/rule'; import { ESQL_QUERY_DETAILS, DEFINITION_DETAILS, RULE_NAME_HEADER, RULE_TYPE_DETAILS, -} from '../../../screens/rule_details'; +} from '../../../../screens/rule_details'; -import { createRule } from '../../../tasks/api_calls/rules'; +import { createRule } from '../../../../tasks/api_calls/rules'; -import { getDetails } from '../../../tasks/rule_details'; -import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; +import { getDetails } from '../../../../tasks/rule_details'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; -import { login } from '../../../tasks/login'; -import { visit } from '../../../tasks/navigation'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; -import { ruleDetailsUrl } from '../../../urls/rule_details'; +import { ruleDetailsUrl } from '../../../../urls/rule_details'; describe('Detection ES|QL rules, details view', { tags: ['@ess'] }, () => { const rule = getEsqlRule(); diff --git a/x-pack/test/security_solution_cypress/package.json b/x-pack/test/security_solution_cypress/package.json index e43f32a447575..e1f552fdba9de 100644 --- a/x-pack/test/security_solution_cypress/package.json +++ b/x-pack/test/security_solution_cypress/package.json @@ -7,9 +7,11 @@ "scripts": { "cypress": "NODE_OPTIONS=--openssl-legacy-provider ../../../node_modules/.bin/cypress", "cypress:open:ess": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel open --spec './cypress/e2e/**/*.cy.ts' --config-file ../../test/security_solution_cypress/cypress/cypress.config.ts --ftr-config-file ../../test/security_solution_cypress/cli_config", - "cypress:run:ess": "yarn cypress:ess --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'", + "cypress:run:ess": "yarn cypress:ess --spec './cypress/e2e/!(investigations|explore|detection_response/rule_management)/**/*.cy.ts'", "cypress:run:cases:ess": "yarn cypress:ess --spec './cypress/e2e/explore/cases/*.cy.ts'", "cypress:ess": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel run --config-file ../../test/security_solution_cypress/cypress/cypress_ci.config.ts --ftr-config-file ../../test/security_solution_cypress/cli_config", + "cypress:rule_management:run:ess":"yarn cypress:ess --spec './cypress/e2e/detection_response/rule_management/!(prebuilt_rules)/**/*.cy.ts'", + "cypress:rule_management:prebuilt_rules:run:ess": "yarn cypress:ess --spec './cypress/e2e/detection_response/rule_management/prebuilt_rules/**/*.cy.ts'", "cypress:run:respops:ess": "yarn cypress:ess --spec './cypress/e2e/(detection_response|exceptions)/**/*.cy.ts'", "cypress:investigations:run:ess": "yarn cypress:ess --spec './cypress/e2e/investigations/**/*.cy.ts'", "cypress:explore:run:ess": "yarn cypress:ess --spec './cypress/e2e/explore/**/*.cy.ts'", @@ -21,16 +23,20 @@ "cypress:cloud:serverless": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider NODE_TLS_REJECT_UNAUTHORIZED=0 ../../../node_modules/.bin/cypress", "cypress:open:cloud:serverless": "yarn cypress:cloud:serverless open --config-file ./cypress/cypress_serverless.config.ts --env CLOUD_SERVERLESS=true", "cypress:open:serverless": "yarn cypress:serverless open --config-file ../../test/security_solution_cypress/cypress/cypress_serverless.config.ts --spec './cypress/e2e/**/*.cy.ts'", - "cypress:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'", + "cypress:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/!(investigations|explore|detection_response/rule_management)/**/*.cy.ts'", "cypress:run:cloud:serverless": "yarn cypress:cloud:serverless run --config-file ./cypress/cypress_ci_serverless.config.ts --env CLOUD_SERVERLESS=true", + "cypress:rule_management:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/detection_response/rule_management/!(prebuilt_rules)/**/*.cy.ts'", + "cypress:rule_management:prebuilt_rules:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/detection_response/rule_management/prebuilt_rules/**/*.cy.ts'", "cypress:investigations:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'", "cypress:explore:run:serverless": "yarn cypress:serverless --spec './cypress/e2e/explore/**/*.cy.ts'", "cypress:changed-specs-only:serverless": "yarn cypress:serverless --changed-specs-only --env burn=5", "cypress:burn:serverless": "yarn cypress:serverless --env burn=2", "cypress:qa:serverless": "TZ=UTC NODE_OPTIONS=--openssl-legacy-provider node ../../plugins/security_solution/scripts/start_cypress_parallel_serverless --config-file ../../test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts", "cypress:open:qa:serverless": "yarn cypress:qa:serverless open", - "cypress:run:qa:serverless": "yarn cypress:qa:serverless --spec './cypress/e2e/!(investigations|explore)/**/*.cy.ts'", + "cypress:run:qa:serverless": "yarn cypress:qa:serverless --spec './cypress/e2e/!(investigations|explore|detection_response/rule_management)/**/*.cy.ts'", "cypress:run:qa:serverless:investigations": "yarn cypress:qa:serverless --spec './cypress/e2e/investigations/**/*.cy.ts'", - "cypress:run:qa:serverless:explore": "yarn cypress:qa:serverless --spec './cypress/e2e/explore/**/*.cy.ts'" + "cypress:run:qa:serverless:explore": "yarn cypress:qa:serverless --spec './cypress/e2e/explore/**/*.cy.ts'", + "cypress:run:qa:serverless:rule_management": "yarn cypress:qa:serverless --spec './cypress/e2e/detection_response/rule_management/!(prebuilt_rules)/**/*.cy.ts'", + "cypress:run:qa:serverless:rule_management:prebuilt_rules": "yarn cypress:qa:serverless --spec './cypress/e2e/detection_response/rule_management/prebuilt_rules/**/*.cy.ts'" } } \ No newline at end of file From e94a977873797b070ca9316c174ed1abb0dc7610 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Tue, 28 Nov 2023 11:15:06 -0500 Subject: [PATCH 14/30] Reorganize new/existing pipeline management screens into tabs (#172027) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split Configure step’s components in pipeline creation into two tabs: - Create new: pipeline name, model selection - Use existing: pipeline selection Keep the split components' content as-is, they are out of scope for this task. Remove the left hand side panels (“Create or select a pipeline”, “Select a trained ML Model” and blurbs underneath), and make the tab full width. Also remove the ELSER text expansion callout from the flyout. --- .../ml_inference/configure_pipeline.tsx | 376 ++++++++---------- .../translations/translations/fr-FR.json | 9 - .../translations/translations/ja-JP.json | 9 - .../translations/translations/zh-CN.json | 9 - 4 files changed, 155 insertions(+), 248 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx index d1056fe8395e4..4b076758d1d4c 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx @@ -11,51 +11,45 @@ import { useValues, useActions } from 'kea'; import { EuiFieldText, - EuiFlexGroup, - EuiFlexItem, EuiForm, EuiFormRow, - EuiLink, - EuiSelect, EuiSuperSelect, EuiSuperSelectOption, EuiSpacer, + EuiTabbedContent, + EuiTabbedContentTab, EuiTitle, EuiText, - EuiPanel, - EuiHorizontalRule, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { docLinks } from '../../../../../shared/doc_links'; - import { IndexNameLogic } from '../../index_name_logic'; import { IndexViewLogic } from '../../index_view_logic'; -import { InferenceConfiguration } from './inference_config'; import { EMPTY_PIPELINE_CONFIGURATION, MLInferenceLogic } from './ml_inference_logic'; import { MlModelSelectOption } from './model_select_option'; import { PipelineSelectOption } from './pipeline_select_option'; -import { TextExpansionCallOut } from './text_expansion_callout/text_expansion_callout'; import { MODEL_REDACTED_VALUE, MODEL_SELECT_PLACEHOLDER } from './utils'; const MODEL_SELECT_PLACEHOLDER_VALUE = 'model_placeholder$$'; const PIPELINE_SELECT_PLACEHOLDER_VALUE = 'pipeline_placeholder$$'; -const CHOOSE_EXISTING_LABEL = i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.chooseLabel', - { defaultMessage: 'Choose' } -); -const CHOOSE_NEW_LABEL = i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.newLabel', - { defaultMessage: 'New pipeline' } +const CREATE_NEW_TAB_NAME = i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.tabs.createNew.name', + { defaultMessage: 'Create new' } ); -const CHOOSE_PIPELINE_LABEL = i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.existingLabel', - { defaultMessage: 'Existing pipeline' } + +const USE_EXISTING_TAB_NAME = i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.tabs.useExisting.name', + { defaultMessage: 'Use existing' } ); +export enum ConfigurePipelineTabId { + CREATE_NEW = 'create_new', + USE_EXISTING = 'use_existing', +} + export const ConfigurePipeline: React.FC = () => { const { addInferencePipelineModal: { configuration }, @@ -113,222 +107,162 @@ export const ConfigurePipeline: React.FC = () => { const inputsDisabled = configuration.existingPipeline !== false; - return ( - <> - - - -

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.title', - { defaultMessage: 'Create or select a pipeline' } - )} -

- + const tabs: EuiTabbedContentTab[] = [ + { + content: ( + <> - -

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.description', - { - defaultMessage: - 'Build or reuse a child pipeline that will be used as a processor in your main pipeline.', - } - )} -

-

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.descriptionUsePipelines', + + - - - - - - - - setInferencePipelineConfiguration({ - ...EMPTY_PIPELINE_CONFIGURATION, - existingPipeline: e.target.value === 'true', - }) - } - value={configuration.existingPipeline?.toString() ?? ''} - /> - - {configuration.existingPipeline === true ? ( - - 0 ? pipelineName : PIPELINE_SELECT_PLACEHOLDER_VALUE - } - options={pipelineOptions} - onChange={(value) => selectExistingPipeline(value)} - /> - - ) : ( - - {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.name.helpText', - { - defaultMessage: - 'Pipeline names are unique within a deployment and can only contain letters, numbers, underscores, and hyphens. This will create a pipeline named {pipelineName}.', - values: { - pipelineName: `ml-inference-${pipelineName}`, - }, - } - )} - - ) - } - error={nameError && formErrors.pipelineName} - isInvalid={nameError} - > - + {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.name.helpText', { - defaultMessage: 'Enter a unique name for this pipeline', + defaultMessage: + 'Pipeline names are unique within a deployment and can only contain letters, numbers, underscores, and hyphens. This will create a pipeline named {pipelineName}.', + values: { + pipelineName: `ml-inference-${pipelineName}`, + }, } )} - value={pipelineName} - onChange={(e) => - setInferencePipelineConfiguration({ - ...configuration, - pipelineName: e.target.value, - }) - } - /> - - )} - - - - - - - - -

- {i18n.translate( + + ) + } + error={nameError && formErrors.pipelineName} + isInvalid={nameError} + > + + setInferencePipelineConfiguration({ + ...configuration, + pipelineName: e.target.value, + }) + } + /> + + - - - -

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.descriptionDeployTrainedModel', - { - defaultMessage: - 'To perform natural language processing tasks in your cluster, you must deploy an appropriate trained model.', + > + + setInferencePipelineConfiguration({ + ...configuration, + inferenceConfig: undefined, + modelID: value, + fieldMappings: undefined, + }) } - )} -

- - {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.docsLink', + options={modelOptions} + valueOfSelected={modelID === '' ? MODEL_SELECT_PLACEHOLDER_VALUE : modelID} + /> + + + + ), + id: ConfigurePipelineTabId.CREATE_NEW, + name: CREATE_NEW_TAB_NAME, + }, + { + content: ( + <> + + + - - - - - - + - - setInferencePipelineConfiguration({ - ...configuration, - inferenceConfig: undefined, - modelID: value, - fieldMappings: undefined, - }) - } - options={modelOptions} - valueOfSelected={modelID === '' ? MODEL_SELECT_PLACEHOLDER_VALUE : modelID} - /> - - - - - - - - + hasDividers + data-telemetry-id={`entSearchContent-${ingestionMethod}-pipelines-configureInferencePipeline-selectExistingPipeline`} + valueOfSelected={ + pipelineName.length > 0 ? pipelineName : PIPELINE_SELECT_PLACEHOLDER_VALUE + } + options={pipelineOptions} + onChange={(value) => selectExistingPipeline(value)} + /> + + + + ), + id: ConfigurePipelineTabId.USE_EXISTING, + name: USE_EXISTING_TAB_NAME, + }, + ]; + + return ( + <> + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.title', + { defaultMessage: 'Configure a pipeline' } + )} +

+ + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.description', + { + defaultMessage: + 'Build or reuse a child pipeline that will be used as a processor in your main pipeline.', + } + )} +

+ + + { + const isExistingPipeline = tab.id === ConfigurePipelineTabId.USE_EXISTING; + if (isExistingPipeline !== configuration.existingPipeline) { + const pipelineConfig = EMPTY_PIPELINE_CONFIGURATION; + pipelineConfig.existingPipeline = isExistingPipeline; + if (!isExistingPipeline) { + pipelineConfig.pipelineName = indexName; + } + + setInferencePipelineConfiguration(pipelineConfig); + } + }} + /> ); }; diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index e72ad4482d77c..f07802a4ef028 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -14584,16 +14584,9 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.createErrors": "Erreur lors de la création d'un pipeline", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.noModels.esDocs.link": "Découvrir comment ajouter un modèle entraîné", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.noModels.imageAlt": "Illustration d'absence de modèles de Machine Learning", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.chooseExistingLabel": "Nouveau ou existant", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.description": "Créez ou réutilisez un pipeline enfant qui servira de processeur dans votre pipeline principal.", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.descriptionDeployTrainedModel": "Pour effectuer des tâches de traitement du langage naturel dans votre cluster, vous devez déployer un modèle entraîné approprié.", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.descriptionUsePipelines": "Les pipelines que vous créez sont enregistrés pour être utilisés ailleurs dans votre déploiement Elastic.", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.docsLink": "Découvrez l'importation et l'utilisation des modèles de ML dans Search", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.emptyValueError": "Champ obligatoire.", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.chooseLabel": "Choisir", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.existingLabel": "Pipeline existant", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.model": "Modèle", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.newLabel": "Nouveau pipeline", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.placeholder": "Effectuez une sélection", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.sourceFields": "Champs sources", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipelineLabel": "Sélectionner un pipeline d'inférence existant", @@ -14603,11 +14596,9 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.invalidPipelineName": "Le nom doit contenir uniquement des lettres, des chiffres, des traits de soulignement et des traits d'union.", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.model.placeholder": "Sélectionner un modèle", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.model.redactedValue": "Ce modèle n'est pas disponible dans l'espace Kibana", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.modelLabel": "Sélectionner un modèle de ML entraîné", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.nameLabel": "Nom", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.namePlaceholder": "Saisir un nom unique pour ce pipeline", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.pipelineNameExistsError": "Ce nom est déjà utilisé par un autre pipeline.", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.title": "Créer ou sélectionner un pipeline", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.titleSelectTrainedModel": "Sélectionner un modèle de ML entraîné", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.actions": "Actions", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.actions.deleteMapping": "Supprimer ce mapping", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 4fb5f53604319..e34f8cdba4c9d 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -14597,16 +14597,9 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.createErrors": "パイプラインの作成エラー", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.noModels.esDocs.link": "学習されたモデルの追加方法の詳細", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.noModels.imageAlt": "機械学習モデル例がありません", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.chooseExistingLabel": "新規または既存", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.description": "メインパイプラインでプロセッサーとして使用される子パイプラインを作成または再利用します。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.descriptionDeployTrainedModel": "クラスターで自然言語処理タスクを実行するには、適切な学習済みモデルをデプロイする必要があります。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.descriptionUsePipelines": "作成するパイプラインはElasticデプロイの場所で使用されるために保存されます。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.docsLink": "SearchでのMLモデルのインポートと使用の詳細", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.emptyValueError": "フィールドが必要です。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.chooseLabel": "選択", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.existingLabel": "既存のパイプライン", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.model": "モデル", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.newLabel": "新しいパイプライン", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.placeholder": "1 つ選択してください", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.sourceFields": "ソースフィールド", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipelineLabel": "既存の推論パイプラインを選択", @@ -14616,11 +14609,9 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.invalidPipelineName": "名前には文字、数字、アンダースコア、ハイフンのみ使用する必要があります。", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.model.placeholder": "モデルを選択", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.model.redactedValue": "このモデルはKibanaスペースで使用できません", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.modelLabel": "学習済みMLモデルを選択", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.nameLabel": "名前", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.namePlaceholder": "このパイプラインの一意の名前を入力", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.pipelineNameExistsError": "名前はすでに別のパイプラインで使用されています。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.title": "パイプラインを作成または入力", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.titleSelectTrainedModel": "学習済みMLモデルを選択", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.actions": "アクション", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.actions.deleteMapping": "このマッピングを削除", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index bf8ca19d55dc6..5af3cf3928ba3 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -14597,16 +14597,9 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.createErrors": "创建管道时出错", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.noModels.esDocs.link": "了解如何添加已训练模型", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.noModels.imageAlt": "无 Machine Learning 模型图示", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.chooseExistingLabel": "新建或现有", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.description": "构建或重复使用将在您的主管道中用作处理器的子管道。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.descriptionDeployTrainedModel": "要在集群中执行自然语言处理任务，必须部署适当的已训练模型。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.descriptionUsePipelines": "将保存您创建的管道，以便在 Elastic 部署中的其他位置使用。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.docsLink": "详细了解如何在 Search 中导入并使用 ML 模型", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.emptyValueError": "“字段”必填。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.chooseLabel": "选择", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.existingLabel": "现有管道", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.model": "模型", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.newLabel": "新建管道", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.placeholder": "选择一个", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.sourceFields": "源字段", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipelineLabel": "选择现有推理管道", @@ -14616,11 +14609,9 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.invalidPipelineName": "名称必须仅包含字母、数字、下划线和连字符。", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.model.placeholder": "选择模型", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.model.redactedValue": "此模型在 Kibana 工作区不可用", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.modelLabel": "选择已训练 ML 模型", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.nameLabel": "名称", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.namePlaceholder": "为此管道输入唯一名称", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.pipelineNameExistsError": "名称已由其他管道使用。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.title": "创建或选择管道", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.titleSelectTrainedModel": "选择已训练 ML 模型", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.actions": "操作", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.actions.deleteMapping": "删除此映射", From 3f9f3e649e3cd8d2069f76ab40927103d8309856 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Tue, 28 Nov 2023 09:16:28 -0700 Subject: [PATCH 15/30] [CSV/Reporting] Refinements for logging hit totals (#171811) ## Summary This PR fixes a **logging** issue noticed when analyzing CSV Export logs. There is an "info" log giving the total number of hits, which includes the `total.relation` value in the message. This log message incorrectly printed the relationship as `eq` even when the relation was not given in the Elasticsearch response. The correction to this log message will help us make sure our search requests are configured to track the total hits of the query. ``` # Before Total hits eq 12345. # After Received total hits: 12345. Accuracy: eq. Received total hits: 12345. Accuracy: unknown. ``` ### Other changes * A new private method for logging the result metadata * Syntax cleanup * More tests ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../__snapshots__/generate_csv.test.ts.snap | 71 ++++++++++++ .../kbn-generate-csv/src/generate_csv.test.ts | 80 +++++++++++++ packages/kbn-generate-csv/src/generate_csv.ts | 109 ++++++++++-------- 3 files changed, 214 insertions(+), 46 deletions(-) diff --git a/packages/kbn-generate-csv/src/__snapshots__/generate_csv.test.ts.snap b/packages/kbn-generate-csv/src/__snapshots__/generate_csv.test.ts.snap index da0f6a4560640..e38a7427e98d6 100644 --- a/packages/kbn-generate-csv/src/__snapshots__/generate_csv.test.ts.snap +++ b/packages/kbn-generate-csv/src/__snapshots__/generate_csv.test.ts.snap @@ -66,6 +66,77 @@ exports[`CsvGenerator formulas escapes formula values in a header, doesn't warn `; exports[`CsvGenerator keeps order of the columns during the scroll 1`] = ` +Array [ + Array [ + "Requesting PIT for: [logstash-*]...", + ], + Array [ + "Opened PIT ID: oju9fs3698s3[39 bytes]", + ], + Array [ + "Executing search request with PIT ID: [oju9fs3698s3[39 bytes]]", + ], + Array [ + "Received total hits: 3. Accuracy: unknown.", + ], + Array [ + "Result details: {\\"rawResponse\\":{\\"took\\":1,\\"timed_out\\":false,\\"_shards\\":{\\"total\\":1,\\"successful\\":1,\\"failed\\":0,\\"skipped\\":0},\\"hits\\":{\\"total\\":3,\\"max_score\\":0},\\"pit_id\\":\\"oju9fs3698s3[39 bytes]\\"}}", + ], + Array [ + "Received PIT ID: [oju9fs3698s3[39 bytes]]", + ], + Array [ + "Received search_after: [undefined]", + ], + Array [ + "Building CSV header row", + ], + Array [ + "Building 1 CSV data rows", + ], + Array [ + "Executing search request with PIT ID: [oju9fs3698s3[39 bytes]]", + ], + Array [ + "Received total hits: 3. Accuracy: unknown.", + ], + Array [ + "Result details: {\\"rawResponse\\":{\\"took\\":1,\\"timed_out\\":false,\\"_shards\\":{\\"total\\":1,\\"successful\\":1,\\"failed\\":0,\\"skipped\\":0},\\"hits\\":{\\"total\\":3,\\"max_score\\":0},\\"pit_id\\":\\"oju9fs3698s3[39 bytes]\\"}}", + ], + Array [ + "Received PIT ID: [oju9fs3698s3[39 bytes]]", + ], + Array [ + "Received search_after: [undefined]", + ], + Array [ + "Building 1 CSV data rows", + ], + Array [ + "Executing search request with PIT ID: [oju9fs3698s3[39 bytes]]", + ], + Array [ + "Received total hits: 3. Accuracy: unknown.", + ], + Array [ + "Result details: {\\"rawResponse\\":{\\"took\\":1,\\"timed_out\\":false,\\"_shards\\":{\\"total\\":1,\\"successful\\":1,\\"failed\\":0,\\"skipped\\":0},\\"hits\\":{\\"total\\":3,\\"max_score\\":0},\\"pit_id\\":\\"oju9fs3698s3[39 bytes]\\"}}", + ], + Array [ + "Received PIT ID: [oju9fs3698s3[39 bytes]]", + ], + Array [ + "Received search_after: [undefined]", + ], + Array [ + "Building 1 CSV data rows", + ], + Array [ + "Closing PIT oju9fs3698s3[39 bytes]", + ], +] +`; + +exports[`CsvGenerator keeps order of the columns during the scroll 2`] = ` "\\"_id\\",\\"_index\\",\\"_score\\",a,b \\"'-\\",\\"'-\\",\\"'-\\",a1,b1 \\"'-\\",\\"'-\\",\\"'-\\",\\"'-\\",b2 diff --git a/packages/kbn-generate-csv/src/generate_csv.test.ts b/packages/kbn-generate-csv/src/generate_csv.test.ts index 22857f37afdaf..22269054a61d9 100644 --- a/packages/kbn-generate-csv/src/generate_csv.test.ts +++ b/packages/kbn-generate-csv/src/generate_csv.test.ts @@ -393,6 +393,8 @@ describe('CsvGenerator', () => { }) ); + const debugLogSpy = jest.spyOn(mockLogger, 'debug'); + const generateCsv = new CsvGenerator( createMockJob({ searchSource: {}, columns: [] }), mockConfig, @@ -411,6 +413,8 @@ describe('CsvGenerator', () => { ); await generateCsv.generateData(); + expect(debugLogSpy.mock.calls).toMatchSnapshot(); + expect(content).toMatchSnapshot(); }); @@ -896,6 +900,82 @@ describe('CsvGenerator', () => { `); }); + describe('debug logging', () => { + it('logs the the total hits relation if relation is provided', async () => { + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: { + took: 1, + timed_out: false, + pit_id: mockPitId, + _shards: { total: 1, successful: 1, failed: 0, skipped: 0 }, + hits: { hits: [], total: { relation: 'eq', value: 12345 }, max_score: 0 }, + }, + }) + ); + + const debugLogSpy = jest.spyOn(mockLogger, 'debug'); + + const generateCsv = new CsvGenerator( + createMockJob({ columns: ['date', 'ip', 'message'] }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + + await generateCsv.generateData(); + + expect(debugLogSpy).toHaveBeenCalledWith('Received total hits: 12345. Accuracy: eq.'); + }); + + it('logs the the total hits relation as "unknown" if relation is not provided', async () => { + mockDataClient.search = jest.fn().mockImplementation(() => + Rx.of({ + rawResponse: { + took: 1, + timed_out: false, + pit_id: mockPitId, + _shards: { total: 1, successful: 1, failed: 0, skipped: 0 }, + hits: { hits: [], total: 12345, max_score: 0 }, + }, + }) + ); + + const debugLogSpy = jest.spyOn(mockLogger, 'debug'); + + const generateCsv = new CsvGenerator( + createMockJob({ columns: ['date', 'ip', 'message'] }), + mockConfig, + { + es: mockEsClient, + data: mockDataClient, + uiSettings: uiSettingsClient, + }, + { + searchSourceStart: mockSearchSourceService, + fieldFormatsRegistry: mockFieldFormatsRegistry, + }, + new CancellationToken(), + mockLogger, + stream + ); + + await generateCsv.generateData(); + + expect(debugLogSpy).toHaveBeenCalledWith('Received total hits: 12345. Accuracy: unknown.'); + }); + }); + it('will return partial data if the scroll or search fails', async () => { mockDataClient.search = jest.fn().mockImplementation(() => { throw new esErrors.ResponseError({ diff --git a/packages/kbn-generate-csv/src/generate_csv.ts b/packages/kbn-generate-csv/src/generate_csv.ts index 59e69e9989c8e..22e2916bf6fd7 100644 --- a/packages/kbn-generate-csv/src/generate_csv.ts +++ b/packages/kbn-generate-csv/src/generate_csv.ts @@ -11,7 +11,11 @@ import type { Writable } from 'stream'; import { errors as esErrors, estypes } from '@elastic/elasticsearch'; import type { IScopedClusterClient, IUiSettingsClient, Logger } from '@kbn/core/server'; -import type { ISearchSource, ISearchStartSearchSource } from '@kbn/data-plugin/common'; +import type { + IKibanaSearchResponse, + ISearchSource, + ISearchStartSearchSource, +} from '@kbn/data-plugin/common'; import { ES_SEARCH_STRATEGY, cellHasFormulas, tabifyDocs } from '@kbn/data-plugin/common'; import type { IScopedSearchClient } from '@kbn/data-plugin/server'; import type { Datatable } from '@kbn/expressions-plugin/server'; @@ -94,6 +98,38 @@ export class CsvGenerator { return pitId; } + /** + * @param clientDetails: Details from the data.search client + * @param results: Raw data from ES + */ + private logResults( + clientDetails: Omit, 'rawResponse'>, + results: estypes.SearchResponse + ) { + const { hits: resultsHits, ...headerWithPit } = results; + const { hits, ...hitsMeta } = resultsHits; + const trackedTotal = resultsHits.total as estypes.SearchTotalHits; + const currentTotal = trackedTotal?.value ?? resultsHits.total; + + const totalAccuracy = trackedTotal?.relation ?? 'unknown'; + this.logger.debug(`Received total hits: ${currentTotal}. Accuracy: ${totalAccuracy}.`); + + // reconstruct the data.search response (w/out the data) for logging + const { pit_id: newPitId, ...header } = headerWithPit; + const logInfo = { + ...clientDetails, + rawResponse: { + ...header, + hits: hitsMeta, + pit_id: `${this.formatPit(newPitId)}`, + }, + }; + this.logger.debug(`Result details: ${JSON.stringify(logInfo)}`); + + // use the most recently received id for the next search request + this.logger.debug(`Received PIT ID: [${this.formatPit(results.pit_id)}]`); + } + private async doSearch( searchSource: ISearchSource, settings: CsvExportSettings, @@ -117,25 +153,20 @@ export class CsvGenerator { throw new Error('Could not retrieve the search body!'); } - const searchParams = { - params: { - body: searchBody, - }, - }; - - let results: estypes.SearchResponse | undefined; + const searchParams = { params: { body: searchBody } }; + let results: estypes.SearchResponse; try { - results = ( - await lastValueFrom( - this.clients.data.search(searchParams, { - strategy: ES_SEARCH_STRATEGY, - transport: { - maxRetries: 0, // retrying reporting jobs is handled in the task manager scheduling logic - requestTimeout: scrollSettings.duration, - }, - }) - ) - ).rawResponse; + const { rawResponse, ...rawDetails } = await lastValueFrom( + this.clients.data.search(searchParams, { + strategy: ES_SEARCH_STRATEGY, + transport: { + maxRetries: 0, // retrying reporting jobs is handled in the task manager scheduling logic + requestTimeout: settings.scroll.duration, + }, + }) + ); + results = rawResponse; + this.logResults(rawDetails, rawResponse); } catch (err) { this.logger.error(`CSV export search error: ${err}`); throw err; @@ -327,7 +358,6 @@ export class CsvGenerator { let first = true; let currentRecord = -1; let totalRecords: number | undefined; - let totalRelation = 'eq'; let searchAfter: estypes.SortResults | undefined; let pitId = await this.openPointInTime(indexPatternTitle, settings); @@ -360,47 +390,31 @@ export class CsvGenerator { searchSource.setField('pit', { id: pitId, keep_alive: settings.scroll.duration }); const results = await this.doSearch(searchSource, settings, searchAfter); - - const { hits } = results; - if (first && hits.total != null) { - if (typeof hits.total === 'number') { - totalRecords = hits.total; - } else { - totalRecords = hits.total?.value; - totalRelation = hits.total?.relation ?? 'unknown'; - } - this.logger.info(`Total hits ${totalRelation} ${totalRecords}.`); - } - if (!results) { this.logger.warn(`Search results are undefined!`); break; } - const { - hits: { hits: _hits, ...hitsMeta }, - ...headerWithPit - } = results; + const { hits: resultsHits } = results; + const { hits, total } = resultsHits; + const trackedTotal = total as estypes.SearchTotalHits; + const currentTotal = trackedTotal?.value ?? total; - const { pit_id: newPitId, ...header } = headerWithPit; - - const logInfo = { - header: { pit_id: `${this.formatPit(newPitId)}`, ...header }, - hitsMeta, - }; - this.logger.debug(`Results metadata: ${JSON.stringify(logInfo)}`); + if (first) { + // export stops when totalRecords have been accumulated (or the results have run out) + totalRecords = currentTotal; + } // use the most recently received id for the next search request - this.logger.debug(`Received PIT ID: [${this.formatPit(results.pit_id)}]`); pitId = results.pit_id ?? pitId; // Update last sort results for next query. PIT is used, so the sort results // automatically include _shard_doc as a tiebreaker - searchAfter = hits.hits[hits.hits.length - 1]?.sort as estypes.SortResults | undefined; + searchAfter = hits[hits.length - 1]?.sort as estypes.SortResults | undefined; this.logger.debug(`Received search_after: [${searchAfter}]`); // check for shard failures, log them and add a warning if found - const { _shards: shards } = header; + const { _shards: shards } = results; if (shards.failures) { shards.failures.forEach(({ reason }) => { warnings.push(`Shard failure: ${JSON.stringify(reason)}`); @@ -499,6 +513,9 @@ export class CsvGenerator { }; } + /** + * Method to avoid logging the entire PIT: it could be megabytes long + */ private formatPit(pitId: string | undefined) { const byteSize = pitId ? Buffer.byteLength(pitId, 'utf-8') : 0; return pitId?.substring(0, 12) + `[${byteSize} bytes]`; From d9ebfd9af1365bba54d5e1ac92f5e53f5fbebea8 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Tue, 28 Nov 2023 11:25:03 -0500 Subject: [PATCH 16/30] [Response Ops][Alerting] Remove echoed field value from bulk error responses when indexing alerts (#172020) ## Summary When alerts are bulk indexed in the rule registry and the alerts client, indexing errors may be returned where the entire field value that failed to be indexed is echoed in the reason. This can cause unnecessarily verbose logging so we want to sanitize the field value. --- .../alerts_client/alerts_client.test.ts | 18 +- .../alerting/server/alerts_client/index.ts | 1 + .../lib/alert_conflict_resolver.ts | 4 +- .../server/alerts_client/lib/index.ts | 1 + .../lib/sanitize_bulk_response.test.ts | 244 ++++++++++++++++++ .../lib/sanitize_bulk_response.ts | 38 +++ x-pack/plugins/alerting/server/index.ts | 1 + .../rule_data_client/rule_data_client.test.ts | 123 +++++++++ .../rule_data_client/rule_data_client.ts | 20 +- 9 files changed, 442 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.test.ts create mode 100644 x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.ts diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts index 9526f87e260d6..86fe855152a9a 100644 --- a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts @@ -1133,6 +1133,22 @@ describe('Alerts Client', () => { }, }, }, + { + index: { + _index: '.internal.alerts-test.alerts-default-000001', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'. Preview of field's value: 'we don't want this field value to be echoed'", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, { index: { _index: '.internal.alerts-test.alerts-default-000002', @@ -1164,7 +1180,7 @@ describe('Alerts Client', () => { expect(clusterClient.bulk).toHaveBeenCalled(); expect(logger.error).toHaveBeenCalledWith( - `Error writing alerts: 1 successful, 0 conflicts, 1 errors: Validation Failed: 1: index is missing;2: type is missing;` + `Error writing alerts: 1 successful, 0 conflicts, 2 errors: Validation Failed: 1: index is missing;2: type is missing;; failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.` ); }); diff --git a/x-pack/plugins/alerting/server/alerts_client/index.ts b/x-pack/plugins/alerting/server/alerts_client/index.ts index 442f8935650f5..a1c0a309e0dc4 100644 --- a/x-pack/plugins/alerting/server/alerts_client/index.ts +++ b/x-pack/plugins/alerting/server/alerts_client/index.ts @@ -8,3 +8,4 @@ export { type LegacyAlertsClientParams, LegacyAlertsClient } from './legacy_alerts_client'; export { AlertsClient } from './alerts_client'; export type { AlertRuleData } from './types'; +export { sanitizeBulkErrorResponse } from './lib'; diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts b/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts index 3c5ce6e25a1a8..383d8dbb103fb 100644 --- a/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts +++ b/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts @@ -23,6 +23,7 @@ import { } from '@kbn/rule-data-utils'; import { zip, get } from 'lodash'; +import { sanitizeBulkErrorResponse } from '../..'; // these fields are the one's we'll refresh from the fresh mget'd docs const REFRESH_FIELDS_ALWAYS = [ALERT_WORKFLOW_STATUS, ALERT_WORKFLOW_TAGS, ALERT_CASE_IDS]; @@ -269,8 +270,9 @@ interface ResponseStatsResult { // generate a summary of the original bulk request attempt, for logging function getResponseStats(bulkResponse: BulkResponse): ResponseStatsResult { + const sanitizedResponse = sanitizeBulkErrorResponse(bulkResponse) as BulkResponse; const stats: ResponseStatsResult = { success: 0, conflicts: 0, errors: 0, messages: [] }; - for (const item of bulkResponse.items) { + for (const item of sanitizedResponse.items) { const op = item.create || item.index || item.update || item.delete; if (op?.error) { if (op?.status === 409 && op === item.index) { diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/index.ts b/x-pack/plugins/alerting/server/alerts_client/lib/index.ts index 7225e87056e4f..6e40e918a8b2c 100644 --- a/x-pack/plugins/alerting/server/alerts_client/lib/index.ts +++ b/x-pack/plugins/alerting/server/alerts_client/lib/index.ts @@ -16,3 +16,4 @@ export { getContinualAlertsQuery, } from './get_summarized_alerts_query'; export { expandFlattenedAlert } from './format_alert'; +export { sanitizeBulkErrorResponse } from './sanitize_bulk_response'; diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.test.ts b/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.test.ts new file mode 100644 index 0000000000000..533bb5b554ae9 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.test.ts @@ -0,0 +1,244 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { TransportResult } from '@elastic/elasticsearch'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { sanitizeBulkErrorResponse } from './sanitize_bulk_response'; + +// Using https://www.elastic.co/guide/en/elasticsearch/reference/8.11/docs-bulk.html +describe('sanitizeBulkErrorResponse', () => { + test('should not modify success response', () => { + const responseBody = { + errors: false, + took: 1, + items: [ + { + index: { + _index: 'test', + _id: '1', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 0, + _primary_term: 1, + }, + }, + { + delete: { + _index: 'test', + _id: '2', + _version: 1, + result: 'not_found', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 404, + _seq_no: 1, + _primary_term: 2, + }, + }, + { + create: { + _index: 'test', + _id: '3', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + { + update: { + _index: 'test', + _id: '1', + _version: 2, + result: 'updated', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 200, + _seq_no: 3, + _primary_term: 4, + }, + }, + ], + }; + const transportResponseBody = wrapResponseBody(responseBody); + + expect(sanitizeBulkErrorResponse(responseBody)).toEqual(responseBody); + expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual(transportResponseBody); + }); + + test('should not modify error response without field preview', () => { + const responseBody = { + took: 486, + errors: true, + items: [ + { + update: { + _index: 'index1', + _id: '5', + status: 404, + error: { + type: 'document_missing_exception', + reason: '[5]: document missing', + index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA', + shard: '0', + index: 'index1', + }, + }, + }, + { + delete: { + _index: 'index1', + _id: '6', + status: 404, + error: { + type: 'document_missing_exception', + reason: '[6]: document missing', + index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA', + shard: '0', + index: 'index1', + }, + }, + }, + { + create: { + _index: 'test', + _id: '3', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + ], + }; + const transportResponseBody = wrapResponseBody(responseBody); + + expect(sanitizeBulkErrorResponse(responseBody)).toEqual(responseBody); + expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual(transportResponseBody); + }); + + test('should sanitize error response with field preview', () => { + const responseBody = { + took: 486, + errors: true, + items: [ + { + update: { + _index: 'index1', + _id: '5', + status: 404, + error: { + type: 'document_missing_exception', + reason: '[5]: document missing', + index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA', + shard: '0', + index: 'index1', + }, + }, + }, + { + update: { + _index: 'index1', + _id: '6', + status: 404, + error: { + type: 'document_missing_exception', + reason: '[6]: document missing', + index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA', + shard: '0', + index: 'index1', + }, + }, + }, + { + create: { + _index: 'index1', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'. Preview of field's value: 'we don't want this field value to be echoed'", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, + ], + }; + const transportResponseBody = wrapResponseBody(responseBody); + + expect(sanitizeBulkErrorResponse(responseBody)).toEqual({ + ...responseBody, + items: [ + responseBody.items[0], + responseBody.items[1], + { + create: { + _index: 'index1', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, + ], + }); + expect(sanitizeBulkErrorResponse(transportResponseBody)).toEqual({ + ...transportResponseBody, + body: { + ...transportResponseBody.body, + items: [ + transportResponseBody.body.items[0], + transportResponseBody.body.items[1], + { + create: { + _index: 'index1', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, + ], + }, + }); + }); +}); + +function wrapResponseBody( + body: estypes.BulkResponse, + statusCode: number = 200 +): TransportResult { + return { + body, + statusCode, + headers: {}, + warnings: null, + // @ts-expect-error + meta: {}, + }; +} diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.ts b/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.ts new file mode 100644 index 0000000000000..2b6d9f6e3c2c3 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_client/lib/sanitize_bulk_response.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { cloneDeep } from 'lodash'; +import { TransportResult } from '@elastic/elasticsearch'; +import { get } from 'lodash'; +import { set } from '@kbn/safer-lodash-set'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +export const sanitizeBulkErrorResponse = ( + response: TransportResult | estypes.BulkResponse +): TransportResult | estypes.BulkResponse => { + const clonedResponse = cloneDeep(response); + const isTransportResponse = !!(response as TransportResult).body; + + const responseToUse: estypes.BulkResponse = isTransportResponse + ? (clonedResponse as TransportResult).body + : (clonedResponse as estypes.BulkResponse); + + if (responseToUse.errors) { + (responseToUse.items ?? []).forEach( + (item: Partial>) => { + for (const [_, responseItem] of Object.entries(item)) { + const reason: string = get(responseItem, 'error.reason'); + const redactIndex = reason ? reason.indexOf(`Preview of field's value:`) : -1; + if (redactIndex > 1) { + set(responseItem, 'error.reason', reason.substring(0, redactIndex - 1)); + } + } + } + ); + } + + return clonedResponse; +}; diff --git a/x-pack/plugins/alerting/server/index.ts b/x-pack/plugins/alerting/server/index.ts index b6962f65b4ab8..13237f92b1c1d 100644 --- a/x-pack/plugins/alerting/server/index.ts +++ b/x-pack/plugins/alerting/server/index.ts @@ -67,6 +67,7 @@ export { isValidAlertIndexName, InstallShutdownError, } from './alerts_service'; +export { sanitizeBulkErrorResponse } from './alerts_client'; export { getDataStreamAdapter } from './alerts_service/lib/data_stream_adapter'; export const plugin = async (initContext: PluginInitializerContext) => { diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.test.ts b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.test.ts index 968a7fbadac0b..8898b8634b293 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.test.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.test.ts @@ -6,6 +6,7 @@ */ import { left, right } from 'fp-ts/lib/Either'; +import { errors } from '@elastic/elasticsearch'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { RuleDataClient, RuleDataClientConstructorOptions, WaitResult } from './rule_data_client'; @@ -382,6 +383,128 @@ describe('RuleDataClient', () => { expect(ruleDataClient.isWriteEnabled()).toBe(true); }); + test('sanitizes error before logging', async () => { + scopedClusterClient.bulk.mockResponseOnce({ + took: 486, + errors: true, + items: [ + { + create: { + _index: 'test', + _id: '3', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + { + create: { + _index: 'test', + _id: '4', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + { + create: { + _index: 'index1', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'. Preview of field's value: 'we don't want this field value to be echoed'", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, + ], + }); + const ruleDataClient = new RuleDataClient( + getRuleDataClientOptions({ isUsingDataStreams }) + ); + expect(ruleDataClient.isWriteEnabled()).toBe(true); + const writer = await ruleDataClient.getWriter(); + + // Previously, a delay between calling getWriter() and using a writer function + // would cause an Unhandled promise rejection if there were any errors getting a writer + // Adding this delay in the tests to ensure this does not pop up again. + await delay(); + + const bulkWriteResponse = await writer.bulk({}); + expect(bulkWriteResponse).toEqual({ + body: { + took: 486, + errors: true, + items: [ + { + create: { + _index: 'test', + _id: '3', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + { + create: { + _index: 'test', + _id: '4', + _version: 1, + result: 'created', + _shards: { total: 2, successful: 1, failed: 0 }, + status: 201, + _seq_no: 2, + _primary_term: 3, + }, + }, + { + create: { + _index: 'index1', + _id: '7', + status: 404, + error: { + type: 'mapper_parsing_exception', + reason: + "failed to parse field [process.command_line] of type [wildcard] in document with id 'f0c9805be95fedbc3c99c663f7f02cc15826c122'.", + caused_by: { + type: 'illegal_state_exception', + reason: "Can't get text on a START_OBJECT at 1:3845", + }, + }, + }, + }, + ], + }, + headers: { + 'x-elastic-product': 'Elasticsearch', + }, + meta: {}, + statusCode: 200, + warnings: [], + }); + + expect(logger.error).toHaveBeenNthCalledWith( + 1, + // @ts-expect-error + new errors.ResponseError(bulkWriteResponse) + ); + expect(ruleDataClient.isWriteEnabled()).toBe(true); + }); + test('waits until cluster client is ready before calling bulk', async () => { scopedClusterClient.bulk.mockResolvedValueOnce( elasticsearchClientMock.createSuccessTransportRequestPromise( diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts index b4d029f4bbe82..329c060426093 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { errors } from '@elastic/elasticsearch'; +import { errors, TransportResult } from '@elastic/elasticsearch'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { Either, isLeft } from 'fp-ts/lib/Either'; @@ -14,6 +14,7 @@ import { Logger } from '@kbn/core/server'; import { IndexPatternsFetcher } from '@kbn/data-plugin/server'; import type { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; +import { sanitizeBulkErrorResponse } from '@kbn/alerting-plugin/server'; import { RuleDataWriteDisabledError, RuleDataWriterInitializationError, @@ -231,13 +232,20 @@ export class RuleDataClient implements IRuleDataClient { meta: true, }); + if (!response.body.errors) { + return response; + } + // TODO: #160572 - add support for version conflict errors, in case alert was updated // some other way between the time it was fetched and the time it was updated. - if (response.body.errors) { - const error = new errors.ResponseError(response); - this.options.logger.error(error); - } - return response; + // Redact part of reason message that echoes back value + const sanitizedResponse = sanitizeBulkErrorResponse(response) as TransportResult< + estypes.BulkResponse, + unknown + >; + const error = new errors.ResponseError(sanitizedResponse); + this.options.logger.error(error); + return sanitizedResponse; } else { this.options.logger.debug(`Writing is disabled, bulk() will not write any data.`); } From 1823d94240abadd456c145329ea83410ec485e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20S=C3=A1nchez?= Date: Tue, 28 Nov 2023 17:27:25 +0100 Subject: [PATCH 17/30] [Security Solution][Endpoint] New enroll endpoint host function CI specific for Cypress tests to use cached agent files (#171399) ## Summary In order to avoid downloading the elastic agent installer file on each Cypress test, we have introduced a new method CI specific that will cache elastic agent files and reuse it across all tests. Old code about `if CI` conditions will be removed in a follow up pr. It also introduces a CLI script to download a specific version of elastic agent using the existing methods in place. --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .buildkite/scripts/packer_cache.sh | 4 + .../disabled/uninstall_agent_from_host.cy.ts | 3 +- .../enabled/uninstall_agent_from_host.cy.ts | 3 +- ...ging_policy_from_enabled_to_disabled.cy.ts | 3 +- .../create_and_enroll_endpoint_host_ci.ts | 132 ++++++++++++++++++ .../cypress/support/data_loaders.ts | 78 ++++++----- .../scripts/endpoint/agent_downloader.js | 9 ++ .../agent_downloader_cli/agent_downloader.ts | 32 +++++ .../endpoint/agent_downloader_cli/index.ts | 31 ++++ .../common/agent_downloads_service.ts | 40 +++++- .../scripts/endpoint/common/fleet_services.ts | 20 ++- .../endpoint/common/vagrant/Vagrantfile | 4 +- .../scripts/endpoint/common/vm_services.ts | 1 + 13 files changed, 307 insertions(+), 53 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/agent_downloader.js create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/index.ts diff --git a/.buildkite/scripts/packer_cache.sh b/.buildkite/scripts/packer_cache.sh index 727accaeead8f..66d663180ec3c 100755 --- a/.buildkite/scripts/packer_cache.sh +++ b/.buildkite/scripts/packer_cache.sh @@ -14,5 +14,9 @@ for version in $(cat versions.json | jq -r '.versions[].version'); do node scripts/es snapshot --download-only --base-path "$ES_CACHE_DIR" --version "$version" done +for version in $(cat versions.json | jq -r '.versions[].version'); do + node x-pack/plugins/security_solution/scripts/endpoint/agent_downloader --version "$version" +done + echo "--- Cloning repos for docs build" node scripts/validate_next_docs --clone-only diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts index 34aba3fcfccf2..ed47855ac894a 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts @@ -21,8 +21,7 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -// FLAKY: https://github.com/elastic/kibana/issues/170667 -describe.skip( +describe( 'Uninstall agent from host when agent tamper protection is disabled', { tags: ['@ess'] }, () => { diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts index 8f45e3d70b5e6..527566bed608b 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts @@ -22,8 +22,7 @@ import { login } from '../../../tasks/login'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -// FLAKY: https://github.com/elastic/kibana/issues/170601 -describe.skip( +describe( 'Uninstall agent from host when agent tamper protection is enabled', { tags: ['@ess'] }, () => { diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts index f5665d830eb4a..0768c4a49ca39 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts @@ -23,8 +23,7 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -// FLAKY: https://github.com/elastic/kibana/issues/170604 -describe.skip( +describe( 'Uninstall agent from host changing agent policy when agent tamper protection is enabled but then is switched to a policy with it disabled', { tags: ['@ess'] }, () => { diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts b/x-pack/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts new file mode 100644 index 0000000000000..df3a5cf6d38a0 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kibanaPackageJson } from '@kbn/repo-info'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { KbnClient } from '@kbn/test/src/kbn_client'; +import { isFleetServerRunning } from '../../../../scripts/endpoint/common/fleet_server/fleet_server_services'; +import type { HostVm } from '../../../../scripts/endpoint/common/types'; +import type { BaseVmCreateOptions } from '../../../../scripts/endpoint/common/vm_services'; +import { createVm } from '../../../../scripts/endpoint/common/vm_services'; +import { + fetchAgentPolicyEnrollmentKey, + fetchFleetServerUrl, + getAgentDownloadUrl, + getAgentFileName, + getOrCreateDefaultAgentPolicy, + waitForHostToEnroll, +} from '../../../../scripts/endpoint/common/fleet_services'; +import type { DownloadedAgentInfo } from '../../../../scripts/endpoint/common/agent_downloads_service'; +import { + downloadAndStoreAgent, + isAgentDownloadFromDiskAvailable, +} from '../../../../scripts/endpoint/common/agent_downloads_service'; + +export interface CreateAndEnrollEndpointHostCIOptions + extends Pick { + kbnClient: KbnClient; + log: ToolingLog; + /** The fleet Agent Policy ID to use for enrolling the agent */ + agentPolicyId: string; + /** version of the Agent to install. Defaults to stack version */ + version?: string; + /** The name for the host. Will also be the name of the VM */ + hostname?: string; + /** If `version` should be exact, or if this is `true`, then the closest version will be used. Defaults to `false` */ + useClosestVersionMatch?: boolean; +} + +export interface CreateAndEnrollEndpointHostCIResponse { + hostname: string; + agentId: string; + hostVm: HostVm; +} + +/** + * Creates a new virtual machine (host) and enrolls that with Fleet + */ +export const createAndEnrollEndpointHostCI = async ({ + kbnClient, + log, + agentPolicyId, + cpus, + disk, + memory, + hostname, + version = kibanaPackageJson.version, + useClosestVersionMatch = true, +}: CreateAndEnrollEndpointHostCIOptions): Promise => { + const vmName = hostname ?? `test-host-${Math.random().toString().substring(2, 6)}`; + + const fileNameNoExtension = getAgentFileName(version); + const agentFileName = `${fileNameNoExtension}.tar.gz`; + let agentDownload: DownloadedAgentInfo | undefined; + + // Check if agent file is already on disk before downloading it again + agentDownload = isAgentDownloadFromDiskAvailable(agentFileName); + + // If it has not been already downloaded, it should be downloaded. + if (!agentDownload) { + log.warning( + `There is no agent installer for ${agentFileName} present on disk, trying to download it now.` + ); + const { url: agentUrl } = await getAgentDownloadUrl(version, useClosestVersionMatch, log); + agentDownload = await downloadAndStoreAgent(agentUrl, agentFileName); + } + + const hostVm = await createVm({ + type: 'vagrant', + name: vmName, + log, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + agentDownload: agentDownload!, + disk, + cpus, + memory, + }); + + if (!(await isFleetServerRunning(kbnClient))) { + throw new Error(`Fleet server does not seem to be running on this instance of kibana!`); + } + + const policyId = agentPolicyId || (await getOrCreateDefaultAgentPolicy({ kbnClient, log })).id; + const [fleetServerUrl, enrollmentToken] = await Promise.all([ + fetchFleetServerUrl(kbnClient), + fetchAgentPolicyEnrollmentKey(kbnClient, policyId), + ]); + + const agentEnrollCommand = [ + 'sudo', + + `./${fileNameNoExtension}/elastic-agent`, + + 'install', + + '--insecure', + + '--force', + + '--url', + fleetServerUrl, + + '--enrollment-token', + enrollmentToken, + ].join(' '); + + log.info(`Enrolling Elastic Agent with Fleet`); + log.verbose('Enrollment command:', agentEnrollCommand); + + await hostVm.exec(agentEnrollCommand); + + const { id: agentId } = await waitForHostToEnroll(kbnClient, log, hostVm.name, 240000); + + return { + hostname: hostVm.name, + agentId, + hostVm, + }; +}; diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts index 99ea877053c91..299d210a65089 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts @@ -26,10 +26,7 @@ import { import type { DeleteAllEndpointDataResponse } from '../../../../scripts/endpoint/common/delete_all_endpoint_data'; import { deleteAllEndpointData } from '../../../../scripts/endpoint/common/delete_all_endpoint_data'; import { waitForEndpointToStreamData } from '../../../../scripts/endpoint/common/endpoint_metadata_services'; -import type { - CreateAndEnrollEndpointHostOptions, - CreateAndEnrollEndpointHostResponse, -} from '../../../../scripts/endpoint/common/endpoint_host_services'; +import type { CreateAndEnrollEndpointHostResponse } from '../../../../scripts/endpoint/common/endpoint_host_services'; import { createAndEnrollEndpointHost, destroyEndpointHost, @@ -66,6 +63,11 @@ import { indexFleetEndpointPolicy, } from '../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy'; import { cyLoadEndpointDataHandler } from './plugin_handlers/endpoint_data_loader'; +import type { + CreateAndEnrollEndpointHostCIOptions, + CreateAndEnrollEndpointHostCIResponse, +} from './create_and_enroll_endpoint_host_ci'; +import { createAndEnrollEndpointHostCI } from './create_and_enroll_endpoint_host_ci'; /** * Test Role/User loader for cypress. Checks to see if running in serverless and handles it as appropriate @@ -290,40 +292,48 @@ export const dataLoadersForRealEndpoints = ( on('task', { createEndpointHost: async ( - options: Omit - ): Promise => { + options: Omit + ): Promise => { const { kbnClient, log } = await stackServicesPromise; let retryAttempt = 0; - const attemptCreateEndpointHost = async (): Promise => { - try { - log.info(`Creating endpoint host, attempt ${retryAttempt}`); - const newHost = await createAndEnrollEndpointHost({ - useClosestVersionMatch: true, - ...options, - log, - kbnClient, - }); - await waitForEndpointToStreamData(kbnClient, newHost.agentId, 360000); - return newHost; - } catch (err) { - log.info(`Caught error when setting up the agent: ${err}`); - if (retryAttempt === 0 && err.agentId) { - retryAttempt++; - await destroyEndpointHost(kbnClient, { - hostname: err.hostname || '', // No hostname in CI env for vagrant - agentId: err.agentId, - }); - log.info(`Deleted endpoint host ${err.agentId} and retrying`); - return attemptCreateEndpointHost(); - } else { - log.info( - `${retryAttempt} attempts of creating endpoint host failed, reason for the last failure was ${err}` - ); - throw err; + const attemptCreateEndpointHost = + async (): Promise => { + try { + log.info(`Creating endpoint host, attempt ${retryAttempt}`); + const newHost = process.env.CI + ? await createAndEnrollEndpointHostCI({ + useClosestVersionMatch: true, + ...options, + log, + kbnClient, + }) + : await createAndEnrollEndpointHost({ + useClosestVersionMatch: true, + ...options, + log, + kbnClient, + }); + await waitForEndpointToStreamData(kbnClient, newHost.agentId, 360000); + return newHost; + } catch (err) { + log.info(`Caught error when setting up the agent: ${err}`); + if (retryAttempt === 0 && err.agentId) { + retryAttempt++; + await destroyEndpointHost(kbnClient, { + hostname: err.hostname || '', // No hostname in CI env for vagrant + agentId: err.agentId, + }); + log.info(`Deleted endpoint host ${err.agentId} and retrying`); + return attemptCreateEndpointHost(); + } else { + log.info( + `${retryAttempt} attempts of creating endpoint host failed, reason for the last failure was ${err}` + ); + throw err; + } } - } - }; + }; return attemptCreateEndpointHost(); }, diff --git a/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader.js b/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader.js new file mode 100644 index 0000000000000..050be8e0cf071 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader.js @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +require('../../../../../src/setup_node_env'); +require('./agent_downloader_cli').cli(); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts b/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts new file mode 100644 index 0000000000000..ab1da6a3f208f --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ok } from 'assert'; +import type { RunFn } from '@kbn/dev-cli-runner'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { getAgentDownloadUrl, getAgentFileName } from '../common/fleet_services'; +import { downloadAndStoreAgent } from '../common/agent_downloads_service'; + +const downloadAndStoreElasticAgent = async ( + version: string, + closestMatch: boolean, + log: ToolingLog +) => { + const downloadUrlResponse = await getAgentDownloadUrl(version, closestMatch, log); + const fileNameNoExtension = getAgentFileName(version); + const agentFile = `${fileNameNoExtension}.tar.gz`; + await downloadAndStoreAgent(downloadUrlResponse.url, agentFile); +}; + +export const agentDownloaderRunner: RunFn = async (cliContext) => { + ok(cliContext.flags.version, 'version argument is required'); + await downloadAndStoreElasticAgent( + cliContext.flags.version as string, + cliContext.flags.closestMatch as boolean, + cliContext.log + ); +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/index.ts b/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/index.ts new file mode 100644 index 0000000000000..7cc6988b63de8 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/agent_downloader_cli/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { run } from '@kbn/dev-cli-runner'; +import { agentDownloaderRunner } from './agent_downloader'; + +export const cli = () => { + run( + agentDownloaderRunner, + + // Options + { + description: `Elastic Agent downloader`, + flags: { + string: ['version'], + boolean: ['closestMatch'], + default: { + closestMatch: true, + }, + help: ` + --version Required. Elastic agent version to be downloaded. + --closestMatch Optional. Use closest elastic agent version to match with. + `, + }, + } + ); +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts index 34f473a854460..410ddd65cf842 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts @@ -64,8 +64,10 @@ class AgentDownloadStorage extends SettingsStorage } } - public getPathsForUrl(agentDownloadUrl: string): DownloadedAgentInfo { - const filename = agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#'); + public getPathsForUrl(agentDownloadUrl: string, agentFileName?: string): DownloadedAgentInfo { + const filename = agentFileName + ? agentFileName + : agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#'); const directory = this.downloadsDirFullPath; const fullFilePath = this.buildPath(join(this.downloadsDirName, filename)); @@ -76,14 +78,17 @@ class AgentDownloadStorage extends SettingsStorage }; } - public async downloadAndStore(agentDownloadUrl: string): Promise { + public async downloadAndStore( + agentDownloadUrl: string, + agentFileName?: string + ): Promise { this.log.debug(`Downloading and storing: ${agentDownloadUrl}`); // TODO: should we add "retry" attempts to file downloads? await this.ensureExists(); - const newDownloadInfo = this.getPathsForUrl(agentDownloadUrl); + const newDownloadInfo = this.getPathsForUrl(agentDownloadUrl, agentFileName); // If download is already present on disk, then just return that info. No need to re-download it if (fs.existsSync(newDownloadInfo.fullFilePath)) { @@ -154,6 +159,18 @@ class AgentDownloadStorage extends SettingsStorage return response; } + + public isAgentDownloadFromDiskAvailable(filename: string): DownloadedAgentInfo | undefined { + if (fs.existsSync(join(this.downloadsDirFullPath, filename))) { + return { + filename, + /** The local directory where downloads are stored */ + directory: this.downloadsDirFullPath, + /** The full local file path and name */ + fullFilePath: join(this.downloadsDirFullPath, filename), + }; + } + } } const handleProcessInterruptions = async ( @@ -203,11 +220,16 @@ export interface DownloadAndStoreAgentResponse extends DownloadedAgentInfo { * already exists on disk, then no download is actually done - the information about the cached * version is returned instead * @param agentDownloadUrl + * @param agentFileName */ export const downloadAndStoreAgent = async ( - agentDownloadUrl: string + agentDownloadUrl: string, + agentFileName?: string ): Promise => { - const downloadedAgent = await agentDownloadsClient.downloadAndStore(agentDownloadUrl); + const downloadedAgent = await agentDownloadsClient.downloadAndStore( + agentDownloadUrl, + agentFileName + ); return { url: agentDownloadUrl, @@ -221,3 +243,9 @@ export const downloadAndStoreAgent = async ( export const cleanupDownloads = async (): ReturnType => { return agentDownloadsClient.cleanupDownloads(); }; + +export const isAgentDownloadFromDiskAvailable = ( + fileName: string +): DownloadedAgentInfo | undefined => { + return agentDownloadsClient.isAgentDownloadFromDiskAvailable(fileName); +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 26ca9d6474393..6477bfc3bebeb 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -373,6 +373,16 @@ export const getAgentVersionMatchingCurrentStack = async ( return version; }; +// Generates a file name using system arch and an agent version. +export const getAgentFileName = (agentVersion: string): string => { + const downloadArch = + { arm64: 'arm64', x64: 'x86_64' }[process.arch as string] ?? + `UNSUPPORTED_ARCHITECTURE_${process.arch}`; + const fileName = `elastic-agent-${agentVersion}-linux-${downloadArch}`; + + return fileName; +}; + interface ElasticArtifactSearchResponse { manifest: { 'last-update-time': string; @@ -414,11 +424,9 @@ export const getAgentDownloadUrl = async ( log?: ToolingLog ): Promise => { const agentVersion = closestMatch ? await getLatestAgentDownloadVersion(version, log) : version; - const downloadArch = - { arm64: 'arm64', x64: 'x86_64' }[process.arch as string] ?? - `UNSUPPORTED_ARCHITECTURE_${process.arch}`; - const fileNameNoExtension = `elastic-agent-${agentVersion}-linux-${downloadArch}`; - const agentFile = `${fileNameNoExtension}.tar.gz`; + + const fileNameWithoutExtension = getAgentFileName(agentVersion); + const agentFile = `${fileNameWithoutExtension}.tar.gz`; const artifactSearchUrl = `https://artifacts-api.elastic.co/v1/search/${agentVersion}/${agentFile}`; log?.verbose(`Retrieving elastic agent download URL from:\n ${artifactSearchUrl}`); @@ -444,7 +452,7 @@ export const getAgentDownloadUrl = async ( return { url: searchResult.packages[agentFile].url, fileName: agentFile, - dirName: fileNameNoExtension, + dirName: fileNameWithoutExtension, }; }; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile b/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile index fd33efaeab625..b058d17ad0764 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile @@ -5,6 +5,7 @@ hostname = ENV["VMNAME"] || 'ubuntu' cachedAgentSource = ENV["CACHED_AGENT_SOURCE"] || '' cachedAgentFilename = ENV["CACHED_AGENT_FILENAME"] || '' +agentDestinationFolder = ENV["AGENT_DESTINATION_FOLDER"] || '' Vagrant.configure("2") do |config| config.vm.hostname = hostname @@ -29,6 +30,7 @@ Vagrant.configure("2") do |config| end config.vm.provision "file", source: cachedAgentSource, destination: "~/#{cachedAgentFilename}" - config.vm.provision "shell", inline: "tar -zxf #{cachedAgentFilename} && rm -f #{cachedAgentFilename}" + config.vm.provision "shell", inline: "mkdir #{agentDestinationFolder}" + config.vm.provision "shell", inline: "tar -zxf #{cachedAgentFilename} --directory #{agentDestinationFolder} --strip-components=1 && rm -f #{cachedAgentFilename}" config.vm.provision "shell", inline: "sudo apt-get install unzip" end diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts index a0efd908f80d7..17c74b1bf6fc3 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -249,6 +249,7 @@ const createVagrantVm = async ({ VMNAME: name, CACHED_AGENT_SOURCE: agentFullFilePath, CACHED_AGENT_FILENAME: agentFileName, + AGENT_DESTINATION_FOLDER: agentFileName.replace('.tar.gz', ''), }, // Only `pipe` STDERR to parent process stdio: ['inherit', 'inherit', 'pipe'], From e64f475a013e1160f0938e14f8da7dcbf97a44bb Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 Nov 2023 11:52:01 -0500 Subject: [PATCH 18/30] [Fleet] Support integration secrets with `required: false` (#172078) ## Summary Support secrets with `required: false` in package manifests. Closes #172061 ## To test 1. Set up an integration in a local package registry with a variable that has `secret: true` and `required: false`, e.g. ```yml - name: secret_token type: password title: (Test) Secret Token description: | Test non-required secret show_user: true secret: true required: false ``` 2. Create a package policy for your test package and note the optional secret is rendered properly 3. Submit the policy editor form without filling out a value for the optional secret 4. Observe the request is successful 5. Edit the package policy and set a value for the optional secret 6. Observe that the secret creation logic works as expected ## Screen recording https://github.com/elastic/kibana/assets/6766512/36e271c5-29d0-49f8-91e8-abc6a7871b20 --- .../package_policy_input_var_field.tsx | 60 +- .../fleet/server/services/secrets.test.ts | 1399 +++++++++-------- .../plugins/fleet/server/services/secrets.ts | 8 +- 3 files changed, 797 insertions(+), 670 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx index c2517c245839d..9321f77886085 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx @@ -338,8 +338,8 @@ const SecretFieldWrapper = ({ children }: { children: React.ReactNode }) => { const SecretFieldLabel = ({ fieldLabel }: { fieldLabel: string }) => { return ( <> - - + + {fieldLabel} @@ -376,12 +376,37 @@ function SecretInputField({ setIsDirty, isDirty, }: InputComponentProps) { - const [editMode, setEditMode] = useState(isEditPage && !value); + const [isReplacing, setIsReplacing] = useState(isEditPage && !value); const valueOnFirstRender = useRef(value); + const hasExistingValue = !!valueOnFirstRender.current; const lowercaseTitle = varDef.title?.toLowerCase(); + const showInactiveReplaceUi = isEditPage && !isReplacing && hasExistingValue; + const valueIsSecretRef = value && value?.isSecretRef; + + const inputComponent = getInputComponent({ + varDef, + value: isReplacing && valueIsSecretRef ? '' : value, + onChange, + frozen, + packageName, + packageType, + datastreams, + isEditPage, + isInvalid, + fieldLabel, + fieldTestSelector, + isDirty, + setIsDirty, + }); + + // If there's no value for this secret, display the input as its "brand new" creation state + // instead of the "replace" state + if (!hasExistingValue) { + return inputComponent; + } - if (isEditPage && !editMode) { + if (showInactiveReplaceUi) { return ( <> @@ -395,7 +420,7 @@ function SecretInputField({ setEditMode(true)} + onClick={() => setIsReplacing(true)} color="primary" iconType="refresh" iconSide="left" @@ -413,28 +438,11 @@ function SecretInputField({ ); } - const valueIsSecretRef = value && value?.isSecretRef; - const field = getInputComponent({ - varDef, - value: editMode && valueIsSecretRef ? '' : value, - onChange, - frozen, - packageName, - packageType, - datastreams, - isEditPage, - isInvalid, - fieldLabel, - fieldTestSelector, - isDirty, - setIsDirty, - }); - - if (editMode) { + if (isReplacing) { const cancelButton = ( { - setEditMode(false); + setIsReplacing(false); setIsDirty(false); onChange(valueOnFirstRender.current); }} @@ -455,12 +463,12 @@ function SecretInputField({ return ( - {field} + {inputComponent} {cancelButton} ); } - return field; + return inputComponent; } diff --git a/x-pack/plugins/fleet/server/services/secrets.test.ts b/x-pack/plugins/fleet/server/services/secrets.test.ts index 66b13638085d3..2a062a5e5a1aa 100644 --- a/x-pack/plugins/fleet/server/services/secrets.test.ts +++ b/x-pack/plugins/fleet/server/services/secrets.test.ts @@ -12,727 +12,732 @@ * 2[0]. */ +import { v4 as uuidv4 } from 'uuid'; + +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; + +import { createAppContextStartContractMock } from '../mocks'; + import type { NewPackagePolicy, PackageInfo } from '../types'; -import { getPolicySecretPaths, diffSecretPaths } from './secrets'; +import { appContextService } from './app_context'; -describe('getPolicySecretPaths', () => { - describe('integration package with one policy template', () => { - const mockIntegrationPackage = { - name: 'mock-package', - title: 'Mock package', - version: '0[0].0', - description: 'description', - type: 'integration', - status: 'not_installed', - vars: [ - { name: 'pkg-secret-1', type: 'text', secret: true }, - { name: 'pkg-secret-2', type: 'text', secret: true }, - ], - data_streams: [ - { - dataset: 'somedataset', - streams: [ +import { getPolicySecretPaths, diffSecretPaths, extractAndWriteSecrets } from './secrets'; + +describe('secrets', () => { + let mockContract: ReturnType; + + beforeEach(async () => { + // prevents `Logger not set.` and other appContext errors + mockContract = createAppContextStartContractMock(); + appContextService.start(mockContract); + }); + + describe('getPolicySecretPaths', () => { + describe('integration package with one policy template', () => { + const mockIntegrationPackage = { + name: 'mock-package', + title: 'Mock package', + version: '0[0].0', + description: 'description', + type: 'integration', + status: 'not_installed', + vars: [ + { name: 'pkg-secret-1', type: 'text', secret: true }, + { name: 'pkg-secret-2', type: 'text', secret: true }, + ], + data_streams: [ + { + dataset: 'somedataset', + streams: [ + { + input: 'foo', + title: 'Foo', + vars: [ + { name: 'stream-secret-1', type: 'text', secret: true }, + { name: 'stream-secret-2', type: 'text', secret: true }, + ], + }, + ], + }, + ], + policy_templates: [ + { + name: 'pkgPolicy1', + title: 'Package policy 1', + description: 'test package policy', + inputs: [ + { + type: 'foo', + title: 'Foo', + vars: [ + { default: 'foo-input-var-value', name: 'foo-input-var-name', type: 'text' }, + { + name: 'input-secret-1', + type: 'text', + secret: true, + }, + { + name: 'input-secret-2', + type: 'text', + secret: true, + }, + { name: 'foo-input3-var-name', type: 'text', multi: true }, + ], + }, + ], + }, + ], + } as unknown as PackageInfo; + it('policy with package level secret vars', () => { + const packagePolicy = { + vars: { + 'pkg-secret-1': { + value: 'pkg-secret-1-val', + }, + 'pkg-secret-2': { + value: 'pkg-secret-2-val', + }, + }, + inputs: [], + } as unknown as NewPackagePolicy; + + expect(getPolicySecretPaths(packagePolicy, mockIntegrationPackage)).toEqual([ + { + path: 'vars.pkg-secret-1', + value: { + value: 'pkg-secret-1-val', + }, + }, + { + path: 'vars.pkg-secret-2', + value: { + value: 'pkg-secret-2-val', + }, + }, + ]); + }); + it('policy with package level secret vars and only one set', () => { + const packagePolicy = { + vars: { + 'pkg-secret-1': { + value: 'pkg-secret-1-val', + }, + }, + inputs: [], + } as unknown as NewPackagePolicy; + + expect(getPolicySecretPaths(packagePolicy, mockIntegrationPackage)).toEqual([ + { + path: 'vars.pkg-secret-1', + value: { + value: 'pkg-secret-1-val', + }, + }, + ]); + }); + it('policy with input level secret vars', () => { + const packagePolicy = { + inputs: [ { - input: 'foo', - title: 'Foo', - vars: [ - { name: 'stream-secret-1', type: 'text', secret: true }, - { name: 'stream-secret-2', type: 'text', secret: true }, - ], + type: 'foo', + policy_template: 'pkgPolicy1', + vars: { + 'input-secret-1': { + value: 'input-secret-1-val', + }, + 'input-secret-2': { + value: 'input-secret-2-val', + }, + }, + streams: [], }, ], - }, - ], - policy_templates: [ - { - name: 'pkgPolicy1', - title: 'Package policy 1', - description: 'test package policy', + } as unknown as NewPackagePolicy; + + expect(getPolicySecretPaths(packagePolicy, mockIntegrationPackage)).toEqual([ + { + path: 'inputs[0].vars.input-secret-1', + value: { value: 'input-secret-1-val' }, + }, + { + path: 'inputs[0].vars.input-secret-2', + value: { value: 'input-secret-2-val' }, + }, + ]); + }); + it('stream level secret vars', () => { + const packagePolicy = { inputs: [ { type: 'foo', - title: 'Foo', - vars: [ - { default: 'foo-input-var-value', name: 'foo-input-var-name', type: 'text' }, + policy_template: 'pkgPolicy1', + streams: [ { - name: 'input-secret-1', - type: 'text', - secret: true, - }, - { - name: 'input-secret-2', - type: 'text', - secret: true, + data_stream: { + dataset: 'somedataset', + type: 'logs', + }, + vars: { + 'stream-secret-1': { + value: 'stream-secret-1-value', + }, + 'stream-secret-2': { + value: 'stream-secret-2-value', + }, + }, }, - { name: 'foo-input3-var-name', type: 'text', multi: true }, ], }, ], - }, - ], - } as unknown as PackageInfo; - it('policy with package level secret vars', () => { - const packagePolicy = { - vars: { - 'pkg-secret-1': { - value: 'pkg-secret-1-val', - }, - 'pkg-secret-2': { - value: 'pkg-secret-2-val', - }, - }, - inputs: [], - } as unknown as NewPackagePolicy; + } as unknown as NewPackagePolicy; - expect(getPolicySecretPaths(packagePolicy, mockIntegrationPackage)).toEqual([ - { - path: 'vars.pkg-secret-1', - value: { - value: 'pkg-secret-1-val', + expect(getPolicySecretPaths(packagePolicy, mockIntegrationPackage)).toEqual([ + { + path: 'inputs[0].streams[0].vars.stream-secret-1', + value: { value: 'stream-secret-1-value' }, }, - }, - { - path: 'vars.pkg-secret-2', - value: { - value: 'pkg-secret-2-val', + { + path: 'inputs[0].streams[0].vars.stream-secret-2', + value: { value: 'stream-secret-2-value' }, }, - }, - ]); + ]); + }); }); - it('policy with package level secret vars and only one set', () => { - const packagePolicy = { - vars: { - 'pkg-secret-1': { - value: 'pkg-secret-1-val', - }, - }, - inputs: [], - } as unknown as NewPackagePolicy; - expect(getPolicySecretPaths(packagePolicy, mockIntegrationPackage)).toEqual([ - { - path: 'vars.pkg-secret-1', - value: { - value: 'pkg-secret-1-val', + describe('integration package with multiple policy templates (e.g AWS)', () => { + const miniAWsPackage = { + name: 'aws', + title: 'AWS', + version: '0.5.3', + release: 'beta', + description: 'AWS Integration', + type: 'integration', + policy_templates: [ + { + name: 'billing', + title: 'AWS Billing', + description: 'Collect AWS billing metrics', + data_streams: ['billing'], + inputs: [ + { + type: 'aws/metrics', + title: 'Collect billing metrics', + description: 'Collect billing metrics', + input_group: 'metrics', + vars: [ + { + name: 'password', + type: 'text', + secret: true, + }, + ], + }, + ], }, - }, - ]); - }); - it('policy with input level secret vars', () => { - const packagePolicy = { - inputs: [ - { - type: 'foo', - policy_template: 'pkgPolicy1', - vars: { - 'input-secret-1': { - value: 'input-secret-1-val', + { + name: 'cloudtrail', + title: 'AWS Cloudtrail', + description: 'Collect logs from AWS Cloudtrail', + data_streams: ['cloudtrail'], + inputs: [ + { + type: 's3', + title: 'Collect logs from Cloudtrail service', + description: 'Collecting Cloudtrail logs using S3 input', + input_group: 'logs', + vars: [ + { + name: 'password', + type: 'text', + secret: true, + }, + ], }, - 'input-secret-2': { - value: 'input-secret-2-val', + { + type: 'httpjson', + title: 'Collect logs from third-party REST API (experimental)', + description: 'Collect logs from third-party REST API (experimental)', + input_group: 'logs', + vars: [ + { + name: 'password', + type: 'text', + secret: true, + }, + ], }, - }, - streams: [], + ], }, ], - } as unknown as NewPackagePolicy; - - expect(getPolicySecretPaths(packagePolicy, mockIntegrationPackage)).toEqual([ - { - path: 'inputs[0].vars.input-secret-1', - value: { value: 'input-secret-1-val' }, - }, - { - path: 'inputs[0].vars.input-secret-2', - value: { value: 'input-secret-2-val' }, - }, - ]); - }); - it('stream level secret vars', () => { - const packagePolicy = { - inputs: [ + vars: [ + { + name: 'secret_access_key', + type: 'text', + title: 'Secret Access Key', + multi: false, + required: false, + show_user: false, + secret: true, + }, + ], + data_streams: [ { - type: 'foo', - policy_template: 'pkgPolicy1', + type: 'metrics', + dataset: 'aws.billing', + title: 'AWS billing metrics', + release: 'beta', streams: [ { - data_stream: { - dataset: 'somedataset', - type: 'logs', - }, - vars: { - 'stream-secret-1': { - value: 'stream-secret-1-value', + input: 'aws/metrics', + vars: [ + { + name: 'password', + type: 'text', + secret: true, }, - 'stream-secret-2': { - value: 'stream-secret-2-value', + ], + template_path: 'stream.yml.hbs', + title: 'AWS Billing metrics', + description: 'Collect AWS billing metrics', + enabled: true, + }, + ], + package: 'aws', + path: 'billing', + }, + { + type: 'logs', + dataset: 'aws.cloudtrail', + title: 'AWS CloudTrail logs', + release: 'beta', + ingest_pipeline: 'default', + streams: [ + { + input: 's3', + vars: [ + { + name: 'password', + type: 'text', + secret: true, }, - }, + ], + template_path: 's3.yml.hbs', + }, + { + input: 'httpjson', + vars: [ + { + name: 'username', + type: 'text', + title: 'Splunk REST API Username', + multi: false, + required: true, + show_user: true, + }, + { + name: 'password', + type: 'password', + title: 'Splunk REST API Password', + multi: false, + required: true, + show_user: true, + secret: true, + }, + ], + template_path: 'httpjson.yml.hbs', }, ], + package: 'aws', + path: 'cloudtrail', }, ], - } as unknown as NewPackagePolicy; - - expect(getPolicySecretPaths(packagePolicy, mockIntegrationPackage)).toEqual([ - { - path: 'inputs[0].streams[0].vars.stream-secret-1', - value: { value: 'stream-secret-1-value' }, - }, - { - path: 'inputs[0].streams[0].vars.stream-secret-2', - value: { value: 'stream-secret-2-value' }, - }, - ]); - }); - }); - - describe('integration package with multiple policy templates (e.g AWS)', () => { - const miniAWsPackage = { - name: 'aws', - title: 'AWS', - version: '0.5.3', - release: 'beta', - description: 'AWS Integration', - type: 'integration', - policy_templates: [ - { - name: 'billing', - title: 'AWS Billing', - description: 'Collect AWS billing metrics', - data_streams: ['billing'], + } as PackageInfo; + it('single policy with package + input + stream level secret var', () => { + const policy = { + vars: { + secret_access_key: { + value: 'my_secret_access_key', + }, + }, inputs: [ { type: 'aws/metrics', - title: 'Collect billing metrics', - description: 'Collect billing metrics', - input_group: 'metrics', - vars: [ + policy_template: 'billing', + enabled: true, + vars: { + password: { value: 'billing_input_password', type: 'text' }, + }, + streams: [ { - name: 'password', - type: 'text', - secret: true, + enabled: true, + data_stream: { type: 'metrics', dataset: 'aws.billing' }, + vars: { + password: { value: 'billing_stream_password', type: 'text' }, + }, }, ], }, ], - }, - { - name: 'cloudtrail', - title: 'AWS Cloudtrail', - description: 'Collect logs from AWS Cloudtrail', - data_streams: ['cloudtrail'], - inputs: [ - { - type: 's3', - title: 'Collect logs from Cloudtrail service', - description: 'Collecting Cloudtrail logs using S3 input', - input_group: 'logs', - vars: [ - { - name: 'password', - type: 'text', - secret: true, - }, - ], + }; + expect( + getPolicySecretPaths( + policy as unknown as NewPackagePolicy, + miniAWsPackage as unknown as PackageInfo + ) + ).toEqual([ + { + path: 'vars.secret_access_key', + value: { + value: 'my_secret_access_key', }, - { - type: 'httpjson', - title: 'Collect logs from third-party REST API (experimental)', - description: 'Collect logs from third-party REST API (experimental)', - input_group: 'logs', - vars: [ - { - name: 'password', - type: 'text', - secret: true, - }, - ], + }, + { + path: 'inputs[0].vars.password', + value: { + type: 'text', + value: 'billing_input_password', }, - ], - }, - ], - vars: [ - { - name: 'secret_access_key', - type: 'text', - title: 'Secret Access Key', - multi: false, - required: false, - show_user: false, - secret: true, - }, - ], - data_streams: [ - { - type: 'metrics', - dataset: 'aws.billing', - title: 'AWS billing metrics', - release: 'beta', - streams: [ - { - input: 'aws/metrics', - vars: [ - { - name: 'password', - type: 'text', - secret: true, - }, - ], - template_path: 'stream.yml.hbs', - title: 'AWS Billing metrics', - description: 'Collect AWS billing metrics', - enabled: true, + }, + { + path: 'inputs[0].streams[0].vars.password', + value: { + type: 'text', + value: 'billing_stream_password', }, - ], - package: 'aws', - path: 'billing', - }, - { - type: 'logs', - dataset: 'aws.cloudtrail', - title: 'AWS CloudTrail logs', - release: 'beta', - ingest_pipeline: 'default', - streams: [ + }, + ]); + }); + it('double policy with package + input + stream level secret var', () => { + const policy = { + vars: { + secret_access_key: { + value: 'my_secret_access_key', + }, + }, + inputs: [ { - input: 's3', - vars: [ + type: 'httpjson', + policy_template: 'cloudtrail', + enabled: false, + vars: { + password: { value: 'cloudtrail_httpjson_input_password' }, + }, + streams: [ { - name: 'password', - type: 'text', - secret: true, + data_stream: { type: 'logs', dataset: 'aws.cloudtrail' }, + vars: { + username: { value: 'hop_dev' }, + password: { value: 'cloudtrail_httpjson_stream_password' }, + }, }, ], - template_path: 's3.yml.hbs', }, { - input: 'httpjson', - vars: [ - { - name: 'username', - type: 'text', - title: 'Splunk REST API Username', - multi: false, - required: true, - show_user: true, - }, + type: 's3', + policy_template: 'cloudtrail', + enabled: true, + vars: { + password: { value: 'cloudtrail_s3_input_password' }, + }, + streams: [ { - name: 'password', - type: 'password', - title: 'Splunk REST API Password', - multi: false, - required: true, - show_user: true, - secret: true, + enabled: true, + data_stream: { type: 'logs', dataset: 'aws.cloudtrail' }, + vars: { + password: { value: 'cloudtrail_s3_stream_password' }, + }, }, ], - template_path: 'httpjson.yml.hbs', }, ], - package: 'aws', - path: 'cloudtrail', - }, - ], - } as PackageInfo; - it('single policy with package + input + stream level secret var', () => { - const policy = { - vars: { - secret_access_key: { - value: 'my_secret_access_key', + }; + + expect( + getPolicySecretPaths( + policy as unknown as NewPackagePolicy, + miniAWsPackage as unknown as PackageInfo + ) + ).toEqual([ + { + path: 'vars.secret_access_key', + value: { + value: 'my_secret_access_key', + }, }, - }, - inputs: [ { - type: 'aws/metrics', - policy_template: 'billing', - enabled: true, - vars: { - password: { value: 'billing_input_password', type: 'text' }, + path: 'inputs[0].vars.password', + value: { + value: 'cloudtrail_httpjson_input_password', }, - streams: [ - { - enabled: true, - data_stream: { type: 'metrics', dataset: 'aws.billing' }, - vars: { - password: { value: 'billing_stream_password', type: 'text' }, - }, - }, - ], }, - ], - }; - expect( - getPolicySecretPaths( - policy as unknown as NewPackagePolicy, - miniAWsPackage as unknown as PackageInfo - ) - ).toEqual([ - { - path: 'vars.secret_access_key', - value: { - value: 'my_secret_access_key', + { + path: 'inputs[0].streams[0].vars.password', + value: { + value: 'cloudtrail_httpjson_stream_password', + }, }, - }, - { - path: 'inputs[0].vars.password', - value: { - type: 'text', - value: 'billing_input_password', + { + path: 'inputs[1].vars.password', + value: { + value: 'cloudtrail_s3_input_password', + }, }, - }, - { - path: 'inputs[0].streams[0].vars.password', - value: { - type: 'text', - value: 'billing_stream_password', + { + path: 'inputs[1].streams[0].vars.password', + value: { + value: 'cloudtrail_s3_stream_password', + }, }, - }, - ]); + ]); + }); }); - it('double policy with package + input + stream level secret var', () => { - const policy = { - vars: { - secret_access_key: { - value: 'my_secret_access_key', - }, - }, - inputs: [ + + describe('input package', () => { + const mockInputPackage = { + name: 'log', + version: '2.0.0', + description: 'Collect custom logs with Elastic Agent.', + title: 'Custom Logs', + format_version: '2.6.0', + owner: { + github: 'elastic/elastic-agent-data-plane', + }, + type: 'input', + categories: ['custom', 'custom_logs'], + conditions: {}, + icons: [], + policy_templates: [ { - type: 'httpjson', - policy_template: 'cloudtrail', - enabled: false, - vars: { - password: { value: 'cloudtrail_httpjson_input_password' }, - }, - streams: [ + name: 'logs', + title: 'Custom log file', + description: 'Collect your custom log files.', + multiple: true, + input: 'logfile', + type: 'logs', + template_path: 'input.yml.hbs', + vars: [ { - data_stream: { type: 'logs', dataset: 'aws.cloudtrail' }, - vars: { - username: { value: 'hop_dev' }, - password: { value: 'cloudtrail_httpjson_stream_password' }, - }, + name: 'paths', + required: true, + title: 'Log file path', + description: 'Path to log files to be collected', + type: 'text', + multi: true, }, - ], - }, - { - type: 's3', - policy_template: 'cloudtrail', - enabled: true, - vars: { - password: { value: 'cloudtrail_s3_input_password' }, - }, - streams: [ { - enabled: true, - data_stream: { type: 'logs', dataset: 'aws.cloudtrail' }, - vars: { - password: { value: 'cloudtrail_s3_stream_password' }, - }, + name: 'data_stream.dataset', + required: true, + title: 'Dataset name', + description: + "Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).\n", + type: 'text', + }, + { + name: 'secret-1', + type: 'text', + secret: true, + }, + { + name: 'secret-2', + type: 'text', + secret: true, }, ], }, ], }; - - expect( - getPolicySecretPaths( - policy as unknown as NewPackagePolicy, - miniAWsPackage as unknown as PackageInfo - ) - ).toEqual([ - { - path: 'vars.secret_access_key', - value: { - value: 'my_secret_access_key', - }, - }, - { - path: 'inputs[0].vars.password', - value: { - value: 'cloudtrail_httpjson_input_password', - }, - }, - { - path: 'inputs[0].streams[0].vars.password', - value: { - value: 'cloudtrail_httpjson_stream_password', - }, - }, - { - path: 'inputs[1].vars.password', - value: { - value: 'cloudtrail_s3_input_password', - }, - }, - { - path: 'inputs[1].streams[0].vars.password', - value: { - value: 'cloudtrail_s3_stream_password', - }, - }, - ]); - }); - }); - - describe('input package', () => { - const mockInputPackage = { - name: 'log', - version: '2.0.0', - description: 'Collect custom logs with Elastic Agent.', - title: 'Custom Logs', - format_version: '2.6.0', - owner: { - github: 'elastic/elastic-agent-data-plane', - }, - type: 'input', - categories: ['custom', 'custom_logs'], - conditions: {}, - icons: [], - policy_templates: [ - { - name: 'logs', - title: 'Custom log file', - description: 'Collect your custom log files.', - multiple: true, - input: 'logfile', - type: 'logs', - template_path: 'input.yml.hbs', - vars: [ - { - name: 'paths', - required: true, - title: 'Log file path', - description: 'Path to log files to be collected', - type: 'text', - multi: true, - }, - { - name: 'data_stream.dataset', - required: true, - title: 'Dataset name', - description: - "Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).\n", - type: 'text', - }, - { - name: 'secret-1', - type: 'text', - secret: true, - }, + it('template level vars', () => { + const policy = { + inputs: [ { - name: 'secret-2', - type: 'text', - secret: true, - }, - ], - }, - ], - }; - it('template level vars', () => { - const policy = { - inputs: [ - { - type: 'logfile', - policy_template: 'logs', - enabled: true, - streams: [ - { - enabled: true, - data_stream: { - type: 'logs', - dataset: 'log.logs', - }, - vars: { - paths: { - value: ['/tmp/test.log'], - }, - 'data_stream.dataset': { - value: 'hello', - }, - 'secret-1': { - value: 'secret-1-value', + type: 'logfile', + policy_template: 'logs', + enabled: true, + streams: [ + { + enabled: true, + data_stream: { + type: 'logs', + dataset: 'log.logs', }, - 'secret-2': { - value: 'secret-2-value', + vars: { + paths: { + value: ['/tmp/test.log'], + }, + 'data_stream.dataset': { + value: 'hello', + }, + 'secret-1': { + value: 'secret-1-value', + }, + 'secret-2': { + value: 'secret-2-value', + }, }, }, - }, - ], - }, - ], - }; + ], + }, + ], + }; - expect( - getPolicySecretPaths( - policy as unknown as NewPackagePolicy, - mockInputPackage as unknown as PackageInfo - ) - ).toEqual([ - { - path: 'inputs[0].streams[0].vars.secret-1', - value: { - value: 'secret-1-value', + expect( + getPolicySecretPaths( + policy as unknown as NewPackagePolicy, + mockInputPackage as unknown as PackageInfo + ) + ).toEqual([ + { + path: 'inputs[0].streams[0].vars.secret-1', + value: { + value: 'secret-1-value', + }, }, - }, - { - path: 'inputs[0].streams[0].vars.secret-2', - value: { - value: 'secret-2-value', + { + path: 'inputs[0].streams[0].vars.secret-2', + value: { + value: 'secret-2-value', + }, }, - }, - ]); + ]); + }); }); }); -}); -describe('diffSecretPaths', () => { - it('should return empty array if no secrets', () => { - expect(diffSecretPaths([], [])).toEqual({ - toCreate: [], - toDelete: [], - noChange: [], + describe('diffSecretPaths', () => { + it('should return empty array if no secrets', () => { + expect(diffSecretPaths([], [])).toEqual({ + toCreate: [], + toDelete: [], + noChange: [], + }); }); - }); - it('should return empty array if single secret not changed', () => { - const paths = [ - { - path: 'somepath', - value: { + it('should return empty array if single secret not changed', () => { + const paths = [ + { + path: 'somepath', value: { - isSecretRef: true, - id: 'secret-1', + value: { + isSecretRef: true, + id: 'secret-1', + }, }, }, - }, - ]; - expect(diffSecretPaths(paths, paths)).toEqual({ - toCreate: [], - toDelete: [], - noChange: paths, + ]; + expect(diffSecretPaths(paths, paths)).toEqual({ + toCreate: [], + toDelete: [], + noChange: paths, + }); }); - }); - it('should return empty array if multiple secrets not changed', () => { - const paths = [ - { - path: 'somepath', - value: { + it('should return empty array if multiple secrets not changed', () => { + const paths = [ + { + path: 'somepath', value: { - isSecretRef: true, - id: 'secret-1', + value: { + isSecretRef: true, + id: 'secret-1', + }, }, }, - }, - { - path: 'somepath2', - value: { + { + path: 'somepath2', value: { - isSecretRef: true, - id: 'secret-2', + value: { + isSecretRef: true, + id: 'secret-2', + }, }, }, - }, - { - path: 'somepath3', - value: { + { + path: 'somepath3', value: { - isSecretRef: true, - id: 'secret-3', + value: { + isSecretRef: true, + id: 'secret-3', + }, }, }, - }, - ]; + ]; - expect(diffSecretPaths(paths, paths.slice().reverse())).toEqual({ - toCreate: [], - toDelete: [], - noChange: paths, + expect(diffSecretPaths(paths, paths.slice().reverse())).toEqual({ + toCreate: [], + toDelete: [], + noChange: paths, + }); }); - }); - it('single secret modified', () => { - const paths1 = [ - { - path: 'somepath1', - value: { + it('single secret modified', () => { + const paths1 = [ + { + path: 'somepath1', value: { - isSecretRef: true, - id: 'secret-1', + value: { + isSecretRef: true, + id: 'secret-1', + }, }, }, - }, - { - path: 'somepath2', - value: { - value: { isSecretRef: true, id: 'secret-2' }, - }, - }, - ]; - - const paths2 = [ - paths1[0], - { - path: 'somepath2', - value: { value: 'newvalue' }, - }, - ]; - - expect(diffSecretPaths(paths1, paths2)).toEqual({ - toCreate: [ { path: 'somepath2', - value: { value: 'newvalue' }, + value: { + value: { isSecretRef: true, id: 'secret-2' }, + }, }, - ], - toDelete: [ + ]; + + const paths2 = [ + paths1[0], { path: 'somepath2', - value: { + value: { value: 'newvalue' }, + }, + ]; + + expect(diffSecretPaths(paths1, paths2)).toEqual({ + toCreate: [ + { + path: 'somepath2', + value: { value: 'newvalue' }, + }, + ], + toDelete: [ + { + path: 'somepath2', value: { - isSecretRef: true, - id: 'secret-2', + value: { + isSecretRef: true, + id: 'secret-2', + }, }, }, - }, - ], - noChange: [paths1[0]], + ], + noChange: [paths1[0]], + }); }); - }); - it('double secret modified', () => { - const paths1 = [ - { - path: 'somepath1', - value: { + it('double secret modified', () => { + const paths1 = [ + { + path: 'somepath1', value: { - isSecretRef: true, - id: 'secret-1', + value: { + isSecretRef: true, + id: 'secret-1', + }, }, }, - }, - { - path: 'somepath2', - value: { + { + path: 'somepath2', value: { - isSecretRef: true, - id: 'secret-2', + value: { + isSecretRef: true, + id: 'secret-2', + }, }, }, - }, - ]; - - const paths2 = [ - { - path: 'somepath1', - value: { value: 'newvalue1' }, - }, - { - path: 'somepath2', - value: { value: 'newvalue2' }, - }, - ]; - - expect(diffSecretPaths(paths1, paths2)).toEqual({ - toCreate: [ + ]; + + const paths2 = [ { path: 'somepath1', value: { value: 'newvalue1' }, @@ -741,8 +746,45 @@ describe('diffSecretPaths', () => { path: 'somepath2', value: { value: 'newvalue2' }, }, - ], - toDelete: [ + ]; + + expect(diffSecretPaths(paths1, paths2)).toEqual({ + toCreate: [ + { + path: 'somepath1', + value: { value: 'newvalue1' }, + }, + { + path: 'somepath2', + value: { value: 'newvalue2' }, + }, + ], + toDelete: [ + { + path: 'somepath1', + value: { + value: { + isSecretRef: true, + id: 'secret-1', + }, + }, + }, + { + path: 'somepath2', + value: { + value: { + isSecretRef: true, + id: 'secret-2', + }, + }, + }, + ], + noChange: [], + }); + }); + + it('single secret added', () => { + const paths1 = [ { path: 'somepath1', value: { @@ -752,50 +794,125 @@ describe('diffSecretPaths', () => { }, }, }, + ]; + + const paths2 = [ + paths1[0], { path: 'somepath2', - value: { - value: { - isSecretRef: true, - id: 'secret-2', - }, - }, + value: { value: 'newvalue' }, }, - ], - noChange: [], + ]; + + expect(diffSecretPaths(paths1, paths2)).toEqual({ + toCreate: [ + { + path: 'somepath2', + value: { value: 'newvalue' }, + }, + ], + toDelete: [], + noChange: [paths1[0]], + }); }); }); - it('single secret added', () => { - const paths1 = [ - { - path: 'somepath1', - value: { - value: { - isSecretRef: true, - id: 'secret-1', - }, + describe('extractAndWriteSecrets', () => { + const esClientMock = elasticsearchServiceMock.createInternalClient(); + + esClientMock.transport.request.mockImplementation(async (req) => { + return { + id: uuidv4(), + }; + }); + + beforeEach(() => { + esClientMock.transport.request.mockClear(); + }); + + const mockIntegrationPackage = { + name: 'mock-package', + title: 'Mock package', + version: '0.0.0', + description: 'description', + type: 'integration', + status: 'not_installed', + vars: [ + { name: 'pkg-secret-1', type: 'text', secret: true, required: true }, + { name: 'pkg-secret-2', type: 'text', secret: true, required: false }, + ], + data_streams: [ + { + dataset: 'somedataset', + streams: [ + { + input: 'foo', + title: 'Foo', + }, + ], }, - }, - ]; - - const paths2 = [ - paths1[0], - { - path: 'somepath2', - value: { value: 'newvalue' }, - }, - ]; - - expect(diffSecretPaths(paths1, paths2)).toEqual({ - toCreate: [ + ], + policy_templates: [ { - path: 'somepath2', - value: { value: 'newvalue' }, + name: 'pkgPolicy1', + title: 'Package policy 1', + description: 'test package policy', + inputs: [ + { + type: 'foo', + title: 'Foo', + vars: [], + }, + ], }, ], - toDelete: [], - noChange: [paths1[0]], + } as unknown as PackageInfo; + + describe('when only required secret value is provided', () => { + it('returns single secret reference for required secret', async () => { + const mockPackagePolicy = { + vars: { + 'pkg-secret-1': { + value: 'pkg-secret-1-val', + }, + }, + inputs: [], + } as unknown as NewPackagePolicy; + + const result = await extractAndWriteSecrets({ + packagePolicy: mockPackagePolicy, + packageInfo: mockIntegrationPackage, + esClient: esClientMock, + }); + + expect(esClientMock.transport.request).toHaveBeenCalledTimes(1); + expect(result.secretReferences).toHaveLength(1); + }); + }); + + describe('when both required and optional secret values are provided', () => { + it('returns secret reference for both required and optional secret', async () => { + const mockPackagePolicy = { + vars: { + 'pkg-secret-1': { + value: 'pkg-secret-1-val', + }, + 'pkg-secret-2': { + value: 'pkg-secret-2-val', + }, + }, + inputs: [], + } as unknown as NewPackagePolicy; + + const result = await extractAndWriteSecrets({ + packagePolicy: mockPackagePolicy, + packageInfo: mockIntegrationPackage, + esClient: esClientMock, + }); + + expect(esClientMock.transport.request).toHaveBeenCalledTimes(2); + expect(result.secretReferences).toHaveLength(2); + }); }); }); }); diff --git a/x-pack/plugins/fleet/server/services/secrets.ts b/x-pack/plugins/fleet/server/services/secrets.ts index 46e166ea4ecf1..24bcabf19b5aa 100644 --- a/x-pack/plugins/fleet/server/services/secrets.ts +++ b/x-pack/plugins/fleet/server/services/secrets.ts @@ -234,13 +234,15 @@ export async function extractAndWriteSecrets(opts: { return { packagePolicy, secretReferences: [] }; } + const secretsToCreate = secretPaths.filter((secretPath) => !!secretPath.value.value); + const secrets = await createSecrets({ esClient, - values: secretPaths.map((secretPath) => secretPath.value.value), + values: secretsToCreate.map((secretPath) => secretPath.value.value), }); const policyWithSecretRefs = JSON.parse(JSON.stringify(packagePolicy)); - secretPaths.forEach((secretPath, i) => { + secretsToCreate.forEach((secretPath, i) => { set(policyWithSecretRefs, secretPath.path + '.value', toVarSecretRef(secrets[i].id)); }); @@ -418,7 +420,7 @@ export async function extractAndUpdateSecrets(opts: { // check if the previous secret is actually a secret refrerence // it may be that secrets were not enabled at the time of creation // in which case they are just stored as plain text - if (secretPath.value.value.isSecretRef) { + if (secretPath.value.value?.isSecretRef) { secretsToDelete.push({ id: secretPath.value.value.id }); } }); From 42253b6aba92454cc8c1d3300b31577953ed2cc1 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Tue, 28 Nov 2023 17:58:22 +0100 Subject: [PATCH 19/30] [Alert As Data] Add match_only_text mapping for reason field (#171969) --- .../src/field_maps/alert_field_map.ts | 8 ++++++++ .../field_maps/mapping_from_field_map.test.ts | 5 +++++ .../assets/field_maps/technical_rule_field_map.test.ts | 7 +++++++ .../tests/trial/__snapshots__/create_rule.snap | 6 ++++++ 4 files changed, 26 insertions(+) diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts index f22e902bbbeaa..8e08439e7450a 100644 --- a/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts @@ -41,6 +41,7 @@ import { EVENT_KIND, TAGS, } from '@kbn/rule-data-utils'; +import { MultiField } from './types'; export const alertFieldMap = { [ALERT_ACTION_GROUP]: { @@ -92,6 +93,13 @@ export const alertFieldMap = { type: 'keyword', array: false, required: false, + multi_fields: [ + { + flat_name: `${ALERT_REASON}.text`, + name: 'text', + type: 'match_only_text', + }, + ] as MultiField[], }, [ALERT_RULE_CATEGORY]: { type: 'keyword', diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts index e58b795863e48..eb3132eba5413 100644 --- a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts @@ -242,6 +242,11 @@ describe('mappingFromFieldMap', () => { }, reason: { type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, }, rule: { properties: { diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts index 4d25b41b6db0e..be24975deae6f 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts @@ -79,6 +79,13 @@ it('matches snapshot', () => { }, "kibana.alert.reason": Object { "array": false, + "multi_fields": Array [ + Object { + "flat_name": "kibana.alert.reason.text", + "name": "text", + "type": "match_only_text", + }, + ], "required": false, "type": "keyword", }, diff --git a/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap b/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap index a66368fe5a8ec..bc4b903dee43e 100644 --- a/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap +++ b/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap @@ -26,6 +26,9 @@ Object { "kibana.alert.reason": Array [ "Failed transactions is 50% in the last 5 mins for service: opbeans-go, env: Not defined, type: request. Alert when > 30%.", ], + "kibana.alert.reason.text": Array [ + "Failed transactions is 50% in the last 5 mins for service: opbeans-go, env: Not defined, type: request. Alert when > 30%.", + ], "kibana.alert.rule.category": Array [ "Failed transaction rate threshold", ], @@ -109,6 +112,9 @@ Object { "kibana.alert.reason": Array [ "Failed transactions is 50% in the last 5 mins for service: opbeans-go, env: Not defined, type: request. Alert when > 30%.", ], + "kibana.alert.reason.text": Array [ + "Failed transactions is 50% in the last 5 mins for service: opbeans-go, env: Not defined, type: request. Alert when > 30%.", + ], "kibana.alert.rule.category": Array [ "Failed transaction rate threshold", ], From 0a7299b0ac2aadfe47aa374002636341a032a6ed Mon Sep 17 00:00:00 2001 From: Jorge Sanz Date: Tue, 28 Nov 2023 17:59:28 +0100 Subject: [PATCH 20/30] [Docs][Maps] Include details about the headers requested and served by EMS (#171659) Fixes #129751 ## Summary Extends the EMS documentation to detail request and response headers the browser sends to EMS resources and includes also a minimal `curl` command to request the response headers for same resource as well. I tried to edit this in a way it does not take the whole page but happy to hear feedback or ideas on how to make this easier to digest. https://github.com/elastic/kibana/assets/188264/27e83a5f-4d01-47a8-af2c-3739576bf56e Also, I am not sure if this is something worth adding to our release notes :thinking: --- docs/maps/connect-to-ems.asciidoc | 47 +++++++++- docs/maps/headers/file-data.asciidoc | 131 ++++++++++++++++++++++++++ docs/maps/headers/file-json.asciidoc | 132 +++++++++++++++++++++++++++ docs/maps/headers/tile-json.asciidoc | 120 ++++++++++++++++++++++++ docs/maps/headers/tile-pbf.asciidoc | 121 ++++++++++++++++++++++++ docs/maps/headers/tile-png.asciidoc | 117 ++++++++++++++++++++++++ 6 files changed, 665 insertions(+), 3 deletions(-) create mode 100644 docs/maps/headers/file-data.asciidoc create mode 100644 docs/maps/headers/file-json.asciidoc create mode 100644 docs/maps/headers/tile-json.asciidoc create mode 100644 docs/maps/headers/tile-pbf.asciidoc create mode 100644 docs/maps/headers/tile-png.asciidoc diff --git a/docs/maps/connect-to-ems.asciidoc b/docs/maps/connect-to-ems.asciidoc index 8db34ee3f61bf..8fac6a6d95c15 100644 --- a/docs/maps/connect-to-ems.asciidoc +++ b/docs/maps/connect-to-ems.asciidoc @@ -3,17 +3,58 @@ :ems-docker-repo: docker.elastic.co/elastic-maps-service/elastic-maps-server-ubi8 :ems-docker-image: {ems-docker-repo}:{version} +:ems-headers-url: https://deployment-host https://www.elastic.co/elastic-maps-service[Elastic Maps Service (EMS)] is a service that hosts tile layers and vector shapes of administrative boundaries. If you are using Kibana's out-of-the-box settings, Maps is already configured to use EMS. +[float] +=== Domains + EMS requests are made to the following domains: -* tiles.maps.elastic.co -* vector.maps.elastic.co +* Tile Service: `tiles.maps.elastic.co` +* File Service: `vector.maps.elastic.co` + +[float] +=== Headers + +Find below examples of the request and response headers from Kibana and a minimal `curl` request example showing the response headers sent by each service. + +WARNING: These headers may change without further notice at anytime and are shared for reference. + +[float] +==== EMS Tile Service + +The EMS Tile Service provides basemaps in three different styles as the default background for Maps visualizations. The basemaps use https://www.openstreetmap.org/about[OpenStreetMap] data following the https://openmaptiles.org/[OpenMapTiles] schema and can be explored at https://maps.elastic.co[maps.elastic.co]. + +Headers for the Tile Service JSON manifest describing the basemaps available. + +include::headers/tile-json.asciidoc[] + +Headers for a vector tile asset in _protobuffer_ format from the Tile Service. + +include::headers/tile-pbf.asciidoc[] + +Headers for an sprite image asset from the Tile Service + +include::headers/tile-png.asciidoc[] + + +[float] +==== EMS File Service + +EMS File Service provides the administrative boundaries used for <> as static assets in GeoJSON or TopoJSON formats and can be explored at https://maps.elastic.co[maps.elastic.co]. + +Headers for the File Service JSON manifest that declares all the datasets available. + +include::headers/file-json.asciidoc[] + +Headers for a sample Dataset from the File Service in TopoJSON format. + +include::headers/file-data.asciidoc[] -Maps makes requests directly from the browser to EMS. [float] === Disable Elastic Maps Service diff --git a/docs/maps/headers/file-data.asciidoc b/docs/maps/headers/file-data.asciidoc new file mode 100644 index 0000000000000..62b47ed714d0a --- /dev/null +++ b/docs/maps/headers/file-data.asciidoc @@ -0,0 +1,131 @@ + +++++ +
+
+ + + +
+
+++++ +[%collapsible] +==== +[source,bash,subs="attributes"] +---------------------------------- +curl -I 'https://vector.maps.elastic.co/files/world_countries_v7.topo.json?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version={version}' \ +-H 'User-Agent: curl/7.81.0' \ +-H 'Accept: */*' \ +-H 'Accept-Encoding: gzip, deflate, br' +---------------------------------- + +Server response + +[source,regex] +---------------------------------- +HTTP/2 200 +x-guploader-uploadid: ABPtcPpmMffchVgfHIr-SSC00WORo145oV-1q0asjqRvjLV_7cIgyfLRfofXV-BG7huMYABFypblcgdgXRBARhpo2c88ow +x-goog-generation: 1689593325442971 +x-goog-metageneration: 1 +x-goog-stored-content-encoding: gzip +x-goog-stored-content-length: 587241 +content-encoding: gzip +x-goog-hash: crc32c=OcROeg== +x-goog-hash: md5=8KKIwD6wbKa3YYXTnnFcZw== +x-goog-storage-class: MULTI_REGIONAL +accept-ranges: bytes +content-length: 587241 +access-control-allow-origin: * +access-control-expose-headers: Authorization, Content-Length, Content-Type, Date, Server, Transfer-Encoding, X-GUploader-UploadID, X-Google-Trace, accept, elastic-api-version, kbn-name, kbn-version, origin +server: UploadServer +date: Tue, 21 Nov 2023 14:22:16 GMT +expires: Tue, 21 Nov 2023 15:22:16 GMT +cache-control: public, max-age=3600,no-transform +age: 2202 +last-modified: Mon, 17 Jul 2023 11:28:45 GMT +etag: "f0a288c03eb06ca6b76185d39e715c67" +content-type: application/json +alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 +---------------------------------- +==== +++++ +
+ + +
+++++ diff --git a/docs/maps/headers/file-json.asciidoc b/docs/maps/headers/file-json.asciidoc new file mode 100644 index 0000000000000..8e08508da14fc --- /dev/null +++ b/docs/maps/headers/file-json.asciidoc @@ -0,0 +1,132 @@ + +++++ +
+
+ + + +
+
+++++ +[%collapsible] +==== +[source,bash,subs="attributes"] +---------------------------------- +curl -I 'https://vector.maps.elastic.co/v{minor-version}/manifest?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version={version}' \ +-H 'User-Agent: curl/7.81.0' \ +-H 'Accept: */*' \ +-H 'Accept-Encoding: gzip, deflate, br' +---------------------------------- + +Server response + +[source,regex] +---------------------------------- +HTTP/2 200 +x-guploader-uploadid: ABPtcPp_BvMdBDO5jVlutETVHmvpOachwjilw4AkIKwMrOQJ4exR9Eln4g0LkW3V_LLSEpvjYLtUtFmO0Uwr61XXUhoP_A +x-goog-generation: 1689593295246576 +x-goog-metageneration: 1 +x-goog-stored-content-encoding: gzip +x-goog-stored-content-length: 108029 +content-encoding: gzip +x-goog-hash: crc32c=T5gVpw== +x-goog-hash: md5=6F8KWV8VTdx8FsN2iFehow== +x-goog-storage-class: MULTI_REGIONAL +accept-ranges: bytes +content-length: 108029 +access-control-allow-origin: * +access-control-expose-headers: Authorization, Content-Length, Content-Type, Date, Server, Transfer-Encoding, X-GUploader-UploadID, X-Google-Trace, accept, elastic-api-version, kbn-name, kbn-version, origin +server: UploadServer +date: Tue, 21 Nov 2023 14:25:07 GMT +expires: Tue, 21 Nov 2023 15:25:07 GMT +cache-control: public, max-age=3600,no-transform +age: 2170 +last-modified: Mon, 17 Jul 2023 11:28:15 GMT +etag: "e85f0a595f154ddc7c16c3768857a1a3" +content-type: application/json +alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 +---------------------------------- +==== +++++ +
+ + +
+++++ diff --git a/docs/maps/headers/tile-json.asciidoc b/docs/maps/headers/tile-json.asciidoc new file mode 100644 index 0000000000000..b34e82dcab3cd --- /dev/null +++ b/docs/maps/headers/tile-json.asciidoc @@ -0,0 +1,120 @@ + +++++ +
+
+ + + +
+
+++++ +[%collapsible] +==== +[source,bash,subs="attributes"] +---------------------------------- +curl -I 'https://tiles.maps.elastic.co/v{minor-version}/manifest?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version={version}' \ +-H 'User-Agent: curl/7.81.0' \ +-H 'Accept: */*' \ +-H 'Accept-Encoding: gzip, deflate, br' +---------------------------------- + +Server response + +[source,regex] +---------------------------------- +HTTP/2 200 +server: BaseHTTP/0.6 Python/3.11.4 +date: Mon, 20 Nov 2023 15:08:46 GMT +content-type: application/json; charset=utf-8 +elastic-api-version: 2023-10-31 +access-control-allow-origin: * +access-control-allow-methods: GET, OPTIONS, HEAD +access-control-allow-headers: Origin, Accept, Content-Type, kbn-version, elastic-api-version +access-control-expose-headers: etag +content-encoding: gzip +vary: Accept-Encoding +x-varnish: 844076 5416505 +accept-ranges: bytes +varnish-age: 85285 +cache-control: private, max-age=86400 +via: 1.1 varnish (Varnish/7.0), 1.1 google +alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 +---------------------------------- +==== +++++ +
+ + +
+++++ diff --git a/docs/maps/headers/tile-pbf.asciidoc b/docs/maps/headers/tile-pbf.asciidoc new file mode 100644 index 0000000000000..52215a714cda1 --- /dev/null +++ b/docs/maps/headers/tile-pbf.asciidoc @@ -0,0 +1,121 @@ + +++++ +
+
+ + + +
+
+++++ +[%collapsible] +==== +[source,bash,subs="attributes"] +---------------------------------- +$ curl -I 'https://tiles.maps.elastic.co/data/v3/1/1/0.pbf?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version={version}' \ +-H 'User-Agent: curl/7.81.0' \ +-H 'Accept: */*' \ +-H 'Accept-Encoding: gzip, deflate, br' +---------------------------------- + +Server response + +[source,regex] +---------------------------------- +HTTP/2 200 +content-encoding: gzip +content-length: 144075 +access-control-allow-origin: * +access-control-allow-methods: GET, OPTIONS, HEAD +access-control-allow-headers: Origin, Accept, Content-Type, kbn-version, elastic-api-version +access-control-expose-headers: etag +x-varnish: 3269455 5976667 +accept-ranges: bytes +varnish-age: 9045 +via: 1.1 varnish (Varnish/7.0), 1.1 google +date: Mon, 20 Nov 2023 15:08:19 GMT +age: 78827 +last-modified: Thu, 16 Sep 2021 17:14:41 GMT +etag: W/"232cb-zYEfNgd8rzHusLotRFzgRDSDDGA" +content-type: application/x-protobuf +vary: Accept-Encoding +cache-control: public,max-age=3600 +alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 +---------------------------------- +==== +++++ +
+ + +
+++++ diff --git a/docs/maps/headers/tile-png.asciidoc b/docs/maps/headers/tile-png.asciidoc new file mode 100644 index 0000000000000..9d972b1ce8c01 --- /dev/null +++ b/docs/maps/headers/tile-png.asciidoc @@ -0,0 +1,117 @@ + +++++ +
+
+ + + +
+
+++++ +[%collapsible] +==== +[source,bash] +---------------------------------- +curl -I 'https://tiles.maps.elastic.co/styles/osm-bright-desaturated/sprite.png' \ +-H 'User-Agent: curl/7.81.0' \ +-H 'Accept: image/avif,image/webp,*/*' \ +-H 'Accept-Encoding: gzip, deflate, br' +---------------------------------- + +Server response + +[source,regex] +---------------------------------- +HTTP/2 200 +content-length: 17181 +access-control-allow-origin: * +access-control-allow-methods: GET, OPTIONS, HEAD +access-control-allow-headers: Origin, Accept, Content-Type, kbn-version, elastic-api-version +access-control-expose-headers: etag +x-varnish: 8769943 4865354 +accept-ranges: bytes +varnish-age: 250 +via: 1.1 varnish (Varnish/7.0), 1.1 google +date: Tue, 21 Nov 2023 14:44:36 GMT +age: 592 +etag: W/"431d-/dqE/W5Q3FqkHikyDQtCuQqAdlY" +content-type: image/png +cache-control: public,max-age=3600 +alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 +---------------------------------- +==== +++++ +
+ + +
+++++ From fc7f40f68bcff77b369c78ee8211c0afc3e77237 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 28 Nov 2023 17:05:24 +0000 Subject: [PATCH 21/30] skip flaky suite (#172031) --- .../cypress/e2e/investigations/timelines/creation.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/creation.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/creation.cy.ts index 8205ef767356b..de0449a455f6b 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/creation.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/creation.cy.ts @@ -147,7 +147,8 @@ describe('Timelines', (): void => { } ); - describe('shows the different timeline states', () => { + // FLAKY: https://github.com/elastic/kibana/issues/172031 + describe.skip('shows the different timeline states', () => { before(() => { login(); visitWithTimeRange(OVERVIEW_URL); From 6c7dbf7adfc49c84ec90d03dfc9028c0719d43fd Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Tue, 28 Nov 2023 17:07:04 +0000 Subject: [PATCH 22/30] [Entity Analytics] Remove unneccesary test hack (#172052) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Remove a test hack that isn't needed. I _thought_ I had the issue that this test was running in serverless config and causing the security service not to be defined, I am starting to doubt myself 😓 CC @WafaaNasr --- .../risk_engine/risk_engine_privileges.ts | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_engine_privileges.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_engine_privileges.ts index d0cc241b559f5..aa6a604dc0c32 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_engine_privileges.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/risk_engine_privileges.ts @@ -5,7 +5,6 @@ * 2.0. */ import expect from '@kbn/expect'; -import type { SecurityService } from '../../../../../../../test/common/services/security/security'; import { riskEngineRouteHelpersFactoryNoAuth } from '../../utils'; import { FtrProviderContext } from '../../../../ftr_provider_context'; @@ -84,18 +83,7 @@ export default ({ getService }: FtrProviderContext) => { describe('@ess privileges_apis', () => { const supertestWithoutAuth = getService('supertestWithoutAuth'); const riskEngineRoutesNoAuth = riskEngineRouteHelpersFactoryNoAuth(supertestWithoutAuth); - const logger = getService('log'); - let security: SecurityService; - try { - security = getService('security'); - } catch (e) { - // even though this test doesn't have the @serverless tag I cannot get it to stop running - // with serverless config. This is a hack to skip the test if security service is not available - logger.info( - 'Skipping privileges test as security service not available (likely run with serverless config)' - ); - return; - } + const security = getService('security'); const createRole = async ({ name, privileges }: { name: string; privileges: any }) => { return await security.role.create(name, privileges); From 92cd3a66e647ba4edfd8e99971d092f1d6a45aab Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 28 Nov 2023 17:08:06 +0000 Subject: [PATCH 23/30] skip flaky suite (#171643) --- .../serverless/endpoint_list_with_security_essentials.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts index 9cd298535df26..a28d710091eb4 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts @@ -15,7 +15,8 @@ import { visitEndpointList, } from '../../screens'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/171643 +describe.skip( 'When on the Endpoint List in Security Essentials PLI', { tags: ['@serverless'], From 4d1adc1b90b28c3205c58b3e3354224cd0d5699f Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 28 Nov 2023 17:10:33 +0000 Subject: [PATCH 24/30] skip flaky suite (#171641) --- .../cypress/e2e/response_actions/reponse_actions_history.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/reponse_actions_history.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/reponse_actions_history.cy.ts index 4efd03c01d05b..0af8ee43f2988 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/reponse_actions_history.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/reponse_actions_history.cy.ts @@ -10,7 +10,8 @@ import { indexEndpointHosts } from '../../tasks/index_endpoint_hosts'; import { login } from '../../tasks/login'; import { loadPage } from '../../tasks/common'; -describe('Response actions history page', { tags: ['@ess', '@serverless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/171641 +describe.skip('Response actions history page', { tags: ['@ess', '@serverless'] }, () => { let endpointData: ReturnTypeFromChainable; // let actionData: ReturnTypeFromChainable; From 9f5651d3bf9979efa6ff347acddc29b0137edcf4 Mon Sep 17 00:00:00 2001 From: Patrick Mueller Date: Tue, 28 Nov 2023 13:28:56 -0500 Subject: [PATCH 25/30] [Task Manager] allow multiple task claiming strategies (#171677) see https://github.com/elastic/kibana/issues/155770 Make the task manager task claiming algorithm selectable, to allow alternative implementations in the future. No other implementations are provided here, this is setup for adding the next algorithm. Task Manager behavior should not be changed by this PR - code has just be re-org'd. This exposes a new config key which is exposed to Docker - `xpack.task_manager.claim_strategy`. The only allowed value is `default`. No plans at present to document this, or allow-list for the cloud. We may end up changing the config key to just test for serverless instead, when we implement the next task claiming algorithm (see referenced issue ^^^, which is aimed for serverless). The jest tests were coarsely re-org'd. Once we have > 1 algorithm, we'll like want to re-org a bit more, so we can test all the implementations "in a loop". --- .../resources/base/bin/kibana-docker | 1 + .../task_manager/server/config.test.ts | 12 + x-pack/plugins/task_manager/server/config.ts | 9 + .../server/ephemeral_task_lifecycle.test.ts | 1 + .../managed_configuration.test.ts | 1 + .../lib/calculate_health_status.test.ts | 1 + .../server/metrics/create_aggregator.test.ts | 1 + .../configuration_statistics.test.ts | 1 + .../monitoring_stats_stream.test.ts | 1 + .../task_manager/server/plugin.test.ts | 1 + .../server/polling_lifecycle.test.ts | 1 + .../task_manager/server/polling_lifecycle.ts | 4 +- .../server/queries/task_claiming.test.ts | 1252 +--------------- .../server/queries/task_claiming.ts | 279 +--- .../server/task_claimers/README.md | 20 + .../server/task_claimers/index.test.ts | 24 + .../server/task_claimers/index.ts | 48 + .../task_claimers/strategy_default.test.ts | 1318 +++++++++++++++++ .../server/task_claimers/strategy_default.ts | 255 ++++ 19 files changed, 1758 insertions(+), 1472 deletions(-) create mode 100644 x-pack/plugins/task_manager/server/task_claimers/README.md create mode 100644 x-pack/plugins/task_manager/server/task_claimers/index.test.ts create mode 100644 x-pack/plugins/task_manager/server/task_claimers/index.ts create mode 100644 x-pack/plugins/task_manager/server/task_claimers/strategy_default.test.ts create mode 100644 x-pack/plugins/task_manager/server/task_claimers/strategy_default.ts diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker index 05b7133519a81..3aa057ac1bd37 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker @@ -402,6 +402,7 @@ kibana_vars=( xpack.securitySolution.packagerTaskInterval xpack.securitySolution.prebuiltRulesPackageVersion xpack.spaces.maxSpaces + xpack.task_manager.claim_strategy xpack.task_manager.max_attempts xpack.task_manager.max_workers xpack.task_manager.monitored_aggregated_stats_refresh_rate diff --git a/x-pack/plugins/task_manager/server/config.test.ts b/x-pack/plugins/task_manager/server/config.test.ts index c196a334931ba..9d85c216546f8 100644 --- a/x-pack/plugins/task_manager/server/config.test.ts +++ b/x-pack/plugins/task_manager/server/config.test.ts @@ -13,6 +13,7 @@ describe('config validation', () => { expect(configSchema.validate(config)).toMatchInlineSnapshot(` Object { "allow_reading_invalid_state": true, + "claim_strategy": "default", "ephemeral_tasks": Object { "enabled": false, "request_capacity": 10, @@ -72,6 +73,7 @@ describe('config validation', () => { expect(configSchema.validate(config)).toMatchInlineSnapshot(` Object { "allow_reading_invalid_state": true, + "claim_strategy": "default", "ephemeral_tasks": Object { "enabled": false, "request_capacity": 10, @@ -129,6 +131,7 @@ describe('config validation', () => { expect(configSchema.validate(config)).toMatchInlineSnapshot(` Object { "allow_reading_invalid_state": true, + "claim_strategy": "default", "ephemeral_tasks": Object { "enabled": false, "request_capacity": 10, @@ -244,4 +247,13 @@ describe('config validation', () => { configSchema.validate(config); }).not.toThrowError(); }); + + test('the claim strategy is validated', () => { + const config = { claim_strategy: 'invalid-strategy' }; + expect(() => { + configSchema.validate(config); + }).toThrowErrorMatchingInlineSnapshot( + `"The claim strategy is invalid: Unknown task claiming strategy (invalid-strategy)"` + ); + }); }); diff --git a/x-pack/plugins/task_manager/server/config.ts b/x-pack/plugins/task_manager/server/config.ts index 490d25a7bdfb0..3be8b341c939e 100644 --- a/x-pack/plugins/task_manager/server/config.ts +++ b/x-pack/plugins/task_manager/server/config.ts @@ -6,6 +6,7 @@ */ import { schema, TypeOf } from '@kbn/config-schema'; +import { getTaskClaimer } from './task_claimers'; export const MAX_WORKERS_LIMIT = 100; export const DEFAULT_MAX_WORKERS = 10; @@ -25,6 +26,8 @@ export const DEFAULT_METRICS_RESET_INTERVAL = 30 * 1000; // 30 seconds // At the default poll interval of 3sec, this averages over the last 15sec. export const DEFAULT_WORKER_UTILIZATION_RUNNING_AVERAGE_WINDOW = 5; +export const CLAIM_STRATEGY_DEFAULT = 'default'; + export const taskExecutionFailureThresholdSchema = schema.object( { error_threshold: schema.number({ @@ -152,6 +155,7 @@ export const configSchema = schema.object( max: 100, min: 1, }), + claim_strategy: schema.string({ defaultValue: CLAIM_STRATEGY_DEFAULT }), }, { validate: (config) => { @@ -162,6 +166,11 @@ export const configSchema = schema.object( ) { return `The specified monitored_stats_required_freshness (${config.monitored_stats_required_freshness}) is invalid, as it is below the poll_interval (${config.poll_interval})`; } + try { + getTaskClaimer(config.claim_strategy); + } catch (err) { + return `The claim strategy is invalid: ${err.message}`; + } }, } ); diff --git a/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts b/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts index 39a9afb0f0d14..6ae1d00a62243 100644 --- a/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts +++ b/x-pack/plugins/task_manager/server/ephemeral_task_lifecycle.test.ts @@ -85,6 +85,7 @@ describe('EphemeralTaskLifecycle', () => { max_attempts: 20, }, metrics_reset_interval: 3000, + claim_strategy: 'default', ...config, }, elasticsearchAndSOAvailability$, diff --git a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts index f034feb154462..2fd4ceb74ca74 100644 --- a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts +++ b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts @@ -80,6 +80,7 @@ describe('managed configuration', () => { max_attempts: 20, }, metrics_reset_interval: 3000, + claim_strategy: 'default', }); logger = context.logger.get('taskManager'); diff --git a/x-pack/plugins/task_manager/server/lib/calculate_health_status.test.ts b/x-pack/plugins/task_manager/server/lib/calculate_health_status.test.ts index 6c9a9efeb558e..228d94ed87daf 100644 --- a/x-pack/plugins/task_manager/server/lib/calculate_health_status.test.ts +++ b/x-pack/plugins/task_manager/server/lib/calculate_health_status.test.ts @@ -57,6 +57,7 @@ const config = { max_attempts: 20, }, metrics_reset_interval: 3000, + claim_strategy: 'default', }; const getStatsWithTimestamp = ({ diff --git a/x-pack/plugins/task_manager/server/metrics/create_aggregator.test.ts b/x-pack/plugins/task_manager/server/metrics/create_aggregator.test.ts index e457797d5ae1b..5becffa4e302c 100644 --- a/x-pack/plugins/task_manager/server/metrics/create_aggregator.test.ts +++ b/x-pack/plugins/task_manager/server/metrics/create_aggregator.test.ts @@ -73,6 +73,7 @@ const config: TaskManagerConfig = { }, version_conflict_threshold: 80, worker_utilization_running_average_window: 5, + claim_strategy: 'default', }; describe('createAggregator', () => { diff --git a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts index 689c9c882bee3..543707bf940b2 100644 --- a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts +++ b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts @@ -53,6 +53,7 @@ describe('Configuration Statistics Aggregator', () => { max_attempts: 20, }, metrics_reset_interval: 3000, + claim_strategy: 'default', }; const managedConfig = { diff --git a/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts index daf3f2baf085d..7dc3460d98388 100644 --- a/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts +++ b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts @@ -58,6 +58,7 @@ describe('createMonitoringStatsStream', () => { max_attempts: 20, }, metrics_reset_interval: 3000, + claim_strategy: 'default', }; it('returns the initial config used to configure Task Manager', async () => { diff --git a/x-pack/plugins/task_manager/server/plugin.test.ts b/x-pack/plugins/task_manager/server/plugin.test.ts index 1e7215d6d7a1b..5e178db3b99ad 100644 --- a/x-pack/plugins/task_manager/server/plugin.test.ts +++ b/x-pack/plugins/task_manager/server/plugin.test.ts @@ -78,6 +78,7 @@ const pluginInitializerContextParams = { max_attempts: 20, }, metrics_reset_interval: 3000, + claim_strategy: 'default', }; describe('TaskManagerPlugin', () => { diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts index 79b153f42a88d..24898d3c99385 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts @@ -83,6 +83,7 @@ describe('TaskPollingLifecycle', () => { max_attempts: 20, }, metrics_reset_interval: 3000, + claim_strategy: 'default', }, taskStore: mockTaskStore, logger: taskManagerLogger, diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.ts index e7509df01cfc8..2024277ea94e0 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.ts @@ -41,7 +41,8 @@ import { identifyEsError, isEsCannotExecuteScriptError } from './lib/identify_es import { BufferedTaskStore } from './buffered_task_store'; import { TaskTypeDictionary } from './task_type_dictionary'; import { delayOnClaimConflicts } from './polling'; -import { TaskClaiming, ClaimOwnershipResult } from './queries/task_claiming'; +import { TaskClaiming } from './queries/task_claiming'; +import { ClaimOwnershipResult } from './task_claimers'; export interface ITaskEventEmitter { get events(): Observable; @@ -132,6 +133,7 @@ export class TaskPollingLifecycle implements ITaskEventEmitter ({ CONCURRENCY_ALLOW_LIST_BY_TASK_TYPE: [ @@ -82,6 +66,23 @@ describe('TaskClaiming', () => { .mockImplementation(() => mockApmTrans as any); }); + test(`should throw an error when invalid strategy specified`, () => { + const definitions = new TaskTypeDictionary(mockLogger()); + + expect(() => { + new TaskClaiming({ + logger: taskManagerLogger, + strategy: 'non-default', + definitions, + excludedTaskTypes: [], + unusedTypes: [], + taskStore: taskStoreMock.create({ taskManagerId: '' }), + maxAttempts: 2, + getCapacity: () => 10, + }); + }).toThrowErrorMatchingInlineSnapshot(`"Unknown task claiming strategy (non-default)"`); + }); + test(`should log when a certain task type is skipped due to having a zero concurency configuration`, () => { const definitions = new TaskTypeDictionary(mockLogger()); definitions.registerTaskDefinitions({ @@ -117,6 +118,7 @@ describe('TaskClaiming', () => { new TaskClaiming({ logger: taskManagerLogger, + strategy: 'default', definitions, excludedTaskTypes: [], unusedTypes: [], @@ -130,1220 +132,4 @@ describe('TaskClaiming', () => { `"Task Manager will never claim tasks of the following types as their \\"maxConcurrency\\" is set to 0: limitedToZero, anotherLimitedToZero"` ); }); - - describe('claimAvailableTasks', () => { - function initialiseTestClaiming({ - storeOpts = {}, - taskClaimingOpts = {}, - hits = [generateFakeTasks(1)], - versionConflicts = 2, - excludedTaskTypes = [], - unusedTaskTypes = [], - }: { - storeOpts: Partial; - taskClaimingOpts: Partial; - hits?: ConcreteTaskInstance[][]; - versionConflicts?: number; - excludedTaskTypes?: string[]; - unusedTaskTypes?: string[]; - }) { - const definitions = storeOpts.definitions ?? taskDefinitions; - const store = taskStoreMock.create({ taskManagerId: storeOpts.taskManagerId }); - store.convertToSavedObjectIds.mockImplementation((ids) => ids.map((id) => `task:${id}`)); - - if (hits.length === 1) { - store.fetch.mockResolvedValue({ docs: hits[0] }); - store.updateByQuery.mockResolvedValue({ - updated: hits[0].length, - version_conflicts: versionConflicts, - total: hits[0].length, - }); - } else { - for (const docs of hits) { - store.fetch.mockResolvedValueOnce({ docs }); - store.updateByQuery.mockResolvedValueOnce({ - updated: docs.length, - version_conflicts: versionConflicts, - total: docs.length, - }); - } - } - - const taskClaiming = new TaskClaiming({ - logger: taskManagerLogger, - definitions, - taskStore: store, - excludedTaskTypes, - unusedTypes: unusedTaskTypes, - maxAttempts: taskClaimingOpts.maxAttempts ?? 2, - getCapacity: taskClaimingOpts.getCapacity ?? (() => 10), - ...taskClaimingOpts, - }); - - return { taskClaiming, store }; - } - - async function testClaimAvailableTasks({ - storeOpts = {}, - taskClaimingOpts = {}, - claimingOpts, - hits = [generateFakeTasks(1)], - versionConflicts = 2, - excludedTaskTypes = [], - unusedTaskTypes = [], - }: { - storeOpts: Partial; - taskClaimingOpts: Partial; - claimingOpts: Omit; - hits?: ConcreteTaskInstance[][]; - versionConflicts?: number; - excludedTaskTypes?: string[]; - unusedTaskTypes?: string[]; - }) { - const getCapacity = taskClaimingOpts.getCapacity ?? (() => 10); - const { taskClaiming, store } = initialiseTestClaiming({ - storeOpts, - taskClaimingOpts, - excludedTaskTypes, - unusedTaskTypes, - hits, - versionConflicts, - }); - - const results = await getAllAsPromise(taskClaiming.claimAvailableTasks(claimingOpts)); - - expect(apm.startTransaction).toHaveBeenCalledWith( - TASK_MANAGER_MARK_AS_CLAIMED, - TASK_MANAGER_TRANSACTION_TYPE - ); - expect(mockApmTrans.end).toHaveBeenCalledWith('success'); - - expect(store.updateByQuery.mock.calls[0][1]).toMatchObject({ - max_docs: getCapacity(), - }); - expect(store.fetch.mock.calls[0][0]).toMatchObject({ size: getCapacity() }); - return results.map((result, index) => ({ - result, - args: { - search: store.fetch.mock.calls[index][0] as SearchOpts & { - query: MustNotCondition; - }, - updateByQuery: store.updateByQuery.mock.calls[index] as [ - UpdateByQuerySearchOpts, - UpdateByQueryOpts - ], - }, - })); - } - - test('makes calls to APM as expected when markAvailableTasksAsClaimed throws error', async () => { - const maxAttempts = _.random(2, 43); - const customMaxAttempts = _.random(44, 100); - - const definitions = new TaskTypeDictionary(mockLogger()); - definitions.registerTaskDefinitions({ - foo: { - title: 'foo', - createTaskRunner: jest.fn(), - }, - bar: { - title: 'bar', - maxAttempts: customMaxAttempts, - createTaskRunner: jest.fn(), - }, - }); - - const { taskClaiming, store } = initialiseTestClaiming({ - storeOpts: { - definitions, - }, - taskClaimingOpts: { - maxAttempts, - }, - }); - - store.updateByQuery.mockRejectedValue(new Error('Oh no')); - - await expect( - getAllAsPromise( - taskClaiming.claimAvailableTasks({ - claimOwnershipUntil: new Date(), - }) - ) - ).rejects.toMatchInlineSnapshot(`[Error: Oh no]`); - - expect(apm.startTransaction).toHaveBeenCalledWith( - TASK_MANAGER_MARK_AS_CLAIMED, - TASK_MANAGER_TRANSACTION_TYPE - ); - expect(mockApmTrans.end).toHaveBeenCalledWith('failure'); - }); - - test('it filters claimed tasks down by supported types, maxAttempts, status, and runAt', async () => { - const maxAttempts = _.random(2, 43); - const customMaxAttempts = _.random(44, 100); - - const definitions = new TaskTypeDictionary(mockLogger()); - definitions.registerTaskDefinitions({ - foo: { - title: 'foo', - createTaskRunner: jest.fn(), - }, - bar: { - title: 'bar', - maxAttempts: customMaxAttempts, - createTaskRunner: jest.fn(), - }, - foobar: { - title: 'foobar', - maxAttempts: customMaxAttempts, - createTaskRunner: jest.fn(), - }, - }); - - const [ - { - args: { - updateByQuery: [{ query, sort }], - }, - }, - ] = await testClaimAvailableTasks({ - storeOpts: { - definitions, - }, - taskClaimingOpts: { - maxAttempts, - }, - claimingOpts: { - claimOwnershipUntil: new Date(), - }, - excludedTaskTypes: ['foobar'], - }); - expect(query).toMatchObject({ - bool: { - must: [ - { - bool: { - must: [ - { - term: { - 'task.enabled': true, - }, - }, - ], - }, - }, - { - bool: { - should: [ - { - bool: { - must: [ - { term: { 'task.status': 'idle' } }, - { range: { 'task.runAt': { lte: 'now' } } }, - ], - }, - }, - { - bool: { - must: [ - { - bool: { - should: [ - { term: { 'task.status': 'running' } }, - { term: { 'task.status': 'claiming' } }, - ], - }, - }, - { range: { 'task.retryAt': { lte: 'now' } } }, - ], - }, - }, - ], - }, - }, - ], - filter: [ - { - bool: { - must_not: [ - { - bool: { - should: [ - { term: { 'task.status': 'running' } }, - { term: { 'task.status': 'claiming' } }, - ], - must: { range: { 'task.retryAt': { gt: 'now' } } }, - }, - }, - ], - }, - }, - ], - }, - }); - expect(sort).toMatchObject([ - { - _script: { - type: 'number', - order: 'asc', - script: { - lang: 'painless', - source: ` -if (doc['task.retryAt'].size()!=0) { - return doc['task.retryAt'].value.toInstant().toEpochMilli(); -} -if (doc['task.runAt'].size()!=0) { - return doc['task.runAt'].value.toInstant().toEpochMilli(); -} - `, - }, - }, - }, - ]); - }); - - test('it should claim in batches partitioned by maxConcurrency', async () => { - const maxAttempts = _.random(2, 43); - const definitions = new TaskTypeDictionary(mockLogger()); - const taskManagerId = uuidv1(); - const fieldUpdates = { - ownerId: taskManagerId, - retryAt: new Date(Date.now()), - }; - definitions.registerTaskDefinitions({ - unlimited: { - title: 'unlimited', - createTaskRunner: jest.fn(), - }, - limitedToZero: { - title: 'limitedToZero', - maxConcurrency: 0, - createTaskRunner: jest.fn(), - }, - anotherUnlimited: { - title: 'anotherUnlimited', - createTaskRunner: jest.fn(), - }, - finalUnlimited: { - title: 'finalUnlimited', - createTaskRunner: jest.fn(), - }, - limitedToOne: { - title: 'limitedToOne', - maxConcurrency: 1, - createTaskRunner: jest.fn(), - }, - anotherLimitedToOne: { - title: 'anotherLimitedToOne', - maxConcurrency: 1, - createTaskRunner: jest.fn(), - }, - limitedToTwo: { - title: 'limitedToTwo', - maxConcurrency: 2, - createTaskRunner: jest.fn(), - }, - }); - const results = await testClaimAvailableTasks({ - storeOpts: { - taskManagerId, - definitions, - }, - taskClaimingOpts: { - maxAttempts, - getCapacity: (type) => { - switch (type) { - case 'limitedToOne': - case 'anotherLimitedToOne': - return 1; - case 'limitedToTwo': - return 2; - default: - return 10; - } - }, - }, - claimingOpts: { - claimOwnershipUntil: new Date(), - }, - }); - - expect(results.length).toEqual(4); - - expect(results[0].args.updateByQuery[1].max_docs).toEqual(10); - expect(results[0].args.updateByQuery[0].script).toMatchObject({ - source: expect.any(String), - lang: 'painless', - params: { - fieldUpdates, - claimableTaskTypes: ['unlimited', 'anotherUnlimited', 'finalUnlimited'], - skippedTaskTypes: [ - 'limitedToZero', - 'limitedToOne', - 'anotherLimitedToOne', - 'limitedToTwo', - ], - unusedTaskTypes: [], - taskMaxAttempts: { - unlimited: maxAttempts, - }, - }, - }); - - expect(results[1].args.updateByQuery[1].max_docs).toEqual(1); - expect(results[1].args.updateByQuery[0].script).toMatchObject({ - source: expect.any(String), - lang: 'painless', - params: { - fieldUpdates, - claimableTaskTypes: ['limitedToOne'], - skippedTaskTypes: [ - 'unlimited', - 'limitedToZero', - 'anotherUnlimited', - 'finalUnlimited', - 'anotherLimitedToOne', - 'limitedToTwo', - ], - taskMaxAttempts: { - limitedToOne: maxAttempts, - }, - }, - }); - - expect(results[2].args.updateByQuery[1].max_docs).toEqual(1); - expect(results[2].args.updateByQuery[0].script).toMatchObject({ - source: expect.any(String), - lang: 'painless', - params: { - fieldUpdates, - claimableTaskTypes: ['anotherLimitedToOne'], - skippedTaskTypes: [ - 'unlimited', - 'limitedToZero', - 'anotherUnlimited', - 'finalUnlimited', - 'limitedToOne', - 'limitedToTwo', - ], - taskMaxAttempts: { - anotherLimitedToOne: maxAttempts, - }, - }, - }); - - expect(results[3].args.updateByQuery[1].max_docs).toEqual(2); - expect(results[3].args.updateByQuery[0].script).toMatchObject({ - source: expect.any(String), - lang: 'painless', - params: { - fieldUpdates, - claimableTaskTypes: ['limitedToTwo'], - skippedTaskTypes: [ - 'unlimited', - 'limitedToZero', - 'anotherUnlimited', - 'finalUnlimited', - 'limitedToOne', - 'anotherLimitedToOne', - ], - taskMaxAttempts: { - limitedToTwo: maxAttempts, - }, - }, - }); - }); - - test('it should reduce the available capacity from batch to batch', async () => { - const maxAttempts = _.random(2, 43); - const definitions = new TaskTypeDictionary(mockLogger()); - const taskManagerId = uuidv1(); - definitions.registerTaskDefinitions({ - unlimited: { - title: 'unlimited', - createTaskRunner: jest.fn(), - }, - limitedToFive: { - title: 'limitedToFive', - maxConcurrency: 5, - createTaskRunner: jest.fn(), - }, - limitedToTwo: { - title: 'limitedToTwo', - maxConcurrency: 2, - createTaskRunner: jest.fn(), - }, - }); - const results = await testClaimAvailableTasks({ - storeOpts: { - taskManagerId, - definitions, - }, - taskClaimingOpts: { - maxAttempts, - getCapacity: (type) => { - switch (type) { - case 'limitedToTwo': - return 2; - case 'limitedToFive': - return 5; - default: - return 10; - } - }, - }, - hits: [ - [ - // 7 returned by unlimited query - mockInstance({ - taskType: 'unlimited', - }), - mockInstance({ - taskType: 'unlimited', - }), - mockInstance({ - taskType: 'unlimited', - }), - mockInstance({ - taskType: 'unlimited', - }), - mockInstance({ - taskType: 'unlimited', - }), - mockInstance({ - taskType: 'unlimited', - }), - mockInstance({ - taskType: 'unlimited', - }), - ], - // 2 returned by limitedToFive query - [ - mockInstance({ - taskType: 'limitedToFive', - }), - mockInstance({ - taskType: 'limitedToFive', - }), - ], - // 1 reterned by limitedToTwo query - [ - mockInstance({ - taskType: 'limitedToTwo', - }), - ], - ], - claimingOpts: { - claimOwnershipUntil: new Date(), - }, - }); - - expect(results.length).toEqual(3); - - expect(results[0].args.updateByQuery[1].max_docs).toEqual(10); - - // only capacity for 3, even though 5 are allowed - expect(results[1].args.updateByQuery[1].max_docs).toEqual(3); - - // only capacity for 1, even though 2 are allowed - expect(results[2].args.updateByQuery[1].max_docs).toEqual(1); - }); - - test('it shuffles the types claimed in batches to ensure no type starves another', async () => { - const maxAttempts = _.random(2, 43); - const definitions = new TaskTypeDictionary(mockLogger()); - const taskManagerId = uuidv1(); - definitions.registerTaskDefinitions({ - unlimited: { - title: 'unlimited', - createTaskRunner: jest.fn(), - }, - anotherUnlimited: { - title: 'anotherUnlimited', - createTaskRunner: jest.fn(), - }, - finalUnlimited: { - title: 'finalUnlimited', - createTaskRunner: jest.fn(), - }, - limitedToOne: { - title: 'limitedToOne', - maxConcurrency: 1, - createTaskRunner: jest.fn(), - }, - anotherLimitedToOne: { - title: 'anotherLimitedToOne', - maxConcurrency: 1, - createTaskRunner: jest.fn(), - }, - limitedToTwo: { - title: 'limitedToTwo', - maxConcurrency: 2, - createTaskRunner: jest.fn(), - }, - }); - - const { taskClaiming, store } = initialiseTestClaiming({ - storeOpts: { - taskManagerId, - definitions, - }, - taskClaimingOpts: { - maxAttempts, - getCapacity: (type) => { - switch (type) { - case 'limitedToOne': - case 'anotherLimitedToOne': - return 1; - case 'limitedToTwo': - return 2; - default: - return 10; - } - }, - }, - }); - - async function getUpdateByQueryScriptParams() { - return ( - await getAllAsPromise( - taskClaiming.claimAvailableTasks({ - claimOwnershipUntil: new Date(), - }) - ) - ).map( - (result, index) => - ( - store.updateByQuery.mock.calls[index][0] as { - query: MustNotCondition; - size: number; - sort: string | string[]; - script: { - params: { - [claimableTaskTypes: string]: string[]; - }; - }; - } - ).script.params.claimableTaskTypes - ); - } - - const firstCycle = await getUpdateByQueryScriptParams(); - store.updateByQuery.mockClear(); - const secondCycle = await getUpdateByQueryScriptParams(); - - expect(firstCycle.length).toEqual(4); - expect(secondCycle.length).toEqual(4); - expect(firstCycle).not.toMatchObject(secondCycle); - }); - - test('it passes any unusedTaskTypes to script', async () => { - const maxAttempts = _.random(2, 43); - const customMaxAttempts = _.random(44, 100); - const taskManagerId = uuidv1(); - const fieldUpdates = { - ownerId: taskManagerId, - retryAt: new Date(Date.now()), - }; - const definitions = new TaskTypeDictionary(mockLogger()); - definitions.registerTaskDefinitions({ - foo: { - title: 'foo', - createTaskRunner: jest.fn(), - }, - bar: { - title: 'bar', - maxAttempts: customMaxAttempts, - createTaskRunner: jest.fn(), - }, - foobar: { - title: 'foobar', - maxAttempts: customMaxAttempts, - createTaskRunner: jest.fn(), - }, - }); - - const [ - { - args: { - updateByQuery: [{ query, script }], - }, - }, - ] = await testClaimAvailableTasks({ - storeOpts: { - definitions, - taskManagerId, - }, - taskClaimingOpts: { - maxAttempts, - }, - claimingOpts: { - claimOwnershipUntil: new Date(), - }, - excludedTaskTypes: ['foobar'], - unusedTaskTypes: ['barfoo'], - }); - expect(query).toMatchObject({ - bool: { - must: [ - { - bool: { - must: [ - { - term: { - 'task.enabled': true, - }, - }, - ], - }, - }, - { - bool: { - should: [ - { - bool: { - must: [ - { term: { 'task.status': 'idle' } }, - { range: { 'task.runAt': { lte: 'now' } } }, - ], - }, - }, - { - bool: { - must: [ - { - bool: { - should: [ - { term: { 'task.status': 'running' } }, - { term: { 'task.status': 'claiming' } }, - ], - }, - }, - { range: { 'task.retryAt': { lte: 'now' } } }, - ], - }, - }, - ], - }, - }, - ], - filter: [ - { - bool: { - must_not: [ - { - bool: { - should: [ - { term: { 'task.status': 'running' } }, - { term: { 'task.status': 'claiming' } }, - ], - must: { range: { 'task.retryAt': { gt: 'now' } } }, - }, - }, - ], - }, - }, - ], - }, - }); - expect(script).toMatchObject({ - source: expect.any(String), - lang: 'painless', - params: { - fieldUpdates, - claimableTaskTypes: ['foo', 'bar'], - skippedTaskTypes: ['foobar'], - unusedTaskTypes: ['barfoo'], - taskMaxAttempts: { - bar: customMaxAttempts, - foo: maxAttempts, - }, - }, - }); - }); - - test('it claims tasks by setting their ownerId, status and retryAt', async () => { - const taskManagerId = uuidv1(); - const claimOwnershipUntil = new Date(Date.now()); - const fieldUpdates = { - ownerId: taskManagerId, - retryAt: claimOwnershipUntil, - }; - const [ - { - args: { - updateByQuery: [{ script }], - }, - }, - ] = await testClaimAvailableTasks({ - storeOpts: { - taskManagerId, - }, - taskClaimingOpts: {}, - claimingOpts: { - claimOwnershipUntil, - }, - }); - expect(script).toMatchObject({ - source: expect.any(String), - lang: 'painless', - params: { - fieldUpdates, - claimableTaskTypes: ['report', 'dernstraight', 'yawn'], - skippedTaskTypes: [], - taskMaxAttempts: { - dernstraight: 2, - report: 2, - yawn: 2, - }, - }, - }); - }); - - test('it filters out running tasks', async () => { - const taskManagerId = uuidv1(); - const claimOwnershipUntil = new Date(Date.now()); - const runAt = new Date(); - const tasks = [ - mockInstance({ - id: 'aaa', - runAt, - taskType: 'yawn', - schedule: undefined, - attempts: 0, - status: TaskStatus.Claiming, - params: { hello: 'world' }, - state: { baby: 'Henhen' }, - user: 'jimbo', - scope: ['reporting'], - ownerId: taskManagerId, - }), - ]; - const [ - { - result: { docs }, - args: { - search: { query }, - }, - }, - ] = await testClaimAvailableTasks({ - storeOpts: { - taskManagerId, - }, - taskClaimingOpts: {}, - claimingOpts: { - claimOwnershipUntil, - }, - hits: [tasks], - }); - - expect(query).toMatchObject({ - bool: { - must: [ - { - term: { - 'task.ownerId': taskManagerId, - }, - }, - { term: { 'task.status': 'claiming' } }, - { - bool: { - should: [ - { - term: { - 'task.taskType': 'report', - }, - }, - { - term: { - 'task.taskType': 'dernstraight', - }, - }, - { - term: { - 'task.taskType': 'yawn', - }, - }, - ], - }, - }, - ], - }, - }); - - expect(docs).toMatchObject([ - { - attempts: 0, - id: 'aaa', - schedule: undefined, - params: { hello: 'world' }, - runAt, - scope: ['reporting'], - state: { baby: 'Henhen' }, - status: 'claiming', - taskType: 'yawn', - user: 'jimbo', - ownerId: taskManagerId, - }, - ]); - }); - - test('it returns task objects', async () => { - const taskManagerId = uuidv1(); - const claimOwnershipUntil = new Date(Date.now()); - const runAt = new Date(); - const tasks = [ - mockInstance({ - id: 'aaa', - runAt, - taskType: 'yawn', - schedule: undefined, - attempts: 0, - status: TaskStatus.Claiming, - params: { hello: 'world' }, - state: { baby: 'Henhen' }, - user: 'jimbo', - scope: ['reporting'], - ownerId: taskManagerId, - }), - mockInstance({ - id: 'bbb', - runAt, - taskType: 'yawn', - schedule: { interval: '5m' }, - attempts: 2, - status: TaskStatus.Claiming, - params: { shazm: 1 }, - state: { henry: 'The 8th' }, - user: 'dabo', - scope: ['reporting', 'ceo'], - ownerId: taskManagerId, - }), - ]; - const [ - { - result: { docs }, - args: { - search: { query }, - }, - }, - ] = await testClaimAvailableTasks({ - storeOpts: { - taskManagerId, - }, - taskClaimingOpts: {}, - claimingOpts: { - claimOwnershipUntil, - }, - hits: [tasks], - }); - - expect(query).toMatchObject({ - bool: { - must: [ - { - term: { - 'task.ownerId': taskManagerId, - }, - }, - { term: { 'task.status': 'claiming' } }, - { - bool: { - should: [ - { - term: { - 'task.taskType': 'report', - }, - }, - { - term: { - 'task.taskType': 'dernstraight', - }, - }, - { - term: { - 'task.taskType': 'yawn', - }, - }, - ], - }, - }, - ], - }, - }); - - expect(docs).toMatchObject([ - { - attempts: 0, - id: 'aaa', - schedule: undefined, - params: { hello: 'world' }, - runAt, - scope: ['reporting'], - state: { baby: 'Henhen' }, - status: 'claiming', - taskType: 'yawn', - user: 'jimbo', - ownerId: taskManagerId, - }, - { - attempts: 2, - id: 'bbb', - schedule: { interval: '5m' }, - params: { shazm: 1 }, - runAt, - scope: ['reporting', 'ceo'], - state: { henry: 'The 8th' }, - status: 'claiming', - taskType: 'yawn', - user: 'dabo', - ownerId: taskManagerId, - }, - ]); - }); - - test('it returns version_conflicts that do not include conflicts that were proceeded against', async () => { - const taskManagerId = uuidv1(); - const claimOwnershipUntil = new Date(Date.now()); - const runAt = new Date(); - const tasks = [ - mockInstance({ - runAt, - taskType: 'foo', - schedule: undefined, - attempts: 0, - status: TaskStatus.Claiming, - params: { hello: 'world' }, - state: { baby: 'Henhen' }, - user: 'jimbo', - scope: ['reporting'], - ownerId: taskManagerId, - }), - mockInstance({ - runAt, - taskType: 'bar', - schedule: { interval: '5m' }, - attempts: 2, - status: TaskStatus.Claiming, - params: { shazm: 1 }, - state: { henry: 'The 8th' }, - user: 'dabo', - scope: ['reporting', 'ceo'], - ownerId: taskManagerId, - }), - ]; - const maxDocs = 10; - const [ - { - result: { - stats: { tasksUpdated, tasksConflicted, tasksClaimed }, - }, - }, - ] = await testClaimAvailableTasks({ - storeOpts: { - taskManagerId, - }, - taskClaimingOpts: { getCapacity: () => maxDocs }, - claimingOpts: { - claimOwnershipUntil, - }, - hits: [tasks], - // assume there were 20 version conflists, but thanks to `conflicts="proceed"` - // we proceeded to claim tasks - versionConflicts: 20, - }); - - expect(tasksUpdated).toEqual(2); - // ensure we only count conflicts that *may* have counted against max_docs, no more than that - expect(tasksConflicted).toEqual(10 - tasksUpdated!); - expect(tasksClaimed).toEqual(2); - }); - }); - - describe('task events', () => { - function generateTasks(taskManagerId: string) { - const runAt = new Date(); - const tasks = [ - { - id: 'claimed-by-id', - runAt, - taskType: 'foo', - schedule: undefined, - attempts: 0, - status: TaskStatus.Claiming, - params: { hello: 'world' }, - state: { baby: 'Henhen' }, - user: 'jimbo', - scope: ['reporting'], - ownerId: taskManagerId, - startedAt: null, - retryAt: null, - scheduledAt: new Date(), - traceparent: 'parent', - }, - { - id: 'claimed-by-schedule', - runAt, - taskType: 'bar', - schedule: { interval: '5m' }, - attempts: 2, - status: TaskStatus.Claiming, - params: { shazm: 1 }, - state: { henry: 'The 8th' }, - user: 'dabo', - scope: ['reporting', 'ceo'], - ownerId: taskManagerId, - startedAt: null, - retryAt: null, - scheduledAt: new Date(), - traceparent: 'newParent', - }, - { - id: 'already-running', - runAt, - taskType: 'bar', - schedule: { interval: '5m' }, - attempts: 2, - status: TaskStatus.Running, - params: { shazm: 1 }, - state: { henry: 'The 8th' }, - user: 'dabo', - scope: ['reporting', 'ceo'], - ownerId: taskManagerId, - startedAt: null, - retryAt: null, - scheduledAt: new Date(), - traceparent: '', - }, - ]; - - return { taskManagerId, runAt, tasks }; - } - - function instantiateStoreWithMockedApiResponses({ - taskManagerId = uuidv4(), - definitions = taskDefinitions, - getCapacity = () => 10, - tasksClaimed, - }: Partial> & { - taskManagerId?: string; - tasksClaimed?: ConcreteTaskInstance[][]; - } = {}) { - const { runAt, tasks: generatedTasks } = generateTasks(taskManagerId); - const taskCycles = tasksClaimed ?? [generatedTasks]; - - const taskStore = taskStoreMock.create({ taskManagerId }); - taskStore.convertToSavedObjectIds.mockImplementation((ids) => ids.map((id) => `task:${id}`)); - for (const docs of taskCycles) { - taskStore.fetch.mockResolvedValueOnce({ docs }); - taskStore.updateByQuery.mockResolvedValueOnce({ - updated: docs.length, - version_conflicts: 0, - total: docs.length, - }); - } - - taskStore.fetch.mockResolvedValue({ docs: [] }); - taskStore.updateByQuery.mockResolvedValue({ - updated: 0, - version_conflicts: 0, - total: 0, - }); - - const taskClaiming = new TaskClaiming({ - logger: taskManagerLogger, - definitions, - excludedTaskTypes: [], - unusedTypes: [], - taskStore, - maxAttempts: 2, - getCapacity, - }); - - return { taskManagerId, runAt, taskClaiming }; - } - - test('emits an event when a task is succesfully by scheduling', async () => { - const { taskManagerId, runAt, taskClaiming } = instantiateStoreWithMockedApiResponses(); - - const promise = taskClaiming.events - .pipe( - filter( - (event: TaskEvent) => event.id === 'claimed-by-schedule' - ), - take(1) - ) - .toPromise(); - - await getFirstAsPromise( - taskClaiming.claimAvailableTasks({ - claimOwnershipUntil: new Date(), - }) - ); - - const event = await promise; - expect(event).toMatchObject( - asTaskClaimEvent( - 'claimed-by-schedule', - asOk({ - id: 'claimed-by-schedule', - runAt, - taskType: 'bar', - schedule: { interval: '5m' }, - attempts: 2, - status: 'claiming' as TaskStatus, - params: { shazm: 1 }, - state: { henry: 'The 8th' }, - user: 'dabo', - scope: ['reporting', 'ceo'], - ownerId: taskManagerId, - startedAt: null, - retryAt: null, - scheduledAt: new Date(), - traceparent: 'newParent', - }) - ) - ); - }); - }); }); - -function generateFakeTasks(count: number = 1) { - return _.times(count, (index) => mockInstance({ id: `task:id-${index}` })); -} - -function mockInstance(instance: Partial = {}) { - return Object.assign( - { - id: uuidv4(), - taskType: 'bar', - sequenceNumber: 32, - primaryTerm: 32, - runAt: new Date(), - scheduledAt: new Date(), - startedAt: null, - retryAt: null, - attempts: 0, - params: {}, - scope: ['reporting'], - state: {}, - status: 'idle', - user: 'example', - ownerId: null, - traceparent: '', - }, - instance - ); -} - -function getFirstAsPromise(obs$: Observable): Promise { - return new Promise((resolve, reject) => { - obs$.subscribe(resolve, reject); - }); -} -function getAllAsPromise(obs$: Observable): Promise { - return new Promise((resolve, reject) => { - obs$.pipe(toArray()).subscribe(resolve, reject); - }); -} diff --git a/x-pack/plugins/task_manager/server/queries/task_claiming.ts b/x-pack/plugins/task_manager/server/queries/task_claiming.ts index 60237338fb09c..1e31bb5a2ed65 100644 --- a/x-pack/plugins/task_manager/server/queries/task_claiming.ts +++ b/x-pack/plugins/task_manager/server/queries/task_claiming.ts @@ -8,41 +8,30 @@ /* * This module contains helpers for managing the task manager storage layer. */ -import apm from 'elastic-apm-node'; -import minimatch from 'minimatch'; -import { Subject, Observable, from, of } from 'rxjs'; -import { map, mergeScan } from 'rxjs/operators'; -import { groupBy, pick, isPlainObject } from 'lodash'; +import { Subject, Observable, of } from 'rxjs'; +import { map } from 'rxjs/operators'; +import { groupBy, isPlainObject } from 'lodash'; import { Logger } from '@kbn/core/server'; import { asOk, asErr, Result } from '../lib/result_type'; import { ConcreteTaskInstance } from '../task'; -import { TaskClaim, asTaskClaimEvent, startTaskTimer, TaskTiming } from '../task_events'; -import { shouldBeOneOf, mustBeAllOf, filterDownBy, matchesClauses } from './query_clauses'; +import { TaskClaim } from '../task_events'; -import { - updateFieldsAndMarkAsFailed, - IdleTaskWithExpiredRunAt, - InactiveTasks, - RunningOrClaimingTaskWithExpiredRetryAt, - SortByRunAtAndRetryAt, - tasksClaimedByOwner, - tasksOfType, - EnabledTask, -} from './mark_available_tasks_as_claimed'; import { TaskTypeDictionary } from '../task_type_dictionary'; -import { - correctVersionConflictsForContinuation, - TaskStore, - UpdateByQueryResult, - SearchOpts, -} from '../task_store'; +import { TaskStore, UpdateByQueryResult } from '../task_store'; import { FillPoolResult } from '../lib/fill_pool'; -import { TASK_MANAGER_TRANSACTION_TYPE } from '../task_running'; +import { + TaskClaimerOpts, + TaskClaimerFn, + ClaimOwnershipResult, + getTaskClaimer, +} from '../task_claimers'; +export type { ClaimOwnershipResult } from '../task_claimers'; export interface TaskClaimingOpts { logger: Logger; + strategy: string; definitions: TaskTypeDictionary; unusedTypes: string[]; taskStore: TaskStore; @@ -67,31 +56,25 @@ export interface FetchResult { docs: ConcreteTaskInstance[]; } -export interface ClaimOwnershipResult { - stats: { - tasksUpdated: number; - tasksConflicted: number; - tasksClaimed: number; - }; - docs: ConcreteTaskInstance[]; - timing?: TaskTiming; +export function isClaimOwnershipResult(result: unknown): result is ClaimOwnershipResult { + return ( + isPlainObject((result as ClaimOwnershipResult).stats) && + Array.isArray((result as ClaimOwnershipResult).docs) + ); } -export const isClaimOwnershipResult = (result: unknown): result is ClaimOwnershipResult => - isPlainObject((result as ClaimOwnershipResult).stats) && - Array.isArray((result as ClaimOwnershipResult).docs); -enum BatchConcurrency { +export enum BatchConcurrency { Unlimited, Limited, } -type TaskClaimingBatches = Array; -interface TaskClaimingBatch { +export type TaskClaimingBatches = Array; +export interface TaskClaimingBatch { concurrency: Concurrency; tasksTypes: TaskType; } -type UnlimitedBatch = TaskClaimingBatch>; -type LimitedBatch = TaskClaimingBatch; +export type UnlimitedBatch = TaskClaimingBatch>; +export type LimitedBatch = TaskClaimingBatch; export const TASK_MANAGER_MARK_AS_CLAIMED = 'mark-available-tasks-as-claimed'; @@ -108,6 +91,7 @@ export class TaskClaiming { private readonly taskMaxAttempts: Record; private readonly excludedTaskTypes: string[]; private readonly unusedTypes: string[]; + private readonly taskClaimer: TaskClaimerFn; /** * Constructs a new TaskStore. @@ -120,12 +104,12 @@ export class TaskClaiming { this.maxAttempts = opts.maxAttempts; this.taskStore = opts.taskStore; this.getCapacity = opts.getCapacity; - this.logger = opts.logger; + this.logger = opts.logger.get('taskClaiming'); this.taskClaimingBatchesByType = this.partitionIntoClaimingBatches(this.definitions); this.taskMaxAttempts = Object.fromEntries(this.normalizeMaxAttempts(this.definitions)); this.excludedTaskTypes = opts.excludedTaskTypes; this.unusedTypes = opts.unusedTypes; - + this.taskClaimer = getTaskClaimer(opts.strategy); this.events$ = new Subject(); } @@ -177,224 +161,43 @@ export class TaskClaiming { return this.events$; } - private emitEvents = (events: TaskClaim[]) => { - events.forEach((event) => this.events$.next(event)); - }; - public claimAvailableTasksIfCapacityIsAvailable( claimingOptions: Omit ): Observable> { if (this.getCapacity()) { - return this.claimAvailableTasks(claimingOptions).pipe( - map((claimResult) => asOk(claimResult)) - ); + const opts: TaskClaimerOpts = { + batches: this.getClaimingBatches(), + claimOwnershipUntil: claimingOptions.claimOwnershipUntil, + taskStore: this.taskStore, + events$: this.events$, + getCapacity: this.getCapacity, + unusedTypes: this.unusedTypes, + definitions: this.definitions, + taskMaxAttempts: this.taskMaxAttempts, + excludedTaskTypes: this.excludedTaskTypes, + }; + return this.taskClaimer(opts).pipe(map((claimResult) => asOk(claimResult))); } this.logger.debug( `[Task Ownership]: Task Manager has skipped Claiming Ownership of available tasks at it has ran out Available Workers.` ); return of(asErr(FillPoolResult.NoAvailableWorkers)); } - - public claimAvailableTasks({ - claimOwnershipUntil, - }: Omit): Observable { - const initialCapacity = this.getCapacity(); - return from(this.getClaimingBatches()).pipe( - mergeScan( - (accumulatedResult, batch) => { - const stopTaskTimer = startTaskTimer(); - const capacity = Math.min( - initialCapacity - accumulatedResult.stats.tasksClaimed, - isLimited(batch) ? this.getCapacity(batch.tasksTypes) : this.getCapacity() - ); - // if we have no more capacity, short circuit here - if (capacity <= 0) { - return of(accumulatedResult); - } - return from( - this.executeClaimAvailableTasks({ - claimOwnershipUntil, - size: capacity, - taskTypes: isLimited(batch) ? new Set([batch.tasksTypes]) : batch.tasksTypes, - }).then((result) => { - const { stats, docs } = accumulateClaimOwnershipResults(accumulatedResult, result); - stats.tasksConflicted = correctVersionConflictsForContinuation( - stats.tasksClaimed, - stats.tasksConflicted, - initialCapacity - ); - return { stats, docs, timing: stopTaskTimer() }; - }) - ); - }, - // initialise the accumulation with no results - accumulateClaimOwnershipResults(), - // only run one batch at a time - 1 - ) - ); - } - - private executeClaimAvailableTasks = async ({ - claimOwnershipUntil, - size, - taskTypes, - }: OwnershipClaimingOpts): Promise => { - const { updated: tasksUpdated, version_conflicts: tasksConflicted } = - await this.markAvailableTasksAsClaimed({ - claimOwnershipUntil, - size, - taskTypes, - }); - - const docs = tasksUpdated > 0 ? await this.sweepForClaimedTasks(taskTypes, size) : []; - - this.emitEvents(docs.map((doc) => asTaskClaimEvent(doc.id, asOk(doc)))); - - const stats = { - tasksUpdated, - tasksConflicted, - tasksClaimed: docs.length, - }; - - return { - stats, - docs, - }; - }; - - private isTaskTypeExcluded(taskType: string) { - for (const excludedType of this.excludedTaskTypes) { - if (minimatch(taskType, excludedType)) { - return true; - } - } - - return false; - } - - private async markAvailableTasksAsClaimed({ - claimOwnershipUntil, - size, - taskTypes, - }: OwnershipClaimingOpts): Promise { - const { taskTypesToSkip = [], taskTypesToClaim = [] } = groupBy( - this.definitions.getAllTypes(), - (type) => - taskTypes.has(type) && !this.isTaskTypeExcluded(type) - ? 'taskTypesToClaim' - : 'taskTypesToSkip' - ); - const queryForScheduledTasks = mustBeAllOf( - // Task must be enabled - EnabledTask, - // Either a task with idle status and runAt <= now or - // status running or claiming with a retryAt <= now. - shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt) - ); - - const sort: NonNullable = [SortByRunAtAndRetryAt]; - const query = matchesClauses(queryForScheduledTasks, filterDownBy(InactiveTasks)); - const script = updateFieldsAndMarkAsFailed({ - fieldUpdates: { - ownerId: this.taskStore.taskManagerId, - retryAt: claimOwnershipUntil, - }, - claimableTaskTypes: taskTypesToClaim, - skippedTaskTypes: taskTypesToSkip, - unusedTaskTypes: this.unusedTypes, - taskMaxAttempts: pick(this.taskMaxAttempts, taskTypesToClaim), - }); - - const apmTrans = apm.startTransaction( - TASK_MANAGER_MARK_AS_CLAIMED, - TASK_MANAGER_TRANSACTION_TYPE - ); - - try { - const result = await this.taskStore.updateByQuery( - { - query, - script, - sort, - }, - { - max_docs: size, - } - ); - apmTrans.end('success'); - return result; - } catch (err) { - apmTrans.end('failure'); - throw err; - } - } - - /** - * Fetches tasks from the index, which are owned by the current Kibana instance - */ - private async sweepForClaimedTasks( - taskTypes: Set, - size: number - ): Promise { - const claimedTasksQuery = tasksClaimedByOwner( - this.taskStore.taskManagerId, - tasksOfType([...taskTypes]) - ); - const { docs } = await this.taskStore.fetch({ - query: claimedTasksQuery, - size, - sort: SortByRunAtAndRetryAt, - seq_no_primary_term: true, - }); - - return docs; - } } -const emptyClaimOwnershipResult = () => { - return { - stats: { - tasksUpdated: 0, - tasksConflicted: 0, - tasksClaimed: 0, - tasksRejected: 0, - }, - docs: [], - }; -}; - -function accumulateClaimOwnershipResults( - prev: ClaimOwnershipResult = emptyClaimOwnershipResult(), - next?: ClaimOwnershipResult -) { - if (next) { - const { stats, docs, timing } = next; - const res = { - stats: { - tasksUpdated: stats.tasksUpdated + prev.stats.tasksUpdated, - tasksConflicted: stats.tasksConflicted + prev.stats.tasksConflicted, - tasksClaimed: stats.tasksClaimed + prev.stats.tasksClaimed, - }, - docs, - timing, - }; - return res; - } - return prev; -} - -function isLimited( +export function isLimited( batch: TaskClaimingBatch ): batch is LimitedBatch { return batch.concurrency === BatchConcurrency.Limited; } + function asLimited(tasksType: string): LimitedBatch { return { concurrency: BatchConcurrency.Limited, tasksTypes: tasksType, }; } + function asUnlimited(tasksTypes: Set): UnlimitedBatch { return { concurrency: BatchConcurrency.Unlimited, diff --git a/x-pack/plugins/task_manager/server/task_claimers/README.md b/x-pack/plugins/task_manager/server/task_claimers/README.md new file mode 100644 index 0000000000000..0c92f02031d2e --- /dev/null +++ b/x-pack/plugins/task_manager/server/task_claimers/README.md @@ -0,0 +1,20 @@ +task_claimers +======================================================================== + +This directory contains code that claims the next tasks to run. + +The code is structured to support multiple strategies, but currently +only supports a `default` strategy. + + +`default` task claiming strategy +------------------------------------------------------------------------ +This has been the strategy for task manager for ... ever? The basic +idea: + +- Run an update by query, for number of available workers, to "mark" + task documents as claimed, by setting task state to `claiming`. + We can do some limited per-task logic in that update script. + +- A search is then run on the documents updated from the update by + query. diff --git a/x-pack/plugins/task_manager/server/task_claimers/index.test.ts b/x-pack/plugins/task_manager/server/task_claimers/index.test.ts new file mode 100644 index 0000000000000..be26f0d2f9efb --- /dev/null +++ b/x-pack/plugins/task_manager/server/task_claimers/index.test.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getTaskClaimer } from '.'; +import { claimAvailableTasksDefault } from './strategy_default'; + +describe('task_claimers/index', () => { + describe('getTaskClaimer()', () => { + test('returns expected result for default', () => { + const taskClaimer = getTaskClaimer('default'); + expect(taskClaimer).toBe(claimAvailableTasksDefault); + }); + + test('throws error for unsupported parameter', () => { + expect(() => getTaskClaimer('not-supported')).toThrowErrorMatchingInlineSnapshot( + `"Unknown task claiming strategy (not-supported)"` + ); + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/task_claimers/index.ts b/x-pack/plugins/task_manager/server/task_claimers/index.ts new file mode 100644 index 0000000000000..8074197a147b6 --- /dev/null +++ b/x-pack/plugins/task_manager/server/task_claimers/index.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Subject, Observable } from 'rxjs'; + +import { TaskStore } from '../task_store'; +import { TaskClaim, TaskTiming } from '../task_events'; +import { TaskTypeDictionary } from '../task_type_dictionary'; +import { TaskClaimingBatches } from '../queries/task_claiming'; +import { ConcreteTaskInstance } from '../task'; +import { claimAvailableTasksDefault } from './strategy_default'; +import { CLAIM_STRATEGY_DEFAULT } from '../config'; + +export interface TaskClaimerOpts { + getCapacity: (taskType?: string | undefined) => number; + claimOwnershipUntil: Date; + batches: TaskClaimingBatches; + events$: Subject; + taskStore: TaskStore; + definitions: TaskTypeDictionary; + unusedTypes: string[]; + excludedTaskTypes: string[]; + taskMaxAttempts: Record; +} + +export interface ClaimOwnershipResult { + stats: { + tasksUpdated: number; + tasksConflicted: number; + tasksClaimed: number; + }; + docs: ConcreteTaskInstance[]; + timing?: TaskTiming; +} + +export type TaskClaimerFn = (opts: TaskClaimerOpts) => Observable; + +export function getTaskClaimer(strategy: string): TaskClaimerFn { + switch (strategy) { + case CLAIM_STRATEGY_DEFAULT: + return claimAvailableTasksDefault; + } + throw new Error(`Unknown task claiming strategy (${strategy})`); +} diff --git a/x-pack/plugins/task_manager/server/task_claimers/strategy_default.test.ts b/x-pack/plugins/task_manager/server/task_claimers/strategy_default.test.ts new file mode 100644 index 0000000000000..c89ecdf669218 --- /dev/null +++ b/x-pack/plugins/task_manager/server/task_claimers/strategy_default.test.ts @@ -0,0 +1,1318 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import _ from 'lodash'; +import { v1 as uuidv1, v4 as uuidv4 } from 'uuid'; +import { filter, take, toArray } from 'rxjs/operators'; + +import { TaskStatus, ConcreteTaskInstance } from '../task'; +import { SearchOpts, StoreOpts, UpdateByQueryOpts, UpdateByQuerySearchOpts } from '../task_store'; +import { asTaskClaimEvent, TaskEvent } from '../task_events'; +import { asOk, isOk, unwrap } from '../lib/result_type'; +import { TaskTypeDictionary } from '../task_type_dictionary'; +import type { MustNotCondition } from '../queries/query_clauses'; +import { mockLogger } from '../test_utils'; +import { + TaskClaiming, + OwnershipClaimingOpts, + TaskClaimingOpts, + TASK_MANAGER_MARK_AS_CLAIMED, +} from '../queries/task_claiming'; +import { Observable } from 'rxjs'; +import { taskStoreMock } from '../task_store.mock'; +import apm from 'elastic-apm-node'; +import { TASK_MANAGER_TRANSACTION_TYPE } from '../task_running'; +import { ClaimOwnershipResult } from '.'; +import { FillPoolResult } from '../lib/fill_pool'; + +jest.mock('../constants', () => ({ + CONCURRENCY_ALLOW_LIST_BY_TASK_TYPE: [ + 'limitedToZero', + 'limitedToOne', + 'anotherLimitedToZero', + 'anotherLimitedToOne', + 'limitedToTwo', + 'limitedToFive', + ], +})); + +const taskManagerLogger = mockLogger(); + +beforeEach(() => jest.clearAllMocks()); + +const mockedDate = new Date('2019-02-12T21:01:22.479Z'); +// eslint-disable-next-line @typescript-eslint/no-explicit-any +(global as any).Date = class Date { + constructor() { + return mockedDate; + } + static now() { + return mockedDate.getTime(); + } +}; + +const taskDefinitions = new TaskTypeDictionary(taskManagerLogger); +taskDefinitions.registerTaskDefinitions({ + report: { + title: 'report', + createTaskRunner: jest.fn(), + }, + dernstraight: { + title: 'dernstraight', + createTaskRunner: jest.fn(), + }, + yawn: { + title: 'yawn', + createTaskRunner: jest.fn(), + }, +}); + +const mockApmTrans = { + end: jest.fn(), +}; + +describe('TaskClaiming', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest + .spyOn(apm, 'startTransaction') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .mockImplementation(() => mockApmTrans as any); + }); + + describe('claimAvailableTasks', () => { + function initialiseTestClaiming({ + storeOpts = {}, + taskClaimingOpts = {}, + hits = [generateFakeTasks(1)], + versionConflicts = 2, + excludedTaskTypes = [], + unusedTaskTypes = [], + }: { + storeOpts: Partial; + taskClaimingOpts: Partial; + hits?: ConcreteTaskInstance[][]; + versionConflicts?: number; + excludedTaskTypes?: string[]; + unusedTaskTypes?: string[]; + }) { + const definitions = storeOpts.definitions ?? taskDefinitions; + const store = taskStoreMock.create({ taskManagerId: storeOpts.taskManagerId }); + store.convertToSavedObjectIds.mockImplementation((ids) => ids.map((id) => `task:${id}`)); + + if (hits.length === 1) { + store.fetch.mockResolvedValue({ docs: hits[0] }); + store.updateByQuery.mockResolvedValue({ + updated: hits[0].length, + version_conflicts: versionConflicts, + total: hits[0].length, + }); + } else { + for (const docs of hits) { + store.fetch.mockResolvedValueOnce({ docs }); + store.updateByQuery.mockResolvedValueOnce({ + updated: docs.length, + version_conflicts: versionConflicts, + total: docs.length, + }); + } + } + + const taskClaiming = new TaskClaiming({ + logger: taskManagerLogger, + strategy: 'default', + definitions, + taskStore: store, + excludedTaskTypes, + unusedTypes: unusedTaskTypes, + maxAttempts: taskClaimingOpts.maxAttempts ?? 2, + getCapacity: taskClaimingOpts.getCapacity ?? (() => 10), + ...taskClaimingOpts, + }); + + return { taskClaiming, store }; + } + + async function testClaimAvailableTasks({ + storeOpts = {}, + taskClaimingOpts = {}, + claimingOpts, + hits = [generateFakeTasks(1)], + versionConflicts = 2, + excludedTaskTypes = [], + unusedTaskTypes = [], + }: { + storeOpts: Partial; + taskClaimingOpts: Partial; + claimingOpts: Omit; + hits?: ConcreteTaskInstance[][]; + versionConflicts?: number; + excludedTaskTypes?: string[]; + unusedTaskTypes?: string[]; + }) { + const getCapacity = taskClaimingOpts.getCapacity ?? (() => 10); + const { taskClaiming, store } = initialiseTestClaiming({ + storeOpts, + taskClaimingOpts, + excludedTaskTypes, + unusedTaskTypes, + hits, + versionConflicts, + }); + + const resultsOrErr = await getAllAsPromise( + taskClaiming.claimAvailableTasksIfCapacityIsAvailable(claimingOpts) + ); + for (const resultOrErr of resultsOrErr) { + if (!isOk(resultOrErr)) { + expect(resultOrErr).toBe(undefined); + } + } + + const results = resultsOrErr.map((resultOrErr) => { + if (!isOk(resultOrErr)) { + expect(resultOrErr).toBe(undefined); + } + return unwrap(resultOrErr) as ClaimOwnershipResult; + }); + + expect(apm.startTransaction).toHaveBeenCalledWith( + TASK_MANAGER_MARK_AS_CLAIMED, + TASK_MANAGER_TRANSACTION_TYPE + ); + expect(mockApmTrans.end).toHaveBeenCalledWith('success'); + + expect(store.updateByQuery.mock.calls[0][1]).toMatchObject({ + max_docs: getCapacity(), + }); + expect(store.fetch.mock.calls[0][0]).toMatchObject({ size: getCapacity() }); + return results.map((result, index) => ({ + result, + args: { + search: store.fetch.mock.calls[index][0] as SearchOpts & { + query: MustNotCondition; + }, + updateByQuery: store.updateByQuery.mock.calls[index] as [ + UpdateByQuerySearchOpts, + UpdateByQueryOpts + ], + }, + })); + } + + test('makes calls to APM as expected when markAvailableTasksAsClaimed throws error', async () => { + const maxAttempts = _.random(2, 43); + const customMaxAttempts = _.random(44, 100); + + const definitions = new TaskTypeDictionary(mockLogger()); + definitions.registerTaskDefinitions({ + foo: { + title: 'foo', + createTaskRunner: jest.fn(), + }, + bar: { + title: 'bar', + maxAttempts: customMaxAttempts, + createTaskRunner: jest.fn(), + }, + }); + + const { taskClaiming, store } = initialiseTestClaiming({ + storeOpts: { + definitions, + }, + taskClaimingOpts: { + maxAttempts, + }, + }); + + store.updateByQuery.mockRejectedValue(new Error('Oh no')); + + await expect( + getAllAsPromise( + taskClaiming.claimAvailableTasksIfCapacityIsAvailable({ + claimOwnershipUntil: new Date(), + }) + ) + ).rejects.toMatchInlineSnapshot(`[Error: Oh no]`); + + expect(apm.startTransaction).toHaveBeenCalledWith( + TASK_MANAGER_MARK_AS_CLAIMED, + TASK_MANAGER_TRANSACTION_TYPE + ); + expect(mockApmTrans.end).toHaveBeenCalledWith('failure'); + }); + + test('it filters claimed tasks down by supported types, maxAttempts, status, and runAt', async () => { + const maxAttempts = _.random(2, 43); + const customMaxAttempts = _.random(44, 100); + + const definitions = new TaskTypeDictionary(mockLogger()); + definitions.registerTaskDefinitions({ + foo: { + title: 'foo', + createTaskRunner: jest.fn(), + }, + bar: { + title: 'bar', + maxAttempts: customMaxAttempts, + createTaskRunner: jest.fn(), + }, + foobar: { + title: 'foobar', + maxAttempts: customMaxAttempts, + createTaskRunner: jest.fn(), + }, + }); + + const [ + { + args: { + updateByQuery: [{ query, sort }], + }, + }, + ] = await testClaimAvailableTasks({ + storeOpts: { + definitions, + }, + taskClaimingOpts: { + maxAttempts, + }, + claimingOpts: { + claimOwnershipUntil: new Date(), + }, + excludedTaskTypes: ['foobar'], + }); + expect(query).toMatchObject({ + bool: { + must: [ + { + bool: { + must: [ + { + term: { + 'task.enabled': true, + }, + }, + ], + }, + }, + { + bool: { + should: [ + { + bool: { + must: [ + { term: { 'task.status': 'idle' } }, + { range: { 'task.runAt': { lte: 'now' } } }, + ], + }, + }, + { + bool: { + must: [ + { + bool: { + should: [ + { term: { 'task.status': 'running' } }, + { term: { 'task.status': 'claiming' } }, + ], + }, + }, + { range: { 'task.retryAt': { lte: 'now' } } }, + ], + }, + }, + ], + }, + }, + ], + filter: [ + { + bool: { + must_not: [ + { + bool: { + should: [ + { term: { 'task.status': 'running' } }, + { term: { 'task.status': 'claiming' } }, + ], + must: { range: { 'task.retryAt': { gt: 'now' } } }, + }, + }, + ], + }, + }, + ], + }, + }); + expect(sort).toMatchObject([ + { + _script: { + type: 'number', + order: 'asc', + script: { + lang: 'painless', + source: ` +if (doc['task.retryAt'].size()!=0) { + return doc['task.retryAt'].value.toInstant().toEpochMilli(); +} +if (doc['task.runAt'].size()!=0) { + return doc['task.runAt'].value.toInstant().toEpochMilli(); +} + `, + }, + }, + }, + ]); + }); + + test('it should claim in batches partitioned by maxConcurrency', async () => { + const maxAttempts = _.random(2, 43); + const definitions = new TaskTypeDictionary(mockLogger()); + const taskManagerId = uuidv1(); + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: new Date(Date.now()), + }; + definitions.registerTaskDefinitions({ + unlimited: { + title: 'unlimited', + createTaskRunner: jest.fn(), + }, + limitedToZero: { + title: 'limitedToZero', + maxConcurrency: 0, + createTaskRunner: jest.fn(), + }, + anotherUnlimited: { + title: 'anotherUnlimited', + createTaskRunner: jest.fn(), + }, + finalUnlimited: { + title: 'finalUnlimited', + createTaskRunner: jest.fn(), + }, + limitedToOne: { + title: 'limitedToOne', + maxConcurrency: 1, + createTaskRunner: jest.fn(), + }, + anotherLimitedToOne: { + title: 'anotherLimitedToOne', + maxConcurrency: 1, + createTaskRunner: jest.fn(), + }, + limitedToTwo: { + title: 'limitedToTwo', + maxConcurrency: 2, + createTaskRunner: jest.fn(), + }, + }); + const results = await testClaimAvailableTasks({ + storeOpts: { + taskManagerId, + definitions, + }, + taskClaimingOpts: { + maxAttempts, + getCapacity: (type) => { + switch (type) { + case 'limitedToOne': + case 'anotherLimitedToOne': + return 1; + case 'limitedToTwo': + return 2; + default: + return 10; + } + }, + }, + claimingOpts: { + claimOwnershipUntil: new Date(), + }, + }); + + expect(results.length).toEqual(4); + + expect(results[0].args.updateByQuery[1].max_docs).toEqual(10); + expect(results[0].args.updateByQuery[0].script).toMatchObject({ + source: expect.any(String), + lang: 'painless', + params: { + fieldUpdates, + claimableTaskTypes: ['unlimited', 'anotherUnlimited', 'finalUnlimited'], + skippedTaskTypes: [ + 'limitedToZero', + 'limitedToOne', + 'anotherLimitedToOne', + 'limitedToTwo', + ], + unusedTaskTypes: [], + taskMaxAttempts: { + unlimited: maxAttempts, + }, + }, + }); + + expect(results[1].args.updateByQuery[1].max_docs).toEqual(1); + expect(results[1].args.updateByQuery[0].script).toMatchObject({ + source: expect.any(String), + lang: 'painless', + params: { + fieldUpdates, + claimableTaskTypes: ['limitedToOne'], + skippedTaskTypes: [ + 'unlimited', + 'limitedToZero', + 'anotherUnlimited', + 'finalUnlimited', + 'anotherLimitedToOne', + 'limitedToTwo', + ], + taskMaxAttempts: { + limitedToOne: maxAttempts, + }, + }, + }); + + expect(results[2].args.updateByQuery[1].max_docs).toEqual(1); + expect(results[2].args.updateByQuery[0].script).toMatchObject({ + source: expect.any(String), + lang: 'painless', + params: { + fieldUpdates, + claimableTaskTypes: ['anotherLimitedToOne'], + skippedTaskTypes: [ + 'unlimited', + 'limitedToZero', + 'anotherUnlimited', + 'finalUnlimited', + 'limitedToOne', + 'limitedToTwo', + ], + taskMaxAttempts: { + anotherLimitedToOne: maxAttempts, + }, + }, + }); + + expect(results[3].args.updateByQuery[1].max_docs).toEqual(2); + expect(results[3].args.updateByQuery[0].script).toMatchObject({ + source: expect.any(String), + lang: 'painless', + params: { + fieldUpdates, + claimableTaskTypes: ['limitedToTwo'], + skippedTaskTypes: [ + 'unlimited', + 'limitedToZero', + 'anotherUnlimited', + 'finalUnlimited', + 'limitedToOne', + 'anotherLimitedToOne', + ], + taskMaxAttempts: { + limitedToTwo: maxAttempts, + }, + }, + }); + }); + + test('it should reduce the available capacity from batch to batch', async () => { + const maxAttempts = _.random(2, 43); + const definitions = new TaskTypeDictionary(mockLogger()); + const taskManagerId = uuidv1(); + definitions.registerTaskDefinitions({ + unlimited: { + title: 'unlimited', + createTaskRunner: jest.fn(), + }, + limitedToFive: { + title: 'limitedToFive', + maxConcurrency: 5, + createTaskRunner: jest.fn(), + }, + limitedToTwo: { + title: 'limitedToTwo', + maxConcurrency: 2, + createTaskRunner: jest.fn(), + }, + }); + const results = await testClaimAvailableTasks({ + storeOpts: { + taskManagerId, + definitions, + }, + taskClaimingOpts: { + maxAttempts, + getCapacity: (type) => { + switch (type) { + case 'limitedToTwo': + return 2; + case 'limitedToFive': + return 5; + default: + return 10; + } + }, + }, + hits: [ + [ + // 7 returned by unlimited query + mockInstance({ + taskType: 'unlimited', + }), + mockInstance({ + taskType: 'unlimited', + }), + mockInstance({ + taskType: 'unlimited', + }), + mockInstance({ + taskType: 'unlimited', + }), + mockInstance({ + taskType: 'unlimited', + }), + mockInstance({ + taskType: 'unlimited', + }), + mockInstance({ + taskType: 'unlimited', + }), + ], + // 2 returned by limitedToFive query + [ + mockInstance({ + taskType: 'limitedToFive', + }), + mockInstance({ + taskType: 'limitedToFive', + }), + ], + // 1 reterned by limitedToTwo query + [ + mockInstance({ + taskType: 'limitedToTwo', + }), + ], + ], + claimingOpts: { + claimOwnershipUntil: new Date(), + }, + }); + + expect(results.length).toEqual(3); + + expect(results[0].args.updateByQuery[1].max_docs).toEqual(10); + + // only capacity for 3, even though 5 are allowed + expect(results[1].args.updateByQuery[1].max_docs).toEqual(3); + + // only capacity for 1, even though 2 are allowed + expect(results[2].args.updateByQuery[1].max_docs).toEqual(1); + }); + + test('it shuffles the types claimed in batches to ensure no type starves another', async () => { + const maxAttempts = _.random(2, 43); + const definitions = new TaskTypeDictionary(mockLogger()); + const taskManagerId = uuidv1(); + definitions.registerTaskDefinitions({ + unlimited: { + title: 'unlimited', + createTaskRunner: jest.fn(), + }, + anotherUnlimited: { + title: 'anotherUnlimited', + createTaskRunner: jest.fn(), + }, + finalUnlimited: { + title: 'finalUnlimited', + createTaskRunner: jest.fn(), + }, + limitedToOne: { + title: 'limitedToOne', + maxConcurrency: 1, + createTaskRunner: jest.fn(), + }, + anotherLimitedToOne: { + title: 'anotherLimitedToOne', + maxConcurrency: 1, + createTaskRunner: jest.fn(), + }, + limitedToTwo: { + title: 'limitedToTwo', + maxConcurrency: 2, + createTaskRunner: jest.fn(), + }, + }); + + const { taskClaiming, store } = initialiseTestClaiming({ + storeOpts: { + taskManagerId, + definitions, + }, + taskClaimingOpts: { + maxAttempts, + getCapacity: (type) => { + switch (type) { + case 'limitedToOne': + case 'anotherLimitedToOne': + return 1; + case 'limitedToTwo': + return 2; + default: + return 10; + } + }, + }, + }); + + async function getUpdateByQueryScriptParams() { + return ( + await getAllAsPromise( + taskClaiming.claimAvailableTasksIfCapacityIsAvailable({ + claimOwnershipUntil: new Date(), + }) + ) + ).map( + (result, index) => + ( + store.updateByQuery.mock.calls[index][0] as { + query: MustNotCondition; + size: number; + sort: string | string[]; + script: { + params: { + [claimableTaskTypes: string]: string[]; + }; + }; + } + ).script.params.claimableTaskTypes + ); + } + + const firstCycle = await getUpdateByQueryScriptParams(); + store.updateByQuery.mockClear(); + const secondCycle = await getUpdateByQueryScriptParams(); + + expect(firstCycle.length).toEqual(4); + expect(secondCycle.length).toEqual(4); + expect(firstCycle).not.toMatchObject(secondCycle); + }); + + test('it passes any unusedTaskTypes to script', async () => { + const maxAttempts = _.random(2, 43); + const customMaxAttempts = _.random(44, 100); + const taskManagerId = uuidv1(); + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: new Date(Date.now()), + }; + const definitions = new TaskTypeDictionary(mockLogger()); + definitions.registerTaskDefinitions({ + foo: { + title: 'foo', + createTaskRunner: jest.fn(), + }, + bar: { + title: 'bar', + maxAttempts: customMaxAttempts, + createTaskRunner: jest.fn(), + }, + foobar: { + title: 'foobar', + maxAttempts: customMaxAttempts, + createTaskRunner: jest.fn(), + }, + }); + + const [ + { + args: { + updateByQuery: [{ query, script }], + }, + }, + ] = await testClaimAvailableTasks({ + storeOpts: { + definitions, + taskManagerId, + }, + taskClaimingOpts: { + maxAttempts, + }, + claimingOpts: { + claimOwnershipUntil: new Date(), + }, + excludedTaskTypes: ['foobar'], + unusedTaskTypes: ['barfoo'], + }); + expect(query).toMatchObject({ + bool: { + must: [ + { + bool: { + must: [ + { + term: { + 'task.enabled': true, + }, + }, + ], + }, + }, + { + bool: { + should: [ + { + bool: { + must: [ + { term: { 'task.status': 'idle' } }, + { range: { 'task.runAt': { lte: 'now' } } }, + ], + }, + }, + { + bool: { + must: [ + { + bool: { + should: [ + { term: { 'task.status': 'running' } }, + { term: { 'task.status': 'claiming' } }, + ], + }, + }, + { range: { 'task.retryAt': { lte: 'now' } } }, + ], + }, + }, + ], + }, + }, + ], + filter: [ + { + bool: { + must_not: [ + { + bool: { + should: [ + { term: { 'task.status': 'running' } }, + { term: { 'task.status': 'claiming' } }, + ], + must: { range: { 'task.retryAt': { gt: 'now' } } }, + }, + }, + ], + }, + }, + ], + }, + }); + expect(script).toMatchObject({ + source: expect.any(String), + lang: 'painless', + params: { + fieldUpdates, + claimableTaskTypes: ['foo', 'bar'], + skippedTaskTypes: ['foobar'], + unusedTaskTypes: ['barfoo'], + taskMaxAttempts: { + bar: customMaxAttempts, + foo: maxAttempts, + }, + }, + }); + }); + + test('it claims tasks by setting their ownerId, status and retryAt', async () => { + const taskManagerId = uuidv1(); + const claimOwnershipUntil = new Date(Date.now()); + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + }; + const [ + { + args: { + updateByQuery: [{ script }], + }, + }, + ] = await testClaimAvailableTasks({ + storeOpts: { + taskManagerId, + }, + taskClaimingOpts: {}, + claimingOpts: { + claimOwnershipUntil, + }, + }); + expect(script).toMatchObject({ + source: expect.any(String), + lang: 'painless', + params: { + fieldUpdates, + claimableTaskTypes: ['report', 'dernstraight', 'yawn'], + skippedTaskTypes: [], + taskMaxAttempts: { + dernstraight: 2, + report: 2, + yawn: 2, + }, + }, + }); + }); + + test('it filters out running tasks', async () => { + const taskManagerId = uuidv1(); + const claimOwnershipUntil = new Date(Date.now()); + const runAt = new Date(); + const tasks = [ + mockInstance({ + id: 'aaa', + runAt, + taskType: 'yawn', + schedule: undefined, + attempts: 0, + status: TaskStatus.Claiming, + params: { hello: 'world' }, + state: { baby: 'Henhen' }, + user: 'jimbo', + scope: ['reporting'], + ownerId: taskManagerId, + }), + ]; + const [ + { + result: { docs }, + args: { + search: { query }, + }, + }, + ] = await testClaimAvailableTasks({ + storeOpts: { + taskManagerId, + }, + taskClaimingOpts: {}, + claimingOpts: { + claimOwnershipUntil, + }, + hits: [tasks], + }); + + expect(query).toMatchObject({ + bool: { + must: [ + { + term: { + 'task.ownerId': taskManagerId, + }, + }, + { term: { 'task.status': 'claiming' } }, + { + bool: { + should: [ + { + term: { + 'task.taskType': 'report', + }, + }, + { + term: { + 'task.taskType': 'dernstraight', + }, + }, + { + term: { + 'task.taskType': 'yawn', + }, + }, + ], + }, + }, + ], + }, + }); + + expect(docs).toMatchObject([ + { + attempts: 0, + id: 'aaa', + schedule: undefined, + params: { hello: 'world' }, + runAt, + scope: ['reporting'], + state: { baby: 'Henhen' }, + status: 'claiming', + taskType: 'yawn', + user: 'jimbo', + ownerId: taskManagerId, + }, + ]); + }); + + test('it returns task objects', async () => { + const taskManagerId = uuidv1(); + const claimOwnershipUntil = new Date(Date.now()); + const runAt = new Date(); + const tasks = [ + mockInstance({ + id: 'aaa', + runAt, + taskType: 'yawn', + schedule: undefined, + attempts: 0, + status: TaskStatus.Claiming, + params: { hello: 'world' }, + state: { baby: 'Henhen' }, + user: 'jimbo', + scope: ['reporting'], + ownerId: taskManagerId, + }), + mockInstance({ + id: 'bbb', + runAt, + taskType: 'yawn', + schedule: { interval: '5m' }, + attempts: 2, + status: TaskStatus.Claiming, + params: { shazm: 1 }, + state: { henry: 'The 8th' }, + user: 'dabo', + scope: ['reporting', 'ceo'], + ownerId: taskManagerId, + }), + ]; + const [ + { + result: { docs }, + args: { + search: { query }, + }, + }, + ] = await testClaimAvailableTasks({ + storeOpts: { + taskManagerId, + }, + taskClaimingOpts: {}, + claimingOpts: { + claimOwnershipUntil, + }, + hits: [tasks], + }); + + expect(query).toMatchObject({ + bool: { + must: [ + { + term: { + 'task.ownerId': taskManagerId, + }, + }, + { term: { 'task.status': 'claiming' } }, + { + bool: { + should: [ + { + term: { + 'task.taskType': 'report', + }, + }, + { + term: { + 'task.taskType': 'dernstraight', + }, + }, + { + term: { + 'task.taskType': 'yawn', + }, + }, + ], + }, + }, + ], + }, + }); + + expect(docs).toMatchObject([ + { + attempts: 0, + id: 'aaa', + schedule: undefined, + params: { hello: 'world' }, + runAt, + scope: ['reporting'], + state: { baby: 'Henhen' }, + status: 'claiming', + taskType: 'yawn', + user: 'jimbo', + ownerId: taskManagerId, + }, + { + attempts: 2, + id: 'bbb', + schedule: { interval: '5m' }, + params: { shazm: 1 }, + runAt, + scope: ['reporting', 'ceo'], + state: { henry: 'The 8th' }, + status: 'claiming', + taskType: 'yawn', + user: 'dabo', + ownerId: taskManagerId, + }, + ]); + }); + + test('it returns version_conflicts that do not include conflicts that were proceeded against', async () => { + const taskManagerId = uuidv1(); + const claimOwnershipUntil = new Date(Date.now()); + const runAt = new Date(); + const tasks = [ + mockInstance({ + runAt, + taskType: 'foo', + schedule: undefined, + attempts: 0, + status: TaskStatus.Claiming, + params: { hello: 'world' }, + state: { baby: 'Henhen' }, + user: 'jimbo', + scope: ['reporting'], + ownerId: taskManagerId, + }), + mockInstance({ + runAt, + taskType: 'bar', + schedule: { interval: '5m' }, + attempts: 2, + status: TaskStatus.Claiming, + params: { shazm: 1 }, + state: { henry: 'The 8th' }, + user: 'dabo', + scope: ['reporting', 'ceo'], + ownerId: taskManagerId, + }), + ]; + const maxDocs = 10; + const [ + { + result: { + stats: { tasksUpdated, tasksConflicted, tasksClaimed }, + }, + }, + ] = await testClaimAvailableTasks({ + storeOpts: { + taskManagerId, + }, + taskClaimingOpts: { getCapacity: () => maxDocs }, + claimingOpts: { + claimOwnershipUntil, + }, + hits: [tasks], + // assume there were 20 version conflists, but thanks to `conflicts="proceed"` + // we proceeded to claim tasks + versionConflicts: 20, + }); + + expect(tasksUpdated).toEqual(2); + // ensure we only count conflicts that *may* have counted against max_docs, no more than that + expect(tasksConflicted).toEqual(10 - tasksUpdated!); + expect(tasksClaimed).toEqual(2); + }); + }); + + describe('task events', () => { + function generateTasks(taskManagerId: string) { + const runAt = new Date(); + const tasks = [ + { + id: 'claimed-by-id', + runAt, + taskType: 'foo', + schedule: undefined, + attempts: 0, + status: TaskStatus.Claiming, + params: { hello: 'world' }, + state: { baby: 'Henhen' }, + user: 'jimbo', + scope: ['reporting'], + ownerId: taskManagerId, + startedAt: null, + retryAt: null, + scheduledAt: new Date(), + traceparent: 'parent', + }, + { + id: 'claimed-by-schedule', + runAt, + taskType: 'bar', + schedule: { interval: '5m' }, + attempts: 2, + status: TaskStatus.Claiming, + params: { shazm: 1 }, + state: { henry: 'The 8th' }, + user: 'dabo', + scope: ['reporting', 'ceo'], + ownerId: taskManagerId, + startedAt: null, + retryAt: null, + scheduledAt: new Date(), + traceparent: 'newParent', + }, + { + id: 'already-running', + runAt, + taskType: 'bar', + schedule: { interval: '5m' }, + attempts: 2, + status: TaskStatus.Running, + params: { shazm: 1 }, + state: { henry: 'The 8th' }, + user: 'dabo', + scope: ['reporting', 'ceo'], + ownerId: taskManagerId, + startedAt: null, + retryAt: null, + scheduledAt: new Date(), + traceparent: '', + }, + ]; + + return { taskManagerId, runAt, tasks }; + } + + function instantiateStoreWithMockedApiResponses({ + taskManagerId = uuidv4(), + definitions = taskDefinitions, + getCapacity = () => 10, + tasksClaimed, + }: Partial> & { + taskManagerId?: string; + tasksClaimed?: ConcreteTaskInstance[][]; + } = {}) { + const { runAt, tasks: generatedTasks } = generateTasks(taskManagerId); + const taskCycles = tasksClaimed ?? [generatedTasks]; + + const taskStore = taskStoreMock.create({ taskManagerId }); + taskStore.convertToSavedObjectIds.mockImplementation((ids) => ids.map((id) => `task:${id}`)); + for (const docs of taskCycles) { + taskStore.fetch.mockResolvedValueOnce({ docs }); + taskStore.updateByQuery.mockResolvedValueOnce({ + updated: docs.length, + version_conflicts: 0, + total: docs.length, + }); + } + + taskStore.fetch.mockResolvedValue({ docs: [] }); + taskStore.updateByQuery.mockResolvedValue({ + updated: 0, + version_conflicts: 0, + total: 0, + }); + + const taskClaiming = new TaskClaiming({ + logger: taskManagerLogger, + strategy: 'default', + definitions, + excludedTaskTypes: [], + unusedTypes: [], + taskStore, + maxAttempts: 2, + getCapacity, + }); + + return { taskManagerId, runAt, taskClaiming }; + } + + test('emits an event when a task is succesfully by scheduling', async () => { + const { taskManagerId, runAt, taskClaiming } = instantiateStoreWithMockedApiResponses(); + + const promise = taskClaiming.events + .pipe( + filter( + (event: TaskEvent) => event.id === 'claimed-by-schedule' + ), + take(1) + ) + .toPromise(); + + await getFirstAsPromise( + taskClaiming.claimAvailableTasksIfCapacityIsAvailable({ + claimOwnershipUntil: new Date(), + }) + ); + + const event = await promise; + expect(event).toMatchObject( + asTaskClaimEvent( + 'claimed-by-schedule', + asOk({ + id: 'claimed-by-schedule', + runAt, + taskType: 'bar', + schedule: { interval: '5m' }, + attempts: 2, + status: 'claiming' as TaskStatus, + params: { shazm: 1 }, + state: { henry: 'The 8th' }, + user: 'dabo', + scope: ['reporting', 'ceo'], + ownerId: taskManagerId, + startedAt: null, + retryAt: null, + scheduledAt: new Date(), + traceparent: 'newParent', + }) + ) + ); + }); + }); +}); + +function generateFakeTasks(count: number = 1) { + return _.times(count, (index) => mockInstance({ id: `task:id-${index}` })); +} + +function mockInstance(instance: Partial = {}) { + return Object.assign( + { + id: uuidv4(), + taskType: 'bar', + sequenceNumber: 32, + primaryTerm: 32, + runAt: new Date(), + scheduledAt: new Date(), + startedAt: null, + retryAt: null, + attempts: 0, + params: {}, + scope: ['reporting'], + state: {}, + status: 'idle', + user: 'example', + ownerId: null, + traceparent: '', + }, + instance + ); +} + +function getFirstAsPromise(obs$: Observable): Promise { + return new Promise((resolve, reject) => { + obs$.subscribe(resolve, reject); + }); +} +function getAllAsPromise(obs$: Observable): Promise { + return new Promise((resolve, reject) => { + obs$.pipe(toArray()).subscribe(resolve, reject); + }); +} diff --git a/x-pack/plugins/task_manager/server/task_claimers/strategy_default.ts b/x-pack/plugins/task_manager/server/task_claimers/strategy_default.ts new file mode 100644 index 0000000000000..0d9ccb2ef723d --- /dev/null +++ b/x-pack/plugins/task_manager/server/task_claimers/strategy_default.ts @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * This module contains helpers for managing the task manager storage layer. + */ +import apm from 'elastic-apm-node'; +import minimatch from 'minimatch'; +import { Subject, Observable, from, of } from 'rxjs'; +import { mergeScan } from 'rxjs/operators'; +import { groupBy, pick } from 'lodash'; + +import { asOk } from '../lib/result_type'; +import { TaskTypeDictionary } from '../task_type_dictionary'; +import { TaskClaimerOpts, ClaimOwnershipResult } from '.'; +import { ConcreteTaskInstance } from '../task'; +import { TASK_MANAGER_TRANSACTION_TYPE } from '../task_running'; +import { isLimited, TASK_MANAGER_MARK_AS_CLAIMED } from '../queries/task_claiming'; +import { TaskClaim, asTaskClaimEvent, startTaskTimer } from '../task_events'; +import { shouldBeOneOf, mustBeAllOf, filterDownBy, matchesClauses } from '../queries/query_clauses'; + +import { + updateFieldsAndMarkAsFailed, + IdleTaskWithExpiredRunAt, + InactiveTasks, + RunningOrClaimingTaskWithExpiredRetryAt, + SortByRunAtAndRetryAt, + tasksClaimedByOwner, + tasksOfType, + EnabledTask, +} from '../queries/mark_available_tasks_as_claimed'; + +import { + correctVersionConflictsForContinuation, + TaskStore, + UpdateByQueryResult, + SearchOpts, +} from '../task_store'; + +interface OwnershipClaimingOpts { + claimOwnershipUntil: Date; + size: number; + taskTypes: Set; + taskStore: TaskStore; + events$: Subject; + definitions: TaskTypeDictionary; + unusedTypes: string[]; + excludedTaskTypes: string[]; + taskMaxAttempts: Record; +} + +export function claimAvailableTasksDefault( + opts: TaskClaimerOpts +): Observable { + const { getCapacity, claimOwnershipUntil, batches, events$, taskStore } = opts; + const { definitions, unusedTypes, excludedTaskTypes, taskMaxAttempts } = opts; + const initialCapacity = getCapacity(); + return from(batches).pipe( + mergeScan( + (accumulatedResult, batch) => { + const stopTaskTimer = startTaskTimer(); + const capacity = Math.min( + initialCapacity - accumulatedResult.stats.tasksClaimed, + isLimited(batch) ? getCapacity(batch.tasksTypes) : getCapacity() + ); + // if we have no more capacity, short circuit here + if (capacity <= 0) { + return of(accumulatedResult); + } + return from( + executeClaimAvailableTasks({ + claimOwnershipUntil, + size: capacity, + events$, + taskTypes: isLimited(batch) ? new Set([batch.tasksTypes]) : batch.tasksTypes, + taskStore, + definitions, + unusedTypes, + excludedTaskTypes, + taskMaxAttempts, + }).then((result) => { + const { stats, docs } = accumulateClaimOwnershipResults(accumulatedResult, result); + stats.tasksConflicted = correctVersionConflictsForContinuation( + stats.tasksClaimed, + stats.tasksConflicted, + initialCapacity + ); + return { stats, docs, timing: stopTaskTimer() }; + }) + ); + }, + // initialise the accumulation with no results + accumulateClaimOwnershipResults(), + // only run one batch at a time + 1 + ) + ); +} + +async function executeClaimAvailableTasks( + opts: OwnershipClaimingOpts +): Promise { + const { taskStore, size, taskTypes, events$ } = opts; + const { updated: tasksUpdated, version_conflicts: tasksConflicted } = + await markAvailableTasksAsClaimed(opts); + + const docs = tasksUpdated > 0 ? await sweepForClaimedTasks(taskStore, taskTypes, size) : []; + + emitEvents( + events$, + docs.map((doc) => asTaskClaimEvent(doc.id, asOk(doc))) + ); + + const stats = { + tasksUpdated, + tasksConflicted, + tasksClaimed: docs.length, + }; + + return { + stats, + docs, + }; +} + +function emitEvents(events$: Subject, events: TaskClaim[]) { + events.forEach((event) => events$.next(event)); +} + +function isTaskTypeExcluded(excludedTaskTypes: string[], taskType: string) { + for (const excludedType of excludedTaskTypes) { + if (minimatch(taskType, excludedType)) { + return true; + } + } + + return false; +} + +async function markAvailableTasksAsClaimed({ + definitions, + excludedTaskTypes, + taskStore, + claimOwnershipUntil, + size, + taskTypes, + unusedTypes, + taskMaxAttempts, +}: OwnershipClaimingOpts): Promise { + const { taskTypesToSkip = [], taskTypesToClaim = [] } = groupBy( + definitions.getAllTypes(), + (type) => + taskTypes.has(type) && !isTaskTypeExcluded(excludedTaskTypes, type) + ? 'taskTypesToClaim' + : 'taskTypesToSkip' + ); + const queryForScheduledTasks = mustBeAllOf( + // Task must be enabled + EnabledTask, + // Either a task with idle status and runAt <= now or + // status running or claiming with a retryAt <= now. + shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt) + ); + + const sort: NonNullable = [SortByRunAtAndRetryAt]; + const query = matchesClauses(queryForScheduledTasks, filterDownBy(InactiveTasks)); + const script = updateFieldsAndMarkAsFailed({ + fieldUpdates: { + ownerId: taskStore.taskManagerId, + retryAt: claimOwnershipUntil, + }, + claimableTaskTypes: taskTypesToClaim, + skippedTaskTypes: taskTypesToSkip, + unusedTaskTypes: unusedTypes, + taskMaxAttempts: pick(taskMaxAttempts, taskTypesToClaim), + }); + + const apmTrans = apm.startTransaction( + TASK_MANAGER_MARK_AS_CLAIMED, + TASK_MANAGER_TRANSACTION_TYPE + ); + + try { + const result = await taskStore.updateByQuery( + { + query, + script, + sort, + }, + { + max_docs: size, + } + ); + apmTrans.end('success'); + return result; + } catch (err) { + apmTrans.end('failure'); + throw err; + } +} + +async function sweepForClaimedTasks( + taskStore: TaskStore, + taskTypes: Set, + size: number +): Promise { + const claimedTasksQuery = tasksClaimedByOwner( + taskStore.taskManagerId, + tasksOfType([...taskTypes]) + ); + const { docs } = await taskStore.fetch({ + query: claimedTasksQuery, + size, + sort: SortByRunAtAndRetryAt, + seq_no_primary_term: true, + }); + + return docs; +} + +function emptyClaimOwnershipResult() { + return { + stats: { + tasksUpdated: 0, + tasksConflicted: 0, + tasksClaimed: 0, + tasksRejected: 0, + }, + docs: [], + }; +} + +function accumulateClaimOwnershipResults( + prev: ClaimOwnershipResult = emptyClaimOwnershipResult(), + next?: ClaimOwnershipResult +) { + if (next) { + const { stats, docs, timing } = next; + const res = { + stats: { + tasksUpdated: stats.tasksUpdated + prev.stats.tasksUpdated, + tasksConflicted: stats.tasksConflicted + prev.stats.tasksConflicted, + tasksClaimed: stats.tasksClaimed + prev.stats.tasksClaimed, + }, + docs, + timing, + }; + return res; + } + return prev; +} From a8647151cb69bbd22f5f1100fe0cfc30599a24f8 Mon Sep 17 00:00:00 2001 From: Drew Tate Date: Tue, 28 Nov 2023 12:30:47 -0700 Subject: [PATCH 26/30] Unify style for embeddable-stack loaders (#171238) ## Summary Fix https://github.com/elastic/kibana/issues/170428 The bug this is intended to resolve requires some in-depth steps to reproduce. Follow the instructions in the issue above. Then, merge in this branch and compare. ### Checklist - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 1 + package.json | 1 + packages/kbn-panel-loader/README.md | 3 ++ .../kbn-panel-loader/index.tsx | 6 ++-- packages/kbn-panel-loader/jest.config.js | 13 +++++++ packages/kbn-panel-loader/kibana.jsonc | 5 +++ packages/kbn-panel-loader/package.json | 6 ++++ packages/kbn-panel-loader/tsconfig.json | 18 ++++++++++ .../component/grid/dashboard_grid_item.tsx | 1 + .../embeddable_panel/embeddable_panel.tsx | 36 ++++++++++++++----- .../public/embeddable_panel/index.tsx | 7 ++-- src/plugins/embeddable/tsconfig.json | 1 + .../react_expression_renderer.tsx | 5 +-- .../react_expression_renderer_wrapper.tsx | 4 +-- src/plugins/expressions/tsconfig.json | 1 + tsconfig.base.json | 2 ++ .../no_change_points_warning.tsx | 6 ++-- .../embeddable_chart_component_wrapper.tsx | 2 +- .../embeddable/embeddable_component.tsx | 3 +- x-pack/plugins/lens/tsconfig.json | 3 +- .../explorer/swimlane_container.tsx | 4 +++ .../embeddable_swim_lane_container.tsx | 1 + yarn.lock | 4 +++ 23 files changed, 111 insertions(+), 22 deletions(-) create mode 100644 packages/kbn-panel-loader/README.md rename src/plugins/embeddable/public/embeddable_panel/embeddable_loading_indicator.tsx => packages/kbn-panel-loader/index.tsx (78%) create mode 100644 packages/kbn-panel-loader/jest.config.js create mode 100644 packages/kbn-panel-loader/kibana.jsonc create mode 100644 packages/kbn-panel-loader/package.json create mode 100644 packages/kbn-panel-loader/tsconfig.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a31103b1f35bc..88e3847749c3b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -566,6 +566,7 @@ packages/kbn-osquery-io-ts-types @elastic/security-asset-management x-pack/plugins/osquery @elastic/security-defend-workflows examples/partial_results_example @elastic/kibana-data-discovery x-pack/plugins/painless_lab @elastic/platform-deployment-management +packages/kbn-panel-loader @elastic/kibana-presentation packages/kbn-peggy @elastic/kibana-operations packages/kbn-peggy-loader @elastic/kibana-operations packages/kbn-performance-testing-dataset-extractor @elastic/kibana-performance-testing diff --git a/package.json b/package.json index 92a0ae4f2be74..bb666f88f738d 100644 --- a/package.json +++ b/package.json @@ -581,6 +581,7 @@ "@kbn/osquery-plugin": "link:x-pack/plugins/osquery", "@kbn/paertial-results-example-plugin": "link:examples/partial_results_example", "@kbn/painless-lab-plugin": "link:x-pack/plugins/painless_lab", + "@kbn/panel-loader": "link:packages/kbn-panel-loader", "@kbn/portable-dashboards-example": "link:examples/portable_dashboards_example", "@kbn/preboot-example-plugin": "link:examples/preboot_example", "@kbn/presentation-util-plugin": "link:src/plugins/presentation_util", diff --git a/packages/kbn-panel-loader/README.md b/packages/kbn-panel-loader/README.md new file mode 100644 index 0000000000000..5b14fa96c6c40 --- /dev/null +++ b/packages/kbn-panel-loader/README.md @@ -0,0 +1,3 @@ +# @kbn/panel-loader + +Contains a generic loader which should be used to indicate that a chart is loading diff --git a/src/plugins/embeddable/public/embeddable_panel/embeddable_loading_indicator.tsx b/packages/kbn-panel-loader/index.tsx similarity index 78% rename from src/plugins/embeddable/public/embeddable_panel/embeddable_loading_indicator.tsx rename to packages/kbn-panel-loader/index.tsx index e0a4ca5cf5fe7..ad4e8751910f3 100644 --- a/src/plugins/embeddable/public/embeddable_panel/embeddable_loading_indicator.tsx +++ b/packages/kbn-panel-loader/index.tsx @@ -9,14 +9,14 @@ import React from 'react'; import { EuiLoadingChart, EuiPanel } from '@elastic/eui'; -export const EmbeddableLoadingIndicator = ({ showShadow }: { showShadow?: boolean }) => { +export const PanelLoader = (props: { showShadow?: boolean; dataTestSubj?: string }) => { return ( diff --git a/packages/kbn-panel-loader/jest.config.js b/packages/kbn-panel-loader/jest.config.js new file mode 100644 index 0000000000000..e8cfab95a0732 --- /dev/null +++ b/packages/kbn-panel-loader/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-panel-loader'], +}; diff --git a/packages/kbn-panel-loader/kibana.jsonc b/packages/kbn-panel-loader/kibana.jsonc new file mode 100644 index 0000000000000..5fc518a8983ca --- /dev/null +++ b/packages/kbn-panel-loader/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-browser", + "id": "@kbn/panel-loader", + "owner": "@elastic/kibana-presentation" +} diff --git a/packages/kbn-panel-loader/package.json b/packages/kbn-panel-loader/package.json new file mode 100644 index 0000000000000..94394420475b1 --- /dev/null +++ b/packages/kbn-panel-loader/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/panel-loader", + "private": true, + "version": "1.0.0", + "license": "SSPL-1.0 OR Elastic License 2.0" +} \ No newline at end of file diff --git a/packages/kbn-panel-loader/tsconfig.json b/packages/kbn-panel-loader/tsconfig.json new file mode 100644 index 0000000000000..f885e788791b7 --- /dev/null +++ b/packages/kbn-panel-loader/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + "**/*.tsx", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [] +} diff --git a/src/plugins/dashboard/public/dashboard_container/component/grid/dashboard_grid_item.tsx b/src/plugins/dashboard/public/dashboard_container/component/grid/dashboard_grid_item.tsx index 9eb12379741ab..083d47ee4e69e 100644 --- a/src/plugins/dashboard/public/dashboard_container/component/grid/dashboard_grid_item.tsx +++ b/src/plugins/dashboard/public/dashboard_container/component/grid/dashboard_grid_item.tsx @@ -108,6 +108,7 @@ export const Item = React.forwardRef( key={type} index={index} showBadges={true} + showShadow={true} showNotifications={true} onPanelStatusChange={onPanelStatusChange} embeddable={() => container.untilEmbeddableLoaded(id)} diff --git a/src/plugins/embeddable/public/embeddable_panel/embeddable_panel.tsx b/src/plugins/embeddable/public/embeddable_panel/embeddable_panel.tsx index 93279e311b065..2c9a30431218d 100644 --- a/src/plugins/embeddable/public/embeddable_panel/embeddable_panel.tsx +++ b/src/plugins/embeddable/public/embeddable_panel/embeddable_panel.tsx @@ -12,9 +12,8 @@ import { distinct, map } from 'rxjs'; import React, { ReactNode, useEffect, useMemo, useState } from 'react'; import { EuiFlexGroup, EuiFlexItem, EuiPanel, htmlIdGenerator } from '@elastic/eui'; -import { isPromise } from '@kbn/std'; import { UI_SETTINGS } from '@kbn/data-plugin/common'; - +import { PanelLoader } from '@kbn/panel-loader'; import { EditPanelAction, RemovePanelAction, @@ -51,6 +50,7 @@ const getEventStatus = (output: EmbeddableOutput): EmbeddablePhase => { export const EmbeddablePanel = (panelProps: UnwrappedEmbeddablePanelProps) => { const { hideHeader, showShadow, embeddable, hideInspector, onPanelStatusChange } = panelProps; const [node, setNode] = useState(); + const [initialLoadComplete, setInitialLoadComplete] = useState(false); const embeddableRoot: React.RefObject = useMemo(() => React.createRef(), []); const headerId = useMemo(() => htmlIdGenerator()(), []); @@ -129,6 +129,11 @@ export const EmbeddablePanel = (panelProps: UnwrappedEmbeddablePanelProps) => { * Select state from the embeddable */ const loading = useSelectFromEmbeddableOutput('loading', embeddable); + + if (loading === false && !initialLoadComplete) { + setInitialLoadComplete(true); + } + const viewMode = useSelectFromEmbeddableInput('viewMode', embeddable); /** @@ -136,21 +141,30 @@ export const EmbeddablePanel = (panelProps: UnwrappedEmbeddablePanelProps) => { */ useEffect(() => { if (!embeddableRoot.current) return; - const nextNode = embeddable.render(embeddableRoot.current) ?? undefined; - if (isPromise(nextNode)) { - nextNode.then((resolved) => setNode(resolved)); - } else { + + let cancelled = false; + + const render = async (root: HTMLDivElement) => { + const nextNode = (await embeddable.render(root)) ?? undefined; + + if (cancelled) return; + setNode(nextNode); - } + }; + + render(embeddableRoot.current); + const errorSubscription = embeddable.getOutput$().subscribe({ next: (output) => { setOutputError(output.error); }, error: (error) => setOutputError(error), }); + return () => { embeddable?.destroy(); errorSubscription?.unsubscribe(); + cancelled = true; }; }, [embeddable, embeddableRoot]); @@ -207,7 +221,13 @@ export const EmbeddablePanel = (panelProps: UnwrappedEmbeddablePanelProps) => { )} -
+ {!initialLoadComplete && } +
{node}
diff --git a/src/plugins/embeddable/public/embeddable_panel/index.tsx b/src/plugins/embeddable/public/embeddable_panel/index.tsx index 9e29d49cbfd5b..7c8c87d255981 100644 --- a/src/plugins/embeddable/public/embeddable_panel/index.tsx +++ b/src/plugins/embeddable/public/embeddable_panel/index.tsx @@ -8,16 +8,19 @@ import React from 'react'; +import { PanelLoader } from '@kbn/panel-loader'; import { EmbeddablePanelProps } from './types'; import { useEmbeddablePanel } from './use_embeddable_panel'; -import { EmbeddableLoadingIndicator } from './embeddable_loading_indicator'; /** * Loads and renders an embeddable. */ export const EmbeddablePanel = (props: EmbeddablePanelProps) => { const result = useEmbeddablePanel({ embeddable: props.embeddable }); - if (!result) return ; + if (!result) + return ( + + ); const { embeddable, ...passThroughProps } = props; return ; }; diff --git a/src/plugins/embeddable/tsconfig.json b/src/plugins/embeddable/tsconfig.json index 9055c59c2d846..8239fba0d0d18 100644 --- a/src/plugins/embeddable/tsconfig.json +++ b/src/plugins/embeddable/tsconfig.json @@ -34,6 +34,7 @@ "@kbn/react-kibana-mount", "@kbn/unified-search-plugin", "@kbn/data-views-plugin", + "@kbn/panel-loader", ], "exclude": ["target/**/*"] } diff --git a/src/plugins/expressions/public/react_expression_renderer/react_expression_renderer.tsx b/src/plugins/expressions/public/react_expression_renderer/react_expression_renderer.tsx index 1d479bd9b4c1c..43723ecc06984 100644 --- a/src/plugins/expressions/public/react_expression_renderer/react_expression_renderer.tsx +++ b/src/plugins/expressions/public/react_expression_renderer/react_expression_renderer.tsx @@ -8,7 +8,8 @@ import React, { useRef } from 'react'; import classNames from 'classnames'; -import { EuiLoadingChart, EuiProgress, useEuiTheme } from '@elastic/eui'; +import { PanelLoader } from '@kbn/panel-loader'; +import { EuiProgress, useEuiTheme } from '@elastic/eui'; import { ExpressionRenderError } from '../types'; import type { ExpressionRendererParams } from './use_expression_renderer'; import { useExpressionRenderer } from './use_expression_renderer'; @@ -56,7 +57,7 @@ export function ReactExpressionRenderer({ return (
- {isEmpty && } + {isEmpty && } {isLoading && ( )} diff --git a/src/plugins/expressions/public/react_expression_renderer_wrapper.tsx b/src/plugins/expressions/public/react_expression_renderer_wrapper.tsx index fe0bfcc42d602..57e2951c12443 100644 --- a/src/plugins/expressions/public/react_expression_renderer_wrapper.tsx +++ b/src/plugins/expressions/public/react_expression_renderer_wrapper.tsx @@ -7,7 +7,7 @@ */ import React, { lazy, Suspense } from 'react'; -import { EuiLoadingSpinner } from '@elastic/eui'; +import { PanelLoader } from '@kbn/panel-loader'; import type { ReactExpressionRendererProps } from './react_expression_renderer'; const ReactExpressionRendererComponent = lazy(async () => { @@ -17,7 +17,7 @@ const ReactExpressionRendererComponent = lazy(async () => { }); export const ReactExpressionRenderer = (props: ReactExpressionRendererProps) => ( - }> + }> ); diff --git a/src/plugins/expressions/tsconfig.json b/src/plugins/expressions/tsconfig.json index 1f1128ae6ab19..94f1d7852da39 100644 --- a/src/plugins/expressions/tsconfig.json +++ b/src/plugins/expressions/tsconfig.json @@ -16,6 +16,7 @@ "@kbn/std", "@kbn/core-execution-context-common", "@kbn/tinymath", + "@kbn/panel-loader", ], "exclude": [ "target/**/*", diff --git a/tsconfig.base.json b/tsconfig.base.json index 3c4a87242841b..526eb297c6022 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1126,6 +1126,8 @@ "@kbn/paertial-results-example-plugin/*": ["examples/partial_results_example/*"], "@kbn/painless-lab-plugin": ["x-pack/plugins/painless_lab"], "@kbn/painless-lab-plugin/*": ["x-pack/plugins/painless_lab/*"], + "@kbn/panel-loader": ["packages/kbn-panel-loader"], + "@kbn/panel-loader/*": ["packages/kbn-panel-loader/*"], "@kbn/peggy": ["packages/kbn-peggy"], "@kbn/peggy/*": ["packages/kbn-peggy/*"], "@kbn/peggy-loader": ["packages/kbn-peggy-loader"], diff --git a/x-pack/plugins/aiops/public/components/change_point_detection/no_change_points_warning.tsx b/x-pack/plugins/aiops/public/components/change_point_detection/no_change_points_warning.tsx index 0f5303725e514..ce15116d968cb 100644 --- a/x-pack/plugins/aiops/public/components/change_point_detection/no_change_points_warning.tsx +++ b/x-pack/plugins/aiops/public/components/change_point_detection/no_change_points_warning.tsx @@ -5,11 +5,13 @@ * 2.0. */ -import React, { type FC } from 'react'; +import React from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiEmptyPrompt } from '@elastic/eui'; -export const NoChangePointsWarning: FC = () => { +export const NoChangePointsWarning = (props: { onRenderComplete?: () => void }) => { + props.onRenderComplete?.(); + return ( + )}
); diff --git a/x-pack/plugins/lens/public/embeddable/embeddable_component.tsx b/x-pack/plugins/lens/public/embeddable/embeddable_component.tsx index d93e3729a6541..986f2f65c693e 100644 --- a/x-pack/plugins/lens/public/embeddable/embeddable_component.tsx +++ b/x-pack/plugins/lens/public/embeddable/embeddable_component.tsx @@ -9,6 +9,7 @@ import React, { FC, useEffect } from 'react'; import type { CoreStart } from '@kbn/core/public'; import type { Action, UiActionsStart } from '@kbn/ui-actions-plugin/public'; import type { Start as InspectorStartContract } from '@kbn/inspector-plugin/public'; +import { PanelLoader } from '@kbn/panel-loader'; import { EuiLoadingChart } from '@elastic/eui'; import { EmbeddableFactory, @@ -160,7 +161,7 @@ const EmbeddablePanelWrapper: FC = ({ }, [embeddable, input]); if (loading || !embeddable) { - return ; + return ; } return ( diff --git a/x-pack/plugins/lens/tsconfig.json b/x-pack/plugins/lens/tsconfig.json index 4f100ff98965c..8958e1db8abfd 100644 --- a/x-pack/plugins/lens/tsconfig.json +++ b/x-pack/plugins/lens/tsconfig.json @@ -92,12 +92,13 @@ "@kbn/core-plugins-server", "@kbn/text-based-languages", "@kbn/field-utils", + "@kbn/panel-loader", "@kbn/shared-ux-button-toolbar", "@kbn/cell-actions", "@kbn/calculate-width-from-char-count", "@kbn/discover-utils" ], "exclude": [ - "target/**/*", + "target/**/*" ] } diff --git a/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx b/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx index d202ffbd38b3f..65ec5c9350269 100644 --- a/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx +++ b/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx @@ -409,6 +409,10 @@ export const SwimlaneContainer: FC = ({ const noSwimLaneData = !isLoading && !showSwimlane && !!noDataWarning; + if (noSwimLaneData) { + onRenderComplete?.(); + } + // A resize observer is required to compute the bucket span based on the chart width to fetch the data accordingly return ( diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.tsx index 27322086dc014..a64a9f0bb859e 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.tsx @@ -108,6 +108,7 @@ export const EmbeddableSwimLaneContainer: FC = ( ); if (error) { + onRenderComplete(); return ( Date: Tue, 28 Nov 2023 12:38:58 -0700 Subject: [PATCH 27/30] [maps] fix tile errors displayed when layer is no longer using tiles (#172019) Closes https://github.com/elastic/kibana/issues/172013 PR updates TileStatusTracker to clear tile error cache when layer is not tiled. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../tile_status_tracker/tile_error_cache.ts | 9 ++++ .../tile_status_tracker.test.tsx | 48 +++++++++++++++++++ .../tile_status_tracker.tsx | 19 ++++++++ 3 files changed, 76 insertions(+) diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_error_cache.ts b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_error_cache.ts index 9d54457056f77..213f883762312 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_error_cache.ts +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_error_cache.ts @@ -18,6 +18,15 @@ export function getErrorCacheTileKey(canonical: { x: number; y: number; z: numbe export class TileErrorCache { private _cache: Record> = {}; + public clearLayer(layerId: string, onClear: () => void) { + if (!(layerId in this._cache)) { + return; + } + + delete this._cache[layerId]; + onClear(); + } + public clearTileError(layerId: string | undefined, tileKey: string, onClear: () => void) { if (!layerId || !(layerId in this._cache)) { return; diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_status_tracker.test.tsx b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_status_tracker.test.tsx index 546001cc23b3b..c8d6395c743ba 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_status_tracker.test.tsx +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_status_tracker.test.tsx @@ -12,6 +12,7 @@ import type { Map as MbMap, MapSourceDataEvent } from '@kbn/mapbox-gl'; import type { TileError, TileMetaFeature } from '../../../../common/descriptor_types'; import { TileStatusTracker } from './tile_status_tracker'; import { ILayer } from '../../../classes/layers/layer'; +import type { IVectorSource } from '../../../classes/sources/vector_source'; class MockMbMap { public listeners: Array<{ type: string; callback: (e: unknown) => void }> = []; @@ -249,6 +250,53 @@ describe('TileStatusTracker', () => { expect(tileErrorsMap.get('layer2')).toBeUndefined(); }); + test('should clear layer tile errors when layer is not tiled', async () => { + const mockMbMap = new MockMbMap(); + const layer1 = createMockLayer('layer1', 'layer1Source'); + + const wrapper = mount( + + ); + + mockMbMap.emit( + 'sourcedataloading', + createSourceDataEvent('layer1Source', IN_VIEW_CANONICAL_TILE) + ); + mockMbMap.emit('error', { + ...createSourceDataEvent('layer1Source', IN_VIEW_CANONICAL_TILE), + error: { + message: 'simulated error', + }, + }); + + // simulate delay. Cache-checking is debounced. + await sleep(300); + + expect(tileErrorsMap.get('layer1')?.length).toBe(1); + + const geojsonLayer1 = createMockLayer('layer1', 'layer1Source'); + geojsonLayer1.getSource = () => { + return { + isESSource() { + return true; + }, + isMvt() { + return false; + }, + } as unknown as IVectorSource; + }; + wrapper.setProps({ layerList: [geojsonLayer1] }); + + // simulate delay. Cache-checking is debounced. + await sleep(300); + + expect(tileErrorsMap.get('layer1')).toBeUndefined(); + }); + test('should only return tile errors within map zoom', async () => { const mockMbMap = new MockMbMap(); diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_status_tracker.tsx b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_status_tracker.tsx index ccb1ca6d06c5d..972e691f6695e 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_status_tracker.tsx +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker/tile_status_tracker.tsx @@ -12,6 +12,7 @@ import type { AJAXError, Map as MbMap, MapSourceDataEvent } from '@kbn/mapbox-gl import type { TileError, TileMetaFeature } from '../../../../common/descriptor_types'; import { SPATIAL_FILTERS_LAYER_ID } from '../../../../common/constants'; import { ILayer } from '../../../classes/layers/layer'; +import { isLayerGroup } from '../../../classes/layers/layer_group'; import { IVectorSource } from '../../../classes/sources/vector_source'; import { getTileKey as getCenterTileKey } from '../../../classes/util/geo_tile_utils'; import { boundsToExtent } from '../../../classes/util/maplibre_utils'; @@ -62,6 +63,24 @@ export class TileStatusTracker extends Component { this.props.mbMap.on('moveend', this._onMoveEnd); } + componentDidUpdate() { + this.props.layerList.forEach((layer) => { + if (isLayerGroup(layer)) { + return; + } + + const source = layer.getSource(); + if ( + source.isESSource() && + typeof (source as IVectorSource).isMvt === 'function' && + !(source as IVectorSource).isMvt() + ) { + // clear tile cache when layer is not tiled + this._tileErrorCache.clearLayer(layer.getId(), this._updateTileStatusForAllLayers); + } + }); + } + componentWillUnmount() { this._isMounted = false; this.props.mbMap.off('error', this._onError); From 2ca730d118fa4394db88406012527315667b45d9 Mon Sep 17 00:00:00 2001 From: "Christiane (Tina) Heiligers" Date: Tue, 28 Nov 2023 12:39:29 -0700 Subject: [PATCH 28/30] [Core] [SOR] BWC bulkUpdate (#171245) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: pgayvallet --- .../src/lib/apis/bulk_update.test.ts | 274 +++++++++++++----- .../src/lib/apis/bulk_update.ts | 232 +++++++++------ .../src/lib/apis/update.ts | 2 +- .../lib/repository.spaces_extension.test.ts | 13 +- .../test_helpers/repository.test.common.ts | 38 ++- .../src/apis/bulk_update.ts | 2 + .../service/lib/bulk_update.test.ts | 234 +++++++++++++++ 7 files changed, 608 insertions(+), 187 deletions(-) create mode 100644 src/core/server/integration_tests/saved_objects/service/lib/bulk_update.test.ts diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/bulk_update.test.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/bulk_update.test.ts index d24c11f190696..60deaa64e3e63 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/bulk_update.test.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/bulk_update.test.ts @@ -75,6 +75,17 @@ describe('SavedObjectsRepository', () => { return expect.toBeDocumentWithoutError(type, id); }; + const expectMigrationArgs = (args: unknown, contains = true, n = 1) => { + const obj = contains ? expect.objectContaining(args) : expect.not.objectContaining(args); + expect(migrator.migrateDocument).toHaveBeenNthCalledWith( + n, + obj, + expect.objectContaining({ + allowDowngrade: expect.any(Boolean), + }) + ); + }; + beforeEach(() => { pointInTimeFinderMock.mockClear(); client = elasticsearchClientMock.createElasticsearchClient(); @@ -121,7 +132,7 @@ describe('SavedObjectsRepository', () => { const originId = 'some-origin-id'; const namespace = 'foo-namespace'; - // bulk create calls have two objects for each source -- the action, and the source + // bulk index calls have two objects for each source -- the action, and the source const expectClientCallArgsAction = ( objects: TypeIdTuple[], { @@ -153,14 +164,26 @@ describe('SavedObjectsRepository', () => { ); }; - const expectObjArgs = ({ type, attributes }: { type: string; attributes: unknown }) => [ - expect.any(Object), + const expectObjArgs = ( { - doc: expect.objectContaining({ - [type]: attributes, - ...mockTimestampFields, - }), + type, + attributes, + references, + }: { + type: string; + attributes: unknown; + references?: SavedObjectReference[]; }, + overrides: Record = {} + ) => [ + expect.any(Object), + expect.objectContaining({ + [type]: attributes, + references, + type, + ...overrides, + ...mockTimestampFields, + }), ]; describe('client calls', () => { @@ -169,13 +192,14 @@ describe('SavedObjectsRepository', () => { expect(client.bulk).toHaveBeenCalled(); }); - it(`should use the ES mget action before bulk action for any types that are multi-namespace`, async () => { + it(`should use the ES mget action before bulk action for any types that are valid`, async () => { const objects = [obj1, { ...obj2, type: MULTI_NAMESPACE_ISOLATED_TYPE }]; await bulkUpdateSuccess(client, repository, registry, objects); expect(client.bulk).toHaveBeenCalled(); expect(client.mget).toHaveBeenCalled(); const docs = [ + expect.objectContaining({ _id: `${obj1.type}:${obj1.id}` }), expect.objectContaining({ _id: `${MULTI_NAMESPACE_ISOLATED_TYPE}:${obj2.id}` }), ]; expect(client.mget).toHaveBeenCalledWith( @@ -186,21 +210,14 @@ describe('SavedObjectsRepository', () => { it(`formats the ES request`, async () => { await bulkUpdateSuccess(client, repository, registry, [obj1, obj2]); - const body = [...expectObjArgs(obj1), ...expectObjArgs(obj2)]; - expect(client.bulk).toHaveBeenCalledWith( - expect.objectContaining({ body }), - expect.anything() - ); + // expect client.bulk call args should include the whole doc + expectClientCallArgsAction([obj1, obj2], { method: 'index' }); }); it(`formats the ES request for any types that are multi-namespace`, async () => { const _obj2 = { ...obj2, type: MULTI_NAMESPACE_ISOLATED_TYPE }; await bulkUpdateSuccess(client, repository, registry, [obj1, _obj2]); - const body = [...expectObjArgs(obj1), ...expectObjArgs(_obj2)]; - expect(client.bulk).toHaveBeenCalledWith( - expect.objectContaining({ body }), - expect.anything() - ); + expectClientCallArgsAction([obj1, _obj2], { method: 'index' }); }); it(`doesnt call Elasticsearch if there are no valid objects to update`, async () => { @@ -211,8 +228,10 @@ describe('SavedObjectsRepository', () => { it(`defaults to no references`, async () => { await bulkUpdateSuccess(client, repository, registry, [obj1, obj2]); - const expected = { doc: expect.not.objectContaining({ references: expect.anything() }) }; - const body = [expect.any(Object), expected, expect.any(Object), expected]; + const body = [ + ...expectObjArgs({ ...obj1, references: [] }), + ...expectObjArgs({ ...obj2, references: [] }), + ]; expect(client.bulk).toHaveBeenCalledWith( expect.objectContaining({ body }), expect.anything() @@ -223,13 +242,16 @@ describe('SavedObjectsRepository', () => { const test = async (references: SavedObjectReference[]) => { const objects = [obj1, obj2].map((obj) => ({ ...obj, references })); await bulkUpdateSuccess(client, repository, registry, objects); - const expected = { doc: expect.objectContaining({ references }) }; - const body = [expect.any(Object), expected, expect.any(Object), expected]; + const body = [ + ...expectObjArgs({ ...obj1, references }), + ...expectObjArgs({ ...obj2, references }), + ]; expect(client.bulk).toHaveBeenCalledWith( expect.objectContaining({ body }), expect.anything() ); client.bulk.mockClear(); + client.mget.mockClear(); }; await test(references); await test([{ type: 'type', id: 'id', name: 'some ref' }]); @@ -238,15 +260,18 @@ describe('SavedObjectsRepository', () => { it(`doesn't accept custom references if not an array`, async () => { const test = async (references: unknown) => { - const objects = [obj1, obj2]; // .map((obj) => ({ ...obj })); + const objects = [obj1, obj2]; await bulkUpdateSuccess(client, repository, registry, objects); - const expected = { doc: expect.not.objectContaining({ references: expect.anything() }) }; - const body = [expect.any(Object), expected, expect.any(Object), expected]; + const body = [ + ...expectObjArgs({ ...obj1, references: expect.not.arrayContaining([references]) }), + ...expectObjArgs({ ...obj2, references: expect.not.arrayContaining([references]) }), + ]; expect(client.bulk).toHaveBeenCalledWith( expect.objectContaining({ body }), expect.anything() ); client.bulk.mockClear(); + client.mget.mockClear(); }; await test('string'); await test(123); @@ -265,7 +290,7 @@ describe('SavedObjectsRepository', () => { it(`defaults to no version for types that are not multi-namespace`, async () => { const objects = [obj1, { ...obj2, type: NAMESPACE_AGNOSTIC_TYPE }]; await bulkUpdateSuccess(client, repository, registry, objects); - expectClientCallArgsAction(objects, { method: 'update' }); + expectClientCallArgsAction(objects, { method: 'index' }); }); it(`accepts version`, async () => { @@ -277,13 +302,13 @@ describe('SavedObjectsRepository', () => { ]; await bulkUpdateSuccess(client, repository, registry, objects); const overrides = { if_seq_no: 100, if_primary_term: 200 }; - expectClientCallArgsAction(objects, { method: 'update', overrides }); + expectClientCallArgsAction(objects, { method: 'index', overrides }); }); it(`prepends namespace to the id when providing namespace for single-namespace type`, async () => { const getId = (type: string, id: string) => `${namespace}:${type}:${id}`; // test that the raw document ID equals this (e.g., has a namespace prefix) await bulkUpdateSuccess(client, repository, registry, [obj1, obj2], { namespace }); - expectClientCallArgsAction([obj1, obj2], { method: 'update', getId }); + expectClientCallArgsAction([obj1, obj2], { method: 'index', getId }); jest.clearAllMocks(); // test again with object namespace string that supersedes the operation's namespace ID @@ -291,13 +316,13 @@ describe('SavedObjectsRepository', () => { { ...obj1, namespace }, { ...obj2, namespace }, ]); - expectClientCallArgsAction([obj1, obj2], { method: 'update', getId }); + expectClientCallArgsAction([obj1, obj2], { method: 'index', getId }); }); it(`doesn't prepend namespace to the id when providing no namespace for single-namespace type`, async () => { const getId = (type: string, id: string) => `${type}:${id}`; // test that the raw document ID equals this (e.g., does not have a namespace prefix) await bulkUpdateSuccess(client, repository, registry, [obj1, obj2]); - expectClientCallArgsAction([obj1, obj2], { method: 'update', getId }); + expectClientCallArgsAction([obj1, obj2], { method: 'index', getId }); jest.clearAllMocks(); // test again with object namespace string that supersedes the operation's namespace ID @@ -311,7 +336,7 @@ describe('SavedObjectsRepository', () => { ], { namespace } ); - expectClientCallArgsAction([obj1, obj2], { method: 'update', getId }); + expectClientCallArgsAction([obj1, obj2], { method: 'index', getId }); }); it(`normalizes options.namespace from 'default' to undefined`, async () => { @@ -319,7 +344,7 @@ describe('SavedObjectsRepository', () => { await bulkUpdateSuccess(client, repository, registry, [obj1, obj2], { namespace: 'default', }); - expectClientCallArgsAction([obj1, obj2], { method: 'update', getId }); + expectClientCallArgsAction([obj1, obj2], { method: 'index', getId }); }); it(`doesn't prepend namespace to the id when not using single-namespace type`, async () => { @@ -328,18 +353,20 @@ describe('SavedObjectsRepository', () => { const _obj2 = { ...obj2, type: MULTI_NAMESPACE_ISOLATED_TYPE }; await bulkUpdateSuccess(client, repository, registry, [_obj1], { namespace }); - expectClientCallArgsAction([_obj1], { method: 'update', getId }); + expectClientCallArgsAction([_obj1], { method: 'index', getId }); client.bulk.mockClear(); + client.mget.mockClear(); await bulkUpdateSuccess(client, repository, registry, [_obj2], { namespace }); - expectClientCallArgsAction([_obj2], { method: 'update', getId }); + expectClientCallArgsAction([_obj2], { method: 'index', getId }); jest.clearAllMocks(); // test again with object namespace string that supersedes the operation's namespace ID await bulkUpdateSuccess(client, repository, registry, [{ ..._obj1, namespace }]); - expectClientCallArgsAction([_obj1], { method: 'update', getId }); + expectClientCallArgsAction([_obj1], { method: 'index', getId }); client.bulk.mockClear(); + client.mget.mockClear(); await bulkUpdateSuccess(client, repository, registry, [{ ..._obj2, namespace }]); - expectClientCallArgsAction([_obj2], { method: 'update', getId }); + expectClientCallArgsAction([_obj2], { method: 'index', getId }); }); }); @@ -359,51 +386,71 @@ describe('SavedObjectsRepository', () => { isBulkError: boolean, expectedErrorResult: ExpectedErrorResult ) => { - const objects = [obj1, obj, obj2]; - const mockResponse = getMockBulkUpdateResponse(registry, objects); + const objects = [obj1, obj2, obj]; + + const mockedMgetResponse = getMockMgetResponse(registry, [obj1, obj2, obj]); + client.bulk.mockClear(); + client.mget.mockClear(); + client.mget.mockResponseOnce(mockedMgetResponse); + + const mockBulkIndexResponse = getMockBulkUpdateResponse(registry, objects); if (isBulkError) { - // mock the bulk error for only the second object + // mock the bulk error for only the third object + mockGetBulkOperationError.mockReturnValueOnce(undefined); mockGetBulkOperationError.mockReturnValueOnce(undefined); mockGetBulkOperationError.mockReturnValueOnce(expectedErrorResult.error as Payload); } - client.bulk.mockResponseOnce(mockResponse); + client.bulk.mockResponseOnce(mockBulkIndexResponse); const result = await repository.bulkUpdate(objects); + + expect(client.mget).toHaveBeenCalled(); expect(client.bulk).toHaveBeenCalled(); - const objCall = isBulkError ? expectObjArgs(obj) : []; - const body = [...expectObjArgs(obj1), ...objCall, ...expectObjArgs(obj2)]; - expect(client.bulk).toHaveBeenCalledWith( - expect.objectContaining({ body }), - expect.anything() - ); + + const expectClientCallObjects = isBulkError ? [obj1, obj2, obj] : [obj1, obj2]; + expectClientCallArgsAction(expectClientCallObjects, { method: 'index' }); + expect(result).toEqual({ - saved_objects: [expectSuccess(obj1), expectedErrorResult, expectSuccess(obj2)], + saved_objects: [expectSuccess(obj1), expectSuccess(obj2), expectedErrorResult], }); }; const bulkUpdateMultiError = async ( - [obj1, _obj, obj2]: SavedObjectsBulkUpdateObject[], + [obj1, obj2, _obj]: SavedObjectsBulkUpdateObject[], options: SavedObjectsBulkUpdateOptions | undefined, mgetResponse: estypes.MgetResponse, mgetOptions?: { statusCode?: number } ) => { + client.bulk.mockClear(); + client.mget.mockClear(); + // we only need to mock the response once. A 404 status code will apply to the response for all client.mget.mockResponseOnce(mgetResponse, { statusCode: mgetOptions?.statusCode }); - const bulkResponse = getMockBulkUpdateResponse(registry, [obj1, obj2], { namespace }); - client.bulk.mockResponseOnce(bulkResponse); + const mockBulkIndexResponse = getMockBulkUpdateResponse(registry, [obj1, obj2], { + namespace, + }); + client.bulk.mockResponseOnce(mockBulkIndexResponse); + + const result = await repository.bulkUpdate([obj1, obj2, _obj], options); - const result = await repository.bulkUpdate([obj1, _obj, obj2], options); - expect(client.bulk).toHaveBeenCalled(); expect(client.mget).toHaveBeenCalled(); - const body = [...expectObjArgs(obj1), ...expectObjArgs(obj2)]; - expect(client.bulk).toHaveBeenCalledWith( - expect.objectContaining({ body }), - expect.anything() - ); - - expect(result).toEqual({ - saved_objects: [expectSuccess(obj1), expectErrorNotFound(_obj), expectSuccess(obj2)], - }); + if (mgetOptions?.statusCode === 404) { + expect(client.bulk).not.toHaveBeenCalled(); + expect(result).toEqual({ + saved_objects: [ + expectErrorNotFound(obj1), + expectErrorNotFound(obj2), + expectErrorNotFound(_obj), + ], + }); + } else { + expect(client.bulk).toHaveBeenCalled(); + expectClientCallArgsAction([obj1, obj2], { method: 'index' }); + + expect(result).toEqual({ + saved_objects: [expectSuccess(obj1), expectSuccess(obj2), expectErrorNotFound(_obj)], + }); + } }; it(`throws when options.namespace is '*'`, async () => { @@ -433,22 +480,22 @@ describe('SavedObjectsRepository', () => { it(`returns error when ES is unable to find the document (mget)`, async () => { const _obj = { ...obj, type: MULTI_NAMESPACE_ISOLATED_TYPE, found: false }; - const mgetResponse = getMockMgetResponse(registry, [_obj]); - await bulkUpdateMultiError([obj1, _obj, obj2], undefined, mgetResponse); + const mgetResponse = getMockMgetResponse(registry, [obj1, obj2, _obj]); + await bulkUpdateMultiError([obj1, obj2, _obj], undefined, mgetResponse); }); it(`returns error when ES is unable to find the index (mget)`, async () => { const _obj = { ...obj, type: MULTI_NAMESPACE_ISOLATED_TYPE }; - const mgetResponse = getMockMgetResponse(registry, [_obj]); - await bulkUpdateMultiError([obj1, _obj, obj2], { namespace }, mgetResponse, { + const mgetResponse = getMockMgetResponse(registry, [obj1, obj2, _obj]); + await bulkUpdateMultiError([obj1, obj2, _obj], { namespace }, mgetResponse, { statusCode: 404, }); }); it(`returns error when there is a conflict with an existing multi-namespace saved object (mget)`, async () => { const _obj = { ...obj, type: MULTI_NAMESPACE_ISOLATED_TYPE }; - const mgetResponse = getMockMgetResponse(registry, [_obj], 'bar-namespace'); - await bulkUpdateMultiError([obj1, _obj, obj2], { namespace }, mgetResponse); + const mgetResponse = getMockMgetResponse(registry, [obj1, obj2, _obj], 'bar-namespace'); + await bulkUpdateMultiError([obj1, obj2, _obj], { namespace }, mgetResponse); }); it(`returns bulk error`, async () => { @@ -460,6 +507,52 @@ describe('SavedObjectsRepository', () => { await bulkUpdateError(obj, true, expectedErrorResult); }); }); + describe('migration', () => { + it('migrates the fetched documents from Mget', async () => { + const modifiedObj2 = { ...obj2, coreMigrationVersion: '8.0.0' }; + const objects = [modifiedObj2]; + migrator.migrateDocument.mockImplementationOnce((doc) => ({ ...doc, migrated: true })); + + await bulkUpdateSuccess(client, repository, registry, objects); + expect(migrator.migrateDocument).toHaveBeenCalledTimes(2); + expectMigrationArgs({ + id: modifiedObj2.id, + type: modifiedObj2.type, + }); + }); + + it('migrates namespace agnostic and multinamespace object documents', async () => { + const modifiedObj2 = { + ...obj2, + coreMigrationVersion: '8.0.0', + type: MULTI_NAMESPACE_ISOLATED_TYPE, + namespace: 'default', + }; + const modifiedObj1 = { ...obj1, type: NAMESPACE_AGNOSTIC_TYPE }; + const objects = [modifiedObj2, modifiedObj1]; + migrator.migrateDocument.mockImplementationOnce((doc) => ({ ...doc, migrated: true })); + + await bulkUpdateSuccess(client, repository, registry, objects, { namespace }); + + expect(migrator.migrateDocument).toHaveBeenCalledTimes(4); + expectMigrationArgs( + { + id: modifiedObj2.id, + type: modifiedObj2.type, + }, + true, + 1 + ); + expectMigrationArgs( + { + id: modifiedObj1.id, + type: modifiedObj1.type, + }, + true, + 2 + ); + }); + }); describe('returns', () => { it(`formats the ES response`, async () => { @@ -483,14 +576,24 @@ describe('SavedObjectsRepository', () => { id: 'three', attributes: {}, }; - const objects = [obj1, obj, obj2]; - const mockResponse = getMockBulkUpdateResponse(registry, objects); - client.bulk.mockResponseOnce(mockResponse); + const objects = [obj1, obj2, obj]; + const mockedMgetResponse = getMockMgetResponse(registry, [obj1, obj2, obj]); + client.bulk.mockClear(); + client.mget.mockClear(); + client.mget.mockResponseOnce(mockedMgetResponse); + const mockBulkIndexResponse = getMockBulkUpdateResponse(registry, objects); + client.bulk.mockResponseOnce(mockBulkIndexResponse); const result = await repository.bulkUpdate(objects); - expect(client.bulk).toHaveBeenCalledTimes(1); + + expect(client.mget).toHaveBeenCalled(); + expect(client.bulk).toHaveBeenCalled(); + + const expectClientCallObjects = [obj1, obj2]; + expectClientCallArgsAction(expectClientCallObjects, { method: 'index' }); + expect(result).toEqual({ - saved_objects: [expectUpdateResult(obj1), expectError(obj), expectUpdateResult(obj2)], + saved_objects: [expectUpdateResult(obj1), expectUpdateResult(obj2), expectError(obj)], }); }); @@ -515,14 +618,25 @@ describe('SavedObjectsRepository', () => { id: 'three', attributes: {}, }; - const result = await bulkUpdateSuccess( - client, - repository, - registry, - [obj1, obj], - {}, - originId - ); + client.bulk.mockClear(); + client.mget.mockClear(); + const objects = [ + { ...obj1, originId }, + { ...obj, originId }, + ]; + const mockedMgetResponse = getMockMgetResponse(registry, objects); + + client.mget.mockResponseOnce(mockedMgetResponse); + + const mockBulkIndexResponse = getMockBulkUpdateResponse(registry, objects, {}, originId); + client.bulk.mockResponseOnce(mockBulkIndexResponse); + const result = await repository.bulkUpdate(objects); + + expect(client.mget).toHaveBeenCalled(); + expect(client.bulk).toHaveBeenCalled(); + + const expectClientCallObjects = objects; + expectClientCallArgsAction(expectClientCallObjects, { method: 'index' }); expect(result).toEqual({ saved_objects: [ expect.objectContaining({ originId }), diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/bulk_update.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/bulk_update.ts index b9c0f10a9021f..9c119ff86e7dd 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/bulk_update.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/bulk_update.ts @@ -14,6 +14,8 @@ import { DecoratedError, AuthorizeUpdateObject, SavedObjectsRawDoc, + SavedObjectsRawDocSource, + SavedObjectSanitizedDoc, } from '@kbn/core-saved-objects-server'; import { ALL_NAMESPACES_STRING, SavedObjectsUtils } from '@kbn/core-saved-objects-utils-server'; import { encodeVersion } from '@kbn/core-saved-objects-base-server-internal'; @@ -35,6 +37,8 @@ import { isLeft, isRight, rawDocExistsInNamespace, + getSavedObjectFromSource, + mergeForUpdate, } from './utils'; import { ApiExecutionContext } from './types'; @@ -43,32 +47,51 @@ export interface PerformUpdateParams { options: SavedObjectsBulkUpdateOptions; } +type DocumentToSave = Record; +type ExpectedBulkGetResult = Either< + { type: string; id: string; error: Payload }, + { + type: string; + id: string; + version?: string; + documentToSave: DocumentToSave; + objectNamespace?: string; + esRequestIndex: number; + migrationVersionCompatibility?: 'raw' | 'compatible'; + } +>; + +type ExpectedBulkUpdateResult = Either< + { type: string; id: string; error: Payload }, + { + type: string; + id: string; + namespaces?: string[]; + documentToSave: DocumentToSave; + esRequestIndex: number; + rawMigratedUpdatedDoc: SavedObjectsRawDoc; + } +>; + export const performBulkUpdate = async ( { objects, options }: PerformUpdateParams, { registry, helpers, allowedTypes, client, serializer, extensions = {} }: ApiExecutionContext ): Promise> => { - const { common: commonHelper, encryption: encryptionHelper } = helpers; + const { + common: commonHelper, + encryption: encryptionHelper, + migration: migrationHelper, + } = helpers; const { securityExtension } = extensions; - + const { migrationVersionCompatibility } = options; const namespace = commonHelper.getCurrentNamespace(options.namespace); const time = getCurrentTime(); let bulkGetRequestIndexCounter = 0; - type DocumentToSave = Record; - type ExpectedBulkGetResult = Either< - { type: string; id: string; error: Payload }, - { - type: string; - id: string; - version?: string; - documentToSave: DocumentToSave; - objectNamespace?: string; - esRequestIndex?: number; - } - >; const expectedBulkGetResults = objects.map((object) => { const { type, id, attributes, references, version, namespace: objectNamespace } = object; let error: DecoratedError | undefined; + if (!allowedTypes.includes(type)) { error = SavedObjectsErrorHelpers.createGenericNotFoundError(type, id); } else { @@ -91,21 +114,19 @@ export const performBulkUpdate = async ( ...(Array.isArray(references) && { references }), }; - const requiresNamespacesCheck = registry.isMultiNamespace(object.type); - return right({ type, id, version, documentToSave, objectNamespace, - ...(requiresNamespacesCheck && { esRequestIndex: bulkGetRequestIndexCounter++ }), + esRequestIndex: bulkGetRequestIndexCounter++, + migrationVersionCompatibility, }); }); const validObjects = expectedBulkGetResults.filter(isRight); if (validObjects.length === 0) { - // We only have error results; return early to avoid potentially trying authZ checks for 0 types which would result in an exception. return { // Technically the returned array should only contain SavedObject results, but for errors this is not true (we cast to 'any' below) saved_objects: expectedBulkGetResults.map>( @@ -117,20 +138,25 @@ export const performBulkUpdate = async ( // `objectNamespace` is a namespace string, while `namespace` is a namespace ID. // The object namespace string, if defined, will supersede the operation's namespace ID. const namespaceString = SavedObjectsUtils.namespaceIdToString(namespace); + const getNamespaceId = (objectNamespace?: string) => objectNamespace !== undefined ? SavedObjectsUtils.namespaceStringToId(objectNamespace) : namespace; + const getNamespaceString = (objectNamespace?: string) => objectNamespace ?? namespaceString; - const bulkGetDocs = validObjects - .filter(({ value }) => value.esRequestIndex !== undefined) - .map(({ value: { type, id, objectNamespace } }) => ({ - _id: serializer.generateRawId(getNamespaceId(objectNamespace), type, id), - _index: commonHelper.getIndexForType(type), - _source: ['type', 'namespaces'], - })); + + const bulkGetDocs = validObjects.map(({ value: { type, id, objectNamespace } }) => ({ + _id: serializer.generateRawId(getNamespaceId(objectNamespace), type, id), + _index: commonHelper.getIndexForType(type), + _source: true, + })); + const bulkGetResponse = bulkGetDocs.length - ? await client.mget({ body: { docs: bulkGetDocs } }, { ignore: [404], meta: true }) + ? await client.mget( + { body: { docs: bulkGetDocs } }, + { ignore: [404], meta: true } + ) : undefined; // fail fast if we can't verify a 404 response is from Elasticsearch if ( @@ -145,14 +171,24 @@ export const performBulkUpdate = async ( const authObjects: AuthorizeUpdateObject[] = validObjects.map((element) => { const { type, id, objectNamespace, esRequestIndex: index } = element.value; - const preflightResult = index !== undefined ? bulkGetResponse?.body.docs[index] : undefined; - return { - type, - id, - objectNamespace, - // @ts-expect-error MultiGetHit._source is optional - existingNamespaces: preflightResult?._source?.namespaces ?? [], - }; + const preflightResult = bulkGetResponse!.body.docs[index]; + + if (registry.isMultiNamespace(type)) { + return { + type, + id, + objectNamespace, + // @ts-expect-error MultiGetHit._source is optional + existingNamespaces: preflightResult._source?.namespaces ?? [], + }; + } else { + return { + type, + id, + objectNamespace, + existingNamespaces: [], + }; + } }); const authorizationResult = await securityExtension?.authorizeBulkUpdate({ @@ -162,16 +198,7 @@ export const performBulkUpdate = async ( let bulkUpdateRequestIndexCounter = 0; const bulkUpdateParams: object[] = []; - type ExpectedBulkUpdateResult = Either< - { type: string; id: string; error: Payload }, - { - type: string; - id: string; - namespaces: string[]; - documentToSave: DocumentToSave; - esRequestIndex: number; - } - >; + const expectedBulkUpdateResults = await Promise.all( expectedBulkGetResults.map>(async (expectedBulkGetResult) => { if (isLeft(expectedBulkGetResult)) { @@ -181,67 +208,105 @@ export const performBulkUpdate = async ( const { esRequestIndex, id, type, version, documentToSave, objectNamespace } = expectedBulkGetResult.value; - let namespaces; - let versionProperties; - if (esRequestIndex !== undefined) { - const indexFound = bulkGetResponse?.statusCode !== 404; - const actualResult = indexFound ? bulkGetResponse?.body.docs[esRequestIndex] : undefined; - const docFound = indexFound && isMgetDoc(actualResult) && actualResult.found; - if ( - !docFound || + let namespaces: string[] | undefined; + const versionProperties = getExpectedVersionProperties(version); + const indexFound = bulkGetResponse?.statusCode !== 404; + const actualResult = indexFound ? bulkGetResponse?.body.docs[esRequestIndex] : undefined; + const docFound = indexFound && isMgetDoc(actualResult) && actualResult.found; + const isMultiNS = registry.isMultiNamespace(type); + + if ( + !docFound || + (isMultiNS && !rawDocExistsInNamespace( registry, actualResult as SavedObjectsRawDoc, getNamespaceId(objectNamespace) - ) - ) { - return left({ - id, - type, - error: errorContent(SavedObjectsErrorHelpers.createGenericNotFoundError(type, id)), - }); - } + )) + ) { + return left({ + id, + type, + error: errorContent(SavedObjectsErrorHelpers.createGenericNotFoundError(type, id)), + }); + } + + if (isMultiNS) { // @ts-expect-error MultiGetHit is incorrectly missing _id, _source namespaces = actualResult!._source.namespaces ?? [ // @ts-expect-error MultiGetHit is incorrectly missing _id, _source SavedObjectsUtils.namespaceIdToString(actualResult!._source.namespace), ]; - versionProperties = getExpectedVersionProperties(version); - } else { - if (registry.isSingleNamespace(type)) { - // if `objectNamespace` is undefined, fall back to `options.namespace` - namespaces = [getNamespaceString(objectNamespace)]; - } - versionProperties = getExpectedVersionProperties(version); + } else if (registry.isSingleNamespace(type)) { + // if `objectNamespace` is undefined, fall back to `options.namespace` + namespaces = [getNamespaceString(objectNamespace)]; + } + + const document = getSavedObjectFromSource( + registry, + type, + id, + actualResult as SavedObjectsRawDoc, + { migrationVersionCompatibility } + ); + + let migrated: SavedObject; + try { + migrated = migrationHelper.migrateStorageDocument(document) as SavedObject; + } catch (migrateStorageDocError) { + throw SavedObjectsErrorHelpers.decorateGeneralError( + migrateStorageDocError, + 'Failed to migrate document to the latest version.' + ); } + const typeDefinition = registry.getType(type)!; + const updatedAttributes = mergeForUpdate({ + targetAttributes: { + ...migrated!.attributes, + }, + updatedAttributes: await encryptionHelper.optionallyEncryptAttributes( + type, + id, + objectNamespace || namespace, + documentToSave[type] + ), + typeMappings: typeDefinition.mappings, + }); + + const migratedUpdatedSavedObjectDoc = migrationHelper.migrateInputDocument({ + ...migrated!, + id, + type, + namespace, + namespaces, + attributes: updatedAttributes, + updated_at: time, + ...(Array.isArray(documentToSave.references) && { references: documentToSave.references }), + }); + const updatedMigratedDocumentToSave = serializer.savedObjectToRaw( + migratedUpdatedSavedObjectDoc as SavedObjectSanitizedDoc + ); + const expectedResult = { type, id, namespaces, esRequestIndex: bulkUpdateRequestIndexCounter++, documentToSave: expectedBulkGetResult.value.documentToSave, + rawMigratedUpdatedDoc: updatedMigratedDocumentToSave, + migrationVersionCompatibility, }; bulkUpdateParams.push( { - update: { + index: { _id: serializer.generateRawId(getNamespaceId(objectNamespace), type, id), _index: commonHelper.getIndexForType(type), ...versionProperties, }, }, - { - doc: { - ...documentToSave, - [type]: await encryptionHelper.optionallyEncryptAttributes( - type, - id, - objectNamespace || namespace, - documentToSave[type] - ), - }, - } + updatedMigratedDocumentToSave._source ); return right(expectedResult); @@ -264,7 +329,8 @@ export const performBulkUpdate = async ( return expectedResult.value as any; } - const { type, id, namespaces, documentToSave, esRequestIndex } = expectedResult.value; + const { type, id, namespaces, documentToSave, esRequestIndex, rawMigratedUpdatedDoc } = + expectedResult.value; const response = bulkUpdateResponse?.items[esRequestIndex] ?? {}; const rawResponse = Object.values(response)[0] as any; @@ -273,14 +339,12 @@ export const performBulkUpdate = async ( return { type, id, error }; } - // When a bulk update operation is completed, any fields specified in `_sourceIncludes` will be found in the "get" value of the - // returned object. We need to retrieve the `originId` if it exists so we can return it to the consumer. - const { _seq_no: seqNo, _primary_term: primaryTerm, get } = rawResponse; + const { _seq_no: seqNo, _primary_term: primaryTerm } = rawResponse; // eslint-disable-next-line @typescript-eslint/naming-convention const { [type]: attributes, references, updated_at } = documentToSave; - const { originId } = get._source; + const { originId } = rawMigratedUpdatedDoc._source; return { id, type, diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/update.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/update.ts index fd9c587502d7b..61f9cb4cfdb27 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/update.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/update.ts @@ -106,7 +106,7 @@ export const executeUpdate = async ( preflightDocResult, }); - const existingNamespaces = preflightDocNSResult?.savedObjectNamespaces ?? []; + const existingNamespaces = preflightDocNSResult.savedObjectNamespaces ?? []; const authorizationResult = await securityExtension?.authorizeUpdate({ namespace, object: { type, id, existingNamespaces }, diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.spaces_extension.test.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.spaces_extension.test.ts index 29983177adc99..82a7d2930f8d5 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.spaces_extension.test.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.spaces_extension.test.ts @@ -696,26 +696,23 @@ describe('SavedObjectsRepository Spaces Extension', () => { expect.objectContaining({ body: expect.arrayContaining([ expect.objectContaining({ - update: expect.objectContaining({ + index: expect.objectContaining({ _id: `${ currentSpace.expectedNamespace ? `${currentSpace.expectedNamespace}:` : '' }${obj1.type}:${obj1.id}`, }), }), expect.objectContaining({ - doc: expect.objectContaining({ - config: obj1.attributes, - }), + config: obj1.attributes, }), + expect.objectContaining({ - update: expect.objectContaining({ + index: expect.objectContaining({ _id: `${obj2.type}:${obj2.id}`, }), }), expect.objectContaining({ - doc: expect.objectContaining({ - multiNamespaceType: obj2.attributes, - }), + multiNamespaceType: obj2.attributes, }), ]), }), diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts index 29c00e9d41ac1..d3a31a905de5c 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts @@ -469,7 +469,9 @@ export const getMockGetResponse = ( export const getMockMgetResponse = ( registry: SavedObjectTypeRegistry, - objects: Array, + objects: Array< + TypeIdTuple & { found?: boolean; initialNamespaces?: string[]; originId?: string } + >, namespace?: string ) => ({ @@ -649,10 +651,10 @@ export const getMockBulkUpdateResponse = ( objects: TypeIdTuple[], options?: SavedObjectsBulkUpdateOptions, originId?: string -) => - ({ +) => { + return { items: objects.map(({ type, id }) => ({ - update: { + index: { _id: `${ registry.isSingleNamespace(type) && options?.namespace ? `${options?.namespace}:` : '' }${type}:${id}`, @@ -667,7 +669,8 @@ export const getMockBulkUpdateResponse = ( result: 'updated', }, })), - } as estypes.BulkResponse); + } as estypes.BulkResponse; +}; export const bulkUpdateSuccess = async ( client: ElasticsearchClientMock, @@ -678,19 +681,26 @@ export const bulkUpdateSuccess = async ( originId?: string, multiNamespaceSpace?: string // the space for multi namespace objects returned by mock mget (this is only needed for space ext testing) ) => { - const multiNamespaceObjects = objects.filter(({ type }) => registry.isMultiNamespace(type)); - if (multiNamespaceObjects?.length) { - const response = getMockMgetResponse( - registry, - multiNamespaceObjects, - multiNamespaceSpace ?? options?.namespace - ); - client.mget.mockResponseOnce(response); + let mockedMgetResponse; + const validObjects = objects.filter(({ type }) => registry.getType(type) !== undefined); + const multiNamespaceObjects = validObjects.filter(({ type }) => registry.isMultiNamespace(type)); + + if (validObjects?.length) { + if (multiNamespaceObjects.length > 0) { + mockedMgetResponse = getMockMgetResponse( + registry, + validObjects, + multiNamespaceSpace ?? options?.namespace + ); + } else { + mockedMgetResponse = getMockMgetResponse(registry, validObjects); + } + client.mget.mockResponseOnce(mockedMgetResponse); } const response = getMockBulkUpdateResponse(registry, objects, options, originId); client.bulk.mockResponseOnce(response); const result = await repository.bulkUpdate(objects, options); - expect(client.mget).toHaveBeenCalledTimes(multiNamespaceObjects?.length ? 1 : 0); + expect(client.mget).toHaveBeenCalledTimes(validObjects?.length ? 1 : 0); return result; }; diff --git a/packages/core/saved-objects/core-saved-objects-api-server/src/apis/bulk_update.ts b/packages/core/saved-objects/core-saved-objects-api-server/src/apis/bulk_update.ts index 6d10aee397b2f..49bc8d769a1d6 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server/src/apis/bulk_update.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server/src/apis/bulk_update.ts @@ -39,6 +39,8 @@ export interface SavedObjectsBulkUpdateObject export interface SavedObjectsBulkUpdateOptions extends SavedObjectsBaseOptions { /** The Elasticsearch Refresh setting for this operation */ refresh?: MutatingOperationRefreshSetting; + /** {@link SavedObjectsRawDocParseOptions.migrationVersionCompatibility} */ + migrationVersionCompatibility?: 'compatible' | 'raw'; } /** diff --git a/src/core/server/integration_tests/saved_objects/service/lib/bulk_update.test.ts b/src/core/server/integration_tests/saved_objects/service/lib/bulk_update.test.ts new file mode 100644 index 0000000000000..dc620f87ea55b --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/service/lib/bulk_update.test.ts @@ -0,0 +1,234 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { pick } from 'lodash'; +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { SavedObjectsType, SavedObjectsModelVersionMap } from '@kbn/core-saved-objects-server'; +import { type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import '../../migrations/jest_matchers'; +import { + getKibanaMigratorTestKit, + startElasticsearch, +} from '../../migrations/kibana_migrator_test_kit'; +import { delay } from '../../migrations/test_utils'; +import { getBaseMigratorParams } from '../../migrations/fixtures/zdt_base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'bulk_update.test.log'); + +describe('SOR - bulk_update API', () => { + let esServer: TestElasticsearchUtils['es']; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + const getType = (version: 'v1' | 'v2'): SavedObjectsType => { + const versionMap: SavedObjectsModelVersionMap = { + 1: { + changes: [], + schemas: { + forwardCompatibility: (attributes) => { + return pick(attributes, 'count'); + }, + }, + }, + }; + + if (version === 'v2') { + versionMap[2] = { + changes: [ + { + type: 'data_backfill', + backfillFn: (document) => { + return { attributes: { even: document.attributes.count % 2 === 0 } }; + }, + }, + ], + }; + } + + return { + name: 'my-test-type', + hidden: false, + namespaceType: 'agnostic', + mappings: { + dynamic: false, + properties: { + count: { type: 'integer' }, + ...(version === 'v2' ? { even: { type: 'boolean' } } : {}), + }, + }, + management: { + importableAndExportable: true, + }, + switchToModelVersionAt: '8.10.0', + modelVersions: versionMap, + }; + }; + + const getOtherType = (version: 'v1' | 'v2'): SavedObjectsType => { + const versionOtherMap: SavedObjectsModelVersionMap = { + 1: { + changes: [], + schemas: { + forwardCompatibility: (attributes) => { + return pick(attributes, 'sum'); + }, + }, + }, + }; + + if (version === 'v2') { + versionOtherMap[2] = { + changes: [ + { + type: 'data_backfill', + backfillFn: (document) => { + return { attributes: { isodd: document.attributes.sum % 2 !== 0 } }; + }, + }, + ], + }; + } + + return { + name: 'my-other-test-type', + hidden: false, + namespaceType: 'agnostic', + mappings: { + dynamic: false, + properties: { + sum: { type: 'integer' }, + ...(version === 'v2' ? { isodd: { type: 'boolean' } } : {}), + }, + }, + management: { + importableAndExportable: true, + }, + switchToModelVersionAt: '8.10.0', + modelVersions: versionOtherMap, + }; + }; + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const setup = async () => { + const { runMigrations: runMigrationV1, savedObjectsRepository: repositoryV1 } = + await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getType('v1'), getOtherType('v1')], + }); + await runMigrationV1(); + + const { + runMigrations: runMigrationV2, + savedObjectsRepository: repositoryV2, + client: esClient, + } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getType('v2'), getOtherType('v2')], + }); + await runMigrationV2(); + + return { repositoryV1, repositoryV2, esClient }; + }; + + it('supports updates between older and newer versions', async () => { + const { repositoryV1, repositoryV2, esClient } = await setup(); + + await repositoryV1.create('my-test-type', { count: 12 }, { id: 'my-id' }); + await repositoryV1.create('my-other-test-type', { sum: 24 }, { id: 'my-other-id' }); + + let repoV2Docs = await repositoryV2.bulkGet([ + { type: 'my-test-type', id: 'my-id' }, + { type: 'my-other-test-type', id: 'my-other-id' }, + ]); + const [doc, otherDoc] = repoV2Docs.saved_objects; + + expect(doc.attributes).toEqual({ + count: 12, + even: true, + }); + expect(otherDoc.attributes).toEqual({ + sum: 24, + isodd: false, + }); + + await repositoryV2.bulkUpdate([ + { type: 'my-test-type', id: doc.id, attributes: { count: 11, even: false } }, + // @ts-expect-error cannot assign to partial + { type: 'my-other-test-type', id: otherDoc.id, attributes: { sum: 23, isodd: true } }, + ]); + + const repoV1Docs = await repositoryV1.bulkGet([ + { type: 'my-test-type', id: 'my-id' }, + { type: 'my-other-test-type', id: 'my-other-id' }, + ]); + const [doc1, otherDoc1] = repoV1Docs.saved_objects; + + expect(doc1.attributes).toEqual({ + count: 11, + }); + expect(otherDoc1.attributes).toEqual({ + sum: 23, + }); + + await repositoryV1.bulkUpdate([ + { type: 'my-test-type', id: doc1.id, attributes: { count: 14 } }, + // @ts-expect-error cannot assign to partial + { type: 'my-other-test-type', id: otherDoc1.id, attributes: { sum: 24 } }, + ]); + + repoV2Docs = await repositoryV2.bulkGet([ + { type: 'my-test-type', id: 'my-id' }, + { type: 'my-other-test-type', id: 'my-other-id' }, + ]); + const [doc2, otherDoc2] = repoV2Docs.saved_objects; + + expect(doc2.attributes).toEqual({ + count: 14, + even: true, + }); + expect(otherDoc2.attributes).toEqual({ + sum: 24, + isodd: false, + }); + + const rawDoc = await fetchDoc(esClient, 'my-test-type', 'my-id'); + expect(rawDoc._source).toEqual( + expect.objectContaining({ + typeMigrationVersion: '10.1.0', + 'my-test-type': { + count: 14, + }, + }) + ); + + const otherRawDoc = await fetchDoc(esClient, 'my-other-test-type', 'my-other-id'); + expect(otherRawDoc._source).toEqual( + expect.objectContaining({ + typeMigrationVersion: '10.1.0', + 'my-other-test-type': { + sum: 24, + }, + }) + ); + }); + + const fetchDoc = async (client: ElasticsearchClient, type: string, id: string) => { + return await client.get({ + index: '.kibana', + id: `${type}:${id}`, + }); + }; +}); From 98b2cfbbb0442898e84189291b7f5c5a725a6115 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Tue, 28 Nov 2023 20:58:46 +0100 Subject: [PATCH 29/30] Enhance plugin documentation (#146678) The PR tackles a couple of improvements for the new `'notifications'` plugin documentation: - Add a link to the plugin API description in the kibana-dev-docs nav bar. - Convert the README to `mdx`. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- docs/developer/plugin-list.asciidoc | 2 +- docs/setup/settings.asciidoc | 3 +-- nav-kibana-dev.docnav.json | 3 +++ x-pack/plugins/notifications/{README.md => README.mdx} | 0 4 files changed, 5 insertions(+), 3 deletions(-) rename x-pack/plugins/notifications/{README.md => README.mdx} (100%) diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index df3f4c8ec855d..cb233fc8c10ce 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -672,7 +672,7 @@ Elastic. |This plugin allows for other plugins to add data to Kibana stack monitoring documents. -|{kib-repo}blob/{branch}/x-pack/plugins/notifications/README.md[notifications] +|{kib-repo}blob/{branch}/x-pack/plugins/notifications/README.mdx[notifications] |The Notifications plugin provides a set of services to help Solutions and plugins send notifications to users. diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc index 9d38b5162f124..13e3fad93cfca 100644 --- a/docs/setup/settings.asciidoc +++ b/docs/setup/settings.asciidoc @@ -323,8 +323,7 @@ run {kib} in different modes. Valid options are `background_tasks` and `ui`, or `*` to select all roles. *Default: `*`* `notifications.connectors.default.email`:: -Specifies the name of the connector that is used to send email notifications. -In {ecloud}, the default value is `elastic-cloud-email`. + Choose the default email connector for user notifications. As of `8.6.0`, {kib} is shipping with a new notification mechanism that will send email notifications for various user actions, e.g. assigning a _Case_ to a user. To enable notifications, an email connector must be <> in the system via `kibana.yml`, and the notifications plugin must be configured to point to the ID of that connector. [[path-data]] `path.data`:: The path where {kib} stores persistent data diff --git a/nav-kibana-dev.docnav.json b/nav-kibana-dev.docnav.json index c85c804cdfda4..56517c4136f7f 100644 --- a/nav-kibana-dev.docnav.json +++ b/nav-kibana-dev.docnav.json @@ -425,6 +425,9 @@ { "id": "kibNewsfeedPluginApi" }, + { + "id": "kibNotificationsPluginApi" + }, { "id": "kibObservabilityPluginApi" }, diff --git a/x-pack/plugins/notifications/README.md b/x-pack/plugins/notifications/README.mdx similarity index 100% rename from x-pack/plugins/notifications/README.md rename to x-pack/plugins/notifications/README.mdx From 8e36ba77f6b2594fb0f1cf2d18c263793c1438c0 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Tue, 28 Nov 2023 21:03:20 +0100 Subject: [PATCH 30/30] [Index Management] Add node plugins serverless tests (#172040) --- .../index_management/cluster_nodes.helpers.ts | 16 ---------- .../index_management/cluster_nodes.ts | 6 ++-- .../index_management/lib/cluster_nodes.api.ts | 23 +++++++++++++ .../services/index_management.ts | 4 +++ .../common/index_management/cluster_nodes.ts | 32 +++++++++++++++++++ .../common/index_management/index.ts | 1 + 6 files changed, 62 insertions(+), 20 deletions(-) delete mode 100644 x-pack/test/api_integration/apis/management/index_management/cluster_nodes.helpers.ts create mode 100644 x-pack/test/api_integration/apis/management/index_management/lib/cluster_nodes.api.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/index_management/cluster_nodes.ts diff --git a/x-pack/test/api_integration/apis/management/index_management/cluster_nodes.helpers.ts b/x-pack/test/api_integration/apis/management/index_management/cluster_nodes.helpers.ts deleted file mode 100644 index 8dabd89bf7c23..0000000000000 --- a/x-pack/test/api_integration/apis/management/index_management/cluster_nodes.helpers.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { API_BASE_PATH } from './constants'; - -export const registerHelpers = ({ supertest }: { supertest: any }) => { - const getNodesPlugins = () => supertest.get(`${API_BASE_PATH}/nodes/plugins`); - - return { - getNodesPlugins, - }; -}; diff --git a/x-pack/test/api_integration/apis/management/index_management/cluster_nodes.ts b/x-pack/test/api_integration/apis/management/index_management/cluster_nodes.ts index e885b677aaffb..72ca2d8fee5ba 100644 --- a/x-pack/test/api_integration/apis/management/index_management/cluster_nodes.ts +++ b/x-pack/test/api_integration/apis/management/index_management/cluster_nodes.ts @@ -8,12 +8,10 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; -import { registerHelpers } from './cluster_nodes.helpers'; +import { clusterNodesApi } from './lib/cluster_nodes.api'; export default function ({ getService }: FtrProviderContext) { - const supertest = getService('supertest'); - - const { getNodesPlugins } = registerHelpers({ supertest }); + const { getNodesPlugins } = clusterNodesApi(getService); describe('nodes', () => { it('should fetch the nodes plugins', async () => { diff --git a/x-pack/test/api_integration/apis/management/index_management/lib/cluster_nodes.api.ts b/x-pack/test/api_integration/apis/management/index_management/lib/cluster_nodes.api.ts new file mode 100644 index 0000000000000..c0bca06cd0e96 --- /dev/null +++ b/x-pack/test/api_integration/apis/management/index_management/lib/cluster_nodes.api.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { API_BASE_PATH } from '../constants'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export function clusterNodesApi(getService: FtrProviderContext['getService']) { + const supertest = getService('supertest'); + + const getNodesPlugins = () => + supertest + .get(`${API_BASE_PATH}/nodes/plugins`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx'); + + return { + getNodesPlugins, + }; +} diff --git a/x-pack/test/api_integration/services/index_management.ts b/x-pack/test/api_integration/services/index_management.ts index f5a57a9b74259..910a22a780479 100644 --- a/x-pack/test/api_integration/services/index_management.ts +++ b/x-pack/test/api_integration/services/index_management.ts @@ -9,6 +9,7 @@ import { FtrProviderContext } from '../ftr_provider_context'; import { indicesApi } from '../apis/management/index_management/lib/indices.api'; import { mappingsApi } from '../apis/management/index_management/lib/mappings.api'; import { indicesHelpers } from '../apis/management/index_management/lib/indices.helpers'; +import { clusterNodesApi } from '../apis/management/index_management/lib/cluster_nodes.api'; import { datastreamsHelpers } from '../apis/management/index_management/lib/datastreams.helpers'; export function IndexManagementProvider({ getService }: FtrProviderContext) { @@ -17,6 +18,9 @@ export function IndexManagementProvider({ getService }: FtrProviderContext) { api: indicesApi(getService), helpers: indicesHelpers(getService), }, + clusterNodes: { + api: clusterNodesApi(getService), + }, datastreams: { helpers: datastreamsHelpers(getService), }, diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/cluster_nodes.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/cluster_nodes.ts new file mode 100644 index 0000000000000..60110dd5af3fc --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/cluster_nodes.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const indexManagementService = getService('indexManagement'); + + describe('nodes', () => { + let getNodesPlugins: typeof indexManagementService['clusterNodes']['api']['getNodesPlugins']; + + before(async () => { + ({ + clusterNodes: { + api: { getNodesPlugins }, + }, + } = indexManagementService); + }); + + it('should fetch the nodes plugins', async () => { + const { body } = await getNodesPlugins().expect(200); + + expect(Array.isArray(body)).to.be(true); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts index 07f5da6aba981..a61822393aee9 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts @@ -14,6 +14,7 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./index_templates')); loadTestFile(require.resolve('./indices')); loadTestFile(require.resolve('./create_enrich_policies')); + loadTestFile(require.resolve('./cluster_nodes')); loadTestFile(require.resolve('./datastreams')); loadTestFile(require.resolve('./mappings')); });