From c89e6e12276705a4abf4b971ad35d5c8a543456c Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 13 Aug 2024 08:49:57 -0700 Subject: [PATCH] [UII] Support integration-level outputs (#189125) ## Summary Resolves #143905. This PR adds support for integration-level outputs. This means that different integrations within the same agent policy can now be configured to send data to different locations. This feature is gated behind `enterprise` level subscription. For each input, the agent policy will configure sending data to the following outputs in decreasing order of priority: 1. Output set specifically on the integration policy 2. Output set specifically on the integration's parent agent policy (including the case where an integration policy belongs to multiple agent policies) 3. Global default data output set via Fleet Settings Integration-level outputs will respect the same rules as agent policy-level outputs: - Certain integrations are disallowed from using certain output types, attempting to add them to each other via creation, updating, or "defaulting", will fail - `fleet-server`, `synthetics`, and `apm` can only use same-cluster Elasticsearch output - When an output is deleted, any integrations that were specifically using it will "clear" their output configuration and revert back to either `#2` or `#3` in the above list - When an output is edited, all agent policies across all spaces that use it will be bumped to a new revision, this includes: - Agent policies that have that output specifically set in their settings (existing behavior) - Agent policies that contain integrations which specifically has that output set (new behavior) - When a proxy is edited, the same new revision bump above will apply for any outputs using that proxy The final agent policy YAML that is generated will have: - `outputs` block that includes: - Data and monitoring outputs set at the agent policy level (existing behavior) - Any additional outputs set at the integration level, if they differ from the above - `outputs_permissions` block that includes permissions for each Elasticsearch output depending on which integrations and/or agent monitoring are assigned to it Integration policies table now includes `Output` column. If the output is defaulting to agent policy-level output, or global setting output, a tooltip is shown: image Configuring an integration-level output is done under Advanced options in the policy editor. Setting to the blank value will "clear" the output configuration. The list of available outputs is filtered by what outputs are available for that integration (see above): image An example of failure: ES output cannot be changed to Kafka while there is an integration image ## TODO - [x] Adjust side effects of editing/deleting output when policies use it across different spaces - [x] Add API integration tests - [x] Update OpenAPI spec - [x] Create doc issue ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../current_fields.json | 1 + .../current_mappings.json | 3 + .../check_registered_types.test.ts | 2 +- .../fleet/common/constants/mappings.ts | 3 +- .../plugins/fleet/common/constants/output.ts | 1 + .../fleet/common/constants/package_policy.ts | 2 + .../plugins/fleet/common/openapi/bundled.json | 3 +- .../plugins/fleet/common/openapi/bundled.yaml | 1 - .../schemas/new_package_policy.yaml | 1 - .../schemas/update_package_policy.yaml | 2 - ...plified_package_policy_helper.test.ts.snap | 1 + .../simplified_package_policy_helper.test.ts | 1 + .../simplified_package_policy_helper.ts | 6 + .../common/types/models/package_policy.ts | 2 + .../common/types/rest_spec/package_policy.ts | 1 + .../components/steps/components/hooks.tsx | 22 +++ .../steps/step_define_package_policy.tsx | 56 +++++- .../package_policies_table.tsx | 75 +++++++- .../use_fleet_proxy_form.tsx | 2 +- .../use_fleet_server_host_form.test.tsx | 4 - .../services/agent_and_policies_count.tsx | 65 +++---- .../hooks/use_multiple_agent_policies.ts | 3 +- x-pack/plugins/fleet/server/errors/index.ts | 2 + x-pack/plugins/fleet/server/mocks/index.ts | 1 + .../server/mocks/package_policy.mocks.ts | 1 + .../routes/package_policy/handlers.test.ts | 78 +------- .../server/routes/package_policy/handlers.ts | 19 -- .../routes/package_policy/utils/index.ts | 17 +- .../fleet/server/saved_objects/index.ts | 11 ++ .../saved_objects/migrations/to_v8_5_0.ts | 1 - .../full_agent_policy.test.ts.snap | 168 ++++++++++++++++++ .../agent_policies/full_agent_policy.test.ts | 120 +++++++++++++ .../agent_policies/full_agent_policy.ts | 53 ++++-- .../agent_policies/output_helpers.test.ts | 46 ++++- .../agent_policies/outputs_helpers.ts | 17 +- .../package_policies_to_agent_inputs.test.ts | 5 +- .../package_policies_to_agent_inputs.ts | 8 +- .../agent_policies/related_saved_objects.ts | 15 +- .../server/services/agent_policy.test.ts | 79 +++++--- .../fleet/server/services/agent_policy.ts | 49 ++++- .../fleet/server/services/output.test.ts | 15 +- .../plugins/fleet/server/services/output.ts | 109 ++++++++---- .../services/package_policies/utils.test.ts | 98 +++++++++- .../server/services/package_policies/utils.ts | 80 ++++++++- .../server/services/package_policy.test.ts | 56 ++++++ .../fleet/server/services/package_policy.ts | 89 +++++++++- .../server/services/package_policy_service.ts | 12 ++ .../services/preconfiguration/outputs.test.ts | 7 + .../server/types/models/package_policy.ts | 6 +- .../server/types/models/preconfiguration.ts | 1 + .../fleet/server/types/so_attributes.ts | 2 + .../__snapshots__/agent_policy.snap | 1 + .../apis/outputs/crud.ts | 122 ++++++++++++- 53 files changed, 1266 insertions(+), 279 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index dd233e14648bd..211777a5274a5 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -630,6 +630,7 @@ "is_managed", "name", "namespace", + "output_id", "overrides", "package", "package.name", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 4ded89844052b..e6e1fef63ee85 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2101,6 +2101,9 @@ "namespace": { "type": "keyword" }, + "output_id": { + "type": "keyword" + }, "overrides": { "index": false, "type": "flattened" diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index af58fd9d383bc..6d978b2d33ca4 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -119,7 +119,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-agent-policies": "90625b4a5ded9d4867358fcccc14a57c0454fcee", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", "ingest-outputs": "daafff49255ab700e07491376fe89f04fc998b91", - "ingest-package-policies": "2c0f7c72d211bb7d3076ce2fc0bd368f9c16d274", + "ingest-package-policies": "53a94064674835fdb35e5186233bcd7052eabd22", "ingest_manager_settings": "91445219e7115ff0c45d1dabd5d614a80b421797", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad", diff --git a/x-pack/plugins/fleet/common/constants/mappings.ts b/x-pack/plugins/fleet/common/constants/mappings.ts index f804dc872e0d1..6499da7f86cc9 100644 --- a/x-pack/plugins/fleet/common/constants/mappings.ts +++ b/x-pack/plugins/fleet/common/constants/mappings.ts @@ -6,7 +6,7 @@ */ /** - * ATTENTION: Mappings for Fleet are defined in the ElasticSearch repo. + * ATTENTION: Mappings for Fleet are defined in the Elasticsearch repo. * * The following mappings declared here closely mirror them * But they are only used to perform validation on the endpoints using ListWithKuery @@ -54,6 +54,7 @@ export const PACKAGE_POLICIES_MAPPINGS = { is_managed: { type: 'boolean' }, policy_id: { type: 'keyword' }, policy_ids: { type: 'keyword' }, + output_id: { type: 'keyword' }, package: { properties: { name: { type: 'keyword' }, diff --git a/x-pack/plugins/fleet/common/constants/output.ts b/x-pack/plugins/fleet/common/constants/output.ts index 8ddb8ed4d2f06..fb01ba991d3d2 100644 --- a/x-pack/plugins/fleet/common/constants/output.ts +++ b/x-pack/plugins/fleet/common/constants/output.ts @@ -29,6 +29,7 @@ export const DEFAULT_OUTPUT: NewOutput = { export const SERVERLESS_DEFAULT_OUTPUT_ID = 'es-default-output'; export const LICENCE_FOR_PER_POLICY_OUTPUT = 'platinum'; +export const LICENCE_FOR_OUTPUT_PER_INTEGRATION = 'enterprise'; /** * Kafka constants diff --git a/x-pack/plugins/fleet/common/constants/package_policy.ts b/x-pack/plugins/fleet/common/constants/package_policy.ts index 1645d1a86efb7..00b41a8a29de2 100644 --- a/x-pack/plugins/fleet/common/constants/package_policy.ts +++ b/x-pack/plugins/fleet/common/constants/package_policy.ts @@ -13,3 +13,5 @@ export const inputsFormat = { Simplified: 'simplified', Legacy: 'legacy', } as const; + +export const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index cf3ee35fca6df..727ef4c30f4fd 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -7484,8 +7484,7 @@ "type": "string" }, "output_id": { - "type": "string", - "deprecated": true + "type": "string" }, "inputs": { "type": "array", diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index ad592b9ea1847..7a19615cfe1da 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -4804,7 +4804,6 @@ components: type: string output_id: type: string - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml index 29a09174385ec..956f51a8016f5 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml @@ -22,7 +22,6 @@ properties: type: string output_id: type: string - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml index 795d92ff1d4a7..f06a2c9ea49f3 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml @@ -23,8 +23,6 @@ properties: type: string output_id: type: string - description: Not supported output can be set at the agent policy level only - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap b/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap index 5d747ebd88ebf..7c549b030a337 100644 --- a/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap +++ b/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap @@ -228,6 +228,7 @@ Object { ], "name": "nginx-1", "namespace": "default", + "output_id": "output123", "package": Object { "name": "nginx", "title": "Nginx", diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts index 0cde9d003dbb1..ff9d6eac7490f 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts @@ -37,6 +37,7 @@ describe('toPackagePolicy', () => { namespace: 'default', policy_id: 'policy123', policy_ids: ['policy123'], + output_id: 'output123', description: 'Test description', inputs: { 'nginx-logfile': { diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts index 98dee1021fe21..8d616bf4cae11 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts @@ -43,6 +43,7 @@ export interface SimplifiedPackagePolicy { id?: string; policy_id?: string; policy_ids: string[]; + output_id?: string; namespace: string; name: string; description?: string; @@ -147,6 +148,7 @@ export function simplifiedPackagePolicytoNewPackagePolicy( const { policy_id: policyId, policy_ids: policyIds, + output_id: outputId, namespace, name, description, @@ -161,6 +163,10 @@ export function simplifiedPackagePolicytoNewPackagePolicy( description ); + if (outputId) { + packagePolicy.output_id = outputId; + } + if (packagePolicy.package && options?.experimental_data_stream_features) { packagePolicy.package.experimental_data_stream_features = options.experimental_data_stream_features; diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 1b8a407ff8dd9..eb054ade02114 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -81,6 +81,8 @@ export interface NewPackagePolicy { /** @deprecated */ policy_id?: string; policy_ids: string[]; + // Nullable to allow user to reset to default outputs + output_id?: string | null; package?: PackagePolicyPackage; inputs: NewPackagePolicyInput[]; vars?: PackagePolicyConfigRecord; diff --git a/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts b/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts index ff190f0a556cd..8b74a3142fa97 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts @@ -63,6 +63,7 @@ export type PostDeletePackagePoliciesResponse = Array<{ package?: PackagePolicyPackage; policy_id?: string; policy_ids?: string[]; + output_id?: string; // Support generic errors statusCode?: number; body?: { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx index 55b8ac90c749c..c63339d682bb7 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx @@ -8,6 +8,10 @@ import { useMemo } from 'react'; import { useHistory } from 'react-router-dom'; +import { LICENCE_FOR_OUTPUT_PER_INTEGRATION } from '../../../../../../../../../common/constants'; +import { getAllowedOutputTypesForIntegration } from '../../../../../../../../../common/services/output_helpers'; +import { useGetOutputs, useLicense } from '../../../../../../hooks'; + export function useDataStreamId() { const history = useHistory(); @@ -16,3 +20,21 @@ export function useDataStreamId() { return searchParams.get('datastreamId') ?? undefined; }, [history.location.search]); } + +export function useOutputs(packageName: string) { + const licenseService = useLicense(); + const canUseOutputPerIntegration = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + const { data: outputsData, isLoading } = useGetOutputs(); + const allowedOutputTypes = getAllowedOutputTypesForIntegration(packageName); + const allowedOutputs = useMemo(() => { + if (!outputsData || !canUseOutputPerIntegration) { + return []; + } + return outputsData.items.filter((output) => allowedOutputTypes.includes(output.type)); + }, [allowedOutputTypes, canUseOutputPerIntegration, outputsData]); + return { + isLoading, + canUseOutputPerIntegration, + allowedOutputs, + }; +} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx index c312b48a2d290..43c3f8092e921 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx @@ -20,6 +20,7 @@ import { EuiLink, EuiCallOut, EuiSpacer, + EuiSelect, } from '@elastic/eui'; import styled from 'styled-components'; @@ -32,6 +33,7 @@ import { isAdvancedVar } from '../../services'; import type { PackagePolicyValidationResults } from '../../services'; import { PackagePolicyInputVarField } from './components'; +import { useOutputs } from './components/hooks'; // on smaller screens, fields should be displayed in one column const FormGroupResponsiveFields = styled(EuiDescribedFormGroup)` @@ -81,6 +83,14 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ }); } + // Outputs + const { + isLoading: isOutputsLoading, + canUseOutputPerIntegration, + allowedOutputs, + } = useOutputs(packageInfo.name); + + // Managed policy const isManaged = packagePolicy.is_managed; return validationResults ? ( @@ -245,6 +255,7 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ {isShowingAdvanced ? ( + {/* Namespace */} + + {/* Output */} + {canUseOutputPerIntegration && ( + + + } + helpText={ + + } + > + ({ + value: output.id, + text: output.name, + })), + ]} + value={packagePolicy.output_id || ''} + onChange={(e) => { + updatePackagePolicy({ + output_id: e.target.value.trim() || null, + }); + }} + /> + + + )} + + {/* Data retention settings info */} = ({ return packagePolicy.policy_ids.length || 0; }, []); + const { data: outputsData, isLoading: isOutputsLoading } = useGetOutputs(); + const { output: defaultOutputData } = useDefaultOutput(); + const outputNamesById = useMemo(() => { + const outputs = outputsData?.items ?? []; + return outputs.reduce>((acc, output) => { + acc[output.id] = output.name; + return acc; + }, {}); + }, [outputsData]); + const columns = useMemo( (): EuiInMemoryTableProps['columns'] => [ { @@ -115,6 +127,7 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.nameColumnTitle', { defaultMessage: 'Integration policy', }), + width: '35%', render: (value: string, packagePolicy: InMemoryPackagePolicy) => ( @@ -278,10 +291,67 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ ); }, }, + { + field: 'output_id', + name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.outputColumnTitle', { + defaultMessage: 'Output', + }), + render: (outputId: InMemoryPackagePolicy['output_id']) => { + if (isOutputsLoading) { + return null; + } + if (outputId) { + return {outputNamesById[outputId] || outputId}; + } + if (agentPolicy.data_output_id) { + return ( + <> + + {outputNamesById[agentPolicy.data_output_id] || agentPolicy.data_output_id} + +   + + + ); + } + if (defaultOutputData) { + return ( + <> + + {outputNamesById[defaultOutputData.id] || defaultOutputData.id} + +   + + + ); + } + }, + }, { name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.actionsColumnTitle', { defaultMessage: 'Actions', }), + width: '70px', actions: [ { render: (packagePolicy: InMemoryPackagePolicy) => { @@ -309,8 +379,11 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ agentPolicy, canUseMultipleAgentPolicies, canReadAgentPolicies, - canWriteIntegrationPolicies, getSharedPoliciesNumber, + canWriteIntegrationPolicies, + isOutputsLoading, + defaultOutputData, + outputNamesById, ] ); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx index 2a4250bd9707a..2d8a5b4aa1952 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx @@ -33,7 +33,7 @@ const ConfirmTitle = () => ( const ConfirmDescription: React.FunctionComponent = ({}) => ( ); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx index 8df533f0206b7..95a94848f4794 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx @@ -11,10 +11,6 @@ import { createFleetTestRendererMock } from '../../../../../../mock'; import { useFleetServerHostsForm } from './use_fleet_server_host_form'; -jest.mock('../../services/agent_and_policies_count', () => ({ - ...jest.requireActual('../../services/agent_and_policies_count'), - getAgentAndPolicyCount: () => ({ agentCount: 0, agentPolicyCount: 0 }), -})); jest.mock('../../hooks/use_confirm_modal', () => ({ ...jest.requireActual('../../hooks/use_confirm_modal'), useConfirmModal: () => ({ confirm: () => true }), diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx index 7e8cdac388cfe..8720ede4f04b8 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx @@ -5,17 +5,24 @@ * 2.0. */ -import { sendGetAgentPolicies, sendGetAgents } from '../../../hooks'; +import { sendGetAgentPolicies, sendGetPackagePolicies, sendGetAgents } from '../../../hooks'; import type { Output } from '../../../types'; -import { AGENT_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../../constants'; +import { + AGENT_POLICY_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, + SO_SEARCH_LIMIT, +} from '../../../constants'; export async function getAgentAndPolicyCountForOutput(output: Output) { - let kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${output.id}" or ${AGENT_POLICY_SAVED_OBJECT_TYPE}.monitoring_output_id:"${output.id}"`; + let agentPolicyKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${output.id}" or ${AGENT_POLICY_SAVED_OBJECT_TYPE}.monitoring_output_id:"${output.id}"`; + const packagePolicyKuery = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.output_id:"${output.id}"`; + if (output.is_default) { - kuery += ` or (not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*)`; + agentPolicyKuery += ` or (not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*)`; } + const agentPolicies = await sendGetAgentPolicies({ - kuery, + kuery: agentPolicyKuery, page: 1, perPage: SO_SEARCH_LIMIT, }); @@ -23,7 +30,26 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { if (agentPolicies.error) { throw agentPolicies.error; } - const agentPolicyCount = agentPolicies.data?.items?.length ?? 0; + + const packagePolicies = await sendGetPackagePolicies({ + kuery: packagePolicyKuery, + page: 1, + perPage: SO_SEARCH_LIMIT, + }); + + if (packagePolicies.error) { + throw agentPolicies.error; + } + + const agentPolicyIds = (agentPolicies.data?.items || []).map((policy) => policy.id); + const agentPolicyIdsFromPackagePolicies = (packagePolicies.data?.items || []).reduce( + (acc: string[], packagePolicy) => { + return [...acc, ...(packagePolicy.policy_ids || [])]; + }, + [] + ); + const uniqueAgentPolicyIds = new Set([...agentPolicyIds, ...agentPolicyIdsFromPackagePolicies]); + const agentPolicyCount = uniqueAgentPolicyIds.size; let agentCount = 0; if (agentPolicyCount > 0) { @@ -31,7 +57,7 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { page: 1, perPage: 0, // We only need the count here showInactive: false, - kuery: agentPolicies.data?.items.map((policy) => `policy_id:"${policy.id}"`).join(' or '), + kuery: [...uniqueAgentPolicyIds].map((id) => `policy_id:"${id}"`).join(' or '), }); if (agents.error) { @@ -43,28 +69,3 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { return { agentPolicyCount, agentCount }; } - -export async function getAgentAndPolicyCount() { - const agentPolicies = await sendGetAgentPolicies({ - perPage: 0, - }); - - if (agentPolicies.error) { - throw agentPolicies.error; - } - const agentPolicyCount = agentPolicies.data?.total ?? 0; - - const agents = await sendGetAgents({ - page: 1, - perPage: 0, // We only need the count here - showInactive: false, - }); - - if (agents.error) { - throw agents.error; - } - - const agentCount = agents.data?.total ?? 0; - - return { agentPolicyCount, agentCount }; -} diff --git a/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts b/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts index 2adf814e8ffba..b85b944c6ee00 100644 --- a/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts +++ b/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts @@ -5,12 +5,11 @@ * 2.0. */ +import { LICENCE_FOR_MULTIPLE_AGENT_POLICIES } from '../../common/constants'; import { ExperimentalFeaturesService } from '../services'; import { useLicense } from './use_license'; -export const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; - export function useMultipleAgentPolicies() { const licenseService = useLicense(); const { enableReusableIntegrationPolicies } = ExperimentalFeaturesService.get(); diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index e2f8b88380484..80d8116baaaa3 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -65,6 +65,8 @@ export class PackagePolicyNameExistsError extends FleetError {} export class BundledPackageLocationNotFoundError extends FleetError {} export class PackagePolicyRequestError extends FleetError {} +export class PackagePolicyMultipleAgentPoliciesError extends FleetError {} +export class PackagePolicyOutputError extends FleetError {} export class EnrollmentKeyNameExistsError extends FleetError {} export class HostedAgentPolicyRestrictionRelatedError extends FleetError { diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 9e517ad928ba3..9dfb920251e76 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -182,6 +182,7 @@ export const createPackagePolicyServiceMock = (): jest.Mocked; @@ -202,6 +201,7 @@ describe('When calling package policy', () => { }; beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); // @ts-ignore const postMock = routerMock.versioned.post.mock; // @ts-ignore @@ -221,29 +221,6 @@ describe('When calling package policy', () => { }, }); }); - - it('should throw if no enterprise license and multiple policy_ids is provided', async () => { - const request = getCreateKibanaRequest({ ...newPolicy, policy_ids: ['1', '2'] } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); - - it('should not throw if enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const request = getCreateKibanaRequest({ ...newPolicy, policy_ids: ['1', '2'] } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).not.toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); }); describe('Update api handler', () => { @@ -308,6 +285,7 @@ describe('When calling package policy', () => { }); beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); packagePolicyServiceMock.update.mockImplementation((soClient, esClient, policyId, newData) => Promise.resolve(newData as PackagePolicy) ); @@ -393,58 +371,6 @@ describe('When calling package policy', () => { }); }); - it('should throw if no enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); - - it('should not throw if enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.ok).toHaveBeenCalled(); - }); - - it('should throw if enterprise license and feature flag is disabled and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: false } as any); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are not supported', - }, - }); - }); - - it('should throw if empty policy_ids are provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); - const request = getUpdateKibanaRequest({ policy_ids: [] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'At least one agent policy id must be provided', - }, - }); - }); - it('should rename the agentless agent policy to sync with the package policy name if agentless is enabled', async () => { jest.spyOn(appContextService, 'getCloud').mockReturnValue({ isCloudEnabled: true } as any); jest diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index 24872b2db72f2..31e38125dead5 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -63,7 +63,6 @@ import { import type { SimplifiedPackagePolicy } from '../../../common/services/simplified_package_policy_helper'; import { - canUseMultipleAgentPolicies, isSimplifiedCreatePackagePolicyRequest, removeFieldsFromInputSchema, renameAgentlessAgentPolicy, @@ -242,21 +241,12 @@ export const createPackagePolicyHandler: FleetRequestHandler< const authorizationHeader = HTTPAuthorizationHeader.parseFromRequest(request, user?.username); let wasPackageAlreadyInstalled = false; - if ('output_id' in newPolicy) { - // TODO Remove deprecated APIs https://github.com/elastic/kibana/issues/121485 - delete newPolicy.output_id; - } const spaceId = fleetContext.spaceId; try { if (!newPolicy.policy_id && (!newPolicy.policy_ids || newPolicy.policy_ids.length === 0)) { throw new PackagePolicyRequestError('Either policy_id or policy_ids must be provided'); } - const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); - if ((newPolicy.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(errorMessage); - } - let newPackagePolicy: NewPackagePolicy; if (isSimplifiedCreatePackagePolicyRequest(newPolicy)) { if (!pkg) { @@ -369,11 +359,6 @@ export const updatePackagePolicyHandler: FleetRequestHandler< try { const { force, package: pkg, ...body } = request.body; - // TODO Remove deprecated APIs https://github.com/elastic/kibana/issues/121485 - if ('output_id' in body) { - delete body.output_id; - } - let newData: NewPackagePolicy; if ( @@ -420,10 +405,6 @@ export const updatePackagePolicyHandler: FleetRequestHandler< } } newData.inputs = alignInputsAndStreams(newData.inputs); - const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); - if ((newData.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(errorMessage); - } if (newData.policy_ids && newData.policy_ids.length === 0) { throw new PackagePolicyRequestError('At least one agent policy id must be provided'); diff --git a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts index f2ffc641c36ab..032dab4a07acc 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts @@ -21,8 +21,7 @@ import type { PackagePolicyInput, NewPackagePolicyInput, } from '../../../types'; -import { appContextService } from '../../../services'; -import { agentPolicyService, licenseService } from '../../../services'; +import { agentPolicyService } from '../../../services'; import type { SimplifiedPackagePolicy } from '../../../../common/services/simplified_package_policy_helper'; import { PackagePolicyRequestError } from '../../../errors'; import type { NewPackagePolicyInputStream } from '../../../../common'; @@ -56,20 +55,6 @@ export function removeFieldsFromInputSchema( }); } -const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; - -export function canUseMultipleAgentPolicies() { - const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_MULTIPLE_AGENT_POLICIES); - const { enableReusableIntegrationPolicies } = appContextService.getExperimentalFeatures(); - - return { - canUseReusablePolicies: hasEnterpriseLicence && enableReusableIntegrationPolicies, - errorMessage: !hasEnterpriseLicence - ? 'Reusable integration policies are only available with an Enterprise license' - : 'Reusable integration policies are not supported', - }; -} - /** * If an agentless agent policy is associated with the package policy, * it will rename the agentless agent policy of a package policy to keep it in sync with the package policy name. diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 4b76a952a0f3b..f6fcae4de6505 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -481,6 +481,7 @@ export const getSavedObjectTypes = ( is_managed: { type: 'boolean' }, policy_id: { type: 'keyword' }, policy_ids: { type: 'keyword' }, + output_id: { type: 'keyword' }, package: { properties: { name: { type: 'keyword' }, @@ -639,6 +640,16 @@ export const getSavedObjectTypes = ( }, ], }, + '14': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + output_id: { type: 'keyword' }, + }, + }, + ], + }, }, migrations: { '7.10.0': migratePackagePolicyToV7100, diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts index cf835ec87c949..d7e73d67a3c60 100644 --- a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts @@ -14,7 +14,6 @@ export const migratePackagePolicyToV850: SavedObjectMigrationFn { - // @ts-expect-error output_id property does not exists anymore delete packagePolicyDoc.attributes.output_id; return packagePolicyDoc; diff --git a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap index 86f8126c1c45a..8518e43fdaf0d 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap @@ -1,5 +1,173 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`getFullAgentPolicy should return the right outputs and permissions when package policies use their own outputs 1`] = ` +Object { + "agent": Object { + "download": Object { + "sourceURI": "http://default-registry.co", + }, + "features": Object {}, + "monitoring": Object { + "enabled": false, + "logs": false, + "metrics": false, + }, + "protection": Object { + "enabled": false, + "signing_key": "", + "uninstall_token_hash": "", + }, + }, + "fleet": Object { + "hosts": Array [ + "http://fleetserver:8220", + ], + }, + "id": "integration-output-policy", + "inputs": Array [ + Object { + "data_stream": Object { + "namespace": "policyspace", + }, + "id": "test-logs-package-policy-using-output", + "meta": Object { + "package": Object { + "name": "test_package", + "version": "0.0.0", + }, + }, + "name": "test-policy-1", + "package_policy_id": "package-policy-using-output", + "revision": 1, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "some-logs", + "type": "logs", + }, + "id": "test-logs", + }, + ], + "type": "test-logs", + "use_output": "test-remote-id", + }, + Object { + "data_stream": Object { + "namespace": "defaultspace", + }, + "id": "test-logs-package-policy-no-output", + "meta": Object { + "package": Object { + "name": "system", + "version": "1.0.0", + }, + }, + "name": "test-policy-2", + "package_policy_id": "package-policy-no-output", + "revision": 1, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "some-logs", + "type": "logs", + }, + "id": "test-logs", + }, + ], + "type": "test-logs", + "use_output": "data-output-id", + }, + ], + "output_permissions": Object { + "data-output-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "package-policy-no-output": Object { + "indices": Array [ + Object { + "names": Array [ + "logs-some-logs-defaultspace", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + "default": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "_elastic_agent_monitoring": Object { + "indices": Array [ + Object { + "names": Array [], + "privileges": Array [], + }, + ], + }, + }, + "test-remote-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "package-policy-using-output": Object { + "indices": Array [ + Object { + "names": Array [ + "logs-some-logs-policyspace", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + }, + "outputs": Object { + "data-output-id": Object { + "hosts": Array [ + "http://es-data.co:9201", + ], + "preset": "balanced", + "type": "elasticsearch", + }, + "default": Object { + "hosts": Array [ + "http://127.0.0.1:9201", + ], + "preset": "balanced", + "type": "elasticsearch", + }, + "test-remote-id": Object { + "hosts": Array [ + "http://127.0.0.1:9201", + ], + "preset": "balanced", + "service_token": undefined, + "type": "remote_elasticsearch", + }, + }, + "revision": 1, + "secret_references": Array [], + "signed": Object { + "data": "", + "signature": "", + }, +} +`; + exports[`getFullAgentPolicy should support a different data output 1`] = ` Object { "agent": Object { diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts index 5701a60b56d03..6a084b5dde586 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -401,6 +401,126 @@ describe('getFullAgentPolicy', () => { expect(agentPolicy?.outputs['test-remote-id']).toBeDefined(); }); + it('should return the right outputs and permissions when package policies use their own outputs', async () => { + mockedGetPackageInfo.mockResolvedValue({ + data_streams: [ + { + type: 'logs', + dataset: 'elastic_agent.metricbeat', + }, + { + type: 'metrics', + dataset: 'elastic_agent.metricbeat', + }, + { + type: 'logs', + dataset: 'elastic_agent.filebeat', + }, + { + type: 'metrics', + dataset: 'elastic_agent.filebeat', + }, + ], + } as PackageInfo); + mockAgentPolicy({ + id: 'integration-output-policy', + status: 'active', + package_policies: [ + { + id: 'package-policy-using-output', + name: 'test-policy-1', + namespace: 'policyspace', + enabled: true, + package: { name: 'test_package', version: '0.0.0', title: 'Test Package' }, + output_id: 'test-remote-id', + inputs: [ + { + type: 'test-logs', + enabled: true, + streams: [ + { + id: 'test-logs', + enabled: true, + data_stream: { type: 'logs', dataset: 'some-logs' }, + }, + ], + }, + { + type: 'test-metrics', + enabled: false, + streams: [ + { + id: 'test-logs', + enabled: false, + data_stream: { type: 'metrics', dataset: 'some-metrics' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + { + id: 'package-policy-no-output', + name: 'test-policy-2', + namespace: '', + enabled: true, + package: { name: 'system', version: '1.0.0', title: 'System' }, + inputs: [ + { + type: 'test-logs', + enabled: true, + streams: [ + { + id: 'test-logs', + enabled: true, + data_stream: { type: 'logs', dataset: 'some-logs' }, + }, + ], + }, + { + type: 'test-metrics', + enabled: false, + streams: [ + { + id: 'test-logs', + enabled: false, + data_stream: { type: 'metrics', dataset: 'some-metrics' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + ], + is_managed: false, + namespace: 'defaultspace', + revision: 1, + name: 'Policy', + updated_at: '2020-01-01', + updated_by: 'qwerty', + is_protected: false, + data_output_id: 'data-output-id', + }); + + const agentPolicy = await getFullAgentPolicy( + savedObjectsClientMock.create(), + 'integration-output-policy' + ); + expect(agentPolicy).toMatchSnapshot(); + }); + it('should return the sourceURI from the agent policy', async () => { mockAgentPolicy({ namespace: 'default', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index efc3a732149d6..0cfce9eb264f4 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -196,16 +196,46 @@ export async function getFullAgentPolicy( fullAgentPolicy.namespaces = [agentPolicy.space_id]; } - const dataPermissions = - (await storedPackagePoliciesToAgentPermissions( + const packagePoliciesByOutputId = Object.keys(fullAgentPolicy.outputs).reduce( + (acc: Record, outputId) => { + acc[outputId] = []; + return acc; + }, + {} + ); + (agentPolicy.package_policies || []).forEach((packagePolicy) => { + const packagePolicyDataOutput = packagePolicy.output_id + ? outputs.find((output) => output.id === packagePolicy.output_id) + : undefined; + if (packagePolicyDataOutput) { + packagePoliciesByOutputId[getOutputIdForAgentPolicy(packagePolicyDataOutput)].push( + packagePolicy + ); + } else { + packagePoliciesByOutputId[getOutputIdForAgentPolicy(dataOutput)].push(packagePolicy); + } + }); + + const dataPermissionsByOutputId = Object.keys(fullAgentPolicy.outputs).reduce( + (acc: Record, outputId) => { + acc[outputId] = {}; + return acc; + }, + {} + ); + for (const [outputId, packagePolicies] of Object.entries(packagePoliciesByOutputId)) { + const dataPermissions = await storedPackagePoliciesToAgentPermissions( packageInfoCache, agentPolicy.namespace, - agentPolicy.package_policies - )) || {}; - - dataPermissions._elastic_agent_checks = { - cluster: DEFAULT_CLUSTER_PERMISSIONS, - }; + packagePolicies + ); + dataPermissionsByOutputId[outputId] = { + _elastic_agent_checks: { + cluster: DEFAULT_CLUSTER_PERMISSIONS, + }, + ...(dataPermissions || {}), + }; + } const monitoringPermissions = await getMonitoringPermissions( soClient, @@ -233,8 +263,11 @@ export async function getFullAgentPolicy( Object.assign(permissions, monitoringPermissions); } - if (outputId === getOutputIdForAgentPolicy(dataOutput)) { - Object.assign(permissions, dataPermissions); + if ( + outputId === getOutputIdForAgentPolicy(dataOutput) || + packagePoliciesByOutputId[outputId].length > 0 + ) { + Object.assign(permissions, dataPermissionsByOutputId[outputId]); } outputPermissions[outputId] = permissions; diff --git a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts index dfdcf26adaa4e..0afef0170a99f 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts @@ -13,7 +13,7 @@ import { appContextService } from '..'; import { outputService } from '../output'; import { validateOutputForPolicy } from '.'; -import { validateOutputForNewPackagePolicy } from './outputs_helpers'; +import { validateAgentPolicyOutputForIntegration } from './outputs_helpers'; jest.mock('../app_context'); jest.mock('../output'); @@ -254,14 +254,14 @@ describe('validateOutputForPolicy', () => { }); }); -describe('validateOutputForNewPackagePolicy', () => { - it('should not allow fleet_server integration to be added to a policy using a logstash output', async () => { +describe('validateAgentPolicyOutputForIntegration', () => { + it('should not allow fleet_server integration to be added or edited to a policy using a logstash output', async () => { mockHasLicence(true); mockedOutputService.get.mockResolvedValue({ type: 'logstash', } as any); await expect( - validateOutputForNewPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -273,15 +273,29 @@ describe('validateOutputForNewPackagePolicy', () => { ).rejects.toThrow( 'Integration "fleet_server" cannot be added to agent policy "Agent policy" because it uses output type "logstash".' ); + await expect( + validateAgentPolicyOutputForIntegration( + savedObjectsClientMock.create(), + { + name: 'Agent policy', + data_output_id: 'test1', + monitoring_output_id: 'test1', + } as any, + 'fleet_server', + false + ) + ).rejects.toThrow( + 'Agent policy "Agent policy" uses output type "logstash" which cannot be used for integration "fleet_server".' + ); }); - it('should not allow apm integration to be added to a policy using a kafka output', async () => { + it('should not allow apm integration to be added or edited to a policy using a kafka output', async () => { mockHasLicence(true); mockedOutputService.get.mockResolvedValue({ type: 'kafka', } as any); await expect( - validateOutputForNewPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -293,6 +307,20 @@ describe('validateOutputForNewPackagePolicy', () => { ).rejects.toThrow( 'Integration "apm" cannot be added to agent policy "Agent policy" because it uses output type "kafka".' ); + await expect( + validateAgentPolicyOutputForIntegration( + savedObjectsClientMock.create(), + { + name: 'Agent policy', + data_output_id: 'test1', + monitoring_output_id: 'test1', + } as any, + 'apm', + false + ) + ).rejects.toThrow( + 'Agent policy "Agent policy" uses output type "kafka" which cannot be used for integration "apm".' + ); }); it('should not allow synthetics integration to be added to a policy using a default logstash output', async () => { @@ -302,7 +330,7 @@ describe('validateOutputForNewPackagePolicy', () => { } as any); mockedOutputService.getDefaultDataOutputId.mockResolvedValue('default'); await expect( - validateOutputForNewPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -320,7 +348,7 @@ describe('validateOutputForNewPackagePolicy', () => { type: 'logstash', } as any); - await validateOutputForNewPackagePolicy( + await validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -335,7 +363,7 @@ describe('validateOutputForNewPackagePolicy', () => { type: 'elasticsearch', } as any); - await validateOutputForNewPackagePolicy( + await validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts index 67f5a7772aa52..66f45427601e6 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts @@ -96,10 +96,11 @@ export async function validateOutputForPolicy( } } -export async function validateOutputForNewPackagePolicy( +export async function validateAgentPolicyOutputForIntegration( soClient: SavedObjectsClientContract, agentPolicy: AgentPolicy, - packageName: string + packageName: string, + isNewPackagePolicy: boolean = true ) { const allowedOutputTypeForPolicy = getAllowedOutputTypesForIntegration(packageName); @@ -109,9 +110,15 @@ export async function validateOutputForNewPackagePolicy( if (isOutputTypeRestricted) { const dataOutput = await getDataOutputForAgentPolicy(soClient, agentPolicy); if (!allowedOutputTypeForPolicy.includes(dataOutput.type)) { - throw new OutputInvalidError( - `Integration "${packageName}" cannot be added to agent policy "${agentPolicy.name}" because it uses output type "${dataOutput.type}".` - ); + if (isNewPackagePolicy) { + throw new OutputInvalidError( + `Integration "${packageName}" cannot be added to agent policy "${agentPolicy.name}" because it uses output type "${dataOutput.type}".` + ); + } else { + throw new OutputInvalidError( + `Agent policy "${agentPolicy.name}" uses output type "${dataOutput.type}" which cannot be used for integration "${packageName}".` + ); + } } } } diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts index 40cda7583a3c1..9004d544f399d 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts @@ -222,6 +222,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { version: '0.0.0', }, inputs: [mockInput, mockInput2], + output_id: 'new-output', }, { ...mockPackagePolicy, @@ -243,7 +244,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { revision: 1, type: 'test-logs', data_stream: { namespace: 'default' }, - use_output: 'default', + use_output: 'new-output', meta: { package: { name: 'mock_package', @@ -270,7 +271,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { revision: 1, type: 'test-metrics', data_stream: { namespace: 'default' }, - use_output: 'default', + use_output: 'new-output', meta: { package: { name: 'mock_package', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts index d7f0c70a0786b..807312fe5e7cb 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts @@ -27,7 +27,7 @@ const isPolicyEnabled = (packagePolicy: PackagePolicy) => { export const storedPackagePolicyToAgentInputs = ( packagePolicy: PackagePolicy, packageInfo?: PackageInfo, - outputId: string = DEFAULT_OUTPUT.name, + agentPolicyOutputId: string = DEFAULT_OUTPUT.name, agentPolicyNamespace?: string, addFields?: FullAgentPolicyAddFields ): FullAgentPolicyInput[] => { @@ -62,7 +62,7 @@ export const storedPackagePolicyToAgentInputs = ( data_stream: { namespace: packagePolicy?.namespace || agentPolicyNamespace || 'default', // custom namespace has precedence on agent policy's one }, - use_output: outputId, + use_output: packagePolicy.output_id || agentPolicyOutputId, package_policy_id: packagePolicy.id, ...getFullInputStreams(input), }; @@ -140,7 +140,7 @@ export const getFullInputStreams = ( export const storedPackagePoliciesToAgentInputs = async ( packagePolicies: PackagePolicy[], packageInfoCache: Map, - outputId: string = DEFAULT_OUTPUT.name, + agentPolicyOutputId: string = DEFAULT_OUTPUT.name, agentPolicyNamespace?: string, globalDataTags?: GlobalDataTag[] ): Promise => { @@ -164,7 +164,7 @@ export const storedPackagePoliciesToAgentInputs = async ( ...storedPackagePolicyToAgentInputs( packagePolicy, packageInfo, - outputId, + agentPolicyOutputId, agentPolicyNamespace, addFields ) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts index 0108e9cd97721..52dd34a757693 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts @@ -35,11 +35,20 @@ export async function fetchRelatedSavedObjects( const monitoringOutputId = agentPolicy.monitoring_output_id || defaultMonitoringOutputId || dataOutputId; + const outputIds = uniq([ + dataOutputId, + monitoringOutputId, + ...(agentPolicy.package_policies || []).reduce((acc: string[], packagePolicy) => { + if (packagePolicy.output_id) { + acc.push(packagePolicy.output_id); + } + return acc; + }, []), + ]); + const [outputs, { host: downloadSourceUri, proxy_id: downloadSourceProxyId }, fleetServerHosts] = await Promise.all([ - outputService.bulkGet(soClient, uniq([dataOutputId, monitoringOutputId]), { - ignoreNotFound: true, - }), + outputService.bulkGet(soClient, outputIds, { ignoreNotFound: true }), getSourceUriForAgentPolicy(soClient, agentPolicy), getFleetServerHostsForAgentPolicy(soClient, agentPolicy).catch((err) => { appContextService diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index a5dbbc6b233b3..2c2c747681887 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -23,7 +23,7 @@ import type { NewAgentPolicy, PreconfiguredAgentPolicy, } from '../types'; -import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../constants'; +import { AGENT_POLICY_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants'; import { AGENT_POLICY_INDEX, SO_SEARCH_LIMIT } from '../../common'; @@ -43,32 +43,63 @@ import type { UninstallTokenServiceInterface } from './security/uninstall_token_ function getSavedObjectMock(agentPolicyAttributes: any) { const mock = savedObjectsClientMock.create(); + const mockPolicy = { + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + references: [], + attributes: agentPolicyAttributes as AgentPolicy, + }; mock.get.mockImplementation(async (type: string, id: string) => { return { - type, id, - references: [], - attributes: agentPolicyAttributes as AgentPolicy, + ...mockPolicy, }; }); - mock.find.mockImplementation(async (options) => { + mock.bulkGet.mockImplementation(async (options) => { return { - saved_objects: [ - { - id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', - attributes: { - fleet_server_hosts: ['http://fleetserver:8220'], - }, - type: 'ingest_manager_settings', - score: 1, - references: [], - }, - ], - total: 1, - page: 1, - per_page: 1, + saved_objects: [], }; }); + mock.find.mockImplementation(async (options) => { + switch (options.type) { + case AGENT_POLICY_SAVED_OBJECT_TYPE: + return { + saved_objects: [ + { + id: 'agent-policy-id', + score: 1, + ...mockPolicy, + }, + ], + total: 1, + page: 1, + per_page: 1, + }; + case PACKAGE_POLICY_SAVED_OBJECT_TYPE: + return { + saved_objects: [], + total: 0, + page: 1, + per_page: 1, + }; + default: + return { + saved_objects: [ + { + id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', + attributes: { + fleet_server_hosts: ['http://fleetserver:8220'], + }, + type: 'ingest_manager_settings', + score: 1, + references: [], + }, + ], + total: 1, + page: 1, + per_page: 1, + }; + } + }); return mock; } @@ -754,12 +785,12 @@ describe('Agent policy', () => { [ expect.objectContaining({ attributes: expect.objectContaining({ - fleet_server_hosts: ['http://fleetserver:8220'], - revision: NaN, - updated_by: 'system', + monitoring_enabled: ['metrics'], }), - id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', - type: 'ingest_manager_settings', + id: 'agent-policy-id', + namespace: undefined, + type: 'ingest-agent-policies', + version: undefined, }), ], expect.objectContaining({ diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index d243ef8b60e16..42bede95986b6 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -16,7 +16,7 @@ import type { SavedObjectsBulkUpdateObject, SavedObjectsBulkUpdateResponse, SavedObjectsClientContract, - SavedObjectsFindResult, + SavedObject, SavedObjectsUpdateResponse, } from '@kbn/core/server'; import { SavedObjectsUtils } from '@kbn/core/server'; @@ -41,6 +41,7 @@ import { import type { HTTPAuthorizationHeader } from '../../common/http_authorization_header'; import { + PACKAGE_POLICY_SAVED_OBJECT_TYPE, AGENT_POLICY_SAVED_OBJECT_TYPE, AGENTS_PREFIX, FLEET_AGENT_POLICIES_SCHEMA_VERSION, @@ -56,6 +57,7 @@ import type { NewAgentPolicy, NewPackagePolicy, PackagePolicy, + PackagePolicySOAttributes, PostAgentPolicyCreateCallback, PostAgentPolicyUpdateCallback, PreconfiguredAgentPolicy, @@ -876,7 +878,7 @@ class AgentPolicyService { private async _bumpPolicies( internalSoClientWithoutSpaceExtension: SavedObjectsClientContract, esClient: ElasticsearchClient, - savedObjectsResults: Array>, + savedObjectsResults: Array>, options?: { user?: AuthenticatedUser } ): Promise> { const bumpedPolicies = savedObjectsResults.map( @@ -938,10 +940,12 @@ class AgentPolicyService { outputId: string, options?: { user?: AuthenticatedUser } ): Promise> { + const { useSpaceAwareness } = appContextService.getExperimentalFeatures(); const internalSoClientWithoutSpaceExtension = appContextService.getInternalUserSOClientWithoutSpaceExtension(); - const currentPolicies = + // All agent policies directly using output + const agentPoliciesUsingOutput = await internalSoClientWithoutSpaceExtension.find({ type: SAVED_OBJECT_TYPE, fields: ['revision', 'data_output_id', 'monitoring_output_id', 'namespaces'], @@ -950,10 +954,47 @@ class AgentPolicyService { perPage: SO_SEARCH_LIMIT, namespaces: ['*'], }); + + // All package policies directly using output + const packagePoliciesUsingOutput = + await internalSoClientWithoutSpaceExtension.find({ + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + fields: ['output_id', 'namespaces', 'policy_ids'], + searchFields: ['output_id'], + search: escapeSearchQueryPhrase(outputId), + perPage: SO_SEARCH_LIMIT, + namespaces: ['*'], + }); + + const agentPolicyIdsDirectlyUsingOutput = agentPoliciesUsingOutput.saved_objects.map( + (agentPolicySO) => agentPolicySO.id + ); + const agentPolicyIdsOfPackagePoliciesUsingOutput = + packagePoliciesUsingOutput.saved_objects.reduce((acc: Set, packagePolicySO) => { + const newIds = packagePolicySO.attributes.policy_ids.filter((policyId) => { + return !agentPolicyIdsDirectlyUsingOutput.includes(policyId); + }); + return new Set([...acc, ...newIds]); + }, new Set()); + + // Agent policies of the identified package policies, excluding ones already retrieved directly + const agentPoliciesOfPackagePoliciesUsingOutput = + await internalSoClientWithoutSpaceExtension.bulkGet( + [...agentPolicyIdsOfPackagePoliciesUsingOutput].map((id) => ({ + type: SAVED_OBJECT_TYPE, + id, + fields: ['revision', 'data_output_id', 'monitoring_output_id', 'namespaces'], + ...(useSpaceAwareness ? { namespaces: ['*'] } : {}), + })) + ); + return this._bumpPolicies( internalSoClientWithoutSpaceExtension, esClient, - currentPolicies.saved_objects, + [ + ...agentPoliciesUsingOutput.saved_objects, + ...agentPoliciesOfPackagePoliciesUsingOutput.saved_objects, + ], options ); } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 3bc9003162f44..0951657fae6dd 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -19,10 +19,12 @@ import { OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; import { outputService, outputIdToUuid } from './output'; import { appContextService } from './app_context'; import { agentPolicyService } from './agent_policy'; +import { packagePolicyService } from './package_policy'; import { auditLoggingService } from './audit_logging'; jest.mock('./app_context'); jest.mock('./agent_policy'); +jest.mock('./package_policy'); jest.mock('./audit_logging'); const mockedAuditLoggingService = auditLoggingService as jest.Mocked; @@ -43,6 +45,7 @@ mockedAppContextService.getLogger.mockImplementation(() => { mockedAppContextService.getExperimentalFeatures.mockReturnValue({} as any); const mockedAgentPolicyService = agentPolicyService as jest.Mocked; +const mockedPackagePolicyService = packagePolicyService as jest.Mocked; const CLOUD_ID = 'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw=='; @@ -219,6 +222,7 @@ function getMockedSoClient( }); mockedAppContextService.getInternalUserSOClient.mockReturnValue(soClient); + mockedAppContextService.getInternalUserSOClientWithoutSpaceExtension.mockReturnValue(soClient); return soClient; } @@ -288,17 +292,23 @@ describe('Output Service', () => { } as unknown as ReturnType; beforeEach(() => { + mockedAgentPolicyService.getByIDs.mockResolvedValue([]); mockedAgentPolicyService.list.mockClear(); mockedAgentPolicyService.hasAPMIntegration.mockClear(); mockedAgentPolicyService.hasFleetServerIntegration.mockClear(); mockedAgentPolicyService.hasSyntheticsIntegration.mockClear(); mockedAgentPolicyService.removeOutputFromAll.mockReset(); + mockedPackagePolicyService.removeOutputFromAll.mockReset(); mockedAppContextService.getInternalUserSOClient.mockReset(); mockedAppContextService.getEncryptedSavedObjectsSetup.mockReset(); mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); mockedAgentPolicyService.update.mockReset(); }); + afterEach(() => { + mockedAgentPolicyService.getByIDs.mockClear(); + }); + describe('create', () => { describe('elasticsearch output', () => { it('works with a predefined id', async () => { @@ -1412,7 +1422,7 @@ describe('Output Service', () => { hosts: ['test:4343'], }) ).rejects.toThrowError( - 'Logstash output cannot be used with Fleet Server integration in fleet server policy. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Fleet Server integration in fleet server policy. Please create a new Elasticsearch output.' ); }); @@ -1428,7 +1438,7 @@ describe('Output Service', () => { hosts: ['test:4343'], }) ).rejects.toThrowError( - 'Logstash output cannot be used with Synthetics integration in synthetics policy. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Synthetics integration in synthetics policy. Please create a new Elasticsearch output.' ); }); @@ -1758,6 +1768,7 @@ describe('Output Service', () => { fromPreconfiguration: true, }); expect(mockedAgentPolicyService.removeOutputFromAll).toBeCalled(); + expect(mockedPackagePolicyService.removeOutputFromAll).toBeCalled(); expect(soClient.delete).toBeCalled(); }); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index f42867f0686bb..2748ad78e765b 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -38,6 +38,7 @@ import type { } from '../types'; import { AGENT_POLICY_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, DEFAULT_OUTPUT, DEFAULT_OUTPUT_ID, OUTPUT_SAVED_OBJECT_TYPE, @@ -63,6 +64,7 @@ import { import type { OutputType } from '../types'; import { agentPolicyService } from './agent_policy'; +import { packagePolicyService } from './package_policy'; import { appContextService } from './app_context'; import { escapeSearchQueryPhrase } from './saved_object'; import { auditLoggingService } from './audit_logging'; @@ -124,40 +126,63 @@ function outputSavedObjectToOutput(so: SavedObject): Output }; } -async function getAgentPoliciesPerOutput( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { - let kuery: string; +async function getAgentPoliciesPerOutput(outputId?: string, isDefault?: boolean) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + let agentPoliciesKuery: string; + const packagePoliciesKuery: string = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.output_id:"${outputId}"`; if (outputId) { if (isDefault) { - kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}" or not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; + agentPoliciesKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}" or not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; } else { - kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}"`; + agentPoliciesKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}"`; } } else { if (isDefault) { - kuery = `not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; + agentPoliciesKuery = `not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; } else { return; } } - const agentPolicySO = await agentPolicyService.list(soClient, { - kuery, + // Get agent policies directly using output + const directAgentPolicies = await agentPolicyService.list(internalSoClientWithoutSpaceExtension, { + kuery: agentPoliciesKuery, perPage: SO_SEARCH_LIMIT, withPackagePolicies: true, }); - return agentPolicySO?.items; + const directAgentPolicyIds = directAgentPolicies?.items.map((policy) => policy.id); + + // Get package policies using output and derive agent policies from that which + // are not already identfied above. The IDs cannot be used as part of the kuery + // above since the underlying saved object client .find() only filters on attributes + const packagePolicySOs = await packagePolicyService.list(internalSoClientWithoutSpaceExtension, { + kuery: packagePoliciesKuery, + perPage: SO_SEARCH_LIMIT, + }); + const agentPolicyIdsFromPackagePolicies = [ + ...new Set( + packagePolicySOs?.items.reduce((acc: string[], packagePolicy) => { + return [ + ...acc, + ...packagePolicy.policy_ids.filter((id) => !directAgentPolicyIds?.includes(id)), + ]; + }, []) + ), + ]; + const agentPoliciesFromPackagePolicies = await agentPolicyService.getByIDs( + internalSoClientWithoutSpaceExtension, + agentPolicyIdsFromPackagePolicies, + { + withPackagePolicies: true, + } + ); + + return [...directAgentPolicies.items, ...agentPoliciesFromPackagePolicies]; } -async function validateLogstashOutputNotUsedInAPMPolicy( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { - const agentPolicies = await getAgentPoliciesPerOutput(soClient, outputId, isDefault); +async function validateLogstashOutputNotUsedInAPMPolicy(outputId?: string, isDefault?: boolean) { + const agentPolicies = await getAgentPoliciesPerOutput(outputId, isDefault); // Validate no policy with APM use that policy if (agentPolicies) { @@ -169,16 +194,19 @@ async function validateLogstashOutputNotUsedInAPMPolicy( } } -async function findPoliciesWithFleetServerOrSynthetics( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { +async function findPoliciesWithFleetServerOrSynthetics(outputId?: string, isDefault?: boolean) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + // find agent policies by outputId // otherwise query all the policies const agentPolicies = outputId - ? await getAgentPoliciesPerOutput(soClient, outputId, isDefault) - : (await agentPolicyService.list(soClient, { withPackagePolicies: true }))?.items; + ? await getAgentPoliciesPerOutput(outputId, isDefault) + : ( + await agentPolicyService.list(internalSoClientWithoutSpaceExtension, { + withPackagePolicies: true, + }) + )?.items; const policiesWithFleetServer = agentPolicies?.filter((policy) => agentPolicyService.hasFleetServerIntegration(policy)) || []; @@ -199,13 +227,12 @@ function validateOutputNotUsedInPolicy( dataOutputType )} output cannot be used with ${integrationName} integration in ${ agentPolicy.name - }. Please create a new ElasticSearch output.` + }. Please create a new Elasticsearch output.` ); } } async function validateTypeChanges( - soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, id: string, data: Nullable>, @@ -213,12 +240,14 @@ async function validateTypeChanges( defaultDataOutputId: string | null, fromPreconfiguration: boolean ) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); const mergedIsDefault = data.is_default ?? originalOutput.is_default; const { policiesWithFleetServer, policiesWithSynthetics } = - await findPoliciesWithFleetServerOrSynthetics(soClient, id, mergedIsDefault); + await findPoliciesWithFleetServerOrSynthetics(id, mergedIsDefault); if (data.type === outputType.Logstash || originalOutput.type === outputType.Logstash) { - await validateLogstashOutputNotUsedInAPMPolicy(soClient, id, mergedIsDefault); + await validateLogstashOutputNotUsedInAPMPolicy(id, mergedIsDefault); } // prevent changing an ES output to logstash or kafka if it's used by fleet server or synthetics policies if ( @@ -230,7 +259,7 @@ async function validateTypeChanges( validateOutputNotUsedInPolicy(policiesWithSynthetics, data.type, 'Synthetics'); } await updateAgentPoliciesDataOutputId( - soClient, + internalSoClientWithoutSpaceExtension, esClient, data, mergedIsDefault, @@ -274,7 +303,7 @@ class OutputService { return appContextService.getInternalUserSOClient(fakeRequest); } - private async _getDefaultDataOutputsSO(soClient: SavedObjectsClientContract) { + private async _getDefaultDataOutputsSO() { const outputs = await this.encryptedSoClient.find({ type: OUTPUT_SAVED_OBJECT_TYPE, searchFields: ['is_default'], @@ -403,7 +432,7 @@ class OutputService { } public async getDefaultDataOutputId(soClient: SavedObjectsClientContract) { - const outputs = await this._getDefaultDataOutputsSO(soClient); + const outputs = await this._getDefaultDataOutputsSO(); if (!outputs.saved_objects.length) { return null; @@ -454,7 +483,7 @@ class OutputService { const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); if (output.type === outputType.Logstash || output.type === outputType.Kafka) { - await validateLogstashOutputNotUsedInAPMPolicy(soClient, undefined, data.is_default); + await validateLogstashOutputNotUsedInAPMPolicy(undefined, data.is_default); if (!appContextService.getEncryptedSavedObjectsSetup()?.canEncrypt) { throw new FleetEncryptedSavedObjectEncryptionKeyRequired( `${output.type} output needs encrypted saved object api key to be set` @@ -462,7 +491,7 @@ class OutputService { } } const { policiesWithFleetServer, policiesWithSynthetics } = - await findPoliciesWithFleetServerOrSynthetics(soClient); + await findPoliciesWithFleetServerOrSynthetics(); await updateAgentPoliciesDataOutputId( soClient, esClient, @@ -740,8 +769,17 @@ class OutputService { throw new OutputUnauthorizedError(`Default monitoring output ${id} cannot be deleted.`); } + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + + await packagePolicyService.removeOutputFromAll( + internalSoClientWithoutSpaceExtension, + appContextService.getInternalUserESClient(), + id + ); + await agentPolicyService.removeOutputFromAll( - soClient, + internalSoClientWithoutSpaceExtension, appContextService.getInternalUserESClient(), id ); @@ -808,7 +846,6 @@ class OutputService { const mergedType = data.type ?? originalOutput.type; const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); await validateTypeChanges( - soClient, esClient, id, updateData, diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts index 9bf58298b09d3..311bfeb339f20 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts @@ -4,10 +4,15 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; import { PackagePolicyMocks } from '../../mocks'; -import { mapPackagePolicySavedObjectToPackagePolicy } from './utils'; +import { appContextService } from '../app_context'; +import { licenseService } from '../license'; +import { outputService } from '../output'; + +import { mapPackagePolicySavedObjectToPackagePolicy, preflightCheckPackagePolicy } from './utils'; describe('Package Policy Utils', () => { describe('mapPackagePolicySavedObjectToPackagePolicy()', () => { @@ -38,6 +43,7 @@ describe('Package Policy Utils', () => { }, policy_id: '444-555-666', policy_ids: ['444-555-666'], + output_id: 'output-123', revision: 1, secret_references: [], updated_at: '2024-01-25T15:21:13.389Z', @@ -47,4 +53,94 @@ describe('Package Policy Utils', () => { }); }); }); + + describe('preflightCheckPackagePolicy', () => { + beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); + jest.spyOn(appContextService, 'getExperimentalFeatures').mockClear(); + }); + const soClient = savedObjectsClientMock.create(); + const testPolicy = { + name: 'Test Package Policy', + namespace: 'test', + enabled: true, + policy_ids: ['test'], + inputs: [], + package: { + name: 'test', + title: 'Test', + version: '0.0.1', + }, + }; + + it('should throw if no enterprise license and multiple policy_ids is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).rejects.toThrowError( + 'Reusable integration policies are only available with an Enterprise license' + ); + }); + + it('should throw if enterprise license and multiple policy_ids is provided but no feature flag', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: false } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).rejects.toThrowError('Reusable integration policies are not supported'); + }); + + it('should not throw if enterprise license and multiple policy_ids is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).resolves.not.toThrow(); + }); + + it('should throw if no valid license and output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, output_id: 'some-output' }) + ).rejects.toThrowError('Output per integration is only available with an enterprise license'); + }); + + it('should throw if valid license and an incompatible output_id for the package is given', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(outputService, 'get') + .mockResolvedValueOnce({ id: 'non-es-output', type: 'kafka' } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { + ...testPolicy, + output_id: 'non-es-output', + package: { name: 'apm', version: '1.0.0', title: 'APM' }, + }) + ).rejects.toThrowError('Output type "kafka" is not usable with package "apm"'); + }); + + it('should not throw if valid license and valid output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(outputService, 'get') + .mockResolvedValueOnce({ id: 'es-output', type: 'elasticsearch' } as any); + await expect( + preflightCheckPackagePolicy(soClient, { + ...testPolicy, + output_id: 'es-output', + }) + ).resolves.not.toThrow(); + }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.ts index cf2d69c8874b9..61ff380fec559 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.ts @@ -6,8 +6,18 @@ */ import type { SavedObject } from '@kbn/core-saved-objects-common/src/server_types'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; -import type { PackagePolicy, PackagePolicySOAttributes } from '../../types'; +import { + LICENCE_FOR_OUTPUT_PER_INTEGRATION, + LICENCE_FOR_MULTIPLE_AGENT_POLICIES, +} from '../../../common/constants'; +import { getAllowedOutputTypesForIntegration } from '../../../common/services/output_helpers'; +import type { PackagePolicy, NewPackagePolicy, PackagePolicySOAttributes } from '../../types'; +import { PackagePolicyMultipleAgentPoliciesError, PackagePolicyOutputError } from '../../errors'; +import { licenseService } from '../license'; +import { outputService } from '../output'; +import { appContextService } from '../app_context'; export const mapPackagePolicySavedObjectToPackagePolicy = ({ /* eslint-disable @typescript-eslint/naming-convention */ @@ -21,6 +31,7 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ is_managed, policy_id, policy_ids, + output_id, // `package` is a reserved keyword package: packageInfo, inputs, @@ -45,6 +56,7 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ is_managed, policy_id, policy_ids, + output_id, package: packageInfo, inputs, vars, @@ -59,3 +71,69 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ created_by, }; }; + +export async function preflightCheckPackagePolicy( + soClient: SavedObjectsClientContract, + packagePolicy: PackagePolicy | NewPackagePolicy +) { + // If package policy has multiple agent policies, check if they can be used + const { canUseReusablePolicies, errorMessage: canUseMultipleAgentPoliciesErrorMessage } = + canUseMultipleAgentPolicies(); + if ((packagePolicy.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { + throw new PackagePolicyMultipleAgentPoliciesError(canUseMultipleAgentPoliciesErrorMessage); + } + + // If package policy has an output_id, check if it can be used + if (packagePolicy.output_id && packagePolicy.package) { + const { canUseOutputForIntegrationResult, errorMessage: outputForIntegrationErrorMessage } = + await canUseOutputForIntegration( + soClient, + packagePolicy.package.name, + packagePolicy.output_id + ); + if (!canUseOutputForIntegrationResult && outputForIntegrationErrorMessage) { + throw new PackagePolicyOutputError(outputForIntegrationErrorMessage); + } + } +} + +export function canUseMultipleAgentPolicies() { + const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_MULTIPLE_AGENT_POLICIES); + const { enableReusableIntegrationPolicies } = appContextService.getExperimentalFeatures(); + + return { + canUseReusablePolicies: hasEnterpriseLicence && enableReusableIntegrationPolicies, + errorMessage: !hasEnterpriseLicence + ? 'Reusable integration policies are only available with an Enterprise license' + : 'Reusable integration policies are not supported', + }; +} + +export async function canUseOutputForIntegration( + soClient: SavedObjectsClientContract, + packageName: string, + outputId: string +) { + const hasAllowedLicense = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + if (!hasAllowedLicense) { + return { + canUseOutputForIntegrationResult: false, + errorMessage: `Output per integration is only available with an ${LICENCE_FOR_OUTPUT_PER_INTEGRATION} license`, + }; + } + + const allowedOutputTypes = getAllowedOutputTypesForIntegration(packageName); + const output = await outputService.get(soClient, outputId); + + if (!allowedOutputTypes.includes(output.type)) { + return { + canUseOutputForIntegrationResult: false, + errorMessage: `Output type "${output.type}" is not usable with package "${packageName}"`, + }; + } + + return { + canUseOutputForIntegrationResult: true, + errorMessage: null, + }; +} diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index a62a5048ec567..9505edd3556cb 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -2921,6 +2921,7 @@ describe('Package policy service', () => { version: 'WzYyMzcsMV0=', name: 'my-cis_kubernetes_benchmark', namespace: 'default', + output_id: null, description: '', package: { name: 'cis_kubernetes_benchmark', @@ -5096,6 +5097,61 @@ describe('Package policy service', () => { ); }); }); + + describe('removeOutputFromAll', () => { + it('should update policies using deleted output', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const updateSpy = jest.spyOn(packagePolicyService, 'update'); + soClient.find.mockResolvedValue({ + saved_objects: [ + { + id: 'package-policy-1', + attributes: { + name: 'policy1', + enabled: true, + policy_ids: ['agent-policy-1'], + output_id: 'output-id-123', + inputs: [], + package: { name: 'test-package', version: '1.0.0' }, + }, + }, + ], + } as any); + soClient.get.mockImplementation((type, id, options): any => { + if (id === 'package-policy-1') { + return Promise.resolve({ + id, + attributes: { + name: 'policy1', + enabled: true, + policy_ids: ['agent-policy-1'], + output_id: 'output-id-123', + inputs: [], + package: { name: 'test-package', version: '1.0.0' }, + }, + }); + } + }); + + await packagePolicyService.removeOutputFromAll(soClient, esClient, 'output-id-123'); + + expect(updateSpy).toHaveBeenCalledTimes(1); + expect(updateSpy).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + 'package-policy-1', + { + name: 'policy1', + enabled: true, + policy_id: 'agent-policy-1', + policy_ids: ['agent-policy-1'], + output_id: null, + inputs: [], + } + ); + }); + }); }); describe('getUpgradeDryRunDiff', () => { diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index d9605880c1572..65495d51f0f60 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -115,6 +115,7 @@ import { sendTelemetryEvents } from './upgrade_sender'; import { handleExperimentalDatastreamFeatureOptIn, mapPackagePolicySavedObjectToPackagePolicy, + preflightCheckPackagePolicy, } from './package_policies'; import { updateDatastreamExperimentalFeatures } from './epm/packages/update'; import type { @@ -131,7 +132,7 @@ import { isSecretStorageEnabled, } from './secrets'; import { getPackageAssetsMap } from './epm/packages/get'; -import { validateOutputForNewPackagePolicy } from './agent_policies/outputs_helpers'; +import { validateAgentPolicyOutputForIntegration } from './agent_policies/outputs_helpers'; import type { PackagePolicyClientFetchAllItemIdsOptions } from './package_policy_service'; import { validatePolicyNamespaceForSpace } from './spaces/policy_namespaces'; @@ -212,6 +213,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { logger.debug(`Creating new package policy`); this.keepPolicyIdInSync(packagePolicy); + await preflightCheckPackagePolicy(soClient, packagePolicy); let enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( 'packagePolicyCreate', @@ -228,8 +230,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const agentPolicy = await agentPolicyService.get(soClient, policyId, true); agentPolicies.push(agentPolicy); - if (agentPolicy && enrichedPackagePolicy.package?.name) { - await validateOutputForNewPackagePolicy( + // If package policy did not set an output_id, see if the agent policy's output is compatible + if (!packagePolicy.output_id && agentPolicy && enrichedPackagePolicy.package?.name) { + await validateAgentPolicyOutputForIntegration( soClient, agentPolicy, enrichedPackagePolicy.package?.name @@ -408,14 +411,14 @@ class PackagePolicyClientImpl implements PackagePolicyClient { if (!packagePolicy.id) { packagePolicy.id = SavedObjectsUtils.generateId(); } - - this.keepPolicyIdInSync(packagePolicy); - auditLoggingService.writeCustomSoAuditLog({ action: 'create', id: packagePolicy.id, savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, }); + + this.keepPolicyIdInSync(packagePolicy); + await preflightCheckPackagePolicy(soClient, packagePolicy); } const agentPolicyIds = new Set(packagePolicies.flatMap((pkgPolicy) => pkgPolicy.policy_ids)); @@ -471,6 +474,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } const { pkgInfo, assetsMap } = packageInfoAndAsset; + validatePackagePolicyOrThrow(packagePolicy, pkgInfo); inputs = pkgInfo ? await _compilePackagePolicyInputs( @@ -822,6 +826,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { try { logger.debug(`Starting update of package policy ${id}`); + await preflightCheckPackagePolicy(soClient, packagePolicyUpdate); enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( 'packagePolicyUpdate', packagePolicyUpdate, @@ -1052,6 +1057,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const id = packagePolicyUpdate.id; this.keepPolicyIdInSync(packagePolicyUpdate); const packagePolicy = { ...packagePolicyUpdate, name: packagePolicyUpdate.name.trim() }; + await preflightCheckPackagePolicy(soClient, packagePolicy); const oldPackagePolicy = oldPackagePolicies.find((p) => p.id === id); if (!oldPackagePolicy) { throw new PackagePolicyNotFoundError('Package policy not found'); @@ -1762,6 +1768,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { }, policy_id: newPolicy.policy_id ?? agentPolicyId, policy_ids: newPolicy.policy_ids ?? [agentPolicyId], + output_id: newPolicy.output_id, inputs: newPolicy.inputs[0]?.streams ? newPolicy.inputs : inputs, vars: newPolicy.vars || newPP.vars, }; @@ -1982,6 +1989,76 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } } + public async removeOutputFromAll( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + outputId: string + ) { + const packagePolicies = ( + await soClient.find({ + type: SAVED_OBJECT_TYPE, + fields: ['name', 'enabled', 'policy_ids', 'inputs', 'output_id'], + searchFields: ['output_id'], + search: escapeSearchQueryPhrase(outputId), + perPage: SO_SEARCH_LIMIT, + }) + ).saved_objects.map(mapPackagePolicySavedObjectToPackagePolicy); + + if (packagePolicies.length > 0) { + const getPackagePolicyUpdate = (packagePolicy: PackagePolicy) => ({ + name: packagePolicy.name, + enabled: packagePolicy.enabled, + policy_ids: packagePolicy.policy_ids, + inputs: packagePolicy.inputs, + output_id: packagePolicy.output_id === outputId ? null : packagePolicy.output_id, + }); + + // Validate that the new cleared/default output is valid for the package policies + // (from each of the associated agent policies) before updating any of them + await pMap( + packagePolicies, + async (packagePolicy) => { + const existingPackagePolicy = await this.get(soClient, packagePolicy.id); + + if (!existingPackagePolicy) { + throw new PackagePolicyNotFoundError('Package policy not found'); + } + + for (const policyId of packagePolicy.policy_ids) { + if (packagePolicy.package?.name) { + const agentPolicy = await agentPolicyService.get(soClient, policyId, true); + if (agentPolicy) { + await validateAgentPolicyOutputForIntegration( + soClient, + agentPolicy, + packagePolicy.package.name, + false + ); + } + } + } + }, + { + concurrency: 50, + } + ); + await pMap( + packagePolicies, + (packagePolicy) => { + return this.update( + soClient, + esClient, + packagePolicy.id, + getPackagePolicyUpdate(packagePolicy) + ); + }, + { + concurrency: 50, + } + ); + } + } + fetchAllItemIds( soClient: SavedObjectsClientContract, { perPage = 1000, kuery }: PackagePolicyClientFetchAllItemIdsOptions = {} diff --git a/x-pack/plugins/fleet/server/services/package_policy_service.ts b/x-pack/plugins/fleet/server/services/package_policy_service.ts index 1462d451388b5..fed46872ab6cb 100644 --- a/x-pack/plugins/fleet/server/services/package_policy_service.ts +++ b/x-pack/plugins/fleet/server/services/package_policy_service.ts @@ -219,6 +219,18 @@ export interface PackagePolicyClient { experimentalDataStreamFeatures: ExperimentalDataStreamFeature[]; }>; + /** + * Remove an output from all package policies that are using it, and replace the output by the default ones. + * @param soClient + * @param esClient + * @param outputId + */ + removeOutputFromAll( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + outputId: string + ): Promise; + /** * Returns an `AsyncIterable` for retrieving all integration policy IDs * @param soClient diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts index f6feffd24df53..3088814c8f8a3 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts @@ -6,6 +6,7 @@ */ import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; + import { appContextService } from '..'; import type { PreconfiguredOutput } from '../../../common/types'; @@ -31,6 +32,9 @@ const mockedOutputService = outputService as jest.Mocked; jest.mock('../app_context', () => ({ appContextService: { getInternalUserSOClientWithoutSpaceExtension: jest.fn(), + getExperimentalFeatures: () => ({ + useSpaceAwareness: true, + }), getLogger: () => new Proxy( {}, @@ -60,6 +64,9 @@ describe('output preconfiguration', () => { per_page: 0, total: 0, }); + internalSoClientWithoutSpaceExtension.bulkGet.mockResolvedValue({ + saved_objects: [], + }); mockedOutputService.create.mockReset(); mockedOutputService.update.mockReset(); mockedOutputService.delete.mockReset(); diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index 54db36b49d043..c80cb45c84ff9 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -98,6 +98,7 @@ const PackagePolicyBaseSchema = { namespace: schema.maybe(PackagePolicyNamespaceSchema), policy_id: schema.maybe(schema.string()), policy_ids: schema.maybe(schema.arrayOf(schema.string())), + output_id: schema.nullable(schema.maybe(schema.string())), enabled: schema.boolean(), is_managed: schema.maybe(schema.boolean()), package: schema.maybe( @@ -109,8 +110,6 @@ const PackagePolicyBaseSchema = { requires_root: schema.maybe(schema.boolean()), }) ), - // Deprecated TODO create remove issue - output_id: schema.maybe(schema.string()), inputs: schema.arrayOf(schema.object(PackagePolicyInputsSchema)), vars: schema.maybe(ConfigRecordSchema), overrides: schema.maybe( @@ -152,8 +151,6 @@ const CreatePackagePolicyProps = { requires_root: schema.maybe(schema.boolean()), }) ), - // Deprecated TODO create remove issue - output_id: schema.maybe(schema.string()), inputs: schema.arrayOf( schema.object({ ...PackagePolicyInputsSchema, @@ -191,6 +188,7 @@ export const SimplifiedPackagePolicyBaseSchema = schema.object({ name: schema.string(), description: schema.maybe(schema.string()), namespace: schema.maybe(schema.string()), + output_id: schema.nullable(schema.maybe(schema.string())), vars: schema.maybe(SimplifiedVarsSchema), inputs: schema.maybe( schema.recordOf( diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts index 6216eb4bbd326..79586b1885ed9 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts @@ -162,6 +162,7 @@ export const PreconfiguredAgentPoliciesSchema = schema.arrayOf( }), description: schema.maybe(schema.string()), namespace: schema.maybe(PackagePolicyNamespaceSchema), + output_id: schema.nullable(schema.maybe(schema.string())), inputs: schema.maybe( schema.arrayOf( schema.object({ diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index 79640a0f90e12..ef39759b206f5 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -121,6 +121,8 @@ export interface PackagePolicySOAttributes { inputs: PackagePolicyInput[]; policy_id?: string; policy_ids: string[]; + // Nullable to allow user to reset to default outputs + output_id?: string | null; updated_at: string; updated_by: string; description?: string; diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap b/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap index 9112ad20ad860..a4d255613133e 100644 --- a/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap +++ b/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap @@ -30,6 +30,7 @@ Object { "enabled": true, "name": "system-1", "namespace": "default", + "output_id": null, "package": Object { "name": "system", "requires_root": true, diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index 76585c1f0099a..b860a774ba122 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -120,7 +120,10 @@ export default function (providerContext: FtrProviderContext) { } }; - const createAgentPolicy = async (spaceId?: string): Promise => { + const createAgentPolicy = async ( + spaceId?: string, + dataOutputId?: string + ): Promise => { const { body: testPolicyRes } = await supertest .post(spaceId ? `/s/${spaceId}/api/fleet/agent_policies` : `/api/fleet/agent_policies`) .set('kbn-xsrf', 'xxxx') @@ -128,6 +131,7 @@ export default function (providerContext: FtrProviderContext) { name: `test ${uuidV4()}`, description: '', namespace: 'default', + ...(dataOutputId ? { data_output_id: dataOutputId } : {}), }) .expect(200); @@ -148,6 +152,33 @@ export default function (providerContext: FtrProviderContext) { .expect(200); }; + const createPackagePolicy = async ( + agentPolicyIds: string[], + spaceId?: string, + outputId?: string + ): Promise => { + const { body: testPolicyRes } = await supertest + .post(spaceId ? `/s/${spaceId}/api/fleet/package_policies` : `/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: `package ${uuidV4()}`, + description: '', + namespace: 'default', + policy_ids: agentPolicyIds, + enabled: true, + inputs: [], + package: { + name: 'filetest', + title: 'For File Tests', + version: '0.1.0', + }, + ...(outputId ? { output_id: outputId } : {}), + }) + .expect(200); + + return testPolicyRes; + }; + const getAgentPolicy = async ( policyId: string, spaceId?: string @@ -396,7 +427,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); expect(body.message).to.eql( - 'Logstash output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new Elasticsearch output.' ); }); @@ -417,7 +448,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); expect(body.message).to.eql( - 'Kafka output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new ElasticSearch output.' + 'Kafka output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new Elasticsearch output.' ); }); @@ -765,10 +796,28 @@ export default function (providerContext: FtrProviderContext) { }); it('should bump all policies in all spaces if updating the default output', async () => { - const [policy1, policy2, policy3] = await Promise.all([ - createAgentPolicy(), + const { body: nonDefaultOutput } = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Nondefault Output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + }) + .expect(200); + + const [policy1, policy2, policy3, policy4] = await Promise.all([ createAgentPolicy(), + createAgentPolicy(undefined, nonDefaultOutput.item.id), createAgentPolicy(TEST_SPACE_ID), + createAgentPolicy(TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + // Create package policies with default output under agent policies not using default output + // to ensure that those agent policies still get bumped + await Promise.all([ + createPackagePolicy([policy2.item.id], undefined, defaultOutputId), + createPackagePolicy([policy4.item.id], TEST_SPACE_ID, defaultOutputId), ]); await supertest @@ -781,20 +830,79 @@ export default function (providerContext: FtrProviderContext) { }) .expect(200); - const [updatedPolicy1, updatedPolicy2, updatedPolicy3] = await Promise.all([ + const [updatedPolicy1, updatedPolicy2, updatedPolicy3, updatedPolicy4] = await Promise.all([ getAgentPolicy(policy1.item.id), getAgentPolicy(policy2.item.id), getAgentPolicy(policy3.item.id, TEST_SPACE_ID), + getAgentPolicy(policy4.item.id, TEST_SPACE_ID), ]); expect(updatedPolicy1.item.revision).to.eql(policy1.item.revision + 1); - expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 1); + expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 2); expect(updatedPolicy3.item.revision).to.eql(policy3.item.revision + 1); + expect(updatedPolicy4.item.revision).to.eql(policy4.item.revision + 2); + + // cleanup + await Promise.all([ + deleteAgentPolicy(policy1.item.id), + deleteAgentPolicy(policy2.item.id), + deleteAgentPolicy(policy3.item.id, TEST_SPACE_ID), + deleteAgentPolicy(policy4.item.id, TEST_SPACE_ID), + ]); + }); + + it('should bump all policies in all spaces if updating non-default output', async () => { + const { body: nonDefaultOutput } = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Nondefault Output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + }) + .expect(200); + + const [policy1, policy2, policy3, policy4] = await Promise.all([ + createAgentPolicy(), + createAgentPolicy(undefined, nonDefaultOutput.item.id), + createAgentPolicy(TEST_SPACE_ID), + createAgentPolicy(TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + // Create package policies under agent policies using default output to ensure those + // agent policies still get bumped + await Promise.all([ + createPackagePolicy([policy1.item.id], undefined, nonDefaultOutput.item.id), + createPackagePolicy([policy3.item.id], TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + await supertest + .put(`/api/fleet/outputs/${nonDefaultOutput.item.id}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Nondefault Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + }) + .expect(200); + + const [updatedPolicy1, updatedPolicy2, updatedPolicy3, updatedPolicy4] = await Promise.all([ + getAgentPolicy(policy1.item.id), + getAgentPolicy(policy2.item.id), + getAgentPolicy(policy3.item.id, TEST_SPACE_ID), + getAgentPolicy(policy4.item.id, TEST_SPACE_ID), + ]); + + expect(updatedPolicy1.item.revision).to.eql(policy1.item.revision + 2); + expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 1); + expect(updatedPolicy3.item.revision).to.eql(policy3.item.revision + 2); + expect(updatedPolicy4.item.revision).to.eql(policy4.item.revision + 1); // cleanup await Promise.all([ deleteAgentPolicy(policy1.item.id), deleteAgentPolicy(policy2.item.id), deleteAgentPolicy(policy3.item.id, TEST_SPACE_ID), + deleteAgentPolicy(policy4.item.id, TEST_SPACE_ID), ]); }); });