From f6d1b171b2336e9bb42e85b3c8e4206bf9397d4a Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Mon, 18 Dec 2023 14:06:29 +0100 Subject: [PATCH 1/2] [Fleet] removed restriction to use remote es as integration data (#173353) Closes https://github.com/elastic/kibana/issues/173237 Removed restriction to allow using remote es output as integration data output. - Create a remote es output, verify that the output is allowed to be set as default for agent integrations - Create an agent policy with system integration and set the remote es output as integration data output - Enroll an agent to the agent policy - Check the remote kibana - Discover, verify that system metrics are coming in from the agent - Install system package on the remote cluster to see dashboards, mappings, etc. - Add nginx integration to the agent policy - Create a dummy nginx log file in `/var/tmp/nginx/access.log` and add some dummy data to it - Verify that the data from the nginx log file appears in the remote kibana Discover in `logs-*` data view. - Change the agent policy integration output back to default - Verify that the system integration data is ingested in the main cluster. - Verify that the API key is invalidated in the remote cluster image image image image System dashboard on remote cluster populated: image - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../fleet/common/services/output_helpers.ts | 4 +- .../hooks.test.tsx | 5 ++- .../agent_policy_advanced_fields/hooks.tsx | 42 +++++++++---------- .../components/edit_output_flyout/index.tsx | 1 - .../fleet/server/services/output.test.ts | 16 +++---- .../plugins/fleet/server/services/output.ts | 17 ++------ 6 files changed, 34 insertions(+), 51 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/output_helpers.ts b/x-pack/plugins/fleet/common/services/output_helpers.ts index 26d97f42c39b4..988be3bc277f7 100644 --- a/x-pack/plugins/fleet/common/services/output_helpers.ts +++ b/x-pack/plugins/fleet/common/services/output_helpers.ts @@ -28,9 +28,9 @@ export function getAllowedOutputTypeForPolicy(agentPolicy: AgentPolicy) { agentPolicy.package_policies && agentPolicy.package_policies.some( (p) => - p.package?.name === FLEET_APM_PACKAGE || p.package?.name === FLEET_SERVER_PACKAGE || - p.package?.name === FLEET_SYNTHETICS_PACKAGE + p.package?.name === FLEET_SYNTHETICS_PACKAGE || + p.package?.name === FLEET_APM_PACKAGE ); if (isRestrictedToSameClusterES) { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx index e027317bf621a..8c2983ee9fd83 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.test.tsx @@ -533,7 +533,7 @@ describe('useOutputOptions', () => { `); }); - it('should only enable remote es output for monitoring output', async () => { + it('should enable remote es output for data and monitoring output', async () => { const testRenderer = createFleetTestRendererMock(); mockedUseLicence.mockReturnValue({ hasAtLeast: () => true, @@ -545,7 +545,8 @@ describe('useOutputOptions', () => { expect(result.current.isLoading).toBeTruthy(); await waitForNextUpdate(); - expect(result.current.dataOutputOptions.length).toEqual(1); + expect(result.current.dataOutputOptions.length).toEqual(2); + expect(result.current.dataOutputOptions[1].value).toEqual('remote1'); expect(result.current.monitoringOutputOptions.length).toEqual(2); expect(result.current.monitoringOutputOptions[1].value).toEqual('remote1'); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx index 9ee8b1f225735..15681ea1dfbb6 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/hooks.tsx @@ -16,7 +16,7 @@ import { useGetDownloadSources, useGetFleetServerHosts, } from '../../../../hooks'; -import { LICENCE_FOR_PER_POLICY_OUTPUT, outputType } from '../../../../../../../common/constants'; +import { LICENCE_FOR_PER_POLICY_OUTPUT } from '../../../../../../../common/constants'; import { getAllowedOutputTypeForPolicy, policyHasFleetServer, @@ -99,28 +99,26 @@ export function useOutputOptions(agentPolicy: Partial item.type !== outputType.RemoteElasticsearch) - .map((item) => { - const isOutputTypeUnsupported = !allowedOutputTypes.includes(item.type); + ...outputsRequest.data.items.map((item) => { + const isOutputTypeUnsupported = !allowedOutputTypes.includes(item.type); - return { - value: item.id, - inputDisplay: getOutputLabel( - item.name, - isOutputTypeUnsupported ? ( - - ) : undefined - ), - disabled: !isPolicyPerOutputAllowed || isOutputTypeUnsupported, - }; - }), + return { + value: item.id, + inputDisplay: getOutputLabel( + item.name, + isOutputTypeUnsupported ? ( + + ) : undefined + ), + disabled: !isPolicyPerOutputAllowed || isOutputTypeUnsupported, + }; + }), ]; }, [outputsRequest, isPolicyPerOutputAllowed, allowedOutputTypes]); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx index 511af850cab46..daebac615d6eb 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx @@ -542,7 +542,6 @@ export const EditOutputFlyout: React.FunctionComponent = }} /> } - disabled={isRemoteESOutput} /> diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index c17cf9d3af210..8c2fee196d8d8 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -705,12 +705,12 @@ describe('Output Service', () => { ); }); - it('should throw when a remote es output is attempted to be created as default data output', async () => { + it('should not throw when a remote es output is attempted to be created as default data output', async () => { const soClient = getMockedSoClient({ defaultOutputId: 'output-test', }); - await expect( + expect( outputService.create( soClient, esClientMock, @@ -722,9 +722,7 @@ describe('Output Service', () => { }, { id: 'output-1' } ) - ).rejects.toThrow( - `Remote elasticsearch output cannot be set as default output for integration data. Please set "is_default" to false.` - ); + ).resolves.not.toThrow(); }); it('should set preset: balanced by default when creating a new ES output', async () => { @@ -1644,21 +1642,19 @@ describe('Output Service', () => { ); }); - it('should throw when a remote es output is attempted to be updated as default data output', async () => { + it('should not throw when a remote es output is attempted to be updated as default data output', async () => { const soClient = getMockedSoClient({ defaultOutputId: 'output-test', }); - await expect( + expect( outputService.update(soClient, esClientMock, 'output-test', { is_default: true, is_default_monitoring: false, name: 'Test', type: 'remote_elasticsearch', }) - ).rejects.toThrow( - `Remote elasticsearch output cannot be set as default output for integration data. Please set "is_default" to false.` - ); + ).resolves.not.toThrow(); }); }); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index f2d17b0187274..7d1e3123de776 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -437,13 +437,6 @@ class OutputService { } ): Promise { const data: OutputSOAttributes = { ...omit(output, ['ssl', 'secrets']) }; - if (output.type === outputType.RemoteElasticsearch) { - if (data.is_default) { - throw new OutputInvalidError( - 'Remote elasticsearch output cannot be set as default output for integration data. Please set "is_default" to false.' - ); - } - } if (outputTypeSupportPresets(data.type)) { if ( @@ -757,13 +750,9 @@ class OutputService { fromPreconfiguration: false, } ) { - if (data.type === outputType.RemoteElasticsearch) { - if (data.is_default) { - throw new OutputInvalidError( - 'Remote elasticsearch output cannot be set as default output for integration data. Please set "is_default" to false.' - ); - } - } + + const logger = appContextService.getLogger(); + logger.debug(`Updating output ${id}`); let secretsToDelete: PolicySecretReference[] = []; const originalOutput = await this.get(soClient, id); From 6e1bb2c2ace5b98b175b44f1ab65e9478c04c48a Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 18 Dec 2023 15:09:21 +0100 Subject: [PATCH 2/2] fix merge --- x-pack/plugins/fleet/server/services/output.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index 7d1e3123de776..a478122286d32 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -750,10 +750,6 @@ class OutputService { fromPreconfiguration: false, } ) { - - const logger = appContextService.getLogger(); - logger.debug(`Updating output ${id}`); - let secretsToDelete: PolicySecretReference[] = []; const originalOutput = await this.get(soClient, id);