From a1789521341cfb21ce0414f1b08ff7b0addfe6a0 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 6 Jul 2020 14:12:15 -0700 Subject: [PATCH] [Ingest Manager] Support limiting integrations on an agent config (#70542) * Add API endpoint and hook for retrieving restricted packages * Filter out restricted packages already in use from list of integrations available for an agent config * Allow list agent configs to optionally return expanded package configs, re * Filter out agent configs which already use the restricted package already from list of agent configs available for an integration * Allow more than 20 agent configs to be shown * Rename restricted to limited; add some common methods to DRY * Add limited package check on server side * Adjust copy wording * Fix typings * Add some package config api integration tests, update es archive mappings * Move test to dockerized integation tests directory; move existing epm tests to their own directory * Remove extra assignPackageConfigs() - already handled in packageConfigService.create() * Review fixes * Fix type, reenabled skipped test * Move new EPM integration test file --- .../ingest_manager/common/constants/routes.ts | 1 + .../ingest_manager/common/services/index.ts | 5 +- .../common/services/limited_package.ts | 23 ++++ .../ingest_manager/common/services/routes.ts | 4 + .../ingest_manager/common/types/models/epm.ts | 1 + .../common/types/rest_spec/agent_config.ts | 4 +- .../common/types/rest_spec/common.ts | 3 +- .../common/types/rest_spec/epm.ts | 5 + .../hooks/use_request/agent_config.ts | 4 +- .../ingest_manager/hooks/use_request/epm.ts | 8 ++ .../step_select_config.tsx | 25 +++- .../step_select_package.tsx | 28 +++- .../ingest_manager/services/index.ts | 4 +- .../ingest_manager/types/index.ts | 2 + .../server/routes/agent_config/handlers.ts | 15 +- .../server/routes/epm/handlers.ts | 35 ++++- .../ingest_manager/server/routes/epm/index.ts | 10 ++ .../routes/package_config/handlers.test.ts | 6 +- .../server/routes/package_config/handlers.ts | 24 +--- .../server/services/agent_config.ts | 44 ++++-- .../server/services/epm/packages/get.ts | 23 ++++ .../server/services/epm/packages/index.ts | 4 +- .../server/services/package_config.ts | 39 +++++- .../ingest_manager/server/services/setup.ts | 8 +- .../server/types/rest_spec/agent_config.ts | 4 +- .../es_archives/fleet/agents/mappings.json | 30 ++-- .../apis/{ => epm}/file.ts | 6 +- .../apis/{ => epm}/ilm.ts | 4 +- .../apis/{ => epm}/install.ts | 4 +- .../apis/{ => epm}/list.ts | 6 +- .../apis/{ => epm}/template.ts | 6 +- .../apis/index.js | 17 ++- .../apis/package_config/create.ts | 130 ++++++++++++++++++ 33 files changed, 429 insertions(+), 103 deletions(-) create mode 100644 x-pack/plugins/ingest_manager/common/services/limited_package.ts rename x-pack/test/ingest_manager_api_integration/apis/{ => epm}/file.ts (94%) rename x-pack/test/ingest_manager_api_integration/apis/{ => epm}/ilm.ts (89%) rename x-pack/test/ingest_manager_api_integration/apis/{ => epm}/install.ts (95%) rename x-pack/test/ingest_manager_api_integration/apis/{ => epm}/list.ts (87%) rename x-pack/test/ingest_manager_api_integration/apis/{ => epm}/template.ts (88%) create mode 100644 x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts diff --git a/x-pack/plugins/ingest_manager/common/constants/routes.ts b/x-pack/plugins/ingest_manager/common/constants/routes.ts index dad3cdce1a497..7c3b5a198571c 100644 --- a/x-pack/plugins/ingest_manager/common/constants/routes.ts +++ b/x-pack/plugins/ingest_manager/common/constants/routes.ts @@ -17,6 +17,7 @@ const EPM_PACKAGES_ONE = `${EPM_PACKAGES_MANY}/{pkgkey}`; const EPM_PACKAGES_FILE = `${EPM_PACKAGES_MANY}/{pkgName}/{pkgVersion}`; export const EPM_API_ROUTES = { LIST_PATTERN: EPM_PACKAGES_MANY, + LIMITED_LIST_PATTERN: `${EPM_PACKAGES_MANY}/limited`, INFO_PATTERN: EPM_PACKAGES_ONE, INSTALL_PATTERN: EPM_PACKAGES_ONE, DELETE_PATTERN: EPM_PACKAGES_ONE, diff --git a/x-pack/plugins/ingest_manager/common/services/index.ts b/x-pack/plugins/ingest_manager/common/services/index.ts index a0db7c20747e2..0c91dbbe10354 100644 --- a/x-pack/plugins/ingest_manager/common/services/index.ts +++ b/x-pack/plugins/ingest_manager/common/services/index.ts @@ -3,11 +3,10 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import * as AgentStatusKueryHelper from './agent_status'; - export * from './routes'; +export * as AgentStatusKueryHelper from './agent_status'; export { packageToPackageConfigInputs, packageToPackageConfig } from './package_to_config'; export { storedPackageConfigsToAgentInputs } from './package_configs_to_agent_inputs'; export { configToYaml } from './config_to_yaml'; -export { AgentStatusKueryHelper }; +export { isPackageLimited, doesAgentConfigAlreadyIncludePackage } from './limited_package'; export { decodeCloudId } from './decode_cloud_id'; diff --git a/x-pack/plugins/ingest_manager/common/services/limited_package.ts b/x-pack/plugins/ingest_manager/common/services/limited_package.ts new file mode 100644 index 0000000000000..7ef445d55063c --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/services/limited_package.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { PackageInfo, AgentConfig, PackageConfig } from '../types'; + +// Assume packages only ever include 1 config template for now +export const isPackageLimited = (packageInfo: PackageInfo): boolean => { + return packageInfo.config_templates?.[0]?.multiple === false; +}; + +export const doesAgentConfigAlreadyIncludePackage = ( + agentConfig: AgentConfig, + packageName: string +): boolean => { + if (agentConfig.package_configs.length && typeof agentConfig.package_configs[0] === 'string') { + throw new Error('Unable to read full package config information'); + } + return (agentConfig.package_configs as PackageConfig[]) + .map((packageConfig) => packageConfig.package?.name || '') + .includes(packageName); +}; diff --git a/x-pack/plugins/ingest_manager/common/services/routes.ts b/x-pack/plugins/ingest_manager/common/services/routes.ts index 463a18887174c..49de9a4d8fd85 100644 --- a/x-pack/plugins/ingest_manager/common/services/routes.ts +++ b/x-pack/plugins/ingest_manager/common/services/routes.ts @@ -27,6 +27,10 @@ export const epmRouteService = { return EPM_API_ROUTES.LIST_PATTERN; }, + getListLimitedPath: () => { + return EPM_API_ROUTES.LIMITED_LIST_PATTERN; + }, + getInfoPath: (pkgkey: string) => { return EPM_API_ROUTES.INFO_PATTERN.replace('{pkgkey}', pkgkey); }, diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index 0d2825f0aa80d..23e31227cbf3c 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -79,6 +79,7 @@ export interface RegistryConfigTemplate { title: string; description: string; inputs: RegistryInput[]; + multiple?: boolean; } export interface RegistryInput { diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts index 86020cb5235ae..4e1612d144ede 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts @@ -7,7 +7,9 @@ import { AgentConfig, NewAgentConfig, FullAgentConfig } from '../models'; import { ListWithKuery } from './common'; export interface GetAgentConfigsRequest { - query: ListWithKuery; + query: ListWithKuery & { + full?: boolean; + }; } export type GetAgentConfigsResponseItem = AgentConfig & { agents?: number }; diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts index 0d1f72afa16f1..a454e39c203ed 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts @@ -3,8 +3,9 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { HttpFetchQuery } from 'src/core/public'; -export interface ListWithKuery { +export interface ListWithKuery extends HttpFetchQuery { page?: number; perPage?: number; sortField?: string; diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts index 5ac7fe9e2779b..c5035d2d44432 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts @@ -34,6 +34,11 @@ export interface GetPackagesResponse { success: boolean; } +export interface GetLimitedPackagesResponse { + response: string[]; + success: boolean; +} + export interface GetFileRequest { params: { pkgkey: string; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/agent_config.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/agent_config.ts index c81303de3d7c3..56b78c6faa93a 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/agent_config.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/agent_config.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { HttpFetchQuery } from 'src/core/public'; import { useRequest, sendRequest, @@ -12,6 +11,7 @@ import { } from './use_request'; import { agentConfigRouteService } from '../../services'; import { + GetAgentConfigsRequest, GetAgentConfigsResponse, GetOneAgentConfigResponse, GetFullAgentConfigResponse, @@ -25,7 +25,7 @@ import { DeleteAgentConfigResponse, } from '../../types'; -export const useGetAgentConfigs = (query: HttpFetchQuery = {}) => { +export const useGetAgentConfigs = (query?: GetAgentConfigsRequest['query']) => { return useRequest({ path: agentConfigRouteService.getListPath(), method: 'get', diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts index 128ef8de68aae..64bee1763b08b 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts @@ -10,6 +10,7 @@ import { epmRouteService } from '../../services'; import { GetCategoriesResponse, GetPackagesResponse, + GetLimitedPackagesResponse, GetInfoResponse, InstallPackageResponse, DeletePackageResponse, @@ -30,6 +31,13 @@ export const useGetPackages = (query: HttpFetchQuery = {}) => { }); }; +export const useGetLimitedPackages = () => { + return useRequest({ + path: epmRouteService.getListLimitedPath(), + method: 'get', + }); +}; + export const useGetPackageInfoByKey = (pkgkey: string) => { return useRequest({ path: epmRouteService.getInfoPath(pkgkey), diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx index 849d7bfc63f34..f6391cf1fa456 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx @@ -9,6 +9,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, EuiFlexItem, EuiSelectable, EuiSpacer, EuiTextColor } from '@elastic/eui'; import { Error } from '../../../components'; import { AgentConfig, PackageInfo, GetAgentConfigsResponseItem } from '../../../types'; +import { isPackageLimited, doesAgentConfigAlreadyIncludePackage } from '../../../services'; import { useGetPackageInfoByKey, useGetAgentConfigs, sendGetOneAgentConfig } from '../../../hooks'; export const StepSelectConfig: React.FunctionComponent<{ @@ -24,7 +25,12 @@ export const StepSelectConfig: React.FunctionComponent<{ const [selectedConfigError, setSelectedConfigError] = useState(); // Fetch package info - const { data: packageInfoData, error: packageInfoError } = useGetPackageInfoByKey(pkgkey); + const { + data: packageInfoData, + error: packageInfoError, + isLoading: packageInfoLoading, + } = useGetPackageInfoByKey(pkgkey); + const isLimitedPackage = (packageInfoData && isPackageLimited(packageInfoData.response)) || false; // Fetch agent configs info const { @@ -36,6 +42,7 @@ export const StepSelectConfig: React.FunctionComponent<{ perPage: 1000, sortField: 'name', sortOrder: 'asc', + full: true, }); const agentConfigs = agentConfigsData?.items || []; const agentConfigsById = agentConfigs.reduce( @@ -112,12 +119,18 @@ export const StepSelectConfig: React.FunctionComponent<{ searchable allowExclusions={false} singleSelection={true} - isLoading={isAgentConfigsLoading} - options={agentConfigs.map(({ id, name, description }) => { + isLoading={isAgentConfigsLoading || packageInfoLoading} + options={agentConfigs.map((agentConf) => { + const alreadyHasLimitedPackage = + (isLimitedPackage && + packageInfoData && + doesAgentConfigAlreadyIncludePackage(agentConf, packageInfoData.response.name)) || + false; return { - label: name, - key: id, - checked: selectedConfigId === id ? 'on' : undefined, + label: agentConf.name, + key: agentConf.id, + checked: selectedConfigId === agentConf.id ? 'on' : undefined, + disabled: alreadyHasLimitedPackage, 'data-test-subj': 'agentConfigItem', }; })} diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx index e4f4c976688b1..204b862bd4dc4 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx @@ -8,8 +8,13 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, EuiFlexItem, EuiSelectable, EuiSpacer } from '@elastic/eui'; import { Error } from '../../../components'; -import { AgentConfig, PackageInfo } from '../../../types'; -import { useGetOneAgentConfig, useGetPackages, sendGetPackageInfoByKey } from '../../../hooks'; +import { AgentConfig, PackageInfo, PackageConfig, GetPackagesResponse } from '../../../types'; +import { + useGetOneAgentConfig, + useGetPackages, + useGetLimitedPackages, + sendGetPackageInfoByKey, +} from '../../../hooks'; import { PackageIcon } from '../../../components/package_icon'; export const StepSelectPackage: React.FunctionComponent<{ @@ -28,12 +33,27 @@ export const StepSelectPackage: React.FunctionComponent<{ const { data: agentConfigData, error: agentConfigError } = useGetOneAgentConfig(agentConfigId); // Fetch packages info + // Filter out limited packages already part of selected agent config + const [packages, setPackages] = useState([]); const { data: packagesData, error: packagesError, isLoading: isPackagesLoading, } = useGetPackages(); - const packages = packagesData?.response || []; + const { + data: limitedPackagesData, + isLoading: isLimitedPackagesLoading, + } = useGetLimitedPackages(); + useEffect(() => { + if (packagesData?.response && limitedPackagesData?.response && agentConfigData?.item) { + const allPackages = packagesData.response; + const limitedPackages = limitedPackagesData.response; + const usedLimitedPackages = (agentConfigData.item.package_configs as PackageConfig[]) + .map((packageConfig) => packageConfig.package?.name || '') + .filter((pkgName) => limitedPackages.includes(pkgName)); + setPackages(allPackages.filter((pkg) => !usedLimitedPackages.includes(pkg.name))); + } + }, [packagesData, limitedPackagesData, agentConfigData]); // Update parent agent config state useEffect(() => { @@ -101,7 +121,7 @@ export const StepSelectPackage: React.FunctionComponent<{ searchable allowExclusions={false} singleSelection={true} - isLoading={isPackagesLoading} + isLoading={isPackagesLoading || isLimitedPackagesLoading} options={packages.map(({ title, name, version, icons }) => { const pkgkey = `${name}-${version}`; return { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts index 5dc9026aebdee..9c3b84d0835b8 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts @@ -7,6 +7,7 @@ export { getFlattenedObject } from '../../../../../../../src/core/public'; export { + AgentStatusKueryHelper, agentConfigRouteService, packageConfigRouteService, dataStreamRouteService, @@ -21,5 +22,6 @@ export { packageToPackageConfigInputs, storedPackageConfigsToAgentInputs, configToYaml, - AgentStatusKueryHelper, + isPackageLimited, + doesAgentConfigAlreadyIncludePackage, } from '../../../../common'; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts index e28d76cae9955..9cd8a75642296 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts @@ -24,6 +24,7 @@ export { // API schema - misc setup, status GetFleetStatusResponse, // API schemas - Agent Config + GetAgentConfigsRequest, GetAgentConfigsResponse, GetAgentConfigsResponseItem, GetOneAgentConfigResponse, @@ -92,6 +93,7 @@ export { ServiceName, GetCategoriesResponse, GetPackagesResponse, + GetLimitedPackagesResponse, GetInfoResponse, InstallPackageResponse, DeletePackageResponse, diff --git a/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts index 7b12a076ff041..110f6b9950829 100644 --- a/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts @@ -38,8 +38,12 @@ export const getAgentConfigsHandler: RequestHandler< TypeOf > = async (context, request, response) => { const soClient = context.core.savedObjects.client; + const { full: withPackageConfigs = false, ...restOfQuery } = request.query; try { - const { items, total, page, perPage } = await agentConfigService.list(soClient, request.query); + const { items, total, page, perPage } = await agentConfigService.list(soClient, { + withPackageConfigs, + ...restOfQuery, + }); const body: GetAgentConfigsResponse = { items, total, @@ -103,6 +107,7 @@ export const createAgentConfigHandler: RequestHandler< TypeOf > = async (context, request, response) => { const soClient = context.core.savedObjects.client; + const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined; const withSysMonitoring = request.query.sys_monitoring ?? false; try { @@ -128,15 +133,9 @@ export const createAgentConfigHandler: RequestHandler< if (withSysMonitoring && newSysPackageConfig !== undefined && agentConfig !== undefined) { newSysPackageConfig.config_id = agentConfig.id; newSysPackageConfig.namespace = agentConfig.namespace; - const sysPackageConfig = await packageConfigService.create(soClient, newSysPackageConfig, { + await packageConfigService.create(soClient, callCluster, newSysPackageConfig, { user, }); - - if (sysPackageConfig) { - agentConfig = await agentConfigService.assignPackageConfigs(soClient, agentConfig.id, [ - sysPackageConfig.id, - ]); - } } const body: CreateAgentConfigResponse = { diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index eaf0e1a104b3e..a50b3b13faeab 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -5,20 +5,21 @@ */ import { TypeOf } from '@kbn/config-schema'; import { RequestHandler, CustomHttpResponseOptions } from 'src/core/server'; -import { - GetPackagesRequestSchema, - GetFileRequestSchema, - GetInfoRequestSchema, - InstallPackageRequestSchema, - DeletePackageRequestSchema, -} from '../../types'; import { GetInfoResponse, InstallPackageResponse, DeletePackageResponse, GetCategoriesResponse, GetPackagesResponse, + GetLimitedPackagesResponse, } from '../../../common'; +import { + GetPackagesRequestSchema, + GetFileRequestSchema, + GetInfoRequestSchema, + InstallPackageRequestSchema, + DeletePackageRequestSchema, +} from '../../types'; import { getCategories, getPackages, @@ -26,6 +27,7 @@ import { getPackageInfo, installPackage, removeInstallation, + getLimitedPackages, } from '../../services/epm/packages'; export const getCategoriesHandler: RequestHandler = async (context, request, response) => { @@ -69,6 +71,25 @@ export const getListHandler: RequestHandler< } }; +export const getLimitedListHandler: RequestHandler = async (context, request, response) => { + try { + const savedObjectsClient = context.core.savedObjects.client; + const res = await getLimitedPackages({ savedObjectsClient }); + const body: GetLimitedPackagesResponse = { + response: res, + success: true, + }; + return response.ok({ + body, + }); + } catch (e) { + return response.customError({ + statusCode: 500, + body: { message: e.message }, + }); + } +}; + export const getFileHandler: RequestHandler> = async ( context, request, diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/index.ts b/x-pack/plugins/ingest_manager/server/routes/epm/index.ts index fcf81f9894d5e..ffaf0ce46c89a 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/index.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/index.ts @@ -8,6 +8,7 @@ import { PLUGIN_ID, EPM_API_ROUTES } from '../../constants'; import { getCategoriesHandler, getListHandler, + getLimitedListHandler, getFileHandler, getInfoHandler, installPackageHandler, @@ -40,6 +41,15 @@ export const registerRoutes = (router: IRouter) => { getListHandler ); + router.get( + { + path: EPM_API_ROUTES.LIMITED_LIST_PATTERN, + validate: false, + options: { tags: [`access:${PLUGIN_ID}`] }, + }, + getLimitedListHandler + ); + router.get( { path: EPM_API_ROUTES.FILEPATH_PATTERN, diff --git a/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.test.ts b/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.test.ts index 6d712ce063290..85ecc5027d64d 100644 --- a/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.test.ts +++ b/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.test.ts @@ -25,7 +25,7 @@ jest.mock('../../services/package_config', (): { assignPackageStream: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)), buildPackageConfigFromPackage: jest.fn(), bulkCreate: jest.fn(), - create: jest.fn((soClient, newData) => + create: jest.fn((soClient, callCluster, newData) => Promise.resolve({ ...newData, id: '1', @@ -213,7 +213,7 @@ describe('When calling package config', () => { const request = getCreateKibanaRequest(); await routeHandler(context, request, response); expect(response.ok).toHaveBeenCalled(); - expect(packageConfigServiceMock.create.mock.calls[0][1]).toEqual({ + expect(packageConfigServiceMock.create.mock.calls[0][2]).toEqual({ config_id: 'a5ca00c0-b30c-11ea-9732-1bb05811278c', description: '', enabled: true, @@ -294,7 +294,7 @@ describe('When calling package config', () => { const request = getCreateKibanaRequest(); await routeHandler(context, request, response); expect(response.ok).toHaveBeenCalled(); - expect(packageConfigServiceMock.create.mock.calls[0][1]).toEqual({ + expect(packageConfigServiceMock.create.mock.calls[0][2]).toEqual({ config_id: 'a5ca00c0-b30c-11ea-9732-1bb05811278c', description: '', enabled: true, diff --git a/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts index f11275c92bb68..6b0c2fe9c2ff7 100644 --- a/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts @@ -7,7 +7,7 @@ import { TypeOf } from '@kbn/config-schema'; import Boom from 'boom'; import { RequestHandler } from 'src/core/server'; import { appContextService, packageConfigService } from '../../services'; -import { ensureInstalledPackage, getPackageInfo } from '../../services/epm/packages'; +import { getPackageInfo } from '../../services/epm/packages'; import { GetPackageConfigsRequestSchema, GetOnePackageConfigRequestSchema, @@ -106,26 +106,10 @@ export const createPackageConfigHandler: RequestHandler< newData = updatedNewData; } - // Make sure the associated package is installed - if (newData.package?.name) { - await ensureInstalledPackage({ - savedObjectsClient: soClient, - pkgName: newData.package.name, - callCluster, - }); - const pkgInfo = await getPackageInfo({ - savedObjectsClient: soClient, - pkgName: newData.package.name, - pkgVersion: newData.package.version, - }); - newData.inputs = (await packageConfigService.assignPackageStream( - pkgInfo, - newData.inputs - )) as TypeOf['inputs']; - } - // Create package config - const packageConfig = await packageConfigService.create(soClient, newData, { user }); + const packageConfig = await packageConfigService.create(soClient, callCluster, newData, { + user, + }); const body: CreatePackageConfigResponse = { item: packageConfig, success: true }; return response.ok({ body, diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.ts index bd00727714c33..fe247d5b91db0 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.ts @@ -141,11 +141,20 @@ class AgentConfigService { public async list( soClient: SavedObjectsClientContract, - options: ListWithKuery + options: ListWithKuery & { + withPackageConfigs?: boolean; + } ): Promise<{ items: AgentConfig[]; total: number; page: number; perPage: number }> { - const { page = 1, perPage = 20, sortField = 'updated_at', sortOrder = 'desc', kuery } = options; - - const agentConfigs = await soClient.find({ + const { + page = 1, + perPage = 20, + sortField = 'updated_at', + sortOrder = 'desc', + kuery, + withPackageConfigs = false, + } = options; + + const agentConfigsSO = await soClient.find({ type: SAVED_OBJECT_TYPE, sortField, sortOrder, @@ -160,12 +169,29 @@ class AgentConfigService { : undefined, }); + const agentConfigs = await Promise.all( + agentConfigsSO.saved_objects.map(async (agentConfigSO) => { + const agentConfig = { + id: agentConfigSO.id, + ...agentConfigSO.attributes, + }; + if (withPackageConfigs) { + const agentConfigWithPackageConfigs = await this.get( + soClient, + agentConfigSO.id, + withPackageConfigs + ); + if (agentConfigWithPackageConfigs) { + agentConfig.package_configs = agentConfigWithPackageConfigs.package_configs; + } + } + return agentConfig; + }) + ); + return { - items: agentConfigs.saved_objects.map((agentConfigSO) => ({ - id: agentConfigSO.id, - ...agentConfigSO.attributes, - })), - total: agentConfigs.total, + items: agentConfigs, + total: agentConfigsSO.total, page, perPage, }; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts index a261eec899d7c..ad9635cc02e06 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts @@ -5,6 +5,7 @@ */ import { SavedObjectsClientContract } from 'src/core/server'; +import { isPackageLimited } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { Installation, InstallationStatus, PackageInfo, KibanaAssetType } from '../../../types'; import * as Registry from '../registry'; @@ -49,6 +50,28 @@ export async function getPackages( return packageList; } +// Get package names for packages which cannot have more than one package config on an agent config +// Assume packages only export one config template for now +export async function getLimitedPackages(options: { + savedObjectsClient: SavedObjectsClientContract; +}): Promise { + const { savedObjectsClient } = options; + const allPackages = await getPackages({ savedObjectsClient }); + const installedPackages = allPackages.filter( + (pkg) => (pkg.status = InstallationStatus.installed) + ); + const installedPackagesInfo = await Promise.all( + installedPackages.map((pkgInstall) => { + return getPackageInfo({ + savedObjectsClient, + pkgName: pkgInstall.name, + pkgVersion: pkgInstall.version, + }); + }) + ); + return installedPackagesInfo.filter((pkgInfo) => isPackageLimited).map((pkgInfo) => pkgInfo.name); +} + export async function getPackageSavedObjects(savedObjectsClient: SavedObjectsClientContract) { return savedObjectsClient.find({ type: PACKAGES_SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts index b79f9178ad6af..4bb803dfaf912 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts @@ -11,8 +11,7 @@ import { Installation, InstallationStatus, KibanaAssetType, -} from '../../../../common/types/models/epm'; - +} from '../../../types'; export { getCategories, getFile, @@ -20,6 +19,7 @@ export { getInstallation, getPackageInfo, getPackages, + getLimitedPackages, SearchParams, } from './get'; diff --git a/x-pack/plugins/ingest_manager/server/services/package_config.ts b/x-pack/plugins/ingest_manager/server/services/package_config.ts index 9fa51d025ad2b..9433a81e74b07 100644 --- a/x-pack/plugins/ingest_manager/server/services/package_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/package_config.ts @@ -7,24 +7,27 @@ import { SavedObjectsClientContract } from 'src/core/server'; import { AuthenticatedUser } from '../../../security/server'; import { DeletePackageConfigsResponse, - packageToPackageConfig, PackageConfigInput, PackageConfigInputStream, PackageInfo, + ListWithKuery, + packageToPackageConfig, + isPackageLimited, + doesAgentConfigAlreadyIncludePackage, } from '../../common'; import { PACKAGE_CONFIG_SAVED_OBJECT_TYPE } from '../constants'; import { NewPackageConfig, UpdatePackageConfig, PackageConfig, - ListWithKuery, PackageConfigSOAttributes, RegistryPackage, + CallESAsCurrentUser, } from '../types'; import { agentConfigService } from './agent_config'; import { outputService } from './output'; import * as Registry from './epm/registry'; -import { getPackageInfo, getInstallation } from './epm/packages'; +import { getPackageInfo, getInstallation, ensureInstalledPackage } from './epm/packages'; import { getAssetsData } from './epm/packages/assets'; import { createStream } from './epm/agent/agent'; @@ -37,9 +40,39 @@ function getDataset(st: string) { class PackageConfigService { public async create( soClient: SavedObjectsClientContract, + callCluster: CallESAsCurrentUser, packageConfig: NewPackageConfig, options?: { id?: string; user?: AuthenticatedUser } ): Promise { + // Make sure the associated package is installed + if (packageConfig.package?.name) { + const [, pkgInfo] = await Promise.all([ + ensureInstalledPackage({ + savedObjectsClient: soClient, + pkgName: packageConfig.package.name, + callCluster, + }), + getPackageInfo({ + savedObjectsClient: soClient, + pkgName: packageConfig.package.name, + pkgVersion: packageConfig.package.version, + }), + ]); + + // Check if it is a limited package, and if so, check that the corresponding agent config does not + // already contain a package config for this package + if (isPackageLimited(pkgInfo)) { + const agentConfig = await agentConfigService.get(soClient, packageConfig.config_id, true); + if (agentConfig && doesAgentConfigAlreadyIncludePackage(agentConfig, pkgInfo.name)) { + throw new Error( + `Unable to create package config. Package '${pkgInfo.name}' already exists on this agent config.` + ); + } + } + + packageConfig.inputs = await this.assignPackageStream(pkgInfo, packageConfig.inputs); + } + const isoDate = new Date().toISOString(); const newSo = await soClient.create( SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/ingest_manager/server/services/setup.ts b/x-pack/plugins/ingest_manager/server/services/setup.ts index 61e1d0ad94db8..e5ed5c589389c 100644 --- a/x-pack/plugins/ingest_manager/server/services/setup.ts +++ b/x-pack/plugins/ingest_manager/server/services/setup.ts @@ -113,6 +113,7 @@ export async function setupIngestManager( if (!isInstalled) { await addPackageToConfig( soClient, + callCluster, installedPackage, configWithPackageConfigs, defaultOutput @@ -192,6 +193,7 @@ function generateRandomPassword() { async function addPackageToConfig( soClient: SavedObjectsClientContract, + callCluster: CallESAsCurrentUser, packageToInstall: Installation, config: AgentConfig, defaultOutput: Output @@ -208,10 +210,6 @@ async function addPackageToConfig( defaultOutput.id, config.namespace ); - newPackageConfig.inputs = await packageConfigService.assignPackageStream( - packageInfo, - newPackageConfig.inputs - ); - await packageConfigService.create(soClient, newPackageConfig); + await packageConfigService.create(soClient, callCluster, newPackageConfig); } diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/agent_config.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/agent_config.ts index 306aefb0d51ff..d076a803f4b53 100644 --- a/x-pack/plugins/ingest_manager/server/types/rest_spec/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/types/rest_spec/agent_config.ts @@ -8,7 +8,9 @@ import { NewAgentConfigSchema } from '../models'; import { ListWithKuerySchema } from './index'; export const GetAgentConfigsRequestSchema = { - query: ListWithKuerySchema, + query: ListWithKuerySchema.extends({ + full: schema.maybe(schema.boolean()), + }), }; export const GetOneAgentConfigRequestSchema = { diff --git a/x-pack/test/functional/es_archives/fleet/agents/mappings.json b/x-pack/test/functional/es_archives/fleet/agents/mappings.json index 0b84514de23f2..1f0aa2f24d6df 100644 --- a/x-pack/test/functional/es_archives/fleet/agents/mappings.json +++ b/x-pack/test/functional/es_archives/fleet/agents/mappings.json @@ -1839,6 +1839,12 @@ "config_id": { "type": "keyword" }, + "created_at": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, "description": { "type": "text" }, @@ -1847,6 +1853,7 @@ }, "inputs": { "type": "nested", + "enabled": false, "properties": { "config": { "type": "flattened" @@ -1854,20 +1861,24 @@ "enabled": { "type": "boolean" }, - "processors": { - "type": "keyword" - }, "streams": { "type": "nested", "properties": { - "agent_stream": { + "compiled_stream": { "type": "flattened" }, "config": { "type": "flattened" }, "dataset": { - "type": "keyword" + "properties": { + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } }, "enabled": { "type": "boolean" @@ -1875,9 +1886,6 @@ "id": { "type": "keyword" }, - "processors": { - "type": "keyword" - }, "vars": { "type": "flattened" } @@ -1915,6 +1923,12 @@ }, "revision": { "type": "integer" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "type": "keyword" } } }, diff --git a/x-pack/test/ingest_manager_api_integration/apis/file.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/file.ts similarity index 94% rename from x-pack/test/ingest_manager_api_integration/apis/file.ts rename to x-pack/test/ingest_manager_api_integration/apis/epm/file.ts index a7462ac51ecc1..733b8d4fd9bd6 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/file.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/file.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; -import { warnAndSkipTest } from '../helpers'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../../helpers'; export default function ({ getService }: FtrProviderContext) { const log = getService('log'); @@ -13,7 +13,7 @@ export default function ({ getService }: FtrProviderContext) { const dockerServers = getService('dockerServers'); const server = dockerServers.get('registry'); - describe('package file', () => { + describe('EPM - package file', () => { it('fetches a .png screenshot image', async function () { if (server.enabled) { await supertest diff --git a/x-pack/test/ingest_manager_api_integration/apis/ilm.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/ilm.ts similarity index 89% rename from x-pack/test/ingest_manager_api_integration/apis/ilm.ts rename to x-pack/test/ingest_manager_api_integration/apis/epm/ilm.ts index b73a9da5fad59..8a801d59eb5b2 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/ilm.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/ilm.ts @@ -5,10 +5,10 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { - describe('ilm', () => { + describe('EPM - ilm', () => { it('setup policy', async () => { const policyName = 'foo'; const es = getService('es'); diff --git a/x-pack/test/ingest_manager_api_integration/apis/install.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/install.ts similarity index 95% rename from x-pack/test/ingest_manager_api_integration/apis/install.ts rename to x-pack/test/ingest_manager_api_integration/apis/epm/install.ts index 92078c25419df..f73ba56c172c4 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/install.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/install.ts @@ -5,8 +5,8 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; -import { warnAndSkipTest } from '../helpers'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../../helpers'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); diff --git a/x-pack/test/ingest_manager_api_integration/apis/list.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/list.ts similarity index 87% rename from x-pack/test/ingest_manager_api_integration/apis/list.ts rename to x-pack/test/ingest_manager_api_integration/apis/epm/list.ts index abed9a7b85959..1ac1474e03700 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/list.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/list.ts @@ -5,8 +5,8 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; -import { warnAndSkipTest } from '../helpers'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../../helpers'; export default function ({ getService }: FtrProviderContext) { const log = getService('log'); @@ -18,7 +18,7 @@ export default function ({ getService }: FtrProviderContext) { // because `this` has to point to the Mocha context // see https://mochajs.org/#arrow-functions - describe('list', async function () { + describe('EPM - list', async function () { it('lists all packages from the registry', async function () { if (server.enabled) { const fetchPackageList = async () => { diff --git a/x-pack/test/ingest_manager_api_integration/apis/template.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/template.ts similarity index 88% rename from x-pack/test/ingest_manager_api_integration/apis/template.ts rename to x-pack/test/ingest_manager_api_integration/apis/epm/template.ts index f7e5a894b83ff..c92dac3334de3 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/template.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/template.ts @@ -5,8 +5,8 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; -import { getTemplate } from '../../../plugins/ingest_manager/server/services/epm/elasticsearch/template/template'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { getTemplate } from '../../../../plugins/ingest_manager/server/services/epm/elasticsearch/template/template'; export default function ({ getService }: FtrProviderContext) { const indexPattern = 'foo'; @@ -20,7 +20,7 @@ export default function ({ getService }: FtrProviderContext) { }, }; // This test was inspired by https://github.com/elastic/kibana/blob/master/x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js - describe('template', async () => { + describe('EPM - template', async () => { it('can be loaded', async () => { const template = getTemplate({ type: 'logs', diff --git a/x-pack/test/ingest_manager_api_integration/apis/index.js b/x-pack/test/ingest_manager_api_integration/apis/index.js index 3f8df8379e743..30c49140c6e2a 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/index.js +++ b/x-pack/test/ingest_manager_api_integration/apis/index.js @@ -5,12 +5,17 @@ */ export default function ({ loadTestFile }) { - describe('EPM Endpoints', function () { + describe('Ingest Manager Endpoints', function () { this.tags('ciGroup7'); - loadTestFile(require.resolve('./list')); - loadTestFile(require.resolve('./file')); - //loadTestFile(require.resolve('./template')); - loadTestFile(require.resolve('./ilm')); - loadTestFile(require.resolve('./install')); + + // EPM + loadTestFile(require.resolve('./epm/list')); + loadTestFile(require.resolve('./epm/file')); + //loadTestFile(require.resolve('./epm/template')); + loadTestFile(require.resolve('./epm/ilm')); + loadTestFile(require.resolve('./epm/install')); + + // Package configs + loadTestFile(require.resolve('./package_config/create')); }); } diff --git a/x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts b/x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts new file mode 100644 index 0000000000000..c7748ab255f43 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../../helpers'; + +export default function ({ getService }: FtrProviderContext) { + const log = getService('log'); + const supertest = getService('supertest'); + const dockerServers = getService('dockerServers'); + + const server = dockerServers.get('registry'); + // use function () {} and not () => {} here + // because `this` has to point to the Mocha context + // see https://mochajs.org/#arrow-functions + + describe('Package Config - create', async function () { + let agentConfigId: string; + + before(async function () { + const { body: agentConfigResponse } = await supertest + .post(`/api/ingest_manager/agent_configs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test config', + namespace: 'default', + }); + agentConfigId = agentConfigResponse.item.id; + }); + + it('should work with valid values', async function () { + if (server.enabled) { + const { body: apiResponse } = await supertest + .post(`/api/ingest_manager/package_configs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'filetest-1', + description: '', + namespace: 'default', + config_id: agentConfigId, + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'filetest', + title: 'For File Tests', + version: '0.1.0', + }, + }) + .expect(200); + + expect(apiResponse.success).to.be(true); + } else { + warnAndSkipTest(this, log); + } + }); + + it('should return a 400 with an invalid namespace', async function () { + if (server.enabled) { + await supertest + .post(`/api/ingest_manager/package_configs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'filetest-1', + description: '', + namespace: '', + config_id: agentConfigId, + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'filetest', + title: 'For File Tests', + version: '0.1.0', + }, + }) + .expect(400); + } else { + warnAndSkipTest(this, log); + } + }); + + it('should not allow multiple limited packages on the same agent config', async function () { + if (server.enabled) { + await supertest + .post(`/api/ingest_manager/package_configs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'endpoint-1', + description: '', + namespace: 'default', + config_id: agentConfigId, + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'endpoint', + title: 'Endpoint', + version: '0.8.0', + }, + }) + .expect(200); + await supertest + .post(`/api/ingest_manager/package_configs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'endpoint-2', + description: '', + namespace: 'default', + config_id: agentConfigId, + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'endpoint', + title: 'Endpoint', + version: '0.8.0', + }, + }) + .expect(500); + } else { + warnAndSkipTest(this, log); + } + }); + }); +}