From 8439e2a346c1adf39402bc305e408e88b09c7964 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 30 Jun 2020 18:20:08 -0700 Subject: [PATCH 01/15] Add API endpoint and hook for retrieving restricted packages --- .../ingest_manager/common/constants/routes.ts | 1 + .../ingest_manager/common/services/routes.ts | 4 +++ .../ingest_manager/common/types/models/epm.ts | 1 + .../common/types/rest_spec/epm.ts | 5 ++++ .../ingest_manager/hooks/use_request/epm.ts | 8 ++++++ .../server/routes/epm/handlers.ts | 25 ++++++++++++++++--- .../ingest_manager/server/routes/epm/index.ts | 10 ++++++++ .../server/services/epm/packages/get.ts | 24 ++++++++++++++++++ .../server/services/epm/packages/index.ts | 4 +-- 9 files changed, 77 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/constants/routes.ts b/x-pack/plugins/ingest_manager/common/constants/routes.ts index dad3cdce1a497..e8d06f3aa5930 100644 --- a/x-pack/plugins/ingest_manager/common/constants/routes.ts +++ b/x-pack/plugins/ingest_manager/common/constants/routes.ts @@ -17,6 +17,7 @@ const EPM_PACKAGES_ONE = `${EPM_PACKAGES_MANY}/{pkgkey}`; const EPM_PACKAGES_FILE = `${EPM_PACKAGES_MANY}/{pkgName}/{pkgVersion}`; export const EPM_API_ROUTES = { LIST_PATTERN: EPM_PACKAGES_MANY, + RESTRICTED_LIST_PATTERN: `${EPM_PACKAGES_MANY}/restricted`, INFO_PATTERN: EPM_PACKAGES_ONE, INSTALL_PATTERN: EPM_PACKAGES_ONE, DELETE_PATTERN: EPM_PACKAGES_ONE, diff --git a/x-pack/plugins/ingest_manager/common/services/routes.ts b/x-pack/plugins/ingest_manager/common/services/routes.ts index 463a18887174c..09077a4d9ad80 100644 --- a/x-pack/plugins/ingest_manager/common/services/routes.ts +++ b/x-pack/plugins/ingest_manager/common/services/routes.ts @@ -27,6 +27,10 @@ export const epmRouteService = { return EPM_API_ROUTES.LIST_PATTERN; }, + getListRestrictedPath: () => { + return EPM_API_ROUTES.RESTRICTED_LIST_PATTERN; + }, + getInfoPath: (pkgkey: string) => { return EPM_API_ROUTES.INFO_PATTERN.replace('{pkgkey}', pkgkey); }, diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index 5b68cd2beeed4..53b08fba8a318 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -79,6 +79,7 @@ export interface RegistryConfigTemplate { title: string; description: string; inputs: RegistryInput[]; + multiple?: boolean; } export interface RegistryInput { diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts index 5ac7fe9e2779b..7d66bd3fa9872 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts @@ -34,6 +34,11 @@ export interface GetPackagesResponse { success: boolean; } +export interface GetRestrictedPackagesResponse { + response: string[]; + success: boolean; +} + export interface GetFileRequest { params: { pkgkey: string; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts index 128ef8de68aae..0f92f25464115 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts @@ -10,6 +10,7 @@ import { epmRouteService } from '../../services'; import { GetCategoriesResponse, GetPackagesResponse, + GetRestrictedPackagesResponse, GetInfoResponse, InstallPackageResponse, DeletePackageResponse, @@ -30,6 +31,13 @@ export const useGetPackages = (query: HttpFetchQuery = {}) => { }); }; +export const useGetRestrictedPackages = () => { + return useRequest({ + path: epmRouteService.getListRestrictedPath(), + method: 'get', + }); +}; + export const useGetPackageInfoByKey = (pkgkey: string) => { return useRequest({ path: epmRouteService.getInfoPath(pkgkey), diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index eaf0e1a104b3e..09a31d401fbc8 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -11,14 +11,13 @@ import { GetInfoRequestSchema, InstallPackageRequestSchema, DeletePackageRequestSchema, -} from '../../types'; -import { GetInfoResponse, InstallPackageResponse, DeletePackageResponse, GetCategoriesResponse, GetPackagesResponse, -} from '../../../common'; + GetRestrictedPackagesResponse, +} from '../../types'; import { getCategories, getPackages, @@ -26,6 +25,7 @@ import { getPackageInfo, installPackage, removeInstallation, + getRestrictedPackages, } from '../../services/epm/packages'; export const getCategoriesHandler: RequestHandler = async (context, request, response) => { @@ -69,6 +69,25 @@ export const getListHandler: RequestHandler< } }; +export const getRestrictedListHandler: RequestHandler = async (context, request, response) => { + try { + const savedObjectsClient = context.core.savedObjects.client; + const res = await getRestrictedPackages({ savedObjectsClient }); + const body: GetRestrictedPackagesResponse = { + response: res, + success: true, + }; + return response.ok({ + body, + }); + } catch (e) { + return response.customError({ + statusCode: 500, + body: { message: e.message }, + }); + } +}; + export const getFileHandler: RequestHandler> = async ( context, request, diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/index.ts b/x-pack/plugins/ingest_manager/server/routes/epm/index.ts index fcf81f9894d5e..c03b2d8af7025 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/index.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/index.ts @@ -8,6 +8,7 @@ import { PLUGIN_ID, EPM_API_ROUTES } from '../../constants'; import { getCategoriesHandler, getListHandler, + getRestrictedListHandler, getFileHandler, getInfoHandler, installPackageHandler, @@ -40,6 +41,15 @@ export const registerRoutes = (router: IRouter) => { getListHandler ); + router.get( + { + path: EPM_API_ROUTES.RESTRICTED_LIST_PATTERN, + validate: false, + options: { tags: [`access:${PLUGIN_ID}`] }, + }, + getRestrictedListHandler + ); + router.get( { path: EPM_API_ROUTES.FILEPATH_PATTERN, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts index a261eec899d7c..f3426b6d25a84 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts @@ -49,6 +49,30 @@ export async function getPackages( return packageList; } +// Get package names for packages which cannot have more than one package config on an agent config +// Assume packages only export one config template for now +export async function getRestrictedPackages(options: { + savedObjectsClient: SavedObjectsClientContract; +}): Promise { + const { savedObjectsClient } = options; + const allPackages = await getPackages({ savedObjectsClient }); + const installedPackages = allPackages.filter( + (pkg) => (pkg.status = InstallationStatus.installed) + ); + const installedPackagesInfo = await Promise.all( + installedPackages.map(async (pkgInstall) => { + return getPackageInfo({ + savedObjectsClient, + pkgName: pkgInstall.name, + pkgVersion: pkgInstall.version, + }); + }) + ); + return installedPackagesInfo + .filter((pkgInfo) => pkgInfo.config_templates?.[0]?.multiple === false) + .map((pkgInfo) => pkgInfo.name); +} + export async function getPackageSavedObjects(savedObjectsClient: SavedObjectsClientContract) { return savedObjectsClient.find({ type: PACKAGES_SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts index b79f9178ad6af..8ef345864d0d6 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts @@ -11,8 +11,7 @@ import { Installation, InstallationStatus, KibanaAssetType, -} from '../../../../common/types/models/epm'; - +} from '../../../types'; export { getCategories, getFile, @@ -20,6 +19,7 @@ export { getInstallation, getPackageInfo, getPackages, + getRestrictedPackages, SearchParams, } from './get'; From 0a2009bc3e204618bcdb7f3e254bf56af61c8939 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 1 Jul 2020 13:14:37 -0700 Subject: [PATCH 02/15] Filter out restricted packages already in use from list of integrations available for an agent config --- .../step_select_package.tsx | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx index e4f4c976688b1..fbb8fe6bef875 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx @@ -8,8 +8,13 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, EuiFlexItem, EuiSelectable, EuiSpacer } from '@elastic/eui'; import { Error } from '../../../components'; -import { AgentConfig, PackageInfo } from '../../../types'; -import { useGetOneAgentConfig, useGetPackages, sendGetPackageInfoByKey } from '../../../hooks'; +import { AgentConfig, PackageInfo, PackageConfig, GetPackagesResponse } from '../../../types'; +import { + useGetOneAgentConfig, + useGetPackages, + useGetRestrictedPackages, + sendGetPackageInfoByKey, +} from '../../../hooks'; import { PackageIcon } from '../../../components/package_icon'; export const StepSelectPackage: React.FunctionComponent<{ @@ -28,12 +33,27 @@ export const StepSelectPackage: React.FunctionComponent<{ const { data: agentConfigData, error: agentConfigError } = useGetOneAgentConfig(agentConfigId); // Fetch packages info + // Filter out restricted packages already part of selected agent config + const [packages, setPackages] = useState([]); const { data: packagesData, error: packagesError, isLoading: isPackagesLoading, } = useGetPackages(); - const packages = packagesData?.response || []; + const { + data: restrictedPackagesData, + isLoading: isRestrictedPackagesLoading, + } = useGetRestrictedPackages(); + useEffect(() => { + if (packagesData?.response && restrictedPackagesData?.response && agentConfigData?.item) { + const allPackages = packagesData.response; + const restrictedPackages = restrictedPackagesData.response; + const usedRestrictedPackages = (agentConfigData.item.package_configs as PackageConfig[]) + .map((packageConfig) => packageConfig.package?.name || '') + .filter((pkgName) => restrictedPackages.includes(pkgName)); + setPackages(allPackages.filter((pkg) => !usedRestrictedPackages.includes(pkg.name))); + } + }, [packagesData, restrictedPackagesData, agentConfigData]); // Update parent agent config state useEffect(() => { @@ -101,7 +121,7 @@ export const StepSelectPackage: React.FunctionComponent<{ searchable allowExclusions={false} singleSelection={true} - isLoading={isPackagesLoading} + isLoading={isPackagesLoading || isRestrictedPackagesLoading} options={packages.map(({ title, name, version, icons }) => { const pkgkey = `${name}-${version}`; return { From 7deee928e9859bcbfe34ef17aa3d12651ce08314 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 1 Jul 2020 15:52:37 -0700 Subject: [PATCH 03/15] Allow list agent configs to optionally return expanded package configs, re --- .../common/types/rest_spec/agent_config.ts | 4 ++- .../common/types/rest_spec/common.ts | 7 ++-- .../hooks/use_request/agent_config.ts | 4 +-- .../server/routes/agent_config/handlers.ts | 6 +++- .../server/services/agent_config.ts | 35 ++++++++++++++----- .../server/types/rest_spec/agent_config.ts | 4 ++- .../server/types/rest_spec/common.ts | 4 +-- 7 files changed, 46 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts index 86020cb5235ae..4e1612d144ede 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/agent_config.ts @@ -7,7 +7,9 @@ import { AgentConfig, NewAgentConfig, FullAgentConfig } from '../models'; import { ListWithKuery } from './common'; export interface GetAgentConfigsRequest { - query: ListWithKuery; + query: ListWithKuery & { + full?: boolean; + }; } export type GetAgentConfigsResponseItem = AgentConfig & { agents?: number }; diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts index c52471ccfb4f5..dfe871b1c1b29 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts @@ -3,9 +3,10 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { HttpFetchQuery } from 'src/core/public'; -export interface ListWithKuery { - page: number; - perPage: number; +export interface ListWithKuery extends HttpFetchQuery { + page?: number; + perPage?: number; kuery?: string; } diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/agent_config.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/agent_config.ts index c81303de3d7c3..56b78c6faa93a 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/agent_config.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/agent_config.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { HttpFetchQuery } from 'src/core/public'; import { useRequest, sendRequest, @@ -12,6 +11,7 @@ import { } from './use_request'; import { agentConfigRouteService } from '../../services'; import { + GetAgentConfigsRequest, GetAgentConfigsResponse, GetOneAgentConfigResponse, GetFullAgentConfigResponse, @@ -25,7 +25,7 @@ import { DeleteAgentConfigResponse, } from '../../types'; -export const useGetAgentConfigs = (query: HttpFetchQuery = {}) => { +export const useGetAgentConfigs = (query?: GetAgentConfigsRequest['query']) => { return useRequest({ path: agentConfigRouteService.getListPath(), method: 'get', diff --git a/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts index 7b12a076ff041..b2a89f96f2802 100644 --- a/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts @@ -38,8 +38,12 @@ export const getAgentConfigsHandler: RequestHandler< TypeOf > = async (context, request, response) => { const soClient = context.core.savedObjects.client; + const { full: withPackageConfigs = false, ...restOfQuery } = request.query; try { - const { items, total, page, perPage } = await agentConfigService.list(soClient, request.query); + const { items, total, page, perPage } = await agentConfigService.list(soClient, { + withPackageConfigs, + ...restOfQuery, + }); const body: GetAgentConfigsResponse = { items, total, diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.ts index ada35d1825069..55d00a267f648 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.ts @@ -141,11 +141,13 @@ class AgentConfigService { public async list( soClient: SavedObjectsClientContract, - options: ListWithKuery + options: ListWithKuery & { + withPackageConfigs?: boolean; + } ): Promise<{ items: AgentConfig[]; total: number; page: number; perPage: number }> { - const { page = 1, perPage = 20, kuery } = options; + const { page = 1, perPage = 20, kuery, withPackageConfigs = false } = options; - const agentConfigs = await soClient.find({ + const agentConfigsSO = await soClient.find({ type: SAVED_OBJECT_TYPE, page, perPage, @@ -158,12 +160,29 @@ class AgentConfigService { : undefined, }); + const agentConfigs = await Promise.all( + agentConfigsSO.saved_objects.map(async (agentConfigSO) => { + const agentConfig = { + id: agentConfigSO.id, + ...agentConfigSO.attributes, + }; + if (withPackageConfigs) { + const agentConfigWithPackageConfigs = await this.get( + soClient, + agentConfigSO.id, + withPackageConfigs + ); + if (agentConfigWithPackageConfigs) { + agentConfig.package_configs = agentConfigWithPackageConfigs.package_configs; + } + } + return agentConfig; + }) + ); + return { - items: agentConfigs.saved_objects.map((agentConfigSO) => ({ - id: agentConfigSO.id, - ...agentConfigSO.attributes, - })), - total: agentConfigs.total, + items: agentConfigs, + total: agentConfigsSO.total, page, perPage, }; diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/agent_config.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/agent_config.ts index 306aefb0d51ff..d076a803f4b53 100644 --- a/x-pack/plugins/ingest_manager/server/types/rest_spec/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/types/rest_spec/agent_config.ts @@ -8,7 +8,9 @@ import { NewAgentConfigSchema } from '../models'; import { ListWithKuerySchema } from './index'; export const GetAgentConfigsRequestSchema = { - query: ListWithKuerySchema, + query: ListWithKuerySchema.extends({ + full: schema.maybe(schema.boolean()), + }), }; export const GetOneAgentConfigRequestSchema = { diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/common.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/common.ts index 2c8134d2e8f92..28292b36ff5d2 100644 --- a/x-pack/plugins/ingest_manager/server/types/rest_spec/common.ts +++ b/x-pack/plugins/ingest_manager/server/types/rest_spec/common.ts @@ -6,8 +6,8 @@ import { schema, TypeOf } from '@kbn/config-schema'; export const ListWithKuerySchema = schema.object({ - page: schema.number({ defaultValue: 1 }), - perPage: schema.number({ defaultValue: 20 }), + page: schema.maybe(schema.number({ defaultValue: 1 })), + perPage: schema.maybe(schema.number({ defaultValue: 20 })), kuery: schema.maybe(schema.string()), }); From 845b7edfd7208b4002cba50e8667f20fe060366b Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 1 Jul 2020 15:53:54 -0700 Subject: [PATCH 04/15] Filter out agent configs which already use the restricted package already from list of agent configs available for an integration --- .../step_select_config.tsx | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx index 70668c2856f98..043cf9f77be58 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx @@ -8,7 +8,12 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, EuiFlexItem, EuiSelectable, EuiSpacer, EuiTextColor } from '@elastic/eui'; import { Error } from '../../../components'; -import { AgentConfig, PackageInfo, GetAgentConfigsResponseItem } from '../../../types'; +import { + AgentConfig, + PackageInfo, + GetAgentConfigsResponseItem, + PackageConfig, +} from '../../../types'; import { useGetPackageInfoByKey, useGetAgentConfigs, sendGetOneAgentConfig } from '../../../hooks'; export const StepSelectConfig: React.FunctionComponent<{ @@ -24,14 +29,19 @@ export const StepSelectConfig: React.FunctionComponent<{ const [selectedConfigError, setSelectedConfigError] = useState(); // Fetch package info - const { data: packageInfoData, error: packageInfoError } = useGetPackageInfoByKey(pkgkey); + const { + data: packageInfoData, + error: packageInfoError, + isLoading: packageInfoLoading, + } = useGetPackageInfoByKey(pkgkey); + const isRestrictedPackage = packageInfoData?.response.config_templates?.[0]?.multiple === false; // Fetch agent configs info const { data: agentConfigsData, error: agentConfigsError, isLoading: isAgentConfigsLoading, - } = useGetAgentConfigs(); + } = useGetAgentConfigs({ full: true }); const agentConfigs = agentConfigsData?.items || []; const agentConfigsById = agentConfigs.reduce( (acc: { [key: string]: GetAgentConfigsResponseItem }, config) => { @@ -107,12 +117,18 @@ export const StepSelectConfig: React.FunctionComponent<{ searchable allowExclusions={false} singleSelection={true} - isLoading={isAgentConfigsLoading} - options={agentConfigs.map(({ id, name, description }) => { + isLoading={isAgentConfigsLoading || packageInfoLoading} + options={agentConfigs.map(({ id, name, package_configs: packageConfigs }) => { + const alreadyHasRestrictedPackage = + isRestrictedPackage && + (packageConfigs as PackageConfig[]) + .map((packageConfig) => packageConfig.package?.name || '') + .includes(packageInfoData!.response.name); return { label: name, key: id, checked: selectedConfigId === id ? 'on' : undefined, + disabled: alreadyHasRestrictedPackage, 'data-test-subj': 'agentConfigItem', }; })} From d9a59ea9537cfb25b1ac36dbef15f362642c9d0f Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 1 Jul 2020 16:08:12 -0700 Subject: [PATCH 05/15] Allow more than 20 agent configs to be shown --- .../create_package_config_page/step_select_config.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx index 043cf9f77be58..1245722a9edbc 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx @@ -41,7 +41,7 @@ export const StepSelectConfig: React.FunctionComponent<{ data: agentConfigsData, error: agentConfigsError, isLoading: isAgentConfigsLoading, - } = useGetAgentConfigs({ full: true }); + } = useGetAgentConfigs({ full: true, perPage: 1000 }); const agentConfigs = agentConfigsData?.items || []; const agentConfigsById = agentConfigs.reduce( (acc: { [key: string]: GetAgentConfigsResponseItem }, config) => { From 03aa609db3a549f8c0726c6b54b2a1a71af70cc5 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 1 Jul 2020 16:45:31 -0700 Subject: [PATCH 06/15] Rename restricted to limited; add some common methods to DRY --- .../ingest_manager/common/constants/routes.ts | 2 +- .../ingest_manager/common/services/index.ts | 5 ++-- .../common/services/limited_package.ts | 23 ++++++++++++++ .../ingest_manager/common/services/routes.ts | 4 +-- .../common/types/rest_spec/epm.ts | 2 +- .../ingest_manager/hooks/use_request/epm.ts | 8 ++--- .../step_select_config.tsx | 30 ++++++++----------- .../step_select_package.tsx | 24 +++++++-------- .../ingest_manager/services/index.ts | 4 ++- .../server/routes/epm/handlers.ts | 10 +++---- .../ingest_manager/server/routes/epm/index.ts | 6 ++-- .../server/services/epm/packages/get.ts | 5 ++-- .../server/services/epm/packages/index.ts | 2 +- 13 files changed, 73 insertions(+), 52 deletions(-) create mode 100644 x-pack/plugins/ingest_manager/common/services/limited_package.ts diff --git a/x-pack/plugins/ingest_manager/common/constants/routes.ts b/x-pack/plugins/ingest_manager/common/constants/routes.ts index e8d06f3aa5930..7c3b5a198571c 100644 --- a/x-pack/plugins/ingest_manager/common/constants/routes.ts +++ b/x-pack/plugins/ingest_manager/common/constants/routes.ts @@ -17,7 +17,7 @@ const EPM_PACKAGES_ONE = `${EPM_PACKAGES_MANY}/{pkgkey}`; const EPM_PACKAGES_FILE = `${EPM_PACKAGES_MANY}/{pkgName}/{pkgVersion}`; export const EPM_API_ROUTES = { LIST_PATTERN: EPM_PACKAGES_MANY, - RESTRICTED_LIST_PATTERN: `${EPM_PACKAGES_MANY}/restricted`, + LIMITED_LIST_PATTERN: `${EPM_PACKAGES_MANY}/limited`, INFO_PATTERN: EPM_PACKAGES_ONE, INSTALL_PATTERN: EPM_PACKAGES_ONE, DELETE_PATTERN: EPM_PACKAGES_ONE, diff --git a/x-pack/plugins/ingest_manager/common/services/index.ts b/x-pack/plugins/ingest_manager/common/services/index.ts index a0db7c20747e2..0c91dbbe10354 100644 --- a/x-pack/plugins/ingest_manager/common/services/index.ts +++ b/x-pack/plugins/ingest_manager/common/services/index.ts @@ -3,11 +3,10 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import * as AgentStatusKueryHelper from './agent_status'; - export * from './routes'; +export * as AgentStatusKueryHelper from './agent_status'; export { packageToPackageConfigInputs, packageToPackageConfig } from './package_to_config'; export { storedPackageConfigsToAgentInputs } from './package_configs_to_agent_inputs'; export { configToYaml } from './config_to_yaml'; -export { AgentStatusKueryHelper }; +export { isPackageLimited, doesAgentConfigAlreadyIncludePackage } from './limited_package'; export { decodeCloudId } from './decode_cloud_id'; diff --git a/x-pack/plugins/ingest_manager/common/services/limited_package.ts b/x-pack/plugins/ingest_manager/common/services/limited_package.ts new file mode 100644 index 0000000000000..7ef445d55063c --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/services/limited_package.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { PackageInfo, AgentConfig, PackageConfig } from '../types'; + +// Assume packages only ever include 1 config template for now +export const isPackageLimited = (packageInfo: PackageInfo): boolean => { + return packageInfo.config_templates?.[0]?.multiple === false; +}; + +export const doesAgentConfigAlreadyIncludePackage = ( + agentConfig: AgentConfig, + packageName: string +): boolean => { + if (agentConfig.package_configs.length && typeof agentConfig.package_configs[0] === 'string') { + throw new Error('Unable to read full package config information'); + } + return (agentConfig.package_configs as PackageConfig[]) + .map((packageConfig) => packageConfig.package?.name || '') + .includes(packageName); +}; diff --git a/x-pack/plugins/ingest_manager/common/services/routes.ts b/x-pack/plugins/ingest_manager/common/services/routes.ts index 09077a4d9ad80..49de9a4d8fd85 100644 --- a/x-pack/plugins/ingest_manager/common/services/routes.ts +++ b/x-pack/plugins/ingest_manager/common/services/routes.ts @@ -27,8 +27,8 @@ export const epmRouteService = { return EPM_API_ROUTES.LIST_PATTERN; }, - getListRestrictedPath: () => { - return EPM_API_ROUTES.RESTRICTED_LIST_PATTERN; + getListLimitedPath: () => { + return EPM_API_ROUTES.LIMITED_LIST_PATTERN; }, getInfoPath: (pkgkey: string) => { diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts index 7d66bd3fa9872..c5035d2d44432 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts @@ -34,7 +34,7 @@ export interface GetPackagesResponse { success: boolean; } -export interface GetRestrictedPackagesResponse { +export interface GetLimitedPackagesResponse { response: string[]; success: boolean; } diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts index 0f92f25464115..64bee1763b08b 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/epm.ts @@ -10,7 +10,7 @@ import { epmRouteService } from '../../services'; import { GetCategoriesResponse, GetPackagesResponse, - GetRestrictedPackagesResponse, + GetLimitedPackagesResponse, GetInfoResponse, InstallPackageResponse, DeletePackageResponse, @@ -31,9 +31,9 @@ export const useGetPackages = (query: HttpFetchQuery = {}) => { }); }; -export const useGetRestrictedPackages = () => { - return useRequest({ - path: epmRouteService.getListRestrictedPath(), +export const useGetLimitedPackages = () => { + return useRequest({ + path: epmRouteService.getListLimitedPath(), method: 'get', }); }; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx index 1245722a9edbc..f0ca72fb8bbc9 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_config.tsx @@ -8,12 +8,8 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, EuiFlexItem, EuiSelectable, EuiSpacer, EuiTextColor } from '@elastic/eui'; import { Error } from '../../../components'; -import { - AgentConfig, - PackageInfo, - GetAgentConfigsResponseItem, - PackageConfig, -} from '../../../types'; +import { AgentConfig, PackageInfo, GetAgentConfigsResponseItem } from '../../../types'; +import { isPackageLimited, doesAgentConfigAlreadyIncludePackage } from '../../../services'; import { useGetPackageInfoByKey, useGetAgentConfigs, sendGetOneAgentConfig } from '../../../hooks'; export const StepSelectConfig: React.FunctionComponent<{ @@ -34,7 +30,7 @@ export const StepSelectConfig: React.FunctionComponent<{ error: packageInfoError, isLoading: packageInfoLoading, } = useGetPackageInfoByKey(pkgkey); - const isRestrictedPackage = packageInfoData?.response.config_templates?.[0]?.multiple === false; + const isLimitedPackage = (packageInfoData && isPackageLimited(packageInfoData.response)) || false; // Fetch agent configs info const { @@ -118,17 +114,17 @@ export const StepSelectConfig: React.FunctionComponent<{ allowExclusions={false} singleSelection={true} isLoading={isAgentConfigsLoading || packageInfoLoading} - options={agentConfigs.map(({ id, name, package_configs: packageConfigs }) => { - const alreadyHasRestrictedPackage = - isRestrictedPackage && - (packageConfigs as PackageConfig[]) - .map((packageConfig) => packageConfig.package?.name || '') - .includes(packageInfoData!.response.name); + options={agentConfigs.map((agentConf) => { + const alreadyHasLimitedPackage = + (isLimitedPackage && + packageInfoData && + doesAgentConfigAlreadyIncludePackage(agentConf, packageInfoData.response.name)) || + false; return { - label: name, - key: id, - checked: selectedConfigId === id ? 'on' : undefined, - disabled: alreadyHasRestrictedPackage, + label: agentConf.name, + key: agentConf.id, + checked: selectedConfigId === agentConf.id ? 'on' : undefined, + disabled: alreadyHasLimitedPackage, 'data-test-subj': 'agentConfigItem', }; })} diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx index fbb8fe6bef875..204b862bd4dc4 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_package_config_page/step_select_package.tsx @@ -12,7 +12,7 @@ import { AgentConfig, PackageInfo, PackageConfig, GetPackagesResponse } from '.. import { useGetOneAgentConfig, useGetPackages, - useGetRestrictedPackages, + useGetLimitedPackages, sendGetPackageInfoByKey, } from '../../../hooks'; import { PackageIcon } from '../../../components/package_icon'; @@ -33,7 +33,7 @@ export const StepSelectPackage: React.FunctionComponent<{ const { data: agentConfigData, error: agentConfigError } = useGetOneAgentConfig(agentConfigId); // Fetch packages info - // Filter out restricted packages already part of selected agent config + // Filter out limited packages already part of selected agent config const [packages, setPackages] = useState([]); const { data: packagesData, @@ -41,19 +41,19 @@ export const StepSelectPackage: React.FunctionComponent<{ isLoading: isPackagesLoading, } = useGetPackages(); const { - data: restrictedPackagesData, - isLoading: isRestrictedPackagesLoading, - } = useGetRestrictedPackages(); + data: limitedPackagesData, + isLoading: isLimitedPackagesLoading, + } = useGetLimitedPackages(); useEffect(() => { - if (packagesData?.response && restrictedPackagesData?.response && agentConfigData?.item) { + if (packagesData?.response && limitedPackagesData?.response && agentConfigData?.item) { const allPackages = packagesData.response; - const restrictedPackages = restrictedPackagesData.response; - const usedRestrictedPackages = (agentConfigData.item.package_configs as PackageConfig[]) + const limitedPackages = limitedPackagesData.response; + const usedLimitedPackages = (agentConfigData.item.package_configs as PackageConfig[]) .map((packageConfig) => packageConfig.package?.name || '') - .filter((pkgName) => restrictedPackages.includes(pkgName)); - setPackages(allPackages.filter((pkg) => !usedRestrictedPackages.includes(pkg.name))); + .filter((pkgName) => limitedPackages.includes(pkgName)); + setPackages(allPackages.filter((pkg) => !usedLimitedPackages.includes(pkg.name))); } - }, [packagesData, restrictedPackagesData, agentConfigData]); + }, [packagesData, limitedPackagesData, agentConfigData]); // Update parent agent config state useEffect(() => { @@ -121,7 +121,7 @@ export const StepSelectPackage: React.FunctionComponent<{ searchable allowExclusions={false} singleSelection={true} - isLoading={isPackagesLoading || isRestrictedPackagesLoading} + isLoading={isPackagesLoading || isLimitedPackagesLoading} options={packages.map(({ title, name, version, icons }) => { const pkgkey = `${name}-${version}`; return { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts index 5dc9026aebdee..9c3b84d0835b8 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts @@ -7,6 +7,7 @@ export { getFlattenedObject } from '../../../../../../../src/core/public'; export { + AgentStatusKueryHelper, agentConfigRouteService, packageConfigRouteService, dataStreamRouteService, @@ -21,5 +22,6 @@ export { packageToPackageConfigInputs, storedPackageConfigsToAgentInputs, configToYaml, - AgentStatusKueryHelper, + isPackageLimited, + doesAgentConfigAlreadyIncludePackage, } from '../../../../common'; diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index 09a31d401fbc8..0f872e3c8d760 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -16,7 +16,7 @@ import { DeletePackageResponse, GetCategoriesResponse, GetPackagesResponse, - GetRestrictedPackagesResponse, + GetLimitedPackagesResponse, } from '../../types'; import { getCategories, @@ -25,7 +25,7 @@ import { getPackageInfo, installPackage, removeInstallation, - getRestrictedPackages, + getLimitedPackages, } from '../../services/epm/packages'; export const getCategoriesHandler: RequestHandler = async (context, request, response) => { @@ -69,11 +69,11 @@ export const getListHandler: RequestHandler< } }; -export const getRestrictedListHandler: RequestHandler = async (context, request, response) => { +export const getLimitedListHandler: RequestHandler = async (context, request, response) => { try { const savedObjectsClient = context.core.savedObjects.client; - const res = await getRestrictedPackages({ savedObjectsClient }); - const body: GetRestrictedPackagesResponse = { + const res = await getLimitedPackages({ savedObjectsClient }); + const body: GetLimitedPackagesResponse = { response: res, success: true, }; diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/index.ts b/x-pack/plugins/ingest_manager/server/routes/epm/index.ts index c03b2d8af7025..ffaf0ce46c89a 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/index.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/index.ts @@ -8,7 +8,7 @@ import { PLUGIN_ID, EPM_API_ROUTES } from '../../constants'; import { getCategoriesHandler, getListHandler, - getRestrictedListHandler, + getLimitedListHandler, getFileHandler, getInfoHandler, installPackageHandler, @@ -43,11 +43,11 @@ export const registerRoutes = (router: IRouter) => { router.get( { - path: EPM_API_ROUTES.RESTRICTED_LIST_PATTERN, + path: EPM_API_ROUTES.LIMITED_LIST_PATTERN, validate: false, options: { tags: [`access:${PLUGIN_ID}`] }, }, - getRestrictedListHandler + getLimitedListHandler ); router.get( diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts index f3426b6d25a84..2f2305d1d005c 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts @@ -5,6 +5,7 @@ */ import { SavedObjectsClientContract } from 'src/core/server'; +import { isPackageLimited } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { Installation, InstallationStatus, PackageInfo, KibanaAssetType } from '../../../types'; import * as Registry from '../registry'; @@ -51,7 +52,7 @@ export async function getPackages( // Get package names for packages which cannot have more than one package config on an agent config // Assume packages only export one config template for now -export async function getRestrictedPackages(options: { +export async function getLimitedPackages(options: { savedObjectsClient: SavedObjectsClientContract; }): Promise { const { savedObjectsClient } = options; @@ -69,7 +70,7 @@ export async function getRestrictedPackages(options: { }) ); return installedPackagesInfo - .filter((pkgInfo) => pkgInfo.config_templates?.[0]?.multiple === false) + .filter((pkgInfo) => isPackageLimited(pkgInfo)) .map((pkgInfo) => pkgInfo.name); } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts index 8ef345864d0d6..4bb803dfaf912 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts @@ -19,7 +19,7 @@ export { getInstallation, getPackageInfo, getPackages, - getRestrictedPackages, + getLimitedPackages, SearchParams, } from './get'; From f307922c0a2609c32159cb64bae4934de8edcc88 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 1 Jul 2020 17:48:27 -0700 Subject: [PATCH 07/15] Add limited package check on server side --- .../server/routes/agent_config/handlers.ts | 12 +++++-- .../server/routes/package_config/handlers.ts | 22 ++---------- .../server/services/package_config.ts | 34 +++++++++++++++++-- .../ingest_manager/server/services/setup.ts | 8 ++--- 4 files changed, 47 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts index b2a89f96f2802..fcc4b5505e7fd 100644 --- a/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts @@ -107,6 +107,7 @@ export const createAgentConfigHandler: RequestHandler< TypeOf > = async (context, request, response) => { const soClient = context.core.savedObjects.client; + const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined; const withSysMonitoring = request.query.sys_monitoring ?? false; try { @@ -132,9 +133,14 @@ export const createAgentConfigHandler: RequestHandler< if (withSysMonitoring && newSysPackageConfig !== undefined && agentConfig !== undefined) { newSysPackageConfig.config_id = agentConfig.id; newSysPackageConfig.namespace = agentConfig.namespace; - const sysPackageConfig = await packageConfigService.create(soClient, newSysPackageConfig, { - user, - }); + const sysPackageConfig = await packageConfigService.create( + soClient, + callCluster, + newSysPackageConfig, + { + user, + } + ); if (sysPackageConfig) { agentConfig = await agentConfigService.assignPackageConfigs(soClient, agentConfig.id, [ diff --git a/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts index e212c861ce770..bcc2e4854d96b 100644 --- a/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts @@ -106,26 +106,10 @@ export const createPackageConfigHandler: RequestHandler< newData = updatedNewData; } - // Make sure the associated package is installed - if (newData.package?.name) { - await ensureInstalledPackage({ - savedObjectsClient: soClient, - pkgName: newData.package.name, - callCluster, - }); - const pkgInfo = await getPackageInfo({ - savedObjectsClient: soClient, - pkgName: newData.package.name, - pkgVersion: newData.package.version, - }); - newData.inputs = (await packageConfigService.assignPackageStream( - pkgInfo, - newData.inputs - )) as TypeOf['inputs']; - } - // Create package config - const packageConfig = await packageConfigService.create(soClient, newData, { user }); + const packageConfig = await packageConfigService.create(soClient, callCluster, newData, { + user, + }); const body: CreatePackageConfigResponse = { item: packageConfig, success: true }; return response.ok({ body, diff --git a/x-pack/plugins/ingest_manager/server/services/package_config.ts b/x-pack/plugins/ingest_manager/server/services/package_config.ts index c886f4868ad30..f9e42a7a7c9b5 100644 --- a/x-pack/plugins/ingest_manager/server/services/package_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/package_config.ts @@ -7,10 +7,12 @@ import { SavedObjectsClientContract } from 'src/core/server'; import { AuthenticatedUser } from '../../../security/server'; import { DeletePackageConfigsResponse, - packageToPackageConfig, PackageConfigInput, PackageConfigInputStream, PackageInfo, + packageToPackageConfig, + isPackageLimited, + doesAgentConfigAlreadyIncludePackage, } from '../../common'; import { PACKAGE_CONFIG_SAVED_OBJECT_TYPE } from '../constants'; import { @@ -23,7 +25,7 @@ import { import { agentConfigService } from './agent_config'; import { outputService } from './output'; import * as Registry from './epm/registry'; -import { getPackageInfo, getInstallation } from './epm/packages'; +import { getPackageInfo, getInstallation, ensureInstalledPackage } from './epm/packages'; import { getAssetsData } from './epm/packages/assets'; import { createStream } from './epm/agent/agent'; @@ -36,9 +38,37 @@ function getDataset(st: string) { class PackageConfigService { public async create( soClient: SavedObjectsClientContract, + callCluster: CallESAsCurrentUser, packageConfig: NewPackageConfig, options?: { id?: string; user?: AuthenticatedUser } ): Promise { + // Make sure the associated package is installed + if (packageConfig.package?.name) { + await ensureInstalledPackage({ + savedObjectsClient: soClient, + pkgName: packageConfig.package.name, + callCluster, + }); + const pkgInfo = await getPackageInfo({ + savedObjectsClient: soClient, + pkgName: packageConfig.package.name, + pkgVersion: packageConfig.package.version, + }); + + // Check if it is a limited package, and if so, check that the corresponding agent config does not + // already contain a package config for this package + if (isPackageLimited(pkgInfo)) { + const agentConfig = await agentConfigService.get(soClient, packageConfig.config_id, true); + if (agentConfig && doesAgentConfigAlreadyIncludePackage(agentConfig, pkgInfo.name)) { + throw new Error( + `Unable to create package config. A config for '${pkgInfo.name}' already exists on this agent config.` + ); + } + } + + packageConfig.inputs = await this.assignPackageStream(pkgInfo, packageConfig.inputs); + } + const isoDate = new Date().toISOString(); const newSo = await soClient.create( SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/ingest_manager/server/services/setup.ts b/x-pack/plugins/ingest_manager/server/services/setup.ts index 61e1d0ad94db8..e5ed5c589389c 100644 --- a/x-pack/plugins/ingest_manager/server/services/setup.ts +++ b/x-pack/plugins/ingest_manager/server/services/setup.ts @@ -113,6 +113,7 @@ export async function setupIngestManager( if (!isInstalled) { await addPackageToConfig( soClient, + callCluster, installedPackage, configWithPackageConfigs, defaultOutput @@ -192,6 +193,7 @@ function generateRandomPassword() { async function addPackageToConfig( soClient: SavedObjectsClientContract, + callCluster: CallESAsCurrentUser, packageToInstall: Installation, config: AgentConfig, defaultOutput: Output @@ -208,10 +210,6 @@ async function addPackageToConfig( defaultOutput.id, config.namespace ); - newPackageConfig.inputs = await packageConfigService.assignPackageStream( - packageInfo, - newPackageConfig.inputs - ); - await packageConfigService.create(soClient, newPackageConfig); + await packageConfigService.create(soClient, callCluster, newPackageConfig); } From a44ec202e5e78bbd5fbde25a659d99fe2bfb2421 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 1 Jul 2020 17:36:07 -0700 Subject: [PATCH 08/15] Adjust copy wording --- x-pack/plugins/ingest_manager/server/services/package_config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/services/package_config.ts b/x-pack/plugins/ingest_manager/server/services/package_config.ts index f9e42a7a7c9b5..26e74ee595be8 100644 --- a/x-pack/plugins/ingest_manager/server/services/package_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/package_config.ts @@ -61,7 +61,7 @@ class PackageConfigService { const agentConfig = await agentConfigService.get(soClient, packageConfig.config_id, true); if (agentConfig && doesAgentConfigAlreadyIncludePackage(agentConfig, pkgInfo.name)) { throw new Error( - `Unable to create package config. A config for '${pkgInfo.name}' already exists on this agent config.` + `Unable to create package config. Package '${pkgInfo.name}' already exists on this agent config.` ); } } From 441f3319902c7e38c403e9c4cd188fa1e46b62dc Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 1 Jul 2020 18:08:05 -0700 Subject: [PATCH 09/15] Fix typings --- .../applications/ingest_manager/types/index.ts | 2 ++ .../ingest_manager/server/routes/epm/handlers.ts | 12 +++++++----- .../server/routes/package_config/handlers.test.ts | 6 +++--- .../server/routes/package_config/handlers.ts | 2 +- .../ingest_manager/server/services/package_config.ts | 3 ++- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts index 43ec2f6d1a74d..7172b052abfe2 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts @@ -23,6 +23,7 @@ export { // API schema - misc setup, status GetFleetStatusResponse, // API schemas - Agent Config + GetAgentConfigsRequest, GetAgentConfigsResponse, GetAgentConfigsResponseItem, GetOneAgentConfigResponse, @@ -91,6 +92,7 @@ export { ServiceName, GetCategoriesResponse, GetPackagesResponse, + GetLimitedPackagesResponse, GetInfoResponse, InstallPackageResponse, DeletePackageResponse, diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index 0f872e3c8d760..a50b3b13faeab 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -6,17 +6,19 @@ import { TypeOf } from '@kbn/config-schema'; import { RequestHandler, CustomHttpResponseOptions } from 'src/core/server'; import { - GetPackagesRequestSchema, - GetFileRequestSchema, - GetInfoRequestSchema, - InstallPackageRequestSchema, - DeletePackageRequestSchema, GetInfoResponse, InstallPackageResponse, DeletePackageResponse, GetCategoriesResponse, GetPackagesResponse, GetLimitedPackagesResponse, +} from '../../../common'; +import { + GetPackagesRequestSchema, + GetFileRequestSchema, + GetInfoRequestSchema, + InstallPackageRequestSchema, + DeletePackageRequestSchema, } from '../../types'; import { getCategories, diff --git a/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.test.ts b/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.test.ts index 6d712ce063290..85ecc5027d64d 100644 --- a/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.test.ts +++ b/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.test.ts @@ -25,7 +25,7 @@ jest.mock('../../services/package_config', (): { assignPackageStream: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)), buildPackageConfigFromPackage: jest.fn(), bulkCreate: jest.fn(), - create: jest.fn((soClient, newData) => + create: jest.fn((soClient, callCluster, newData) => Promise.resolve({ ...newData, id: '1', @@ -213,7 +213,7 @@ describe('When calling package config', () => { const request = getCreateKibanaRequest(); await routeHandler(context, request, response); expect(response.ok).toHaveBeenCalled(); - expect(packageConfigServiceMock.create.mock.calls[0][1]).toEqual({ + expect(packageConfigServiceMock.create.mock.calls[0][2]).toEqual({ config_id: 'a5ca00c0-b30c-11ea-9732-1bb05811278c', description: '', enabled: true, @@ -294,7 +294,7 @@ describe('When calling package config', () => { const request = getCreateKibanaRequest(); await routeHandler(context, request, response); expect(response.ok).toHaveBeenCalled(); - expect(packageConfigServiceMock.create.mock.calls[0][1]).toEqual({ + expect(packageConfigServiceMock.create.mock.calls[0][2]).toEqual({ config_id: 'a5ca00c0-b30c-11ea-9732-1bb05811278c', description: '', enabled: true, diff --git a/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts index bcc2e4854d96b..430b153a9a786 100644 --- a/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/package_config/handlers.ts @@ -7,7 +7,7 @@ import { TypeOf } from '@kbn/config-schema'; import Boom from 'boom'; import { RequestHandler } from 'src/core/server'; import { appContextService, packageConfigService } from '../../services'; -import { ensureInstalledPackage, getPackageInfo } from '../../services/epm/packages'; +import { getPackageInfo } from '../../services/epm/packages'; import { GetPackageConfigsRequestSchema, GetOnePackageConfigRequestSchema, diff --git a/x-pack/plugins/ingest_manager/server/services/package_config.ts b/x-pack/plugins/ingest_manager/server/services/package_config.ts index 26e74ee595be8..06ab308ede1b7 100644 --- a/x-pack/plugins/ingest_manager/server/services/package_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/package_config.ts @@ -10,6 +10,7 @@ import { PackageConfigInput, PackageConfigInputStream, PackageInfo, + ListWithKuery, packageToPackageConfig, isPackageLimited, doesAgentConfigAlreadyIncludePackage, @@ -18,9 +19,9 @@ import { PACKAGE_CONFIG_SAVED_OBJECT_TYPE } from '../constants'; import { NewPackageConfig, PackageConfig, - ListWithKuery, PackageConfigSOAttributes, RegistryPackage, + CallESAsCurrentUser, } from '../types'; import { agentConfigService } from './agent_config'; import { outputService } from './output'; From e88f80bcb6f56caa95ff94a6ea93368e8a44b4a6 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 2 Jul 2020 12:35:43 -0700 Subject: [PATCH 10/15] Add some package config api integration tests, update es archive mappings --- .../apis/ingest_manager/index.js | 1 + .../apis/ingest_manager/package_config.ts | 107 ++++++++++++++++++ .../es_archives/fleet/agents/mappings.json | 30 +++-- 3 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 x-pack/test/api_integration/apis/ingest_manager/package_config.ts diff --git a/x-pack/test/api_integration/apis/ingest_manager/index.js b/x-pack/test/api_integration/apis/ingest_manager/index.js index 1671c423c03c0..ffbe1ada6d6b1 100644 --- a/x-pack/test/api_integration/apis/ingest_manager/index.js +++ b/x-pack/test/api_integration/apis/ingest_manager/index.js @@ -7,5 +7,6 @@ export default function loadTests({ loadTestFile }) { describe('Ingest Manager Endpoints', () => { loadTestFile(require.resolve('./agent_config')); + loadTestFile(require.resolve('./package_config')); }); } diff --git a/x-pack/test/api_integration/apis/ingest_manager/package_config.ts b/x-pack/test/api_integration/apis/ingest_manager/package_config.ts new file mode 100644 index 0000000000000..e9829c34762c2 --- /dev/null +++ b/x-pack/test/api_integration/apis/ingest_manager/package_config.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + describe('ingest_manager_package_configs', () => { + describe('POST /api/ingest_manager/package_configs', () => { + before(async () => { + await esArchiver.loadIfNeeded('fleet/agents'); + }); + after(async () => { + await esArchiver.unload('fleet/agents'); + }); + + it('should work with valid values', async () => { + const { body: apiResponse } = await supertest + .post(`/api/ingest_manager/package_configs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'log-1', + description: '', + namespace: 'default', + config_id: 'config1', + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'log', + title: 'Customs logs', + version: '0.1.0', + }, + }) + .expect(200); + + expect(apiResponse.success).to.be(true); + }); + + it('should return a 400 with an invalid namespace', async () => { + await supertest + .post(`/api/ingest_manager/package_configs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'log-1', + description: '', + namespace: '', + config_id: 'config1', + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'log', + title: 'Customs logs', + version: '0.1.0', + }, + }) + .expect(400); + }); + + it('should not allow multiple limited packages on the same agent config', async () => { + await supertest + .post(`/api/ingest_manager/package_configs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'endpoint-1', + description: '', + namespace: 'default', + config_id: 'config1', + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'endpoint', + title: 'Elastic Endpoint', + version: '0.8.0', + }, + }) + .expect(200); + await supertest + .post(`/api/ingest_manager/package_configs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'endpoint-2', + description: '', + namespace: 'default', + config_id: 'config1', + enabled: true, + output_id: '', + inputs: [], + package: { + name: 'endpoint', + title: 'Elastic Endpoint', + version: '0.8.0', + }, + }) + .expect(500); + }); + }); + }); +} diff --git a/x-pack/test/functional/es_archives/fleet/agents/mappings.json b/x-pack/test/functional/es_archives/fleet/agents/mappings.json index 0b84514de23f2..1f0aa2f24d6df 100644 --- a/x-pack/test/functional/es_archives/fleet/agents/mappings.json +++ b/x-pack/test/functional/es_archives/fleet/agents/mappings.json @@ -1839,6 +1839,12 @@ "config_id": { "type": "keyword" }, + "created_at": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, "description": { "type": "text" }, @@ -1847,6 +1853,7 @@ }, "inputs": { "type": "nested", + "enabled": false, "properties": { "config": { "type": "flattened" @@ -1854,20 +1861,24 @@ "enabled": { "type": "boolean" }, - "processors": { - "type": "keyword" - }, "streams": { "type": "nested", "properties": { - "agent_stream": { + "compiled_stream": { "type": "flattened" }, "config": { "type": "flattened" }, "dataset": { - "type": "keyword" + "properties": { + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } }, "enabled": { "type": "boolean" @@ -1875,9 +1886,6 @@ "id": { "type": "keyword" }, - "processors": { - "type": "keyword" - }, "vars": { "type": "flattened" } @@ -1915,6 +1923,12 @@ }, "revision": { "type": "integer" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "type": "keyword" } } }, From 7da46ac0064689b923237e68797cc180d50d474c Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 2 Jul 2020 13:36:55 -0700 Subject: [PATCH 11/15] Move test to dockerized integation tests directory; move existing epm tests to their own directory --- .../apis/ingest_manager/index.js | 1 - .../apis/{ => epm}/file.ts | 6 +- .../apis/{ => epm}/ilm.ts | 4 +- .../apis/{ => epm}/list.ts | 6 +- .../apis/{ => epm}/template.ts | 6 +- .../apis/index.js | 15 ++-- .../apis/package_config/create.ts} | 85 ++++++++++++------- 7 files changed, 76 insertions(+), 47 deletions(-) rename x-pack/test/ingest_manager_api_integration/apis/{ => epm}/file.ts (94%) rename x-pack/test/ingest_manager_api_integration/apis/{ => epm}/ilm.ts (89%) rename x-pack/test/ingest_manager_api_integration/apis/{ => epm}/list.ts (87%) rename x-pack/test/ingest_manager_api_integration/apis/{ => epm}/template.ts (87%) rename x-pack/test/{api_integration/apis/ingest_manager/package_config.ts => ingest_manager_api_integration/apis/package_config/create.ts} (51%) diff --git a/x-pack/test/api_integration/apis/ingest_manager/index.js b/x-pack/test/api_integration/apis/ingest_manager/index.js index ffbe1ada6d6b1..1671c423c03c0 100644 --- a/x-pack/test/api_integration/apis/ingest_manager/index.js +++ b/x-pack/test/api_integration/apis/ingest_manager/index.js @@ -7,6 +7,5 @@ export default function loadTests({ loadTestFile }) { describe('Ingest Manager Endpoints', () => { loadTestFile(require.resolve('./agent_config')); - loadTestFile(require.resolve('./package_config')); }); } diff --git a/x-pack/test/ingest_manager_api_integration/apis/file.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/file.ts similarity index 94% rename from x-pack/test/ingest_manager_api_integration/apis/file.ts rename to x-pack/test/ingest_manager_api_integration/apis/epm/file.ts index 33eeda1ee274d..3f99f91394d2c 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/file.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/file.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; -import { warnAndSkipTest } from '../helpers'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../../helpers'; export default function ({ getService }: FtrProviderContext) { const log = getService('log'); @@ -13,7 +13,7 @@ export default function ({ getService }: FtrProviderContext) { const dockerServers = getService('dockerServers'); const server = dockerServers.get('registry'); - describe('package file', () => { + describe('EPM - package file', () => { it('fetches a .png screenshot image', async function () { if (server.enabled) { await supertest diff --git a/x-pack/test/ingest_manager_api_integration/apis/ilm.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/ilm.ts similarity index 89% rename from x-pack/test/ingest_manager_api_integration/apis/ilm.ts rename to x-pack/test/ingest_manager_api_integration/apis/epm/ilm.ts index b73a9da5fad59..8a801d59eb5b2 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/ilm.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/ilm.ts @@ -5,10 +5,10 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { - describe('ilm', () => { + describe('EPM - ilm', () => { it('setup policy', async () => { const policyName = 'foo'; const es = getService('es'); diff --git a/x-pack/test/ingest_manager_api_integration/apis/list.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/list.ts similarity index 87% rename from x-pack/test/ingest_manager_api_integration/apis/list.ts rename to x-pack/test/ingest_manager_api_integration/apis/epm/list.ts index 200358cb6f8f0..669a10df398a8 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/list.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/list.ts @@ -5,8 +5,8 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; -import { warnAndSkipTest } from '../helpers'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../../helpers'; export default function ({ getService }: FtrProviderContext) { const log = getService('log'); @@ -18,7 +18,7 @@ export default function ({ getService }: FtrProviderContext) { // because `this` has to point to the Mocha context // see https://mochajs.org/#arrow-functions - describe('list', async function () { + describe('EPM - list', async function () { it('lists all packages from the registry', async function () { if (server.enabled) { const fetchPackageList = async () => { diff --git a/x-pack/test/ingest_manager_api_integration/apis/template.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/template.ts similarity index 87% rename from x-pack/test/ingest_manager_api_integration/apis/template.ts rename to x-pack/test/ingest_manager_api_integration/apis/epm/template.ts index 8911dd28dc243..1884a2b7c45b0 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/template.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/template.ts @@ -5,8 +5,8 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; -import { getTemplate } from '../../../plugins/ingest_manager/server/services/epm/elasticsearch/template/template'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { getTemplate } from '../../../../plugins/ingest_manager/server/services/epm/elasticsearch/template/template'; export default function ({ getService }: FtrProviderContext) { const indexPattern = 'foo'; @@ -20,7 +20,7 @@ export default function ({ getService }: FtrProviderContext) { }, }; // This test was inspired by https://github.com/elastic/kibana/blob/master/x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js - describe('template', async () => { + describe('EPM - template', async () => { it('can be loaded', async () => { const template = getTemplate({ type: 'logs', diff --git a/x-pack/test/ingest_manager_api_integration/apis/index.js b/x-pack/test/ingest_manager_api_integration/apis/index.js index ef8880f86078b..dc8f04e35c278 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/index.js +++ b/x-pack/test/ingest_manager_api_integration/apis/index.js @@ -5,11 +5,16 @@ */ export default function ({ loadTestFile }) { - describe('EPM Endpoints', function () { + describe('Ingest Manager Endpoints', function () { this.tags('ciGroup7'); - loadTestFile(require.resolve('./list')); - loadTestFile(require.resolve('./file')); - //loadTestFile(require.resolve('./template')); - loadTestFile(require.resolve('./ilm')); + + // EPM + loadTestFile(require.resolve('./epm/list')); + loadTestFile(require.resolve('./epm/list')); + //loadTestFile(require.resolve('./epm/template')); + loadTestFile(require.resolve('./epm/ilm')); + + // Package configs + loadTestFile(require.resolve('./package_config/create')); }); } diff --git a/x-pack/test/api_integration/apis/ingest_manager/package_config.ts b/x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts similarity index 51% rename from x-pack/test/api_integration/apis/ingest_manager/package_config.ts rename to x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts index e9829c34762c2..b533425950faa 100644 --- a/x-pack/test/api_integration/apis/ingest_manager/package_config.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts @@ -5,66 +5,89 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../ftr_provider_context'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../../helpers'; export default function ({ getService }: FtrProviderContext) { + const log = getService('log'); const supertest = getService('supertest'); - const esArchiver = getService('esArchiver'); + const dockerServers = getService('dockerServers'); - describe('ingest_manager_package_configs', () => { - describe('POST /api/ingest_manager/package_configs', () => { - before(async () => { - await esArchiver.loadIfNeeded('fleet/agents'); - }); - after(async () => { - await esArchiver.unload('fleet/agents'); - }); + const server = dockerServers.get('registry'); + // use function () {} and not () => {} here + // because `this` has to point to the Mocha context + // see https://mochajs.org/#arrow-functions - it('should work with valid values', async () => { + describe('Package Config - create', async function () { + let agentConfigId: string; + + before(async function () { + const { body: agentConfigResponse } = await supertest + .post(`/api/ingest_manager/agent_configs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test config', + namespace: 'default', + }); + agentConfigId = agentConfigResponse.item.id; + }); + + it('should work with valid values', async function () { + if (server.enabled) { const { body: apiResponse } = await supertest .post(`/api/ingest_manager/package_configs`) .set('kbn-xsrf', 'xxxx') .send({ - name: 'log-1', + name: 'filetest-1', description: '', namespace: 'default', - config_id: 'config1', + config_id: agentConfigId, enabled: true, output_id: '', inputs: [], package: { - name: 'log', - title: 'Customs logs', + name: 'filetest', + title: 'For File Tests', version: '0.1.0', }, }) .expect(200); expect(apiResponse.success).to.be(true); - }); + } else { + warnAndSkipTest(this, log); + } + }); - it('should return a 400 with an invalid namespace', async () => { + it('should return a 400 with an invalid namespace', async function () { + if (server.enabled) { await supertest .post(`/api/ingest_manager/package_configs`) .set('kbn-xsrf', 'xxxx') .send({ - name: 'log-1', + name: 'filetest-1', description: '', namespace: '', - config_id: 'config1', + config_id: agentConfigId, enabled: true, output_id: '', inputs: [], package: { - name: 'log', - title: 'Customs logs', + name: 'filetest', + title: 'For File Tests', version: '0.1.0', }, }) .expect(400); - }); + } else { + warnAndSkipTest(this, log); + } + }); - it('should not allow multiple limited packages on the same agent config', async () => { + // This test is disabled as the docker image used does not contain necessary package registry changes + // to support this functionality + it.skip('should not allow multiple limited packages on the same agent config', async function () { + if (server.enabled) { await supertest .post(`/api/ingest_manager/package_configs`) .set('kbn-xsrf', 'xxxx') @@ -72,14 +95,14 @@ export default function ({ getService }: FtrProviderContext) { name: 'endpoint-1', description: '', namespace: 'default', - config_id: 'config1', + config_id: agentConfigId, enabled: true, output_id: '', inputs: [], package: { name: 'endpoint', - title: 'Elastic Endpoint', - version: '0.8.0', + title: 'Endpoint', + version: '0.5.0', }, }) .expect(200); @@ -90,18 +113,20 @@ export default function ({ getService }: FtrProviderContext) { name: 'endpoint-2', description: '', namespace: 'default', - config_id: 'config1', + config_id: agentConfigId, enabled: true, output_id: '', inputs: [], package: { name: 'endpoint', - title: 'Elastic Endpoint', - version: '0.8.0', + title: 'Endpoint', + version: '0.5.0', }, }) .expect(500); - }); + } else { + warnAndSkipTest(this, log); + } }); }); } From 6d3006ed034ad93a3194befcedda22ef60ccf0f0 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 2 Jul 2020 14:29:52 -0700 Subject: [PATCH 12/15] Remove extra assignPackageConfigs() - already handled in packageConfigService.create() --- .../server/routes/agent_config/handlers.ts | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts index fcc4b5505e7fd..110f6b9950829 100644 --- a/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts @@ -133,20 +133,9 @@ export const createAgentConfigHandler: RequestHandler< if (withSysMonitoring && newSysPackageConfig !== undefined && agentConfig !== undefined) { newSysPackageConfig.config_id = agentConfig.id; newSysPackageConfig.namespace = agentConfig.namespace; - const sysPackageConfig = await packageConfigService.create( - soClient, - callCluster, - newSysPackageConfig, - { - user, - } - ); - - if (sysPackageConfig) { - agentConfig = await agentConfigService.assignPackageConfigs(soClient, agentConfig.id, [ - sysPackageConfig.id, - ]); - } + await packageConfigService.create(soClient, callCluster, newSysPackageConfig, { + user, + }); } const body: CreateAgentConfigResponse = { From b4e522f6959a53a6aac744b2be923caef1184ca0 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 6 Jul 2020 09:59:49 -0700 Subject: [PATCH 13/15] Review fixes --- .../common/types/rest_spec/common.ts | 1 - .../server/services/epm/packages/get.ts | 6 ++--- .../server/services/package_config.ts | 22 ++++++++++--------- .../apis/index.js | 2 +- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts index 05d6b7ecfdd77..0d1f72afa16f1 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { HttpFetchQuery } from 'src/core/public'; export interface ListWithKuery { page?: number; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts index 2f2305d1d005c..ad9635cc02e06 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts @@ -61,7 +61,7 @@ export async function getLimitedPackages(options: { (pkg) => (pkg.status = InstallationStatus.installed) ); const installedPackagesInfo = await Promise.all( - installedPackages.map(async (pkgInstall) => { + installedPackages.map((pkgInstall) => { return getPackageInfo({ savedObjectsClient, pkgName: pkgInstall.name, @@ -69,9 +69,7 @@ export async function getLimitedPackages(options: { }); }) ); - return installedPackagesInfo - .filter((pkgInfo) => isPackageLimited(pkgInfo)) - .map((pkgInfo) => pkgInfo.name); + return installedPackagesInfo.filter((pkgInfo) => isPackageLimited).map((pkgInfo) => pkgInfo.name); } export async function getPackageSavedObjects(savedObjectsClient: SavedObjectsClientContract) { diff --git a/x-pack/plugins/ingest_manager/server/services/package_config.ts b/x-pack/plugins/ingest_manager/server/services/package_config.ts index 1e35587b1f468..9433a81e74b07 100644 --- a/x-pack/plugins/ingest_manager/server/services/package_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/package_config.ts @@ -46,16 +46,18 @@ class PackageConfigService { ): Promise { // Make sure the associated package is installed if (packageConfig.package?.name) { - await ensureInstalledPackage({ - savedObjectsClient: soClient, - pkgName: packageConfig.package.name, - callCluster, - }); - const pkgInfo = await getPackageInfo({ - savedObjectsClient: soClient, - pkgName: packageConfig.package.name, - pkgVersion: packageConfig.package.version, - }); + const [, pkgInfo] = await Promise.all([ + ensureInstalledPackage({ + savedObjectsClient: soClient, + pkgName: packageConfig.package.name, + callCluster, + }), + getPackageInfo({ + savedObjectsClient: soClient, + pkgName: packageConfig.package.name, + pkgVersion: packageConfig.package.version, + }), + ]); // Check if it is a limited package, and if so, check that the corresponding agent config does not // already contain a package config for this package diff --git a/x-pack/test/ingest_manager_api_integration/apis/index.js b/x-pack/test/ingest_manager_api_integration/apis/index.js index dc8f04e35c278..83bfd9b21cdf9 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/index.js +++ b/x-pack/test/ingest_manager_api_integration/apis/index.js @@ -10,7 +10,7 @@ export default function ({ loadTestFile }) { // EPM loadTestFile(require.resolve('./epm/list')); - loadTestFile(require.resolve('./epm/list')); + loadTestFile(require.resolve('./epm/file')); //loadTestFile(require.resolve('./epm/template')); loadTestFile(require.resolve('./epm/ilm')); From c18ec7624047573d3ff0fa6350d5ebc700958370 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 6 Jul 2020 10:17:01 -0700 Subject: [PATCH 14/15] Fix type, reenabled skipped test --- .../ingest_manager/common/types/rest_spec/common.ts | 3 ++- .../apis/package_config/create.ts | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts index 0d1f72afa16f1..a454e39c203ed 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/common.ts @@ -3,8 +3,9 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { HttpFetchQuery } from 'src/core/public'; -export interface ListWithKuery { +export interface ListWithKuery extends HttpFetchQuery { page?: number; perPage?: number; sortField?: string; diff --git a/x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts b/x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts index b533425950faa..c7748ab255f43 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/package_config/create.ts @@ -84,9 +84,7 @@ export default function ({ getService }: FtrProviderContext) { } }); - // This test is disabled as the docker image used does not contain necessary package registry changes - // to support this functionality - it.skip('should not allow multiple limited packages on the same agent config', async function () { + it('should not allow multiple limited packages on the same agent config', async function () { if (server.enabled) { await supertest .post(`/api/ingest_manager/package_configs`) @@ -102,7 +100,7 @@ export default function ({ getService }: FtrProviderContext) { package: { name: 'endpoint', title: 'Endpoint', - version: '0.5.0', + version: '0.8.0', }, }) .expect(200); @@ -120,7 +118,7 @@ export default function ({ getService }: FtrProviderContext) { package: { name: 'endpoint', title: 'Endpoint', - version: '0.5.0', + version: '0.8.0', }, }) .expect(500); From fa768a1f8475b6add6fe3b7efde47dc19b9dcbc6 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 6 Jul 2020 12:18:39 -0700 Subject: [PATCH 15/15] Move new EPM integration test file --- .../ingest_manager_api_integration/apis/{ => epm}/install.ts | 4 ++-- x-pack/test/ingest_manager_api_integration/apis/index.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename x-pack/test/ingest_manager_api_integration/apis/{ => epm}/install.ts (95%) diff --git a/x-pack/test/ingest_manager_api_integration/apis/install.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/install.ts similarity index 95% rename from x-pack/test/ingest_manager_api_integration/apis/install.ts rename to x-pack/test/ingest_manager_api_integration/apis/epm/install.ts index 92078c25419df..f73ba56c172c4 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/install.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/install.ts @@ -5,8 +5,8 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; -import { warnAndSkipTest } from '../helpers'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../../helpers'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); diff --git a/x-pack/test/ingest_manager_api_integration/apis/index.js b/x-pack/test/ingest_manager_api_integration/apis/index.js index 716905b92f2a7..30c49140c6e2a 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/index.js +++ b/x-pack/test/ingest_manager_api_integration/apis/index.js @@ -13,7 +13,7 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./epm/file')); //loadTestFile(require.resolve('./epm/template')); loadTestFile(require.resolve('./epm/ilm')); - loadTestFile(require.resolve('./install')); + loadTestFile(require.resolve('./epm/install')); // Package configs loadTestFile(require.resolve('./package_config/create'));