From 293eeafe1fc9f540d65301d6b59fb04c628ad0e5 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 3 Aug 2020 13:36:52 -0400 Subject: [PATCH 01/24] Create Policies for each generated host --- .../common/endpoint/generate_data.ts | 19 ++- .../common/endpoint/index_data.ts | 140 +++++++++++++++++- .../endpoint/resolver_generator_script.ts | 4 + 3 files changed, 152 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 7f31c71fe712b..2260c38581e6d 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -110,6 +110,12 @@ const Mac: OSFields[] = []; const OS: OSFields[] = [...Windows, ...Mac, ...Linux]; +const POLICY_RESPONSE_STATUSES: HostPolicyResponseActionStatus[] = [ + HostPolicyResponseActionStatus.success, + HostPolicyResponseActionStatus.failure, + HostPolicyResponseActionStatus.warning, +]; + const APPLIED_POLICIES: Array<{ name: string; id: string; @@ -364,15 +370,12 @@ export class EndpointDocGenerator { } /** - * Creates new random policy id for the host to simulate new policy application + * Updates the current Host common record applied Policy to a different one from the list + * of random choices and gives it a random policy response status. */ - public updatePolicyId() { - this.commonInfo.Endpoint.policy.applied.id = this.randomChoice(APPLIED_POLICIES).id; - this.commonInfo.Endpoint.policy.applied.status = this.randomChoice([ - HostPolicyResponseActionStatus.success, - HostPolicyResponseActionStatus.failure, - HostPolicyResponseActionStatus.warning, - ]); + public updateHostPolicyData() { + this.commonInfo.Endpoint.policy.applied = this.randomChoice(APPLIED_POLICIES); + this.commonInfo.Endpoint.policy.applied.status = this.randomChoice(POLICY_RESPONSE_STATUSES); } private createHostData(): HostInfo { diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index b8c2fdbe65f1e..63b34f62f725a 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -6,11 +6,27 @@ import { Client } from '@elastic/elasticsearch'; import seedrandom from 'seedrandom'; +import { KbnClient } from '@kbn/dev-utils'; +import { AxiosResponse } from 'axios'; import { EndpointDocGenerator, TreeOptions, Event } from './generate_data'; import { firstNonNullValue } from './models/ecs_safety_helpers'; +import { + CreateAgentConfigRequest, + CreateAgentConfigResponse, + CreatePackageConfigRequest, + CreatePackageConfigResponse, + GetPackagesResponse, +} from '../../../ingest_manager/common/types/rest_spec'; +import { + AGENT_CONFIG_API_ROUTES, + EPM_API_ROUTES, + PACKAGE_CONFIG_API_ROUTES, +} from '../../../ingest_manager/common'; +import { factory as policyConfigFactory } from './models/policy_config'; export async function indexHostsAndAlerts( client: Client, + kbnClient: KbnClient, seed: string, numHosts: number, numDocs: number, @@ -22,9 +38,22 @@ export async function indexHostsAndAlerts( options: TreeOptions = {} ) { const random = seedrandom(seed); + const epmEndpointPackage = await getEndpointPackageInfo(kbnClient); + // Keep a map of host applied policy ids (fake) to real ingest package configs (policy record) + const realPolicies: Record = {}; + for (let i = 0; i < numHosts; i++) { const generator = new EndpointDocGenerator(random); - await indexHostDocs(numDocs, client, metadataIndex, policyIndex, generator); + await indexHostDocs( + numDocs, + client, + kbnClient, + realPolicies, + epmEndpointPackage, + metadataIndex, + policyIndex, + generator + ); await indexAlerts(client, eventIndex, alertIndex, generator, alertsPerHost, options); } await client.indices.refresh({ @@ -43,18 +72,55 @@ function delay(ms: number) { async function indexHostDocs( numDocs: number, client: Client, + kbnClient: KbnClient, + realPolicies: Record, + epmEndpointPackage: GetPackagesResponse['response'][0], metadataIndex: string, policyIndex: string, generator: EndpointDocGenerator ) { const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents const timestamp = new Date().getTime(); + for (let j = 0; j < numDocs; j++) { generator.updateHostData(); - generator.updatePolicyId(); + generator.updateHostPolicyData(); + + let hostMetadata = generator.generateHostMetadata( + timestamp - timeBetweenDocs * (numDocs - j - 1) + ); + const { id: appliedPolicyId, name: appliedPolicyName } = hostMetadata.Endpoint.policy.applied; + + if (appliedPolicyId !== '00000000-0000-0000-0000-000000000000') { + // If we don't yet have a "real" policy record, then create it now in ingest (package config) + if (!realPolicies[appliedPolicyId]) { + // eslint-disable-next-line require-atomic-updates + realPolicies[appliedPolicyId] = await createPolicy( + kbnClient, + appliedPolicyName, + epmEndpointPackage.version + ); + } + + // Update the Host metadata record with the ID of the "real" policy + hostMetadata = { + ...hostMetadata, + Endpoint: { + ...hostMetadata.Endpoint, + policy: { + ...hostMetadata.Endpoint.policy, + applied: { + ...hostMetadata.Endpoint.policy.applied, + id: realPolicies[appliedPolicyId].id, + }, + }, + }, + }; + } + await client.index({ index: metadataIndex, - body: generator.generateHostMetadata(timestamp - timeBetweenDocs * (numDocs - j - 1)), + body: hostMetadata, op_type: 'create', }); await client.index({ @@ -98,3 +164,71 @@ async function indexAlerts( await client.bulk({ body, refresh: true }); } } + +const createPolicy = async ( + kbnClient: KbnClient, + policyName: string, + endpointPackageVersion: string +): Promise => { + // Create Agent Configuration first + const newAgentconfigData: CreateAgentConfigRequest['body'] = { + name: `Config for ${policyName}`, + description: '', + namespace: 'default', + }; + const agentConfig = (await kbnClient.request({ + path: AGENT_CONFIG_API_ROUTES.CREATE_PATTERN, + method: 'POST', + body: newAgentconfigData, + })) as AxiosResponse; + + // Create Package Configuration + const newPackageConfigData: CreatePackageConfigRequest['body'] = { + name: policyName, + description: 'Protect the worlds data', + config_id: agentConfig.data.item.id, + enabled: true, + output_id: '', + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + policy: { + value: policyConfigFactory(), + }, + }, + }, + ], + namespace: 'default', + package: { + name: 'endpoint', + title: 'endpoint', + version: endpointPackageVersion, + }, + }; + const packageConfig = (await kbnClient.request({ + path: PACKAGE_CONFIG_API_ROUTES.CREATE_PATTERN, + method: 'POST', + body: newPackageConfigData, + })) as AxiosResponse; + return packageConfig.data.item; +}; + +const getEndpointPackageInfo = async ( + kbnClient: KbnClient +): Promise => { + const endpointPackage = ((await kbnClient.request({ + path: `${EPM_API_ROUTES.LIST_PATTERN}?category=security`, + method: 'GET', + })) as AxiosResponse).data.response.find( + (epmPackage) => epmPackage.name === 'endpoint' + ); + + if (!endpointPackage) { + throw new Error('EPM Endpoint package was not found!'); + } + + return endpointPackage; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index cfe1c741ef3f1..7952ef0ecfb0f 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -10,6 +10,7 @@ import * as url from 'url'; import fetch from 'node-fetch'; import { Client, ClientOptions } from '@elastic/elasticsearch'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; +import { KbnClient, ToolingLog } from '@kbn/dev-utils'; import { indexHostsAndAlerts } from '../../common/endpoint/index_data'; import { ANCESTRY_LIMIT } from '../../common/endpoint/generate_data'; @@ -203,6 +204,8 @@ async function main() { node: argv.node, }; + const kibana = new KbnClient(new ToolingLog(), { url: argv.kibana }); + const client = new Client(clientOptions); if (argv.delete) { await deleteIndices( @@ -219,6 +222,7 @@ async function main() { const startTime = new Date().getTime(); await indexHostsAndAlerts( client, + kibana, seed, argv.numHosts, argv.numDocs, From 2b5ab651fb173973b62892700ae2d054d26779ed Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 3 Aug 2020 14:08:54 -0400 Subject: [PATCH 02/24] Refactor Ingest setup to also setup Fleet --- .../common/endpoint/generate_data.ts | 5 ++ .../endpoint/resolver_generator_script.ts | 85 ++++++++++--------- 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 2260c38581e6d..6db33d4abfc56 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -131,6 +131,11 @@ const APPLIED_POLICIES: Array<{ id: 'C2A9093E-E289-4C0A-AA44-8C32A414FA7A', status: HostPolicyResponseActionStatus.success, }, + { + name: 'Detect Malware Only', + id: '47d7965d-6869-478b-bd9c-fb0d2bb3959f', + status: HostPolicyResponseActionStatus.success, + }, ]; const FILE_OPERATIONS: string[] = ['creation', 'open', 'rename', 'execution', 'deletion']; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index 7952ef0ecfb0f..c3afb38d04952 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -4,15 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ /* eslint-disable no-console */ -import * as path from 'path'; import yargs from 'yargs'; -import * as url from 'url'; -import fetch from 'node-fetch'; import { Client, ClientOptions } from '@elastic/elasticsearch'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { KbnClient, ToolingLog } from '@kbn/dev-utils'; +import { AxiosResponse } from 'axios'; import { indexHostsAndAlerts } from '../../common/endpoint/index_data'; import { ANCESTRY_LIMIT } from '../../common/endpoint/generate_data'; +import { FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../../ingest_manager/common/constants'; +import { + CreateFleetSetupResponse, + PostIngestSetupResponse, +} from '../../../ingest_manager/common/types/rest_spec'; main(); @@ -36,42 +39,37 @@ async function deleteIndices(indices: string[], client: Client) { } } -async function doIngestSetup(kibanaURL: string) { +async function doIngestSetup(kbnClient: KbnClient) { + // Setup Ingest try { - const kbURL = new url.URL(kibanaURL); - // this includes the base path that is randomly generated by Kibana - const pathname = path.posix.join(path.posix.sep, kbURL.pathname, 'api/ingest_manager/setup'); - const connectURL = new url.URL(pathname, kbURL); - console.log('Calling ingest manager setup at ', connectURL.toString()); - const response = await fetch( - // wrap base url in URL class because the kibana basepath will get removed otherwise - connectURL.toString(), - { - method: 'POST', - headers: { - 'kbn-xsrf': 'blah', - }, - } - ); - if (response.status !== 200) { - console.log('POST response ', response); - console.log( - 'Request failed please check that you have the correct base path and credentials for the kibana URL' - ); - // eslint-disable-next-line no-process-exit - process.exit(1); + const setupResponse = (await kbnClient.request({ + path: SETUP_API_ROUTE, + method: 'POST', + })) as AxiosResponse; + + if (!setupResponse.data.isInitialized) { + console.error(setupResponse.data); + throw new Error('Initializing the ingest manager failed, existing'); } - const setupResponse = await response.json(); - console.log('Ingest setup response ', setupResponse); - if (!setupResponse?.isInitialized) { - console.log('Initializing the ingest manager failed, existing'); - // eslint-disable-next-line no-process-exit - process.exit(1); + } catch (error) { + console.error(error); + throw error; + } + + // Setup Fleet + try { + const setupResponse = (await kbnClient.request({ + path: FLEET_SETUP_API_ROUTES.CREATE_PATTERN, + method: 'POST', + })) as AxiosResponse; + + if (!setupResponse.data.isInitialized) { + console.error(setupResponse.data); + throw new Error('Initializing Fleet failed, existing'); } } catch (error) { - console.log(JSON.stringify(error, null, 2)); - // eslint-disable-next-line no-process-exit - process.exit(1); + console.error(error); + throw error; } } @@ -198,15 +196,18 @@ async function main() { default: false, }, }).argv; - await doIngestSetup(argv.kibana); + const kbnClient = new KbnClient(new ToolingLog(), { url: argv.kibana }); - const clientOptions: ClientOptions = { - node: argv.node, - }; - - const kibana = new KbnClient(new ToolingLog(), { url: argv.kibana }); + try { + await doIngestSetup(kbnClient); + } catch (error) { + // eslint-disable-next-line no-process-exit + process.exit(1); + } + const clientOptions: ClientOptions = { node: argv.node }; const client = new Client(clientOptions); + if (argv.delete) { await deleteIndices( [argv.eventIndex, argv.metadataIndex, argv.policyIndex, argv.alertIndex], @@ -222,7 +223,7 @@ async function main() { const startTime = new Date().getTime(); await indexHostsAndAlerts( client, - kibana, + kbnClient, seed, argv.numHosts, argv.numDocs, From ad39d54d05d5d50e7b8cc03a17ffb8532f915f3e Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 3 Aug 2020 14:40:23 -0400 Subject: [PATCH 03/24] Rename prop name --- .../security_solution/common/endpoint/index_data.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 63b34f62f725a..a2817e376c8f4 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -31,7 +31,7 @@ export async function indexHostsAndAlerts( numHosts: number, numDocs: number, metadataIndex: string, - policyIndex: string, + policyResponseIndex: string, eventIndex: string, alertIndex: string, alertsPerHost: number, @@ -51,7 +51,7 @@ export async function indexHostsAndAlerts( realPolicies, epmEndpointPackage, metadataIndex, - policyIndex, + policyResponseIndex, generator ); await indexAlerts(client, eventIndex, alertIndex, generator, alertsPerHost, options); @@ -76,7 +76,7 @@ async function indexHostDocs( realPolicies: Record, epmEndpointPackage: GetPackagesResponse['response'][0], metadataIndex: string, - policyIndex: string, + policyResponseIndex: string, generator: EndpointDocGenerator ) { const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents @@ -124,7 +124,7 @@ async function indexHostDocs( op_type: 'create', }); await client.index({ - index: policyIndex, + index: policyResponseIndex, body: generator.generatePolicyResponse(timestamp - timeBetweenDocs * (numDocs - j - 1)), op_type: 'create', }); From c2aea571fe2c6335037953ec8200a123260e955f Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 3 Aug 2020 18:19:46 -0400 Subject: [PATCH 04/24] Add generic response type to KbnClient.request + support for headers --- packages/kbn-dev-utils/src/kbn_client/kbn_client.ts | 4 ++-- packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts index 861ea0988692c..7184727fc53de 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts @@ -54,8 +54,8 @@ export class KbnClient { /** * Make a direct request to the Kibana server */ - async request(options: ReqOptions) { - return await this.requester.request(options); + async request(options: ReqOptions) { + return await this.requester.request(options); } resolveUrl(relativeUrl: string) { diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts index 2aba2be56f277..f4d7c9b7f287d 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts @@ -60,6 +60,7 @@ export interface ReqOptions { path: string; query?: Record; method: 'GET' | 'POST' | 'PUT' | 'DELETE'; + headers?: Record; body?: any; retries?: number; } @@ -117,6 +118,7 @@ export class KbnClientRequester { params: options.query, headers: { 'kbn-xsrf': 'kbn-client', + ...options.headers, }, httpsAgent: this.httpsAgent, }); From b71e43864042a0a39f4a5cd3d24350d6445c030e Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 4 Aug 2020 07:32:31 -0400 Subject: [PATCH 05/24] first attempt at adding fleet agent registration --- .../common/endpoint/index_data.ts | 89 +++++++++++++++++-- 1 file changed, 84 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index a2817e376c8f4..9319aa80b365a 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -16,13 +16,19 @@ import { CreatePackageConfigRequest, CreatePackageConfigResponse, GetPackagesResponse, -} from '../../../ingest_manager/common/types/rest_spec'; -import { + PostAgentEnrollRequest, + AGENT_API_ROUTES, AGENT_CONFIG_API_ROUTES, EPM_API_ROUTES, PACKAGE_CONFIG_API_ROUTES, + ENROLLMENT_API_KEY_ROUTES, + GetEnrollmentAPIKeysResponse, + GetOnePackageConfigResponse, + GetOneEnrollmentAPIKeyResponse, + PostAgentEnrollResponse, } from '../../../ingest_manager/common'; import { factory as policyConfigFactory } from './models/policy_config'; +import { HostMetadata } from './types'; export async function indexHostsAndAlerts( client: Client, @@ -81,14 +87,13 @@ async function indexHostDocs( ) { const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents const timestamp = new Date().getTime(); + let hostMetadata: HostMetadata; for (let j = 0; j < numDocs; j++) { generator.updateHostData(); generator.updateHostPolicyData(); - let hostMetadata = generator.generateHostMetadata( - timestamp - timeBetweenDocs * (numDocs - j - 1) - ); + hostMetadata = generator.generateHostMetadata(timestamp - timeBetweenDocs * (numDocs - j - 1)); const { id: appliedPolicyId, name: appliedPolicyName } = hostMetadata.Endpoint.policy.applied; if (appliedPolicyId !== '00000000-0000-0000-0000-000000000000') { @@ -129,6 +134,10 @@ async function indexHostDocs( op_type: 'create', }); } + + if (hostMetadata!.Endpoint.policy.applied.id !== '00000000-0000-0000-0000-000000000000') { + await fleetEnrollAgentForHost(kbnClient, hostMetadata!); + } } async function indexAlerts( @@ -232,3 +241,73 @@ const getEndpointPackageInfo = async ( return endpointPackage; }; + +const fleetEnrollAgentForHost = async (kbnClient: KbnClient, host: HostMetadata) => { + // Get Enrollement key for host's applied policy + const enrollmentApiKey = await kbnClient + .request({ + path: PACKAGE_CONFIG_API_ROUTES.INFO_PATTERN.replace( + '{packageConfigId}', + host.Endpoint.policy.applied.id + ), + method: 'GET', + }) + .then((packageConfigResponse) => { + return kbnClient.request({ + path: ENROLLMENT_API_KEY_ROUTES.LIST_PATTERN, + method: 'GET', + query: { + kuery: `fleet-enrollment-api-keys.config_id:"${packageConfigResponse.data.item.config_id}"`, + }, + }); + }) + .then((apiKeysResponse) => { + const apiKey = apiKeysResponse.data.list[0]; + + // TODO: Handle if it does not exist + + return kbnClient.request({ + path: ENROLLMENT_API_KEY_ROUTES.INFO_PATTERN.replace('{keyId}', apiKey.id), + method: 'GET', + }); + }) + .then((apiKeyDetailsResponse) => { + return apiKeyDetailsResponse.data.item.api_key; + }) + .catch(() => { + return ''; + }); + + if (enrollmentApiKey.length === 0) { + return; + } + + // Enroll an agent for the Host + const body: PostAgentEnrollRequest['body'] = { + type: 'PERMANENT', + shared_id: host.elastic.agent.id, + metadata: { + local: { + ...host.host, + }, + user_provided: { + dev_agent_version: '0.0.1', + region: 'us-east', + }, + }, + }; + + try { + await kbnClient.request({ + path: AGENT_API_ROUTES.ENROLL_PATTERN, + method: 'POST', + body, + headers: { + Authorization: `ApiKey ${enrollmentApiKey}`, + 'Content-Type': 'application/json', + }, + }); + } catch (error) { + // debugger; + } +}; From 73a0754448d81d95a310daa97072df06b3fa2fa8 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 4 Aug 2020 09:25:17 -0400 Subject: [PATCH 06/24] a little closer with fleet integration --- .../common/endpoint/index_data.ts | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 9319aa80b365a..bcedf49163750 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -8,6 +8,7 @@ import { Client } from '@elastic/elasticsearch'; import seedrandom from 'seedrandom'; import { KbnClient } from '@kbn/dev-utils'; import { AxiosResponse } from 'axios'; +import fetch from 'node-fetch'; import { EndpointDocGenerator, TreeOptions, Event } from './generate_data'; import { firstNonNullValue } from './models/ecs_safety_helpers'; import { @@ -288,7 +289,13 @@ const fleetEnrollAgentForHost = async (kbnClient: KbnClient, host: HostMetadata) shared_id: host.elastic.agent.id, metadata: { local: { - ...host.host, + host: host.host, + elastic: { + agent: { + ...host.agent, + version: '8.0.0', + }, + }, }, user_provided: { dev_agent_version: '0.0.1', @@ -298,16 +305,39 @@ const fleetEnrollAgentForHost = async (kbnClient: KbnClient, host: HostMetadata) }; try { - await kbnClient.request({ - path: AGENT_API_ROUTES.ENROLL_PATTERN, + // FIXME: should we use `fetch()` in this module? + // FIXME: need way to get kibana URL without auth in it + const res = await fetch(`http://localhost:5601${AGENT_API_ROUTES.ENROLL_PATTERN}`, { method: 'POST', - body, + body: JSON.stringify(body), headers: { + 'kbn-xsrf': 'xxx', Authorization: `ApiKey ${enrollmentApiKey}`, 'Content-Type': 'application/json', }, }); + + if (res) { + const obj: PostAgentEnrollResponse = await res.json(); + if (!obj.success) { + // eslint-disable-next-line no-console + console.error(obj); + } + } + + // THIS DOES NOT WORK because its sent with basic auth (Authorization header is stripped out) + // + // await kbnClient.request({ + // path: AGENT_API_ROUTES.ENROLL_PATTERN, + // method: 'POST', + // body, + // headers: { + // Authorization: `ApiKey ${enrollmentApiKey}`, + // 'Content-Type': 'application/json', + // }, + // }); } catch (error) { - // debugger; + // eslint-disable-next-line no-console + console.error(error); } }; From a8221bb033f9e4f264cc7cbe302c7c062b55db0b Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 4 Aug 2020 17:37:49 -0400 Subject: [PATCH 07/24] SUCCESS. Able to enroll agent and set it to online --- .../common/endpoint/index_data.ts | 228 +++++++++++++----- 1 file changed, 165 insertions(+), 63 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index bcedf49163750..365ba62963afc 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -24,9 +24,12 @@ import { PACKAGE_CONFIG_API_ROUTES, ENROLLMENT_API_KEY_ROUTES, GetEnrollmentAPIKeysResponse, - GetOnePackageConfigResponse, GetOneEnrollmentAPIKeyResponse, PostAgentEnrollResponse, + PostAgentCheckinRequest, + PostAgentCheckinResponse, + PostAgentAcksResponse, + PostAgentAcksRequest, } from '../../../ingest_manager/common'; import { factory as policyConfigFactory } from './models/policy_config'; import { HostMetadata } from './types'; @@ -89,6 +92,8 @@ async function indexHostDocs( const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents const timestamp = new Date().getTime(); let hostMetadata: HostMetadata; + let wasAgentEnrolled = false; + let enrolledAgent: undefined | PostAgentEnrollResponse['item']; for (let j = 0; j < numDocs; j++) { generator.updateHostData(); @@ -97,32 +102,47 @@ async function indexHostDocs( hostMetadata = generator.generateHostMetadata(timestamp - timeBetweenDocs * (numDocs - j - 1)); const { id: appliedPolicyId, name: appliedPolicyName } = hostMetadata.Endpoint.policy.applied; - if (appliedPolicyId !== '00000000-0000-0000-0000-000000000000') { - // If we don't yet have a "real" policy record, then create it now in ingest (package config) - if (!realPolicies[appliedPolicyId]) { - // eslint-disable-next-line require-atomic-updates - realPolicies[appliedPolicyId] = await createPolicy( - kbnClient, - appliedPolicyName, - epmEndpointPackage.version - ); - } + // If we don't yet have a "real" policy record, then create it now in ingest (package config) + if (!realPolicies[appliedPolicyId]) { + // eslint-disable-next-line require-atomic-updates + realPolicies[appliedPolicyId] = await createPolicy( + kbnClient, + appliedPolicyName, + epmEndpointPackage.version + ); + } - // Update the Host metadata record with the ID of the "real" policy - hostMetadata = { - ...hostMetadata, - Endpoint: { - ...hostMetadata.Endpoint, - policy: { - ...hostMetadata.Endpoint.policy, - applied: { - ...hostMetadata.Endpoint.policy.applied, - id: realPolicies[appliedPolicyId].id, - }, + // If we did not yet enroll an agent for this Host, do it now that we have good policy id + if (!wasAgentEnrolled) { + wasAgentEnrolled = true; + enrolledAgent = await fleetEnrollAgentForHost( + kbnClient, + hostMetadata!, + realPolicies[appliedPolicyId].config_id + ); + } + + // Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id + hostMetadata = { + ...hostMetadata, + elastic: { + ...hostMetadata.elastic, + agent: { + ...hostMetadata.elastic.agent, + id: enrolledAgent?.id ?? hostMetadata.elastic.agent.id, + }, + }, + Endpoint: { + ...hostMetadata.Endpoint, + policy: { + ...hostMetadata.Endpoint.policy, + applied: { + ...hostMetadata.Endpoint.policy.applied, + id: realPolicies[appliedPolicyId].id, }, }, - }; - } + }, + }; await client.index({ index: metadataIndex, @@ -135,10 +155,6 @@ async function indexHostDocs( op_type: 'create', }); } - - if (hostMetadata!.Endpoint.policy.applied.id !== '00000000-0000-0000-0000-000000000000') { - await fleetEnrollAgentForHost(kbnClient, hostMetadata!); - } } async function indexAlerts( @@ -243,29 +259,28 @@ const getEndpointPackageInfo = async ( return endpointPackage; }; -const fleetEnrollAgentForHost = async (kbnClient: KbnClient, host: HostMetadata) => { +const fleetEnrollAgentForHost = async ( + kbnClient: KbnClient, + endpointHost: HostMetadata, + agentConfigId: string +): Promise => { // Get Enrollement key for host's applied policy const enrollmentApiKey = await kbnClient - .request({ - path: PACKAGE_CONFIG_API_ROUTES.INFO_PATTERN.replace( - '{packageConfigId}', - host.Endpoint.policy.applied.id - ), + .request({ + path: ENROLLMENT_API_KEY_ROUTES.LIST_PATTERN, method: 'GET', - }) - .then((packageConfigResponse) => { - return kbnClient.request({ - path: ENROLLMENT_API_KEY_ROUTES.LIST_PATTERN, - method: 'GET', - query: { - kuery: `fleet-enrollment-api-keys.config_id:"${packageConfigResponse.data.item.config_id}"`, - }, - }); + query: { + kuery: `fleet-enrollment-api-keys.config_id:"${agentConfigId}"`, + }, }) .then((apiKeysResponse) => { const apiKey = apiKeysResponse.data.list[0]; - // TODO: Handle if it does not exist + if (!apiKey) { + return Promise.reject( + new Error(`no API enrolment key found for agent config id ${agentConfigId}`) + ); + } return kbnClient.request({ path: ENROLLMENT_API_KEY_ROUTES.INFO_PATTERN.replace('{keyId}', apiKey.id), @@ -275,7 +290,9 @@ const fleetEnrollAgentForHost = async (kbnClient: KbnClient, host: HostMetadata) .then((apiKeyDetailsResponse) => { return apiKeyDetailsResponse.data.item.api_key; }) - .catch(() => { + .catch((error) => { + // eslint-disable-next-line no-console + console.error(error); return ''; }); @@ -286,13 +303,11 @@ const fleetEnrollAgentForHost = async (kbnClient: KbnClient, host: HostMetadata) // Enroll an agent for the Host const body: PostAgentEnrollRequest['body'] = { type: 'PERMANENT', - shared_id: host.elastic.agent.id, metadata: { local: { - host: host.host, + host: endpointHost.host, elastic: { agent: { - ...host.agent, version: '8.0.0', }, }, @@ -307,6 +322,9 @@ const fleetEnrollAgentForHost = async (kbnClient: KbnClient, host: HostMetadata) try { // FIXME: should we use `fetch()` in this module? // FIXME: need way to get kibana URL without auth in it + + // ------------------------------------------------ + // First enroll the agent const res = await fetch(`http://localhost:5601${AGENT_API_ROUTES.ENROLL_PATTERN}`, { method: 'POST', body: JSON.stringify(body), @@ -318,24 +336,108 @@ const fleetEnrollAgentForHost = async (kbnClient: KbnClient, host: HostMetadata) }); if (res) { - const obj: PostAgentEnrollResponse = await res.json(); - if (!obj.success) { + const enrollObj: PostAgentEnrollResponse = await res.json(); + if (!enrollObj.success) { // eslint-disable-next-line no-console - console.error(obj); + console.error(enrollObj); + return; } - } - // THIS DOES NOT WORK because its sent with basic auth (Authorization header is stripped out) - // - // await kbnClient.request({ - // path: AGENT_API_ROUTES.ENROLL_PATTERN, - // method: 'POST', - // body, - // headers: { - // Authorization: `ApiKey ${enrollmentApiKey}`, - // 'Content-Type': 'application/json', - // }, - // }); + // ------------------------------------------------ + // now check the agent in so that it can complete enrollment + const checkinBody: PostAgentCheckinRequest['body'] = { + events: [ + { + type: 'STATE', + subtype: 'RUNNING', + message: 'state changed from STOPPED to RUNNING', + timestamp: new Date().toISOString(), + payload: { + random: 'data', + state: 'RUNNING', + previous_state: 'STOPPED', + }, + agent_id: enrollObj.item.id, + }, + ], + }; + const checkinRes = await fetch( + `http://localhost:5601${AGENT_API_ROUTES.CHECKIN_PATTERN.replace( + '{agentId}', + enrollObj.item.id + )}`, + { + method: 'POST', + body: JSON.stringify(checkinBody), + headers: { + 'kbn-xsrf': 'xxx', + Authorization: `ApiKey ${enrollObj.item.access_api_key}`, + 'Content-Type': 'application/json', + }, + } + ); + + // Agent unenrolling? + if (checkinRes.status === 403) { + return; + } + + const checkinObj: PostAgentCheckinResponse = await checkinRes.json(); + if (!checkinObj.success) { + // eslint-disable-next-line no-console + console.error( + `failed to checkin agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` + ); + return enrollObj.item; + } + + // ------------------------------------------------ + // If we have an action to ack(), then do it now + if (checkinObj.actions.length) { + const ackActionBody: PostAgentAcksRequest['body'] = { + // @ts-ignore + events: checkinObj.actions.map((action) => { + return { + action_id: action.id, + type: 'ACTION_RESULT', + subtype: 'CONFIG', + timestamp: new Date().toISOString(), + agent_id: action.agent_id, + config_id: agentConfigId, + message: `endpoint generator: Endpoint Started`, + }; + }), + }; + const ackActionResp = await fetch( + `http://localhost:5601${AGENT_API_ROUTES.ACKS_PATTERN.replace( + '{agentId}', + enrollObj.item.id + )}`, + { + method: 'POST', + body: JSON.stringify(ackActionBody), + headers: { + 'kbn-xsrf': 'xxx', + Authorization: `ApiKey ${enrollObj.item.access_api_key}`, + 'Content-Type': 'application/json', + }, + } + ); + + const ackActionObj: PostAgentAcksResponse = await ackActionResp.json(); + if (!ackActionObj.success) { + // eslint-disable-next-line no-console + console.error( + `failed to ACK Actions provided to agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` + ); + // eslint-disable-next-line no-console + console.error(JSON.stringify(ackActionObj, null, 2)); + return enrollObj.item; + } + } + + return enrollObj.item; + } } catch (error) { // eslint-disable-next-line no-console console.error(error); From e3ef5096de26934ac39c172ecfadf5e6c717a987 Mon Sep 17 00:00:00 2001 From: kevinlog Date: Sun, 23 Aug 2020 15:55:28 -0400 Subject: [PATCH 08/24] update names to be policy --- .../common/endpoint/index_data.ts | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 365ba62963afc..07c4186472ef7 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -12,16 +12,16 @@ import fetch from 'node-fetch'; import { EndpointDocGenerator, TreeOptions, Event } from './generate_data'; import { firstNonNullValue } from './models/ecs_safety_helpers'; import { - CreateAgentConfigRequest, - CreateAgentConfigResponse, - CreatePackageConfigRequest, - CreatePackageConfigResponse, + CreateAgentPolicyRequest, + CreateAgentPolicyResponse, + CreatePackagePolicyRequest, + CreatePackagePolicyResponse, GetPackagesResponse, PostAgentEnrollRequest, AGENT_API_ROUTES, - AGENT_CONFIG_API_ROUTES, + AGENT_POLICY_API_ROUTES, EPM_API_ROUTES, - PACKAGE_CONFIG_API_ROUTES, + PACKAGE_POLICY_API_ROUTES, ENROLLMENT_API_KEY_ROUTES, GetEnrollmentAPIKeysResponse, GetOneEnrollmentAPIKeyResponse, @@ -50,7 +50,7 @@ export async function indexHostsAndAlerts( const random = seedrandom(seed); const epmEndpointPackage = await getEndpointPackageInfo(kbnClient); // Keep a map of host applied policy ids (fake) to real ingest package configs (policy record) - const realPolicies: Record = {}; + const realPolicies: Record = {}; for (let i = 0; i < numHosts; i++) { const generator = new EndpointDocGenerator(random); @@ -83,7 +83,7 @@ async function indexHostDocs( numDocs: number, client: Client, kbnClient: KbnClient, - realPolicies: Record, + realPolicies: Record, epmEndpointPackage: GetPackagesResponse['response'][0], metadataIndex: string, policyResponseIndex: string, @@ -118,7 +118,7 @@ async function indexHostDocs( enrolledAgent = await fleetEnrollAgentForHost( kbnClient, hostMetadata!, - realPolicies[appliedPolicyId].config_id + realPolicies[appliedPolicyId].policy_id ); } @@ -195,24 +195,29 @@ const createPolicy = async ( kbnClient: KbnClient, policyName: string, endpointPackageVersion: string -): Promise => { - // Create Agent Configuration first - const newAgentconfigData: CreateAgentConfigRequest['body'] = { - name: `Config for ${policyName}`, +): Promise => { + // Create Agent Policy first + const newAgentPolicyData: CreateAgentPolicyRequest['body'] = { + name: `Policy for ${policyName}`, description: '', namespace: 'default', }; - const agentConfig = (await kbnClient.request({ - path: AGENT_CONFIG_API_ROUTES.CREATE_PATTERN, - method: 'POST', - body: newAgentconfigData, - })) as AxiosResponse; + let agentPolicy; + try { + agentPolicy = (await kbnClient.request({ + path: AGENT_POLICY_API_ROUTES.CREATE_PATTERN, + method: 'POST', + body: newAgentPolicyData, + })) as AxiosResponse; + } catch (error) { + throw new Error(`create policy ${error}`); + } // Create Package Configuration - const newPackageConfigData: CreatePackageConfigRequest['body'] = { + const newPackagePolicyData: CreatePackagePolicyRequest['body'] = { name: policyName, description: 'Protect the worlds data', - config_id: agentConfig.data.item.id, + policy_id: agentPolicy.data.item.id, enabled: true, output_id: '', inputs: [ @@ -234,12 +239,12 @@ const createPolicy = async ( version: endpointPackageVersion, }, }; - const packageConfig = (await kbnClient.request({ - path: PACKAGE_CONFIG_API_ROUTES.CREATE_PATTERN, + const packagePolicy = (await kbnClient.request({ + path: PACKAGE_POLICY_API_ROUTES.CREATE_PATTERN, method: 'POST', - body: newPackageConfigData, - })) as AxiosResponse; - return packageConfig.data.item; + body: newPackagePolicyData, + })) as AxiosResponse; + return packagePolicy.data.item; }; const getEndpointPackageInfo = async ( @@ -262,7 +267,7 @@ const getEndpointPackageInfo = async ( const fleetEnrollAgentForHost = async ( kbnClient: KbnClient, endpointHost: HostMetadata, - agentConfigId: string + agentPolicyId: string ): Promise => { // Get Enrollement key for host's applied policy const enrollmentApiKey = await kbnClient @@ -270,7 +275,7 @@ const fleetEnrollAgentForHost = async ( path: ENROLLMENT_API_KEY_ROUTES.LIST_PATTERN, method: 'GET', query: { - kuery: `fleet-enrollment-api-keys.config_id:"${agentConfigId}"`, + kuery: `fleet-enrollment-api-keys.policy_id:"${agentPolicyId}"`, }, }) .then((apiKeysResponse) => { @@ -278,7 +283,7 @@ const fleetEnrollAgentForHost = async ( if (!apiKey) { return Promise.reject( - new Error(`no API enrolment key found for agent config id ${agentConfigId}`) + new Error(`no API enrolment key found for agent policy id ${agentPolicyId}`) ); } @@ -403,7 +408,7 @@ const fleetEnrollAgentForHost = async ( subtype: 'CONFIG', timestamp: new Date().toISOString(), agent_id: action.agent_id, - config_id: agentConfigId, + policy_id: agentPolicyId, message: `endpoint generator: Endpoint Started`, }; }), From 4deebbf5c00f50757402a83577d0e7c3a3c3d98c Mon Sep 17 00:00:00 2001 From: Candace Park Date: Mon, 31 Aug 2020 12:35:40 -0400 Subject: [PATCH 09/24] policy generator has advanced types in endpoint confit --- .../common/endpoint/models/policy_config.ts | 24 +++++++++++++++++++ .../common/endpoint/types/index.ts | 11 +++++++++ 2 files changed, 35 insertions(+) diff --git a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts index 37b7308856196..a2a024e3e5c0c 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts @@ -12,6 +12,14 @@ import { PolicyConfig, ProtectionModes } from '../types'; export const factory = (): PolicyConfig => { return { windows: { + advanced: { + elasticsearch: { + tls: { + verify_hostname: 'false', + verify_peer: '', + }, + }, + }, events: { dll_and_driver_load: true, dns: true, @@ -29,6 +37,14 @@ export const factory = (): PolicyConfig => { }, }, mac: { + advanced: { + elasticsearch: { + tls: { + verify_hostname: '', + verify_peer: '', + }, + }, + }, events: { process: true, file: true, @@ -42,6 +58,14 @@ export const factory = (): PolicyConfig => { }, }, linux: { + advanced: { + elasticsearch: { + tls: { + verify_hostname: '', + verify_peer: '', + }, + }, + }, events: { process: true, file: true, diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index 6afec75903477..c762126c69aa5 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -843,6 +843,7 @@ type KbnConfigSchemaNonOptionalProps> = Pi */ export interface PolicyConfig { windows: { + advanced: AdvancedFields; events: { dll_and_driver_load: boolean; dns: boolean; @@ -858,6 +859,7 @@ export interface PolicyConfig { }; }; mac: { + advanced: AdvancedFields; events: { file: boolean; process: boolean; @@ -869,6 +871,7 @@ export interface PolicyConfig { }; }; linux: { + advanced: AdvancedFields; events: { file: boolean; process: boolean; @@ -880,6 +883,14 @@ export interface PolicyConfig { }; } +export interface AdvancedFields { + elasticsearch: { + tls: { + verify_hostname: string; + verify_peer: string; + }; + }; +} /** * The set of Policy configuration settings that are show/edited via the UI */ From 0e61d00303cb2e049c1f7c243a99d050951de941 Mon Sep 17 00:00:00 2001 From: Candace Park Date: Wed, 16 Sep 2020 17:40:36 -0400 Subject: [PATCH 10/24] use KbnClientWithRequestAPI --- .../common/endpoint/index_data.ts | 38 +++++++++++-------- .../endpoint/resolver_generator_script.ts | 19 +++++++++- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 07c4186472ef7..78b4f8ba2554b 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -8,7 +8,6 @@ import { Client } from '@elastic/elasticsearch'; import seedrandom from 'seedrandom'; import { KbnClient } from '@kbn/dev-utils'; import { AxiosResponse } from 'axios'; -import fetch from 'node-fetch'; import { EndpointDocGenerator, TreeOptions, Event } from './generate_data'; import { firstNonNullValue } from './models/ecs_safety_helpers'; import { @@ -36,7 +35,9 @@ import { HostMetadata } from './types'; export async function indexHostsAndAlerts( client: Client, - kbnClient: KbnClient, + kbnClient: KbnClient & { + requestWithApiKey: (path: string, init?: RequestInit | undefined) => Promise; + }, seed: string, numHosts: number, numDocs: number, @@ -82,7 +83,9 @@ function delay(ms: number) { async function indexHostDocs( numDocs: number, client: Client, - kbnClient: KbnClient, + kbnClient: KbnClient & { + requestWithApiKey: (path: string, init?: RequestInit | undefined) => Promise; + }, realPolicies: Record, epmEndpointPackage: GetPackagesResponse['response'][0], metadataIndex: string, @@ -265,7 +268,9 @@ const getEndpointPackageInfo = async ( }; const fleetEnrollAgentForHost = async ( - kbnClient: KbnClient, + kbnClient: KbnClient & { + requestWithApiKey: (path: string, init?: RequestInit | undefined) => Promise; + }, endpointHost: HostMetadata, agentPolicyId: string ): Promise => { @@ -330,15 +335,18 @@ const fleetEnrollAgentForHost = async ( // ------------------------------------------------ // First enroll the agent - const res = await fetch(`http://localhost:5601${AGENT_API_ROUTES.ENROLL_PATTERN}`, { - method: 'POST', - body: JSON.stringify(body), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollmentApiKey}`, - 'Content-Type': 'application/json', - }, - }); + const res = await kbnClient.requestWithApiKey( + `http://localhost:5601${AGENT_API_ROUTES.ENROLL_PATTERN}`, + { + method: 'POST', + body: JSON.stringify(body), + headers: { + 'kbn-xsrf': 'xxx', + Authorization: `ApiKey ${enrollmentApiKey}`, + 'Content-Type': 'application/json', + }, + } + ); if (res) { const enrollObj: PostAgentEnrollResponse = await res.json(); @@ -366,7 +374,7 @@ const fleetEnrollAgentForHost = async ( }, ], }; - const checkinRes = await fetch( + const checkinRes = await kbnClient.requestWithApiKey( `http://localhost:5601${AGENT_API_ROUTES.CHECKIN_PATTERN.replace( '{agentId}', enrollObj.item.id @@ -413,7 +421,7 @@ const fleetEnrollAgentForHost = async ( }; }), }; - const ackActionResp = await fetch( + const ackActionResp = await kbnClient.requestWithApiKey( `http://localhost:5601${AGENT_API_ROUTES.ACKS_PATTERN.replace( '{agentId}', enrollObj.item.id diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index c3afb38d04952..723e94c7305e5 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -9,6 +9,8 @@ import { Client, ClientOptions } from '@elastic/elasticsearch'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { KbnClient, ToolingLog } from '@kbn/dev-utils'; import { AxiosResponse } from 'axios'; +import { KibanaConfig } from '@kbn/dev-utils/target/kbn_client/kbn_client_requester'; +import fetch from 'node-fetch'; import { indexHostsAndAlerts } from '../../common/endpoint/index_data'; import { ANCESTRY_LIMIT } from '../../common/endpoint/generate_data'; import { FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../../ingest_manager/common/constants'; @@ -17,6 +19,21 @@ import { PostIngestSetupResponse, } from '../../../ingest_manager/common/types/rest_spec'; +class KbnClientWithApiKeySupport extends KbnClient { + private kibanaUrlNoAuth: string; + constructor(log: ToolingLog, kibanaConfig: KibanaConfig) { + super(log, kibanaConfig); + // is resolveUrl necessary? + const kibanaUrl = this.resolveUrl(kibanaConfig.url); + const matches = kibanaUrl.match(/(https?:\/\/)(.*\:.*\@)(.*)/); + // strip auth from url + this.kibanaUrlNoAuth = matches && matches.length === 3 ? matches[1] + matches[3] : kibanaUrl; + } + requestWithApiKey(path: string, init?: RequestInit | undefined) { + return fetch(`${this.kibanaUrlNoAuth}/${path}`, init); + } +} + main(); async function deleteIndices(indices: string[], client: Client) { @@ -196,7 +213,7 @@ async function main() { default: false, }, }).argv; - const kbnClient = new KbnClient(new ToolingLog(), { url: argv.kibana }); + const kbnClient = new KbnClientWithApiKeySupport(new ToolingLog(), { url: argv.kibana }); try { await doIngestSetup(kbnClient); From dfb96272e712989cc2630ef9994f824daf33ab8c Mon Sep 17 00:00:00 2001 From: Candace Park Date: Thu, 17 Sep 2020 16:52:30 -0400 Subject: [PATCH 11/24] fix typecheck errors --- .../security_solution/common/endpoint/index_data.ts | 6 +++--- .../scripts/endpoint/resolver_generator_script.ts | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 78b4f8ba2554b..859f15108b101 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -350,7 +350,7 @@ const fleetEnrollAgentForHost = async ( if (res) { const enrollObj: PostAgentEnrollResponse = await res.json(); - if (!enrollObj.success) { + if (!res.ok) { // eslint-disable-next-line no-console console.error(enrollObj); return; @@ -396,7 +396,7 @@ const fleetEnrollAgentForHost = async ( } const checkinObj: PostAgentCheckinResponse = await checkinRes.json(); - if (!checkinObj.success) { + if (!checkinRes.ok) { // eslint-disable-next-line no-console console.error( `failed to checkin agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` @@ -438,7 +438,7 @@ const fleetEnrollAgentForHost = async ( ); const ackActionObj: PostAgentAcksResponse = await ackActionResp.json(); - if (!ackActionObj.success) { + if (!ackActionResp.ok) { // eslint-disable-next-line no-console console.error( `failed to ACK Actions provided to agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index 723e94c7305e5..fa64ba4dfe727 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -10,7 +10,7 @@ import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { KbnClient, ToolingLog } from '@kbn/dev-utils'; import { AxiosResponse } from 'axios'; import { KibanaConfig } from '@kbn/dev-utils/target/kbn_client/kbn_client_requester'; -import fetch from 'node-fetch'; +import fetch, { RequestInit as FetchRequestInit } from 'node-fetch'; import { indexHostsAndAlerts } from '../../common/endpoint/index_data'; import { ANCESTRY_LIMIT } from '../../common/endpoint/generate_data'; import { FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../../ingest_manager/common/constants'; @@ -29,8 +29,11 @@ class KbnClientWithApiKeySupport extends KbnClient { // strip auth from url this.kibanaUrlNoAuth = matches && matches.length === 3 ? matches[1] + matches[3] : kibanaUrl; } - requestWithApiKey(path: string, init?: RequestInit | undefined) { - return fetch(`${this.kibanaUrlNoAuth}/${path}`, init); + requestWithApiKey(path: string, init?: RequestInit | undefined): Promise { + return (fetch( + `${this.kibanaUrlNoAuth}/${path}`, + init as FetchRequestInit + ) as unknown) as Promise; } } From 76db23b31d90060362d18a191234f99425d6a8fb Mon Sep 17 00:00:00 2001 From: Candace Park Date: Thu, 17 Sep 2020 18:13:14 -0400 Subject: [PATCH 12/24] add flag to skip fleet --- .../common/endpoint/index_data.ts | 21 ++++++++++++------- .../endpoint/resolver_generator_script.ts | 7 +++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 859f15108b101..5aed9c8d0c675 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -46,6 +46,7 @@ export async function indexHostsAndAlerts( eventIndex: string, alertIndex: string, alertsPerHost: number, + fleet: boolean, options: TreeOptions = {} ) { const random = seedrandom(seed); @@ -63,6 +64,7 @@ export async function indexHostsAndAlerts( epmEndpointPackage, metadataIndex, policyResponseIndex, + fleet, generator ); await indexAlerts(client, eventIndex, alertIndex, generator, alertsPerHost, options); @@ -90,6 +92,7 @@ async function indexHostDocs( epmEndpointPackage: GetPackagesResponse['response'][0], metadataIndex: string, policyResponseIndex: string, + enrollFleet: boolean, generator: EndpointDocGenerator ) { const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents @@ -115,14 +118,16 @@ async function indexHostDocs( ); } - // If we did not yet enroll an agent for this Host, do it now that we have good policy id - if (!wasAgentEnrolled) { - wasAgentEnrolled = true; - enrolledAgent = await fleetEnrollAgentForHost( - kbnClient, - hostMetadata!, - realPolicies[appliedPolicyId].policy_id - ); + if (enrollFleet) { + // If we did not yet enroll an agent for this Host, do it now that we have good policy id + if (!wasAgentEnrolled) { + wasAgentEnrolled = true; + enrolledAgent = await fleetEnrollAgentForHost( + kbnClient, + hostMetadata!, + realPolicies[appliedPolicyId].policy_id + ); + } } // Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index fa64ba4dfe727..dfebc4a5ce7aa 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -215,6 +215,12 @@ async function main() { type: 'boolean', default: false, }, + fleet: { + alias: 'f', + describe: 'enroll fleet agents for hosts', + type: 'boolean', + default: true, + }, }).argv; const kbnClient = new KbnClientWithApiKeySupport(new ToolingLog(), { url: argv.kibana }); @@ -252,6 +258,7 @@ async function main() { argv.eventIndex, argv.alertIndex, argv.alertsPerHost, + argv.fleet, { ancestors: argv.ancestors, generations: argv.generations, From 848eec584db6be524747b6e2c71794b4b1eee3c0 Mon Sep 17 00:00:00 2001 From: Candace Park Date: Mon, 21 Sep 2020 11:16:55 -0400 Subject: [PATCH 13/24] fix tests --- .../common/endpoint/models/policy_config.ts | 2 +- .../policy/store/policy_details/index.test.ts | 24 +++++++++++++++++++ .../apps/endpoint/policy_details.ts | 24 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts index a2a024e3e5c0c..e6d21485fa5e9 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts @@ -15,7 +15,7 @@ export const factory = (): PolicyConfig => { advanced: { elasticsearch: { tls: { - verify_hostname: 'false', + verify_hostname: '', verify_peer: '', }, }, diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index 0eedecef22170..32aab58fad76d 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -228,6 +228,14 @@ describe('policy details: ', () => { policy: { value: { windows: { + advanced: { + elasticsearch: { + tls: { + verify_hostname: '', + verify_peer: '', + }, + }, + }, events: { dll_and_driver_load: true, dns: false, @@ -241,11 +249,27 @@ describe('policy details: ', () => { logging: { file: 'info' }, }, mac: { + advanced: { + elasticsearch: { + tls: { + verify_hostname: '', + verify_peer: '', + }, + }, + }, events: { process: true, file: true, network: true }, malware: { mode: 'prevent' }, logging: { file: 'info' }, }, linux: { + advanced: { + elasticsearch: { + tls: { + verify_hostname: '', + verify_peer: '', + }, + }, + }, events: { process: true, file: true, network: true }, logging: { file: 'info' }, }, diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index 325283f5e3440..717ccedda9ca3 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -199,15 +199,39 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }, policy: { linux: { + advanced: { + elasticsearch: { + tls: { + verify_hostname: '', + verify_peer: '', + }, + }, + }, events: { file: false, network: true, process: true }, logging: { file: 'info' }, }, mac: { + advanced: { + elasticsearch: { + tls: { + verify_hostname: '', + verify_peer: '', + }, + }, + }, events: { file: false, network: true, process: true }, logging: { file: 'info' }, malware: { mode: 'prevent' }, }, windows: { + advanced: { + elasticsearch: { + tls: { + verify_hostname: '', + verify_peer: '', + }, + }, + }, events: { dll_and_driver_load: true, dns: true, From 4caaf963349c029852a4d1c306a232d23cc63baa Mon Sep 17 00:00:00 2001 From: Candace Park Date: Tue, 22 Sep 2020 10:03:28 -0400 Subject: [PATCH 14/24] make advanced field type optional --- .../security_solution/common/endpoint/types/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index c762126c69aa5..db14b6bda7ea7 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -843,7 +843,7 @@ type KbnConfigSchemaNonOptionalProps> = Pi */ export interface PolicyConfig { windows: { - advanced: AdvancedFields; + advanced?: AdvancedFields; // will be introduced in 7.11+ events: { dll_and_driver_load: boolean; dns: boolean; @@ -859,7 +859,7 @@ export interface PolicyConfig { }; }; mac: { - advanced: AdvancedFields; + advanced?: AdvancedFields; events: { file: boolean; process: boolean; @@ -871,7 +871,7 @@ export interface PolicyConfig { }; }; linux: { - advanced: AdvancedFields; + advanced?: AdvancedFields; events: { file: boolean; process: boolean; From 02cd8fa4425a7ca547b63e18e9aa69653b93788d Mon Sep 17 00:00:00 2001 From: Candace Park Date: Tue, 22 Sep 2020 15:53:07 -0400 Subject: [PATCH 15/24] remove advanced fields --- .../common/endpoint/models/policy_config.ts | 24 ------------------- .../common/endpoint/types/index.ts | 11 --------- .../apps/endpoint/policy_details.ts | 24 ------------------- 3 files changed, 59 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts index e6d21485fa5e9..37b7308856196 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts @@ -12,14 +12,6 @@ import { PolicyConfig, ProtectionModes } from '../types'; export const factory = (): PolicyConfig => { return { windows: { - advanced: { - elasticsearch: { - tls: { - verify_hostname: '', - verify_peer: '', - }, - }, - }, events: { dll_and_driver_load: true, dns: true, @@ -37,14 +29,6 @@ export const factory = (): PolicyConfig => { }, }, mac: { - advanced: { - elasticsearch: { - tls: { - verify_hostname: '', - verify_peer: '', - }, - }, - }, events: { process: true, file: true, @@ -58,14 +42,6 @@ export const factory = (): PolicyConfig => { }, }, linux: { - advanced: { - elasticsearch: { - tls: { - verify_hostname: '', - verify_peer: '', - }, - }, - }, events: { process: true, file: true, diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index db14b6bda7ea7..6afec75903477 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -843,7 +843,6 @@ type KbnConfigSchemaNonOptionalProps> = Pi */ export interface PolicyConfig { windows: { - advanced?: AdvancedFields; // will be introduced in 7.11+ events: { dll_and_driver_load: boolean; dns: boolean; @@ -859,7 +858,6 @@ export interface PolicyConfig { }; }; mac: { - advanced?: AdvancedFields; events: { file: boolean; process: boolean; @@ -871,7 +869,6 @@ export interface PolicyConfig { }; }; linux: { - advanced?: AdvancedFields; events: { file: boolean; process: boolean; @@ -883,14 +880,6 @@ export interface PolicyConfig { }; } -export interface AdvancedFields { - elasticsearch: { - tls: { - verify_hostname: string; - verify_peer: string; - }; - }; -} /** * The set of Policy configuration settings that are show/edited via the UI */ diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index 717ccedda9ca3..325283f5e3440 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -199,39 +199,15 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }, policy: { linux: { - advanced: { - elasticsearch: { - tls: { - verify_hostname: '', - verify_peer: '', - }, - }, - }, events: { file: false, network: true, process: true }, logging: { file: 'info' }, }, mac: { - advanced: { - elasticsearch: { - tls: { - verify_hostname: '', - verify_peer: '', - }, - }, - }, events: { file: false, network: true, process: true }, logging: { file: 'info' }, malware: { mode: 'prevent' }, }, windows: { - advanced: { - elasticsearch: { - tls: { - verify_hostname: '', - verify_peer: '', - }, - }, - }, events: { dll_and_driver_load: true, dns: true, From 58ef6d5d1284b2c66dfb54ba99c3168f9623c2fc Mon Sep 17 00:00:00 2001 From: Candace Park Date: Tue, 22 Sep 2020 16:00:45 -0400 Subject: [PATCH 16/24] another advanced removal --- .../policy/store/policy_details/index.test.ts | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index 32aab58fad76d..0eedecef22170 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -228,14 +228,6 @@ describe('policy details: ', () => { policy: { value: { windows: { - advanced: { - elasticsearch: { - tls: { - verify_hostname: '', - verify_peer: '', - }, - }, - }, events: { dll_and_driver_load: true, dns: false, @@ -249,27 +241,11 @@ describe('policy details: ', () => { logging: { file: 'info' }, }, mac: { - advanced: { - elasticsearch: { - tls: { - verify_hostname: '', - verify_peer: '', - }, - }, - }, events: { process: true, file: true, network: true }, malware: { mode: 'prevent' }, logging: { file: 'info' }, }, linux: { - advanced: { - elasticsearch: { - tls: { - verify_hostname: '', - verify_peer: '', - }, - }, - }, events: { process: true, file: true, network: true }, logging: { file: 'info' }, }, From 7b82e054c2f1dc56b815a291a0f1f06f93b71ae6 Mon Sep 17 00:00:00 2001 From: Candace Park Date: Tue, 22 Sep 2020 16:27:10 -0400 Subject: [PATCH 17/24] revert experimental change --- packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts index f4d7c9b7f287d..3ad152a6d3288 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts @@ -60,7 +60,6 @@ export interface ReqOptions { path: string; query?: Record; method: 'GET' | 'POST' | 'PUT' | 'DELETE'; - headers?: Record; body?: any; retries?: number; } From e8ead7fa5a9c6bfd7899dcd06e71154b6bbc56ce Mon Sep 17 00:00:00 2001 From: Candace Park Date: Tue, 22 Sep 2020 16:44:35 -0400 Subject: [PATCH 18/24] shorten urls that use requestWithApiKey --- .../common/endpoint/index_data.ts | 31 +++++++------------ 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 5aed9c8d0c675..d060ffb257f47 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -340,18 +340,15 @@ const fleetEnrollAgentForHost = async ( // ------------------------------------------------ // First enroll the agent - const res = await kbnClient.requestWithApiKey( - `http://localhost:5601${AGENT_API_ROUTES.ENROLL_PATTERN}`, - { - method: 'POST', - body: JSON.stringify(body), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollmentApiKey}`, - 'Content-Type': 'application/json', - }, - } - ); + const res = await kbnClient.requestWithApiKey(`${AGENT_API_ROUTES.ENROLL_PATTERN}`, { + method: 'POST', + body: JSON.stringify(body), + headers: { + 'kbn-xsrf': 'xxx', + Authorization: `ApiKey ${enrollmentApiKey}`, + 'Content-Type': 'application/json', + }, + }); if (res) { const enrollObj: PostAgentEnrollResponse = await res.json(); @@ -380,10 +377,7 @@ const fleetEnrollAgentForHost = async ( ], }; const checkinRes = await kbnClient.requestWithApiKey( - `http://localhost:5601${AGENT_API_ROUTES.CHECKIN_PATTERN.replace( - '{agentId}', - enrollObj.item.id - )}`, + `${AGENT_API_ROUTES.CHECKIN_PATTERN.replace('{agentId}', enrollObj.item.id)}`, { method: 'POST', body: JSON.stringify(checkinBody), @@ -427,10 +421,7 @@ const fleetEnrollAgentForHost = async ( }), }; const ackActionResp = await kbnClient.requestWithApiKey( - `http://localhost:5601${AGENT_API_ROUTES.ACKS_PATTERN.replace( - '{agentId}', - enrollObj.item.id - )}`, + `${AGENT_API_ROUTES.ACKS_PATTERN.replace('{agentId}', enrollObj.item.id)}`, { method: 'POST', body: JSON.stringify(ackActionBody), From 65596eca13faa39dca7afaf6ea9941fc0a9315ac Mon Sep 17 00:00:00 2001 From: Candace Park Date: Wed, 23 Sep 2020 09:57:09 -0400 Subject: [PATCH 19/24] remove excess quaotes --- .../plugins/security_solution/common/endpoint/index_data.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index d060ffb257f47..c010d7e992ee7 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -340,7 +340,7 @@ const fleetEnrollAgentForHost = async ( // ------------------------------------------------ // First enroll the agent - const res = await kbnClient.requestWithApiKey(`${AGENT_API_ROUTES.ENROLL_PATTERN}`, { + const res = await kbnClient.requestWithApiKey(AGENT_API_ROUTES.ENROLL_PATTERN, { method: 'POST', body: JSON.stringify(body), headers: { @@ -377,7 +377,7 @@ const fleetEnrollAgentForHost = async ( ], }; const checkinRes = await kbnClient.requestWithApiKey( - `${AGENT_API_ROUTES.CHECKIN_PATTERN.replace('{agentId}', enrollObj.item.id)}`, + AGENT_API_ROUTES.CHECKIN_PATTERN.replace('{agentId}', enrollObj.item.id), { method: 'POST', body: JSON.stringify(checkinBody), @@ -421,7 +421,7 @@ const fleetEnrollAgentForHost = async ( }), }; const ackActionResp = await kbnClient.requestWithApiKey( - `${AGENT_API_ROUTES.ACKS_PATTERN.replace('{agentId}', enrollObj.item.id)}`, + AGENT_API_ROUTES.ACKS_PATTERN.replace('{agentId}', enrollObj.item.id), { method: 'POST', body: JSON.stringify(ackActionBody), From 2f4d043d3568576b8b98dfe669e1ae92d27ee9c4 Mon Sep 17 00:00:00 2001 From: Candace Park Date: Wed, 23 Sep 2020 16:56:24 -0400 Subject: [PATCH 20/24] fix ci errors --- packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts index 3ad152a6d3288..2aba2be56f277 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts @@ -117,7 +117,6 @@ export class KbnClientRequester { params: options.query, headers: { 'kbn-xsrf': 'kbn-client', - ...options.headers, }, httpsAgent: this.httpsAgent, }); From c3d92826b13eba73fe9352d124697feec4410132 Mon Sep 17 00:00:00 2001 From: Candace Park Date: Mon, 28 Sep 2020 11:00:44 -0400 Subject: [PATCH 21/24] agent enrollment fix --- .../common/endpoint/index_data.ts | 113 +++++++++--------- .../endpoint/resolver_generator_script.ts | 10 +- 2 files changed, 65 insertions(+), 58 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index c010d7e992ee7..4985d2c169c29 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -106,19 +106,20 @@ async function indexHostDocs( generator.updateHostPolicyData(); hostMetadata = generator.generateHostMetadata(timestamp - timeBetweenDocs * (numDocs - j - 1)); - const { id: appliedPolicyId, name: appliedPolicyName } = hostMetadata.Endpoint.policy.applied; - - // If we don't yet have a "real" policy record, then create it now in ingest (package config) - if (!realPolicies[appliedPolicyId]) { - // eslint-disable-next-line require-atomic-updates - realPolicies[appliedPolicyId] = await createPolicy( - kbnClient, - appliedPolicyName, - epmEndpointPackage.version - ); - } if (enrollFleet) { + const { id: appliedPolicyId, name: appliedPolicyName } = hostMetadata.Endpoint.policy.applied; + + // If we don't yet have a "real" policy record, then create it now in ingest (package config) + if (!realPolicies[appliedPolicyId]) { + // eslint-disable-next-line require-atomic-updates + realPolicies[appliedPolicyId] = await createPolicy( + kbnClient, + appliedPolicyName, + epmEndpointPackage.version + ); + } + // If we did not yet enroll an agent for this Host, do it now that we have good policy id if (!wasAgentEnrolled) { wasAgentEnrolled = true; @@ -128,29 +129,28 @@ async function indexHostDocs( realPolicies[appliedPolicyId].policy_id ); } - } - - // Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id - hostMetadata = { - ...hostMetadata, - elastic: { - ...hostMetadata.elastic, - agent: { - ...hostMetadata.elastic.agent, - id: enrolledAgent?.id ?? hostMetadata.elastic.agent.id, + // Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id + hostMetadata = { + ...hostMetadata, + elastic: { + ...hostMetadata.elastic, + agent: { + ...hostMetadata.elastic.agent, + id: enrolledAgent?.id ?? hostMetadata.elastic.agent.id, + }, }, - }, - Endpoint: { - ...hostMetadata.Endpoint, - policy: { - ...hostMetadata.Endpoint.policy, - applied: { - ...hostMetadata.Endpoint.policy.applied, - id: realPolicies[appliedPolicyId].id, + Endpoint: { + ...hostMetadata.Endpoint, + policy: { + ...hostMetadata.Endpoint.policy, + applied: { + ...hostMetadata.Endpoint.policy.applied, + id: realPolicies[appliedPolicyId].id, + }, }, }, - }, - }; + }; + } await client.index({ index: metadataIndex, @@ -293,14 +293,20 @@ const fleetEnrollAgentForHost = async ( if (!apiKey) { return Promise.reject( - new Error(`no API enrolment key found for agent policy id ${agentPolicyId}`) + new Error(`no API enrollment key found for agent policy id ${agentPolicyId}`) ); } - return kbnClient.request({ - path: ENROLLMENT_API_KEY_ROUTES.INFO_PATTERN.replace('{keyId}', apiKey.id), - method: 'GET', - }); + return kbnClient + .request({ + path: ENROLLMENT_API_KEY_ROUTES.INFO_PATTERN.replace('{keyId}', apiKey.id), + method: 'GET', + }) + .catch((error) => { + // eslint-disable-next-line no-console + console.log('unable to retrieve enrollment api key for policy'); + return Promise.reject(error); + }); }) .then((apiKeyDetailsResponse) => { return apiKeyDetailsResponse.data.item.api_key; @@ -335,10 +341,6 @@ const fleetEnrollAgentForHost = async ( }; try { - // FIXME: should we use `fetch()` in this module? - // FIXME: need way to get kibana URL without auth in it - - // ------------------------------------------------ // First enroll the agent const res = await kbnClient.requestWithApiKey(AGENT_API_ROUTES.ENROLL_PATTERN, { method: 'POST', @@ -354,10 +356,9 @@ const fleetEnrollAgentForHost = async ( const enrollObj: PostAgentEnrollResponse = await res.json(); if (!res.ok) { // eslint-disable-next-line no-console - console.error(enrollObj); + console.error('unable to enroll agent', enrollObj); return; } - // ------------------------------------------------ // now check the agent in so that it can complete enrollment const checkinBody: PostAgentCheckinRequest['body'] = { @@ -376,18 +377,22 @@ const fleetEnrollAgentForHost = async ( }, ], }; - const checkinRes = await kbnClient.requestWithApiKey( - AGENT_API_ROUTES.CHECKIN_PATTERN.replace('{agentId}', enrollObj.item.id), - { - method: 'POST', - body: JSON.stringify(checkinBody), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollObj.item.access_api_key}`, - 'Content-Type': 'application/json', - }, - } - ); + const checkinRes = await kbnClient + .requestWithApiKey( + AGENT_API_ROUTES.CHECKIN_PATTERN.replace('{agentId}', enrollObj.item.id), + { + method: 'POST', + body: JSON.stringify(checkinBody), + headers: { + 'kbn-xsrf': 'xxx', + Authorization: `ApiKey ${enrollObj.item.access_api_key}`, + 'Content-Type': 'application/json', + }, + } + ) + .catch((error) => { + return Promise.reject(error); + }); // Agent unenrolling? if (checkinRes.status === 403) { diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index dfebc4a5ce7aa..f705104e62d15 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -23,15 +23,17 @@ class KbnClientWithApiKeySupport extends KbnClient { private kibanaUrlNoAuth: string; constructor(log: ToolingLog, kibanaConfig: KibanaConfig) { super(log, kibanaConfig); - // is resolveUrl necessary? const kibanaUrl = this.resolveUrl(kibanaConfig.url); const matches = kibanaUrl.match(/(https?:\/\/)(.*\:.*\@)(.*)/); // strip auth from url - this.kibanaUrlNoAuth = matches && matches.length === 3 ? matches[1] + matches[3] : kibanaUrl; + this.kibanaUrlNoAuth = + matches && matches.length >= 3 + ? matches[1] + matches[3].replace('/', '') + : kibanaUrl.replace('/', ''); } requestWithApiKey(path: string, init?: RequestInit | undefined): Promise { return (fetch( - `${this.kibanaUrlNoAuth}/${path}`, + `${this.kibanaUrlNoAuth}${path}`, init as FetchRequestInit ) as unknown) as Promise; } @@ -219,7 +221,7 @@ async function main() { alias: 'f', describe: 'enroll fleet agents for hosts', type: 'boolean', - default: true, + default: false, }, }).argv; const kbnClient = new KbnClientWithApiKeySupport(new ToolingLog(), { url: argv.kibana }); From 56d2c996c3b27d40e263dbe7841c0fd504583f76 Mon Sep 17 00:00:00 2001 From: Candace Park Date: Tue, 29 Sep 2020 13:44:00 -0400 Subject: [PATCH 22/24] marshall's changes + edits to hostname --- .../common/endpoint/index_data.ts | 31 +++++++++++++------ .../endpoint/resolver_generator_script.ts | 5 ++- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 4985d2c169c29..e3d34206f695a 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -32,12 +32,11 @@ import { } from '../../../ingest_manager/common'; import { factory as policyConfigFactory } from './models/policy_config'; import { HostMetadata } from './types'; +import { KbnClientWithApiKeySupport } from '../../scripts/endpoint/resolver_generator_script'; export async function indexHostsAndAlerts( client: Client, - kbnClient: KbnClient & { - requestWithApiKey: (path: string, init?: RequestInit | undefined) => Promise; - }, + kbnClient: KbnClientWithApiKeySupport, seed: string, numHosts: number, numDocs: number, @@ -85,9 +84,7 @@ function delay(ms: number) { async function indexHostDocs( numDocs: number, client: Client, - kbnClient: KbnClient & { - requestWithApiKey: (path: string, init?: RequestInit | undefined) => Promise; - }, + kbnClient: KbnClientWithApiKeySupport, realPolicies: Record, epmEndpointPackage: GetPackagesResponse['response'][0], metadataIndex: string, @@ -273,9 +270,7 @@ const getEndpointPackageInfo = async ( }; const fleetEnrollAgentForHost = async ( - kbnClient: KbnClient & { - requestWithApiKey: (path: string, init?: RequestInit | undefined) => Promise; - }, + kbnClient: KbnClientWithApiKeySupport, endpointHost: HostMetadata, agentPolicyId: string ): Promise => { @@ -321,17 +316,33 @@ const fleetEnrollAgentForHost = async ( return; } + const kibanaVersion = await kbnClient.fetchKibanaVersion().number; // Enroll an agent for the Host const body: PostAgentEnrollRequest['body'] = { type: 'PERMANENT', metadata: { local: { - host: endpointHost.host, elastic: { agent: { version: '8.0.0', }, }, + host: { + architecture: 'x86_64', + hostname: endpointHost.host, + name: endpointHost.host, + id: '1c032ec0-3a94-4d54-9ad2-c5610c0eaba4', + ip: ['fe80::703b:b9e6:887d:7f5/64', '10.0.2.15/24', '::1/128', '127.0.0.1/8'], + mac: ['08:00:27:d8:c5:c0'], + }, + os: { + family: 'windows', + kernel: '10.0.19041.388 (WinBuild.160101.0800)', + platform: 'windows', + version: '10.0', + name: 'Windows 10 Pro', + full: 'Windows 10 Pro(10.0)', + }, }, user_provided: { dev_agent_version: '0.0.1', diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index f705104e62d15..1ea7b5b0648fb 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -19,7 +19,7 @@ import { PostIngestSetupResponse, } from '../../../ingest_manager/common/types/rest_spec'; -class KbnClientWithApiKeySupport extends KbnClient { +export class KbnClientWithApiKeySupport extends KbnClient { private kibanaUrlNoAuth: string; constructor(log: ToolingLog, kibanaConfig: KibanaConfig) { super(log, kibanaConfig); @@ -31,6 +31,9 @@ class KbnClientWithApiKeySupport extends KbnClient { ? matches[1] + matches[3].replace('/', '') : kibanaUrl.replace('/', ''); } + /** + * The fleet api to enroll and agent requires an api key when you mke the request, however KbnClient currently does not support sending an api key with the request. This function allows you to send an api key with a request. + */ requestWithApiKey(path: string, init?: RequestInit | undefined): Promise { return (fetch( `${this.kibanaUrlNoAuth}${path}`, From edd7bdeed41793d41ca647f1abb3150b2afa1391 Mon Sep 17 00:00:00 2001 From: Candace Park Date: Wed, 30 Sep 2020 10:36:34 -0400 Subject: [PATCH 23/24] fetch kibana version --- .../common/endpoint/index_data.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index e3d34206f695a..10aa170a1df8e 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -316,7 +316,18 @@ const fleetEnrollAgentForHost = async ( return; } - const kibanaVersion = await kbnClient.fetchKibanaVersion().number; + const fetchKibanaVersion = async () => { + const version = ((await kbnClient.request({ + path: '/api/status', + method: 'GET', + })) as AxiosResponse).data.version.number; + if (!version) { + // eslint-disable-next-line no-console + console.log('failed to retrieve kibana version'); + } + return version; + }; + // Enroll an agent for the Host const body: PostAgentEnrollRequest['body'] = { type: 'PERMANENT', @@ -324,7 +335,7 @@ const fleetEnrollAgentForHost = async ( local: { elastic: { agent: { - version: '8.0.0', + version: await fetchKibanaVersion(), }, }, host: { From 369318ca4e6812e0b10ff4f3b904adfb8b1f8b71 Mon Sep 17 00:00:00 2001 From: Candace Park Date: Wed, 30 Sep 2020 14:24:03 -0400 Subject: [PATCH 24/24] remove circular dependency --- .../common/endpoint/index_data.ts | 2 +- .../kbn_client_with_api_key_support.ts | 32 +++++++++++++++++++ .../endpoint/resolver_generator_script.ts | 26 +-------------- 3 files changed, 34 insertions(+), 26 deletions(-) create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 10aa170a1df8e..bf3d12f231c86 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -32,7 +32,7 @@ import { } from '../../../ingest_manager/common'; import { factory as policyConfigFactory } from './models/policy_config'; import { HostMetadata } from './types'; -import { KbnClientWithApiKeySupport } from '../../scripts/endpoint/resolver_generator_script'; +import { KbnClientWithApiKeySupport } from '../../scripts/endpoint/kbn_client_with_api_key_support'; export async function indexHostsAndAlerts( client: Client, diff --git a/x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts b/x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts new file mode 100644 index 0000000000000..526c0eb4a9055 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { KbnClient, ToolingLog } from '@kbn/dev-utils'; +import { KibanaConfig } from '@kbn/dev-utils/target/kbn_client/kbn_client_requester'; +import fetch, { RequestInit as FetchRequestInit } from 'node-fetch'; + +export class KbnClientWithApiKeySupport extends KbnClient { + private kibanaUrlNoAuth: string; + constructor(log: ToolingLog, kibanaConfig: KibanaConfig) { + super(log, kibanaConfig); + const kibanaUrl = this.resolveUrl(kibanaConfig.url); + const matches = kibanaUrl.match(/(https?:\/\/)(.*\:.*\@)(.*)/); + // strip auth from url + this.kibanaUrlNoAuth = + matches && matches.length >= 3 + ? matches[1] + matches[3].replace('/', '') + : kibanaUrl.replace('/', ''); + } + /** + * The fleet api to enroll and agent requires an api key when you mke the request, however KbnClient currently does not support sending an api key with the request. This function allows you to send an api key with a request. + */ + requestWithApiKey(path: string, init?: RequestInit | undefined): Promise { + return (fetch( + `${this.kibanaUrlNoAuth}${path}`, + init as FetchRequestInit + ) as unknown) as Promise; + } +} diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index 1ea7b5b0648fb..1c2c4a857451b 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -9,8 +9,6 @@ import { Client, ClientOptions } from '@elastic/elasticsearch'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { KbnClient, ToolingLog } from '@kbn/dev-utils'; import { AxiosResponse } from 'axios'; -import { KibanaConfig } from '@kbn/dev-utils/target/kbn_client/kbn_client_requester'; -import fetch, { RequestInit as FetchRequestInit } from 'node-fetch'; import { indexHostsAndAlerts } from '../../common/endpoint/index_data'; import { ANCESTRY_LIMIT } from '../../common/endpoint/generate_data'; import { FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../../ingest_manager/common/constants'; @@ -18,29 +16,7 @@ import { CreateFleetSetupResponse, PostIngestSetupResponse, } from '../../../ingest_manager/common/types/rest_spec'; - -export class KbnClientWithApiKeySupport extends KbnClient { - private kibanaUrlNoAuth: string; - constructor(log: ToolingLog, kibanaConfig: KibanaConfig) { - super(log, kibanaConfig); - const kibanaUrl = this.resolveUrl(kibanaConfig.url); - const matches = kibanaUrl.match(/(https?:\/\/)(.*\:.*\@)(.*)/); - // strip auth from url - this.kibanaUrlNoAuth = - matches && matches.length >= 3 - ? matches[1] + matches[3].replace('/', '') - : kibanaUrl.replace('/', ''); - } - /** - * The fleet api to enroll and agent requires an api key when you mke the request, however KbnClient currently does not support sending an api key with the request. This function allows you to send an api key with a request. - */ - requestWithApiKey(path: string, init?: RequestInit | undefined): Promise { - return (fetch( - `${this.kibanaUrlNoAuth}${path}`, - init as FetchRequestInit - ) as unknown) as Promise; - } -} +import { KbnClientWithApiKeySupport } from './kbn_client_with_api_key_support'; main();