From 2f083bfcc715eb4d34a045e0264c3303fd2e8513 Mon Sep 17 00:00:00 2001 From: Chris Donaher Date: Wed, 2 Mar 2022 17:39:14 -0700 Subject: [PATCH] Add Internal Alert/Signal ID to Endpoint Alert telemetry (#126216) * Attach the internal signal_id to the endpoint alert to join with insights * Ensure we forward signal_id field properly * Don't think we need to explicitly define the field in the top-level since it satisfies the key:string * Updated unit test to check for signal id enrichment * Addressed some comments about alert_id enrichment * Refactored send_telemetry_events to be more performant and idiomatic * Added test cases with a non-matching enrichment or non-existing enrichment * Broke some tests that don't assume QueueTelemetryEvents are endpoint.alerts * my types were still off * Addressed comments to use more idiomatic 'toString' function * Fixed 'Cannot access signalIdMap before initialization name' in reduce by instatiating map prior to reduce Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../signals/search_after_bulk_create.ts | 8 +- .../signals/send_telemetry_events.test.ts | 76 ++++++++++++++++++- .../signals/send_telemetry_events.ts | 40 +++++++++- .../server/lib/telemetry/filters.ts | 1 + .../server/lib/telemetry/sender.test.ts | 2 + .../server/lib/telemetry/types.ts | 4 + 6 files changed, 123 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index f8270c53b07ae..99230627cb6b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -176,7 +176,13 @@ export const searchAfterAndBulkCreate = async ({ buildRuleMessage(`enrichedEvents.hits.hits: ${enrichedEvents.hits.hits.length}`) ); - sendAlertTelemetryEvents(logger, eventsTelemetry, enrichedEvents, buildRuleMessage); + sendAlertTelemetryEvents( + logger, + eventsTelemetry, + enrichedEvents, + createdItems, + buildRuleMessage + ); } if (!hasSortId) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.test.ts index 991378983e1b2..36bb90936620b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { selectEvents } from './send_telemetry_events'; +import { selectEvents, enrichEndpointAlertsSignalID } from './send_telemetry_events'; describe('sendAlertTelemetry', () => { it('selectEvents', () => { @@ -33,6 +33,9 @@ describe('sendAlertTelemetry', () => { data_stream: { dataset: 'endpoint.events', }, + event: { + id: 'foo', + }, }, }, { @@ -47,6 +50,9 @@ describe('sendAlertTelemetry', () => { dataset: 'endpoint.alerts', other: 'x', }, + event: { + id: 'bar', + }, }, }, { @@ -58,13 +64,52 @@ describe('sendAlertTelemetry', () => { '@timestamp': 'x', key3: 'hello', data_stream: {}, + event: { + id: 'baz', + }, + }, + }, + { + _index: 'y', + _type: 'y', + _id: 'y', + _score: 0, + _source: { + '@timestamp': 'y', + key3: 'hello', + data_stream: { + dataset: 'endpoint.alerts', + other: 'y', + }, + event: { + id: 'not-in-map', + }, + }, + }, + { + _index: 'z', + _type: 'z', + _id: 'z', + _score: 0, + _source: { + '@timestamp': 'z', + key3: 'no-event-id', + data_stream: { + dataset: 'endpoint.alerts', + other: 'z', + }, }, }, ], }, }; - - const sources = selectEvents(filteredEvents); + const joinMap = new Map([ + ['foo', '1234'], + ['bar', 'abcd'], + ['baz', '4567'], + ]); + const subsetEvents = selectEvents(filteredEvents); + const sources = enrichEndpointAlertsSignalID(subsetEvents, joinMap); expect(sources).toStrictEqual([ { '@timestamp': 'x', @@ -73,6 +118,31 @@ describe('sendAlertTelemetry', () => { dataset: 'endpoint.alerts', other: 'x', }, + event: { + id: 'bar', + }, + signal_id: 'abcd', + }, + { + '@timestamp': 'y', + key3: 'hello', + data_stream: { + dataset: 'endpoint.alerts', + other: 'y', + }, + event: { + id: 'not-in-map', + }, + signal_id: undefined, + }, + { + '@timestamp': 'z', + key3: 'no-event-id', + data_stream: { + dataset: 'endpoint.alerts', + other: 'z', + }, + signal_id: undefined, }, ]); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts index 5904f943183c3..fc3aed36939cd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts @@ -11,14 +11,17 @@ import { BuildRuleMessage } from './rule_messages'; import { SignalSearchResponse, SignalSource } from './types'; import { Logger } from '../../../../../../../src/core/server'; -export interface SearchResultWithSource { +interface SearchResultSource { _source: SignalSource; } +type CreatedSignalId = string; +type AlertId = string; + export function selectEvents(filteredEvents: SignalSearchResponse): TelemetryEvent[] { // @ts-expect-error @elastic/elasticsearch _source is optional const sources: TelemetryEvent[] = filteredEvents.hits.hits.map(function ( - obj: SearchResultWithSource + obj: SearchResultSource ): TelemetryEvent { return obj._source; }); @@ -27,20 +30,49 @@ export function selectEvents(filteredEvents: SignalSearchResponse): TelemetryEve return sources.filter((obj: TelemetryEvent) => obj.data_stream?.dataset === 'endpoint.alerts'); } +export function enrichEndpointAlertsSignalID( + events: TelemetryEvent[], + signalIdMap: Map +): TelemetryEvent[] { + return events.map(function (obj: TelemetryEvent): TelemetryEvent { + obj.signal_id = undefined; + if (obj?.event?.id !== undefined) { + obj.signal_id = signalIdMap.get(obj.event.id); + } + return obj; + }); +} + export function sendAlertTelemetryEvents( logger: Logger, eventsTelemetry: ITelemetryEventsSender | undefined, filteredEvents: SignalSearchResponse, + createdEvents: SignalSource[], buildRuleMessage: BuildRuleMessage ) { if (eventsTelemetry === undefined) { return; } - const sources = selectEvents(filteredEvents); + let selectedEvents = selectEvents(filteredEvents); + if (selectedEvents.length > 0) { + // Create map of ancenstor_id -> alert_id + let signalIdMap = new Map(); + /* eslint-disable no-param-reassign */ + signalIdMap = createdEvents.reduce((signalMap, obj) => { + const ancestorId = obj['kibana.alert.original_event.id']?.toString(); + const alertId = obj._id?.toString(); + if (ancestorId !== null && ancestorId !== undefined && alertId !== undefined) { + signalMap = signalIdMap.set(ancestorId, alertId); + } + + return signalMap; + }, new Map()); + selectedEvents = enrichEndpointAlertsSignalID(selectedEvents, signalIdMap); + } try { - eventsTelemetry.queueTelemetryEvents(sources); + eventsTelemetry.queueTelemetryEvents(selectedEvents); } catch (exc) { logger.error(buildRuleMessage(`[-] queing telemetry events failed ${exc}`)); } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts b/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts index 452717f1efb4f..bd41bc454e876 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts @@ -108,6 +108,7 @@ const allowlistBaseEventFields: AllowlistFields = { export const allowlistEventFields: AllowlistFields = { _id: true, '@timestamp': true, + signal_id: true, agent: true, Endpoint: true, /* eslint-disable @typescript-eslint/naming-convention */ diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts index 70852aa3093c6..d055f3843d479 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts @@ -35,6 +35,7 @@ describe('TelemetryEventsSender', () => { { event: { kind: 'alert', + id: 'test', }, dns: { question: { @@ -108,6 +109,7 @@ describe('TelemetryEventsSender', () => { { event: { kind: 'alert', + id: 'test', }, dns: { question: { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index 35b701552b6ba..35b531ae6941c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -58,6 +58,10 @@ export interface TelemetryEvent { }; }; license?: ESLicense; + event?: { + id?: string; + kind?: string; + }; } // EP Policy Response