From f96873b5169d8812a216778c803bd67568e46f32 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Fri, 30 Oct 2020 14:00:55 -0600 Subject: [PATCH] Copies any foreign signals to a original_signals section within signals --- .../signals/build_bulk_body.test.ts | 202 +++++++++++++++++- .../signals/build_bulk_body.ts | 2 +- .../signals/build_event_type_signal.test.ts | 56 ++++- .../signals/build_event_type_signal.ts | 13 ++ .../signals/build_signal.test.ts | 70 +++++- .../detection_engine/signals/build_signal.ts | 31 ++- .../signals/single_bulk_create.test.ts | 12 ++ .../signals/single_bulk_create.ts | 3 +- .../lib/detection_engine/signals/types.ts | 1 + 9 files changed, 381 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 75a7de8cd2c44..ad060a9304e84 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -20,7 +20,7 @@ import { objectPairIntersection, objectArrayIntersection, } from './build_bulk_body'; -import { SignalHit } from './types'; +import { SignalHit, SignalSourceHit } from './types'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; describe('buildBulkBody', () => { @@ -441,6 +441,206 @@ describe('buildBulkBody', () => { }; expect(fakeSignalSourceHit).toEqual(expected); }); + + test('bulk body builds "original_signal" if it exists already as a numeric', () => { + const sampleParams = sampleRuleAlertParams(); + const sampleDoc = sampleDocNoSortId(); + delete sampleDoc._source.source; + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: 123, + }, + } as unknown) as SignalSourceHit; + const { '@timestamp': timestamp, ...fakeSignalSourceHit } = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + const expected: Omit & { someKey: string } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_signal: 123, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + original_time: '2020-04-20T21:27:45+0000', + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + version: 1, + updated_at: fakeSignalSourceHit.signal.rule?.updated_at, + created_at: fakeSignalSourceHit.signal.rule?.created_at, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 1, + }, + }; + expect(fakeSignalSourceHit).toEqual(expected); + }); + + test('bulk body builds "original_signal" if it exists already as an object', () => { + const sampleParams = sampleRuleAlertParams(); + const sampleDoc = sampleDocNoSortId(); + delete sampleDoc._source.source; + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { child_1: { child_2: 'nested data' } }, + }, + } as unknown) as SignalSourceHit; + const { '@timestamp': timestamp, ...fakeSignalSourceHit } = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + const expected: Omit & { someKey: string } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_signal: { child_1: { child_2: 'nested data' } }, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + original_time: '2020-04-20T21:27:45+0000', + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + version: 1, + updated_at: fakeSignalSourceHit.signal.rule?.updated_at, + created_at: fakeSignalSourceHit.signal.rule?.created_at, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 1, + }, + }; + expect(fakeSignalSourceHit).toEqual(expected); + }); }); describe('buildSignalFromSequence', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index cc454ac1e9462..a704d076880bf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -154,7 +154,7 @@ export const buildSignalFromEvent = ( const rule = applyOverrides ? buildRuleWithOverrides(ruleSO, event._source) : buildRuleWithoutOverrides(ruleSO); - const signal = { + const signal: Signal = { ...buildSignal([event], rule), ...additionalSignalFields(event), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts index 106a049002e05..ada939ed0941a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts @@ -5,7 +5,8 @@ */ import { sampleDocNoSortId } from './__mocks__/es_results'; -import { buildEventTypeSignal } from './build_event_type_signal'; +import { buildEventTypeSignal, isEventTypeSignal } from './build_event_type_signal'; +import { BaseSignalHit } from './types'; describe('buildEventTypeSignal', () => { beforeEach(() => { @@ -44,4 +45,57 @@ describe('buildEventTypeSignal', () => { }; expect(eventType).toEqual(expected); }); + + test('It validates a sample doc with no signal type as "false"', () => { + const doc = sampleDocNoSortId(); + expect(isEventTypeSignal(doc)).toEqual(false); + }); + + test('It validates a sample doc with a signal type as "true"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: { + rule: { id: 'id-123' }, + }, + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(true); + }); + + test('It validates a numeric signal string as "false"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: 'something', + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(false); + }); + + test('It validates an empty object as "false"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: {}, + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(false); + }); + + test('It validates an empty rule object as "false"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: { + rule: {}, + }, + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(false); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts index 81c9d1dedcc56..3d78cf5ce5e46 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts @@ -13,3 +13,16 @@ export const buildEventTypeSignal = (doc: BaseSignalHit): object => { return { kind: 'signal' }; } }; + +/** + * Given a document this will return true if that document is a signal + * document. We can't guarantee the code will call this function with a document + * before adding the _source.event.kind = "signal" from "buildEventTypeSignal" + * so we do basic testing to ensure that if the object has the fields of: + * "signal.rule.id" then it will be one of our signals rather than a customer + * overwritten signal. + * @param doc The document which might be a signal or it might be a regular log + */ +export const isEventTypeSignal = (doc: BaseSignalHit): boolean => { + return doc._source.signal?.rule?.id != null && typeof doc._source.signal?.rule?.id === 'string'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index d0c451bbdf2e2..c5e6bc9f157e0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -5,8 +5,14 @@ */ import { sampleDocNoSortId } from './__mocks__/es_results'; -import { buildSignal, buildParent, buildAncestors, additionalSignalFields } from './build_signal'; -import { Signal, Ancestor } from './types'; +import { + buildSignal, + buildParent, + buildAncestors, + additionalSignalFields, + removeClashes, +} from './build_signal'; +import { Signal, Ancestor, BaseSignalHit } from './types'; import { getRulesSchemaMock, ANCHOR_DATE, @@ -302,4 +308,64 @@ describe('buildSignal', () => { ]; expect(signal).toEqual(expected); }); + + describe('removeClashes', () => { + test('it will call renameClashes with a regular doc and not mutate it if it does not have a signal clash', () => { + const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const output = removeClashes(doc); + expect(output).toBe(doc); // reference check + }); + + test('it will call renameClashes with a regular doc and not change anything', () => { + const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const output = removeClashes(doc); + expect(output).toEqual(doc); // deep equal check + }); + + test('it will remove a "signal" numeric clash', () => { + const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: 127, + }, + } as unknown) as BaseSignalHit; + const output = removeClashes(doc); + expect(output).toEqual(sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')); + }); + + test('it will remove a "signal" object clash', () => { + const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { child_1: { child_2: 'Test nesting' } }, + }, + } as unknown) as BaseSignalHit; + const output = removeClashes(doc); + expect(output).toEqual(sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')); + }); + + test('it will not remove a "signal" if that is signal is one of our signals', () => { + const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { rule: { id: '123' } }, + }, + } as unknown) as BaseSignalHit; + const output = removeClashes(doc); + const expected = { + ...sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'), + _source: { + ...sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')._source, + signal: { rule: { id: '123' } }, + }, + }; + expect(output).toEqual(expected); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 947938de6caca..b36a1cbb4a6b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,6 +5,7 @@ */ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; +import { isEventTypeSignal } from './build_event_type_signal'; import { Signal, Ancestor, BaseSignalHit } from './types'; /** @@ -48,15 +49,37 @@ export const buildAncestors = (doc: BaseSignalHit): Ancestor[] => { } }; +/** + * This removes any signal named clashes such as if a source index has + * "signal" but is not a signal object we put onto the object. If this + * is our "signal object" then we don't want to remove it. + * @param doc The source index doc to a signal. + */ +export const removeClashes = (doc: BaseSignalHit): BaseSignalHit => { + const { signal, ...noSignal } = doc._source; + if (signal == null || isEventTypeSignal(doc)) { + return doc; + } else { + return { + ...doc, + _source: { ...noSignal }, + }; + } +}; + /** * Builds the `signal.*` fields that are common across all signals. * @param docs The parent signals/events of the new signal to be built. * @param rule The rule that is generating the new signal. */ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => { - const parents = docs.map(buildParent); + const removedClashes = docs.map(removeClashes); + const parents = removedClashes.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; - const ancestors = docs.reduce((acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), []); + const ancestors = removedClashes.reduce( + (acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), + [] + ); return { parents, ancestors, @@ -72,9 +95,11 @@ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => */ export const additionalSignalFields = (doc: BaseSignalHit) => { return { - parent: buildParent(doc), + parent: buildParent(removeClashes(doc)), original_time: doc._source['@timestamp'], original_event: doc._source.event ?? undefined, threshold_count: doc._source.threshold_count ?? undefined, + original_signal: + doc._source.signal != null && !isEventTypeSignal(doc) ? doc._source.signal : undefined, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts index b7cc13fd13a01..eeeda6561892d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -330,6 +330,18 @@ describe('singleBulkCreate', () => { ]); }); + test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own numeric signal type', () => { + const doc = { ...sampleDocWithAncestors(), _source: { signal: 1234 } }; + const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', doc); + expect(filtered).toEqual([]); + }); + + test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own object signal type', () => { + const doc = { ...sampleDocWithAncestors(), _source: { signal: {} } }; + const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', doc); + expect(filtered).toEqual([]); + }); + test('create successful and returns proper createdItemsCount', async () => { const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockResolvedValue(sampleBulkCreateDuplicateResult); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index 759890cc9d074..d8889dcfcf471 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -14,6 +14,7 @@ import { generateId, makeFloatString, errorAggregator } from './utils'; import { buildBulkBody } from './build_bulk_body'; import { BuildRuleMessage } from './rule_messages'; import { Logger } from '../../../../../../../src/core/server'; +import { isEventTypeSignal } from './build_event_type_signal'; interface SingleBulkCreateParams { filteredEvents: SignalSearchResponse; @@ -50,7 +51,7 @@ export const filterDuplicateRules = ( signalSearchResponse: SignalSearchResponse ) => { return signalSearchResponse.hits.hits.filter((doc) => { - if (doc._source.signal == null) { + if (doc._source.signal == null || !isEventTypeSignal(doc)) { return true; } else { return !( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 9d4e7d8a81051..7128feb80ab3c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -157,6 +157,7 @@ export interface Signal { original_event?: SearchTypes; status: Status; threshold_count?: SearchTypes; + original_signal?: SearchTypes; depth: number; }