From 39a5f49483dee6c6758f3196945b76372ce96fe4 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 2 Sep 2020 11:14:19 -0400 Subject: [PATCH 1/9] Refactors signal ancestry to allow multiple parents --- .../routes/index/signals_mapping.json | 25 ++ .../signals/__mocks__/es_results.ts | 10 +- .../signals/build_bulk_body.test.ts | 76 +++--- .../signals/build_bulk_body.ts | 9 +- .../signals/build_rule.test.ts | 102 +++++++- .../detection_engine/signals/build_rule.ts | 16 +- .../signals/build_signal.test.ts | 243 ++++-------------- .../detection_engine/signals/build_signal.ts | 63 ++--- .../signals/signal_rule_alert_type.ts | 3 +- .../signals/single_bulk_create.test.ts | 32 +-- .../signals/single_bulk_create.ts | 5 +- .../lib/detection_engine/signals/types.ts | 16 +- 12 files changed, 283 insertions(+), 317 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 7d80a319e9e52..cfce019910071 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -22,11 +22,33 @@ } } }, + "parents": { + "properties": { + "rule": { + "type": "keyword" + }, + "index": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "depth": { + "type": "long" + } + } + }, "ancestors": { "properties": { "rule": { "type": "keyword" }, + "index": { + "type": "keyword" + }, "id": { "type": "keyword" }, @@ -299,6 +321,9 @@ }, "threshold_count": { "type": "float" + }, + "depth": { + "type": "integer" } } } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 95ec753c21fd8..9d3eb29be08dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -149,21 +149,23 @@ export const sampleDocWithAncestors = (): SignalSearchResponse => { delete sampleDoc._source.source; sampleDoc._source.signal = { parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], + rule: { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + }, + depth: 1, }; return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index fe2e0f2d96fd8..8c711303b7493 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -46,20 +46,20 @@ describe('buildBulkBody', () => { kind: 'signal', }, signal: { - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: sampleIdGuid, - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -101,6 +101,7 @@ describe('buildBulkBody', () => { updated_at: fakeSignalSourceHit.signal.rule?.updated_at, exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); @@ -148,20 +149,20 @@ describe('buildBulkBody', () => { kind: 'event', module: 'system', }, - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: sampleIdGuid, - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -203,6 +204,7 @@ describe('buildBulkBody', () => { threat: [], exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); @@ -248,20 +250,20 @@ describe('buildBulkBody', () => { dataset: 'socket', module: 'system', }, - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: sampleIdGuid, - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -303,6 +305,7 @@ describe('buildBulkBody', () => { throttle: 'no_actions', exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); @@ -341,20 +344,20 @@ describe('buildBulkBody', () => { original_event: { kind: 'event', }, - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: sampleIdGuid, - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -396,6 +399,7 @@ describe('buildBulkBody', () => { throttle: 'no_actions', exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 218750ac30a2a..7be97e46f91f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalHit } from './types'; +import { SignalSourceHit, SignalHit, Signal } from './types'; import { buildRule } from './build_rule'; -import { buildSignal } from './build_signal'; +import { additionalSignalFields, buildSignal } from './build_signal'; import { buildEventTypeSignal } from './build_event_type_signal'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; @@ -58,7 +58,10 @@ export const buildBulkBody = ({ tags, throttle, }); - const signal = buildSignal(doc, rule); + const signal: Signal = { + ...buildSignal([doc], rule), + ...additionalSignalFields(doc), + }; const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index 7257e5952ff05..ba815a0b62f0d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -4,10 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { buildRule } from './build_rule'; +import { buildRule, removeInternalTagsFromRule } from './build_rule'; import { sampleDocNoSortId, sampleRuleAlertParams, sampleRuleGuid } from './__mocks__/es_results'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; +import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; +import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; describe('buildRule', () => { beforeEach(() => { @@ -208,4 +210,102 @@ describe('buildRule', () => { }; expect(rule).toEqual(expected); }); + + test('it builds a rule and removes internal tags', () => { + const ruleParams = sampleRuleAlertParams(); + const rule = buildRule({ + actions: [], + doc: sampleDocNoSortId(), + ruleParams, + name: 'some-name', + id: sampleRuleGuid, + enabled: false, + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: 'some interval', + tags: [ + 'some fake tag 1', + 'some fake tag 2', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:true`, + ], + throttle: 'no_actions', + }); + const expected: Partial = { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: false, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: 'some interval', + language: 'kuery', + license: 'Elastic License', + max_signals: 10000, + name: 'some-name', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + risk_score: 50, + risk_score_mapping: [], + rule_id: 'rule-1', + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + to: 'now', + type: 'query', + note: '', + updated_by: 'elastic', + updated_at: rule.updated_at, + created_at: rule.created_at, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + version: 1, + }; + expect(rule).toEqual(expected); + }); + + test('it removes internal tags from a typical rule', () => { + const rule = getPartialRulesSchemaMock(); + rule.tags = [ + 'some fake tag 1', + 'some fake tag 2', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:true`, + ]; + const noInternals = removeInternalTagsFromRule(rule); + expect(noInternals).toEqual(getPartialRulesSchemaMock()); + }); + + test('it works with an empty array', () => { + const rule = getPartialRulesSchemaMock(); + rule.tags = []; + const noInternals = removeInternalTagsFromRule(rule); + const expected = getPartialRulesSchemaMock(); + expected.tags = []; + expect(noInternals).toEqual(expected); + }); + + test('it works if tags does not exist', () => { + const rule = getPartialRulesSchemaMock(); + delete rule.tags; + const noInternals = removeInternalTagsFromRule(rule); + const expected = getPartialRulesSchemaMock(); + delete expected.tags; + expect(noInternals).toEqual(expected); + }); + + test('it works if tags contains normal values and no internal values', () => { + const rule = getPartialRulesSchemaMock(); + const noInternals = removeInternalTagsFromRule(rule); + expect(noInternals).toEqual(rule); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index e02a0154d63c9..aacf9b8be31b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -12,6 +12,7 @@ import { buildRiskScoreFromMapping } from './mappings/build_risk_score_from_mapp import { SignalSourceHit } from './types'; import { buildSeverityFromMapping } from './mappings/build_severity_from_mapping'; import { buildRuleNameFromMapping } from './mappings/build_rule_name_from_mapping'; +import { INTERNAL_IDENTIFIER } from '../../../../common/constants'; interface BuildRuleParams { ruleParams: RuleTypeParams; @@ -64,7 +65,7 @@ export const buildRule = ({ const meta = { ...ruleParams.meta, ...riskScoreMeta, ...severityMeta, ...ruleNameMeta }; - return pickBy((value: unknown) => value != null, { + const rule = pickBy((value: unknown) => value != null, { id, rule_id: ruleParams.ruleId ?? '(unknown rule_id)', actions, @@ -111,4 +112,17 @@ export const buildRule = ({ anomaly_threshold: ruleParams.anomalyThreshold, threshold: ruleParams.threshold, }); + return removeInternalTagsFromRule(rule); +}; + +export const removeInternalTagsFromRule = (rule: Partial): Partial => { + if (rule.tags == null) { + return rule; + } else { + const ruleWithoutInternalTags: Partial = { + ...rule, + tags: rule.tags.filter((tag) => !tag.startsWith(INTERNAL_IDENTIFIER)), + }; + return ruleWithoutInternalTags; + } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index 6aebf8815659a..f3806b458ef73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -7,12 +7,11 @@ import { sampleDocNoSortId } from './__mocks__/es_results'; import { buildSignal, - buildAncestor, + buildParent, buildAncestorsSignal, - removeInternalTagsFromRule, + additionalSignalFields, } from './build_signal'; import { Signal, Ancestor } from './types'; -import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; describe('buildSignal', () => { @@ -24,22 +23,25 @@ describe('buildSignal', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); delete doc._source.event; const rule = getPartialRulesSchemaMock(); - const signal = buildSignal(doc, rule); + const signal = { + ...buildSignal([doc], rule), + ...additionalSignalFields(doc), + }; const expected: Signal = { - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, + parents: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -71,6 +73,7 @@ describe('buildSignal', () => { updated_at: signal.rule.updated_at, created_at: signal.rule.created_at, }, + depth: 1, }; expect(signal).toEqual(expected); }); @@ -84,94 +87,25 @@ describe('buildSignal', () => { module: 'system', }; const rule = getPartialRulesSchemaMock(); - const signal = buildSignal(doc, rule); + const signal = { + ...buildSignal([doc], rule), + ...additionalSignalFields(doc), + }; const expected: Signal = { - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, - ancestors: [ + parents: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', - original_event: { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }, - status: 'open', - rule: { - created_by: 'elastic', - description: 'Detecting root and admin users', - enabled: true, - false_positives: [], - from: 'now-6m', - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - immutable: false, - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - interval: '5m', - risk_score: 50, - rule_id: 'rule-1', - language: 'kuery', - max_signals: 100, - name: 'Detect Root/Admin Users', - output_index: '.siem-signals', - query: 'user.name: root or user.name: admin', - references: ['http://www.example.com', 'https://ww.example.com'], - severity: 'high', - updated_by: 'elastic', - tags: ['some fake tag 1', 'some fake tag 2'], - to: 'now', - type: 'query', - note: '', - updated_at: signal.rule.updated_at, - created_at: signal.rule.created_at, - }, - }; - expect(signal).toEqual(expected); - }); - - test('it builds a signal as expected with original_event if is present and without internal tags in them', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - const rule = getPartialRulesSchemaMock(); - rule.tags = [ - 'some fake tag 1', - 'some fake tag 2', - `${INTERNAL_RULE_ID_KEY}:rule-1`, - `${INTERNAL_IMMUTABLE_KEY}:true`, - ]; - const signal = buildSignal(doc, rule); - const expected: Signal = { - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -209,6 +143,7 @@ describe('buildSignal', () => { updated_at: signal.rule.updated_at, created_at: signal.rule.created_at, }, + depth: 1, }; expect(signal).toEqual(expected); }); @@ -221,14 +156,12 @@ describe('buildSignal', () => { kind: 'event', module: 'system', }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestor(doc, rule); + const signal = buildParent(doc); const expected: Ancestor = { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }; expect(signal).toEqual(expected); }); @@ -242,76 +175,34 @@ describe('buildSignal', () => { module: 'system', }; doc._source.signal = { - parent: { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, - ancestors: [ + parents: [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], - }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestor(doc, rule); - const expected: Ancestor = { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'signal', - index: 'myFakeSignalIndex', - depth: 2, - }; - expect(signal).toEqual(expected); - }); - - test('it builds a ancestor correctly if the parent does exist without internal tags in them', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - doc._source.signal = { - parent: { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, ancestors: [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], + depth: 1, + rule: { + id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', + }, }; - const rule = getPartialRulesSchemaMock(); - rule.tags = [ - 'some fake tag 1', - 'some fake tag 2', - `${INTERNAL_RULE_ID_KEY}:rule-1`, - `${INTERNAL_IMMUTABLE_KEY}:true`, - ]; - - const signal = buildAncestor(doc, rule); + const signal = buildParent(doc); const expected: Ancestor = { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'signal', index: 'myFakeSignalIndex', - depth: 2, + depth: 1, }; expect(signal).toEqual(expected); }); @@ -324,15 +215,13 @@ describe('buildSignal', () => { kind: 'event', module: 'system', }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestorsSignal(doc, rule); + const signal = buildAncestorsSignal(doc); const expected: Ancestor[] = [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ]; expect(signal).toEqual(expected); @@ -348,76 +237,40 @@ describe('buildSignal', () => { }; doc._source.signal = { parent: { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ancestors: [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], + rule: { + id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', + }, + depth: 1, }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestorsSignal(doc, rule); + const signal = buildAncestorsSignal(doc); const expected: Ancestor[] = [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'signal', index: 'myFakeSignalIndex', - depth: 2, + depth: 1, }, ]; expect(signal).toEqual(expected); }); - - test('it removes internal tags from a typical rule', () => { - const rule = getPartialRulesSchemaMock(); - rule.tags = [ - 'some fake tag 1', - 'some fake tag 2', - `${INTERNAL_RULE_ID_KEY}:rule-1`, - `${INTERNAL_IMMUTABLE_KEY}:true`, - ]; - const noInternals = removeInternalTagsFromRule(rule); - expect(noInternals).toEqual(getPartialRulesSchemaMock()); - }); - - test('it works with an empty array', () => { - const rule = getPartialRulesSchemaMock(); - rule.tags = []; - const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); - expected.tags = []; - expect(noInternals).toEqual(expected); - }); - - test('it works if tags does not exist', () => { - const rule = getPartialRulesSchemaMock(); - delete rule.tags; - const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); - delete expected.tags; - expect(noInternals).toEqual(expected); - }); - - test('it works if tags contains normal values and no internal values', () => { - const rule = getPartialRulesSchemaMock(); - const noInternals = removeInternalTagsFromRule(rule); - expect(noInternals).toEqual(rule); - }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index e7098c015c165..c0b46de1d956f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,35 +5,29 @@ */ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; -import { INTERNAL_IDENTIFIER } from '../../../../common/constants'; import { SignalSourceHit, Signal, Ancestor } from './types'; -export const buildAncestor = (doc: SignalSourceHit, rule: Partial): Ancestor => { - const existingSignal = doc._source.signal?.parent; - if (existingSignal != null) { +export const buildParent = (doc: SignalSourceHit): Ancestor => { + if (doc._source.signal != null) { return { - rule: rule.id != null ? rule.id : '', + rule: doc._source.signal.rule.id, id: doc._id, type: 'signal', index: doc._index, - depth: existingSignal.depth + 1, + depth: doc._source.signal.depth, }; } else { return { - rule: rule.id != null ? rule.id : '', id: doc._id, type: 'event', index: doc._index, - depth: 1, + depth: 0, }; } }; -export const buildAncestorsSignal = ( - doc: SignalSourceHit, - rule: Partial -): Signal['ancestors'] => { - const newAncestor = buildAncestor(doc, rule); +export const buildAncestorsSignal = (doc: SignalSourceHit): Signal['ancestors'] => { + const newAncestor = buildParent(doc); const existingAncestors = doc._source.signal?.ancestors; if (existingAncestors != null) { return [...existingAncestors, newAncestor]; @@ -42,35 +36,26 @@ export const buildAncestorsSignal = ( } }; -export const buildSignal = (doc: SignalSourceHit, rule: Partial): Signal => { - const ruleWithoutInternalTags = removeInternalTagsFromRule(rule); - const parent = buildAncestor(doc, rule); - const ancestors = buildAncestorsSignal(doc, rule); - let signal: Signal = { - parent, +export const buildSignal = (docs: SignalSourceHit[], rule: Partial): Signal => { + const depth = docs.reduce((acc, doc) => Math.max(doc._source.signal?.depth ?? 0, acc), 0) + 1; + const parents = docs.map(buildParent); + const ancestors = docs.reduce( + (acc: Ancestor[], doc) => acc.concat(buildAncestorsSignal(doc)), + [] + ); + return { + parents, ancestors, - original_time: doc._source['@timestamp'], status: 'open', - rule: ruleWithoutInternalTags, + rule, + depth, }; - if (doc._source.event != null) { - signal = { ...signal, original_event: doc._source.event }; - } - if (doc._source.threshold_count != null) { - signal = { ...signal, threshold_count: doc._source.threshold_count }; - delete doc._source.threshold_count; - } - return signal; }; -export const removeInternalTagsFromRule = (rule: Partial): Partial => { - if (rule.tags == null) { - return rule; - } else { - const ruleWithoutInternalTags: Partial = { - ...rule, - tags: rule.tags.filter((tag) => !tag.startsWith(INTERNAL_IDENTIFIER)), - }; - return ruleWithoutInternalTags; - } +export const additionalSignalFields = (doc: SignalSourceHit) => { + return { + original_time: doc._source['@timestamp'], + original_event: doc._source.event ?? undefined, + threshold_count: doc._source.threshold_count ?? undefined, + }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 0e859ecef31c6..b11ec964fabf2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -120,7 +120,6 @@ export const signalRulesAlertType = ({ enabled, schedule: { interval }, throttle, - params: ruleParams, } = savedObject.attributes; const updatedAt = savedObject.updated_at ?? ''; const refresh = actions.length ? 'wait_for' : false; @@ -344,7 +343,7 @@ export const signalRulesAlertType = ({ if (result.success) { if (actions.length) { const notificationRuleParams: NotificationRuleTypeParams = { - ...ruleParams, + ...params, name, id: savedObject.id, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts index 8b9fb0574efe9..41c825ea4d978 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -291,37 +291,7 @@ describe('singleBulkCreate', () => { test('filter duplicate rules will return nothing filtered when the two rule ids do not match with each other', () => { const filtered = filterDuplicateRules('some id', sampleDocWithAncestors()); - expect(filtered).toEqual([ - { - _index: 'myFakeSignalIndex', - _type: 'doc', - _score: 100, - _version: 1, - _id: 'e1e08ddc-5e37-49ff-a258-5393aa44435a', - _source: { - someKey: 'someValue', - '@timestamp': '2020-04-20T21:27:45+0000', - signal: { - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, - ancestors: [ - { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, - ], - }, - }, - }, - ]); + expect(filtered).toEqual(sampleDocWithAncestors().hits.hits); }); test('filters duplicate rules will return empty array when the two rule ids match each other', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index 74709f31563ee..be71c67615a4c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -51,7 +51,10 @@ export const filterDuplicateRules = ( if (doc._source.signal == null) { return true; } else { - return !doc._source.signal.ancestors.some((ancestor) => ancestor.rule === ruleId); + return !( + doc._source.signal.ancestors.some((ancestor) => ancestor.rule === ruleId) || + doc._source.signal.rule.id === ruleId + ); } }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index aecdbe10695d2..b02e19752faf8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -44,8 +44,15 @@ export interface SignalSource { [key: string]: SearchTypes; '@timestamp': string; signal?: { - parent: Ancestor; + // parent is deprecated: new signals should populate parents instead + // both are optional until all signals with parent are gone and we can safely remove it + parent?: Ancestor; + parents?: Ancestor[]; ancestors: Ancestor[]; + rule: { + id: string; + }; + depth: number; }; } @@ -113,7 +120,7 @@ export type SignalRuleAlertTypeDefinition = Omit & { }; export interface Ancestor { - rule: string; + rule?: string; id: string; type: string; index: string; @@ -122,12 +129,13 @@ export interface Ancestor { export interface Signal { rule: Partial; - parent: Ancestor; + parents: Ancestor[]; ancestors: Ancestor[]; - original_time: string; + original_time?: string; original_event?: SearchTypes; status: Status; threshold_count?: SearchTypes; + depth: number; } export interface SignalHit { From de9debb3e2aa594dc282799158cc85fab90d340c Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 2 Sep 2020 16:03:52 -0400 Subject: [PATCH 2/9] Fix depth calculation for 7.10+ signals on pre-7.10 signals --- .../server/lib/detection_engine/signals/build_signal.ts | 6 ++++-- .../server/lib/detection_engine/signals/types.ts | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index c0b46de1d956f..00ad0ea0c52eb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -14,7 +14,9 @@ export const buildParent = (doc: SignalSourceHit): Ancestor => { id: doc._id, type: 'signal', index: doc._index, - depth: doc._source.signal.depth, + // We first look for signal.depth and use that if it exists. If it doesn't exist, this should be a pre-7.10 signal + // and should have signal.parent.depth instead. signal.parent.depth is treated as equivalent to signal.depth. + depth: doc._source.signal.depth ?? doc._source.signal?.parent?.depth ?? 1, }; } else { return { @@ -37,8 +39,8 @@ export const buildAncestorsSignal = (doc: SignalSourceHit): Signal['ancestors'] }; export const buildSignal = (docs: SignalSourceHit[], rule: Partial): Signal => { - const depth = docs.reduce((acc, doc) => Math.max(doc._source.signal?.depth ?? 0, acc), 0) + 1; const parents = docs.map(buildParent); + const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; const ancestors = docs.reduce( (acc: Ancestor[], doc) => acc.concat(buildAncestorsSignal(doc)), [] diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index b02e19752faf8..1e9f9389bab50 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -52,7 +52,8 @@ export interface SignalSource { rule: { id: string; }; - depth: number; + // signal.depth doesn't exist on pre-7.10 signals + depth?: number; }; } From a3e1559ececfd337de1e65c15f3dafab17e0f8be Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 2 Sep 2020 16:18:35 -0400 Subject: [PATCH 3/9] Comment build_signal functions --- .../detection_engine/signals/build_signal.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 00ad0ea0c52eb..dd5ce3de79f52 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -7,6 +7,11 @@ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { SignalSourceHit, Signal, Ancestor } from './types'; +/** + * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child + * signal's `signal.parents` array. + * @param doc The parent signal or event + */ export const buildParent = (doc: SignalSourceHit): Ancestor => { if (doc._source.signal != null) { return { @@ -28,6 +33,11 @@ export const buildParent = (doc: SignalSourceHit): Ancestor => { } }; +/** + * Takes a parent signal or event document with N ancestors and adds the parent document to the ancestry array, + * creating an array of N+1 ancestors. + * @param doc The parent signal/event for which to extend the ancestry. + */ export const buildAncestorsSignal = (doc: SignalSourceHit): Signal['ancestors'] => { const newAncestor = buildParent(doc); const existingAncestors = doc._source.signal?.ancestors; @@ -38,6 +48,11 @@ export const buildAncestorsSignal = (doc: SignalSourceHit): Signal['ancestors'] } }; +/** + * Builds the `signal.*` fields that are common across all signals. + * @param docs The parent signals/events of the new signal to be built. + * @param rule The rule that is generating the new signal. + */ export const buildSignal = (docs: SignalSourceHit[], rule: Partial): Signal => { const parents = docs.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; @@ -54,6 +69,10 @@ export const buildSignal = (docs: SignalSourceHit[], rule: Partial) }; }; +/** + * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. + * @param doc The parent signal/event of the new signal to be built. + */ export const additionalSignalFields = (doc: SignalSourceHit) => { return { original_time: doc._source['@timestamp'], From 2c9d0388870a48a96230c372eb732a9f3d8a7c52 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 3 Sep 2020 16:01:20 -0400 Subject: [PATCH 4/9] Rename buildAncestorsSignal to buildAncestors --- .../lib/detection_engine/signals/build_signal.test.ts | 11 +++-------- .../lib/detection_engine/signals/build_signal.ts | 7 ++----- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index f3806b458ef73..dbc635e3815fd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -5,12 +5,7 @@ */ import { sampleDocNoSortId } from './__mocks__/es_results'; -import { - buildSignal, - buildParent, - buildAncestorsSignal, - additionalSignalFields, -} from './build_signal'; +import { buildSignal, buildParent, buildAncestors, additionalSignalFields } from './build_signal'; import { Signal, Ancestor } from './types'; import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; @@ -215,7 +210,7 @@ describe('buildSignal', () => { kind: 'event', module: 'system', }; - const signal = buildAncestorsSignal(doc); + const signal = buildAncestors(doc); const expected: Ancestor[] = [ { id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', @@ -255,7 +250,7 @@ describe('buildSignal', () => { }, depth: 1, }; - const signal = buildAncestorsSignal(doc); + const signal = buildAncestors(doc); const expected: Ancestor[] = [ { id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index dd5ce3de79f52..c78425f35773a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -38,7 +38,7 @@ export const buildParent = (doc: SignalSourceHit): Ancestor => { * creating an array of N+1 ancestors. * @param doc The parent signal/event for which to extend the ancestry. */ -export const buildAncestorsSignal = (doc: SignalSourceHit): Signal['ancestors'] => { +export const buildAncestors = (doc: SignalSourceHit): Signal['ancestors'] => { const newAncestor = buildParent(doc); const existingAncestors = doc._source.signal?.ancestors; if (existingAncestors != null) { @@ -56,10 +56,7 @@ export const buildAncestorsSignal = (doc: SignalSourceHit): Signal['ancestors'] export const buildSignal = (docs: SignalSourceHit[], rule: Partial): Signal => { const parents = docs.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; - const ancestors = docs.reduce( - (acc: Ancestor[], doc) => acc.concat(buildAncestorsSignal(doc)), - [] - ); + const ancestors = docs.reduce((acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), []); return { parents, ancestors, From 93094ab396da02708936d1a5aca55e2101c6a048 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Fri, 4 Sep 2020 12:48:44 -0400 Subject: [PATCH 5/9] Update detection engine depth test scripts and docs --- .../signals_on_signals/depth_test/README.md | 239 ++++++++++-------- .../depth_test/query_single_id.json | 2 +- .../depth_test/signal_on_signal_depth_1.json | 4 +- .../depth_test/signal_on_signal_depth_2.json | 4 +- 4 files changed, 141 insertions(+), 108 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md index 2310ba979da20..f829aef567755 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md @@ -22,7 +22,7 @@ which will write a single signal document into the signals index by searching fo signal_on_signal_depth_1.json ``` -which has this key part of its query: `"query": "signal.parent.depth: 1 and _id: *"` which will only create signals +which has this key part of its query: `"query": "signal.depth: 1 and _id: *"` which will only create signals from all signals that point directly to an event (signal -> event). Then a second rule called @@ -34,7 +34,7 @@ signal_on_signal_depth_2.json which will only create signals from all signals that point directly to another signal (signal -> signal) with this query ```json -"query": "signal.parent.depth: 2 and _id: *" +"query": "signal.depth: 2 and _id: *" ``` ## Setup @@ -90,38 +90,39 @@ And then you can query against that: GET .siem-signals-default/_search ``` -Check your parent section of the signal and you will see something like this: +Check your signal.parents section of the signal and you will see something like this: ```json -"parent" : { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, ] ``` -The parent and ancestors structure is defined as: +The parents structure is defined as: ``` -rule -> The id of the rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 +rule -> The id of the rule, if the parent was generated by a rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 id -> The original _id of the document type -> The type of the document, it will be either event or signal index -> The original location of the index -depth -> The depth of this signal. It will be at least 1 to indicate it is a signal generated from a event. Otherwise 2 or more to indicate a signal on signal and what depth we are at -ancestors -> An array tracking all of the parents of this particular signal. As depth increases this will too. +depth -> The depth of the parent event/signal. It will be 0 if the parent is an event, or 1+ if the parent is another signal. ``` +The ancestors structure has the same fields as parents, but is an array of all ancestors (parents, grandparents, etc) of the signal. + This is indicating that you have a single parent of an event from the signal (signal -> event) and this document has a single ancestor of that event. Each 30 seconds that goes it will use de-duplication technique to ensure that this signal is not re-inserted. If after each 30 seconds you DO SEE multiple signals then the bug is a de-duplication bug and a critical bug. If you ever see a duplicate rule in the @@ -138,55 +139,63 @@ running in the system which are generating signals on top of signals. After 30 s documents in the signals index. The first signal is our original (signal -> event) document with a rule id: ```json -"parent" : { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, +"parents" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] -``` +], +"depth": 1, +"rule": { + "id": "74e0dd0c-4609-416f-b65e-90f8b2564612" +} and the second document is a signal on top of a signal like so: ```json -"parent" : { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", - "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 2 -}, -"ancestors" : [ +"parents" : [ { "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", + "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 1 + } +] +"ancestors" : [ + { "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "1d3b3735-66ef-4e53-b7f5-4340026cc40c" +} ``` Notice that the depth indicates it is at level 2 and its parent is that of a signal. Also notice that the ancestors is an array of size 2 indicating that this signal terminates at an event. Each and every signal ancestors array should terminate at an event and should ONLY contain 1 -event and NEVER 2 or more events. After 30+ seconds you should NOT see any new documents being created and you should be stable +event and NEVER 2 or more events for KQL query based rules. EQL query based rules that use sequences may have multiple parents at the same level. After 30+ seconds you should NOT see any new documents being created and you should be stable at 2. Otherwise we have AND/OR a de-duplication issue, signal on signal issue. Now, post this same rule a second time as a second instance which is going to run against these two documents. @@ -212,79 +221,93 @@ The expected behavior is that eventually you will get 3 total documents but not The original event rule 74e0dd0c-4609-416f-b65e-90f8b2564612 (event -> signal) ```json -"parent" : { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, +"parents" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"depth": 1, +"rule": { + "id": "74e0dd0c-4609-416f-b65e-90f8b2564612" +} ``` The first signal to signal rule 1d3b3735-66ef-4e53-b7f5-4340026cc40c (signal -> event) ```json -"parent" : { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", - "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 2 -}, -"ancestors" : [ +"parents" : [ { "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", + "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 1 + } +] +"ancestors" : [ + { "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "1d3b3735-66ef-4e53-b7f5-4340026cc40c" +} ``` Then our second signal to signal rule c93ddb57-e7e9-4973-9886-72ddefb4d22e (signal -> event) which finds the same thing as the first signal to signal ```json -"parent" : { - "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", - "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 2 -}, -"ancestors" : [ +"parents" : [ { "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", + "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 1 + } +], +"ancestors" : [ + { "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "c93ddb57-e7e9-4973-9886-72ddefb4d22e" +} ``` We should be able to post this depth level as many times as we want and get only 1 new document each time. If we decide though to @@ -298,69 +321,79 @@ The expectation is that a document for each of the previous depth 1 documents wo depth 1 rules running then the signals at depth 2 will produce two new ones and those two will look like so: ```json -"parent" : { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", - "id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 3 -}, +"parents" : [ + { + "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 2 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", + "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", "id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "a1f7b520-5bfd-451d-af59-428f60753fee" +} ``` ```json -"parent" : { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", - "id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 3 -}, +"parents" : [ + { + "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", + "id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 2 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", + "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", "id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "a1f7b520-5bfd-451d-af59-428f60753fee" +} ``` The total number of documents should be 5 at this point. If you were to post this same rule a second time to get a second instance diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json index dc05c656d7cf1..305aa34992623 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json @@ -7,6 +7,6 @@ "from": "now-1d", "interval": "30s", "to": "now", - "query": "_id: o8G7vm8BvLT8jmu5B1-M", + "query": "event.id: 08cde4aa-d249-4e6b-8300-06f3d56c7fe7", "enabled": true } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json index fb13413a02791..c9132ddb0a590 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json @@ -7,7 +7,7 @@ "from": "now-1d", "interval": "30s", "to": "now", - "query": "signal.parent.depth: 1 and _id: *", + "query": "signal.depth: 1 and _id: *", "enabled": true, - "index": ".siem-signals-default" + "index": [".siem-signals-default"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json index c1b7594653ec7..d1a2749792686 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json @@ -7,7 +7,7 @@ "from": "now-1d", "interval": "30s", "to": "now", - "query": "signal.parent.depth: 2 and _id: *", + "query": "signal.depth: 2 and _id: *", "enabled": true, - "index": ".siem-signals-default" + "index": [".siem-signals-default"] } From 08a2a4d0e97ecd6b4b4508b0fd0d44b902a53754 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Fri, 4 Sep 2020 13:07:08 -0400 Subject: [PATCH 6/9] Update halting test readme --- .../signals_on_signals/depth_test/README.md | 9 +- .../signals_on_signals/halting_test/README.md | 182 ++++++++++-------- 2 files changed, 112 insertions(+), 79 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md index f829aef567755..112a1d9248d8e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md @@ -90,7 +90,7 @@ And then you can query against that: GET .siem-signals-default/_search ``` -Check your signal.parents section of the signal and you will see something like this: +Check your `signal` section of the signal and you will see something like this: ```json "parents" : [ @@ -108,7 +108,11 @@ Check your signal.parents section of the signal and you will see something like "index" : "filebeat-8.0.0-2019.12.18-000001", "depth" : 0 }, -] +], +"depth": 1, +"rule": { + "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" +} ``` The parents structure is defined as: @@ -159,6 +163,7 @@ documents in the signals index. The first signal is our original (signal -> even "rule": { "id": "74e0dd0c-4609-416f-b65e-90f8b2564612" } +``` and the second document is a signal on top of a signal like so: diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md index b1a83f5317776..01b21bf762e44 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md @@ -69,38 +69,43 @@ And then you can query against that: GET .siem-signals-default/_search ``` -Check your parent section of the signal and you will see something like this: +Check your `signal` section of the signal and you will see something like this: ```json -"parent" : { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" +} ``` -The parent and ancestors structure is defined as: +The parents structure is defined as: ``` -rule -> The id of the rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 +rule -> The id of the rule, if the parent was generated by a rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 id -> The original _id of the document type -> The type of the document, it will be either event or signal index -> The original location of the index -depth -> The depth of this signal. It will be at least 1 to indicate it is a signal generated from a event. Otherwise 2 or more to indicate a signal on signal and what depth we are at -ancestors -> An array tracking all of the parents of this particular signal. As depth increases this will too. +depth -> The depth of the parent event/signal. It will be 0 if the parent is an event, or 1+ if the parent is another signal. ``` +The ancestors structure has the same fields as parents, but is an array of all ancestors (parents, grandparents, etc) of the signal. + This is indicating that you have a single parent of an event from the signal (signal -> event) and this document has a single ancestor of that event. Each 30 seconds that goes it will use de-duplication technique to ensure that this signal is not re-inserted. If after each 30 seconds you DO SEE multiple signals then the bug is a de-duplication bug and a critical bug. If you ever see a duplicate rule in the @@ -119,22 +124,26 @@ documents in the signals index. The first signal is our original (signal -> even (signal -> event) ```json -"parent" : { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" +} ``` and the second document is a signal on top of a signal like so: @@ -143,28 +152,31 @@ and the second document is a signal on top of a signal like so: ```json "parent" : { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "161fa5b8-0b96-4985-b066-0d99b2bcb904" +} ``` Notice that the depth indicates it is at level 2 and its parent is that of a signal. Also notice that the ancestors is an array of size 2 @@ -195,50 +207,57 @@ The expected behavior is that eventually you will get 5 total documents but not The original event rule ded57b36-9c4e-4ee4-805d-be4e92033e41 (event -> signal) ```json -"parent" : { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" +} ``` The first signal to signal rule 161fa5b8-0b96-4985-b066-0d99b2bcb904 (signal -> event) ```json "parent" : { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "161fa5b8-0b96-4985-b066-0d99b2bcb904" +} ``` Then our second signal to signal rule f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406 (signal -> event) which finds the same thing as the first @@ -246,28 +265,31 @@ signal to signal ```json "parent" : { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406" +} ``` But then f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406 also finds the first signal to signal rule from 161fa5b8-0b96-4985-b066-0d99b2bcb904 @@ -275,35 +297,38 @@ and writes that document out with a depth of 3. (signal -> signal -> event) ```json "parent" : { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", "id" : "c627e5e2576f1b10952c6c57249947e89b6153b763a59fb9e391d0b56be8e7fe", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", "id" : "c627e5e2576f1b10952c6c57249947e89b6153b763a59fb9e391d0b56be8e7fe", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406" +} ``` Since it wrote that document, the first signal to signal 161fa5b8-0b96-4985-b066-0d99b2bcb904 writes out it found this newly created signal @@ -311,35 +336,38 @@ Since it wrote that document, the first signal to signal 161fa5b8-0b96-4985-b066 ```json "parent" : { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", "id" : "efbe514e8d806a5ef3da7658cfa73961e25befefc84f622e963b45dcac798868", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", "id" : "efbe514e8d806a5ef3da7658cfa73961e25befefc84f622e963b45dcac798868", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "161fa5b8-0b96-4985-b066-0d99b2bcb904" +} ``` You will be "halted" at this point as the signal ancestry and de-duplication ensures that we do not report twice on signals and that we do not From 9143432795b244c1792954fc260e4ab553ffff90 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Fri, 4 Sep 2020 13:10:35 -0400 Subject: [PATCH 7/9] Match up rule ids in readme --- .../rules/test_cases/signals_on_signals/depth_test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md index 112a1d9248d8e..7cf7d11e4c1f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md @@ -111,7 +111,7 @@ Check your `signal` section of the signal and you will see something like this: ], "depth": 1, "rule": { - "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" + "id": "74e0dd0c-4609-416f-b65e-90f8b2564612" } ``` From f3b294e888eb70fd8da846a8ef3c4c17be1dc2b1 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Fri, 4 Sep 2020 18:41:21 -0400 Subject: [PATCH 8/9] Continue populating signal.parent along with signal.parents --- .../signals/build_bulk_body.test.ts | 24 +++++++++++++++++ .../signals/build_signal.test.ts | 26 ++++++++++++++----- .../detection_engine/signals/build_signal.ts | 3 ++- .../lib/detection_engine/signals/types.ts | 2 ++ 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 0840c4592676a..967dc5331e46b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -47,6 +47,12 @@ describe('buildBulkBody', () => { kind: 'signal', }, signal: { + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, parents: [ { id: sampleIdGuid, @@ -151,6 +157,12 @@ describe('buildBulkBody', () => { kind: 'event', module: 'system', }, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, parents: [ { id: sampleIdGuid, @@ -253,6 +265,12 @@ describe('buildBulkBody', () => { dataset: 'socket', module: 'system', }, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, parents: [ { id: sampleIdGuid, @@ -348,6 +366,12 @@ describe('buildBulkBody', () => { original_event: { kind: 'event', }, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, parents: [ { id: sampleIdGuid, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index dbc635e3815fd..d684807a09126 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -23,6 +23,12 @@ describe('buildSignal', () => { ...additionalSignalFields(doc), }; const expected: Signal = { + parent: { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, parents: [ { id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', @@ -87,6 +93,12 @@ describe('buildSignal', () => { ...additionalSignalFields(doc), }; const expected: Signal = { + parent: { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, parents: [ { id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', @@ -231,12 +243,14 @@ describe('buildSignal', () => { module: 'system', }; doc._source.signal = { - parent: { - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, + parents: [ + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index c78425f35773a..6d35c604d91b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -20,7 +20,7 @@ export const buildParent = (doc: SignalSourceHit): Ancestor => { type: 'signal', index: doc._index, // We first look for signal.depth and use that if it exists. If it doesn't exist, this should be a pre-7.10 signal - // and should have signal.parent.depth instead. signal.parent.depth is treated as equivalent to signal.depth. + // and should have signal.parent.depth instead. signal.parent.depth in this case is treated as equivalent to signal.depth. depth: doc._source.signal.depth ?? doc._source.signal?.parent?.depth ?? 1, }; } else { @@ -72,6 +72,7 @@ export const buildSignal = (docs: SignalSourceHit[], rule: Partial) */ export const additionalSignalFields = (doc: SignalSourceHit) => { return { + parent: buildParent(doc), original_time: doc._source['@timestamp'], original_event: doc._source.event ?? undefined, threshold_count: doc._source.threshold_count ?? undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 1e9f9389bab50..700a8fb5022d7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -130,6 +130,8 @@ export interface Ancestor { export interface Signal { rule: Partial; + // DEPRECATED: use parents instead of parent + parent?: Ancestor; parents: Ancestor[]; ancestors: Ancestor[]; original_time?: string; From 0a8caa56634f59ccc2f0fe44f96552a9bcb71fc0 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 8 Sep 2020 13:53:18 -0400 Subject: [PATCH 9/9] pr comments --- .../server/lib/detection_engine/signals/build_signal.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 6d35c604d91b3..78818779dd661 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -21,7 +21,7 @@ export const buildParent = (doc: SignalSourceHit): Ancestor => { index: doc._index, // We first look for signal.depth and use that if it exists. If it doesn't exist, this should be a pre-7.10 signal // and should have signal.parent.depth instead. signal.parent.depth in this case is treated as equivalent to signal.depth. - depth: doc._source.signal.depth ?? doc._source.signal?.parent?.depth ?? 1, + depth: doc._source.signal.depth ?? doc._source.signal.parent?.depth ?? 1, }; } else { return { @@ -38,7 +38,7 @@ export const buildParent = (doc: SignalSourceHit): Ancestor => { * creating an array of N+1 ancestors. * @param doc The parent signal/event for which to extend the ancestry. */ -export const buildAncestors = (doc: SignalSourceHit): Signal['ancestors'] => { +export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => { const newAncestor = buildParent(doc); const existingAncestors = doc._source.signal?.ancestors; if (existingAncestors != null) {