From 34c35aef3585aad054b133add122c74731a7f67c Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Fri, 4 Jun 2021 16:19:59 -0600 Subject: [PATCH] [Security Solutions][Detection Engine] Fixes timestamp bugs within source indexes when the formats are not ISO8601 format (#101349) ## Summary We have a few bugs where when the source index for detections is not `"strict_date_optional_time"` it is possible that we will misinterpret the format to be epoch milliseconds when it could be epoch seconds or another ambiguous format or blow up when trying to write out the signals index. This fixes it to where we query for the source index format as an ISO8601 and when we copy the date time format we copy it back out as ISO8601 and insert it into the signal index as ISO8601. See this [gist](https://gist.github.com/FrankHassanabad/f614ec9762d59cd1129b3269f5bae41c) for more details of how this was accidentally introduced when we added support for runtime fields and the general idea of the fix. * Removes `docvalue_field` and we now only use `fields` in detection engine search requests * Splits out the timestamp e2e tests into their own file for `timestamps` file * Adds more tests to ensure we copy what we expect and we are converting to ISO8601 in the signals * Removes `ts-expect-error` in a lot of areas including tests and then I fix the types and issues once it is removed. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../signals/__mocks__/es_results.ts | 12 +- .../signals/build_bulk_body.test.ts | 60 ++--- .../signals/build_event_type_signal.test.ts | 3 - .../signals/build_events_query.test.ts | 63 +++--- .../signals/build_events_query.ts | 6 +- .../signals/build_rule.test.ts | 1 - .../signals/build_signal.test.ts | 12 +- .../detection_engine/signals/build_signal.ts | 9 +- .../build_rule_name_from_mapping.test.ts | 1 - .../signals/single_search_after.test.ts | 10 +- .../signals/single_search_after.ts | 2 +- .../detection_engine/signals/utils.test.ts | 86 +++++++- .../lib/detection_engine/signals/utils.ts | 58 +++-- .../tests/generating_signals.ts | 113 ---------- .../security_and_spaces/tests/index.ts | 1 + .../security_and_spaces/tests/timestamps.ts | 208 ++++++++++++++++++ .../es_archives/security_solution/README.md | 11 + .../timestamp_in_seconds/data.json | 10 + .../timestamp_in_seconds/mappings.json | 22 ++ .../timestamp_override/mappings.json | 1 + .../timestamp_override_1/mappings.json | 1 + .../timestamp_override_2/mappings.json | 1 + .../timestamp_override_3/mappings.json | 1 + .../timestamp_override_4/data.json | 2 +- .../timestamp_override_4/mappings.json | 1 + .../timestamp_override_5/data.json | 14 ++ .../timestamp_override_5/mappings.json | 39 ++++ 27 files changed, 505 insertions(+), 243 deletions(-) create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts create mode 100644 x-pack/test/functional/es_archives/security_solution/README.md create mode 100644 x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json create mode 100644 x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json create mode 100644 x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json create mode 100644 x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 1590a4f0fbb04..0fed141ca4dbc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -146,7 +146,7 @@ export const sampleDocWithSortId = ( export const sampleDocNoSortId = ( someUuid: string = sampleIdGuid, ip?: string -): SignalSourceHit => ({ +): SignalSourceHit & { _source: Required['_source'] } => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -225,12 +225,12 @@ export const sampleWrappedSignalHit = (): WrappedSignalHit => { }; }; -export const sampleDocWithAncestors = (): SignalSearchResponse => { +export const sampleDocWithAncestors = (): SignalSearchResponse & { + hits: { hits: Array> }; +} => { const sampleDoc = sampleDocNoSortId(); delete sampleDoc.sort; - // @ts-expect-error @elastic/elasticsearch _source is optional delete sampleDoc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional sampleDoc._source.signal = { parent: { id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', @@ -562,7 +562,9 @@ export const sampleBulkCreateErrorResult = { export const sampleDocSearchResultsNoSortId = ( someUuid: string = sampleIdGuid -): SignalSearchResponse => ({ +): SignalSearchResponse & { + hits: { hits: Array> }; +} => ({ took: 10, timed_out: false, _shards: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 743d9580218a3..4d3ca26f5a71e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -24,6 +24,11 @@ import { SignalHit, SignalSourceHit } from './types'; import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; import { getQueryRuleParams, getThresholdRuleParams } from '../schemas/rule_schemas.mock'; +// This allows us to not have to use ts-expect-error with delete in the code. +type SignalHitOptionalTimestamp = Omit & { + '@timestamp'?: SignalHit['@timestamp']; +}; + describe('buildBulkBody', () => { beforeEach(() => { jest.clearAllMocks(); @@ -32,11 +37,9 @@ describe('buildBulkBody', () => { test('bulk body builds well-defined body', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -69,7 +72,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -81,9 +84,8 @@ describe('buildBulkBody', () => { test('bulk body builds well-defined body with threshold results', () => { const ruleSO = sampleRuleSO(getThresholdRuleParams()); const baseDoc = sampleDocNoSortId(); - const doc: SignalSourceHit = { + const doc: SignalSourceHit & { _source: Required['_source'] } = { ...baseDoc, - // @ts-expect-error @elastic/elasticsearch _source is optional _source: { ...baseDoc._source, threshold_result: { @@ -96,11 +98,9 @@ describe('buildBulkBody', () => { }, }, }; - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -133,7 +133,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: { ...expectedRule(), @@ -167,18 +167,15 @@ describe('buildBulkBody', () => { test('bulk body builds original_event if it exists on the event to begin with', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', module: 'system', dataset: 'socket', kind: 'event', }; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -220,7 +217,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -232,17 +229,14 @@ describe('buildBulkBody', () => { test('bulk body builds original_event if it exists on the event to begin with but no kind information', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', module: 'system', dataset: 'socket', }; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -283,7 +277,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -295,15 +289,12 @@ describe('buildBulkBody', () => { test('bulk body builds original_event if it exists on the event to begin with with only kind information', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.source; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { kind: 'event', }; - const fakeSignalSourceHit = buildBulkBody(ruleSO, doc); + const fakeSignalSourceHit: SignalHitOptionalTimestamp = buildBulkBody(ruleSO, doc); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete fakeSignalSourceHit['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -339,7 +330,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -351,7 +342,6 @@ describe('buildBulkBody', () => { test('bulk body builds "original_signal" if it exists already as a numeric', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const sampleDoc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete sampleDoc._source.source; const doc = ({ ...sampleDoc, @@ -393,7 +383,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -405,7 +395,6 @@ describe('buildBulkBody', () => { test('bulk body builds "original_signal" if it exists already as an object', () => { const ruleSO = sampleRuleSO(getQueryRuleParams()); const sampleDoc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete sampleDoc._source.source; const doc = ({ ...sampleDoc, @@ -447,7 +436,7 @@ describe('buildBulkBody', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: expectedRule(), depth: 1, @@ -466,9 +455,8 @@ describe('buildSignalFromSequence', () => { block2._source.new_key = 'new_key_value'; const blocks = [block1, block2]; const ruleSO = sampleRuleSO(getQueryRuleParams()); - const signal = buildSignalFromSequence(blocks, ruleSO); + const signal: SignalHitOptionalTimestamp = buildSignalFromSequence(blocks, ruleSO); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete signal['@timestamp']; const expected: Omit & { new_key: string } = { new_key: 'new_key_value', @@ -552,9 +540,8 @@ describe('buildSignalFromSequence', () => { block2._source['@timestamp'] = '2021-05-20T22:28:46+0000'; block2._source.someKey = 'someOtherValue'; const ruleSO = sampleRuleSO(getQueryRuleParams()); - const signal = buildSignalFromSequence([block1, block2], ruleSO); + const signal: SignalHitOptionalTimestamp = buildSignalFromSequence([block1, block2], ruleSO); // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete signal['@timestamp']; const expected: Omit = { event: { @@ -635,12 +622,11 @@ describe('buildSignalFromSequence', () => { describe('buildSignalFromEvent', () => { test('builds a basic signal from a single event', () => { const ancestor = sampleDocWithAncestors().hits.hits[0]; - // @ts-expect-error @elastic/elasticsearch _source is optional delete ancestor._source.source; const ruleSO = sampleRuleSO(getQueryRuleParams()); - const signal = buildSignalFromEvent(ancestor, ruleSO, true); + const signal: SignalHitOptionalTimestamp = buildSignalFromEvent(ancestor, ruleSO, true); + // Timestamp will potentially always be different so remove it for the test - // @ts-expect-error delete signal['@timestamp']; const expected: Omit & { someKey: 'someValue' } = { someKey: 'someValue', @@ -651,7 +637,7 @@ describe('buildSignalFromEvent', () => { _meta: { version: SIGNALS_TEMPLATE_VERSION, }, - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', parent: { id: sampleIdGuid, rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts index 185c165442921..0ae81770e83c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts @@ -16,7 +16,6 @@ describe('buildEventTypeSignal', () => { test('it returns the event appended of kind signal if it does not exist', () => { const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.event; const eventType = buildEventTypeSignal(doc); const expected: object = { kind: 'signal' }; @@ -25,7 +24,6 @@ describe('buildEventTypeSignal', () => { test('it returns the event appended of kind signal if it is an empty object', () => { const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = {}; const eventType = buildEventTypeSignal(doc); const expected: object = { kind: 'signal' }; @@ -34,7 +32,6 @@ describe('buildEventTypeSignal', () => { test('it returns the event with kind signal and other properties if they exist', () => { const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', module: 'system', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts index 3f4a17dc091ab..28cea9ea22b0d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts @@ -24,12 +24,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -67,6 +61,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], sort: [ { @@ -96,16 +94,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: 'event.ingested', - format: 'strict_date_optional_time', - }, - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -167,6 +155,14 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: 'event.ingested', + format: 'strict_date_optional_time', + }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], sort: [ { @@ -203,12 +199,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -246,6 +236,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], search_after: [fakeSortId], sort: [ @@ -276,12 +270,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -319,6 +307,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], search_after: [fakeSortIdNumber], sort: [ @@ -348,12 +340,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [ - { - field: '@timestamp', - format: 'strict_date_optional_time', - }, - ], query: { bool: { filter: [ @@ -391,6 +377,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], sort: [ { @@ -427,7 +417,6 @@ describe('create_signals', () => { size: 100, ignore_unavailable: true, body: { - docvalue_fields: [{ field: '@timestamp', format: 'strict_date_optional_time' }], query: { bool: { filter: [ @@ -465,6 +454,10 @@ describe('create_signals', () => { field: '*', include_unmapped: true, }, + { + field: '@timestamp', + format: 'strict_date_optional_time', + }, ], aggregations: { tags: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 86fb51e4785ad..0414439580361 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -17,7 +17,7 @@ interface BuildEventsSearchQuery { index: string[]; from: string; to: string; - filter?: estypes.QueryContainer; + filter: estypes.QueryContainer; size: number; sortOrder?: SortOrderOrUndefined; searchAfterSortIds: SortResults | undefined; @@ -94,8 +94,6 @@ export const buildEventsSearchQuery = ({ ]; const filterWithTime: estypes.QueryContainer[] = [ - // but tests contain undefined, so I suppose it's desired behaviour - // @ts-expect-error undefined in not assignable to QueryContainer filter, { bool: { filter: [{ bool: { should: [...rangeFilter], minimum_should_match: 1 } }] } }, ]; @@ -106,7 +104,6 @@ export const buildEventsSearchQuery = ({ size, ignore_unavailable: true, body: { - docvalue_fields: docFields, query: { bool: { filter: [ @@ -122,6 +119,7 @@ export const buildEventsSearchQuery = ({ field: '*', include_unmapped: true, }, + ...docFields, ], ...(aggregations ? { aggregations } : {}), sort: [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index 412ccf7a40e33..bd5444a325128 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -134,7 +134,6 @@ describe('buildRuleWithOverrides', () => { }, ]; const doc = sampleDocNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.new_risk_score = newRiskScore; const rule = buildRuleWithOverrides(ruleSO, doc._source!); const expected = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index 6408b5fe9de10..3a30da170d3f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -28,7 +28,6 @@ describe('buildSignal', () => { test('it builds a signal as expected without original_event if event does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional delete doc._source.event; const rule = getRulesSchemaMock(); const signal = { @@ -61,7 +60,7 @@ describe('buildSignal', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', status: 'open', rule: { author: [], @@ -105,7 +104,6 @@ describe('buildSignal', () => { test('it builds a signal as expected with original_event if is present', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', @@ -143,7 +141,7 @@ describe('buildSignal', () => { depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', + original_time: '2020-04-20T21:27:45.000Z', original_event: { action: 'socket_opened', dataset: 'socket', @@ -193,7 +191,6 @@ describe('buildSignal', () => { test('it builds a ancestor correctly if the parent does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', @@ -212,14 +209,12 @@ describe('buildSignal', () => { test('it builds a ancestor correctly if the parent does exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', kind: 'event', module: 'system', }; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.signal = { parents: [ { @@ -255,7 +250,6 @@ describe('buildSignal', () => { test('it builds a signal ancestor correctly if the parent does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', @@ -276,14 +270,12 @@ describe('buildSignal', () => { test('it builds a signal ancestor correctly if the parent does exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.event = { action: 'socket_opened', dataset: 'socket', kind: 'event', module: 'system', }; - // @ts-expect-error @elastic/elasticsearch _source is optional doc._source.signal = { parents: [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 237536a99c0f0..a415c83e857c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -10,6 +10,7 @@ import { RulesSchema } from '../../../../common/detection_engine/schemas/respons import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; import { isEventTypeSignal } from './build_event_type_signal'; import { Signal, Ancestor, BaseSignalHit, ThresholdResult } from './types'; +import { getValidDateFromDoc } from './utils'; /** * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child @@ -103,6 +104,7 @@ const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is Thr /** * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. + * We copy the original time from the document as "original_time" since we override the timestamp with the current date time. * @param doc The parent signal/event of the new signal to be built. */ export const additionalSignalFields = (doc: BaseSignalHit) => { @@ -110,10 +112,13 @@ export const additionalSignalFields = (doc: BaseSignalHit) => { if (thresholdResult != null && !isThresholdResult(thresholdResult)) { throw new Error(`threshold_result failed to validate: ${thresholdResult}`); } + const originalTime = getValidDateFromDoc({ + doc, + timestampOverride: undefined, + }); return { parent: buildParent(removeClashes(doc)), - // @ts-expect-error @elastic/elasticsearch _source is optional - original_time: doc._source['@timestamp'], // This field has already been replaced with timestampOverride, if provided. + original_time: originalTime != null ? originalTime.toISOString() : undefined, original_event: doc._source?.event ?? undefined, threshold_result: thresholdResult, original_signal: diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts index b6281b637d434..23e5aecc5c553 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/mappings/build_rule_name_from_mapping.test.ts @@ -15,7 +15,6 @@ describe('buildRuleNameFromMapping', () => { test('rule name defaults to provided if mapping is incomplete', () => { const ruleName = buildRuleNameFromMapping({ - // @ts-expect-error @elastic/elasticsearch _source is optional eventSource: sampleDocNoSortId()._source, ruleName: 'rule-name', ruleNameMapping: 'message', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts index a40459d312b9f..a67016491aaef 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts @@ -41,7 +41,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -59,7 +59,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -109,7 +109,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -132,7 +132,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }); @@ -152,7 +152,7 @@ describe('singleSearchAfter', () => { services: mockService, logger: mockLogger, pageSize: 1, - filter: undefined, + filter: {}, timestampOverride: undefined, buildRuleMessage, }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 57ed05bcb27cf..ae22964eced92 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -32,7 +32,7 @@ interface SingleSearchAfterParams { logger: Logger; pageSize: number; sortOrder?: SortOrderOrUndefined; - filter?: estypes.QueryContainer; + filter: estypes.QueryContainer; timestampOverride: TimestampOverrideOrUndefined; buildRuleMessage: BuildRuleMessage; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index f49492939eeb1..60bf0ec337f3d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -40,6 +40,7 @@ import { lastValidDate, calculateThresholdSignalUuid, buildChunkedOrFilter, + getValidDateFromDoc, } from './utils'; import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; import { @@ -54,6 +55,7 @@ import { sampleDocSearchResultsNoSortIdNoHits, repeatedSearchResultsWithSortId, sampleDocSearchResultsNoSortId, + sampleDocNoSortId, } from './__mocks__/es_results'; import { ShardError } from '../../types'; @@ -1172,7 +1174,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a non-existent @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; @@ -1186,7 +1187,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a null @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; @@ -1200,7 +1200,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from an invalid @timestamp string', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid'; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid']; @@ -1216,7 +1215,6 @@ describe('utils', () => { describe('lastValidDate', () => { test('It returns undefined if the search result contains a null timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; @@ -1227,7 +1225,6 @@ describe('utils', () => { test('It returns undefined if the search result contains a undefined timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; @@ -1238,7 +1235,6 @@ describe('utils', () => { test('It returns undefined if the search result contains an invalid string value', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid value'; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid value']; @@ -1294,6 +1290,84 @@ describe('utils', () => { }); }); + describe('getValidDateFromDoc', () => { + test('It returns undefined if the search result contains a null timestamp', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = null; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = null; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns undefined if the search result contains a undefined timestamp', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = undefined; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = undefined; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns undefined if the search result contains an invalid string value', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = 'invalid value'; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = ['invalid value']; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns normal date time if set', () => { + const doc = sampleDocNoSortId(); + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date?.toISOString()).toEqual('2020-04-20T21:27:45.000Z'); + }); + + test('It returns date time from field if set there', () => { + const timestamp = '2020-10-07T19:27:19.136Z'; + let doc = sampleDocNoSortId(); + if (doc == null) { + throw new TypeError('Test requires one element'); + } + doc = { + ...doc, + fields: { + '@timestamp': [timestamp], + }, + }; + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date?.toISOString()).toEqual(timestamp); + }); + + test('It returns timestampOverride date time if set', () => { + const override = '2020-10-07T19:20:28.049Z'; + const doc = sampleDocNoSortId(); + doc._source.different_timestamp = new Date(override).toISOString(); + const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' }); + expect(date?.toISOString()).toEqual(override); + }); + + test('It returns timestampOverride date time from fields if set on it', () => { + const override = '2020-10-07T19:36:31.110Z'; + let doc = sampleDocNoSortId(); + if (doc == null) { + throw new TypeError('Test requires one element'); + } + doc = { + ...doc, + fields: { + different_timestamp: [override], + }, + }; + const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' }); + expect(date?.toISOString()).toEqual(override); + }); + }); + describe('createSearchAfterReturnType', () => { test('createSearchAfterReturnType will return full object when nothing is passed', () => { const searchAfterReturnType = createSearchAfterReturnType(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index cc4ed6a45807b..dde9986e8bdf5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -38,6 +38,7 @@ import { Signal, WrappedSignalHit, RuleRangeTuple, + BaseSignalHit, } from './types'; import { BuildRuleMessage } from './rule_messages'; import { ShardError } from '../../types'; @@ -577,30 +578,49 @@ export const lastValidDate = ({ searchResult, timestampOverride, }: { - searchResult: estypes.SearchResponse; + searchResult: SignalSearchResponse; timestampOverride: TimestampOverrideOrUndefined; }): Date | undefined => { if (searchResult.hits.hits.length === 0) { return undefined; } else { const lastRecord = searchResult.hits.hits[searchResult.hits.hits.length - 1]; - const timestamp = timestampOverride ?? '@timestamp'; - const timestampValue = - lastRecord.fields != null && lastRecord.fields[timestamp] != null - ? lastRecord.fields[timestamp][0] - : // @ts-expect-error @elastic/elasticsearch _source is optional - lastRecord._source[timestamp]; - const lastTimestamp = - typeof timestampValue === 'string' || typeof timestampValue === 'number' - ? timestampValue - : undefined; - if (lastTimestamp != null) { - const tempMoment = moment(lastTimestamp); - if (tempMoment.isValid()) { - return tempMoment.toDate(); - } else { - return undefined; - } + return getValidDateFromDoc({ doc: lastRecord, timestampOverride }); + } +}; + +/** + * Given a search hit this will return a valid last date if it can find one, otherwise it + * will return undefined. This tries the "fields" first to get a formatted date time if it can, but if + * it cannot it will resort to using the "_source" fields second which can be problematic if the date time + * is not correctly ISO8601 or epoch milliseconds formatted. + * @param searchResult The result to try and parse out the timestamp. + * @param timestampOverride The timestamp override to use its values if we have it. + */ +export const getValidDateFromDoc = ({ + doc, + timestampOverride, +}: { + doc: BaseSignalHit; + timestampOverride: TimestampOverrideOrUndefined; +}): Date | undefined => { + const timestamp = timestampOverride ?? '@timestamp'; + const timestampValue = + doc.fields != null && doc.fields[timestamp] != null + ? doc.fields[timestamp][0] + : doc._source != null + ? doc._source[timestamp] + : undefined; + const lastTimestamp = + typeof timestampValue === 'string' || typeof timestampValue === 'number' + ? timestampValue + : undefined; + if (lastTimestamp != null) { + const tempMoment = moment(lastTimestamp); + if (tempMoment.isValid()) { + return tempMoment.toDate(); + } else { + return undefined; } } }; @@ -609,7 +629,7 @@ export const createSearchAfterReturnTypeFromResponse = ({ searchResult, timestampOverride, }: { - searchResult: estypes.SearchResponse; + searchResult: SignalSearchResponse; timestampOverride: TimestampOverrideOrUndefined; }): SearchAfterAndBulkCreateReturnType => { return createSearchAfterReturnType({ diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts index f9f378bc4bfa8..8638f6c1bd7ed 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -1616,119 +1616,6 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe('Signals generated from events with timestamp override field and ensures search_after continues to work when documents are missing timestamp override field', () => { - beforeEach(async () => { - await createSignalsIndex(supertest); - await esArchiver.load('auditbeat/hosts'); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest); - await deleteAllAlerts(supertest); - await esArchiver.unload('auditbeat/hosts'); - }); - - /** - * This represents our worst case scenario where this field is not mapped on any index - * We want to check that our logic continues to function within the constraints of search after - * Elasticsearch returns java's long.MAX_VALUE for unmapped date fields - * Javascript does not support numbers this large, but without passing in a number of this size - * The search_after will continue to return the same results and not iterate to the next set - * So to circumvent this limitation of javascript we return the stringified version of Java's - * Long.MAX_VALUE so that search_after does not enter into an infinite loop. - * - * ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620 - */ - it('should generate 200 signals when timestamp override does not exist', async () => { - const rule: QueryCreateSchema = { - ...getRuleForSignalTesting(['auditbeat-*']), - timestamp_override: 'event.fakeingested', - max_signals: 200, - }; - - const { id } = await createRule(supertest, rule); - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 200, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id], 200); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - - expect(signals.length).equal(200); - }); - }); - - /** - * Here we test the functionality of timestamp overrides. If the rule specifies a timestamp override, - * then the documents will be queried and sorted using the timestamp override field. - * If no timestamp override field exists in the indices but one was provided to the rule, - * the rule's query will additionally search for events using the `@timestamp` field - */ - describe('Signals generated from events with timestamp override field', async () => { - beforeEach(async () => { - await deleteSignalsIndex(supertest); - await createSignalsIndex(supertest); - await esArchiver.load('security_solution/timestamp_override_1'); - await esArchiver.load('security_solution/timestamp_override_2'); - await esArchiver.load('security_solution/timestamp_override_3'); - await esArchiver.load('security_solution/timestamp_override_4'); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest); - await deleteAllAlerts(supertest); - await esArchiver.unload('security_solution/timestamp_override_1'); - await esArchiver.unload('security_solution/timestamp_override_2'); - await esArchiver.unload('security_solution/timestamp_override_3'); - await esArchiver.unload('security_solution/timestamp_override_4'); - }); - - it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => { - const rule: QueryCreateSchema = { - ...getRuleForSignalTesting(['myfa*']), - timestamp_override: 'event.ingested', - }; - - const { id } = await createRule(supertest, rule); - - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 3, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id], 3); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - - expect(signalsOrderedByEventId.length).equal(3); - }); - - it('should generate 2 signals with @timestamp', async () => { - const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']); - - const { id } = await createRule(supertest, rule); - - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 2, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id]); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - - expect(signalsOrderedByEventId.length).equal(2); - }); - - it('should generate 2 signals when timestamp override does not exist', async () => { - const rule: QueryCreateSchema = { - ...getRuleForSignalTesting(['myfa*']), - timestamp_override: 'event.fakeingestfield', - }; - const { id } = await createRule(supertest, rule); - - await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); - await waitForSignalsToBePresent(supertest, 2, [id]); - const signalsResponse = await getSignalsByIds(supertest, [id, id]); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - - expect(signalsOrderedByEventId.length).equal(2); - }); - }); - describe('Signals generated from events with name override field', async () => { beforeEach(async () => { await deleteSignalsIndex(supertest); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 5756b02c238ae..00b289a89e4c8 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -42,6 +42,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./create_signals_migrations')); loadTestFile(require.resolve('./finalize_signals_migrations')); loadTestFile(require.resolve('./delete_signals_migrations')); + loadTestFile(require.resolve('./timestamps')); }); // That split here enable us on using a different ciGroup to run the tests diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts new file mode 100644 index 0000000000000..16610e6a44915 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { orderBy } from 'lodash'; +import { QueryCreateSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/request'; + +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + createRule, + waitForRuleSuccessOrStatus, + waitForSignalsToBePresent, + getRuleForSignalTesting, + getSignalsByIds, +} from '../../utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + /** + * Tests around timestamps within signals such as the copying of timestamps correctly into + * the "signal.original_time" field, ensuring that timestamp overrides operate, and ensuring that + * partial errors happen correctly + */ + describe('timestamp tests', () => { + describe('Signals generated from events with a timestamp in seconds is converted correctly into the forced ISO8601 format when copying', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await esArchiver.load('security_solution/timestamp_in_seconds'); + await esArchiver.load('security_solution/timestamp_override_5'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('security_solution/timestamp_in_seconds'); + await esArchiver.unload('security_solution/timestamp_override_5'); + }); + + it('should convert the @timestamp which is epoch_seconds into the correct ISO format', async () => { + const rule = getRuleForSignalTesting(['timestamp_in_seconds']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort(); + expect(hits).to.eql(['2021-06-02T23:33:15.000Z']); + }); + + it('should still use the @timestamp field even with an override field. It should never use the override field', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfakeindex-5']), + timestamp_override: 'event.ingested', + }; + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort(); + expect(hits).to.eql(['2020-12-16T15:16:18.000Z']); + }); + }); + + /** + * Here we test the functionality of timestamp overrides. If the rule specifies a timestamp override, + * then the documents will be queried and sorted using the timestamp override field. + * If no timestamp override field exists in the indices but one was provided to the rule, + * the rule's query will additionally search for events using the `@timestamp` field + */ + describe('Signals generated from events with timestamp override field', async () => { + beforeEach(async () => { + await deleteSignalsIndex(supertest); + await createSignalsIndex(supertest); + await esArchiver.load('security_solution/timestamp_override_1'); + await esArchiver.load('security_solution/timestamp_override_2'); + await esArchiver.load('security_solution/timestamp_override_3'); + await esArchiver.load('security_solution/timestamp_override_4'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('security_solution/timestamp_override_1'); + await esArchiver.unload('security_solution/timestamp_override_2'); + await esArchiver.unload('security_solution/timestamp_override_3'); + await esArchiver.unload('security_solution/timestamp_override_4'); + }); + + it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfa*']), + timestamp_override: 'event.ingested', + }; + + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id], 3); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(3); + }); + + it('should generate 2 signals with @timestamp', async () => { + const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']); + + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id]); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(2); + }); + + it('should generate 2 signals when timestamp override does not exist', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfa*']), + timestamp_override: 'event.fakeingestfield', + }; + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id, id]); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(2); + }); + + /** + * We should not use the timestamp override as the "original_time" as that can cause + * confusion if you have both a timestamp and an override in the source event. Instead the "original_time" + * field should only be overridden by the "timestamp" since when we generate a signal + * and we add a new timestamp to the signal. + */ + it('should NOT use the timestamp override as the "original_time"', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfakeindex-2']), + timestamp_override: 'event.ingested', + }; + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id, id]); + const hits = signalsResponse.hits.hits + .map((hit) => hit._source.signal.original_time) + .sort(); + expect(hits).to.eql([undefined]); + }); + }); + + describe('Signals generated from events with timestamp override field and ensures search_after continues to work when documents are missing timestamp override field', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await esArchiver.load('auditbeat/hosts'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('auditbeat/hosts'); + }); + + /** + * This represents our worst case scenario where this field is not mapped on any index + * We want to check that our logic continues to function within the constraints of search after + * Elasticsearch returns java's long.MAX_VALUE for unmapped date fields + * Javascript does not support numbers this large, but without passing in a number of this size + * The search_after will continue to return the same results and not iterate to the next set + * So to circumvent this limitation of javascript we return the stringified version of Java's + * Long.MAX_VALUE so that search_after does not enter into an infinite loop. + * + * ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620 + */ + it('should generate 200 signals when timestamp override does not exist', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['auditbeat-*']), + timestamp_override: 'event.fakeingested', + max_signals: 200, + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 200, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id], 200); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + + expect(signals.length).equal(200); + }); + }); + }); +}; diff --git a/x-pack/test/functional/es_archives/security_solution/README.md b/x-pack/test/functional/es_archives/security_solution/README.md new file mode 100644 index 0000000000000..c832e0835bbbc --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/README.md @@ -0,0 +1,11 @@ +Collection of data sets for use within various tests. Most of the tests to these live in either: + +``` +x-pack/test/detection_engine_api_integrations/security_and_spaces/tests +``` + +or + +``` +x-pack/test/api_integration/apis/security_solution +``` diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json new file mode 100644 index 0000000000000..46b30b239bbc7 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/data.json @@ -0,0 +1,10 @@ +{ + "type": "doc", + "value": { + "index": "timestamp_in_seconds", + "source": { + "@timestamp": 1622676795 + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json new file mode 100644 index 0000000000000..fd8880fe0bc49 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_in_seconds/mappings.json @@ -0,0 +1,22 @@ +{ + "type": "index", + "value": { + "index": "timestamp_in_seconds", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_second" + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json index 085ab34a3d58a..092519a792863 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-1", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json index 085ab34a3d58a..092519a792863 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-1", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json index 49a27a423cdaa..1f1c1673fe1a2 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-2", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json index 736584386a705..a0409280c34eb 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-3", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json index ca7025b36154c..ad0e7cbab7d2b 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json @@ -6,7 +6,7 @@ "message": "hello world 4", "@timestamp": "2020-12-16T15:16:18.570Z", "event": { - "ingested": "2020-12-16T15:16:18.570Z" + "ingested": "2020-12-16T16:16:18.570Z" } }, "type": "_doc" diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json index ab4edc9f300e1..a4e021e45ff9e 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-4", "mappings": { + "dynamic": "strict", "properties": { "@timestamp": { "type": "date" diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json new file mode 100644 index 0000000000000..f2c81e9b5e45e --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json @@ -0,0 +1,14 @@ +{ + "type": "doc", + "value": { + "index": "myfakeindex-5", + "source": { + "@timestamp": 1608131778, + "message": "hello world 4", + "event": { + "ingested": 1622676795 + } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json new file mode 100644 index 0000000000000..a9735aaeca8ef --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json @@ -0,0 +1,39 @@ +{ + "type": "index", + "value": { + "index": "myfakeindex-5", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_second" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "event": { + "properties": { + "ingested": { + "type": "date", + "format": "epoch_second" + } + } + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}