From 6562e56bdf23b63d7c45e30fffc4b3af4352bf15 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 25 Oct 2022 13:28:27 +0100 Subject: [PATCH 01/36] [Security Solution][Alerts] adds support for multi fields in new terms rule --- .../new_terms_attributes.ts | 2 +- .../common/field_maps/field_names.ts | 2 + .../event_details/get_alert_summary_rows.tsx | 11 +- .../components/alerts_table/translations.ts | 7 + .../rules/step_define_rule/schema.tsx | 4 +- .../build_new_terms_aggregation.test.ts.snap | 34 +++++ .../build_new_terms_aggregation.test.ts | 13 +- .../new_terms/build_new_terms_aggregation.ts | 22 +-- .../new_terms/create_new_terms_alert_type.ts | 32 +++-- .../rule_types/new_terms/utils-base-64.ts | 126 ++++++++++++++++++ .../rule_types/new_terms/utils.ts | 69 ++++++++++ 11 files changed, 294 insertions(+), 28 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils-base-64.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts index 15bf73ba150e5..faffffaf50e1f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts @@ -14,7 +14,7 @@ import { LimitedSizeArray, NonEmptyString } from '@kbn/securitysolution-io-ts-ty * New terms rule type currently only supports a single term, but should support more in the future */ export type NewTermsFields = t.TypeOf; -export const NewTermsFields = LimitedSizeArray({ codec: t.string, minSize: 1, maxSize: 1 }); +export const NewTermsFields = LimitedSizeArray({ codec: t.string, minSize: 1, maxSize: 3 }); export type HistoryWindowStart = t.TypeOf; export const HistoryWindowStart = NonEmptyString; diff --git a/x-pack/plugins/security_solution/common/field_maps/field_names.ts b/x-pack/plugins/security_solution/common/field_maps/field_names.ts index 6a7b4efff8a7c..e74c12f187c73 100644 --- a/x-pack/plugins/security_solution/common/field_maps/field_names.ts +++ b/x-pack/plugins/security_solution/common/field_maps/field_names.ts @@ -16,6 +16,8 @@ export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const; export const ALERT_THRESHOLD_RESULT = `${ALERT_NAMESPACE}.threshold_result` as const; export const ALERT_THRESHOLD_RESULT_COUNT = `${ALERT_THRESHOLD_RESULT}.count` as const; export const ALERT_NEW_TERMS = `${ALERT_NAMESPACE}.new_terms` as const; +export const ALERT_NEW_TERMS_FIELDS = + `${ALERT_RULE_NAMESPACE}.parameters.new_terms_fields` as const; export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const; export const ALERT_ORIGINAL_EVENT_ACTION = `${ALERT_ORIGINAL_EVENT}.action` as const; diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx index 5caca1fcd7253..6e2efdafb174c 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx @@ -16,8 +16,13 @@ import { ALERTS_HEADERS_THRESHOLD_TERMS, ALERTS_HEADERS_RULE_DESCRIPTION, ALERTS_HEADERS_NEW_TERMS, + ALERTS_HEADERS_NEW_TERMS_FIELDS, } from '../../../detections/components/alerts_table/translations'; -import { ALERT_NEW_TERMS, ALERT_THRESHOLD_RESULT } from '../../../../common/field_maps/field_names'; +import { + ALERT_NEW_TERMS, + ALERT_NEW_TERMS_FIELDS, + ALERT_THRESHOLD_RESULT, +} from '../../../../common/field_maps/field_names'; import { AGENT_STATUS_FIELD_NAME } from '../../../timelines/components/timeline/body/renderers/constants'; import type { AlertSummaryRow } from './helpers'; import { getEnrichedFieldInfo } from './helpers'; @@ -172,6 +177,10 @@ function getFieldsByRuleType(ruleType?: string): EventSummaryField[] { ]; case 'new_terms': return [ + { + id: ALERT_NEW_TERMS_FIELDS, + label: ALERTS_HEADERS_NEW_TERMS_FIELDS, + }, { id: ALERT_NEW_TERMS, label: ALERTS_HEADERS_NEW_TERMS, diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts index efbe86244ab58..97e642d8fd720 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts @@ -123,6 +123,13 @@ export const ALERTS_HEADERS_NEW_TERMS = i18n.translate( } ); +export const ALERTS_HEADERS_NEW_TERMS_FIELDS = i18n.translate( + 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTermsFields', + { + defaultMessage: 'New Terms fields', + } +); + export const ACTION_INVESTIGATE_IN_TIMELINE = i18n.translate( 'xpack.securitySolution.detectionEngine.alerts.actions.investigateInTimelineTitle', { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx index b23b496eae82e..bfb6750e5ac6b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx @@ -601,11 +601,11 @@ export const schema: FormSchema = { return; } return fieldValidators.maxLengthField({ - length: 1, + length: 3, message: i18n.translate( 'xpack.securitySolution.detectionEngine.validations.stepDefineRule.newTermsFieldsMax', { - defaultMessage: 'Number of fields must be 1.', + defaultMessage: 'Number of fields must not be greater than 3.', } ), })(...args); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/__snapshots__/build_new_terms_aggregation.test.ts.snap b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/__snapshots__/build_new_terms_aggregation.test.ts.snap index 72df8a34cfa18..022f4eba9365c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/__snapshots__/build_new_terms_aggregation.test.ts.snap +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/__snapshots__/build_new_terms_aggregation.test.ts.snap @@ -135,3 +135,37 @@ Object { }, } `; + +exports[`aggregations buildRecentTermsAgg builds a correct composite aggregation with multiple fields 1`] = ` +Object { + "new_terms": Object { + "composite": Object { + "after": undefined, + "size": 10000, + "sources": Array [ + Object { + "host.name": Object { + "terms": Object { + "field": "host.name", + }, + }, + }, + Object { + "host.port": Object { + "terms": Object { + "field": "host.port", + }, + }, + }, + Object { + "host.url": Object { + "terms": Object { + "field": "host.url", + }, + }, + }, + ], + }, + }, +} +`; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.test.ts index 9b853a730ba4c..ec81c06b92837 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.test.ts @@ -16,7 +16,7 @@ describe('aggregations', () => { describe('buildRecentTermsAgg', () => { test('builds a correct composite agg without `after`', () => { const aggregation = buildRecentTermsAgg({ - field: 'host.name', + fields: ['host.name'], after: undefined, }); @@ -25,12 +25,21 @@ describe('aggregations', () => { test('builds a correct composite aggregation with `after`', () => { const aggregation = buildRecentTermsAgg({ - field: 'host.name', + fields: ['host.name'], after: { 'host.name': 'myHost' }, }); expect(aggregation).toMatchSnapshot(); }); + + test('builds a correct composite aggregation with multiple fields', () => { + const aggregation = buildRecentTermsAgg({ + fields: ['host.name', 'host.port', 'host.url'], + after: undefined, + }); + + expect(aggregation).toMatchSnapshot(); + }); }); describe('buildNewTermsAggregation', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts index 41f2f7c6dc0ab..e9bf89554941f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts @@ -31,24 +31,24 @@ const PAGE_SIZE = 10000; * without regard to whether or not they're actually new. */ export const buildRecentTermsAgg = ({ - field, + fields, after, }: { - field: string; + fields: string[]; after: Record | undefined; }) => { + const sources = fields.map((field) => ({ + [field]: { + terms: { + field, + }, + }, + })); + return { new_terms: { composite: { - sources: [ - { - [field]: { - terms: { - field, - }, - }, - }, - ], + sources, size: PAGE_SIZE, after, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 630553bc4d78c..f49c95ef12377 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -28,7 +28,14 @@ import { } from './build_new_terms_aggregation'; import type { SignalSource } from '../../signals/types'; import { validateIndexPatterns } from '../utils'; -import { parseDateString, validateHistoryWindowStart } from './utils'; +import { + parseDateString, + validateHistoryWindowStart, + retrieveValuesFromBuckets, + getRuntimeMappings as getRuntimeMappingsForNewTerms, + getAggregationField, + decodeMatchedBucketKey, +} from './utils-base-64'; import { addToSearchAfterReturn, createSearchAfterReturnType, @@ -152,7 +159,7 @@ export const createNewTermsAlertType = ( // ones are new. const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ aggregations: buildRecentTermsAgg({ - field: params.newTermsFields[0], + fields: params.newTermsFields, after: afterKey, }), searchAfterSortIds: undefined, @@ -185,10 +192,7 @@ export const createNewTermsAlertType = ( break; } const bucketsForField = searchResultWithAggs.aggregations.new_terms.buckets; - const includeValues = bucketsForField - .map((bucket) => Object.values(bucket.key)[0]) - .filter((value): value is string | number => value != null); - + const includeValues = retrieveValuesFromBuckets(params.newTermsFields, bucketsForField); // PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window. // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the // response correspond to each new term. @@ -200,10 +204,13 @@ export const createNewTermsAlertType = ( aggregations: buildNewTermsAgg({ newValueWindowStart: tuple.from, timestampField: aggregatableTimestampField, - field: params.newTermsFields[0], + field: getAggregationField(params.newTermsFields), include: includeValues, }), - runtimeMappings, + runtimeMappings: { + ...runtimeMappings, + ...getRuntimeMappingsForNewTerms(params.newTermsFields), + }, searchAfterSortIds: undefined, index: inputIndex, // For Phase 2, we expand the time range to aggregate over the history window @@ -243,10 +250,13 @@ export const createNewTermsAlertType = ( } = await singleSearchAfter({ aggregations: buildDocFetchAgg({ timestampField: aggregatableTimestampField, - field: params.newTermsFields[0], + field: getAggregationField(params.newTermsFields), include: actualNewTerms, }), - runtimeMappings, + runtimeMappings: { + ...runtimeMappings, + ...getRuntimeMappingsForNewTerms(params.newTermsFields), + }, searchAfterSortIds: undefined, index: inputIndex, // For phase 3, we go back to aggregating only over the rule interval - excluding the history window @@ -273,7 +283,7 @@ export const createNewTermsAlertType = ( newTerms: Array; }> = docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => ({ event: bucket.docs.hits.hits[0], - newTerms: [bucket.key], + newTerms: decodeMatchedBucketKey(params.newTermsFields, bucket.key), })); const wrappedAlerts = wrapNewTermsAlerts({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils-base-64.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils-base-64.ts new file mode 100644 index 0000000000000..316848f7d84ff --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils-base-64.ts @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import dateMath from '@elastic/datemath'; +import moment from 'moment'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +const AGGR_FIELD = 'new_terms_values'; +const DELIMITER = '_'; + +export const parseDateString = ({ + date, + forceNow, + name, +}: { + date: string; + forceNow: Date; + name?: string; +}): moment.Moment => { + const parsedDate = dateMath.parse(date, { + forceNow, + }); + if (parsedDate == null || !parsedDate.isValid()) { + throw Error(`Failed to parse '${name ?? 'date string'}'`); + } + return parsedDate; +}; + +export const validateHistoryWindowStart = ({ + historyWindowStart, + from, +}: { + historyWindowStart: string; + from: string; +}) => { + const forceNow = moment().toDate(); + const parsedHistoryWindowStart = parseDateString({ + date: historyWindowStart, + forceNow, + name: 'historyWindowStart', + }); + const parsedFrom = parseDateString({ date: from, forceNow, name: 'from' }); + if (parsedHistoryWindowStart.isSameOrAfter(parsedFrom)) { + throw Error( + `History window size is smaller than rule interval + additional lookback, 'historyWindowStart' must be earlier than 'from'` + ); + } +}; + +export const retrieveValuesFromBuckets = ( + newTermsFields: string[], + buckets: Array<{ key: Record }> +): Array => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length === 1) { + return buckets + .map((bucket) => Object.values(bucket.key)[0]) + .filter((value): value is string | number => value != null); + } + + return buckets.map((bucket) => + Object.values(bucket.key) + .filter((value): value is string | number => value != null) + .map((value) => + Buffer.from(typeof value !== 'string' ? value.toString() : value).toString('base64') + ) + .join(DELIMITER) + ); +}; + +export const getRuntimeMappings = ( + newTermsFields: string[] +): undefined | { [AGGR_FIELD]: estypes.MappingRuntimeField } => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length === 1) { + return undefined; + } + + const fields = newTermsFields.map((field) => `'${field}'`).join(', '); + return { + [AGGR_FIELD]: { + type: 'keyword', + script: ` + String[] fields = new String[] {${fields}}; + String acc = doc[fields[0]].value.encodeBase64(); + + for (int i = 1; i < fields.length; i++) { + acc = acc + '${DELIMITER}' + doc[fields[i]].value.encodeBase64(); + } + + String[] arr = new String[1]; + arr[0] = acc; + + emit(arr[0]) + `, + }, + }; +}; + +export const getAggregationField = (newTermsFields: string[]): string => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length === 1) { + return newTermsFields[0]; + } + + return AGGR_FIELD; +}; + +export const decodeMatchedBucketKey = ( + newTermsFields: string[], + bucketKey: string | number +): Array => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length === 1) { + return [bucketKey]; + } + + // if newTermsFields has length greater than 1, bucketKey can't be umber, so casting is safe here + return (bucketKey as string) + .split(DELIMITER) + .map((encodedValue) => Buffer.from(encodedValue, 'base64').toString()); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 4a87ec9edbbda..d56ae93d957aa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -7,6 +7,10 @@ import dateMath from '@elastic/datemath'; import moment from 'moment'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +const AGGR_FIELD = 'new_terms_values'; +const DELIMITER = '_______'; export const parseDateString = ({ date, @@ -46,3 +50,68 @@ export const validateHistoryWindowStart = ({ ); } }; + +export const retrieveValuesFromBuckets = ( + newTermsFields: string[], + buckets: Array<{ key: Record }> +): Array => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length === 1) { + return buckets + .map((bucket) => Object.values(bucket.key)[0]) + .filter((value): value is string | number => value != null); + } + + return buckets.map((bucket) => Object.values(bucket.key).join(DELIMITER)); +}; + +export const getRuntimeMappings = ( + newTermsFields: string[] +): undefined | { [AGGR_FIELD]: estypes.MappingRuntimeField } => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length === 1) { + return undefined; + } + + const fields = newTermsFields.map((field) => `'${field}'`).join(', '); + return { + [AGGR_FIELD]: { + type: 'keyword', + script: ` + String[] fields = new String[] {${fields}}; + String acc = doc[fields[0]].value; + + for (int i = 1; i < fields.length; i++) { + acc = acc + '${DELIMITER}' + doc[fields[i]].value + } + + String[] arr = new String[1]; + arr[0] = acc; + + emit(arr[0]) + `, + }, + }; +}; + +export const getAggregationField = (newTermsFields: string[]): string => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length === 1) { + return newTermsFields[0]; + } + + return AGGR_FIELD; +}; + +export const decodeMatchedBucketKey = ( + newTermsFields: string[], + bucketKey: string | number +): Array => { + // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + if (newTermsFields.length === 1) { + return [bucketKey]; + } + + // if newTermsFields has length greater than 1, bucketKey can't be umber, so casting is safe here + return (bucketKey as string).split(DELIMITER); +}; From 948ad8bc9eadc50f9a8ff75166dd6263ab21128e Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 25 Oct 2022 13:51:49 +0100 Subject: [PATCH 02/36] fix buiold checks --- .../rule_types/new_terms/create_new_terms_alert_type.ts | 3 ++- .../new_terms/{utils-base-64.ts => utils_base_64.ts} | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/{utils-base-64.ts => utils_base_64.ts} (100%) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index f49c95ef12377..37c8e2934ecd0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -35,7 +35,8 @@ import { getRuntimeMappings as getRuntimeMappingsForNewTerms, getAggregationField, decodeMatchedBucketKey, -} from './utils-base-64'; +} from './utils'; +// } from './utils_base_64'; import { addToSearchAfterReturn, createSearchAfterReturnType, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils-base-64.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils-base-64.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts From 7655cf11f97171b84e1d7975a446c7d6251874c4 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 26 Oct 2022 17:24:05 +0100 Subject: [PATCH 03/36] handle arrays and empty fields --- .../rule_types/new_terms/utils_base_64.ts | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts index 316848f7d84ff..a213236714492 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts @@ -85,17 +85,23 @@ export const getRuntimeMappings = ( [AGGR_FIELD]: { type: 'keyword', script: ` - String[] fields = new String[] {${fields}}; - String acc = doc[fields[0]].value.encodeBase64(); - for (int i = 1; i < fields.length; i++) { - acc = acc + '${DELIMITER}' + doc[fields[i]].value.encodeBase64(); + void traverseDocFields(def doc, def fields, def index, def line) { + if (index === fields.length) { + emit(line); + } else { + for (field in doc[fields[index]]) { + def delimiter = index === 0 ? '' : '${DELIMITER}'; + def nextLine = line + delimiter + String.valueOf(field).encodeBase64(); + + traverseDocFields(doc, fields, index + 1, nextLine); + } + } } - - String[] arr = new String[1]; - arr[0] = acc; - - emit(arr[0]) + + String[] fields = new String[] {${fields}}; + + traverseDocFields(doc, fields, 0, ''); `, }, }; From 1a4cbb42c106d506e54551f3a69784bd0dc48890 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 26 Oct 2022 19:51:52 +0100 Subject: [PATCH 04/36] UX changes --- .../common/field_maps/field_names.ts | 2 -- .../event_details/get_alert_summary_rows.tsx | 11 +---------- .../components/alerts_table/translations.ts | 7 ------- .../rule_types/new_terms/utils_base_64.ts | 4 +++- 4 files changed, 4 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/security_solution/common/field_maps/field_names.ts b/x-pack/plugins/security_solution/common/field_maps/field_names.ts index e74c12f187c73..6a7b4efff8a7c 100644 --- a/x-pack/plugins/security_solution/common/field_maps/field_names.ts +++ b/x-pack/plugins/security_solution/common/field_maps/field_names.ts @@ -16,8 +16,6 @@ export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const; export const ALERT_THRESHOLD_RESULT = `${ALERT_NAMESPACE}.threshold_result` as const; export const ALERT_THRESHOLD_RESULT_COUNT = `${ALERT_THRESHOLD_RESULT}.count` as const; export const ALERT_NEW_TERMS = `${ALERT_NAMESPACE}.new_terms` as const; -export const ALERT_NEW_TERMS_FIELDS = - `${ALERT_RULE_NAMESPACE}.parameters.new_terms_fields` as const; export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const; export const ALERT_ORIGINAL_EVENT_ACTION = `${ALERT_ORIGINAL_EVENT}.action` as const; diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx index 6e2efdafb174c..5caca1fcd7253 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx @@ -16,13 +16,8 @@ import { ALERTS_HEADERS_THRESHOLD_TERMS, ALERTS_HEADERS_RULE_DESCRIPTION, ALERTS_HEADERS_NEW_TERMS, - ALERTS_HEADERS_NEW_TERMS_FIELDS, } from '../../../detections/components/alerts_table/translations'; -import { - ALERT_NEW_TERMS, - ALERT_NEW_TERMS_FIELDS, - ALERT_THRESHOLD_RESULT, -} from '../../../../common/field_maps/field_names'; +import { ALERT_NEW_TERMS, ALERT_THRESHOLD_RESULT } from '../../../../common/field_maps/field_names'; import { AGENT_STATUS_FIELD_NAME } from '../../../timelines/components/timeline/body/renderers/constants'; import type { AlertSummaryRow } from './helpers'; import { getEnrichedFieldInfo } from './helpers'; @@ -177,10 +172,6 @@ function getFieldsByRuleType(ruleType?: string): EventSummaryField[] { ]; case 'new_terms': return [ - { - id: ALERT_NEW_TERMS_FIELDS, - label: ALERTS_HEADERS_NEW_TERMS_FIELDS, - }, { id: ALERT_NEW_TERMS, label: ALERTS_HEADERS_NEW_TERMS, diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts index 97e642d8fd720..efbe86244ab58 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts @@ -123,13 +123,6 @@ export const ALERTS_HEADERS_NEW_TERMS = i18n.translate( } ); -export const ALERTS_HEADERS_NEW_TERMS_FIELDS = i18n.translate( - 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTermsFields', - { - defaultMessage: 'New Terms fields', - } -); - export const ACTION_INVESTIGATE_IN_TIMELINE = i18n.translate( 'xpack.securitySolution.detectionEngine.alerts.actions.investigateInTimelineTitle', { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts index a213236714492..10d426d570a9b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts @@ -128,5 +128,7 @@ export const decodeMatchedBucketKey = ( // if newTermsFields has length greater than 1, bucketKey can't be umber, so casting is safe here return (bucketKey as string) .split(DELIMITER) - .map((encodedValue) => Buffer.from(encodedValue, 'base64').toString()); + .map((encodedValue, i) => + [newTermsFields[i], Buffer.from(encodedValue, 'base64').toString()].join(': ') + ); }; From 9bb3f44dbeba390e1ed7a5f38fc8dd53807794ee Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 27 Oct 2022 10:33:28 +0100 Subject: [PATCH 05/36] unify codebasese --- .../new_terms/create_new_terms_alert_type.ts | 7 +- .../rule_types/new_terms/utils.ts | 58 +++++--- .../rule_types/new_terms/utils_base_64.ts | 134 ------------------ 3 files changed, 43 insertions(+), 156 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 37c8e2934ecd0..95c78d0457714 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -32,11 +32,10 @@ import { parseDateString, validateHistoryWindowStart, retrieveValuesFromBuckets, - getRuntimeMappings as getRuntimeMappingsForNewTerms, + getNewTermsRuntimeMappings, getAggregationField, decodeMatchedBucketKey, } from './utils'; -// } from './utils_base_64'; import { addToSearchAfterReturn, createSearchAfterReturnType, @@ -210,7 +209,7 @@ export const createNewTermsAlertType = ( }), runtimeMappings: { ...runtimeMappings, - ...getRuntimeMappingsForNewTerms(params.newTermsFields), + ...getNewTermsRuntimeMappings(params.newTermsFields), }, searchAfterSortIds: undefined, index: inputIndex, @@ -256,7 +255,7 @@ export const createNewTermsAlertType = ( }), runtimeMappings: { ...runtimeMappings, - ...getRuntimeMappingsForNewTerms(params.newTermsFields), + ...getNewTermsRuntimeMappings(params.newTermsFields), }, searchAfterSortIds: undefined, index: inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index d56ae93d957aa..3ac63df3e45d4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -9,8 +9,8 @@ import dateMath from '@elastic/datemath'; import moment from 'moment'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -const AGGR_FIELD = 'new_terms_values'; -const DELIMITER = '_______'; +const AGG_FIELD_NAME = 'new_terms_values'; +const DELIMITER = '_'; export const parseDateString = ({ date, @@ -62,12 +62,24 @@ export const retrieveValuesFromBuckets = ( .filter((value): value is string | number => value != null); } - return buckets.map((bucket) => Object.values(bucket.key).join(DELIMITER)); + return buckets.map((bucket) => + Object.values(bucket.key) + .filter((value): value is string | number => value != null) + .map((value) => + Buffer.from(typeof value !== 'string' ? value.toString() : value).toString('base64') + ) + .join(DELIMITER) + ); }; -export const getRuntimeMappings = ( +/** + * creates runtime field + * @param newTermsFields + * @returns + */ +export const getNewTermsRuntimeMappings = ( newTermsFields: string[] -): undefined | { [AGGR_FIELD]: estypes.MappingRuntimeField } => { +): undefined | { [AGG_FIELD_NAME]: estypes.MappingRuntimeField } => { // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together if (newTermsFields.length === 1) { return undefined; @@ -75,20 +87,26 @@ export const getRuntimeMappings = ( const fields = newTermsFields.map((field) => `'${field}'`).join(', '); return { - [AGGR_FIELD]: { + [AGG_FIELD_NAME]: { type: 'keyword', script: ` - String[] fields = new String[] {${fields}}; - String acc = doc[fields[0]].value; - for (int i = 1; i < fields.length; i++) { - acc = acc + '${DELIMITER}' + doc[fields[i]].value + void traverseDocFields(def doc, def fields, def index, def line) { + if (index === fields.length) { + emit(line); + } else { + for (field in doc[fields[index]]) { + def delimiter = index === 0 ? '' : '${DELIMITER}'; + def nextLine = line + delimiter + String.valueOf(field).encodeBase64(); + + traverseDocFields(doc, fields, index + 1, nextLine); + } + } } - - String[] arr = new String[1]; - arr[0] = acc; - - emit(arr[0]) + + String[] fields = new String[] {${fields}}; + + traverseDocFields(doc, fields, 0, ''); `, }, }; @@ -100,7 +118,7 @@ export const getAggregationField = (newTermsFields: string[]): string => { return newTermsFields[0]; } - return AGGR_FIELD; + return AGG_FIELD_NAME; }; export const decodeMatchedBucketKey = ( @@ -112,6 +130,10 @@ export const decodeMatchedBucketKey = ( return [bucketKey]; } - // if newTermsFields has length greater than 1, bucketKey can't be umber, so casting is safe here - return (bucketKey as string).split(DELIMITER); + // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here + return (bucketKey as string) + .split(DELIMITER) + .map((encodedValue, i) => + [newTermsFields[i], Buffer.from(encodedValue, 'base64').toString()].join(': ') + ); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts deleted file mode 100644 index 10d426d570a9b..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils_base_64.ts +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import dateMath from '@elastic/datemath'; -import moment from 'moment'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - -const AGGR_FIELD = 'new_terms_values'; -const DELIMITER = '_'; - -export const parseDateString = ({ - date, - forceNow, - name, -}: { - date: string; - forceNow: Date; - name?: string; -}): moment.Moment => { - const parsedDate = dateMath.parse(date, { - forceNow, - }); - if (parsedDate == null || !parsedDate.isValid()) { - throw Error(`Failed to parse '${name ?? 'date string'}'`); - } - return parsedDate; -}; - -export const validateHistoryWindowStart = ({ - historyWindowStart, - from, -}: { - historyWindowStart: string; - from: string; -}) => { - const forceNow = moment().toDate(); - const parsedHistoryWindowStart = parseDateString({ - date: historyWindowStart, - forceNow, - name: 'historyWindowStart', - }); - const parsedFrom = parseDateString({ date: from, forceNow, name: 'from' }); - if (parsedHistoryWindowStart.isSameOrAfter(parsedFrom)) { - throw Error( - `History window size is smaller than rule interval + additional lookback, 'historyWindowStart' must be earlier than 'from'` - ); - } -}; - -export const retrieveValuesFromBuckets = ( - newTermsFields: string[], - buckets: Array<{ key: Record }> -): Array => { - // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together - if (newTermsFields.length === 1) { - return buckets - .map((bucket) => Object.values(bucket.key)[0]) - .filter((value): value is string | number => value != null); - } - - return buckets.map((bucket) => - Object.values(bucket.key) - .filter((value): value is string | number => value != null) - .map((value) => - Buffer.from(typeof value !== 'string' ? value.toString() : value).toString('base64') - ) - .join(DELIMITER) - ); -}; - -export const getRuntimeMappings = ( - newTermsFields: string[] -): undefined | { [AGGR_FIELD]: estypes.MappingRuntimeField } => { - // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together - if (newTermsFields.length === 1) { - return undefined; - } - - const fields = newTermsFields.map((field) => `'${field}'`).join(', '); - return { - [AGGR_FIELD]: { - type: 'keyword', - script: ` - - void traverseDocFields(def doc, def fields, def index, def line) { - if (index === fields.length) { - emit(line); - } else { - for (field in doc[fields[index]]) { - def delimiter = index === 0 ? '' : '${DELIMITER}'; - def nextLine = line + delimiter + String.valueOf(field).encodeBase64(); - - traverseDocFields(doc, fields, index + 1, nextLine); - } - } - } - - String[] fields = new String[] {${fields}}; - - traverseDocFields(doc, fields, 0, ''); - `, - }, - }; -}; - -export const getAggregationField = (newTermsFields: string[]): string => { - // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together - if (newTermsFields.length === 1) { - return newTermsFields[0]; - } - - return AGGR_FIELD; -}; - -export const decodeMatchedBucketKey = ( - newTermsFields: string[], - bucketKey: string | number -): Array => { - // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together - if (newTermsFields.length === 1) { - return [bucketKey]; - } - - // if newTermsFields has length greater than 1, bucketKey can't be umber, so casting is safe here - return (bucketKey as string) - .split(DELIMITER) - .map((encodedValue, i) => - [newTermsFields[i], Buffer.from(encodedValue, 'base64').toString()].join(': ') - ); -}; From d7b4a65a2eeededca2d3aa313ec13bf45cb44d40 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 27 Oct 2022 13:01:06 +0100 Subject: [PATCH 06/36] use params in ES query --- .../rule_types/new_terms/utils.ts | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 3ac63df3e45d4..8e648f4f30ff8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -85,29 +85,28 @@ export const getNewTermsRuntimeMappings = ( return undefined; } - const fields = newTermsFields.map((field) => `'${field}'`).join(', '); return { [AGG_FIELD_NAME]: { type: 'keyword', - script: ` - - void traverseDocFields(def doc, def fields, def index, def line) { - if (index === fields.length) { - emit(line); - } else { - for (field in doc[fields[index]]) { - def delimiter = index === 0 ? '' : '${DELIMITER}'; - def nextLine = line + delimiter + String.valueOf(field).encodeBase64(); - - traverseDocFields(doc, fields, index + 1, nextLine); + script: { + params: { fields: newTermsFields }, + source: ` + void traverseDocFields(def doc, def fields, def index, def line) { + if (index === fields.length) { + emit(line); + } else { + for (field in doc[fields[index]]) { + def delimiter = index === 0 ? '' : '${DELIMITER}'; + def nextLine = line + delimiter + String.valueOf(field).encodeBase64(); + + traverseDocFields(doc, fields, index + 1, nextLine); + } + } } - } - } - - String[] fields = new String[] {${fields}}; - traverseDocFields(doc, fields, 0, ''); - `, + traverseDocFields(doc, params['fields'], 0, ''); + `, + }, }, }; }; @@ -127,7 +126,7 @@ export const decodeMatchedBucketKey = ( ): Array => { // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together if (newTermsFields.length === 1) { - return [bucketKey]; + return newTermsFields.map((field) => [field, bucketKey].join(': ')); } // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here From 1cc69e65514b959aeaa68bfebf455f89db1c3517 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 27 Oct 2022 13:12:04 +0100 Subject: [PATCH 07/36] code improvenemts --- .../new_terms/create_new_terms_alert_type.ts | 4 +-- .../rule_types/new_terms/utils.ts | 28 ++++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 95c78d0457714..c6fefa9491607 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -34,7 +34,7 @@ import { retrieveValuesFromBuckets, getNewTermsRuntimeMappings, getAggregationField, - decodeMatchedBucketKey, + prepareNewTerms, } from './utils'; import { addToSearchAfterReturn, @@ -283,7 +283,7 @@ export const createNewTermsAlertType = ( newTerms: Array; }> = docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => ({ event: bucket.docs.hits.hits[0], - newTerms: decodeMatchedBucketKey(params.newTermsFields, bucket.key), + newTerms: prepareNewTerms(params.newTermsFields, bucket.key), })); const wrappedAlerts = wrapNewTermsAlerts({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 8e648f4f30ff8..1821478e17468 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -120,19 +120,21 @@ export const getAggregationField = (newTermsFields: string[]): string => { return AGG_FIELD_NAME; }; -export const decodeMatchedBucketKey = ( - newTermsFields: string[], - bucketKey: string | number -): Array => { - // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together - if (newTermsFields.length === 1) { - return newTermsFields.map((field) => [field, bucketKey].join(': ')); - } - +const decodeBucketKey = (bucketKey: string): string[] => { // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here - return (bucketKey as string) + return bucketKey .split(DELIMITER) - .map((encodedValue, i) => - [newTermsFields[i], Buffer.from(encodedValue, 'base64').toString()].join(': ') - ); + .map((encodedValue) => Buffer.from(encodedValue, 'base64').toString()); +}; + +/** + * returns new term fields and values in following format + * @example + * [ 'field1: new_value1', 'field2: new_value2'] + */ +export const prepareNewTerms = (newTermsFields: string[], bucketKey: string | number) => { + // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here + const values = newTermsFields.length === 1 ? [bucketKey] : decodeBucketKey(bucketKey as string); + + return newTermsFields.map((field, i) => [field, values[i]].join(': ')); }; From 517898fbf66c0d7c086a592f319aebdd4a01538d Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Oct 2022 11:16:20 +0100 Subject: [PATCH 08/36] filter out types for terms aggregation --- .../rules/step_define_rule/index.tsx | 8 +++++++- .../rules/step_define_rule/utils.ts | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index dc926fd4f74e2..9cbe2362acb6d 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -59,6 +59,7 @@ import { useFormData, } from '../../../../shared_imports'; import { schema } from './schema'; +import { getTermsAggregationFields } from './utils'; import * as i18n from './translations'; import { isEqlRule, @@ -297,6 +298,11 @@ const StepDefineRuleComponent: FC = ({ setAggregatableFields(aggregatableFields(fields as BrowserField[])); }, [indexPattern]); + const termsAggregationFields: BrowserField[] = useMemo( + () => getTermsAggregationFields(aggFields), + [aggFields] + ); + const [ threatIndexPatternsLoading, { browserFields: threatBrowserFields, indexPatterns: threatIndexPatterns }, @@ -836,7 +842,7 @@ const StepDefineRuleComponent: FC = ({ path="newTermsFields" component={NewTermsFields} componentProps={{ - browserFields: aggFields, + browserFields: termsAggregationFields, }} /> { + const allowedTypesSet = new Set(['string', 'number', 'ip', 'boolean', 'binary']); + + return fields.filter((field) => field.aggregatable === true && allowedTypesSet.has(field.type)); +}; From 2006059e2f7d0b3e7aaef0fba1d193db7d491176 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Oct 2022 11:21:06 +0100 Subject: [PATCH 09/36] rewrite tests assertions --- .../rule_execution_logic/new_terms.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 4bfbe92118599..b6e7a0873031d 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -76,7 +76,7 @@ export default ({ getService }: FtrProviderContext) => { expect(alerts.hits.hits.length).eql(1); expect(removeRandomValuedProperties(alerts.hits.hits[0]._source)).eql({ - 'kibana.alert.new_terms': ['zeek-newyork-sha-aa8df15'], + 'kibana.alert.new_terms': ['host.name: zeek-newyork-sha-aa8df15'], 'kibana.alert.rule.category': 'New Terms Rule', 'kibana.alert.rule.consumer': 'siem', 'kibana.alert.rule.name': 'Query with a rule id', @@ -218,13 +218,13 @@ export default ({ getService }: FtrProviderContext) => { 'asc' ); expect(previewAlertsOrderedByHostIp[0]._source?.['kibana.alert.new_terms']).eql([ - '10.10.0.6', + 'host.ip: 10.10.0.6', ]); expect(previewAlertsOrderedByHostIp[1]._source?.['kibana.alert.new_terms']).eql([ - '157.230.208.30', + 'host.ip: 157.230.208.30', ]); expect(previewAlertsOrderedByHostIp[2]._source?.['kibana.alert.new_terms']).eql([ - 'fe80::24ce:f7ff:fede:a571', + 'host.ip: fe80::24ce:f7ff:fede:a571', ]); }); @@ -244,11 +244,11 @@ export default ({ getService }: FtrProviderContext) => { const hostNames = previewAlerts .map((signal) => signal._source?.['kibana.alert.new_terms']) .sort(); - expect(hostNames[0]).eql(['suricata-sensor-amsterdam']); - expect(hostNames[1]).eql(['suricata-sensor-san-francisco']); - expect(hostNames[2]).eql(['zeek-newyork-sha-aa8df15']); - expect(hostNames[3]).eql(['zeek-sensor-amsterdam']); - expect(hostNames[4]).eql(['zeek-sensor-san-francisco']); + expect(hostNames[0]).eql(['host.name: suricata-sensor-amsterdam']); + expect(hostNames[1]).eql(['host.name: suricata-sensor-san-francisco']); + expect(hostNames[2]).eql(['host.name: zeek-newyork-sha-aa8df15']); + expect(hostNames[3]).eql(['host.name: zeek-sensor-amsterdam']); + expect(hostNames[4]).eql(['host.name: zeek-sensor-san-francisco']); }); describe('timestamp override and fallback', () => { @@ -289,8 +289,8 @@ export default ({ getService }: FtrProviderContext) => { const hostNames = previewAlerts .map((signal) => signal._source?.['kibana.alert.new_terms']) .sort(); - expect(hostNames[0]).eql(['host-3']); - expect(hostNames[1]).eql(['host-4']); + expect(hostNames[0]).eql(['host.name: host-3']); + expect(hostNames[1]).eql(['host.name: host-4']); }); }); @@ -329,10 +329,10 @@ export default ({ getService }: FtrProviderContext) => { const hostNames = previewAlerts .map((signal) => signal._source?.['kibana.alert.new_terms']) .sort(); - expect(hostNames[0]).eql(['suricata-sensor-amsterdam']); - expect(hostNames[1]).eql(['suricata-sensor-san-francisco']); - expect(hostNames[2]).eql(['zeek-newyork-sha-aa8df15']); - expect(hostNames[3]).eql(['zeek-sensor-amsterdam']); + expect(hostNames[0]).eql(['host.name: suricata-sensor-amsterdam']); + expect(hostNames[1]).eql(['host.name: suricata-sensor-san-francisco']); + expect(hostNames[2]).eql(['host.name: zeek-newyork-sha-aa8df15']); + expect(hostNames[3]).eql(['host.name: zeek-sensor-amsterdam']); }); }); @@ -354,7 +354,7 @@ export default ({ getService }: FtrProviderContext) => { const processPids = previewAlerts .map((signal) => signal._source?.['kibana.alert.new_terms']) .sort(); - expect(processPids[0]).eql([1]); + expect(processPids[0]).eql(['process.pid: 1']); }); describe('alerts should be be enriched', () => { From 910c207c2e1564094f9eb8847addea2ddb6f3e85 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Oct 2022 16:24:19 +0100 Subject: [PATCH 10/36] add new tests --- .../rule_execution_logic/new_terms.ts | 92 +++++++++++++++ .../security_solution/new_terms/data.json | 108 ++++++++++++++++++ .../security_solution/new_terms/mappings.json | 46 ++++++++ 3 files changed, 246 insertions(+) create mode 100644 x-pack/test/functional/es_archives/security_solution/new_terms/data.json create mode 100644 x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index b6e7a0873031d..056ada779e3b8 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -50,10 +50,12 @@ export default ({ getService }: FtrProviderContext) => { describe('New terms type rules', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/new_terms'); }); after(async () => { await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/new_terms'); await deleteSignalsIndex(supertest, log); await deleteAllAlerts(supertest, log); }); @@ -228,6 +230,96 @@ export default ({ getService }: FtrProviderContext) => { ]); }); + it('should generate 3 alerts when 1 document has 3 new values for multiple fields', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + new_terms_fields: ['host.name', 'host.ip'], + from: '2019-02-19T20:42:00.000Z', + history_window_start: '2019-01-19T20:42:00.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(3); + + expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql([ + 'host.name: zeek-newyork-sha-aa8df15', + 'host.ip: 10.10.0.6', + ]); + expect(previewAlerts[1]._source?.['kibana.alert.new_terms']).eql([ + 'host.name: zeek-newyork-sha-aa8df15', + 'host.ip: 157.230.208.30', + ]); + expect(previewAlerts[2]._source?.['kibana.alert.new_terms']).eql([ + 'host.name: zeek-newyork-sha-aa8df15', + 'host.ip: fe80::24ce:f7ff:fede:a571', + ]); + }); + + it.only('should generate 1 alert for unique combination of existing terms', async () => { + // ensure there are no alerts for single new terms fields, it means values are not new + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['host.name', 'host.ip'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + // shouldn't be terms for 'host.ip' + const hostIpPreview = await previewRule({ + supertest, + rule: { ...rule, new_terms_fields: ['host.ip'] }, + }); + const hostIpPreviewAlerts = await getPreviewAlerts({ + es, + previewId: hostIpPreview.previewId, + }); + expect(hostIpPreviewAlerts.length).eql(0); + + // shouldn't be terms for 'host.name' + const hostNamePreview = await previewRule({ + supertest, + rule: { ...rule, new_terms_fields: ['host.name'] }, + }); + const hostNamePreviewAlerts = await getPreviewAlerts({ + es, + previewId: hostNamePreview.previewId, + }); + expect(hostNamePreviewAlerts.length).eql(0); + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(1); + + expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql([ + 'host.name: host-0', + 'host.ip: 127.0.0.2', + ]); + }); + + it('should generate alert for each new unique combination', async () => { + // ensure there are no alerts for single new terms fields, it means values are not new + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['source.ip', 'tags'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(4); + + expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql([ + 'host.name: host-0', + 'host.ip: 127.0.0.2', + ]); + }); + it('should generate alerts for every term when history window is small', async () => { const rule: NewTermsRuleCreateProps = { ...getCreateNewTermsRulesSchemaMock('rule-1', true), diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json new file mode 100644 index 0000000000000..b20058031c617 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json @@ -0,0 +1,108 @@ +{ + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:01.000Z", + "host": { + "name": "host-0", + "ip": "127.0.0.1" + }, + "user.name": "user-0", + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:02.000Z", + "host": { + "name": "host-1", + "ip": "127.0.0.1" + }, + "user.name": "user-0", + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:03.000Z", + "host": { + "name": "host-1", + "ip": "127.0.0.2" + }, + "user.name": "user-0", + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:04.000Z", + "host": { + "name": "host-0", + "ip": "127.0.0.1" + }, + "user.name": "user-0", + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-20T05:00:04.000Z", + "host": { + "name": "host-0", + "ip": "127.0.0.2" + }, + "user.name": "user-0", + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-20T05:00:04.000Z", + "host": { + "name": "host-0", + "ip": "127.0.0.2" + }, + "user.name": "user-0", + "source.ip": ["192.168.1.1", "192.168.1.2"], + "tags": ["tag-new-1", "tag-2", "tag-new-3"] + }, + "type": "_doc" + } + } \ No newline at end of file diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json new file mode 100644 index 0000000000000..664f3a9866aa7 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json @@ -0,0 +1,46 @@ +{ + "type": "index", + "value": { + "index": "new_terms", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "properties": { + "name": { + "type": "keyword" + }, + "ip": { + "type": "ip" + } + } + }, + "user": { + "properties": { + "name": { + "type": "keyword" + } + } + }, + "source": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "tags": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } + } \ No newline at end of file From c973bed3c4ff39eae38862e770e9ca9e634749ea Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Oct 2022 18:05:07 +0100 Subject: [PATCH 11/36] more tests --- .../rule_types/new_terms/utils.ts | 3 +- .../group1/create_new_terms.ts | 17 +++ .../rule_execution_logic/new_terms.ts | 126 +++++++++++++----- .../security_solution/new_terms/data.json | 79 +++++++++++ .../security_solution/new_terms/mappings.json | 11 +- 5 files changed, 198 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 1821478e17468..0822e010e2fc8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -136,5 +136,6 @@ export const prepareNewTerms = (newTermsFields: string[], bucketKey: string | nu // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here const values = newTermsFields.length === 1 ? [bucketKey] : decodeBucketKey(bucketKey as string); - return newTermsFields.map((field, i) => [field, values[i]].join(': ')); + return values; + // return newTermsFields.map((field, i) => [field, values[i]].join(': ')); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts index 095ce3766918d..7f85acb3fa42d 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts @@ -40,5 +40,22 @@ export default ({ getService }: FtrProviderContext) => { "params invalid: History window size is smaller than rule interval + additional lookback, 'historyWindowStart' must be earlier than 'from'" ); }); + + it('should not be able to create a new terms rule new terms number greater than 3', async () => { + const rule = { + ...getCreateNewTermsRulesSchemaMock('rule-1'), + history_window_start: 'now-5m', + new_terms_fields: ['field1', 'field2', 'field3', 'field4'], + }; + const response = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(rule); + + expect(response.status).to.equal(400); + expect(response.body.message).to.be( + '[request body]: Array size (4) is out of bounds: min: 1, max: 3' + ); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 056ada779e3b8..8a12efd39a297 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -78,7 +78,7 @@ export default ({ getService }: FtrProviderContext) => { expect(alerts.hits.hits.length).eql(1); expect(removeRandomValuedProperties(alerts.hits.hits[0]._source)).eql({ - 'kibana.alert.new_terms': ['host.name: zeek-newyork-sha-aa8df15'], + 'kibana.alert.new_terms': ['zeek-newyork-sha-aa8df15'], 'kibana.alert.rule.category': 'New Terms Rule', 'kibana.alert.rule.consumer': 'siem', 'kibana.alert.rule.name': 'Query with a rule id', @@ -220,13 +220,13 @@ export default ({ getService }: FtrProviderContext) => { 'asc' ); expect(previewAlertsOrderedByHostIp[0]._source?.['kibana.alert.new_terms']).eql([ - 'host.ip: 10.10.0.6', + '10.10.0.6', ]); expect(previewAlertsOrderedByHostIp[1]._source?.['kibana.alert.new_terms']).eql([ - 'host.ip: 157.230.208.30', + '157.230.208.30', ]); expect(previewAlertsOrderedByHostIp[2]._source?.['kibana.alert.new_terms']).eql([ - 'host.ip: fe80::24ce:f7ff:fede:a571', + 'fe80::24ce:f7ff:fede:a571', ]); }); @@ -243,21 +243,19 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts.length).eql(3); - expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql([ - 'host.name: zeek-newyork-sha-aa8df15', - 'host.ip: 10.10.0.6', - ]); - expect(previewAlerts[1]._source?.['kibana.alert.new_terms']).eql([ - 'host.name: zeek-newyork-sha-aa8df15', - 'host.ip: 157.230.208.30', - ]); - expect(previewAlerts[2]._source?.['kibana.alert.new_terms']).eql([ - 'host.name: zeek-newyork-sha-aa8df15', - 'host.ip: fe80::24ce:f7ff:fede:a571', + const newTerms = orderBy( + previewAlerts.map((item) => item._source?.['kibana.alert.new_terms']), + ['0', '1'] + ); + + expect(newTerms).eql([ + ['zeek-newyork-sha-aa8df15', '10.10.0.6'], + ['zeek-newyork-sha-aa8df15', '157.230.208.30'], + ['zeek-newyork-sha-aa8df15', 'fe80::24ce:f7ff:fede:a571'], ]); }); - it.only('should generate 1 alert for unique combination of existing terms', async () => { + it('should generate 1 alert for unique combination of existing terms', async () => { // ensure there are no alerts for single new terms fields, it means values are not new const rule: NewTermsRuleCreateProps = { ...getCreateNewTermsRulesSchemaMock('rule-1', true), @@ -293,13 +291,10 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts.length).eql(1); - expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql([ - 'host.name: host-0', - 'host.ip: 127.0.0.2', - ]); + expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['host-0', '127.0.0.2']); }); - it('should generate alert for each new unique combination', async () => { + it('should generate 5 alerts, 1 for each new unique combination in 2 fields', async () => { // ensure there are no alerts for single new terms fields, it means values are not new const rule: NewTermsRuleCreateProps = { ...getCreateNewTermsRulesSchemaMock('rule-1', true), @@ -312,11 +307,70 @@ export default ({ getService }: FtrProviderContext) => { const { previewId } = await previewRule({ supertest, rule }); const previewAlerts = await getPreviewAlerts({ es, previewId }); - expect(previewAlerts.length).eql(4); + expect(previewAlerts.length).eql(5); + + const newTerms = orderBy( + previewAlerts.map((item) => item._source?.['kibana.alert.new_terms']), + ['0', '1'] + ); + + expect(newTerms).eql([ + ['192.168.1.1', 'tag-new-1'], + ['192.168.1.1', 'tag-new-3'], + ['192.168.1.2', 'tag-2'], + ['192.168.1.2', 'tag-new-1'], + ['192.168.1.2', 'tag-new-3'], + ]); + }); + + it('should generate 1 alert for unique combination of terms, one if which is a number', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['user.name', 'user.id'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(1); + expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', '1']); + }); + + it('should generate 1 alert for unique combination of terms, one if which is a boolean', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['user.name', 'user.enabled'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(1); + expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', 'false']); + }); + + it('should generate 1 alert for unique combination of terms, one if which is of a binary type', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['user.name', 'blob'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(1); expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql([ - 'host.name: host-0', - 'host.ip: 127.0.0.2', + 'user-0', + 'bmV3IHRlcm1zIHRlc3Q=', ]); }); @@ -336,11 +390,11 @@ export default ({ getService }: FtrProviderContext) => { const hostNames = previewAlerts .map((signal) => signal._source?.['kibana.alert.new_terms']) .sort(); - expect(hostNames[0]).eql(['host.name: suricata-sensor-amsterdam']); - expect(hostNames[1]).eql(['host.name: suricata-sensor-san-francisco']); - expect(hostNames[2]).eql(['host.name: zeek-newyork-sha-aa8df15']); - expect(hostNames[3]).eql(['host.name: zeek-sensor-amsterdam']); - expect(hostNames[4]).eql(['host.name: zeek-sensor-san-francisco']); + expect(hostNames[0]).eql(['suricata-sensor-amsterdam']); + expect(hostNames[1]).eql(['suricata-sensor-san-francisco']); + expect(hostNames[2]).eql(['zeek-newyork-sha-aa8df15']); + expect(hostNames[3]).eql(['zeek-sensor-amsterdam']); + expect(hostNames[4]).eql(['zeek-sensor-san-francisco']); }); describe('timestamp override and fallback', () => { @@ -381,8 +435,8 @@ export default ({ getService }: FtrProviderContext) => { const hostNames = previewAlerts .map((signal) => signal._source?.['kibana.alert.new_terms']) .sort(); - expect(hostNames[0]).eql(['host.name: host-3']); - expect(hostNames[1]).eql(['host.name: host-4']); + expect(hostNames[0]).eql(['host-3']); + expect(hostNames[1]).eql(['host-4']); }); }); @@ -421,10 +475,10 @@ export default ({ getService }: FtrProviderContext) => { const hostNames = previewAlerts .map((signal) => signal._source?.['kibana.alert.new_terms']) .sort(); - expect(hostNames[0]).eql(['host.name: suricata-sensor-amsterdam']); - expect(hostNames[1]).eql(['host.name: suricata-sensor-san-francisco']); - expect(hostNames[2]).eql(['host.name: zeek-newyork-sha-aa8df15']); - expect(hostNames[3]).eql(['host.name: zeek-sensor-amsterdam']); + expect(hostNames[0]).eql(['suricata-sensor-amsterdam']); + expect(hostNames[1]).eql(['suricata-sensor-san-francisco']); + expect(hostNames[2]).eql(['zeek-newyork-sha-aa8df15']); + expect(hostNames[3]).eql(['zeek-sensor-amsterdam']); }); }); @@ -446,7 +500,7 @@ export default ({ getService }: FtrProviderContext) => { const processPids = previewAlerts .map((signal) => signal._source?.['kibana.alert.new_terms']) .sort(); - expect(processPids[0]).eql(['process.pid: 1']); + expect(processPids[0]).eql([1]); }); describe('alerts should be be enriched', () => { diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json index b20058031c617..c4f36a288354a 100644 --- a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json @@ -9,6 +9,9 @@ "ip": "127.0.0.1" }, "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, @@ -27,6 +30,28 @@ "ip": "127.0.0.1" }, "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "blob": "U29tZSBiaW5hcnkgYmxvYg==", + "source.ip": "192.168.1.1" + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:02.000Z", + "host": { + "name": "host-1" + }, + "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, @@ -45,6 +70,9 @@ "ip": "127.0.0.2" }, "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, @@ -63,6 +91,9 @@ "ip": "127.0.0.1" }, "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, @@ -82,6 +113,9 @@ "ip": "127.0.0.2" }, "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, @@ -100,9 +134,54 @@ "ip": "127.0.0.2" }, "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": ["192.168.1.1", "192.168.1.2"], "tags": ["tag-new-1", "tag-2", "tag-new-3"] }, "type": "_doc" } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-20T05:00:04.000Z", + "host": { + "name": "host-0", + "ip": "127.0.0.2" + }, + "user.name": "user-0", + "user.id": 1, + "user.enabled": false, + "blob": "bmV3IHRlcm1zIHRlc3QgZm9yIHRoZSB3aW4=", + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-20T05:00:04.000Z", + "host": { + "name": "host-0", + "ip": "127.0.0.2" + }, + "user.name": "user-0", + "user.id": 0, + "user.enabled": true, + "blob": "bmV3IHRlcm1zIHRlc3Q=", + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } } \ No newline at end of file diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json index 664f3a9866aa7..b8de01c6c66e3 100644 --- a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json @@ -21,8 +21,14 @@ "properties": { "name": { "type": "keyword" + }, + "id": { + "type": "integer" + }, + "enabled": { + "type": "boolean" + } } - } }, "source": { "properties": { @@ -33,6 +39,9 @@ }, "tags": { "type": "keyword" + }, + "blob": { + "type": "binary" } } }, From 9448e2b594eaab3bc89f84ae8a8e0ef49f31760c Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Oct 2022 11:31:11 +0000 Subject: [PATCH 12/36] fix tests --- .../rules/step_define_rule/utils.ts | 3 +- .../rule_execution_logic/new_terms.ts | 19 ------- .../security_solution/new_terms/data.json | 49 ++++++++----------- 3 files changed, 22 insertions(+), 49 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts index 571a59167665d..d8b63f5801159 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts @@ -14,7 +14,8 @@ import type { BrowserField } from '../../../../common/containers/source'; * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html */ export const getTermsAggregationFields = (fields: BrowserField[]): BrowserField[] => { - const allowedTypesSet = new Set(['string', 'number', 'ip', 'boolean', 'binary']); + // binary types is excluded, as binary field has property aggregatable === false + const allowedTypesSet = new Set(['string', 'number', 'ip', 'boolean']); return fields.filter((field) => field.aggregatable === true && allowedTypesSet.has(field.type)); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 8a12efd39a297..10d718c215106 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -355,25 +355,6 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', 'false']); }); - it('should generate 1 alert for unique combination of terms, one if which is of a binary type', async () => { - const rule: NewTermsRuleCreateProps = { - ...getCreateNewTermsRulesSchemaMock('rule-1', true), - index: ['new_terms'], - new_terms_fields: ['user.name', 'blob'], - from: '2020-10-19T05:00:04.000Z', - history_window_start: '2020-10-13T05:00:04.000Z', - }; - - const { previewId } = await previewRule({ supertest, rule }); - const previewAlerts = await getPreviewAlerts({ es, previewId }); - - expect(previewAlerts.length).eql(1); - expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql([ - 'user-0', - 'bmV3IHRlcm1zIHRlc3Q=', - ]); - }); - it('should generate alerts for every term when history window is small', async () => { const rule: NewTermsRuleCreateProps = { ...getCreateNewTermsRulesSchemaMock('rule-1', true), diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json index c4f36a288354a..2953a286c97fb 100644 --- a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json @@ -11,7 +11,26 @@ "user.name": "user-0", "user.id": 0, "user.enabled": true, - "blob": "U29tZSBiaW5hcnkgYmxvYg==", + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2"] + }, + "type": "_doc" + } + } + + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:01.000Z", + "host": { + "name": "host-0", + "ip": "127.0.0.1" + }, + "user.name": "user-1", + "user.id": 1, + "user.enabled": false, "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, @@ -32,7 +51,6 @@ "user.name": "user-0", "user.id": 0, "user.enabled": true, - "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": "192.168.1.1" }, "type": "_doc" @@ -51,7 +69,6 @@ "user.name": "user-0", "user.id": 0, "user.enabled": true, - "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, @@ -72,7 +89,6 @@ "user.name": "user-0", "user.id": 0, "user.enabled": true, - "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, @@ -93,7 +109,6 @@ "user.name": "user-0", "user.id": 0, "user.enabled": true, - "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, @@ -115,7 +130,6 @@ "user.name": "user-0", "user.id": 0, "user.enabled": true, - "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, @@ -136,7 +150,6 @@ "user.name": "user-0", "user.id": 0, "user.enabled": true, - "blob": "U29tZSBiaW5hcnkgYmxvYg==", "source.ip": ["192.168.1.1", "192.168.1.2"], "tags": ["tag-new-1", "tag-2", "tag-new-3"] }, @@ -157,28 +170,6 @@ "user.name": "user-0", "user.id": 1, "user.enabled": false, - "blob": "bmV3IHRlcm1zIHRlc3QgZm9yIHRoZSB3aW4=", - "source.ip": "192.168.1.1", - "tags": ["tag-1", "tag-2"] - }, - "type": "_doc" - } - } - - { - "type": "doc", - "value": { - "index": "new_terms", - "source": { - "@timestamp": "2020-10-20T05:00:04.000Z", - "host": { - "name": "host-0", - "ip": "127.0.0.2" - }, - "user.name": "user-0", - "user.id": 0, - "user.enabled": true, - "blob": "bmV3IHRlcm1zIHRlc3Q=", "source.ip": "192.168.1.1", "tags": ["tag-1", "tag-2"] }, From 7076c7c97bec0c51196d119461fa39a8b3955e24 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Oct 2022 12:32:11 +0000 Subject: [PATCH 13/36] UX improvements --- .../common/field_maps/field_names.ts | 1 + .../event_details/get_alert_summary_rows.tsx | 11 ++++++---- .../components/alerts_table/translations.ts | 4 ++-- .../factories/utils/wrap_new_terms_alerts.ts | 7 ++++++- .../new_terms/create_new_terms_alert_type.ts | 20 ++++++++++--------- .../rule_types/new_terms/utils.ts | 16 ++++++++++++--- 6 files changed, 40 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/common/field_maps/field_names.ts b/x-pack/plugins/security_solution/common/field_maps/field_names.ts index 6a7b4efff8a7c..8e006191ea259 100644 --- a/x-pack/plugins/security_solution/common/field_maps/field_names.ts +++ b/x-pack/plugins/security_solution/common/field_maps/field_names.ts @@ -16,6 +16,7 @@ export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const; export const ALERT_THRESHOLD_RESULT = `${ALERT_NAMESPACE}.threshold_result` as const; export const ALERT_THRESHOLD_RESULT_COUNT = `${ALERT_THRESHOLD_RESULT}.count` as const; export const ALERT_NEW_TERMS = `${ALERT_NAMESPACE}.new_terms` as const; +export const ALERT_NEW_TERMS_FIELDS_VALUES = `${ALERT_NAMESPACE}.new_terms_fields_values` as const; export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const; export const ALERT_ORIGINAL_EVENT_ACTION = `${ALERT_ORIGINAL_EVENT}.action` as const; diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx index 5caca1fcd7253..7277f2f91c473 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx @@ -15,9 +15,12 @@ import { ALERTS_HEADERS_THRESHOLD_COUNT, ALERTS_HEADERS_THRESHOLD_TERMS, ALERTS_HEADERS_RULE_DESCRIPTION, - ALERTS_HEADERS_NEW_TERMS, + ALERTS_HEADERS_NEW_TERMS_FIELDS_VALUES, } from '../../../detections/components/alerts_table/translations'; -import { ALERT_NEW_TERMS, ALERT_THRESHOLD_RESULT } from '../../../../common/field_maps/field_names'; +import { + ALERT_NEW_TERMS_FIELDS_VALUES, + ALERT_THRESHOLD_RESULT, +} from '../../../../common/field_maps/field_names'; import { AGENT_STATUS_FIELD_NAME } from '../../../timelines/components/timeline/body/renderers/constants'; import type { AlertSummaryRow } from './helpers'; import { getEnrichedFieldInfo } from './helpers'; @@ -173,8 +176,8 @@ function getFieldsByRuleType(ruleType?: string): EventSummaryField[] { case 'new_terms': return [ { - id: ALERT_NEW_TERMS, - label: ALERTS_HEADERS_NEW_TERMS, + id: ALERT_NEW_TERMS_FIELDS_VALUES, + label: ALERTS_HEADERS_NEW_TERMS_FIELDS_VALUES, }, ]; default: diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts index efbe86244ab58..0fa95c7936971 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts @@ -116,8 +116,8 @@ export const ALERTS_HEADERS_THRESHOLD_CARDINALITY = i18n.translate( } ); -export const ALERTS_HEADERS_NEW_TERMS = i18n.translate( - 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTerms', +export const ALERTS_HEADERS_NEW_TERMS_FIELDS_VALUES = i18n.translate( + 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTermsFieldsValues', { defaultMessage: 'New Terms', } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts index e39bcf67909ae..45d90e7bc94e4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts @@ -13,7 +13,10 @@ import type { NewTermsFieldsLatest, WrappedFieldsLatest, } from '../../../../../../common/detection_engine/schemas/alerts'; -import { ALERT_NEW_TERMS } from '../../../../../../common/field_maps/field_names'; +import { + ALERT_NEW_TERMS, + ALERT_NEW_TERMS_FIELDS_VALUES, +} from '../../../../../../common/field_maps/field_names'; import type { ConfigType } from '../../../../../config'; import type { CompleteRule, RuleParams } from '../../../rule_schema'; import { buildReasonMessageForNewTermsAlert } from '../../../signals/reason_formatters'; @@ -23,6 +26,7 @@ import { buildBulkBody } from './build_bulk_body'; export interface EventsAndTerms { event: estypes.SearchHit; newTerms: Array; + newTermsFieldsValues: string[]; } export const wrapNewTermsAlerts = ({ @@ -65,6 +69,7 @@ export const wrapNewTermsAlerts = ({ _source: { ...baseAlert, [ALERT_NEW_TERMS]: eventAndTerms.newTerms, + [ALERT_NEW_TERMS_FIELDS_VALUES]: eventAndTerms.newTermsFieldsValues, [ALERT_UUID]: id, }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 1851584859539..073095a1e1d6c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -5,7 +5,6 @@ * 2.0. */ -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { validateNonExact } from '@kbn/securitysolution-io-ts-utils'; import { NEW_TERMS_RULE_TYPE_ID } from '@kbn/securitysolution-rules'; import { SERVER_APP_ID } from '../../../../../common/constants'; @@ -16,6 +15,7 @@ import type { CreateRuleOptions, SecurityAlertType } from '../types'; import { singleSearchAfter } from '../../signals/single_search_after'; import { getFilter } from '../../signals/get_filter'; import { wrapNewTermsAlerts } from '../factories/utils/wrap_new_terms_alerts'; +import type { EventsAndTerms } from '../factories/utils/wrap_new_terms_alerts'; import type { DocFetchAggResult, RecentTermsAggResult, @@ -26,7 +26,6 @@ import { buildRecentTermsAgg, buildNewTermsAgg, } from './build_new_terms_aggregation'; -import type { SignalSource } from '../../signals/types'; import { validateIndexPatterns } from '../utils'; import { parseDateString, @@ -35,6 +34,7 @@ import { getNewTermsRuntimeMappings, getAggregationField, prepareNewTerms, + prepareNewTermsFieldsValues, } from './utils'; import { addToSearchAfterReturn, @@ -280,13 +280,15 @@ export const createNewTermsAlertType = ( throw new Error('Aggregations were missing on document fetch search result'); } - const eventsAndTerms: Array<{ - event: estypes.SearchHit; - newTerms: Array; - }> = docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => ({ - event: bucket.docs.hits.hits[0], - newTerms: prepareNewTerms(params.newTermsFields, bucket.key), - })); + const eventsAndTerms: EventsAndTerms[] = + docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => { + const newTerms = prepareNewTerms(params.newTermsFields, bucket.key); + return { + event: bucket.docs.hits.hits[0], + newTerms, + newTermsFieldsValues: prepareNewTermsFieldsValues(params.newTermsFields, newTerms), + }; + }); const alertTimestampOverride = isPreview ? startedAt : undefined; const wrappedAlerts = wrapNewTermsAlerts({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 0822e010e2fc8..ea3dcc2f8ad88 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -128,9 +128,7 @@ const decodeBucketKey = (bucketKey: string): string[] => { }; /** - * returns new term fields and values in following format - * @example - * [ 'field1: new_value1', 'field2: new_value2'] + * decodes bucket key and returns fields as array */ export const prepareNewTerms = (newTermsFields: string[], bucketKey: string | number) => { // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here @@ -139,3 +137,15 @@ export const prepareNewTerms = (newTermsFields: string[], bucketKey: string | nu return values; // return newTermsFields.map((field, i) => [field, values[i]].join(': ')); }; + +/** + * returns new term fields and values in following format + * @example + * [ 'field1: new_value1', 'field2: new_value2'] + */ +export const prepareNewTermsFieldsValues = ( + newTermsFields: string[], + newTermsValues: Array +) => { + return newTermsFields.map((field, i) => [field, newTermsValues[i]].join(': ')); +}; From 8d39471123b560d56af47628e6339dbd1f549dcf Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Oct 2022 12:45:49 +0000 Subject: [PATCH 14/36] validation messages --- .../detections/components/rules/step_define_rule/schema.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx index bfb6750e5ac6b..fe7fc617e0e49 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx @@ -585,7 +585,7 @@ export const schema: FormSchema = { i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.newTermsFieldsMin', { - defaultMessage: 'Number of fields must be 1.', + defaultMessage: 'A minimum of one field is required.', } ) )(...args); @@ -605,7 +605,7 @@ export const schema: FormSchema = { message: i18n.translate( 'xpack.securitySolution.detectionEngine.validations.stepDefineRule.newTermsFieldsMax', { - defaultMessage: 'Number of fields must not be greater than 3.', + defaultMessage: 'Number of fields must be 3 or less.', } ), })(...args); From e4348cb47a3fbc14958b49383f950dc882bf4a57 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Oct 2022 13:22:23 +0000 Subject: [PATCH 15/36] comments --- .../rule_types/new_terms/utils.ts | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index ea3dcc2f8ad88..44e7bf1b3d940 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -51,6 +51,12 @@ export const validateHistoryWindowStart = ({ } }; +/** + * Takes a list of buckets and creates value from them to be used in 'include' clause of terms aggregation. + * For a single new terms field, value equals to bucket name + * For multiple new terms fields and buckets, value equals to concatenated base64 encoded bucket names + * @returns for buckets('host-0', 'test'), resulted value equals to: 'aG9zdC0w_dGVzdA==' + */ export const retrieveValuesFromBuckets = ( newTermsFields: string[], buckets: Array<{ key: Record }> @@ -72,11 +78,6 @@ export const retrieveValuesFromBuckets = ( ); }; -/** - * creates runtime field - * @param newTermsFields - * @returns - */ export const getNewTermsRuntimeMappings = ( newTermsFields: string[] ): undefined | { [AGG_FIELD_NAME]: estypes.MappingRuntimeField } => { @@ -111,6 +112,10 @@ export const getNewTermsRuntimeMappings = ( }; }; +/** + * For a single new terms field, aggregation field equals to new terms field + * For multiple new terms fields, aggregation field equals to defined AGG_FIELD_NAME, which is runtime field + */ export const getAggregationField = (newTermsFields: string[]): string => { // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together if (newTermsFields.length === 1) { @@ -129,19 +134,18 @@ const decodeBucketKey = (bucketKey: string): string[] => { /** * decodes bucket key and returns fields as array + * @returns 'aG9zdC0w_dGVzdA==' bucket key will result in ['host-0', 'test'] */ export const prepareNewTerms = (newTermsFields: string[], bucketKey: string | number) => { // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here const values = newTermsFields.length === 1 ? [bucketKey] : decodeBucketKey(bucketKey as string); return values; - // return newTermsFields.map((field, i) => [field, values[i]].join(': ')); }; /** * returns new term fields and values in following format - * @example - * [ 'field1: new_value1', 'field2: new_value2'] + * @returns fields(['field1', 'field2'] and values(['new_value1', 'new_value2']) will result in ['field1: new_value1', 'field2: new_value2'] */ export const prepareNewTermsFieldsValues = ( newTermsFields: string[], From 7ea355457d8aa74c0a74b251871376ac72db84e1 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Oct 2022 16:31:22 +0000 Subject: [PATCH 16/36] add unit tests --- .../new_terms/create_new_terms_alert_type.ts | 8 +- .../rule_types/new_terms/utils.test.ts | 92 ++++++++++++++++++- .../rule_types/new_terms/utils.ts | 10 +- .../rule_execution_logic/new_terms.ts | 1 + 4 files changed, 101 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 073095a1e1d6c..9ce6f8c908c33 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -30,10 +30,10 @@ import { validateIndexPatterns } from '../utils'; import { parseDateString, validateHistoryWindowStart, - retrieveValuesFromBuckets, + transformBucketsToValues, getNewTermsRuntimeMappings, getAggregationField, - prepareNewTerms, + decodeMatchedValues, prepareNewTermsFieldsValues, } from './utils'; import { @@ -194,7 +194,7 @@ export const createNewTermsAlertType = ( break; } const bucketsForField = searchResultWithAggs.aggregations.new_terms.buckets; - const includeValues = retrieveValuesFromBuckets(params.newTermsFields, bucketsForField); + const includeValues = transformBucketsToValues(params.newTermsFields, bucketsForField); // PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window. // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the // response correspond to each new term. @@ -282,7 +282,7 @@ export const createNewTermsAlertType = ( const eventsAndTerms: EventsAndTerms[] = docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => { - const newTerms = prepareNewTerms(params.newTermsFields, bucket.key); + const newTerms = decodeMatchedValues(params.newTermsFields, bucket.key); return { event: bucket.docs.hits.hits[0], newTerms, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index e1207eccf82b0..9235201d97dd3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -5,7 +5,15 @@ * 2.0. */ -import { parseDateString, validateHistoryWindowStart } from './utils'; +import { + parseDateString, + validateHistoryWindowStart, + transformBucketsToValues, + getAggregationField, + prepareNewTermsFieldsValues, + decodeMatchedValues, + AGG_FIELD_NAME, +} from './utils'; describe('new terms utils', () => { describe('parseDateString', () => { @@ -64,4 +72,86 @@ describe('new terms utils', () => { ); }); }); + + describe('transformBucketsToValues', () => { + it('should return correct value for a single new terms field', () => { + expect( + transformBucketsToValues( + ['source.host'], + [ + { + key: { + 'source.host': 'host-0', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + }, + doc_count: 3, + }, + ] + ) + ).toEqual(['host-0', 'host-1']); + }); + + it('should return correct value for multiple new terms fields', () => { + expect( + transformBucketsToValues( + ['source.host', 'source.ip'], + [ + { + key: { + 'source.host': 'host-0', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + ] + ) + ).toEqual(['aG9zdC0w_MTI3LjAuMC4x', 'aG9zdC0x_MTI3LjAuMC4x']); + }); + }); + + describe('getAggregationField', () => { + it('should return correct value for a single new terms field', () => { + expect(getAggregationField(['source.ip'])).toBe('source.ip'); + }); + it('should return correct value for multiple new terms fields', () => { + expect(getAggregationField(['source.host', 'source.ip'])).toBe(AGG_FIELD_NAME); + }); + }); + + describe('decodeMatchedValues', () => { + it('should return correct value for a single new terms field', () => { + expect(decodeMatchedValues(['source.ip'], '127.0.0.1')).toEqual(['127.0.0.1']); + }); + it('should return correct value for multiple new terms fields', () => { + expect(decodeMatchedValues(['source.host', 'source.ip'], 'aG9zdC0w_MTI3LjAuMC4x')).toEqual([ + 'host-0', + '127.0.0.1', + ]); + }); + }); + + describe('prepareNewTermsFieldsValues', () => { + it('should return correct value for a single new terms field', () => { + expect(prepareNewTermsFieldsValues(['source.ip'], ['127.0.0.1'])).toEqual([ + 'source.ip: 127.0.0.1', + ]); + }); + it('should return correct value for multiple new terms fields', () => { + expect( + prepareNewTermsFieldsValues(['source.host', 'source.ip'], ['host-0', '127.0.0.1']) + ).toEqual(['source.host: host-0', 'source.ip: 127.0.0.1']); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 44e7bf1b3d940..bbd704be6734d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -9,7 +9,7 @@ import dateMath from '@elastic/datemath'; import moment from 'moment'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -const AGG_FIELD_NAME = 'new_terms_values'; +export const AGG_FIELD_NAME = 'new_terms_values'; const DELIMITER = '_'; export const parseDateString = ({ @@ -57,9 +57,9 @@ export const validateHistoryWindowStart = ({ * For multiple new terms fields and buckets, value equals to concatenated base64 encoded bucket names * @returns for buckets('host-0', 'test'), resulted value equals to: 'aG9zdC0w_dGVzdA==' */ -export const retrieveValuesFromBuckets = ( +export const transformBucketsToValues = ( newTermsFields: string[], - buckets: Array<{ key: Record }> + buckets: estypes.AggregationsCompositeBucket[] ): Array => { // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together if (newTermsFields.length === 1) { @@ -133,10 +133,10 @@ const decodeBucketKey = (bucketKey: string): string[] => { }; /** - * decodes bucket key and returns fields as array + * decodes matched values(bucket keys) from terms aggregation and returns fields as array * @returns 'aG9zdC0w_dGVzdA==' bucket key will result in ['host-0', 'test'] */ -export const prepareNewTerms = (newTermsFields: string[], bucketKey: string | number) => { +export const decodeMatchedValues = (newTermsFields: string[], bucketKey: string | number) => { // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here const values = newTermsFields.length === 1 ? [bucketKey] : decodeBucketKey(bucketKey as string); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 10d718c215106..eea8d842878ac 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -79,6 +79,7 @@ export default ({ getService }: FtrProviderContext) => { expect(alerts.hits.hits.length).eql(1); expect(removeRandomValuedProperties(alerts.hits.hits[0]._source)).eql({ 'kibana.alert.new_terms': ['zeek-newyork-sha-aa8df15'], + 'kibana.alert.new_terms_fields_values': ['host.name: zeek-newyork-sha-aa8df15'], 'kibana.alert.rule.category': 'New Terms Rule', 'kibana.alert.rule.consumer': 'siem', 'kibana.alert.rule.name': 'Query with a rule id', From 4dc48129eacfa7840e74297643dd2fae599eee76 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Oct 2022 16:42:12 +0000 Subject: [PATCH 17/36] add functional test --- .../rule_execution_logic/new_terms.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index eea8d842878ac..e8db667e69de3 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -231,6 +231,31 @@ export default ({ getService }: FtrProviderContext) => { ]); }); + it('should generate combined property that combines new terms fields and values', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + new_terms_fields: ['host.name', 'host.ip'], + from: '2019-02-19T20:42:00.000Z', + history_window_start: '2019-01-19T20:42:00.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(3); + + const newTermsFieldsValues = orderBy( + previewAlerts.map((item) => item._source?.['kibana.alert.new_terms_fields_values']), + ['0', '1'] + ); + + expect(newTermsFieldsValues).eql([ + ['host.name: zeek-newyork-sha-aa8df15', 'host.ip: 10.10.0.6'], + ['host.name: zeek-newyork-sha-aa8df15', 'host.ip: 157.230.208.30'], + ['host.name: zeek-newyork-sha-aa8df15', 'host.ip: fe80::24ce:f7ff:fede:a571'], + ]); + }); + it('should generate 3 alerts when 1 document has 3 new values for multiple fields', async () => { const rule: NewTermsRuleCreateProps = { ...getCreateNewTermsRulesSchemaMock('rule-1', true), From 24a806bf386d2ffdd7e7911616b0d892e31a8e91 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Oct 2022 17:21:43 +0000 Subject: [PATCH 18/36] update README --- .../rule_types/new_terms/README.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index 50ec5e7682a28..e4f7f7b40eba9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -2,7 +2,7 @@ The rule accepts 2 new parameters that are unique to the new_terms rule type, in addition to common Security rule parameters such as query, index, and filters, to, from, etc. The new parameters are: -- `new_terms_fields`: an array of field names, currently limited to an array of size 1. In the future we will likely allow multiple field names to be specified here. +- `new_terms_fields`: an array of field names, currently limited to an array of size 3. Example: ['host.ip'] - `history_window_start`: defines the additional time range to search over when determining if a term is "new". If a term is found between the times `history_window_start` and from then it will not be classified as a new term. Example: now-30d @@ -12,12 +12,21 @@ Each page is evaluated in 3 phases. Phase 1: Collect "recent" terms - terms that have appeared in the last rule interval, without regard to whether or not they have appeared in historical data. This is done using a composite aggregation to ensure we can iterate over every term. Phase 2: Check if the page of terms contains any new terms. This uses a regular terms agg with the include parameter - every term is added to the array of include values, so the terms agg is limited to only aggregating on the terms of interest from phase 1. This avoids issues with the terms agg providing approximate results due to getting different terms from different shards. +For multiple new terms fields(['source.host', 'source.ip']), in terms aggregation uses a runtime field. Which is created by joining values from new terms fields into one single keyword value. Fields values encoded in base64 and joined with configured a delimiter symbol, which is not part of base64 symbols(a–Z, 0–9, +, /, =) to avoid a situation when delimiter can be part of field value. Include parameter consists of encoded in base64 results from Phase 1. Phase 3: Any new terms from phase 2 are processed and the first document to contain that term is retrieved. The document becomes the basis of the generated alert. This is done with an aggregation query that is very similar to the agg used in phase 2, except it also includes a top_hits agg. top_hits is moved to a separate, later phase for efficiency - top_hits is slow and most terms will not be new in phase 2. This means we only execute the top_hits agg on the terms that are actually new which is faster. ## Alert schema -New terms alerts have one special field at the moment: `kibana.alert.new_terms`. This field contains the detected term that caused the alert. A single source document may have multiple new terms if the source document contains an array of values in the specified field. In that case, multiple alerts will be generated from the single source document - one for each new value. +New terms alerts have following fields: +- `kibana.alert.new_terms`. This field contains the detected term that caused the alert. A single source document may have multiple new terms if the source document contains an array of values in the specified field. In that case, multiple alerts will be generated from the single source document - one for each new value. + +- `kibana.alert.new_terms_fields_values`. It contains the detected new term and corresponding field, combined in one string and split by colons. For example: +```JSON +... +"kibana.alert.new_terms_fields_values": ["host.name: host-0", "host.ip: 127.0.01"], +... +``` ## Timestamp override and fallback @@ -25,5 +34,4 @@ The new terms rule type reuses the singleSearchAfter function which implements t ## Limitations and future enhancements -- Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. -- In the future we may want to support searching for new sets of terms, e.g. a pair of `host.ip` and `host.id` that has never been seen together before. +- Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. \ No newline at end of file From a64e6d1736f279bca30539c1d2f1f120dd202f62 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Oct 2022 17:36:41 +0000 Subject: [PATCH 19/36] [Possible revert] fix tests and type check --- .../utils/wrap_new_terms_alerts.test.ts | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts index cfa696d71c47f..cc13c3cebab73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts @@ -6,7 +6,10 @@ */ import { ALERT_UUID } from '@kbn/rule-data-utils'; -import { ALERT_NEW_TERMS } from '../../../../../../common/field_maps/field_names'; +import { + ALERT_NEW_TERMS, + ALERT_NEW_TERMS_FIELDS_VALUES, +} from '../../../../../../common/field_maps/field_names'; import { getCompleteRuleMock, getNewTermsRuleParams } from '../../../rule_schema/mocks'; import { sampleDocNoSortIdWithTimestamp } from '../../../signals/__mocks__/es_results'; import { wrapNewTermsAlerts } from './wrap_new_terms_alerts'; @@ -17,7 +20,9 @@ describe('wrapNewTermsAlerts', () => { const doc = sampleDocNoSortIdWithTimestamp(docId); const completeRule = getCompleteRuleMock(getNewTermsRuleParams()); const alerts = wrapNewTermsAlerts({ - eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.1'] }], + eventsAndTerms: [ + { event: doc, newTerms: ['127.0.0.1'], newTermsFieldsValues: ['host.ip: 127.0.0.1'] }, + ], spaceId: 'default', mergeStrategy: 'missingFields', completeRule, @@ -28,13 +33,16 @@ describe('wrapNewTermsAlerts', () => { expect(alerts[0]._id).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58'); expect(alerts[0]._source[ALERT_UUID]).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58'); expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1']); + expect(alerts[0]._source[ALERT_NEW_TERMS_FIELDS_VALUES]).toEqual(['host.ip: 127.0.0.1']); }); test('should create an alert with a different _id if the space is different', () => { const doc = sampleDocNoSortIdWithTimestamp(docId); const completeRule = getCompleteRuleMock(getNewTermsRuleParams()); const alerts = wrapNewTermsAlerts({ - eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.1'] }], + eventsAndTerms: [ + { event: doc, newTerms: ['127.0.0.1'], newTermsFieldsValues: ['host.ip: 127.0.0.1'] }, + ], spaceId: 'otherSpace', mergeStrategy: 'missingFields', completeRule, @@ -45,13 +53,16 @@ describe('wrapNewTermsAlerts', () => { expect(alerts[0]._id).toEqual('f7877a31b1cc83373dbc9ba5939ebfab1db66545'); expect(alerts[0]._source[ALERT_UUID]).toEqual('f7877a31b1cc83373dbc9ba5939ebfab1db66545'); expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1']); + expect(alerts[0]._source[ALERT_NEW_TERMS_FIELDS_VALUES]).toEqual(['host.ip: 127.0.0.1']); }); test('should create an alert with a different _id if the newTerms array is different', () => { const doc = sampleDocNoSortIdWithTimestamp(docId); const completeRule = getCompleteRuleMock(getNewTermsRuleParams()); const alerts = wrapNewTermsAlerts({ - eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.2'] }], + eventsAndTerms: [ + { event: doc, newTerms: ['127.0.0.2'], newTermsFieldsValues: ['host.ip: 127.0.0.2'] }, + ], spaceId: 'otherSpace', mergeStrategy: 'missingFields', completeRule, @@ -62,13 +73,20 @@ describe('wrapNewTermsAlerts', () => { expect(alerts[0]._id).toEqual('75e5a507a4bc48bcd983820c7fd2d9621ff4e2ea'); expect(alerts[0]._source[ALERT_UUID]).toEqual('75e5a507a4bc48bcd983820c7fd2d9621ff4e2ea'); expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.2']); + expect(alerts[0]._source[ALERT_NEW_TERMS_FIELDS_VALUES]).toEqual(['host.ip: 127.0.0.2']); }); test('should create an alert with a different _id if the newTerms array contains multiple terms', () => { const doc = sampleDocNoSortIdWithTimestamp(docId); const completeRule = getCompleteRuleMock(getNewTermsRuleParams()); const alerts = wrapNewTermsAlerts({ - eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.1', '127.0.0.2'] }], + eventsAndTerms: [ + { + event: doc, + newTerms: ['127.0.0.1', 'host-0'], + newTermsFieldsValues: ['host.ip: 127.0.0.1', 'host.name: host-0'], + }, + ], spaceId: 'otherSpace', mergeStrategy: 'missingFields', completeRule, @@ -76,8 +94,12 @@ describe('wrapNewTermsAlerts', () => { alertTimestampOverride: undefined, }); - expect(alerts[0]._id).toEqual('86a216cfa4884767d9bb26d2b8db911cb4aa85ce'); - expect(alerts[0]._source[ALERT_UUID]).toEqual('86a216cfa4884767d9bb26d2b8db911cb4aa85ce'); - expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1', '127.0.0.2']); + expect(alerts[0]._id).toEqual('d2e9f981f173ef54904e36ba09802e42788bc2a9'); + expect(alerts[0]._source[ALERT_UUID]).toEqual('d2e9f981f173ef54904e36ba09802e42788bc2a9'); + expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1', 'host-0']); + expect(alerts[0]._source[ALERT_NEW_TERMS_FIELDS_VALUES]).toEqual([ + 'host.ip: 127.0.0.1', + 'host.name: host-0', + ]); }); }); From 6ecab5378ae53da387ced024b33f5184b1aa2f99 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 31 Oct 2022 17:51:56 +0000 Subject: [PATCH 20/36] Update create_new_terms.ts --- .../security_and_spaces/group1/create_new_terms.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts index 7f85acb3fa42d..9fcf96371495b 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts @@ -41,7 +41,7 @@ export default ({ getService }: FtrProviderContext) => { ); }); - it('should not be able to create a new terms rule new terms number greater than 3', async () => { + it('should not be able to create a new terms rule with field number greater than 3', async () => { const rule = { ...getCreateNewTermsRulesSchemaMock('rule-1'), history_window_start: 'now-5m', From 24f1db409eef1ee2be6dc1d12c9824d1d1277c9d Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Nov 2022 11:38:00 +0000 Subject: [PATCH 21/36] enric new terms --- .../common/field_maps/field_names.ts | 4 +- .../event_details/alert_summary_view.test.tsx | 44 +++++++++++++++++- .../event_details/get_alert_summary_rows.tsx | 46 +++++++++++++++++-- .../components/alerts_table/translations.ts | 2 +- .../utils/wrap_new_terms_alerts.test.ts | 38 ++++----------- .../factories/utils/wrap_new_terms_alerts.ts | 7 +-- .../rule_types/new_terms/README.md | 6 --- .../new_terms/create_new_terms_alert_type.ts | 2 - .../rule_types/new_terms/utils.test.ts | 14 ------ .../rule_types/new_terms/utils.ts | 11 ----- .../rule_execution_logic/new_terms.ts | 26 ----------- 11 files changed, 97 insertions(+), 103 deletions(-) diff --git a/x-pack/plugins/security_solution/common/field_maps/field_names.ts b/x-pack/plugins/security_solution/common/field_maps/field_names.ts index 8e006191ea259..53ebfc5c188d1 100644 --- a/x-pack/plugins/security_solution/common/field_maps/field_names.ts +++ b/x-pack/plugins/security_solution/common/field_maps/field_names.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE } from '@kbn/rule-data-utils'; +import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE, ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils'; export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors` as const; export const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as const; @@ -16,7 +16,7 @@ export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const; export const ALERT_THRESHOLD_RESULT = `${ALERT_NAMESPACE}.threshold_result` as const; export const ALERT_THRESHOLD_RESULT_COUNT = `${ALERT_THRESHOLD_RESULT}.count` as const; export const ALERT_NEW_TERMS = `${ALERT_NAMESPACE}.new_terms` as const; -export const ALERT_NEW_TERMS_FIELDS_VALUES = `${ALERT_NAMESPACE}.new_terms_fields_values` as const; +export const ALERT_NEW_TERMS_FIELDS = `${ALERT_RULE_PARAMETERS}.new_terms_fields` as const; export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const; export const ALERT_ORIGINAL_EVENT_ACTION = `${ALERT_ORIGINAL_EVENT}.action` as const; diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx index 662e1983680bd..9748e1da3ef3f 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx @@ -708,13 +708,55 @@ describe('AlertSummaryView', () => { ...props, data: enhancedData, }; + + const { getByText } = render( + + + + ); + + ['New Terms', '127.0.0.1'].forEach((fieldId) => { + expect(getByText(fieldId)); + }); + }); + + test('enriches New terms with new terms field names', () => { + const enhancedData = [ + ...mockAlertDetailsData.map((item) => { + if (item.category === 'kibana' && item.field === 'kibana.alert.rule.type') { + return { + ...item, + values: ['new_terms'], + originalValue: ['new_terms'], + }; + } + return item; + }), + { + category: 'kibana', + field: 'kibana.alert.new_terms', + values: ['127.0.0.1'], + originalValue: ['127.0.0.1'], + }, + { + category: 'kibana', + field: 'kibana.alert.rule.parameters.new_terms_fields', + values: ['host.ip'], + originalValue: ['host.ip'], + }, + ] as TimelineEventsDetailsItem[]; + const renderProps = { + ...props, + data: enhancedData, + }; + const { getByText } = render( ); - ['New Terms'].forEach((fieldId) => { + ['New Terms', 'host.ip: 127.0.0.1'].forEach((fieldId) => { expect(getByText(fieldId)); }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx index 7277f2f91c473..89ed961460a06 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx @@ -15,10 +15,11 @@ import { ALERTS_HEADERS_THRESHOLD_COUNT, ALERTS_HEADERS_THRESHOLD_TERMS, ALERTS_HEADERS_RULE_DESCRIPTION, - ALERTS_HEADERS_NEW_TERMS_FIELDS_VALUES, + ALERTS_HEADERS_NEW_TERMS, } from '../../../detections/components/alerts_table/translations'; import { - ALERT_NEW_TERMS_FIELDS_VALUES, + ALERT_NEW_TERMS_FIELDS, + ALERT_NEW_TERMS, ALERT_THRESHOLD_RESULT, } from '../../../../common/field_maps/field_names'; import { AGENT_STATUS_FIELD_NAME } from '../../../timelines/components/timeline/body/renderers/constants'; @@ -176,8 +177,8 @@ function getFieldsByRuleType(ruleType?: string): EventSummaryField[] { case 'new_terms': return [ { - id: ALERT_NEW_TERMS_FIELDS_VALUES, - label: ALERTS_HEADERS_NEW_TERMS_FIELDS_VALUES, + id: ALERT_NEW_TERMS, + label: ALERTS_HEADERS_NEW_TERMS, }, ]; default: @@ -323,6 +324,11 @@ export const getSummaryRows = ({ } } + if (field.id === ALERT_NEW_TERMS) { + const enrichedInfo = enrichNewTerms(item, data, description); + return [...acc, enrichedInfo]; + } + return [ ...acc, { @@ -398,3 +404,35 @@ function enrichThresholdCardinality( }; } } + +/** + * if new terms fields present and have the same length as new terms + * transform new terms in following format: + * new_terms_fields: ['field-1', 'field-2'] + * new_terms: ['value-1', 'value-2'] + * enriched new_terms: ['field-1: value-1', 'field-2: value-2'] + */ +function enrichNewTerms( + { values: newTerms }: TimelineEventsDetailsItem, + data: TimelineEventsDetailsItem[], + description: EnrichedFieldInfo +) { + const newTermsFields = data.find(({ field }) => field === ALERT_NEW_TERMS_FIELDS)?.values; + + // terms values and terms fields arrays must have the same length + if ( + !Array.isArray(newTerms) || + !Array.isArray(newTermsFields) || + newTerms.length !== newTermsFields.length + ) { + return { title: ALERTS_HEADERS_NEW_TERMS, description }; + } + + return { + title: ALERTS_HEADERS_NEW_TERMS, + description: { + ...description, + values: newTerms.map((newTerm, i) => `${newTermsFields[i]}: ${newTerm}`), + }, + }; +} diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts index 0fa95c7936971..1dcef47e8551b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts @@ -116,7 +116,7 @@ export const ALERTS_HEADERS_THRESHOLD_CARDINALITY = i18n.translate( } ); -export const ALERTS_HEADERS_NEW_TERMS_FIELDS_VALUES = i18n.translate( +export const ALERTS_HEADERS_NEW_TERMS = i18n.translate( 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTermsFieldsValues', { defaultMessage: 'New Terms', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts index cc13c3cebab73..cfa696d71c47f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.test.ts @@ -6,10 +6,7 @@ */ import { ALERT_UUID } from '@kbn/rule-data-utils'; -import { - ALERT_NEW_TERMS, - ALERT_NEW_TERMS_FIELDS_VALUES, -} from '../../../../../../common/field_maps/field_names'; +import { ALERT_NEW_TERMS } from '../../../../../../common/field_maps/field_names'; import { getCompleteRuleMock, getNewTermsRuleParams } from '../../../rule_schema/mocks'; import { sampleDocNoSortIdWithTimestamp } from '../../../signals/__mocks__/es_results'; import { wrapNewTermsAlerts } from './wrap_new_terms_alerts'; @@ -20,9 +17,7 @@ describe('wrapNewTermsAlerts', () => { const doc = sampleDocNoSortIdWithTimestamp(docId); const completeRule = getCompleteRuleMock(getNewTermsRuleParams()); const alerts = wrapNewTermsAlerts({ - eventsAndTerms: [ - { event: doc, newTerms: ['127.0.0.1'], newTermsFieldsValues: ['host.ip: 127.0.0.1'] }, - ], + eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.1'] }], spaceId: 'default', mergeStrategy: 'missingFields', completeRule, @@ -33,16 +28,13 @@ describe('wrapNewTermsAlerts', () => { expect(alerts[0]._id).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58'); expect(alerts[0]._source[ALERT_UUID]).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58'); expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1']); - expect(alerts[0]._source[ALERT_NEW_TERMS_FIELDS_VALUES]).toEqual(['host.ip: 127.0.0.1']); }); test('should create an alert with a different _id if the space is different', () => { const doc = sampleDocNoSortIdWithTimestamp(docId); const completeRule = getCompleteRuleMock(getNewTermsRuleParams()); const alerts = wrapNewTermsAlerts({ - eventsAndTerms: [ - { event: doc, newTerms: ['127.0.0.1'], newTermsFieldsValues: ['host.ip: 127.0.0.1'] }, - ], + eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.1'] }], spaceId: 'otherSpace', mergeStrategy: 'missingFields', completeRule, @@ -53,16 +45,13 @@ describe('wrapNewTermsAlerts', () => { expect(alerts[0]._id).toEqual('f7877a31b1cc83373dbc9ba5939ebfab1db66545'); expect(alerts[0]._source[ALERT_UUID]).toEqual('f7877a31b1cc83373dbc9ba5939ebfab1db66545'); expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1']); - expect(alerts[0]._source[ALERT_NEW_TERMS_FIELDS_VALUES]).toEqual(['host.ip: 127.0.0.1']); }); test('should create an alert with a different _id if the newTerms array is different', () => { const doc = sampleDocNoSortIdWithTimestamp(docId); const completeRule = getCompleteRuleMock(getNewTermsRuleParams()); const alerts = wrapNewTermsAlerts({ - eventsAndTerms: [ - { event: doc, newTerms: ['127.0.0.2'], newTermsFieldsValues: ['host.ip: 127.0.0.2'] }, - ], + eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.2'] }], spaceId: 'otherSpace', mergeStrategy: 'missingFields', completeRule, @@ -73,20 +62,13 @@ describe('wrapNewTermsAlerts', () => { expect(alerts[0]._id).toEqual('75e5a507a4bc48bcd983820c7fd2d9621ff4e2ea'); expect(alerts[0]._source[ALERT_UUID]).toEqual('75e5a507a4bc48bcd983820c7fd2d9621ff4e2ea'); expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.2']); - expect(alerts[0]._source[ALERT_NEW_TERMS_FIELDS_VALUES]).toEqual(['host.ip: 127.0.0.2']); }); test('should create an alert with a different _id if the newTerms array contains multiple terms', () => { const doc = sampleDocNoSortIdWithTimestamp(docId); const completeRule = getCompleteRuleMock(getNewTermsRuleParams()); const alerts = wrapNewTermsAlerts({ - eventsAndTerms: [ - { - event: doc, - newTerms: ['127.0.0.1', 'host-0'], - newTermsFieldsValues: ['host.ip: 127.0.0.1', 'host.name: host-0'], - }, - ], + eventsAndTerms: [{ event: doc, newTerms: ['127.0.0.1', '127.0.0.2'] }], spaceId: 'otherSpace', mergeStrategy: 'missingFields', completeRule, @@ -94,12 +76,8 @@ describe('wrapNewTermsAlerts', () => { alertTimestampOverride: undefined, }); - expect(alerts[0]._id).toEqual('d2e9f981f173ef54904e36ba09802e42788bc2a9'); - expect(alerts[0]._source[ALERT_UUID]).toEqual('d2e9f981f173ef54904e36ba09802e42788bc2a9'); - expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1', 'host-0']); - expect(alerts[0]._source[ALERT_NEW_TERMS_FIELDS_VALUES]).toEqual([ - 'host.ip: 127.0.0.1', - 'host.name: host-0', - ]); + expect(alerts[0]._id).toEqual('86a216cfa4884767d9bb26d2b8db911cb4aa85ce'); + expect(alerts[0]._source[ALERT_UUID]).toEqual('86a216cfa4884767d9bb26d2b8db911cb4aa85ce'); + expect(alerts[0]._source[ALERT_NEW_TERMS]).toEqual(['127.0.0.1', '127.0.0.2']); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts index 45d90e7bc94e4..e39bcf67909ae 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_new_terms_alerts.ts @@ -13,10 +13,7 @@ import type { NewTermsFieldsLatest, WrappedFieldsLatest, } from '../../../../../../common/detection_engine/schemas/alerts'; -import { - ALERT_NEW_TERMS, - ALERT_NEW_TERMS_FIELDS_VALUES, -} from '../../../../../../common/field_maps/field_names'; +import { ALERT_NEW_TERMS } from '../../../../../../common/field_maps/field_names'; import type { ConfigType } from '../../../../../config'; import type { CompleteRule, RuleParams } from '../../../rule_schema'; import { buildReasonMessageForNewTermsAlert } from '../../../signals/reason_formatters'; @@ -26,7 +23,6 @@ import { buildBulkBody } from './build_bulk_body'; export interface EventsAndTerms { event: estypes.SearchHit; newTerms: Array; - newTermsFieldsValues: string[]; } export const wrapNewTermsAlerts = ({ @@ -69,7 +65,6 @@ export const wrapNewTermsAlerts = ({ _source: { ...baseAlert, [ALERT_NEW_TERMS]: eventAndTerms.newTerms, - [ALERT_NEW_TERMS_FIELDS_VALUES]: eventAndTerms.newTermsFieldsValues, [ALERT_UUID]: id, }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index e4f7f7b40eba9..077820f91070f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -21,12 +21,6 @@ Phase 3: Any new terms from phase 2 are processed and the first document to cont New terms alerts have following fields: - `kibana.alert.new_terms`. This field contains the detected term that caused the alert. A single source document may have multiple new terms if the source document contains an array of values in the specified field. In that case, multiple alerts will be generated from the single source document - one for each new value. -- `kibana.alert.new_terms_fields_values`. It contains the detected new term and corresponding field, combined in one string and split by colons. For example: -```JSON -... -"kibana.alert.new_terms_fields_values": ["host.name: host-0", "host.ip: 127.0.01"], -... -``` ## Timestamp override and fallback diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 9ce6f8c908c33..bc2746ddf7888 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -34,7 +34,6 @@ import { getNewTermsRuntimeMappings, getAggregationField, decodeMatchedValues, - prepareNewTermsFieldsValues, } from './utils'; import { addToSearchAfterReturn, @@ -286,7 +285,6 @@ export const createNewTermsAlertType = ( return { event: bucket.docs.hits.hits[0], newTerms, - newTermsFieldsValues: prepareNewTermsFieldsValues(params.newTermsFields, newTerms), }; }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index 9235201d97dd3..f47dd30ca7368 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -10,7 +10,6 @@ import { validateHistoryWindowStart, transformBucketsToValues, getAggregationField, - prepareNewTermsFieldsValues, decodeMatchedValues, AGG_FIELD_NAME, } from './utils'; @@ -141,17 +140,4 @@ describe('new terms utils', () => { ]); }); }); - - describe('prepareNewTermsFieldsValues', () => { - it('should return correct value for a single new terms field', () => { - expect(prepareNewTermsFieldsValues(['source.ip'], ['127.0.0.1'])).toEqual([ - 'source.ip: 127.0.0.1', - ]); - }); - it('should return correct value for multiple new terms fields', () => { - expect( - prepareNewTermsFieldsValues(['source.host', 'source.ip'], ['host-0', '127.0.0.1']) - ).toEqual(['source.host: host-0', 'source.ip: 127.0.0.1']); - }); - }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index bbd704be6734d..2c810618f089a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -142,14 +142,3 @@ export const decodeMatchedValues = (newTermsFields: string[], bucketKey: string return values; }; - -/** - * returns new term fields and values in following format - * @returns fields(['field1', 'field2'] and values(['new_value1', 'new_value2']) will result in ['field1: new_value1', 'field2: new_value2'] - */ -export const prepareNewTermsFieldsValues = ( - newTermsFields: string[], - newTermsValues: Array -) => { - return newTermsFields.map((field, i) => [field, newTermsValues[i]].join(': ')); -}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index e8db667e69de3..10d718c215106 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -79,7 +79,6 @@ export default ({ getService }: FtrProviderContext) => { expect(alerts.hits.hits.length).eql(1); expect(removeRandomValuedProperties(alerts.hits.hits[0]._source)).eql({ 'kibana.alert.new_terms': ['zeek-newyork-sha-aa8df15'], - 'kibana.alert.new_terms_fields_values': ['host.name: zeek-newyork-sha-aa8df15'], 'kibana.alert.rule.category': 'New Terms Rule', 'kibana.alert.rule.consumer': 'siem', 'kibana.alert.rule.name': 'Query with a rule id', @@ -231,31 +230,6 @@ export default ({ getService }: FtrProviderContext) => { ]); }); - it('should generate combined property that combines new terms fields and values', async () => { - const rule: NewTermsRuleCreateProps = { - ...getCreateNewTermsRulesSchemaMock('rule-1', true), - new_terms_fields: ['host.name', 'host.ip'], - from: '2019-02-19T20:42:00.000Z', - history_window_start: '2019-01-19T20:42:00.000Z', - }; - - const { previewId } = await previewRule({ supertest, rule }); - const previewAlerts = await getPreviewAlerts({ es, previewId }); - - expect(previewAlerts.length).eql(3); - - const newTermsFieldsValues = orderBy( - previewAlerts.map((item) => item._source?.['kibana.alert.new_terms_fields_values']), - ['0', '1'] - ); - - expect(newTermsFieldsValues).eql([ - ['host.name: zeek-newyork-sha-aa8df15', 'host.ip: 10.10.0.6'], - ['host.name: zeek-newyork-sha-aa8df15', 'host.ip: 157.230.208.30'], - ['host.name: zeek-newyork-sha-aa8df15', 'host.ip: fe80::24ce:f7ff:fede:a571'], - ]); - }); - it('should generate 3 alerts when 1 document has 3 new values for multiple fields', async () => { const rule: NewTermsRuleCreateProps = { ...getCreateNewTermsRulesSchemaMock('rule-1', true), From 9e6dc8556f210058cc221a0a0018394dba73a7ba Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 1 Nov 2022 11:40:17 +0000 Subject: [PATCH 22/36] Update translations.ts --- .../public/detections/components/alerts_table/translations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts index 1dcef47e8551b..efbe86244ab58 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts @@ -117,7 +117,7 @@ export const ALERTS_HEADERS_THRESHOLD_CARDINALITY = i18n.translate( ); export const ALERTS_HEADERS_NEW_TERMS = i18n.translate( - 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTermsFieldsValues', + 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTerms', { defaultMessage: 'New Terms', } From 6be3c6ac2eb53e6f214afaf70bdcb523f115400c Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 1 Nov 2022 11:41:41 +0000 Subject: [PATCH 23/36] Update README.md --- .../lib/detection_engine/rule_types/new_terms/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index 077820f91070f..5a473a389fe9d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -18,9 +18,7 @@ Phase 3: Any new terms from phase 2 are processed and the first document to cont ## Alert schema -New terms alerts have following fields: -- `kibana.alert.new_terms`. This field contains the detected term that caused the alert. A single source document may have multiple new terms if the source document contains an array of values in the specified field. In that case, multiple alerts will be generated from the single source document - one for each new value. - +New terms alerts have one special field at the moment: `kibana.alert.new_terms`. This field contains the detected term that caused the alert. A single source document may have multiple new terms if the source document contains an array of values in the specified field. In that case, multiple alerts will be generated from the single source document - one for each new value. ## Timestamp override and fallback @@ -28,4 +26,4 @@ The new terms rule type reuses the singleSearchAfter function which implements t ## Limitations and future enhancements -- Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. \ No newline at end of file +- Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. From 624855803efa285fd4aed892baa4380dc393aab4 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Nov 2022 11:44:54 +0000 Subject: [PATCH 24/36] nits --- .../server/lib/detection_engine/rule_types/new_terms/utils.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 2c810618f089a..0c4483875f818 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -126,7 +126,6 @@ export const getAggregationField = (newTermsFields: string[]): string => { }; const decodeBucketKey = (bucketKey: string): string[] => { - // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here return bucketKey .split(DELIMITER) .map((encodedValue) => Buffer.from(encodedValue, 'base64').toString()); From 8038e480bacbe7e64ebb8d616d4ca98b1c00d914 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 1 Nov 2022 11:45:17 +0000 Subject: [PATCH 25/36] Update create_new_terms.ts --- .../security_and_spaces/group1/create_new_terms.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts index 9fcf96371495b..cd961fce7aed0 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts @@ -41,7 +41,7 @@ export default ({ getService }: FtrProviderContext) => { ); }); - it('should not be able to create a new terms rule with field number greater than 3', async () => { + it('should not be able to create a new terms rule with fields number greater than 3', async () => { const rule = { ...getCreateNewTermsRulesSchemaMock('rule-1'), history_window_start: 'now-5m', From 889763564147917e23f9a590204eaa4056ec32cc Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 1 Nov 2022 11:46:23 +0000 Subject: [PATCH 26/36] Update new_terms.ts --- .../security_and_spaces/rule_execution_logic/new_terms.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 10d718c215106..ada583879d359 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -295,7 +295,6 @@ export default ({ getService }: FtrProviderContext) => { }); it('should generate 5 alerts, 1 for each new unique combination in 2 fields', async () => { - // ensure there are no alerts for single new terms fields, it means values are not new const rule: NewTermsRuleCreateProps = { ...getCreateNewTermsRulesSchemaMock('rule-1', true), index: ['new_terms'], From 0c6b9d6dcc144a37d65ee35aa428a78d7612d6b8 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Nov 2022 11:49:09 +0000 Subject: [PATCH 27/36] fix test naming --- .../security_and_spaces/rule_execution_logic/new_terms.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index ada583879d359..30fd36af531d1 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -322,7 +322,7 @@ export default ({ getService }: FtrProviderContext) => { ]); }); - it('should generate 1 alert for unique combination of terms, one if which is a number', async () => { + it('should generate 1 alert for unique combination of terms, one of which is a number', async () => { const rule: NewTermsRuleCreateProps = { ...getCreateNewTermsRulesSchemaMock('rule-1', true), index: ['new_terms'], @@ -338,7 +338,7 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', '1']); }); - it('should generate 1 alert for unique combination of terms, one if which is a boolean', async () => { + it('should generate 1 alert for unique combination of terms, one of which is a boolean', async () => { const rule: NewTermsRuleCreateProps = { ...getCreateNewTermsRulesSchemaMock('rule-1', true), index: ['new_terms'], From e1deea8b097a4b9a98324d3d819242f919b8b7af Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Nov 2022 11:57:09 +0000 Subject: [PATCH 28/36] move to constnat --- x-pack/plugins/security_solution/common/constants.ts | 2 ++ .../model/specific_attributes/new_terms_attributes.ts | 7 ++++++- .../components/rules/step_define_rule/schema.tsx | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index 65c5757a1b1bb..051b7d36de41e 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -462,3 +462,5 @@ export const RISKY_HOSTS_DOC_LINK = 'https://www.elastic.co/guide/en/security/current/host-risk-score.html'; export const RISKY_USERS_DOC_LINK = 'https://www.elastic.co/guide/en/security/current/user-risk-score.html'; + +export const MAX_NUMBER_OF_NEW_TERMS_FIELDS = 3; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts index faffffaf50e1f..32360f262e350 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts @@ -7,6 +7,7 @@ import * as t from 'io-ts'; import { LimitedSizeArray, NonEmptyString } from '@kbn/securitysolution-io-ts-types'; +import { MAX_NUMBER_OF_NEW_TERMS_FIELDS } from '../../../../constants'; // Attributes specific to New Terms rules @@ -14,7 +15,11 @@ import { LimitedSizeArray, NonEmptyString } from '@kbn/securitysolution-io-ts-ty * New terms rule type currently only supports a single term, but should support more in the future */ export type NewTermsFields = t.TypeOf; -export const NewTermsFields = LimitedSizeArray({ codec: t.string, minSize: 1, maxSize: 3 }); +export const NewTermsFields = LimitedSizeArray({ + codec: t.string, + minSize: 1, + maxSize: MAX_NUMBER_OF_NEW_TERMS_FIELDS, +}); export type HistoryWindowStart = t.TypeOf; export const HistoryWindowStart = NonEmptyString; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx index fe7fc617e0e49..c1d3d6cd8666f 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx @@ -22,6 +22,7 @@ import { isThreatMatchRule, isThresholdRule, } from '../../../../../common/detection_engine/utils'; +import { MAX_NUMBER_OF_NEW_TERMS_FIELDS } from '../../../../../common/constants'; import { isMlRule } from '../../../../../common/machine_learning/helpers'; import type { FieldValueQueryBar } from '../query_bar'; import type { ERROR_CODE, FormSchema, ValidationFunc } from '../../../../shared_imports'; @@ -601,7 +602,7 @@ export const schema: FormSchema = { return; } return fieldValidators.maxLengthField({ - length: 3, + length: MAX_NUMBER_OF_NEW_TERMS_FIELDS, message: i18n.translate( 'xpack.securitySolution.detectionEngine.validations.stepDefineRule.newTermsFieldsMax', { From f6e760b0808d8ef1dc7fedea5b1a6cdaa6074a98 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 2 Nov 2022 14:07:49 +0000 Subject: [PATCH 29/36] Update new_terms_attributes.ts --- .../model/specific_attributes/new_terms_attributes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts index 32360f262e350..6d9f39011b675 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts @@ -12,7 +12,7 @@ import { MAX_NUMBER_OF_NEW_TERMS_FIELDS } from '../../../../constants'; // Attributes specific to New Terms rules /** - * New terms rule type currently only supports a single term, but should support more in the future + * New terms rule type supports a limited number of fields. Max number of fields is 3 and defined in common constants as MAX_NUMBER_OF_NEW_TERMS_FIELDS */ export type NewTermsFields = t.TypeOf; export const NewTermsFields = LimitedSizeArray({ From 0dae92015ae57370290d1ad99e1a4ea48504c402 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 9 Nov 2022 14:38:25 +0000 Subject: [PATCH 30/36] fix action issue --- .../common/components/event_details/get_alert_summary_rows.tsx | 1 + .../public/common/components/event_details/helpers.tsx | 2 ++ 2 files changed, 3 insertions(+) diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx index 89ed961460a06..ef97751bcc13a 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx @@ -433,6 +433,7 @@ function enrichNewTerms( description: { ...description, values: newTerms.map((newTerm, i) => `${newTermsFields[i]}: ${newTerm}`), + data: { ...description.data, field: `${ALERT_NEW_TERMS}_enriched` }, }, }; } diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx index 97ccc01a8e179..94570a9b937c8 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx @@ -14,6 +14,7 @@ import { handleSkipFocus, stopPropagationAndPreventDefault, } from '@kbn/timelines-plugin/public'; +import { ALERT_NEW_TERMS } from '../../../../common/field_maps/field_names'; import type { BrowserFields } from '../../containers/source'; import type { TimelineEventsDetailsItem } from '../../../../common/search_strategy/timeline'; import type { EnrichedFieldInfo, EventSummaryField } from './types'; @@ -176,6 +177,7 @@ export function getEnrichedFieldInfo({ */ export const FIELDS_WITHOUT_ACTIONS: { [field: string]: boolean } = { [AGENT_STATUS_FIELD_NAME]: true, + [`${ALERT_NEW_TERMS}_enriched`]: true, }; /** From 92c57a5762020f0cfa5fe3635ee2fb0e331d3901 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 9 Nov 2022 19:04:56 +0000 Subject: [PATCH 31/36] remove enrichment row in alert summary --- .../event_details/alert_summary_view.test.tsx | 37 +--------------- .../event_details/get_alert_summary_rows.tsx | 43 +++---------------- .../components/event_details/helpers.tsx | 2 - .../components/alerts_table/translations.ts | 7 +++ 4 files changed, 13 insertions(+), 76 deletions(-) diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx index 9748e1da3ef3f..beda9c0aa4d13 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx @@ -686,41 +686,6 @@ describe('AlertSummaryView', () => { }); test('New terms events have special fields', () => { - const enhancedData = [ - ...mockAlertDetailsData.map((item) => { - if (item.category === 'kibana' && item.field === 'kibana.alert.rule.type') { - return { - ...item, - values: ['new_terms'], - originalValue: ['new_terms'], - }; - } - return item; - }), - { - category: 'kibana', - field: 'kibana.alert.new_terms', - values: ['127.0.0.1'], - originalValue: ['127.0.0.1'], - }, - ] as TimelineEventsDetailsItem[]; - const renderProps = { - ...props, - data: enhancedData, - }; - - const { getByText } = render( - - - - ); - - ['New Terms', '127.0.0.1'].forEach((fieldId) => { - expect(getByText(fieldId)); - }); - }); - - test('enriches New terms with new terms field names', () => { const enhancedData = [ ...mockAlertDetailsData.map((item) => { if (item.category === 'kibana' && item.field === 'kibana.alert.rule.type') { @@ -756,7 +721,7 @@ describe('AlertSummaryView', () => { ); - ['New Terms', 'host.ip: 127.0.0.1'].forEach((fieldId) => { + ['New Terms', '127.0.0.1', 'New Terms fields', 'host.ip'].forEach((fieldId) => { expect(getByText(fieldId)); }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx index ef97751bcc13a..96bc3e030f7a2 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx @@ -16,6 +16,7 @@ import { ALERTS_HEADERS_THRESHOLD_TERMS, ALERTS_HEADERS_RULE_DESCRIPTION, ALERTS_HEADERS_NEW_TERMS, + ALERTS_HEADERS_NEW_TERMS_FIELDS, } from '../../../detections/components/alerts_table/translations'; import { ALERT_NEW_TERMS_FIELDS, @@ -176,6 +177,10 @@ function getFieldsByRuleType(ruleType?: string): EventSummaryField[] { ]; case 'new_terms': return [ + { + id: ALERT_NEW_TERMS_FIELDS, + label: ALERTS_HEADERS_NEW_TERMS_FIELDS, + }, { id: ALERT_NEW_TERMS, label: ALERTS_HEADERS_NEW_TERMS, @@ -324,11 +329,6 @@ export const getSummaryRows = ({ } } - if (field.id === ALERT_NEW_TERMS) { - const enrichedInfo = enrichNewTerms(item, data, description); - return [...acc, enrichedInfo]; - } - return [ ...acc, { @@ -404,36 +404,3 @@ function enrichThresholdCardinality( }; } } - -/** - * if new terms fields present and have the same length as new terms - * transform new terms in following format: - * new_terms_fields: ['field-1', 'field-2'] - * new_terms: ['value-1', 'value-2'] - * enriched new_terms: ['field-1: value-1', 'field-2: value-2'] - */ -function enrichNewTerms( - { values: newTerms }: TimelineEventsDetailsItem, - data: TimelineEventsDetailsItem[], - description: EnrichedFieldInfo -) { - const newTermsFields = data.find(({ field }) => field === ALERT_NEW_TERMS_FIELDS)?.values; - - // terms values and terms fields arrays must have the same length - if ( - !Array.isArray(newTerms) || - !Array.isArray(newTermsFields) || - newTerms.length !== newTermsFields.length - ) { - return { title: ALERTS_HEADERS_NEW_TERMS, description }; - } - - return { - title: ALERTS_HEADERS_NEW_TERMS, - description: { - ...description, - values: newTerms.map((newTerm, i) => `${newTermsFields[i]}: ${newTerm}`), - data: { ...description.data, field: `${ALERT_NEW_TERMS}_enriched` }, - }, - }; -} diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx index 94570a9b937c8..97ccc01a8e179 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx @@ -14,7 +14,6 @@ import { handleSkipFocus, stopPropagationAndPreventDefault, } from '@kbn/timelines-plugin/public'; -import { ALERT_NEW_TERMS } from '../../../../common/field_maps/field_names'; import type { BrowserFields } from '../../containers/source'; import type { TimelineEventsDetailsItem } from '../../../../common/search_strategy/timeline'; import type { EnrichedFieldInfo, EventSummaryField } from './types'; @@ -177,7 +176,6 @@ export function getEnrichedFieldInfo({ */ export const FIELDS_WITHOUT_ACTIONS: { [field: string]: boolean } = { [AGENT_STATUS_FIELD_NAME]: true, - [`${ALERT_NEW_TERMS}_enriched`]: true, }; /** diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts index 3f2f3ca8c5a15..21e80a6770c1f 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts @@ -123,6 +123,13 @@ export const ALERTS_HEADERS_NEW_TERMS = i18n.translate( } ); +export const ALERTS_HEADERS_NEW_TERMS_FIELDS = i18n.translate( + 'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTermsFields', + { + defaultMessage: 'New Terms fields', + } +); + export const ACTION_INVESTIGATE_IN_TIMELINE = i18n.translate( 'xpack.securitySolution.detectionEngine.alerts.actions.investigateInTimelineTitle', { From 4f8eefc962c1346abd6f4c70b41ded8464008aa0 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 10 Nov 2022 16:36:04 +0000 Subject: [PATCH 32/36] filter null values --- .../rule_types/new_terms/utils.test.ts | 46 +++++++++++++++++++ .../rule_types/new_terms/utils.ts | 18 ++++---- .../rule_execution_logic/new_terms.ts | 31 +++++++++++++ .../security_solution/new_terms/data.json | 9 ++++ .../security_solution/new_terms/mappings.json | 3 ++ 5 files changed, 99 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index f47dd30ca7368..65b948e2743ad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -95,6 +95,28 @@ describe('new terms utils', () => { ).toEqual(['host-0', 'host-1']); }); + it('should filter null values for a single new terms field', () => { + expect( + transformBucketsToValues( + ['source.host'], + [ + { + key: { + 'source.host': 'host-0', + }, + doc_count: 1, + }, + { + key: { + 'source.host': null, + }, + doc_count: 3, + }, + ] + ) + ).toEqual(['host-0']); + }); + it('should return correct value for multiple new terms fields', () => { expect( transformBucketsToValues( @@ -118,6 +140,30 @@ describe('new terms utils', () => { ) ).toEqual(['aG9zdC0w_MTI3LjAuMC4x', 'aG9zdC0x_MTI3LjAuMC4x']); }); + + it('should filter null values for multiple new terms fields', () => { + expect( + transformBucketsToValues( + ['source.host', 'source.ip'], + [ + { + key: { + 'source.host': 'host-0', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + 'source.ip': null, + }, + doc_count: 1, + }, + ] + ) + ).toEqual(['aG9zdC0w_MTI3LjAuMC4x']); + }); }); describe('getAggregationField', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 0c4483875f818..252b855664f70 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -68,14 +68,16 @@ export const transformBucketsToValues = ( .filter((value): value is string | number => value != null); } - return buckets.map((bucket) => - Object.values(bucket.key) - .filter((value): value is string | number => value != null) - .map((value) => - Buffer.from(typeof value !== 'string' ? value.toString() : value).toString('base64') - ) - .join(DELIMITER) - ); + return buckets + .map((bucket) => Object.values(bucket.key)) + .filter((values) => !values.some((value) => value == null)) + .map((values) => + values + .map((value) => + Buffer.from(typeof value !== 'string' ? value.toString() : value).toString('base64') + ) + .join(DELIMITER) + ); }; export const getNewTermsRuntimeMappings = ( diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 30fd36af531d1..f3cb4199452a7 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -377,6 +377,37 @@ export default ({ getService }: FtrProviderContext) => { expect(hostNames[4]).eql(['zeek-sensor-san-francisco']); }); + describe('null values', () => { + it('should not generate alerts with null values for single field', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['possibly_null_field'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(0); + }); + + it('should not generate alerts with null values for multiple fields', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['possibly_null_field', 'host.name'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(0); + }); + }); describe('timestamp override and fallback', () => { before(async () => { await esArchiver.load( diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json index 2953a286c97fb..a2495931dc9af 100644 --- a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json @@ -4,6 +4,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-14T05:00:01.000Z", + "possibly_null_field": "test-value", "host": { "name": "host-0", "ip": "127.0.0.1" @@ -24,6 +25,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-14T05:00:01.000Z", + "possibly_null_field": "test-value", "host": { "name": "host-0", "ip": "127.0.0.1" @@ -44,6 +46,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-14T05:00:02.000Z", + "possibly_null_field": "test-value", "host": { "name": "host-1", "ip": "127.0.0.1" @@ -63,6 +66,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-14T05:00:02.000Z", + "possibly_null_field": "test-value", "host": { "name": "host-1" }, @@ -82,6 +86,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-14T05:00:03.000Z", + "possibly_null_field": "test-value", "host": { "name": "host-1", "ip": "127.0.0.2" @@ -102,6 +107,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-14T05:00:04.000Z", + "possibly_null_field": "test-value", "host": { "name": "host-0", "ip": "127.0.0.1" @@ -123,6 +129,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-20T05:00:04.000Z", + "possibly_null_field": "test-value", "host": { "name": "host-0", "ip": "127.0.0.2" @@ -143,6 +150,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-20T05:00:04.000Z", + "possibly_null_field": "test-value", "host": { "name": "host-0", "ip": "127.0.0.2" @@ -163,6 +171,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-20T05:00:04.000Z", + "possibly_null_field": null, "host": { "name": "host-0", "ip": "127.0.0.2" diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json index b8de01c6c66e3..acac4c50ce7af 100644 --- a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json @@ -42,6 +42,9 @@ }, "blob": { "type": "binary" + }, + "possibly_null_field": { + "type": "keyword" } } }, From 28d029e7b2e7424c2b8f65dcc04cd65071e27737 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 10 Nov 2022 19:28:32 +0000 Subject: [PATCH 33/36] cover emit fields limit --- .../rule_types/new_terms/utils.ts | 34 +++++++---- .../rule_execution_logic/new_terms.ts | 58 +++++++++++++++++++ .../security_solution/new_terms/data.json | 13 ++++- .../security_solution/new_terms/mappings.json | 9 +++ 4 files changed, 99 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 252b855664f70..ec44654a52d0d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -94,20 +94,30 @@ export const getNewTermsRuntimeMappings = ( script: { params: { fields: newTermsFields }, source: ` - void traverseDocFields(def doc, def fields, def index, def line) { - if (index === fields.length) { - emit(line); - } else { - for (field in doc[fields[index]]) { - def delimiter = index === 0 ? '' : '${DELIMITER}'; - def nextLine = line + delimiter + String.valueOf(field).encodeBase64(); - - traverseDocFields(doc, fields, index + 1, nextLine); + def stack = new Stack(); + // ES has limit in 100 values for runtime field, after this query will fail + int emitLimit = 100; + stack.add([0, '']); + + while (stack.length > 0) { + if (emitLimit == 0) { + break; + } + def tuple = stack.pop(); + def index = tuple[0]; + def line = tuple[1]; + if (index === params['fields'].length) { + emit(line); + emitLimit = emitLimit - 1; + } else { + for (field in doc[params['fields'][index]]) { + def delimiter = index === 0 ? '' : '${DELIMITER}'; + def nextLine = line + delimiter + String.valueOf(field).encodeBase64(); + + stack.add([index + 1, nextLine]) + } } - } } - - traverseDocFields(doc, params['fields'], 0, ''); `, }, }, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index f3cb4199452a7..30a18254862c6 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -408,6 +408,64 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts.length).eql(0); }); }); + + describe('large arrays values', () => { + it('should generate alerts for unique values in large array for single field from a single document', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['large_array_20'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + // there are 20 unique values for the document, but only 10 alerts generated + expect(previewAlerts.length).eql(10); + }); + + // There is a limit in ES for a number of emitted values in runtime field (100) + // This test ensures rule run doesn't fail if processed fields in runtime script generates 100 values, hard limit for ES + // For this test case: large_array_10 & large_array_5 have 100 unique combination in total + it('should generate alerts for array fields that have 100 unique combination of values in runtime field', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['large_array_10', 'large_array_5'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + // there are 100 unique values for the document, but only 10 alerts generated + expect(previewAlerts.length).eql(10); + }); + + // There is a limit in ES for a number of emitted values in runtime field (100) + // This test ensures rule run doesn't fail if processed fields in runtime script generates 200 values + // In case of this test case: large_array_10 & large_array_20 have 200 unique combination in total + // Rule run should not fail and should generate alerts + it('should generate alert for array fields that have more than 200 unique combination of values in runtime field', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['large_array_10', 'large_array_20'], + from: '2020-10-19T05:00:04.000Z', + history_window_start: '2020-10-13T05:00:04.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + // there are 200 unique values for the document, but only 10 alerts generated + expect(previewAlerts.length).eql(10); + }); + }); + describe('timestamp override and fallback', () => { before(async () => { await esArchiver.load( diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json index a2495931dc9af..6dccf315e2e05 100644 --- a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json @@ -13,8 +13,12 @@ "user.id": 0, "user.enabled": true, "source.ip": "192.168.1.1", - "tags": ["tag-1", "tag-2"] - }, + "tags": ["tag-1", "tag-2"], + "large_array_10": ["value-of-10-0","value-of-10-1","value-of-10-2","value-of-10-3","value-of-10-4","value-of-10-5","value-of-10-6","value-of-10-7","value-of-10-8","value-of-10-9"], + "large_array_5": ["value-of-5-0","value-of-5-1","value-of-5-2","value-of-5-3","value-of-5-4"], + "large_array_20": ["value-of-20-0","value-of-20-1","value-of-20-2","value-of-20-3","value-of-20-4","value-of-20-5","value-of-20-6","value-of-20-7","value-of-20-8","value-of-20-9","value-of-20-10","value-of-20-11","value-of-20-12","value-of-20-13","value-of-20-14","value-of-20-15","value-of-20-16","value-of-20-17","value-of-20-18","value-of-20-19"] + } + , "type": "_doc" } } @@ -180,7 +184,10 @@ "user.id": 1, "user.enabled": false, "source.ip": "192.168.1.1", - "tags": ["tag-1", "tag-2"] + "tags": ["tag-1", "tag-2"], + "large_array_10": ["a-new-value-of-10-0","a-new-value-of-10-1","a-new-value-of-10-2","a-new-value-of-10-3","a-new-value-of-10-4","a-new-value-of-10-5","a-new-value-of-10-6","a-new-value-of-10-7","a-new-value-of-10-8","a-new-value-of-10-9"], + "large_array_5": ["another-new-value-of-10-0","another-new-value-of-10-1","another-new-value-of-10-2","another-new-value-of-10-3","another-new-value-of-10-4","another-new-value-of-10-5","another-new-value-of-10-6","another-new-value-of-10-7","another-new-value-of-10-8","another-new-value-of-10-9"], + "large_array_20": ["a-new-value-of-20-0","a-new-value-of-20-1","a-new-value-of-20-2","a-new-value-of-20-3","a-new-value-of-20-4","a-new-value-of-20-5","a-new-value-of-20-6","a-new-value-of-20-7","a-new-value-of-20-8","a-new-value-of-20-9","a-new-value-of-20-10","a-new-value-of-20-11","a-new-value-of-20-12","a-new-value-of-20-13","a-new-value-of-20-14","a-new-value-of-20-15","a-new-value-of-20-16","a-new-value-of-20-17","a-new-value-of-20-18","a-new-value-of-20-19"] }, "type": "_doc" } diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json index acac4c50ce7af..4af549454c474 100644 --- a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json @@ -45,6 +45,15 @@ }, "possibly_null_field": { "type": "keyword" + }, + "large_array_10": { + "type": "keyword" + }, + "large_array_20": { + "type": "keyword" + }, + "large_array_5": { + "type": "keyword" } } }, From fb3d08b37bbc4f3eed6f1ba6ae1f9363cf8c53b5 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 10 Nov 2022 19:49:00 +0000 Subject: [PATCH 34/36] fix tests --- .../rule_execution_logic/new_terms.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 30a18254862c6..5a0bf346b5c75 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -420,10 +420,9 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); - const previewAlerts = await getPreviewAlerts({ es, previewId }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 100 }); - // there are 20 unique values for the document, but only 10 alerts generated - expect(previewAlerts.length).eql(10); + expect(previewAlerts.length).eql(20); }); // There is a limit in ES for a number of emitted values in runtime field (100) @@ -439,10 +438,9 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); - const previewAlerts = await getPreviewAlerts({ es, previewId }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 }); - // there are 100 unique values for the document, but only 10 alerts generated - expect(previewAlerts.length).eql(10); + expect(previewAlerts.length).eql(100); }); // There is a limit in ES for a number of emitted values in runtime field (100) @@ -459,10 +457,9 @@ export default ({ getService }: FtrProviderContext) => { }; const { previewId } = await previewRule({ supertest, rule }); - const previewAlerts = await getPreviewAlerts({ es, previewId }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 }); - // there are 200 unique values for the document, but only 10 alerts generated - expect(previewAlerts.length).eql(10); + expect(previewAlerts.length).eql(100); }); }); From bc50e91d1fea8b57fa62f733dddd6a08b8274b88 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 14 Nov 2022 15:19:02 +0000 Subject: [PATCH 35/36] add tests for runtime script --- .../rule_types/new_terms/utils.test.ts | 22 +++ .../rule_types/new_terms/utils.ts | 2 +- .../rule_execution_logic/new_terms.ts | 144 ++++++++++++++++++ .../utils/index.ts | 1 + .../utils/perform_search_query.ts | 44 ++++++ .../security_solution/new_terms/data.json | 26 ++++ .../security_solution/new_terms/mappings.json | 3 + 7 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 x-pack/test/detection_engine_api_integration/utils/perform_search_query.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index 65b948e2743ad..2b04b617ba9ba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -11,6 +11,7 @@ import { transformBucketsToValues, getAggregationField, decodeMatchedValues, + getNewTermsRuntimeMappings, AGG_FIELD_NAME, } from './utils'; @@ -186,4 +187,25 @@ describe('new terms utils', () => { ]); }); }); + + describe('getNewTermsRuntimeMappings', () => { + it('should not return runtime field if new terms fields is empty', () => { + expect(getNewTermsRuntimeMappings([])).toBeUndefined(); + }); + it('should not return runtime field if new terms fields has only one field', () => { + expect(getNewTermsRuntimeMappings(['host.name'])).toBeUndefined(); + }); + + it('should return runtime field if new terms fields has more than one field', () => { + const runtimeMappings = getNewTermsRuntimeMappings(['host.name', 'host.ip']); + + expect(runtimeMappings?.[AGG_FIELD_NAME]).toMatchObject({ + type: 'keyword', + script: { + params: { fields: ['host.name', 'host.ip'] }, + source: expect.any(String), + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index ec44654a52d0d..cebd63f17e663 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -84,7 +84,7 @@ export const getNewTermsRuntimeMappings = ( newTermsFields: string[] ): undefined | { [AGG_FIELD_NAME]: estypes.MappingRuntimeField } => { // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together - if (newTermsFields.length === 1) { + if (newTermsFields.length <= 1) { return undefined; } diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 5a0bf346b5c75..3b1304f12e6c3 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -11,6 +11,10 @@ import { NewTermsRuleCreateProps } from '@kbn/security-solution-plugin/common/de import { orderBy } from 'lodash'; import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema/mocks'; import { DetectionAlert } from '@kbn/security-solution-plugin/common/detection_engine/schemas/alerts'; +import { + getNewTermsRuntimeMappings, + AGG_FIELD_NAME, +} from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/new_terms/utils'; import { createRule, deleteAllAlerts, @@ -18,6 +22,7 @@ import { getOpenSignals, getPreviewAlerts, previewRule, + performSearchQuery, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { previewRuleWithExceptionEntries } from '../../utils/preview_rule_with_exception_entries'; @@ -593,5 +598,144 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts[0]?._source?.host?.risk?.calculated_score_norm).to.eql(23); }); }); + + describe('runtime field', () => { + it('should return runtime field created from 2 single values', async () => { + // encoded base64 values of "host-0" and "127.0.0.1" joined with underscore + const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should return runtime field created from 2 single values, including number value', async () => { + // encoded base64 values of "user-0" and 0 joined with underscore + const expectedEncodedValues = ['dXNlci0w_MA==']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.id']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should return runtime field created from 2 single values, including boolean value', async () => { + // encoded base64 values of "user-0" and true joined with underscore + const expectedEncodedValues = ['dXNlci0w_dHJ1ZQ==']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.enabled']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should return runtime field created from 3 single values', async () => { + // encoded base64 values of "host-0" and "127.0.0.1" and "user-0" joined with underscore + const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x_dXNlci0w']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip', 'user.name']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should return runtime field created from fields of arrays', async () => { + // encoded base64 values of all combinations of ["192.168.1.1", "192.168.1.2"] + // and ["tag-new-1", "tag-2", "tag-new-3"] joined with underscore + const expectedEncodedValues = [ + 'MTkyLjE2OC4xLjE=_dGFnLTI=', + 'MTkyLjE2OC4xLjE=_dGFnLW5ldy0x', + 'MTkyLjE2OC4xLjE=_dGFnLW5ldy0z', + 'MTkyLjE2OC4xLjI=_dGFnLTI=', + 'MTkyLjE2OC4xLjI=_dGFnLW5ldy0x', + 'MTkyLjE2OC4xLjI=_dGFnLW5ldy0z', + ]; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'doc_with_source_ip_as_array' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['source.ip', 'tags']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should return runtime field without duplicated values', async () => { + // encoded base64 values of "host-0" and ["tag-1", "tag-2", "tag-2", "tag-1", "tag-1"] + // joined with underscore, without duplicates in tags + const expectedEncodedValues = ['aG9zdC0w_dGFnLTE=', 'aG9zdC0w_dGFnLTI=']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'doc_with_duplicated_tags' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'tags']), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + + it('should not return runtime field if one of fields is null', async () => { + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'doc_with_null_field' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME, 'possibly_null_field', 'host.name'], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'possibly_null_field']), + }); + + expect(hits.hits.length).to.be(1); + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.be(undefined); + expect(hits.hits[0].fields?.possibly_null_field).to.be(undefined); + expect(hits.hits[0].fields?.['host.name']).to.eql(['host-0']); + }); + + it('should not return runtime field if one of fields is not defined', async () => { + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'doc_without_large_arrays' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'large_array_5']), + }); + + expect(hits.hits.length).to.be(1); + expect(hits.hits[0].fields).to.be(undefined); + }); + + // There is a limit in ES for a number of emitted values in runtime field (100) + // This test makes sure runtime script doesn't cause query failure and returns first 100 results + it('should return runtime field if number of emitted values greater than 100', async () => { + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['large_array_20', 'large_array_10']), + }); + + // runtime field should have 100 values, as large_array_20 and large_array_10 + // give in total 200 combinations + expect(hits.hits[0].fields?.[AGG_FIELD_NAME].length).to.be(100); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/utils/index.ts b/x-pack/test/detection_engine_api_integration/utils/index.ts index b686589addc09..7d03141f58f10 100644 --- a/x-pack/test/detection_engine_api_integration/utils/index.ts +++ b/x-pack/test/detection_engine_api_integration/utils/index.ts @@ -81,6 +81,7 @@ export * from './get_web_hook_action'; export * from './index_event_log_execution_events'; export * from './install_prepackaged_rules'; export * from './machine_learning_setup'; +export * from './perform_search_query'; export * from './preview_rule_with_exception_entries'; export * from './preview_rule'; export * from './refresh_index'; diff --git a/x-pack/test/detection_engine_api_integration/utils/perform_search_query.ts b/x-pack/test/detection_engine_api_integration/utils/perform_search_query.ts new file mode 100644 index 0000000000000..6afd1eebb501c --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/perform_search_query.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; + +import type { + QueryDslQueryContainer, + MappingRuntimeFields, + IndexName, + Field, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +interface PerformSearchQueryArgs { + es: Client; + query: QueryDslQueryContainer; + index: IndexName; + size?: number; + runtimeMappings?: MappingRuntimeFields; + fields?: Field[]; +} + +/** + * run ES search query + */ +export const performSearchQuery = async ({ + es, + query, + index, + size = 10, + runtimeMappings, + fields, +}: PerformSearchQueryArgs) => { + return es.search({ + index, + size, + fields, + query, + runtime_mappings: runtimeMappings, + }); +}; diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json index 6dccf315e2e05..6970a37472c3b 100644 --- a/x-pack/test/functional/es_archives/security_solution/new_terms/data.json +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/data.json @@ -4,6 +4,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-14T05:00:01.000Z", + "id": "first_doc", "possibly_null_field": "test-value", "host": { "name": "host-0", @@ -29,6 +30,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-14T05:00:01.000Z", + "id": "doc_without_large_arrays", "possibly_null_field": "test-value", "host": { "name": "host-0", @@ -44,6 +46,28 @@ } } + { + "type": "doc", + "value": { + "index": "new_terms", + "source": { + "@timestamp": "2020-10-14T05:00:01.000Z", + "id": "doc_with_duplicated_tags", + "possibly_null_field": "test-value", + "host": { + "name": "host-0", + "ip": "127.0.0.1" + }, + "user.name": "user-1", + "user.id": 1, + "user.enabled": false, + "source.ip": "192.168.1.1", + "tags": ["tag-1", "tag-2", "tag-2", "tag-1", "tag-1"] + }, + "type": "_doc" + } + } + { "type": "doc", "value": { @@ -154,6 +178,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-20T05:00:04.000Z", + "id": "doc_with_source_ip_as_array", "possibly_null_field": "test-value", "host": { "name": "host-0", @@ -175,6 +200,7 @@ "index": "new_terms", "source": { "@timestamp": "2020-10-20T05:00:04.000Z", + "id": "doc_with_null_field", "possibly_null_field": null, "host": { "name": "host-0", diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json index 4af549454c474..2f156ddedf580 100644 --- a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json @@ -4,6 +4,9 @@ "index": "new_terms", "mappings": { "properties": { + "id": { + "type": "keyword" + }, "@timestamp": { "type": "date" }, From cb135ad1073f6de7c6c46b25d98959c2f996db1c Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 14 Nov 2022 15:29:39 +0000 Subject: [PATCH 36/36] Update README.md --- .../server/lib/detection_engine/rule_types/new_terms/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index 5a473a389fe9d..694fdd53fe2f4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -27,3 +27,4 @@ The new terms rule type reuses the singleSearchAfter function which implements t ## Limitations and future enhancements - Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. +- Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields