From e918ed0d3bf1cc3061d59742784d6d1f3b40070e Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 14 Nov 2022 17:58:00 +0000 Subject: [PATCH 01/14] POC for performancd improvements --- .../new_terms/create_new_terms_alert_type.ts | 6 ++- .../rule_types/new_terms/utils.ts | 41 +++++++++++++++++-- .../rule_execution_logic/new_terms.ts | 17 ++++++++ 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index bc2746ddf7888..6737b492f4a00 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -34,6 +34,7 @@ import { getNewTermsRuntimeMappings, getAggregationField, decodeMatchedValues, + createFieldValuesMap, } from './utils'; import { addToSearchAfterReturn, @@ -194,6 +195,7 @@ export const createNewTermsAlertType = ( } const bucketsForField = searchResultWithAggs.aggregations.new_terms.buckets; const includeValues = transformBucketsToValues(params.newTermsFields, bucketsForField); + const fieldsValuesMap = createFieldValuesMap(params.newTermsFields, bucketsForField); // PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window. // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the // response correspond to each new term. @@ -210,7 +212,7 @@ export const createNewTermsAlertType = ( }), runtimeMappings: { ...runtimeMappings, - ...getNewTermsRuntimeMappings(params.newTermsFields), + ...getNewTermsRuntimeMappings(params.newTermsFields, fieldsValuesMap), }, searchAfterSortIds: undefined, index: inputIndex, @@ -256,7 +258,7 @@ export const createNewTermsAlertType = ( }), runtimeMappings: { ...runtimeMappings, - ...getNewTermsRuntimeMappings(params.newTermsFields), + ...getNewTermsRuntimeMappings(params.newTermsFields, fieldsValuesMap), }, searchAfterSortIds: undefined, index: inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index cebd63f17e663..cbca9353d31b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -80,8 +80,36 @@ export const transformBucketsToValues = ( ); }; +export const createFieldValuesMap = ( + newTermsFields: string[], + buckets: estypes.AggregationsCompositeBucket[] +) => { + if (newTermsFields.length === 1) { + return undefined; + } + + const valuesMap = newTermsFields.reduce>>( + (acc, field) => ({ ...acc, [field]: {} }), + {} + ); + + buckets + .map((bucket) => bucket.key) + .forEach((bucket) => { + Object.entries(bucket).forEach(([key, value]) => { + const strValue = typeof value !== 'string' ? value.toString() : value; + if (strValue != null) { + valuesMap[key][strValue] = true; + } + }); + }); + + return valuesMap; +}; + export const getNewTermsRuntimeMappings = ( - newTermsFields: string[] + newTermsFields: string[], + values?: Record> ): undefined | { [AGG_FIELD_NAME]: estypes.MappingRuntimeField } => { // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together if (newTermsFields.length <= 1) { @@ -92,7 +120,7 @@ export const getNewTermsRuntimeMappings = ( [AGG_FIELD_NAME]: { type: 'keyword', script: { - params: { fields: newTermsFields }, + params: { fields: newTermsFields, values }, source: ` def stack = new Stack(); // ES has limit in 100 values for runtime field, after this query will fail @@ -110,9 +138,14 @@ export const getNewTermsRuntimeMappings = ( emit(line); emitLimit = emitLimit - 1; } else { - for (field in doc[params['fields'][index]]) { + def fieldName = params['fields'][index]; + for (field in doc[fieldName]) { + def fieldStr = String.valueOf(field); + if (!params['values'][fieldName].containsKey(fieldStr)) { + continue; + } def delimiter = index === 0 ? '' : '${DELIMITER}'; - def nextLine = line + delimiter + String.valueOf(field).encodeBase64(); + def nextLine = line + delimiter + fieldStr.encodeBase64(); stack.add([index + 1, nextLine]) } diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 3b1304f12e6c3..3eff5aa1961dc 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -600,6 +600,23 @@ export default ({ getService }: FtrProviderContext) => { }); describe('runtime field', () => { + it('[POC] should return runtime field created from 2 single values', async () => { + // encoded base64 values of "host-0" and "127.0.0.1" joined with underscore + const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip'], { + 'host.name': { 'host-0': true }, + 'host.ip': { '127.0.0.1': true }, + }), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + it('should return runtime field created from 2 single values', async () => { // encoded base64 values of "host-0" and "127.0.0.1" joined with underscore const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x']; From f3bf6ac0602ae750ad7cf0b60096bba0881e831d Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 15 Nov 2022 14:07:55 +0000 Subject: [PATCH 02/14] fix tests --- .../rule_execution_logic/new_terms.ts | 86 ++++++++++++++++--- 1 file changed, 72 insertions(+), 14 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 3eff5aa1961dc..2fa681f2843f1 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -600,7 +600,7 @@ export default ({ getService }: FtrProviderContext) => { }); describe('runtime field', () => { - it('[POC] should return runtime field created from 2 single values', async () => { + it('should return runtime field created from 2 single values', async () => { // encoded base64 values of "host-0" and "127.0.0.1" joined with underscore const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x']; const { hits } = await performSearchQuery({ @@ -617,18 +617,19 @@ export default ({ getService }: FtrProviderContext) => { expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); }); - it('should return runtime field created from 2 single values', async () => { - // encoded base64 values of "host-0" and "127.0.0.1" joined with underscore - const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x']; + it('should not return runtime field created from 2 single values if its value is not in value map', async () => { const { hits } = await performSearchQuery({ es, query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip']), + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip'], { + 'host.name': { 'host-0': true }, + 'host.ip': {}, + }), }); - expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.be(undefined); }); it('should return runtime field created from 2 single values, including number value', async () => { @@ -639,7 +640,10 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.id']), + runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.id'], { + 'user.name': { 'user-0': true }, + 'user.id': { '0': true }, + }), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -653,7 +657,10 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.enabled']), + runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.enabled'], { + 'user.name': { 'user-0': true }, + 'user.enabled': { ['true']: true }, + }), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -667,7 +674,11 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip', 'user.name']), + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip', 'user.name'], { + 'host.name': { 'host-0': true }, + 'host.ip': { '127.0.0.1': true }, + 'user.name': { 'user-0': true }, + }), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -689,7 +700,10 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'doc_with_source_ip_as_array' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['source.ip', 'tags']), + runtimeMappings: getNewTermsRuntimeMappings(['source.ip', 'tags'], { + 'source.ip': { '192.168.1.1': true, '192.168.1.2': true }, + tags: { 'tag-new-1': true, 'tag-2': true, 'tag-new-3': true }, + }), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -704,7 +718,10 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'doc_with_duplicated_tags' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'tags']), + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'tags'], { + 'host.name': { 'host-0': true }, + tags: { 'tag-1': true, 'tag-2': true }, + }), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -716,7 +733,10 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'doc_with_null_field' } }, index: 'new_terms', fields: [AGG_FIELD_NAME, 'possibly_null_field', 'host.name'], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'possibly_null_field']), + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'possibly_null_field'], { + 'host.name': { 'host-0': true }, + tags: { ['null']: true }, + }), }); expect(hits.hits.length).to.be(1); @@ -731,7 +751,10 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'doc_without_large_arrays' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'large_array_5']), + runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'large_array_5'], { + 'host.name': { 'host-0': true }, + large_array_5: {}, + }), }); expect(hits.hits.length).to.be(1); @@ -746,7 +769,42 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['large_array_20', 'large_array_10']), + runtimeMappings: getNewTermsRuntimeMappings(['large_array_20', 'large_array_10'], { + large_array_20: { + 'value-of-20-0': true, + 'value-of-20-1': true, + 'value-of-20-2': true, + 'value-of-20-3': true, + 'value-of-20-4': true, + 'value-of-20-5': true, + 'value-of-20-6': true, + 'value-of-20-7': true, + 'value-of-20-8': true, + 'value-of-20-9': true, + 'value-of-20-10': true, + 'value-of-20-11': true, + 'value-of-20-12': true, + 'value-of-20-13': true, + 'value-of-20-14': true, + 'value-of-20-15': true, + 'value-of-20-16': true, + 'value-of-20-17': true, + 'value-of-20-18': true, + 'value-of-20-19': true, + }, + large_array_10: { + 'value-of-10-0': true, + 'value-of-10-1': true, + 'value-of-10-2': true, + 'value-of-10-3': true, + 'value-of-10-4': true, + 'value-of-10-5': true, + 'value-of-10-6': true, + 'value-of-10-7': true, + 'value-of-10-8': true, + 'value-of-10-9': true, + }, + }), }); // runtime field should have 100 values, as large_array_20 and large_array_10 From 53083017f40b675c6ad3a4b26100cbc87b54e357 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 15 Nov 2022 17:23:31 +0000 Subject: [PATCH 03/14] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- .../page_steps/install_agent/install_agent_managed.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx index aa3b0830aa200..68f78017c962c 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx @@ -19,9 +19,10 @@ import { } from '../../../../../../../../../components/agent_enrollment_flyout/steps'; import { ManualInstructions } from '../../../../../../../../../components/enrollment_instructions'; -import type { InstallAgentPageProps } from './types'; import { KubernetesManifestApplyStep } from '../../../../../../../../../components/agent_enrollment_flyout/steps/run_k8s_apply_command_step'; +import type { InstallAgentPageProps } from './types'; + export const InstallElasticAgentManagedPageStep: React.FC = (props) => { const { cancelUrl, From f28d14416b92ab00a6c647e96aeeddf9b3cc50d8 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 16 Nov 2022 11:19:08 +0000 Subject: [PATCH 04/14] add tests/update README --- .../rule_types/new_terms/README.md | 1 - .../rule_types/new_terms/utils.test.ts | 135 ++++++++++++++++++ .../rule_types/new_terms/utils.ts | 13 +- 3 files changed, 145 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index 694fdd53fe2f4..5a473a389fe9d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -27,4 +27,3 @@ The new terms rule type reuses the singleSearchAfter function which implements t ## Limitations and future enhancements - Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. -- Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index 2b04b617ba9ba..a4f0650db8413 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -12,6 +12,7 @@ import { getAggregationField, decodeMatchedValues, getNewTermsRuntimeMappings, + createFieldValuesMap, AGG_FIELD_NAME, } from './utils'; @@ -209,3 +210,137 @@ describe('new terms utils', () => { }); }); }); + +describe('createFieldValuesMap', () => { + it('should return undefined if new terms fields has only one field', () => { + expect( + createFieldValuesMap( + ['host.name'], + [ + { + key: { + 'source.host': 'host-0', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + }, + doc_count: 3, + }, + ] + ) + ).toBeUndefined(); + }); + + it('should return values map if new terms fields has more than one field', () => { + expect( + createFieldValuesMap( + ['source.host', 'source.ip'], + [ + { + key: { + 'source.host': 'host-0', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + ] + ) + ).toEqual({ + 'source.host': { + 'host-0': true, + 'host-1': true, + }, + 'source.ip': { + '127.0.0.1': true, + }, + }); + }); + + it('should not put value in map if it is null', () => { + expect( + createFieldValuesMap( + ['source.host', 'source.ip'], + [ + { + key: { + 'source.host': 'host-1', + 'source.ip': null, + }, + doc_count: 1, + }, + ] + ) + ).toEqual({ + 'source.host': { + 'host-1': true, + }, + 'source.ip': {}, + }); + }); + + it('should not put value in map if it is a number', () => { + expect( + createFieldValuesMap( + ['source.host', 'source.id'], + [ + { + key: { + 'source.host': 'host-1', + 'source.id': 100, + }, + doc_count: 1, + }, + ] + ) + ).toEqual({ + 'source.host': { + 'host-1': true, + }, + 'source.id': { + '100': true, + }, + }); + }); + + it('should not put value in map if it is a boolean', () => { + expect( + createFieldValuesMap( + ['source.host', 'user.enabled'], + [ + { + key: { + 'source.host': 'host-1', + 'user.enabled': true, + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + 'user.enabled': false, + }, + doc_count: 1, + }, + ] + ) + ).toEqual({ + 'source.host': { + 'host-1': true, + }, + 'user.enabled': { + true: true, + false: true, + }, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index cbca9353d31b8..24232b30500e2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -80,6 +80,12 @@ export const transformBucketsToValues = ( ); }; +/** + * transforms arrays of new terms fields and its values in object + * [new_terms_field]: { [value1]: true, [value1]: true } + * It's needed to have constant time complexity of accessing whether value is present in new terms + * It will be passed to Painless script used in runtime field + */ export const createFieldValuesMap = ( newTermsFields: string[], buckets: estypes.AggregationsCompositeBucket[] @@ -97,10 +103,11 @@ export const createFieldValuesMap = ( .map((bucket) => bucket.key) .forEach((bucket) => { Object.entries(bucket).forEach(([key, value]) => { - const strValue = typeof value !== 'string' ? value.toString() : value; - if (strValue != null) { - valuesMap[key][strValue] = true; + if (value == null) { + return; } + const strValue = typeof value !== 'string' ? value.toString() : value; + valuesMap[key][strValue] = true; }); }); From 1b0ce67d7031e69b583e2efdd28df9a0ae41d6d1 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 16 Nov 2022 12:16:05 +0000 Subject: [PATCH 05/14] null tests --- .../security_and_spaces/rule_execution_logic/new_terms.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 2fa681f2843f1..c007215e31332 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -735,7 +735,7 @@ export default ({ getService }: FtrProviderContext) => { fields: [AGG_FIELD_NAME, 'possibly_null_field', 'host.name'], runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'possibly_null_field'], { 'host.name': { 'host-0': true }, - tags: { ['null']: true }, + tags: {}, }), }); From 29c026bfdcfaaf7bee52f9b69e52e24b52c5c29d Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 16 Nov 2022 13:13:43 +0000 Subject: [PATCH 06/14] Revert "[CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix'" This reverts commit 53083017f40b675c6ad3a4b26100cbc87b54e357. --- .../page_steps/install_agent/install_agent_managed.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx index 68f78017c962c..aa3b0830aa200 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx @@ -19,9 +19,8 @@ import { } from '../../../../../../../../../components/agent_enrollment_flyout/steps'; import { ManualInstructions } from '../../../../../../../../../components/enrollment_instructions'; -import { KubernetesManifestApplyStep } from '../../../../../../../../../components/agent_enrollment_flyout/steps/run_k8s_apply_command_step'; - import type { InstallAgentPageProps } from './types'; +import { KubernetesManifestApplyStep } from '../../../../../../../../../components/agent_enrollment_flyout/steps/run_k8s_apply_command_step'; export const InstallElasticAgentManagedPageStep: React.FC = (props) => { const { From 02e29483fa408dfcb38cb8f019d29922c4609116 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 16 Nov 2022 13:20:09 +0000 Subject: [PATCH 07/14] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- .../page_steps/install_agent/install_agent_managed.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx index aa3b0830aa200..68f78017c962c 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_managed.tsx @@ -19,9 +19,10 @@ import { } from '../../../../../../../../../components/agent_enrollment_flyout/steps'; import { ManualInstructions } from '../../../../../../../../../components/enrollment_instructions'; -import type { InstallAgentPageProps } from './types'; import { KubernetesManifestApplyStep } from '../../../../../../../../../components/agent_enrollment_flyout/steps/run_k8s_apply_command_step'; +import type { InstallAgentPageProps } from './types'; + export const InstallElasticAgentManagedPageStep: React.FC = (props) => { const { cancelUrl, From dbb71e367448337313b0fc763e89852fb924f7fb Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 17 Nov 2022 09:35:44 +0000 Subject: [PATCH 08/14] Update x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts Co-authored-by: Marshall Main <55718608+marshallmain@users.noreply.github.com> --- .../lib/detection_engine/rule_types/new_terms/utils.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index a4f0650db8413..f315dabdbcafc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -288,7 +288,7 @@ describe('createFieldValuesMap', () => { }); }); - it('should not put value in map if it is a number', () => { + it('should put value in map if it is a number', () => { expect( createFieldValuesMap( ['source.host', 'source.id'], From 3b26fa6cb4c770d872d29577fe4c43d3c1c1d5f7 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 17 Nov 2022 09:35:50 +0000 Subject: [PATCH 09/14] Update x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts Co-authored-by: Marshall Main <55718608+marshallmain@users.noreply.github.com> --- .../lib/detection_engine/rule_types/new_terms/utils.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index f315dabdbcafc..c5035abffae2c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -312,7 +312,7 @@ describe('createFieldValuesMap', () => { }); }); - it('should not put value in map if it is a boolean', () => { + it('should put value in map if it is a boolean', () => { expect( createFieldValuesMap( ['source.host', 'user.enabled'], From da52b8dbd7287a2cb9c39ee28fb787c83f066373 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 17 Nov 2022 10:33:18 +0000 Subject: [PATCH 10/14] CR feedback --- .../new_terms/create_new_terms_alert_type.ts | 11 +- .../rule_types/new_terms/utils.test.ts | 37 +- .../rule_types/new_terms/utils.ts | 3 +- .../mocks/new_terms_large_array_buckets.json | 1402 +++++++++++++++++ .../rule_execution_logic/new_terms.ts | 226 +-- 5 files changed, 1582 insertions(+), 97 deletions(-) create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms_large_array_buckets.json diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index e7ae34fe6fda5..acfb9f725d3f9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -34,7 +34,6 @@ import { getNewTermsRuntimeMappings, getAggregationField, decodeMatchedValues, - createFieldValuesMap, } from './utils'; import { addToSearchAfterReturn, @@ -194,7 +193,11 @@ export const createNewTermsAlertType = ( } const bucketsForField = searchResultWithAggs.aggregations.new_terms.buckets; const includeValues = transformBucketsToValues(params.newTermsFields, bucketsForField); - const fieldsValuesMap = createFieldValuesMap(params.newTermsFields, bucketsForField); + const newTermsRuntimeMappings = getNewTermsRuntimeMappings( + params.newTermsFields, + bucketsForField + ); + // PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window. // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the // response correspond to each new term. @@ -211,7 +214,7 @@ export const createNewTermsAlertType = ( }), runtimeMappings: { ...runtimeMappings, - ...getNewTermsRuntimeMappings(params.newTermsFields, fieldsValuesMap), + ...newTermsRuntimeMappings, }, searchAfterSortIds: undefined, index: inputIndex, @@ -257,7 +260,7 @@ export const createNewTermsAlertType = ( }), runtimeMappings: { ...runtimeMappings, - ...getNewTermsRuntimeMappings(params.newTermsFields, fieldsValuesMap), + ...newTermsRuntimeMappings, }, searchAfterSortIds: undefined, index: inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index c5035abffae2c..9d9993d471e30 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -191,19 +191,48 @@ describe('new terms utils', () => { describe('getNewTermsRuntimeMappings', () => { it('should not return runtime field if new terms fields is empty', () => { - expect(getNewTermsRuntimeMappings([])).toBeUndefined(); + expect(getNewTermsRuntimeMappings([], [])).toBeUndefined(); }); it('should not return runtime field if new terms fields has only one field', () => { - expect(getNewTermsRuntimeMappings(['host.name'])).toBeUndefined(); + expect(getNewTermsRuntimeMappings(['host.name'], [])).toBeUndefined(); }); it('should return runtime field if new terms fields has more than one field', () => { - const runtimeMappings = getNewTermsRuntimeMappings(['host.name', 'host.ip']); + const runtimeMappings = getNewTermsRuntimeMappings( + ['source.host', 'source.ip'], + [ + { + key: { + 'source.host': 'host-0', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + ] + ); expect(runtimeMappings?.[AGG_FIELD_NAME]).toMatchObject({ type: 'keyword', script: { - params: { fields: ['host.name', 'host.ip'] }, + params: { + fields: ['source.host', 'source.ip'], + values: { + 'source.host': { + 'host-0': true, + 'host-1': true, + }, + 'source.ip': { + '127.0.0.1': true, + }, + }, + }, source: expect.any(String), }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index 24232b30500e2..de5822d29b1b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -116,13 +116,14 @@ export const createFieldValuesMap = ( export const getNewTermsRuntimeMappings = ( newTermsFields: string[], - values?: Record> + buckets: estypes.AggregationsCompositeBucket[] ): undefined | { [AGG_FIELD_NAME]: estypes.MappingRuntimeField } => { // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together if (newTermsFields.length <= 1) { return undefined; } + const values = createFieldValuesMap(newTermsFields, buckets); return { [AGG_FIELD_NAME]: { type: 'keyword', diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms_large_array_buckets.json b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms_large_array_buckets.json new file mode 100644 index 0000000000000..44b489face246 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms_large_array_buckets.json @@ -0,0 +1,1402 @@ +[ + { + "key":{ + "large_array_20":"value-of-20-0", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-1", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-2", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-3", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-4", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-5", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-6", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-7", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-8", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-9", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-10", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-11", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-12", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-13", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-14", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-15", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-16", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-17", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-18", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-19", + "large_array_10":"value-of-10-0" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-0", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-1", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-2", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-3", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-4", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-5", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-6", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-7", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-8", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-9", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-10", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-11", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-12", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-13", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-14", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-15", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-16", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-17", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-18", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-19", + "large_array_10":"value-of-10-1" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-0", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-1", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-2", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-3", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-4", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-5", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-6", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-7", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-8", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-9", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-10", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-11", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-12", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-13", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-14", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-15", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-16", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-17", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-18", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-19", + "large_array_10":"value-of-10-2" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-0", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-1", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-2", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-3", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-4", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-5", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-6", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-7", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-8", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-9", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-10", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-11", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-12", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-13", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-14", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-15", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-16", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-17", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-18", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-19", + "large_array_10":"value-of-10-3" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-0", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-1", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-2", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-3", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-4", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-5", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-6", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-7", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-8", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-9", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-10", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-11", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-12", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-13", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-14", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-15", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-16", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-17", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-18", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-19", + "large_array_10":"value-of-10-4" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-0", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-1", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-2", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-3", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-4", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-5", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-6", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-7", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-8", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-9", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-10", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-11", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-12", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-13", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-14", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-15", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-16", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-17", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-18", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-19", + "large_array_10":"value-of-10-5" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-0", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-1", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-2", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-3", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-4", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-5", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-6", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-7", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-8", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-9", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-10", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-11", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-12", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-13", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-14", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-15", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-16", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-17", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-18", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-19", + "large_array_10":"value-of-10-6" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-0", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-1", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-2", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-3", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-4", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-5", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-6", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-7", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-8", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-9", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-10", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-11", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-12", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-13", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-14", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-15", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-16", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-17", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-18", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-19", + "large_array_10":"value-of-10-7" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-0", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-1", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-2", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-3", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-4", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-5", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-6", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-7", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-8", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-9", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-10", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-11", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-12", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-13", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-14", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-15", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-16", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-17", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-18", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-19", + "large_array_10":"value-of-10-8" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-0", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-1", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-2", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-3", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-4", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-5", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-6", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-7", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-8", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-9", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-10", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-11", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-12", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-13", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-14", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-15", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-16", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-17", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-18", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + }, + { + "key":{ + "large_array_20":"value-of-20-19", + "large_array_10":"value-of-10-9" + }, + "doc_count":1 + } + ] \ No newline at end of file diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index c007215e31332..51b0246be0e06 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -28,6 +28,8 @@ import { FtrProviderContext } from '../../common/ftr_provider_context'; import { previewRuleWithExceptionEntries } from '../../utils/preview_rule_with_exception_entries'; import { deleteAllExceptions } from '../../../lists_api_integration/utils'; +import newTermsLargeArrayBuckets from './mocks/new_terms_large_array_buckets.json'; + const removeRandomValuedProperties = (alert: DetectionAlert | undefined) => { if (!alert) { return undefined; @@ -608,25 +610,40 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip'], { - 'host.name': { 'host-0': true }, - 'host.ip': { '127.0.0.1': true }, - }), + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'host.ip'], + [ + { + key: { + 'host.name': 'host-0', + 'host.ip': '127.0.0.1', + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); }); - it('should not return runtime field created from 2 single values if its value is not in value map', async () => { + it('should not return runtime field created from 2 single values if its value is not in buckets', async () => { const { hits } = await performSearchQuery({ es, query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip'], { - 'host.name': { 'host-0': true }, - 'host.ip': {}, - }), + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'host.ip'], + [ + { + key: { + 'host.name': 'host-0', + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.be(undefined); @@ -640,10 +657,18 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.id'], { - 'user.name': { 'user-0': true }, - 'user.id': { '0': true }, - }), + runtimeMappings: getNewTermsRuntimeMappings( + ['user.name', 'user.id'], + [ + { + key: { + 'user.name': 'user-0', + 'user.id': 0, + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -657,10 +682,18 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.enabled'], { - 'user.name': { 'user-0': true }, - 'user.enabled': { ['true']: true }, - }), + runtimeMappings: getNewTermsRuntimeMappings( + ['user.name', 'user.enabled'], + [ + { + key: { + 'user.name': 'user-0', + 'user.enabled': true, + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -674,11 +707,19 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip', 'user.name'], { - 'host.name': { 'host-0': true }, - 'host.ip': { '127.0.0.1': true }, - 'user.name': { 'user-0': true }, - }), + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'host.ip', 'user.name'], + [ + { + key: { + 'host.name': 'host-0', + 'host.ip': '127.0.0.1', + 'user.name': 'user-0', + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -700,28 +741,54 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'doc_with_source_ip_as_array' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['source.ip', 'tags'], { - 'source.ip': { '192.168.1.1': true, '192.168.1.2': true }, - tags: { 'tag-new-1': true, 'tag-2': true, 'tag-new-3': true }, - }), - }); + runtimeMappings: getNewTermsRuntimeMappings( + ['source.ip', 'tags'], - expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); - }); - - it('should return runtime field without duplicated values', async () => { - // encoded base64 values of "host-0" and ["tag-1", "tag-2", "tag-2", "tag-1", "tag-1"] - // joined with underscore, without duplicates in tags - const expectedEncodedValues = ['aG9zdC0w_dGFnLTE=', 'aG9zdC0w_dGFnLTI=']; - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'doc_with_duplicated_tags' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'tags'], { - 'host.name': { 'host-0': true }, - tags: { 'tag-1': true, 'tag-2': true }, - }), + [ + { + key: { + tags: 'tag-new-1', + 'source.ip': '192.168.1.1', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-2', + 'source.ip': '192.168.1.1', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-new-3', + 'source.ip': '192.168.1.1', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-new-1', + 'source.ip': '192.168.1.2', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-2', + 'source.ip': '192.168.1.2', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-new-3', + 'source.ip': '192.168.1.2', + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -733,10 +800,18 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'doc_with_null_field' } }, index: 'new_terms', fields: [AGG_FIELD_NAME, 'possibly_null_field', 'host.name'], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'possibly_null_field'], { - 'host.name': { 'host-0': true }, - tags: {}, - }), + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'possibly_null_field'], + [ + { + key: { + 'host.name': 'host-0', + possibly_null_field: null, + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits.length).to.be(1); @@ -745,16 +820,23 @@ export default ({ getService }: FtrProviderContext) => { expect(hits.hits[0].fields?.['host.name']).to.eql(['host-0']); }); - it('should not return runtime field if one of fields is not defined', async () => { + it('should not return runtime field if one of fields is not defined in a document', async () => { const { hits } = await performSearchQuery({ es, query: { match: { id: 'doc_without_large_arrays' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'large_array_5'], { - 'host.name': { 'host-0': true }, - large_array_5: {}, - }), + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'large_array_5'], + [ + { + key: { + 'host.name': 'host-0', + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits.length).to.be(1); @@ -769,42 +851,10 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['large_array_20', 'large_array_10'], { - large_array_20: { - 'value-of-20-0': true, - 'value-of-20-1': true, - 'value-of-20-2': true, - 'value-of-20-3': true, - 'value-of-20-4': true, - 'value-of-20-5': true, - 'value-of-20-6': true, - 'value-of-20-7': true, - 'value-of-20-8': true, - 'value-of-20-9': true, - 'value-of-20-10': true, - 'value-of-20-11': true, - 'value-of-20-12': true, - 'value-of-20-13': true, - 'value-of-20-14': true, - 'value-of-20-15': true, - 'value-of-20-16': true, - 'value-of-20-17': true, - 'value-of-20-18': true, - 'value-of-20-19': true, - }, - large_array_10: { - 'value-of-10-0': true, - 'value-of-10-1': true, - 'value-of-10-2': true, - 'value-of-10-3': true, - 'value-of-10-4': true, - 'value-of-10-5': true, - 'value-of-10-6': true, - 'value-of-10-7': true, - 'value-of-10-8': true, - 'value-of-10-9': true, - }, - }), + runtimeMappings: getNewTermsRuntimeMappings( + ['large_array_20', 'large_array_10'], + newTermsLargeArrayBuckets + ), }); // runtime field should have 100 values, as large_array_20 and large_array_10 From 705984d25b2a0b272c447307e812283b61892004 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 17 Nov 2022 10:41:18 +0000 Subject: [PATCH 11/14] CR feedback --- .../server/lib/detection_engine/rule_types/new_terms/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index 5a473a389fe9d..1e7f5603a5765 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -27,3 +27,5 @@ The new terms rule type reuses the singleSearchAfter function which implements t ## Limitations and future enhancements - Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. +- Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields. +As there is limit in 100 alerts per rule execution, it should not be an issue at this moment. From 26cb0b8d28088b018a650522012afc2469a3e7c6 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 17 Nov 2022 11:30:46 +0000 Subject: [PATCH 12/14] tests --- .../rule_execution_logic/mocks/new_terms.ts | 1409 +++++++++++++++++ .../mocks/new_terms_large_array_buckets.json | 1402 ---------------- .../rule_execution_logic/new_terms.ts | 38 +- 3 files changed, 1444 insertions(+), 1405 deletions(-) create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms.ts delete mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms_large_array_buckets.json diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms.ts new file mode 100644 index 0000000000000..15c63546a4be7 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms.ts @@ -0,0 +1,1409 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const largeArraysBuckets = [ + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, +]; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms_large_array_buckets.json b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms_large_array_buckets.json deleted file mode 100644 index 44b489face246..0000000000000 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms_large_array_buckets.json +++ /dev/null @@ -1,1402 +0,0 @@ -[ - { - "key":{ - "large_array_20":"value-of-20-0", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-1", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-2", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-3", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-4", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-5", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-6", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-7", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-8", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-9", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-10", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-11", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-12", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-13", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-14", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-15", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-16", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-17", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-18", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-19", - "large_array_10":"value-of-10-0" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-0", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-1", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-2", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-3", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-4", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-5", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-6", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-7", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-8", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-9", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-10", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-11", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-12", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-13", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-14", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-15", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-16", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-17", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-18", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-19", - "large_array_10":"value-of-10-1" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-0", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-1", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-2", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-3", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-4", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-5", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-6", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-7", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-8", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-9", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-10", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-11", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-12", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-13", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-14", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-15", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-16", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-17", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-18", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-19", - "large_array_10":"value-of-10-2" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-0", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-1", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-2", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-3", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-4", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-5", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-6", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-7", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-8", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-9", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-10", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-11", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-12", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-13", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-14", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-15", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-16", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-17", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-18", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-19", - "large_array_10":"value-of-10-3" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-0", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-1", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-2", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-3", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-4", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-5", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-6", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-7", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-8", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-9", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-10", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-11", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-12", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-13", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-14", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-15", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-16", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-17", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-18", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-19", - "large_array_10":"value-of-10-4" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-0", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-1", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-2", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-3", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-4", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-5", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-6", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-7", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-8", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-9", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-10", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-11", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-12", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-13", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-14", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-15", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-16", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-17", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-18", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-19", - "large_array_10":"value-of-10-5" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-0", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-1", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-2", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-3", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-4", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-5", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-6", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-7", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-8", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-9", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-10", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-11", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-12", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-13", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-14", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-15", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-16", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-17", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-18", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-19", - "large_array_10":"value-of-10-6" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-0", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-1", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-2", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-3", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-4", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-5", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-6", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-7", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-8", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-9", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-10", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-11", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-12", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-13", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-14", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-15", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-16", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-17", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-18", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-19", - "large_array_10":"value-of-10-7" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-0", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-1", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-2", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-3", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-4", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-5", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-6", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-7", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-8", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-9", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-10", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-11", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-12", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-13", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-14", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-15", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-16", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-17", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-18", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-19", - "large_array_10":"value-of-10-8" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-0", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-1", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-2", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-3", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-4", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-5", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-6", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-7", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-8", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-9", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-10", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-11", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-12", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-13", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-14", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-15", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-16", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-17", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-18", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - }, - { - "key":{ - "large_array_20":"value-of-20-19", - "large_array_10":"value-of-10-9" - }, - "doc_count":1 - } - ] \ No newline at end of file diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 51b0246be0e06..fe21bae00a30b 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -28,7 +28,7 @@ import { FtrProviderContext } from '../../common/ftr_provider_context'; import { previewRuleWithExceptionEntries } from '../../utils/preview_rule_with_exception_entries'; import { deleteAllExceptions } from '../../../lists_api_integration/utils'; -import newTermsLargeArrayBuckets from './mocks/new_terms_large_array_buckets.json'; +import { largeArraysBuckets } from './mocks/new_terms'; const removeRandomValuedProperties = (alert: DetectionAlert | undefined) => { if (!alert) { @@ -743,7 +743,6 @@ export default ({ getService }: FtrProviderContext) => { fields: [AGG_FIELD_NAME], runtimeMappings: getNewTermsRuntimeMappings( ['source.ip', 'tags'], - [ { key: { @@ -794,6 +793,39 @@ export default ({ getService }: FtrProviderContext) => { expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); }); + it('should return runtime field without duplicated values', async () => { + // encoded base64 values of "host-0" and ["tag-1", "tag-2", "tag-2", "tag-1", "tag-1"] + // joined with underscore, without duplicates in tags + const expectedEncodedValues = ['aG9zdC0w_dGFnLTE=', 'aG9zdC0w_dGFnLTI=']; + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'doc_with_duplicated_tags' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'tags'], + [ + { + key: { + tags: 'tag-1', + 'host.name': 'host-0', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-2', + 'host.name': 'host-0', + }, + doc_count: 1, + }, + ] + ), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); + }); + it('should not return runtime field if one of fields is null', async () => { const { hits } = await performSearchQuery({ es, @@ -853,7 +885,7 @@ export default ({ getService }: FtrProviderContext) => { fields: [AGG_FIELD_NAME], runtimeMappings: getNewTermsRuntimeMappings( ['large_array_20', 'large_array_10'], - newTermsLargeArrayBuckets + largeArraysBuckets ), }); From 8f105280232c8c7a554878be37428457cbc41a8b Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 17 Nov 2022 19:35:44 +0000 Subject: [PATCH 13/14] Update README.md --- .../lib/detection_engine/rule_types/new_terms/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index 1e7f5603a5765..61bbfc8738627 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -28,4 +28,6 @@ The new terms rule type reuses the singleSearchAfter function which implements t - Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. - Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields. -As there is limit in 100 alerts per rule execution, it should not be an issue at this moment. +Following edge cases possible: + - false negatives (alert is not generated) if too many fields were emitted and actual new values are not getting evaluated if it happened in document in rule run window. + - false positives (wrong alert generated) if too many fields were emitted in historical document and some old terms are not getting evaluated against values in new documents. From a153d7791167118adc5ebb9cc681983cdb6bd93c Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 17 Nov 2022 19:37:55 +0000 Subject: [PATCH 14/14] Update README.md --- .../lib/detection_engine/rule_types/new_terms/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index 61bbfc8738627..70c00d6203c8b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -28,6 +28,6 @@ The new terms rule type reuses the singleSearchAfter function which implements t - Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. - Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields. -Following edge cases possible: - - false negatives (alert is not generated) if too many fields were emitted and actual new values are not getting evaluated if it happened in document in rule run window. - - false positives (wrong alert generated) if too many fields were emitted in historical document and some old terms are not getting evaluated against values in new documents. + Following edge cases possible: + - false negatives (alert is not generated) if too many fields were emitted and actual new values are not getting evaluated if it happened in document in rule run window. + - false positives (wrong alert generated) if too many fields were emitted in historical document and some old terms are not getting evaluated against values in new documents.