From 6b66276ce88f92dc4c24b21a027197f163d1e428 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 18 Nov 2022 09:02:15 +0000 Subject: [PATCH] [Security Solution][Alerts] improves performance of new terms multi fields (#145167) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR improves performance of new terms multiple fields implementation released in https://github.com/elastic/kibana/pull/143943 In comparison with single value fields it's faster in 2-2.5 times In comparison with array field the win even more significant: 3-4 times ### Technical implementation: Value for runtime field is emitted only if any of new terms fields present in `include` clause of terms aggregation. It's achieved by passing a values map of new terms fields to the runtime script and checking new terms values in it. So, there is a significant performance wins if runtime field is not matched with includes values: - new terms fields are not encoded in base64 within runtime script - runtime script doesn't emit any values - runtime field doesn't have any value to be compared against during aggregation, as its empty - it eliminates possible unyielding execution branches early: if one of items in array of first new terms field is not present in values map, no need to run through the rest of combinations As a result, this implementation overcomes the issue with non exhaustive results due to the large number of emitted fields. ES doesn't allow emitting more than 100 values in the runtime script, so if the number of all combinations in new terms fields is greater than 100, only the first 100 combinations will be used for terms aggregation. With this new implementation only matched fields will be emitting. Even if its number is greater than 100, we will hit circuit breaker in Security Solution: as rule run can't generate more than 100 alerts ### Performance measurements Implementation | Shards | Docs per shard | Simultaneous Rule Executions | Fields cardinality | Rule Execution Time (improved) | Rule Execution Time (old) -- | -- | -- | -- | -- | -- | -- Terms 1 field | 10 | 900,000 | 1 | 100,000 | 7s |   Terms 2 fields | 10 | 900,000 | 1 | 100,000 | 17s | 33s Terms 2 fields | 10 | 900,000 | 2 | 100,000 | 19s |   Terms 3 fields | 10 | 900,000 | 1 | 100,000 | 18s | 46s Terms 3 fields | 10 | 900,000 | 2 | 100,000 | 20s |     |   |   |   |   |   |   Terms 1 field | 20 | 900,000 | 1 | 100,000 | 10.5s |   Terms 2 fields | 20 | 900,000 | 1 | 100,000 | 28s | 55s Terms 2 fields | 20 | 900,000 | 2 | 100,000 | 28.5s | 56s Terms 3 fields | 20 | 900,000 | 1 | 100,000 | 30s | 75s Terms 3 fields | 20 | 900,000 | 2 | 100,000 | 31s | 75s   |   |   |   |   |   |   Terms 1 field | 10 | 1,800,000 | 1 | 100,000 | 7s |   Terms 2 fields | 10 | 1,800,000 | 1 | 100,000 | 24s | 50s Terms 3 fields | 10 | 1,800,000 | 1 | 100,000 | 26s | 68s   |   |   |   |   |   |   array of unique values length 10 |   |   |   |   |   |   Terms 1 field | 10 | 900,000 | 1 | 100,000 | 9.5s |   Terms 2 fields | 10 | 900,000 | 1 | 100,000 | 75s | 3.5m Terms 3 fields | 10 | 900,000 | 1 | 100,000 | 83s | 6m ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Marshall Main <55718608+marshallmain@users.noreply.github.com> (cherry picked from commit b985ec1735d432bf4cf13fb4a4610dabb5c987c6) --- .../rule_types/new_terms/README.md | 5 +- .../new_terms/create_new_terms_alert_type.ts | 9 +- .../rule_types/new_terms/utils.test.ts | 172 +- .../rule_types/new_terms/utils.ts | 49 +- .../rule_execution_logic/mocks/new_terms.ts | 1409 +++++++++++++++++ .../rule_execution_logic/new_terms.ts | 177 ++- 6 files changed, 1800 insertions(+), 21 deletions(-) create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index 694fdd53fe2f4..70c00d6203c8b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -27,4 +27,7 @@ The new terms rule type reuses the singleSearchAfter function which implements t ## Limitations and future enhancements - Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. -- Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields +- Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields. + Following edge cases possible: + - false negatives (alert is not generated) if too many fields were emitted and actual new values are not getting evaluated if it happened in document in rule run window. + - false positives (wrong alert generated) if too many fields were emitted in historical document and some old terms are not getting evaluated against values in new documents. diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 8ef5b3acf4b2c..acfb9f725d3f9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -193,6 +193,11 @@ export const createNewTermsAlertType = ( } const bucketsForField = searchResultWithAggs.aggregations.new_terms.buckets; const includeValues = transformBucketsToValues(params.newTermsFields, bucketsForField); + const newTermsRuntimeMappings = getNewTermsRuntimeMappings( + params.newTermsFields, + bucketsForField + ); + // PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window. // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the // response correspond to each new term. @@ -209,7 +214,7 @@ export const createNewTermsAlertType = ( }), runtimeMappings: { ...runtimeMappings, - ...getNewTermsRuntimeMappings(params.newTermsFields), + ...newTermsRuntimeMappings, }, searchAfterSortIds: undefined, index: inputIndex, @@ -255,7 +260,7 @@ export const createNewTermsAlertType = ( }), runtimeMappings: { ...runtimeMappings, - ...getNewTermsRuntimeMappings(params.newTermsFields), + ...newTermsRuntimeMappings, }, searchAfterSortIds: undefined, index: inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index 2b04b617ba9ba..9d9993d471e30 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -12,6 +12,7 @@ import { getAggregationField, decodeMatchedValues, getNewTermsRuntimeMappings, + createFieldValuesMap, AGG_FIELD_NAME, } from './utils'; @@ -190,22 +191,185 @@ describe('new terms utils', () => { describe('getNewTermsRuntimeMappings', () => { it('should not return runtime field if new terms fields is empty', () => { - expect(getNewTermsRuntimeMappings([])).toBeUndefined(); + expect(getNewTermsRuntimeMappings([], [])).toBeUndefined(); }); it('should not return runtime field if new terms fields has only one field', () => { - expect(getNewTermsRuntimeMappings(['host.name'])).toBeUndefined(); + expect(getNewTermsRuntimeMappings(['host.name'], [])).toBeUndefined(); }); it('should return runtime field if new terms fields has more than one field', () => { - const runtimeMappings = getNewTermsRuntimeMappings(['host.name', 'host.ip']); + const runtimeMappings = getNewTermsRuntimeMappings( + ['source.host', 'source.ip'], + [ + { + key: { + 'source.host': 'host-0', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + ] + ); expect(runtimeMappings?.[AGG_FIELD_NAME]).toMatchObject({ type: 'keyword', script: { - params: { fields: ['host.name', 'host.ip'] }, + params: { + fields: ['source.host', 'source.ip'], + values: { + 'source.host': { + 'host-0': true, + 'host-1': true, + }, + 'source.ip': { + '127.0.0.1': true, + }, + }, + }, source: expect.any(String), }, }); }); }); }); + +describe('createFieldValuesMap', () => { + it('should return undefined if new terms fields has only one field', () => { + expect( + createFieldValuesMap( + ['host.name'], + [ + { + key: { + 'source.host': 'host-0', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + }, + doc_count: 3, + }, + ] + ) + ).toBeUndefined(); + }); + + it('should return values map if new terms fields has more than one field', () => { + expect( + createFieldValuesMap( + ['source.host', 'source.ip'], + [ + { + key: { + 'source.host': 'host-0', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + 'source.ip': '127.0.0.1', + }, + doc_count: 1, + }, + ] + ) + ).toEqual({ + 'source.host': { + 'host-0': true, + 'host-1': true, + }, + 'source.ip': { + '127.0.0.1': true, + }, + }); + }); + + it('should not put value in map if it is null', () => { + expect( + createFieldValuesMap( + ['source.host', 'source.ip'], + [ + { + key: { + 'source.host': 'host-1', + 'source.ip': null, + }, + doc_count: 1, + }, + ] + ) + ).toEqual({ + 'source.host': { + 'host-1': true, + }, + 'source.ip': {}, + }); + }); + + it('should put value in map if it is a number', () => { + expect( + createFieldValuesMap( + ['source.host', 'source.id'], + [ + { + key: { + 'source.host': 'host-1', + 'source.id': 100, + }, + doc_count: 1, + }, + ] + ) + ).toEqual({ + 'source.host': { + 'host-1': true, + }, + 'source.id': { + '100': true, + }, + }); + }); + + it('should put value in map if it is a boolean', () => { + expect( + createFieldValuesMap( + ['source.host', 'user.enabled'], + [ + { + key: { + 'source.host': 'host-1', + 'user.enabled': true, + }, + doc_count: 1, + }, + { + key: { + 'source.host': 'host-1', + 'user.enabled': false, + }, + doc_count: 1, + }, + ] + ) + ).toEqual({ + 'source.host': { + 'host-1': true, + }, + 'user.enabled': { + true: true, + false: true, + }, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index cebd63f17e663..de5822d29b1b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -80,19 +80,55 @@ export const transformBucketsToValues = ( ); }; +/** + * transforms arrays of new terms fields and its values in object + * [new_terms_field]: { [value1]: true, [value1]: true } + * It's needed to have constant time complexity of accessing whether value is present in new terms + * It will be passed to Painless script used in runtime field + */ +export const createFieldValuesMap = ( + newTermsFields: string[], + buckets: estypes.AggregationsCompositeBucket[] +) => { + if (newTermsFields.length === 1) { + return undefined; + } + + const valuesMap = newTermsFields.reduce>>( + (acc, field) => ({ ...acc, [field]: {} }), + {} + ); + + buckets + .map((bucket) => bucket.key) + .forEach((bucket) => { + Object.entries(bucket).forEach(([key, value]) => { + if (value == null) { + return; + } + const strValue = typeof value !== 'string' ? value.toString() : value; + valuesMap[key][strValue] = true; + }); + }); + + return valuesMap; +}; + export const getNewTermsRuntimeMappings = ( - newTermsFields: string[] + newTermsFields: string[], + buckets: estypes.AggregationsCompositeBucket[] ): undefined | { [AGG_FIELD_NAME]: estypes.MappingRuntimeField } => { // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together if (newTermsFields.length <= 1) { return undefined; } + const values = createFieldValuesMap(newTermsFields, buckets); return { [AGG_FIELD_NAME]: { type: 'keyword', script: { - params: { fields: newTermsFields }, + params: { fields: newTermsFields, values }, source: ` def stack = new Stack(); // ES has limit in 100 values for runtime field, after this query will fail @@ -110,9 +146,14 @@ export const getNewTermsRuntimeMappings = ( emit(line); emitLimit = emitLimit - 1; } else { - for (field in doc[params['fields'][index]]) { + def fieldName = params['fields'][index]; + for (field in doc[fieldName]) { + def fieldStr = String.valueOf(field); + if (!params['values'][fieldName].containsKey(fieldStr)) { + continue; + } def delimiter = index === 0 ? '' : '${DELIMITER}'; - def nextLine = line + delimiter + String.valueOf(field).encodeBase64(); + def nextLine = line + delimiter + fieldStr.encodeBase64(); stack.add([index + 1, nextLine]) } diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms.ts new file mode 100644 index 0000000000000..15c63546a4be7 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/mocks/new_terms.ts @@ -0,0 +1,1409 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const largeArraysBuckets = [ + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-0', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-1', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-2', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-3', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-4', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-5', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-6', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-7', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-8', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-0', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-1', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-2', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-3', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-4', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-5', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-6', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-7', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-8', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-9', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-10', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-11', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-12', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-13', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-14', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-15', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-16', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-17', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-18', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, + { + key: { + large_array_20: 'value-of-20-19', + large_array_10: 'value-of-10-9', + }, + doc_count: 1, + }, +]; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 3b1304f12e6c3..fe21bae00a30b 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -28,6 +28,8 @@ import { FtrProviderContext } from '../../common/ftr_provider_context'; import { previewRuleWithExceptionEntries } from '../../utils/preview_rule_with_exception_entries'; import { deleteAllExceptions } from '../../../lists_api_integration/utils'; +import { largeArraysBuckets } from './mocks/new_terms'; + const removeRandomValuedProperties = (alert: DetectionAlert | undefined) => { if (!alert) { return undefined; @@ -608,12 +610,45 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip']), + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'host.ip'], + [ + { + key: { + 'host.name': 'host-0', + 'host.ip': '127.0.0.1', + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); }); + it('should not return runtime field created from 2 single values if its value is not in buckets', async () => { + const { hits } = await performSearchQuery({ + es, + query: { match: { id: 'first_doc' } }, + index: 'new_terms', + fields: [AGG_FIELD_NAME], + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'host.ip'], + [ + { + key: { + 'host.name': 'host-0', + }, + doc_count: 1, + }, + ] + ), + }); + + expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.be(undefined); + }); + it('should return runtime field created from 2 single values, including number value', async () => { // encoded base64 values of "user-0" and 0 joined with underscore const expectedEncodedValues = ['dXNlci0w_MA==']; @@ -622,7 +657,18 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.id']), + runtimeMappings: getNewTermsRuntimeMappings( + ['user.name', 'user.id'], + [ + { + key: { + 'user.name': 'user-0', + 'user.id': 0, + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -636,7 +682,18 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.enabled']), + runtimeMappings: getNewTermsRuntimeMappings( + ['user.name', 'user.enabled'], + [ + { + key: { + 'user.name': 'user-0', + 'user.enabled': true, + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -650,7 +707,19 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip', 'user.name']), + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'host.ip', 'user.name'], + [ + { + key: { + 'host.name': 'host-0', + 'host.ip': '127.0.0.1', + 'user.name': 'user-0', + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -672,7 +741,53 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'doc_with_source_ip_as_array' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['source.ip', 'tags']), + runtimeMappings: getNewTermsRuntimeMappings( + ['source.ip', 'tags'], + [ + { + key: { + tags: 'tag-new-1', + 'source.ip': '192.168.1.1', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-2', + 'source.ip': '192.168.1.1', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-new-3', + 'source.ip': '192.168.1.1', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-new-1', + 'source.ip': '192.168.1.2', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-2', + 'source.ip': '192.168.1.2', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-new-3', + 'source.ip': '192.168.1.2', + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -687,7 +802,25 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'doc_with_duplicated_tags' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'tags']), + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'tags'], + [ + { + key: { + tags: 'tag-1', + 'host.name': 'host-0', + }, + doc_count: 1, + }, + { + key: { + tags: 'tag-2', + 'host.name': 'host-0', + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); @@ -699,7 +832,18 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'doc_with_null_field' } }, index: 'new_terms', fields: [AGG_FIELD_NAME, 'possibly_null_field', 'host.name'], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'possibly_null_field']), + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'possibly_null_field'], + [ + { + key: { + 'host.name': 'host-0', + possibly_null_field: null, + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits.length).to.be(1); @@ -708,13 +852,23 @@ export default ({ getService }: FtrProviderContext) => { expect(hits.hits[0].fields?.['host.name']).to.eql(['host-0']); }); - it('should not return runtime field if one of fields is not defined', async () => { + it('should not return runtime field if one of fields is not defined in a document', async () => { const { hits } = await performSearchQuery({ es, query: { match: { id: 'doc_without_large_arrays' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'large_array_5']), + runtimeMappings: getNewTermsRuntimeMappings( + ['host.name', 'large_array_5'], + [ + { + key: { + 'host.name': 'host-0', + }, + doc_count: 1, + }, + ] + ), }); expect(hits.hits.length).to.be(1); @@ -729,7 +883,10 @@ export default ({ getService }: FtrProviderContext) => { query: { match: { id: 'first_doc' } }, index: 'new_terms', fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings(['large_array_20', 'large_array_10']), + runtimeMappings: getNewTermsRuntimeMappings( + ['large_array_20', 'large_array_10'], + largeArraysBuckets + ), }); // runtime field should have 100 values, as large_array_20 and large_array_10