[Security Solution][Alerts] adds support for multi fields in new terms rule

x-pack/plugins/security_solution/common/constants.ts

@@ -469,4 +469,6 @@ export const RISKY_HOSTS_DOC_LINK =
 export const RISKY_USERS_DOC_LINK =
   'https://www.elastic.co/guide/en/security/current/user-risk-score.html';
 
+export const MAX_NUMBER_OF_NEW_TERMS_FIELDS = 3;
+
 export const BULK_ADD_TO_TIMELINE_LIMIT = 2000;

x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts

@@ -7,14 +7,19 @@
 
 import * as t from 'io-ts';
 import { LimitedSizeArray, NonEmptyString } from '@kbn/securitysolution-io-ts-types';
+import { MAX_NUMBER_OF_NEW_TERMS_FIELDS } from '../../../../constants';
 
 // Attributes specific to New Terms rules
 
 /**
- * New terms rule type currently only supports a single term, but should support more in the future
+ * New terms rule type supports a limited number of fields. Max number of fields is 3 and defined in common constants as MAX_NUMBER_OF_NEW_TERMS_FIELDS
  */
 export type NewTermsFields = t.TypeOf<typeof NewTermsFields>;
-export const NewTermsFields = LimitedSizeArray({ codec: t.string, minSize: 1, maxSize: 1 });
+export const NewTermsFields = LimitedSizeArray({
+  codec: t.string,
+  minSize: 1,
+  maxSize: MAX_NUMBER_OF_NEW_TERMS_FIELDS,
+});
 
 export type HistoryWindowStart = t.TypeOf<typeof HistoryWindowStart>;
 export const HistoryWindowStart = NonEmptyString;

x-pack/plugins/security_solution/common/field_maps/field_names.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE } from '@kbn/rule-data-utils';
+import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE, ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils';
 
 export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors` as const;
 export const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as const;
@@ -16,6 +16,7 @@ export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const;
 export const ALERT_THRESHOLD_RESULT = `${ALERT_NAMESPACE}.threshold_result` as const;
 export const ALERT_THRESHOLD_RESULT_COUNT = `${ALERT_THRESHOLD_RESULT}.count` as const;
 export const ALERT_NEW_TERMS = `${ALERT_NAMESPACE}.new_terms` as const;
+export const ALERT_NEW_TERMS_FIELDS = `${ALERT_RULE_PARAMETERS}.new_terms_fields` as const;
 
 export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const;
 export const ALERT_ORIGINAL_EVENT_ACTION = `${ALERT_ORIGINAL_EVENT}.action` as const;

x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx

@@ -703,18 +703,25 @@ describe('AlertSummaryView', () => {
         values: ['127.0.0.1'],
         originalValue: ['127.0.0.1'],
       },
+      {
+        category: 'kibana',
+        field: 'kibana.alert.rule.parameters.new_terms_fields',
+        values: ['host.ip'],
+        originalValue: ['host.ip'],
+      },
     ] as TimelineEventsDetailsItem[];
     const renderProps = {
       ...props,
       data: enhancedData,
     };
+
     const { getByText } = render(
       <TestProvidersComponent>
         <AlertSummaryView {...renderProps} />
       </TestProvidersComponent>
     );
 
-    ['New Terms'].forEach((fieldId) => {
+    ['New Terms', '127.0.0.1', 'New Terms fields', 'host.ip'].forEach((fieldId) => {
       expect(getByText(fieldId));
     });
   });

x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx

@@ -16,8 +16,13 @@ import {
   ALERTS_HEADERS_THRESHOLD_TERMS,
   ALERTS_HEADERS_RULE_DESCRIPTION,
   ALERTS_HEADERS_NEW_TERMS,
+  ALERTS_HEADERS_NEW_TERMS_FIELDS,
 } from '../../../detections/components/alerts_table/translations';
-import { ALERT_NEW_TERMS, ALERT_THRESHOLD_RESULT } from '../../../../common/field_maps/field_names';
+import {
+  ALERT_NEW_TERMS_FIELDS,
+  ALERT_NEW_TERMS,
+  ALERT_THRESHOLD_RESULT,
+} from '../../../../common/field_maps/field_names';
 import { AGENT_STATUS_FIELD_NAME } from '../../../timelines/components/timeline/body/renderers/constants';
 import type { AlertSummaryRow } from './helpers';
 import { getEnrichedFieldInfo } from './helpers';
@@ -172,6 +177,10 @@ function getFieldsByRuleType(ruleType?: string): EventSummaryField[] {
       ];
     case 'new_terms':
       return [
+        {
+          id: ALERT_NEW_TERMS_FIELDS,
+          label: ALERTS_HEADERS_NEW_TERMS_FIELDS,
+        },
         {
           id: ALERT_NEW_TERMS,
           label: ALERTS_HEADERS_NEW_TERMS,

x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts

@@ -123,6 +123,13 @@ export const ALERTS_HEADERS_NEW_TERMS = i18n.translate(
   }
 );
 
+export const ALERTS_HEADERS_NEW_TERMS_FIELDS = i18n.translate(
+  'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTermsFields',
+  {
+    defaultMessage: 'New Terms fields',
+  }
+);
+
 export const ACTION_INVESTIGATE_IN_TIMELINE = i18n.translate(
   'xpack.securitySolution.detectionEngine.alerts.actions.investigateInTimelineTitle',
   {

x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx

@@ -59,6 +59,7 @@ import {
   useFormData,
 } from '../../../../shared_imports';
 import { schema } from './schema';
+import { getTermsAggregationFields } from './utils';
 import * as i18n from './translations';
 import {
   isEqlRule,
@@ -297,6 +298,11 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
     setAggregatableFields(aggregatableFields(fields as BrowserField[]));
   }, [indexPattern]);
 
+  const termsAggregationFields: BrowserField[] = useMemo(
+    () => getTermsAggregationFields(aggFields),
+    [aggFields]
+  );
+
   const [
     threatIndexPatternsLoading,
     { browserFields: threatBrowserFields, indexPatterns: threatIndexPatterns },
@@ -836,7 +842,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
                 path="newTermsFields"
                 component={NewTermsFields}
                 componentProps={{
-                  browserFields: aggFields,
+                  browserFields: termsAggregationFields,
                 }}
               />
               <UseField

x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx

@@ -22,6 +22,7 @@ import {
   isThreatMatchRule,
   isThresholdRule,
 } from '../../../../../common/detection_engine/utils';
+import { MAX_NUMBER_OF_NEW_TERMS_FIELDS } from '../../../../../common/constants';
 import { isMlRule } from '../../../../../common/machine_learning/helpers';
 import type { FieldValueQueryBar } from '../query_bar';
 import type { ERROR_CODE, FormSchema, ValidationFunc } from '../../../../shared_imports';
@@ -585,7 +586,7 @@ export const schema: FormSchema<DefineStepRule> = {
             i18n.translate(
               'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.newTermsFieldsMin',
               {
-                defaultMessage: 'Number of fields must be 1.',
+                defaultMessage: 'A minimum of one field is required.',
               }
             )
           )(...args);
@@ -601,11 +602,11 @@ export const schema: FormSchema<DefineStepRule> = {
             return;
           }
           return fieldValidators.maxLengthField({
-            length: 1,
+            length: MAX_NUMBER_OF_NEW_TERMS_FIELDS,
             message: i18n.translate(
               'xpack.securitySolution.detectionEngine.validations.stepDefineRule.newTermsFieldsMax',
               {
-                defaultMessage: 'Number of fields must be 1.',
+                defaultMessage: 'Number of fields must be 3 or less.',
               }
             ),
           })(...args);

x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { BrowserField } from '../../../../common/containers/source';
+
+/**
+ * Filters out fields, that are not supported in terms aggregation.
+ * Terms aggregation supports limited number of types:
+ * Keyword, Numeric, ip, boolean, or binary.
+ * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html
+ */
+export const getTermsAggregationFields = (fields: BrowserField[]): BrowserField[] => {
+  // binary types is excluded, as binary field has property aggregatable === false
+  const allowedTypesSet = new Set(['string', 'number', 'ip', 'boolean']);
+
+  return fields.filter((field) => field.aggregatable === true && allowedTypesSet.has(field.type));
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md

@@ -2,7 +2,7 @@
 
 The rule accepts 2 new parameters that are unique to the new_terms rule type, in addition to common Security rule parameters such as query, index, and filters, to, from, etc. The new parameters are:
 
-- `new_terms_fields`: an array of field names, currently limited to an array of size 1. In the future we will likely allow multiple field names to be specified here.
+- `new_terms_fields`: an array of field names, currently limited to an array of size 3.
   Example: ['host.ip']
 - `history_window_start`: defines the additional time range to search over when determining if a term is "new". If a term is found between the times `history_window_start` and from then it will not be classified as a new term.
   Example: now-30d
@@ -12,6 +12,7 @@ Each page is evaluated in 3 phases.
 Phase 1: Collect "recent" terms - terms that have appeared in the last rule interval, without regard to whether or not they have appeared in historical data. This is done using a composite aggregation to ensure we can iterate over every term.
 
 Phase 2: Check if the page of terms contains any new terms. This uses a regular terms agg with the include parameter - every term is added to the array of include values, so the terms agg is limited to only aggregating on the terms of interest from phase 1. This avoids issues with the terms agg providing approximate results due to getting different terms from different shards.
+For multiple new terms fields(['source.host', 'source.ip']), in terms aggregation uses a runtime field. Which is created by joining values from new terms fields into one single keyword value. Fields values encoded in base64 and joined with configured a delimiter symbol, which is not part of base64 symbols(a–Z, 0–9, +, /,  =) to avoid a situation when delimiter can be part of field value. Include parameter consists of encoded in base64 results from Phase 1.
 
 Phase 3: Any new terms from phase 2 are processed and the first document to contain that term is retrieved. The document becomes the basis of the generated alert. This is done with an aggregation query that is very similar to the agg used in phase 2, except it also includes a top_hits agg. top_hits is moved to a separate, later phase for efficiency - top_hits is slow and most terms will not be new in phase 2. This means we only execute the top_hits agg on the terms that are actually new which is faster.
 
@@ -26,4 +27,4 @@ The new terms rule type reuses the singleSearchAfter function which implements t
 ## Limitations and future enhancements
 
 - Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions.
-- In the future we may want to support searching for new sets of terms, e.g. a pair of `host.ip` and `host.id` that has never been seen together before.
+- Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.test.ts

@@ -16,7 +16,7 @@ describe('aggregations', () => {
   describe('buildRecentTermsAgg', () => {
     test('builds a correct composite agg without `after`', () => {
       const aggregation = buildRecentTermsAgg({
-        field: 'host.name',
+        fields: ['host.name'],
         after: undefined,
       });
 
@@ -25,12 +25,21 @@ describe('aggregations', () => {
 
     test('builds a correct composite aggregation with `after`', () => {
       const aggregation = buildRecentTermsAgg({
-        field: 'host.name',
+        fields: ['host.name'],
         after: { 'host.name': 'myHost' },
       });
 
       expect(aggregation).toMatchSnapshot();
     });
+
+    test('builds a correct composite aggregation with multiple fields', () => {
+      const aggregation = buildRecentTermsAgg({
+        fields: ['host.name', 'host.port', 'host.url'],
+        after: undefined,
+      });
+
+      expect(aggregation).toMatchSnapshot();
+    });
   });
 
   describe('buildNewTermsAggregation', () => {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts

@@ -31,24 +31,24 @@ const PAGE_SIZE = 10000;
  * without regard to whether or not they're actually new.
  */
 export const buildRecentTermsAgg = ({
-  field,
+  fields,
   after,
 }: {
-  field: string;
+  fields: string[];
   after: Record<string, string | number | null> | undefined;
 }) => {
+  const sources = fields.map((field) => ({
+    [field]: {
+      terms: {
+        field,
+      },
+    },
+  }));
+
   return {
     new_terms: {
       composite: {
-        sources: [
-          {
-            [field]: {
-              terms: {
-                field,
-              },
-            },
-          },
-        ],
+        sources,
         size: PAGE_SIZE,
         after,
       },