[7.x] [Security] Adds field mapping support to rule creation (#70288) (…

…#70603)

* [Security] Adds field mapping support to rule creation (#70288)

## Summary

Resolves: #65941, #66317, and `Add support for "building block" alerts`

This PR is `Part I` and adds additional fields to the `rules schema` in supporting the ability to map and override fields when generating alerts. A few bookkeeping fields like `license` and `author` have been added as well. The new fields are as follows:

``` ts
export interface TheseAreTheNewFields {
  author: string[];
  building_block_type: string; // 'default'
  license: string;
  risk_score_mapping: Array<
    {
      field: string;
      operator: string; // 'equals'
      value: string;
    }
  >;
  rule_name_override: string;
  severity_mapping: Array<
    {
      field: string;
      operator: string; // 'equals'
      value: string;
      severity: string; // 'low' | 'medium' | 'high' | 'critical'
    }
  >;
  timestamp_override: string;
}
```

These new fields are exposed as additional settings on the `About rule` section of the Rule Creation UI.

##### Default collapsed view, no severity or risk score override specified:
<p align="center">
  <img width="500" src="https://user-images.githubusercontent.com/2946766/86090417-49c0ee80-ba67-11ea-898f-a43af6d9383f.png" />
</p>

##### Severity & risk score override specified:
<p align="center">
  <img width="500" src="https://user-images.githubusercontent.com/2946766/86091165-a8d33300-ba68-11ea-86ac-89393a7ca3f5.png" />
</p>

##### Additional fields in Advanced settings:
<p align="center">
  <img width="500" src="https://user-images.githubusercontent.com/2946766/86091256-cbfde280-ba68-11ea-9b63-acf2524039bd.png" />
</p>


Note: This PR adds the fields to the `Rules Schema`, the `signals index mapping`,  and creates the UI for adding these fields during Rule Creation/Editing. The follow-up `Part II` will add the business logic for mapping fields during `rule execution`, and also add UI validation/additional tests.

### Checklist

Delete any items that are not applicable to this PR.

- [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)
- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials
  - Syncing w/ @benskelker 
- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios
- [x] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)

### For maintainers

- [x] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)

* Fixed i18n keys

Co-authored-by: Garrett Spong <spong@users.noreply.github.com>

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -13,6 +13,18 @@ import { IsoDateString } from '../types/iso_date_string';
 import { PositiveIntegerGreaterThanZero } from '../types/positive_integer_greater_than_zero';
 import { PositiveInteger } from '../types/positive_integer';
 
+export const author = t.array(t.string);
+export type Author = t.TypeOf<typeof author>;
+
+export const authorOrUndefined = t.union([author, t.undefined]);
+export type AuthorOrUndefined = t.TypeOf<typeof authorOrUndefined>;
+
+export const building_block_type = t.string;
+export type BuildingBlockType = t.TypeOf<typeof building_block_type>;
+
+export const buildingBlockTypeOrUndefined = t.union([building_block_type, t.undefined]);
+export type BuildingBlockTypeOrUndefined = t.TypeOf<typeof buildingBlockTypeOrUndefined>;
+
 export const description = t.string;
 export type Description = t.TypeOf<typeof description>;
 
@@ -111,6 +123,12 @@ export type Language = t.TypeOf<typeof language>;
 export const languageOrUndefined = t.union([language, t.undefined]);
 export type LanguageOrUndefined = t.TypeOf<typeof languageOrUndefined>;
 
+export const license = t.string;
+export type License = t.TypeOf<typeof license>;
+
+export const licenseOrUndefined = t.union([license, t.undefined]);
+export type LicenseOrUndefined = t.TypeOf<typeof licenseOrUndefined>;
+
 export const objects = t.array(t.type({ rule_id }));
 
 export const output_index = t.string;
@@ -137,6 +155,12 @@ export type TimelineTitle = t.TypeOf<typeof t.string>;
 export const timelineTitleOrUndefined = t.union([timeline_title, t.undefined]);
 export type TimelineTitleOrUndefined = t.TypeOf<typeof timelineTitleOrUndefined>;
 
+export const timestamp_override = t.string;
+export type TimestampOverride = t.TypeOf<typeof timestamp_override>;
+
+export const timestampOverrideOrUndefined = t.union([timestamp_override, t.undefined]);
+export type TimestampOverrideOrUndefined = t.TypeOf<typeof timestampOverrideOrUndefined>;
+
 export const throttle = t.string;
 export type Throttle = t.TypeOf<typeof throttle>;
 
@@ -179,18 +203,65 @@ export type Name = t.TypeOf<typeof name>;
 export const nameOrUndefined = t.union([name, t.undefined]);
 export type NameOrUndefined = t.TypeOf<typeof nameOrUndefined>;
 
+export const operator = t.keyof({
+  equals: null,
+});
+export type Operator = t.TypeOf<typeof operator>;
+export enum OperatorEnum {
+  EQUALS = 'equals',
+}
+
 export const risk_score = RiskScore;
 export type RiskScore = t.TypeOf<typeof risk_score>;
 
 export const riskScoreOrUndefined = t.union([risk_score, t.undefined]);
 export type RiskScoreOrUndefined = t.TypeOf<typeof riskScoreOrUndefined>;
 
+export const risk_score_mapping_field = t.string;
+export const risk_score_mapping_value = t.string;
+export const risk_score_mapping_item = t.exact(
+  t.type({
+    field: risk_score_mapping_field,
+    operator,
+    value: risk_score_mapping_value,
+  })
+);
+
+export const risk_score_mapping = t.array(risk_score_mapping_item);
+export type RiskScoreMapping = t.TypeOf<typeof risk_score_mapping>;
+
+export const riskScoreMappingOrUndefined = t.union([risk_score_mapping, t.undefined]);
+export type RiskScoreMappingOrUndefined = t.TypeOf<typeof riskScoreMappingOrUndefined>;
+
+export const rule_name_override = t.string;
+export type RuleNameOverride = t.TypeOf<typeof rule_name_override>;
+
+export const ruleNameOverrideOrUndefined = t.union([rule_name_override, t.undefined]);
+export type RuleNameOverrideOrUndefined = t.TypeOf<typeof ruleNameOverrideOrUndefined>;
+
 export const severity = t.keyof({ low: null, medium: null, high: null, critical: null });
 export type Severity = t.TypeOf<typeof severity>;
 
 export const severityOrUndefined = t.union([severity, t.undefined]);
 export type SeverityOrUndefined = t.TypeOf<typeof severityOrUndefined>;
 
+export const severity_mapping_field = t.string;
+export const severity_mapping_value = t.string;
+export const severity_mapping_item = t.exact(
+  t.type({
+    field: severity_mapping_field,
+    operator,
+    value: severity_mapping_value,
+    severity,
+  })
+);
+
+export const severity_mapping = t.array(severity_mapping_item);
+export type SeverityMapping = t.TypeOf<typeof severity_mapping>;
+
+export const severityMappingOrUndefined = t.union([severity_mapping, t.undefined]);
+export type SeverityMappingOrUndefined = t.TypeOf<typeof severityMappingOrUndefined>;
+
 export const status = t.keyof({ open: null, closed: null, 'in-progress': null });
 export type Status = t.TypeOf<typeof status>;
 

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts

@@ -23,12 +23,15 @@ export const getAddPrepackagedRulesSchemaMock = (): AddPrepackagedRulesSchema =>
 });
 
 export const getAddPrepackagedRulesSchemaDecodedMock = (): AddPrepackagedRulesSchemaDecoded => ({
+  author: [],
   description: 'some description',
   name: 'Query with a rule id',
   query: 'user.name: root or user.name: admin',
   severity: 'high',
+  severity_mapping: [],
   type: 'query',
   risk_score: 55,
+  risk_score_mapping: [],
   language: 'kuery',
   references: [],
   actions: [],

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -37,6 +37,13 @@ import {
   query,
   rule_id,
   version,
+  building_block_type,
+  license,
+  rule_name_override,
+  timestamp_override,
+  Author,
+  RiskScoreMapping,
+  SeverityMapping,
 } from '../common/schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
@@ -52,6 +59,8 @@ import {
   DefaultThrottleNull,
   DefaultListArray,
   ListArray,
+  DefaultRiskScoreMappingArray,
+  DefaultSeverityMappingArray,
 } from '../types';
 
 /**
@@ -79,6 +88,8 @@ export const addPrepackagedRulesSchema = t.intersection([
     t.partial({
       actions: DefaultActionsArray, // defaults to empty actions array if not set during decode
       anomaly_threshold, // defaults to undefined if not set during decode
+      author: DefaultStringArray, // defaults to empty array of strings if not set during decode
+      building_block_type, // defaults to undefined if not set during decode
       enabled: DefaultBooleanFalse, // defaults to false if not set during decode
       false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
       filters, // defaults to undefined if not set during decode
@@ -87,16 +98,21 @@ export const addPrepackagedRulesSchema = t.intersection([
       interval: DefaultIntervalString, // defaults to "5m" if not set during decode
       query, // defaults to undefined if not set during decode
       language, // defaults to undefined if not set during decode
+      license, // defaults to "undefined" if not set during decode
       saved_id, // defaults to "undefined" if not set during decode
       timeline_id, // defaults to "undefined" if not set during decode
       timeline_title, // defaults to "undefined" if not set during decode
       meta, // defaults to "undefined" if not set during decode
       machine_learning_job_id, // defaults to "undefined" if not set during decode
       max_signals: DefaultMaxSignalsNumber, // defaults to DEFAULT_MAX_SIGNALS (100) if not set during decode
+      risk_score_mapping: DefaultRiskScoreMappingArray, // defaults to empty risk score mapping array if not set during decode
+      rule_name_override, // defaults to "undefined" if not set during decode
+      severity_mapping: DefaultSeverityMappingArray, // defaults to empty actions array if not set during decode
       tags: DefaultStringArray, // defaults to empty string array if not set during decode
       to: DefaultToString, // defaults to "now" if not set during decode
       threat: DefaultThreatArray, // defaults to empty array if not set during decode
       throttle: DefaultThrottleNull, // defaults to "null" if not set during decode
+      timestamp_override, // defaults to "undefined" if not set during decode
       references: DefaultStringArray, // defaults to empty array of strings if not set during decode
       note, // defaults to "undefined" if not set during decode
       exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
@@ -109,26 +125,32 @@ export type AddPrepackagedRulesSchema = t.TypeOf<typeof addPrepackagedRulesSchem
 // This type is used after a decode since some things are defaults after a decode.
 export type AddPrepackagedRulesSchemaDecoded = Omit<
   AddPrepackagedRulesSchema,
+  | 'author'
   | 'references'
   | 'actions'
   | 'enabled'
   | 'false_positives'
   | 'from'
   | 'interval'
   | 'max_signals'
+  | 'risk_score_mapping'
+  | 'severity_mapping'
   | 'tags'
   | 'to'
   | 'threat'
   | 'throttle'
   | 'exceptions_list'
 > & {
+  author: Author;
   references: References;
   actions: Actions;
   enabled: Enabled;
   false_positives: FalsePositives;
   from: From;
   interval: Interval;
   max_signals: MaxSignals;
+  risk_score_mapping: RiskScoreMapping;
+  severity_mapping: SeverityMapping;
   tags: Tags;
   to: To;
   threat: Threat;

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts

@@ -261,6 +261,9 @@ describe('add prepackaged rules schema', () => {
     const message = pipe(checked, foldLeftRight);
     expect(getPaths(left(message.errors))).toEqual([]);
     const expected: AddPrepackagedRulesSchemaDecoded = {
+      author: [],
+      severity_mapping: [],
+      risk_score_mapping: [],
       rule_id: 'rule-1',
       risk_score: 50,
       description: 'some description',
@@ -333,6 +336,9 @@ describe('add prepackaged rules schema', () => {
     const message = pipe(checked, foldLeftRight);
     expect(getPaths(left(message.errors))).toEqual([]);
     const expected: AddPrepackagedRulesSchemaDecoded = {
+      author: [],
+      severity_mapping: [],
+      risk_score_mapping: [],
       rule_id: 'rule-1',
       risk_score: 50,
       description: 'some description',
@@ -430,6 +436,9 @@ describe('add prepackaged rules schema', () => {
     const message = pipe(checked, foldLeftRight);
     expect(getPaths(left(message.errors))).toEqual([]);
     const expected: AddPrepackagedRulesSchemaDecoded = {
+      author: [],
+      severity_mapping: [],
+      risk_score_mapping: [],
       rule_id: 'rule-1',
       description: 'some description',
       from: 'now-5m',
@@ -508,6 +517,9 @@ describe('add prepackaged rules schema', () => {
     const message = pipe(checked, foldLeftRight);
     expect(getPaths(left(message.errors))).toEqual([]);
     const expected: AddPrepackagedRulesSchemaDecoded = {
+      author: [],
+      severity_mapping: [],
+      risk_score_mapping: [],
       rule_id: 'rule-1',
       risk_score: 50,
       description: 'some description',
@@ -1354,6 +1366,9 @@ describe('add prepackaged rules schema', () => {
       const message = pipe(checked, foldLeftRight);
       expect(getPaths(left(message.errors))).toEqual([]);
       const expected: AddPrepackagedRulesSchemaDecoded = {
+        author: [],
+        severity_mapping: [],
+        risk_score_mapping: [],
         rule_id: 'rule-1',
         description: 'some description',
         from: 'now-5m',
@@ -1404,6 +1419,9 @@ describe('add prepackaged rules schema', () => {
       const message = pipe(checked, foldLeftRight);
       expect(getPaths(left(message.errors))).toEqual([]);
       const expected: AddPrepackagedRulesSchemaDecoded = {
+        author: [],
+        severity_mapping: [],
+        risk_score_mapping: [],
         rule_id: 'rule-1',
         description: 'some description',
         from: 'now-5m',
@@ -1462,6 +1480,9 @@ describe('add prepackaged rules schema', () => {
       const message = pipe(checked, foldLeftRight);
       expect(getPaths(left(message.errors))).toEqual([]);
       const expected: AddPrepackagedRulesSchemaDecoded = {
+        author: [],
+        severity_mapping: [],
+        risk_score_mapping: [],
         rule_id: 'rule-1',
         description: 'some description',
         from: 'now-5m',
@@ -1539,6 +1560,9 @@ describe('add prepackaged rules schema', () => {
       const message = pipe(checked, foldLeftRight);
       expect(getPaths(left(message.errors))).toEqual([]);
       const expected: AddPrepackagedRulesSchemaDecoded = {
+        author: [],
+        severity_mapping: [],
+        risk_score_mapping: [],
         rule_id: 'rule-1',
         description: 'some description',
         from: 'now-5m',

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts

@@ -30,6 +30,9 @@ export const getCreateMlRulesSchemaMock = (ruleId = 'rule-1') => {
 };
 
 export const getCreateRulesSchemaDecodedMock = (): CreateRulesSchemaDecoded => ({
+  author: [],
+  severity_mapping: [],
+  risk_score_mapping: [],
   description: 'Detecting root and admin users',
   name: 'Query with a rule id',
   query: 'user.name: root or user.name: admin',