[RAC][Security Solution] Add RAC support to rule routes (#108053)

* prototyping

* how dis

* RAC rules create API

* Find rules (in progress)

* Finalize find_rules route

* A couple more routes, and type error fixes

* Fix integration tests?

* Fix tests

* Fix imports

* Add ref

* Test fixes

* Fix refs

* Type fixes

* Test fixes

* Remove console log

* Update rule changes

* Test and type fixes

* Fix patch rule tests

* Fix types

* Begin removing namespace as required param

* Remove generics

* Support RAC everywhere

* Tests passing

* Types

* Keep on passing isRuleRegistryEnabled around

* Rewrite install_prepackaged_timelines helper tests

x-pack/plugins/rule_registry/server/index.ts

@@ -12,6 +12,9 @@ import { PluginInitializerContext } from 'src/core/server';
 import { RuleRegistryPlugin } from './plugin';
 
 export type { RuleRegistryPluginSetupContract, RuleRegistryPluginStartContract } from './plugin';
+export { RuleDataPluginService } from './rule_data_plugin_service';
+export { RuleDataClient } from './rule_data_client';
+export { IRuleDataClient } from './rule_data_client/types';
 export type {
   RacRequestHandlerContext,
   RacApiRequestHandlerContext,

x-pack/plugins/security_solution/common/constants.ts

@@ -187,19 +187,18 @@ export const DEFAULT_TRANSFORMS_SETTING = JSON.stringify(defaultTransformsSettin
 /**
  * Id for the signals alerting type
  */
-export const SIGNALS_ID = `siem.signals`;
+export const SIGNALS_ID = `siem.signals` as const;
 
 /**
- * Id's for reference rule types
+ * IDs for RAC rule types
  */
-export const REFERENCE_RULE_ALERT_TYPE_ID = `siem.referenceRule`;
-export const REFERENCE_RULE_PERSISTENCE_ALERT_TYPE_ID = `siem.referenceRulePersistence`;
-
-export const QUERY_ALERT_TYPE_ID = `siem.queryRule`;
-export const EQL_ALERT_TYPE_ID = `siem.eqlRule`;
-export const INDICATOR_ALERT_TYPE_ID = `siem.indicatorRule`;
-export const ML_ALERT_TYPE_ID = `siem.mlRule`;
-export const THRESHOLD_ALERT_TYPE_ID = `siem.thresholdRule`;
+const RULE_TYPE_PREFIX = `siem` as const;
+export const EQL_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.eqlRule` as const;
+export const INDICATOR_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.indicatorRule` as const;
+export const ML_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.mlRule` as const;
+export const QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.queryRule` as const;
+export const SAVED_QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.savedQueryRule` as const;
+export const THRESHOLD_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.thresholdRule` as const;
 
 /**
  * Id for the notifications alerting type

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -59,6 +59,9 @@ export type FileName = t.TypeOf<typeof file_name>;
 export const exclude_export_details = t.boolean;
 export type ExcludeExportDetails = t.TypeOf<typeof exclude_export_details>;
 
+export const namespace = t.string;
+export type Namespace = t.TypeOf<typeof namespace>;
+
 /**
  * TODO: Right now the filters is an "unknown", when it could more than likely
  * become the actual ESFilter as a type.
@@ -352,6 +355,9 @@ export const timelines_not_updated = PositiveInteger;
 export const note = t.string;
 export type Note = t.TypeOf<typeof note>;
 
+export const namespaceOrUndefined = t.union([namespace, t.undefined]);
+export type NamespaceOrUndefined = t.TypeOf<typeof namespaceOrUndefined>;
+
 export const noteOrUndefined = t.union([note, t.undefined]);
 export type NoteOrUndefined = t.TypeOf<typeof noteOrUndefined>;
 

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts → x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.test.ts

@@ -465,6 +465,23 @@ describe('add prepackaged rules schema', () => {
     expect(message.schema).toEqual(expected);
   });
 
+  test('You can send in a namespace', () => {
+    const payload: AddPrepackagedRulesSchema = {
+      ...getAddPrepackagedRulesSchemaMock(),
+      namespace: 'a namespace',
+    };
+
+    const decoded = addPrepackagedRulesSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([]);
+    const expected: AddPrepackagedRulesSchemaDecoded = {
+      ...getAddPrepackagedRulesSchemaDecodedMock(),
+      namespace: 'a namespace',
+    };
+    expect(message.schema).toEqual(expected);
+  });
+
   test('You can send in an empty array to threat', () => {
     const payload: AddPrepackagedRulesSchema = {
       ...getAddPrepackagedRulesSchemaMock(),

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -71,6 +71,7 @@ import {
   timestamp_override,
   Author,
   event_category_override,
+  namespace,
 } from '../common/schemas';
 
 /**
@@ -136,10 +137,10 @@ export const addPrepackagedRulesSchema = t.intersection([
       threat_indicator_path, // defaults "undefined" if not set during decode
       concurrent_searches, // defaults to "undefined" if not set during decode
       items_per_search, // defaults to "undefined" if not set during decode
+      namespace, // defaults to "undefined" if not set during decode
     })
   ),
 ]);
-
 export type AddPrepackagedRulesSchema = t.TypeOf<typeof addPrepackagedRulesSchema>;
 
 // This type is used after a decode since some things are defaults after a decode.
@@ -153,6 +154,7 @@ export type AddPrepackagedRulesSchemaDecoded = Omit<
   | 'from'
   | 'interval'
   | 'max_signals'
+  | 'namespace'
   | 'risk_score_mapping'
   | 'severity_mapping'
   | 'tags'
@@ -176,4 +178,5 @@ export type AddPrepackagedRulesSchemaDecoded = Omit<
   threat: Threats;
   throttle: ThrottleOrNull;
   exceptions_list: ListArray;
+  namespace?: string;
 };

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts

@@ -330,6 +330,19 @@ describe('create rules schema', () => {
     expect(message.schema).toEqual(payload);
   });
 
+  test('You can send in a namespace', () => {
+    const payload: CreateRulesSchema = {
+      ...getCreateRulesSchemaMock(),
+      namespace: 'a namespace',
+    };
+
+    const decoded = createRulesSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
   test('You can send in an empty array to threat', () => {
     const payload: CreateRulesSchema = {
       ...getCreateRulesSchemaMock(),

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts

@@ -68,6 +68,7 @@ import {
   last_success_message,
   last_failure_at,
   last_failure_message,
+  namespace,
 } from '../common/schemas';
 
 export const createSchema = <
@@ -155,6 +156,7 @@ const baseParams = {
     meta,
     rule_name_override,
     timestamp_override,
+    namespace,
   },
   defaultable: {
     tags,

x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_bulk_schema.test.ts

@@ -187,6 +187,18 @@ describe('update_rules_bulk_schema', () => {
     expect(output.schema).toEqual({});
   });
 
+  test('You can set "namespace" to a string', () => {
+    const payload: UpdateRulesBulkSchema = [
+      { ...getUpdateRulesSchemaMock(), namespace: 'a namespace' },
+    ];
+
+    const decoded = updateRulesBulkSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const output = foldLeftRight(checked);
+    expect(formatErrors(output.errors)).toEqual([]);
+    expect(output.schema).toEqual(payload);
+  });
+
   test('You can set "note" to a string', () => {
     const payload: UpdateRulesBulkSchema = [
       { ...getUpdateRulesSchemaMock(), note: '# test markdown' },

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts

@@ -404,6 +404,19 @@ describe('rules_schema', () => {
       expect(message.schema).toEqual(expected);
     });
 
+    test('it should validate a namespace as string', () => {
+      const payload = {
+        ...getRulesSchemaMock(),
+        namespace: 'a namespace',
+      };
+      const dependents = getDependents(payload);
+      const decoded = dependents.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(payload);
+    });
+
     test('it should NOT validate invalid_data for the type', () => {
       const payload: Omit<RulesSchema, 'type'> & { type: string } = getRulesSchemaMock();
       payload.type = 'invalid_data';

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts

@@ -75,6 +75,7 @@ import {
   license,
   rule_name_override,
   timestamp_override,
+  namespace,
 } from '../common/schemas';
 
 import { typeAndTimelineOnlySchema, TypeAndTimelineOnly } from './type_timeline_only_schema';
@@ -174,6 +175,7 @@ export const partialRulesSchema = t.partial({
   filters,
   meta,
   index,
+  namespace,
   note,
 });
 

x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_throttle_notification_actions.test.ts

@@ -53,6 +53,7 @@ describe('schedule_throttle_notification_actions', () => {
       to: 'now',
       type: 'query',
       references: ['http://www.example.com'],
+      namespace: 'a namespace',
       note: '# sample markdown',
       version: 1,
       exceptionsList: [],

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts

@@ -39,6 +39,7 @@ import { getQueryRuleParams } from '../../schemas/rule_schemas.mock';
 import { getPerformBulkActionSchemaMock } from '../../../../../common/detection_engine/schemas/request/perform_bulk_action_schema.mock';
 import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas';
 import { FindBulkExecutionLogResponse } from '../../rule_execution_log/types';
+import { ruleTypeMappings } from '../../signals/utils';
 
 export const typicalSetStatusSignalByIdsPayload = (): SetSignalsStatusSchemaDecoded => ({
   signal_ids: ['somefakeid1', 'somefakeid2'],
@@ -179,18 +180,18 @@ export const getEmptyFindResult = (): FindHit => ({
   data: [],
 });
 
-export const getFindResultWithSingleHit = (): FindHit => ({
+export const getFindResultWithSingleHit = (isRuleRegistryEnabled: boolean): FindHit => ({
   page: 1,
   perPage: 1,
   total: 1,
-  data: [getAlertMock(getQueryRuleParams())],
+  data: [getAlertMock(isRuleRegistryEnabled, getQueryRuleParams())],
 });
 
-export const nonRuleFindResult = (): FindHit => ({
+export const nonRuleFindResult = (isRuleRegistryEnabled: boolean): FindHit => ({
   page: 1,
   perPage: 1,
   total: 1,
-  data: [nonRuleAlert()],
+  data: [nonRuleAlert(isRuleRegistryEnabled)],
 });
 
 export const getFindResultWithMultiHits = ({
@@ -348,19 +349,22 @@ export const createActionResult = (): ActionResult => ({
   isPreconfigured: false,
 });
 
-export const nonRuleAlert = () => ({
+export const nonRuleAlert = (isRuleRegistryEnabled: boolean) => ({
   // Defaulting to QueryRuleParams because ts doesn't like empty objects
-  ...getAlertMock(getQueryRuleParams()),
+  ...getAlertMock(isRuleRegistryEnabled, getQueryRuleParams()),
   id: '04128c15-0d1b-4716-a4c5-46997ac7f3bc',
   name: 'Non-Rule Alert',
   alertTypeId: 'something',
 });
 
-export const getAlertMock = <T extends RuleParams>(params: T): Alert<T> => ({
+export const getAlertMock = <T extends RuleParams>(
+  isRuleRegistryEnabled: boolean,
+  params: T
+): Alert<T> => ({
   id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
   name: 'Detect Root/Admin Users',
   tags: [`${INTERNAL_RULE_ID_KEY}:rule-1`, `${INTERNAL_IMMUTABLE_KEY}:false`],
-  alertTypeId: 'siem.signals',
+  alertTypeId: isRuleRegistryEnabled ? ruleTypeMappings[params.type] : 'siem.signals',
   consumer: 'siem',
   params,
   createdAt: new Date('2019-12-13T16:40:33.400Z'),

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts

@@ -72,12 +72,16 @@ jest.mock('../../../timeline/routes/prepackaged_timelines/install_prepackaged_ti
   };
 });
 
-describe('add_prepackaged_rules_route', () => {
+describe.each([
+  ['Legacy', false],
+  ['RAC', true],
+])('add_prepackaged_rules_route - %s', (_, isRuleRegistryEnabled) => {
   const siemMockClient = siemMock.createClient();
   let server: ReturnType<typeof serverMock.create>;
   let { clients, context } = requestContextMock.createTools();
   let securitySetup: SecurityPluginSetup;
   let mockExceptionsClient: ExceptionListClient;
+  const testif = isRuleRegistryEnabled ? test.skip : test;
 
   beforeEach(() => {
     server = serverMock.create();
@@ -91,8 +95,10 @@ describe('add_prepackaged_rules_route', () => {
 
     mockExceptionsClient = listMock.getExceptionListClient();
 
-    clients.rulesClient.find.mockResolvedValue(getFindResultWithSingleHit());
-    clients.rulesClient.update.mockResolvedValue(getAlertMock(getQueryRuleParams()));
+    clients.rulesClient.find.mockResolvedValue(getFindResultWithSingleHit(isRuleRegistryEnabled));
+    clients.rulesClient.update.mockResolvedValue(
+      getAlertMock(isRuleRegistryEnabled, getQueryRuleParams())
+    );
 
     (installPrepackagedTimelines as jest.Mock).mockReset();
     (installPrepackagedTimelines as jest.Mock).mockResolvedValue({
@@ -106,7 +112,7 @@ describe('add_prepackaged_rules_route', () => {
     context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValue(
       elasticsearchClientMock.createSuccessTransportRequestPromise({ _shards: { total: 1 } })
     );
-    addPrepackedRulesRoute(server.router, createMockConfig(), securitySetup);
+    addPrepackedRulesRoute(server.router, createMockConfig(), securitySetup, isRuleRegistryEnabled);
   });
 
   describe('status codes', () => {
@@ -129,23 +135,25 @@ describe('add_prepackaged_rules_route', () => {
       });
     });
 
-    test('it returns a 400 if the index does not exist', async () => {
+    test('it returns a 400 if the index does not exist when rule registry not enabled', async () => {
       const request = addPrepackagedRulesRequest();
       context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValueOnce(
         elasticsearchClientMock.createSuccessTransportRequestPromise({ _shards: { total: 0 } })
       );
       const response = await server.inject(request, context);
 
-      expect(response.status).toEqual(400);
-      expect(response.body).toEqual({
-        status_code: 400,
-        message: expect.stringContaining(
-          'Pre-packaged rules cannot be installed until the signals index is created'
-        ),
-      });
+      expect(response.status).toEqual(isRuleRegistryEnabled ? 200 : 400);
+      if (!isRuleRegistryEnabled) {
+        expect(response.body).toEqual({
+          status_code: 400,
+          message: expect.stringContaining(
+            'Pre-packaged rules cannot be installed until the signals index is created'
+          ),
+        });
+      }
     });
 
-    it('returns 404 if siem client is unavailable', async () => {
+    test('returns 404 if siem client is unavailable', async () => {
       const { securitySolution, ...contextWithoutSecuritySolution } = context;
       const response = await server.inject(
         addPrepackagedRulesRequest(),
@@ -185,16 +193,19 @@ describe('add_prepackaged_rules_route', () => {
       });
     });
 
-    test('catches errors if payloads cause errors to be thrown', async () => {
-      context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValue(
-        elasticsearchClientMock.createErrorTransportRequestPromise(new Error('Test error'))
-      );
-      const request = addPrepackagedRulesRequest();
-      const response = await server.inject(request, context);
-
-      expect(response.status).toEqual(500);
-      expect(response.body).toEqual({ message: 'Test error', status_code: 500 });
-    });
+    testif(
+      'catches errors if signals index does not exist when rule registry not enabled',
+      async () => {
+        context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValue(
+          elasticsearchClientMock.createErrorTransportRequestPromise(new Error('Test error'))
+        );
+        const request = addPrepackagedRulesRequest();
+        const response = await server.inject(request, context);
+
+        expect(response.status).toEqual(500);
+        expect(response.body).toEqual({ message: 'Test error', status_code: 500 });
+      }
+    );
   });
 
   test('should install prepackaged timelines', async () => {