diff --git a/x-pack/legacy/plugins/siem/public/components/ml/helpers/index.test.ts b/x-pack/legacy/plugins/siem/common/detection_engine/ml_helpers.test.ts similarity index 96% rename from x-pack/legacy/plugins/siem/public/components/ml/helpers/index.test.ts rename to x-pack/legacy/plugins/siem/common/detection_engine/ml_helpers.test.ts index 693f0bd0dd0fd..ba93b2e4b8a0d 100644 --- a/x-pack/legacy/plugins/siem/public/components/ml/helpers/index.test.ts +++ b/x-pack/legacy/plugins/siem/common/detection_engine/ml_helpers.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { isJobStarted, isJobLoading, isJobFailed } from './'; +import { isJobStarted, isJobLoading, isJobFailed } from './ml_helpers'; describe('isJobStarted', () => { test('returns false if only jobState is enabled', () => { diff --git a/x-pack/legacy/plugins/siem/public/components/ml/helpers/index.ts b/x-pack/legacy/plugins/siem/common/detection_engine/ml_helpers.ts similarity index 89% rename from x-pack/legacy/plugins/siem/public/components/ml/helpers/index.ts rename to x-pack/legacy/plugins/siem/common/detection_engine/ml_helpers.ts index c06596b49317d..e4158d08d448d 100644 --- a/x-pack/legacy/plugins/siem/public/components/ml/helpers/index.ts +++ b/x-pack/legacy/plugins/siem/common/detection_engine/ml_helpers.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { RuleType } from './types'; + // Based on ML Job/Datafeed States from x-pack/legacy/plugins/ml/common/constants/states.js const enabledStates = ['started', 'opened']; const loadingStates = ['starting', 'stopping', 'opening', 'closing']; @@ -20,3 +22,5 @@ export const isJobLoading = (jobState: string, datafeedState: string): boolean = export const isJobFailed = (jobState: string, datafeedState: string): boolean => { return failureStates.includes(jobState) || failureStates.includes(datafeedState); }; + +export const isMlRule = (ruleType: RuleType) => ruleType === 'machine_learning'; diff --git a/x-pack/legacy/plugins/siem/common/detection_engine/types.ts b/x-pack/legacy/plugins/siem/common/detection_engine/types.ts index 0de370b11cdaf..39012d0b4b683 100644 --- a/x-pack/legacy/plugins/siem/common/detection_engine/types.ts +++ b/x-pack/legacy/plugins/siem/common/detection_engine/types.ts @@ -3,9 +3,17 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import * as t from 'io-ts'; import { AlertAction } from '../../../../../plugins/alerting/common'; export type RuleAlertAction = Omit & { action_type_id: string; }; + +export const RuleTypeSchema = t.keyof({ + query: null, + saved_query: null, + machine_learning: null, +}); +export type RuleType = t.TypeOf; diff --git a/x-pack/legacy/plugins/siem/public/components/ml_popover/jobs_table/job_switch.tsx b/x-pack/legacy/plugins/siem/public/components/ml_popover/jobs_table/job_switch.tsx index e5066eef18c8b..a0e0c70d2f204 100644 --- a/x-pack/legacy/plugins/siem/public/components/ml_popover/jobs_table/job_switch.tsx +++ b/x-pack/legacy/plugins/siem/public/components/ml_popover/jobs_table/job_switch.tsx @@ -8,7 +8,11 @@ import styled from 'styled-components'; import React, { useState, useCallback } from 'react'; import { EuiFlexGroup, EuiFlexItem, EuiLoadingSpinner, EuiSwitch } from '@elastic/eui'; import { SiemJob } from '../types'; -import { isJobLoading, isJobStarted, isJobFailed } from '../../ml/helpers'; +import { + isJobLoading, + isJobFailed, + isJobStarted, +} from '../../../../common/detection_engine/ml_helpers'; const StaticSwitch = styled(EuiSwitch)` .euiSwitch__thumb, diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts index f676ab944fce4..bc559c5ac4972 100644 --- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts +++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts @@ -6,12 +6,7 @@ import * as t from 'io-ts'; -export const RuleTypeSchema = t.keyof({ - query: null, - saved_query: null, - machine_learning: null, -}); -export type RuleType = t.TypeOf; +import { RuleTypeSchema } from '../../../../common/detection_engine/types'; /** * Params is an "record", since it is a type of AlertActionParams which is action templates. diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rule_status.test.tsx b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rule_status.test.tsx index 3db39dac78b8e..0a30329baf68d 100644 --- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rule_status.test.tsx +++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rule_status.test.tsx @@ -12,11 +12,11 @@ import { ReturnRulesStatuses, } from './use_rule_status'; import * as api from './api'; -import { RuleType } from '../rules/types'; +import { Rule } from '../rules/types'; jest.mock('./api'); -const testRule = { +const testRule: Rule = { actions: [ { group: 'fake group', @@ -57,7 +57,7 @@ const testRule = { threat: [], throttle: null, to: 'now', - type: 'query' as RuleType, + type: 'query', updated_at: 'mm/dd/yyyyTHH:MM:sssz', updated_by: 'mockUser', }; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx index f9b255a95d869..79da7999b081a 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx @@ -21,13 +21,13 @@ import styled from 'styled-components'; import { esFilters } from '../../../../../../../../../../src/plugins/data/public'; +import { RuleType } from '../../../../../../common/detection_engine/types'; import { tacticsOptions, techniquesOptions } from '../../../mitre/mitre_tactics_techniques'; import * as i18n from './translations'; import { BuildQueryBarDescription, BuildThreatDescription, ListItems } from './types'; import { SeverityBadge } from '../severity_badge'; import ListTreeIcon from './assets/list_tree_icon.svg'; -import { RuleType } from '../../../../../containers/detection_engine/rules'; import { assertUnreachable } from '../../../../../lib/helpers'; const NoteDescriptionContainer = styled(EuiFlexItem)` diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx index 69c4ee1017155..05e47225c8f4b 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx @@ -15,7 +15,7 @@ import { esFilters, FilterManager, } from '../../../../../../../../../../src/plugins/data/public'; -import { RuleType } from '../../../../../containers/detection_engine/rules'; +import { RuleType } from '../../../../../../common/detection_engine/types'; import { DEFAULT_TIMELINE_TITLE } from '../../../../../components/timeline/translations'; import { useKibana } from '../../../../../lib/kibana'; import { IMitreEnterpriseAttack } from '../../types'; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/ml_job_description.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/ml_job_description.tsx index 947bf29c07148..8276aa3578563 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/ml_job_description.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/ml_job_description.tsx @@ -11,8 +11,8 @@ import { EuiBadge, EuiIcon, EuiLink, EuiToolTip } from '@elastic/eui'; import { useKibana } from '../../../../../lib/kibana'; import { SiemJob } from '../../../../../components/ml_popover/types'; import { ListItems } from './types'; -import { isJobStarted } from '../../../../../components/ml/helpers'; import { ML_JOB_STARTED, ML_JOB_STOPPED } from './translations'; +import { isJobStarted } from '../../../../../../common/detection_engine/ml_helpers'; enum MessageLevels { info = 'info', diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx index 4ccde78f3cda7..9d3b37f1788fa 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx @@ -16,10 +16,10 @@ import { EuiText, } from '@elastic/eui'; +import { isMlRule } from '../../../../../../common/detection_engine/ml_helpers'; +import { RuleType } from '../../../../../../common/detection_engine/types'; import { FieldHook } from '../../../../../shared_imports'; -import { RuleType } from '../../../../../containers/detection_engine/rules/types'; import * as i18n from './translations'; -import { isMlRule } from '../../helpers'; const MlCardDescription = ({ hasValidLicense = false }: { hasValidLicense?: boolean }) => ( diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx index 6c46ab0b171a2..05043e5b96a30 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx @@ -12,10 +12,11 @@ import deepEqual from 'fast-deep-equal'; import { IIndexPattern } from '../../../../../../../../../../src/plugins/data/public'; import { useFetchIndexPatterns } from '../../../../../containers/detection_engine/rules'; import { DEFAULT_INDEX_KEY } from '../../../../../../common/constants'; +import { isMlRule } from '../../../../../../common/detection_engine/ml_helpers'; import { DEFAULT_TIMELINE_TITLE } from '../../../../../components/timeline/translations'; import { useMlCapabilities } from '../../../../../components/ml_popover/hooks/use_ml_capabilities'; import { useUiSetting$ } from '../../../../../lib/kibana'; -import { setFieldValue, isMlRule } from '../../helpers'; +import { setFieldValue } from '../../helpers'; import { DefineStepRule, RuleStep, RuleStepProps } from '../../types'; import { StepRuleDescription } from '../description_step'; import { QueryBarDefineRule } from '../query_bar'; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx index 271c8fabed3a5..4a132f94a9871 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx @@ -10,6 +10,7 @@ import { isEmpty } from 'lodash/fp'; import React from 'react'; import { esKuery } from '../../../../../../../../../../src/plugins/data/public'; +import { isMlRule } from '../../../../../../common/detection_engine/ml_helpers'; import { FieldValueQueryBar } from '../query_bar'; import { ERROR_CODE, @@ -19,7 +20,6 @@ import { ValidationFunc, } from '../../../../../shared_imports'; import { CUSTOM_QUERY_REQUIRED, INVALID_CUSTOM_QUERY, INDEX_HELPER_TEXT } from './translations'; -import { isMlRule } from '../../helpers'; export const schema: FormSchema = { index: { diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts index 7abe5a576c0e5..1bc5d85258ffd 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts @@ -12,8 +12,10 @@ import { NOTIFICATION_THROTTLE_RULE, NOTIFICATION_THROTTLE_NO_ACTIONS, } from '../../../../../common/constants'; -import { NewRule, RuleType } from '../../../../containers/detection_engine/rules'; import { transformAlertToRuleAction } from '../../../../../common/detection_engine/transform_actions'; +import { RuleType } from '../../../../../common/detection_engine/types'; +import { isMlRule } from '../../../../../common/detection_engine/ml_helpers'; +import { NewRule } from '../../../../containers/detection_engine/rules'; import { AboutStepRule, @@ -25,7 +27,6 @@ import { AboutStepRuleJson, ActionsStepRuleJson, } from '../types'; -import { isMlRule } from '../helpers'; export const getTimeTypeValue = (time: string): { unit: string; value: number } => { const timeObj = { diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx index 50b76552ddc8f..710dd2cabeb65 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx @@ -10,10 +10,11 @@ import moment from 'moment'; import memoizeOne from 'memoize-one'; import { useLocation } from 'react-router-dom'; -import { RuleAlertAction } from '../../../../common/detection_engine/types'; +import { RuleAlertAction, RuleType } from '../../../../common/detection_engine/types'; +import { isMlRule } from '../../../../common/detection_engine/ml_helpers'; import { transformRuleToAlertAction } from '../../../../common/detection_engine/transform_actions'; import { Filter } from '../../../../../../../../src/plugins/data/public'; -import { Rule, RuleType } from '../../../containers/detection_engine/rules'; +import { Rule } from '../../../containers/detection_engine/rules'; import { FormData, FormHook, FormSchema } from '../../../shared_imports'; import { AboutStepRule, @@ -214,8 +215,6 @@ export const setFieldValue = ( } }); -export const isMlRule = (ruleType: RuleType) => ruleType === 'machine_learning'; - export const redirectToDetections = ( isSignalIndexExists: boolean | null, isAuthenticated: boolean | null, diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts index c1db24991c17c..1c366e6640b29 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts @@ -5,9 +5,8 @@ */ import { AlertAction } from '../../../../../../../plugins/alerting/common'; -import { RuleAlertAction } from '../../../../common/detection_engine/types'; +import { RuleAlertAction, RuleType } from '../../../../common/detection_engine/types'; import { Filter } from '../../../../../../../../src/plugins/data/common'; -import { RuleType } from '../../../containers/detection_engine/rules/types'; import { FieldValueQueryBar } from './components/query_bar'; import { FormData, FormHook } from '../../../shared_imports'; import { FieldValueTimeline } from './components/pick_timeline'; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts index ca0d133627210..a0458dc3a133d 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts @@ -19,12 +19,7 @@ import { isRuleStatusFindTypes, isRuleStatusSavedObjectType, } from '../../rules/types'; -import { - OutputRuleAlertRest, - ImportRuleAlertRest, - RuleAlertParamsRest, - RuleType, -} from '../../types'; +import { OutputRuleAlertRest, ImportRuleAlertRest, RuleAlertParamsRest } from '../../types'; import { createBulkErrorObject, BulkError, @@ -300,5 +295,3 @@ export const getTupleDuplicateErrorsAndUniqueRules = ( return [Array.from(errors.values()), Array.from(rulesAcc.values())]; }; - -export const isMlRule = (ruleType: RuleType) => ruleType === 'machine_learning'; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts index b5a01e3e5c6df..25e76f367037a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts @@ -8,6 +8,7 @@ import * as t from 'io-ts'; import { Either, left, fold } from 'fp-ts/lib/Either'; import { pipe } from 'fp-ts/lib/pipeable'; +import { isMlRule } from '../../../../../../common/detection_engine/ml_helpers'; import { dependentRulesSchema, RequiredRulesSchema, @@ -47,7 +48,7 @@ export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixe }; export const addMlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { - if (typeAndTimelineOnly.type === 'machine_learning') { + if (isMlRule(typeAndTimelineOnly.type)) { return [ t.exact(t.type({ anomaly_threshold: dependentRulesSchema.props.anomaly_threshold })), t.exact( diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts index 90c7d4a07ddf8..8d7360bae8eb9 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts @@ -16,9 +16,9 @@ import { } from '../../../../../../../../src/core/server'; import { ILicense } from '../../../../../../../plugins/licensing/server'; import { MINIMUM_ML_LICENSE } from '../../../../common/constants'; +import { RuleType } from '../../../../common/detection_engine/types'; +import { isMlRule } from '../../../../common/detection_engine/ml_helpers'; import { BadRequestError } from '../errors/bad_request_error'; -import { RuleType } from '../types'; -import { isMlRule } from './rules/utils'; export interface OutputError { message: string; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts index 68716bb4e3795..d8dacc7c64397 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts @@ -84,7 +84,7 @@ export interface IRuleStatusFindType { saved_objects: IRuleStatusSavedObject[]; } -export type RuleStatusString = 'succeeded' | 'failed' | 'going to run' | 'executing'; +export type RuleStatusString = 'succeeded' | 'failed' | 'going to run'; export interface HapiReadableStream extends Readable { hapi: { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts index 31b922e0067cd..6d7d7e93d7e6e 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -5,9 +5,15 @@ */ import { SignalSourceHit, SignalSearchResponse } from '../types'; -import { Logger } from 'kibana/server'; +import { + Logger, + SavedObject, + SavedObjectsFindResponse, +} from '../../../../../../../../../src/core/server'; import { loggingServiceMock } from '../../../../../../../../../src/core/server/mocks'; import { RuleTypeParams, OutputRuleAlertRest } from '../../types'; +import { IRuleStatusAttributes } from '../../rules/types'; +import { ruleStatusSavedObjectType } from '../../../../saved_objects'; export const sampleRuleAlertParams = ( maxSignals?: number | undefined, @@ -373,4 +379,34 @@ export const sampleRule = (): Partial => { }; }; +export const exampleRuleStatus: () => SavedObject = () => ({ + type: ruleStatusSavedObjectType, + id: '042e6d90-7069-11ea-af8b-0f8ae4fa817e', + attributes: { + alertId: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', + statusDate: '2020-03-27T22:55:59.517Z', + status: 'succeeded', + lastFailureAt: null, + lastSuccessAt: '2020-03-27T22:55:59.517Z', + lastFailureMessage: null, + lastSuccessMessage: 'succeeded', + gap: null, + bulkCreateTimeDurations: [], + searchAfterTimeDurations: [], + lastLookBackDate: null, + }, + references: [], + updated_at: '2020-03-27T22:55:59.577Z', + version: 'WzgyMiwxXQ==', +}); + +export const exampleFindRuleStatusResponse: ( + mockStatuses: Array> +) => SavedObjectsFindResponse = (mockStatuses = [exampleRuleStatus()]) => ({ + total: 1, + per_page: 6, + page: 1, + saved_objects: mockStatuses, +}); + export const mockLogger: Logger = loggingServiceMock.createLogger(); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/rule_status_saved_objects_client.mock.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/rule_status_saved_objects_client.mock.ts new file mode 100644 index 0000000000000..7528dc8b656ec --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/rule_status_saved_objects_client.mock.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RuleStatusSavedObjectsClient } from '../rule_status_saved_objects_client'; + +const createMockRuleStatusSavedObjectsClient = (): jest.Mocked => ({ + find: jest.fn(), + create: jest.fn(), + update: jest.fn(), + delete: jest.fn(), +}); + +export const ruleStatusSavedObjectsClientMock = { + create: createMockRuleStatusSavedObjectsClient, +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_current_status_saved_object.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_current_status_saved_object.ts deleted file mode 100644 index 1fee8bcd6c2f0..0000000000000 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_current_status_saved_object.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SavedObjectsFindResponse, SavedObject } from 'src/core/server'; - -import { AlertServices } from '../../../../../../../plugins/alerting/server'; -import { IRuleSavedAttributesSavedObjectAttributes } from '../rules/types'; -import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings'; - -interface CurrentStatusSavedObjectParams { - alertId: string; - services: AlertServices; - ruleStatusSavedObjects: SavedObjectsFindResponse; -} - -export const getCurrentStatusSavedObject = async ({ - alertId, - services, - ruleStatusSavedObjects, -}: CurrentStatusSavedObjectParams): Promise> => { - if (ruleStatusSavedObjects.saved_objects.length === 0) { - // create - const date = new Date().toISOString(); - const currentStatusSavedObject = await services.savedObjectsClient.create< - IRuleSavedAttributesSavedObjectAttributes - >(ruleStatusSavedObjectType, { - alertId, // do a search for this id. - statusDate: date, - status: 'going to run', - lastFailureAt: null, - lastSuccessAt: null, - lastFailureMessage: null, - lastSuccessMessage: null, - gap: null, - bulkCreateTimeDurations: [], - searchAfterTimeDurations: [], - lastLookBackDate: null, - }); - return currentStatusSavedObject; - } else { - // update 0th to executing. - const currentStatusSavedObject = ruleStatusSavedObjects.saved_objects[0]; - const sDate = new Date().toISOString(); - currentStatusSavedObject.attributes.status = 'going to run'; - currentStatusSavedObject.attributes.statusDate = sDate; - await services.savedObjectsClient.update( - ruleStatusSavedObjectType, - currentStatusSavedObject.id, - { - ...currentStatusSavedObject.attributes, - } - ); - return currentStatusSavedObject; - } -}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts new file mode 100644 index 0000000000000..913efbe04aa16 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObject } from 'src/core/server'; + +import { IRuleStatusAttributes } from '../rules/types'; +import { RuleStatusSavedObjectsClient } from './rule_status_saved_objects_client'; +import { getRuleStatusSavedObjects } from './get_rule_status_saved_objects'; + +interface RuleStatusParams { + alertId: string; + ruleStatusClient: RuleStatusSavedObjectsClient; +} + +export const createNewRuleStatus = async ({ + alertId, + ruleStatusClient, +}: RuleStatusParams): Promise> => { + const now = new Date().toISOString(); + return ruleStatusClient.create({ + alertId, + statusDate: now, + status: 'going to run', + lastFailureAt: null, + lastSuccessAt: null, + lastFailureMessage: null, + lastSuccessMessage: null, + gap: null, + bulkCreateTimeDurations: [], + searchAfterTimeDurations: [], + lastLookBackDate: null, + }); +}; + +export const getOrCreateRuleStatuses = async ({ + alertId, + ruleStatusClient, +}: RuleStatusParams): Promise>> => { + const ruleStatuses = await getRuleStatusSavedObjects({ + alertId, + ruleStatusClient, + }); + if (ruleStatuses.saved_objects.length > 0) { + return ruleStatuses.saved_objects; + } + const newStatus = await createNewRuleStatus({ alertId, ruleStatusClient }); + + return [newStatus]; +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts index 5a59d0413cfb9..828b4ea41096e 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts @@ -5,24 +5,21 @@ */ import { SavedObjectsFindResponse } from 'kibana/server'; -import { AlertServices } from '../../../../../../../plugins/alerting/server'; -import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings'; -import { IRuleSavedAttributesSavedObjectAttributes } from '../rules/types'; +import { IRuleStatusAttributes } from '../rules/types'; +import { MAX_RULE_STATUSES } from './rule_status_service'; +import { RuleStatusSavedObjectsClient } from './rule_status_saved_objects_client'; interface GetRuleStatusSavedObject { alertId: string; - services: AlertServices; + ruleStatusClient: RuleStatusSavedObjectsClient; } export const getRuleStatusSavedObjects = async ({ alertId, - services, -}: GetRuleStatusSavedObject): Promise> => { - return services.savedObjectsClient.find({ - type: ruleStatusSavedObjectType, - perPage: 6, // 0th element is current status, 1-5 is last 5 failures. + ruleStatusClient, +}: GetRuleStatusSavedObject): Promise> => { + return ruleStatusClient.find({ + perPage: MAX_RULE_STATUSES, sortField: 'statusDate', sortOrder: 'desc', search: `${alertId}`, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_messages.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_messages.test.ts new file mode 100644 index 0000000000000..8e4b5ce3c9924 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_messages.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { BuildRuleMessageFactoryParams, buildRuleMessageFactory } from './rule_messages'; + +describe('buildRuleMessageFactory', () => { + let factoryParams: BuildRuleMessageFactoryParams; + beforeEach(() => { + factoryParams = { + name: 'name', + id: 'id', + ruleId: 'ruleId', + index: 'index', + }; + }); + + it('appends rule attributes to the provided message', () => { + const buildMessage = buildRuleMessageFactory(factoryParams); + + const message = buildMessage('my message'); + expect(message).toEqual(expect.stringContaining('my message')); + expect(message).toEqual(expect.stringContaining('name: "name"')); + expect(message).toEqual(expect.stringContaining('id: "id"')); + expect(message).toEqual(expect.stringContaining('rule id: "ruleId"')); + expect(message).toEqual(expect.stringContaining('signals index: "index"')); + }); + + it('joins message parts with newlines', () => { + const buildMessage = buildRuleMessageFactory(factoryParams); + + const message = buildMessage('my message'); + const messageParts = message.split('\n'); + expect(messageParts).toContain('my message'); + expect(messageParts).toContain('name: "name"'); + expect(messageParts).toContain('id: "id"'); + expect(messageParts).toContain('rule id: "ruleId"'); + expect(messageParts).toContain('signals index: "index"'); + }); + + it('joins multiple arguments with newlines', () => { + const buildMessage = buildRuleMessageFactory(factoryParams); + + const message = buildMessage('my message', 'here is more'); + const messageParts = message.split('\n'); + expect(messageParts).toContain('my message'); + expect(messageParts).toContain('here is more'); + }); + + it('defaults the rule ID if not provided ', () => { + const buildMessage = buildRuleMessageFactory({ + ...factoryParams, + ruleId: undefined, + }); + + const message = buildMessage('my message', 'here is more'); + expect(message).toEqual(expect.stringContaining('rule id: "(unknown rule id)"')); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_messages.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_messages.ts new file mode 100644 index 0000000000000..d5f9d332bbcdd --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_messages.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export type BuildRuleMessage = (...messages: string[]) => string; +export interface BuildRuleMessageFactoryParams { + name: string; + id: string; + ruleId: string | null | undefined; + index: string; +} + +export const buildRuleMessageFactory = ({ + id, + ruleId, + index, + name, +}: BuildRuleMessageFactoryParams): BuildRuleMessage => (...messages) => + [ + ...messages, + `name: "${name}"`, + `id: "${id}"`, + `rule id: "${ruleId ?? '(unknown rule id)'}"`, + `signals index: "${index}"`, + ].join('\n'); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts new file mode 100644 index 0000000000000..11cbf67304409 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + SavedObjectsClientContract, + SavedObject, + SavedObjectsUpdateResponse, + SavedObjectsFindOptions, + SavedObjectsFindResponse, +} from '../../../../../../../../src/core/server'; +import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings'; +import { IRuleStatusAttributes } from '../rules/types'; + +export interface RuleStatusSavedObjectsClient { + find: ( + options?: Omit + ) => Promise>; + create: (attributes: IRuleStatusAttributes) => Promise>; + update: ( + id: string, + attributes: Partial + ) => Promise>; + delete: (id: string) => Promise<{}>; +} + +export const ruleStatusSavedObjectsClientFactory = ( + savedObjectsClient: SavedObjectsClientContract +): RuleStatusSavedObjectsClient => ({ + find: options => + savedObjectsClient.find({ ...options, type: ruleStatusSavedObjectType }), + create: attributes => savedObjectsClient.create(ruleStatusSavedObjectType, attributes), + update: (id, attributes) => savedObjectsClient.update(ruleStatusSavedObjectType, id, attributes), + delete: id => savedObjectsClient.delete(ruleStatusSavedObjectType, id), +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_status_service.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_status_service.test.ts new file mode 100644 index 0000000000000..ea9534710d418 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_status_service.test.ts @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ruleStatusSavedObjectsClientMock } from './__mocks__/rule_status_saved_objects_client.mock'; +import { + buildRuleStatusAttributes, + RuleStatusService, + ruleStatusServiceFactory, + MAX_RULE_STATUSES, +} from './rule_status_service'; +import { exampleRuleStatus, exampleFindRuleStatusResponse } from './__mocks__/es_results'; + +const expectIsoDateString = expect.stringMatching(/Z$/); +const buildStatuses = (n: number) => + Array(n) + .fill(exampleRuleStatus()) + .map((status, index) => ({ + ...status, + id: `status-index-${index}`, + })); + +describe('buildRuleStatusAttributes', () => { + it('generates a new date on each call', async () => { + const { statusDate } = buildRuleStatusAttributes('going to run'); + await new Promise(resolve => setTimeout(resolve, 10)); // ensure time has passed + const { statusDate: statusDate2 } = buildRuleStatusAttributes('going to run'); + + expect(statusDate).toEqual(expectIsoDateString); + expect(statusDate2).toEqual(expectIsoDateString); + expect(statusDate).not.toEqual(statusDate2); + }); + + it('returns a status and statusDate if "going to run"', () => { + const result = buildRuleStatusAttributes('going to run'); + expect(result).toEqual({ + status: 'going to run', + statusDate: expectIsoDateString, + }); + }); + + it('returns success fields if "success"', () => { + const result = buildRuleStatusAttributes('succeeded', 'success message'); + expect(result).toEqual({ + status: 'succeeded', + statusDate: expectIsoDateString, + lastSuccessAt: expectIsoDateString, + lastSuccessMessage: 'success message', + }); + + expect(result.statusDate).toEqual(result.lastSuccessAt); + }); + + it('returns failure fields if "failed"', () => { + const result = buildRuleStatusAttributes('failed', 'failure message'); + expect(result).toEqual({ + status: 'failed', + statusDate: expectIsoDateString, + lastFailureAt: expectIsoDateString, + lastFailureMessage: 'failure message', + }); + + expect(result.statusDate).toEqual(result.lastFailureAt); + }); +}); + +describe('ruleStatusService', () => { + let currentStatus: ReturnType; + let ruleStatusClient: ReturnType; + let service: RuleStatusService; + + beforeEach(async () => { + currentStatus = exampleRuleStatus(); + ruleStatusClient = ruleStatusSavedObjectsClientMock.create(); + ruleStatusClient.find.mockResolvedValue(exampleFindRuleStatusResponse([currentStatus])); + service = await ruleStatusServiceFactory({ alertId: 'mock-alert-id', ruleStatusClient }); + }); + + describe('goingToRun', () => { + it('updates the current status to "going to run"', async () => { + await service.goingToRun(); + + expect(ruleStatusClient.update).toHaveBeenCalledWith( + currentStatus.id, + expect.objectContaining({ + status: 'going to run', + statusDate: expectIsoDateString, + }) + ); + }); + }); + + describe('success', () => { + it('updates the current status to "succeeded"', async () => { + await service.success('hey, it worked'); + + expect(ruleStatusClient.update).toHaveBeenCalledWith( + currentStatus.id, + expect.objectContaining({ + status: 'succeeded', + statusDate: expectIsoDateString, + lastSuccessAt: expectIsoDateString, + lastSuccessMessage: 'hey, it worked', + }) + ); + }); + }); + + describe('error', () => { + beforeEach(() => { + // mock the creation of our new status + ruleStatusClient.create.mockResolvedValue(exampleRuleStatus()); + }); + + it('updates the current status to "failed"', async () => { + await service.error('oh no, it broke'); + + expect(ruleStatusClient.update).toHaveBeenCalledWith( + currentStatus.id, + expect.objectContaining({ + status: 'failed', + statusDate: expectIsoDateString, + lastFailureAt: expectIsoDateString, + lastFailureMessage: 'oh no, it broke', + }) + ); + }); + + it('does not delete statuses if we have less than the max number of statuses', async () => { + await service.error('oh no, it broke'); + + expect(ruleStatusClient.delete).not.toHaveBeenCalled(); + }); + + it('does not delete rule statuses when we just hit the limit', async () => { + // max - 1 in store, meaning our new error will put us at max + ruleStatusClient.find.mockResolvedValue( + exampleFindRuleStatusResponse(buildStatuses(MAX_RULE_STATUSES - 1)) + ); + service = await ruleStatusServiceFactory({ alertId: 'mock-alert-id', ruleStatusClient }); + + await service.error('oh no, it broke'); + + expect(ruleStatusClient.delete).not.toHaveBeenCalled(); + }); + + it('deletes stale rule status when we already have max statuses', async () => { + // max in store, meaning our new error will push one off the end + ruleStatusClient.find.mockResolvedValue( + exampleFindRuleStatusResponse(buildStatuses(MAX_RULE_STATUSES)) + ); + service = await ruleStatusServiceFactory({ alertId: 'mock-alert-id', ruleStatusClient }); + + await service.error('oh no, it broke'); + + expect(ruleStatusClient.delete).toHaveBeenCalledTimes(1); + // we should delete the 6th (index 5) + expect(ruleStatusClient.delete).toHaveBeenCalledWith('status-index-5'); + }); + + it('deletes any number of rule statuses in excess of the max', async () => { + // max + 1 in store, meaning our new error will put us two over + ruleStatusClient.find.mockResolvedValue( + exampleFindRuleStatusResponse(buildStatuses(MAX_RULE_STATUSES + 1)) + ); + service = await ruleStatusServiceFactory({ alertId: 'mock-alert-id', ruleStatusClient }); + + await service.error('oh no, it broke'); + + expect(ruleStatusClient.delete).toHaveBeenCalledTimes(2); + // we should delete the 6th (index 5) + expect(ruleStatusClient.delete).toHaveBeenCalledWith('status-index-5'); + // we should delete the 7th (index 6) + expect(ruleStatusClient.delete).toHaveBeenCalledWith('status-index-6'); + }); + + it('handles multiple error calls', async () => { + // max in store, meaning our new error will push one off the end + ruleStatusClient.find.mockResolvedValue( + exampleFindRuleStatusResponse(buildStatuses(MAX_RULE_STATUSES)) + ); + service = await ruleStatusServiceFactory({ alertId: 'mock-alert-id', ruleStatusClient }); + + await service.error('oh no, it broke'); + await service.error('oh no, it broke'); + + expect(ruleStatusClient.delete).toHaveBeenCalledTimes(2); + // we should delete the 6th (index 5) + expect(ruleStatusClient.delete).toHaveBeenCalledWith('status-index-5'); + expect(ruleStatusClient.delete).toHaveBeenCalledWith('status-index-5'); + }); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_status_service.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_status_service.ts new file mode 100644 index 0000000000000..5bfef134b0bae --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/rule_status_service.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { assertUnreachable } from '../../../utils/build_query'; +import { IRuleStatusAttributes, RuleStatusString } from '../rules/types'; +import { getOrCreateRuleStatuses } from './get_or_create_rule_statuses'; +import { RuleStatusSavedObjectsClient } from './rule_status_saved_objects_client'; + +// 1st is mutable status, followed by 5 most recent failures +export const MAX_RULE_STATUSES = 6; + +interface Attributes { + searchAfterTimeDurations?: string[]; + bulkCreateTimeDurations?: string[]; + lastLookBackDate?: string; + gap?: string; +} + +export interface RuleStatusService { + goingToRun: () => Promise; + success: (message: string, attributes?: Attributes) => Promise; + error: (message: string, attributes?: Attributes) => Promise; +} + +export const buildRuleStatusAttributes: ( + status: RuleStatusString, + message?: string, + attributes?: Attributes +) => Partial = (status, message, attributes = {}) => { + const now = new Date().toISOString(); + const baseAttributes: Partial = { + ...attributes, + status, + statusDate: now, + }; + + switch (status) { + case 'succeeded': { + return { + ...baseAttributes, + lastSuccessAt: now, + lastSuccessMessage: message, + }; + } + case 'failed': { + return { + ...baseAttributes, + lastFailureAt: now, + lastFailureMessage: message, + }; + } + case 'going to run': { + return baseAttributes; + } + } + + assertUnreachable(status); +}; + +export const ruleStatusServiceFactory = async ({ + alertId, + ruleStatusClient, +}: { + alertId: string; + ruleStatusClient: RuleStatusSavedObjectsClient; +}): Promise => { + return { + goingToRun: async () => { + const [currentStatus] = await getOrCreateRuleStatuses({ + alertId, + ruleStatusClient, + }); + + await ruleStatusClient.update(currentStatus.id, { + ...currentStatus.attributes, + ...buildRuleStatusAttributes('going to run'), + }); + }, + + success: async (message, attributes) => { + const [currentStatus] = await getOrCreateRuleStatuses({ + alertId, + ruleStatusClient, + }); + + await ruleStatusClient.update(currentStatus.id, { + ...currentStatus.attributes, + ...buildRuleStatusAttributes('succeeded', message, attributes), + }); + }, + + error: async (message, attributes) => { + const ruleStatuses = await getOrCreateRuleStatuses({ + alertId, + ruleStatusClient, + }); + const [currentStatus] = ruleStatuses; + + const failureAttributes = { + ...currentStatus.attributes, + ...buildRuleStatusAttributes('failed', message, attributes), + }; + + // We always update the newest status, so to 'persist' a failure we push a copy to the head of the list + await ruleStatusClient.update(currentStatus.id, failureAttributes); + const newStatus = await ruleStatusClient.create(failureAttributes); + + // drop oldest failures + const oldStatuses = [newStatus, ...ruleStatuses].slice(MAX_RULE_STATUSES); + await Promise.all(oldStatuses.map(status => ruleStatusClient.delete(status.id))); + }, + }; +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts index ab9def14bef65..de4ec68e8fc8a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -6,11 +6,14 @@ import { performance } from 'perf_hooks'; import { Logger } from 'src/core/server'; + import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE, NOTIFICATION_THROTTLE_RULE, } from '../../../../common/constants'; +import { isJobStarted, isMlRule } from '../../../../common/detection_engine/ml_helpers'; +import { SetupPlugins } from '../../../plugin'; import { buildEventsSearchQuery } from './build_events_query'; import { getInputIndex } from './get_input_output_index'; @@ -21,24 +24,24 @@ import { import { getFilter } from './get_filter'; import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types'; import { getGapBetweenRuns, makeFloatString } from './utils'; -import { writeSignalRuleExceptionToSavedObject } from './write_signal_rule_exception_to_saved_object'; import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; -import { writeGapErrorToSavedObject } from './write_gap_error_to_saved_object'; -import { getRuleStatusSavedObjects } from './get_rule_status_saved_objects'; -import { getCurrentStatusSavedObject } from './get_current_status_saved_object'; -import { writeCurrentStatusSucceeded } from './write_current_status_succeeded'; import { findMlSignals } from './find_ml_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; import { getSignalsCount } from '../notifications/get_signals_count'; import { scheduleNotificationActions } from '../notifications/schedule_notification_actions'; +import { ruleStatusServiceFactory } from './rule_status_service'; +import { buildRuleMessageFactory } from './rule_messages'; +import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; export const signalRulesAlertType = ({ logger, version, + ml, }: { logger: Logger; version: string; + ml: SetupPlugins['ml']; }): SignalRuleAlertTypeDefinition => { return { id: SIGNALS_ID, @@ -64,22 +67,15 @@ export const signalRulesAlertType = ({ to, type, } = params; + const ruleStatusClient = ruleStatusSavedObjectsClientFactory(services.savedObjectsClient); + const ruleStatusService = await ruleStatusServiceFactory({ + alertId, + ruleStatusClient, + }); const savedObject = await services.savedObjectsClient.get( 'alert', alertId ); - - const ruleStatusSavedObjects = await getRuleStatusSavedObjects({ - alertId, - services, - }); - - const currentStatusSavedObject = await getCurrentStatusSavedObject({ - alertId, - services, - ruleStatusSavedObjects, - }); - const { actions, name, @@ -92,23 +88,31 @@ export const signalRulesAlertType = ({ throttle, params: ruleParams, } = savedObject.attributes; - const updatedAt = savedObject.updated_at ?? ''; - - const gap = getGapBetweenRuns({ previousStartedAt, interval, from, to }); - await writeGapErrorToSavedObject({ - alertId, - logger, - ruleId: ruleId ?? '(unknown rule id)', - currentStatusSavedObject, - services, - gap, - ruleStatusSavedObjects, + const buildRuleMessage = buildRuleMessageFactory({ + id: alertId, + ruleId, name, + index: outputIndex, }); + logger.debug(buildRuleMessage('[+] Starting Signal Rule execution')); + await ruleStatusService.goingToRun(); + + const gap = getGapBetweenRuns({ previousStartedAt, interval, from, to }); + if (gap != null && gap.asMilliseconds() > 0) { + const gapString = gap.humanize(); + const gapMessage = buildRuleMessage( + `${gapString} (${gap.asMilliseconds()}ms) has passed since last rule execution, and signals may have been missed.`, + 'Consider increasing your look behind time or adding more Kibana instances.' + ); + logger.warn(gapMessage); + + await ruleStatusService.error(gapMessage, { gap: gapString }); + } + const searchAfterSize = Math.min(params.maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); - let creationSucceeded: SearchAfterAndBulkCreateReturnType = { + let result: SearchAfterAndBulkCreateReturnType = { success: false, bulkCreateTimes: [], searchAfterTimes: [], @@ -116,11 +120,34 @@ export const signalRulesAlertType = ({ }; try { - if (type === 'machine_learning') { + if (isMlRule(type)) { + if (ml == null) { + throw new Error('ML plugin unavailable during rule execution'); + } if (machineLearningJobId == null || anomalyThreshold == null) { throw new Error( - `Attempted to execute machine learning rule, but it is missing job id and/or anomaly threshold for rule id: "${ruleId}", name: "${name}", signals index: "${outputIndex}", job id: "${machineLearningJobId}", anomaly threshold: "${anomalyThreshold}"` + [ + 'Machine learning rule is missing job id and/or anomaly threshold:', + `job id: "${machineLearningJobId}"`, + `anomaly threshold: "${anomalyThreshold}"`, + ].join('\n') + ); + } + + const summaryJobs = await ml + .jobServiceProvider(ml.mlClient.callAsInternalUser) + .jobsSummary([machineLearningJobId]); + const jobSummary = summaryJobs.find(job => job.id === machineLearningJobId); + + if (jobSummary == null || !isJobStarted(jobSummary.jobState, jobSummary.datafeedState)) { + const errorMessage = buildRuleMessage( + 'Machine learning job is not started:', + `job id: "${machineLearningJobId}"`, + `job status: "${jobSummary?.jobState}"`, + `datafeed status: "${jobSummary?.datafeedState}"` ); + logger.warn(errorMessage); + await ruleStatusService.error(errorMessage); } const anomalyResults = await findMlSignals( @@ -130,12 +157,9 @@ export const signalRulesAlertType = ({ to, services.callCluster ); - const anomalyCount = anomalyResults.hits.hits.length; if (anomalyCount) { - logger.info( - `Found ${anomalyCount} signals from ML anomalies for signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", pushing signals to index "${outputIndex}"` - ); + logger.info(buildRuleMessage(`Found ${anomalyCount} signals from ML anomalies.`)); } const { success, bulkCreateDuration } = await bulkCreateMlSignals({ @@ -156,9 +180,9 @@ export const signalRulesAlertType = ({ enabled, tags, }); - creationSucceeded.success = success; + result.success = success; if (bulkCreateDuration) { - creationSucceeded.bulkCreateTimes.push(bulkCreateDuration); + result.bulkCreateTimes.push(bulkCreateDuration); } } else { const inputIndex = await getInputIndex(services, version, index); @@ -181,27 +205,21 @@ export const signalRulesAlertType = ({ searchAfterSortId: undefined, }); - logger.debug( - `Starting signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"` - ); - logger.debug( - `[+] Initial search call of signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"` - ); + logger.debug(buildRuleMessage('[+] Initial search call')); const start = performance.now(); const noReIndexResult = await services.callCluster('search', noReIndex); const end = performance.now(); - if (noReIndexResult.hits.total.value !== 0) { + const signalCount = noReIndexResult.hits.total.value; + if (signalCount !== 0) { logger.info( - `Found ${ - noReIndexResult.hits.total.value - } signals from the indexes of "[${inputIndex.join( - ', ' - )}]" using signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", pushing signals to index "${outputIndex}"` + buildRuleMessage( + `Found ${signalCount} signals from the indexes of "[${inputIndex.join(', ')}]"` + ) ); } - creationSucceeded = await searchAfterAndBulkCreate({ + result = await searchAfterAndBulkCreate({ someResult: noReIndexResult, ruleParams: params, services, @@ -222,10 +240,10 @@ export const signalRulesAlertType = ({ tags, throttle, }); - creationSucceeded.searchAfterTimes.push(makeFloatString(end - start)); + result.searchAfterTimes.push(makeFloatString(end - start)); } - if (creationSucceeded.success) { + if (result.success) { if (meta?.throttle === NOTIFICATION_THROTTLE_RULE && actions.length) { const notificationRuleParams = { ...ruleParams, @@ -242,9 +260,7 @@ export const signalRulesAlertType = ({ callCluster: services.callCluster, }); - logger.info( - `Found ${signalsCount} signals using signal rule name: "${notificationRuleParams.name}", id: "${notificationRuleParams.ruleId}", rule_id: "${notificationRuleParams.ruleId}" in "${notificationRuleParams.outputIndex}" index` - ); + logger.info(buildRuleMessage(`Found ${signalsCount} signals for notification.`)); if (signalsCount) { const alertInstance = services.alertInstanceFactory(alertId); @@ -257,44 +273,35 @@ export const signalRulesAlertType = ({ } } - logger.debug( - `Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"` - ); - await writeCurrentStatusSucceeded({ - services, - currentStatusSavedObject, - bulkCreateTimes: creationSucceeded.bulkCreateTimes, - searchAfterTimes: creationSucceeded.searchAfterTimes, - lastLookBackDate: creationSucceeded.lastLookBackDate?.toISOString() ?? null, + logger.debug(buildRuleMessage('[+] Signal Rule execution completed.')); + await ruleStatusService.success('succeeded', { + bulkCreateTimeDurations: result.bulkCreateTimes, + searchAfterTimeDurations: result.searchAfterTimes, + lastLookBackDate: result.lastLookBackDate?.toISOString(), }); } else { - await writeSignalRuleExceptionToSavedObject({ - name, - alertId, - currentStatusSavedObject, - logger, - message: `Bulk Indexing signals failed. Check logs for further details \nRule name: "${name}"\nid: "${alertId}"\nrule_id: "${ruleId}"\n`, - services, - ruleStatusSavedObjects, - ruleId: ruleId ?? '(unknown rule id)', - bulkCreateTimes: creationSucceeded.bulkCreateTimes, - searchAfterTimes: creationSucceeded.searchAfterTimes, - lastLookBackDate: creationSucceeded.lastLookBackDate?.toISOString() ?? null, + const errorMessage = buildRuleMessage( + 'Bulk Indexing of signals failed. Check logs for further details.' + ); + logger.error(errorMessage); + await ruleStatusService.error(errorMessage, { + bulkCreateTimeDurations: result.bulkCreateTimes, + searchAfterTimeDurations: result.searchAfterTimes, + lastLookBackDate: result.lastLookBackDate?.toISOString(), }); } - } catch (err) { - await writeSignalRuleExceptionToSavedObject({ - name, - alertId, - currentStatusSavedObject, - logger, - message: `Bulk Indexing signals failed. Check logs for further details \nRule name: "${name}"\nid: "${alertId}"\nrule_id: "${ruleId}"\n`, - services, - ruleStatusSavedObjects, - ruleId: ruleId ?? '(unknown rule id)', - bulkCreateTimes: creationSucceeded.bulkCreateTimes, - searchAfterTimes: creationSucceeded.searchAfterTimes, - lastLookBackDate: creationSucceeded.lastLookBackDate?.toISOString() ?? null, + } catch (error) { + const errorMessage = error.message ?? '(no error message given)'; + const message = buildRuleMessage( + 'An error occurred during rule execution:', + `message: "${errorMessage}"` + ); + + logger.error(message); + await ruleStatusService.error(message, { + bulkCreateTimeDurations: result.bulkCreateTimes, + searchAfterTimeDurations: result.searchAfterTimes, + lastLookBackDate: result.lastLookBackDate?.toISOString(), }); } }, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/write_current_status_succeeded.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/write_current_status_succeeded.ts deleted file mode 100644 index 50136790c3479..0000000000000 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/write_current_status_succeeded.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SavedObject } from 'src/core/server'; -import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings'; - -import { AlertServices } from '../../../../../../../plugins/alerting/server'; -import { IRuleSavedAttributesSavedObjectAttributes } from '../rules/types'; - -interface GetRuleStatusSavedObject { - services: AlertServices; - currentStatusSavedObject: SavedObject; - lastLookBackDate: string | null | undefined; - bulkCreateTimes: string[] | null | undefined; - searchAfterTimes: string[] | null | undefined; -} - -export const writeCurrentStatusSucceeded = async ({ - services, - currentStatusSavedObject, - lastLookBackDate, - bulkCreateTimes, - searchAfterTimes, -}: GetRuleStatusSavedObject): Promise => { - const sDate = new Date().toISOString(); - currentStatusSavedObject.attributes.status = 'succeeded'; - currentStatusSavedObject.attributes.statusDate = sDate; - currentStatusSavedObject.attributes.lastSuccessAt = sDate; - currentStatusSavedObject.attributes.lastSuccessMessage = 'succeeded'; - if (lastLookBackDate != null) { - currentStatusSavedObject.attributes.lastLookBackDate = lastLookBackDate; - } - if (bulkCreateTimes != null) { - currentStatusSavedObject.attributes.bulkCreateTimeDurations = bulkCreateTimes; - } - if (searchAfterTimes != null) { - currentStatusSavedObject.attributes.searchAfterTimeDurations = searchAfterTimes; - } - await services.savedObjectsClient.update(ruleStatusSavedObjectType, currentStatusSavedObject.id, { - ...currentStatusSavedObject.attributes, - }); -}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/write_gap_error_to_saved_object.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/write_gap_error_to_saved_object.ts deleted file mode 100644 index e47e5388527da..0000000000000 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/write_gap_error_to_saved_object.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import moment from 'moment'; -import { Logger, SavedObject, SavedObjectsFindResponse } from 'src/core/server'; - -import { AlertServices } from '../../../../../../../plugins/alerting/server'; -import { IRuleSavedAttributesSavedObjectAttributes } from '../rules/types'; -import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings'; - -interface WriteGapErrorToSavedObjectParams { - logger: Logger; - alertId: string; - ruleId: string; - currentStatusSavedObject: SavedObject; - ruleStatusSavedObjects: SavedObjectsFindResponse; - services: AlertServices; - gap: moment.Duration | null | undefined; - name: string; -} - -export const writeGapErrorToSavedObject = async ({ - alertId, - currentStatusSavedObject, - logger, - services, - ruleStatusSavedObjects, - ruleId, - gap, - name, -}: WriteGapErrorToSavedObjectParams): Promise => { - if (gap != null && gap.asMilliseconds() > 0) { - logger.warn( - `Signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}" has a time gap of ${gap.humanize()} (${gap.asMilliseconds()}ms), and could be missing signals within that time. Consider increasing your look behind time or adding more Kibana instances.` - ); - // write a failure status whenever we have a time gap - // this is a temporary solution until general activity - // monitoring is developed as a feature - const gapDate = new Date().toISOString(); - await services.savedObjectsClient.create(ruleStatusSavedObjectType, { - alertId, - statusDate: gapDate, - status: 'failed', - lastFailureAt: gapDate, - lastSuccessAt: currentStatusSavedObject.attributes.lastSuccessAt, - lastFailureMessage: `Signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}" has a time gap of ${gap.humanize()} (${gap.asMilliseconds()}ms), and could be missing signals within that time. Consider increasing your look behind time or adding more Kibana instances.`, - lastSuccessMessage: currentStatusSavedObject.attributes.lastSuccessMessage, - gap: gap.humanize(), - }); - - if (ruleStatusSavedObjects.saved_objects.length >= 6) { - // delete fifth status and prepare to insert a newer one. - const toDelete = ruleStatusSavedObjects.saved_objects.slice(5); - await toDelete.forEach(async item => - services.savedObjectsClient.delete(ruleStatusSavedObjectType, item.id) - ); - } - } -}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/write_signal_rule_exception_to_saved_object.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/write_signal_rule_exception_to_saved_object.ts deleted file mode 100644 index 2a14184859591..0000000000000 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/write_signal_rule_exception_to_saved_object.ts +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { Logger, SavedObject, SavedObjectsFindResponse } from 'src/core/server'; - -import { AlertServices } from '../../../../../../../plugins/alerting/server'; -import { IRuleSavedAttributesSavedObjectAttributes } from '../rules/types'; -import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings'; - -interface SignalRuleExceptionParams { - logger: Logger; - alertId: string; - ruleId: string; - currentStatusSavedObject: SavedObject; - ruleStatusSavedObjects: SavedObjectsFindResponse; - message: string; - services: AlertServices; - name: string; - lastLookBackDate?: string | null | undefined; - bulkCreateTimes?: string[] | null | undefined; - searchAfterTimes?: string[] | null | undefined; -} - -export const writeSignalRuleExceptionToSavedObject = async ({ - alertId, - currentStatusSavedObject, - logger, - message, - services, - ruleStatusSavedObjects, - ruleId, - name, - lastLookBackDate, - bulkCreateTimes, - searchAfterTimes, -}: SignalRuleExceptionParams): Promise => { - logger.error( - `Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}" message: ${message}` - ); - const sDate = new Date().toISOString(); - currentStatusSavedObject.attributes.status = 'failed'; - currentStatusSavedObject.attributes.statusDate = sDate; - currentStatusSavedObject.attributes.lastFailureAt = sDate; - currentStatusSavedObject.attributes.lastFailureMessage = message; - if (lastLookBackDate) { - currentStatusSavedObject.attributes.lastLookBackDate = lastLookBackDate; - } - if (bulkCreateTimes) { - currentStatusSavedObject.attributes.bulkCreateTimeDurations = bulkCreateTimes; - } - if (searchAfterTimes) { - currentStatusSavedObject.attributes.searchAfterTimeDurations = searchAfterTimes; - } - // current status is failing - await services.savedObjectsClient.update(ruleStatusSavedObjectType, currentStatusSavedObject.id, { - ...currentStatusSavedObject.attributes, - }); - // create new status for historical purposes - await services.savedObjectsClient.create(ruleStatusSavedObjectType, { - ...currentStatusSavedObject.attributes, - }); - - if (ruleStatusSavedObjects.saved_objects.length >= 6) { - // delete fifth status and prepare to insert a newer one. - const toDelete = ruleStatusSavedObjects.saved_objects.slice(5); - await toDelete.forEach(async item => - services.savedObjectsClient.delete(ruleStatusSavedObjectType, item.id) - ); - } -}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts index aae8763a7ea39..08b3f864314f9 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts @@ -8,7 +8,7 @@ import { CallAPIOptions } from '../../../../../../../src/core/server'; import { Filter } from '../../../../../../../src/plugins/data/server'; import { IRuleStatusAttributes } from './rules/types'; import { ListsDefaultArraySchema } from './routes/schemas/types/lists_default_array'; -import { RuleAlertAction } from '../../../common/detection_engine/types'; +import { RuleAlertAction, RuleType } from '../../../common/detection_engine/types'; export type PartialFilter = Partial; @@ -28,7 +28,6 @@ export interface ThreatParams { // TODO: Eventually this whole RuleAlertParams will be replaced with io-ts. For now we can slowly strangle it out and reduce duplicate types // We don't have the input types defined through io-ts just yet but as we being introducing types from there we will more and more remove // types and share them between input and output schema but have an input Rule Schema and an output Rule Schema. -export type RuleType = 'query' | 'saved_query' | 'machine_learning'; export interface RuleAlertParams { actions: RuleAlertAction[]; diff --git a/x-pack/legacy/plugins/siem/server/plugin.ts b/x-pack/legacy/plugins/siem/server/plugin.ts index 98631ea220a54..dcaf805c54f6e 100644 --- a/x-pack/legacy/plugins/siem/server/plugin.ts +++ b/x-pack/legacy/plugins/siem/server/plugin.ts @@ -18,6 +18,7 @@ import { } from '../../../../../src/core/server'; import { SecurityPluginSetup as SecuritySetup } from '../../../../plugins/security/server'; import { PluginSetupContract as FeaturesSetup } from '../../../../plugins/features/server'; +import { MlPluginSetup as MlSetup } from '../../../../plugins/ml/server'; import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from '../../../../plugins/encrypted_saved_objects/server'; import { SpacesPluginSetup as SpacesSetup } from '../../../../plugins/spaces/server'; import { PluginStartContract as ActionsStart } from '../../../../plugins/actions/server'; @@ -48,6 +49,7 @@ export interface SetupPlugins { licensing: LicensingPluginSetup; security?: SecuritySetup; spaces?: SpacesSetup; + ml?: MlSetup; } export interface StartPlugins { @@ -164,6 +166,7 @@ export class Plugin { const signalRuleType = signalRulesAlertType({ logger: this.logger, version: this.context.env.packageInfo.version, + ml: plugins.ml, }); const ruleNotificationType = rulesNotificationAlertType({ logger: this.logger, diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index 674c3886c12f8..7d3ef116e67ab 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -11,6 +11,7 @@ import { IScopedClusterClient, Logger, PluginInitializerContext, + ICustomClusterClient, } from 'kibana/server'; import { PluginsSetup, RouteInitialization } from './types'; import { PLUGIN_ID, PLUGIN_ICON } from '../common/constants/app'; @@ -49,7 +50,9 @@ declare module 'kibana/server' { } } -export type MlPluginSetup = SharedServices; +export interface MlPluginSetup extends SharedServices { + mlClient: ICustomClusterClient; +} export type MlPluginStart = void; export class MlServerPlugin implements Plugin { @@ -135,7 +138,10 @@ export class MlServerPlugin implements Plugin