ML Rule Suppression UI Improvements

x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx

@@ -6,6 +6,7 @@
  */
 
 import type { MlSummaryJob } from '@kbn/ml-plugin/public';
+import { isSecurityJob } from '../../../../../common/machine_learning/is_security_job';
 import type {
   AugmentedSecurityJobFields,
   Module,
@@ -111,13 +112,11 @@ export const getInstalledJobs = (
   moduleJobs: SecurityJob[],
   compatibleModuleIds: string[]
 ): SecurityJob[] =>
-  jobSummaryData
-    .filter(({ groups }) => groups.includes('siem') || groups.includes('security'))
-    .map<SecurityJob>((jobSummary) => ({
-      ...jobSummary,
-      ...getAugmentedFields(jobSummary.id, moduleJobs, compatibleModuleIds),
-      isInstalled: true,
-    }));
+  jobSummaryData.filter(isSecurityJob).map((jobSummary) => ({
+    ...jobSummary,
+    ...getAugmentedFields(jobSummary.id, moduleJobs, compatibleModuleIds),
+    isInstalled: true,
+  }));
 
 /**
  * Combines installed jobs + moduleSecurityJobs that don't overlap and sorts by name asc

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx

@@ -35,7 +35,7 @@ import type { DataViewBase } from '@kbn/es-query';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { useSetFieldValueWithCallback } from '../../../../common/utils/use_set_field_value_cb';
 import { useRuleFromTimeline } from '../../../../detections/containers/detection_engine/rules/use_rule_from_timeline';
-import { isMlRule } from '../../../../../common/machine_learning/helpers';
+import { isJobStarted, isMlRule } from '../../../../../common/machine_learning/helpers';
 import { hasMlAdminPermissions } from '../../../../../common/machine_learning/has_ml_admin_permissions';
 import { hasMlLicense } from '../../../../../common/machine_learning/has_ml_license';
 import { useMlCapabilities } from '../../../../common/components/ml/hooks/use_ml_capabilities';
@@ -93,6 +93,7 @@ import { NewTermsFields } from '../new_terms_fields';
 import { ScheduleItem } from '../../../rule_creation/components/schedule_item_form';
 import { RequiredFields } from '../../../rule_creation/components/required_fields';
 import { DocLink } from '../../../../common/components/links_to_docs/doc_link';
+import { useInstalledSecurityJobs } from '../../../../common/components/ml/hooks/use_installed_security_jobs';
 import { defaultCustomQuery } from '../../../../detections/pages/detection_engine/rules/utils';
 import { MultiSelectFieldsAutocomplete } from '../multi_select_fields';
 import { useLicense } from '../../../../common/hooks/use_license';
@@ -103,6 +104,7 @@ import { useUpsellingMessage } from '../../../../common/hooks/use_upselling';
 import { useAllEsqlRuleFields } from '../../hooks';
 import { useAlertSuppression } from '../../../rule_management/logic/use_alert_suppression';
 import { RelatedIntegrations } from '../../../rule_creation/components/related_integrations';
+import { useRuleFields } from '../../../rule_management/logic/use_rule_fields';
 
 const CommonUseField = getUseField({ component: Field });
 
@@ -167,41 +169,58 @@ const IntendedRuleTypeEuiFormRow = styled(RuleTypeEuiFormRow)`
 
 // eslint-disable-next-line complexity
 const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
-  isLoading,
-  isUpdateView = false,
-  kibanaDataViews,
-  indicesConfig,
-  threatIndicesConfig,
+  browserFields,
+  dataSourceType,
   defaultSavedQuery,
+  enableThresholdSuppression,
   form,
-  optionsSelected,
-  setOptionsSelected,
+  groupByFields,
+  index,
   indexPattern,
+  indicesConfig,
   isIndexPatternLoading,
-  browserFields,
+  isLoading,
   isQueryBarValid,
+  isUpdateView = false,
+  kibanaDataViews,
+  optionsSelected,
+  queryBarSavedId,
+  queryBarTitle,
+  ruleType,
   setIsQueryBarValid,
   setIsThreatQueryBarValid,
-  ruleType,
-  index,
-  threatIndex,
-  groupByFields,
-  dataSourceType,
+  setOptionsSelected,
   shouldLoadQueryDynamically,
-  queryBarTitle,
-  queryBarSavedId,
+  threatIndex,
+  threatIndicesConfig,
   thresholdFields,
-  enableThresholdSuppression,
 }) => {
   const queryClient = useQueryClient();
 
   const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(ruleType);
-  const mlCapabilities = useMlCapabilities();
   const [openTimelineSearch, setOpenTimelineSearch] = useState(false);
   const [indexModified, setIndexModified] = useState(false);
   const [threatIndexModified, setThreatIndexModified] = useState(false);
   const license = useLicense();
 
+  const mlCapabilities = useMlCapabilities();
+  const installedMlJobs = useInstalledSecurityJobs();
+  const [{ machineLearningJobId }] = useFormData<DefineStepRule>({
+    form,
+    watch: ['machineLearningJobId'],
+  });
+  const ruleMlJobs = installedMlJobs.jobs.filter((job) => machineLearningJobId.includes(job.id));
+  const numberOfRuleMlJobsStarted = ruleMlJobs.filter((job) =>
+    isJobStarted(job.jobState, job.datafeedState)
+  ).length;
+  const noMlJobsStarted = numberOfRuleMlJobsStarted === 0;
+  const someMlJobsStarted = !noMlJobsStarted && numberOfRuleMlJobsStarted !== ruleMlJobs.length;
+  const { loading: mlFieldsLoading, fields: mlFields } = useRuleFields({ machineLearningJobId });
+  const mlSuppressionFields = useMemo(
+    () => getTermsAggregationFields(mlFields as BrowserField[]),
+    [mlFields]
+  );
+
   const esqlQueryRef = useRef<DefineStepRule['queryBar'] | undefined>(undefined);
 
   const isAlertSuppressionLicenseValid = license.isAtLeast(MINIMUM_LICENSE_FOR_SUPPRESSION);
@@ -1068,22 +1087,35 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
                 </EuiText>
               }
             >
-              <UseField
-                path="groupByFields"
-                component={MultiSelectFieldsAutocomplete}
-                componentProps={{
-                  browserFields: isEsqlRule(ruleType)
-                    ? esqlSuppressionFields
-                    : termsAggregationFields,
-                  isDisabled:
-                    !isAlertSuppressionLicenseValid ||
-                    areSuppressionFieldsDisabledBySequence ||
-                    isEsqlSuppressionLoading,
-                  disabledText: areSuppressionFieldsDisabledBySequence
-                    ? i18n.EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP
-                    : alertSuppressionUpsellingMessage,
-                }}
-              />
+              <>
+                <UseField
+                  path="groupByFields"
+                  component={MultiSelectFieldsAutocomplete}
+                  componentProps={{
+                    browserFields: isEsqlRule(ruleType)
+                      ? esqlSuppressionFields
+                      : isMlRule(ruleType)
+                      ? mlSuppressionFields
+                      : termsAggregationFields,
+                    isDisabled:
+                      !isAlertSuppressionLicenseValid ||
+                      areSuppressionFieldsDisabledBySequence ||
+                      isEsqlSuppressionLoading ||
+                      mlFieldsLoading ||
+                      noMlJobsStarted,
+                    disabledText: areSuppressionFieldsDisabledBySequence
+                      ? i18n.EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP
+                      : noMlJobsStarted
+                      ? i18n.MACHINE_LEARNING_SUPPRESSION_DISABLED_LABEL
+                      : alertSuppressionUpsellingMessage,
+                  }}
+                />
+                {someMlJobsStarted && (
+                  <EuiText size="xs" color="warning">
+                    {i18n.MACHINE_LEARNING_SUPPRESSION_INCOMPLETE_LABEL}
+                  </EuiText>
+                )}
+              </>
             </RuleTypeEuiFormRow>
 
             <IntendedRuleTypeEuiFormRow

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx

@@ -234,6 +234,21 @@ export const EQL_SEQUENCE_SUPPRESSION_GROUPBY_VALIDATION_TEXT = i18n.translate(
   }
 );
 
+export const MACHINE_LEARNING_SUPPRESSION_DISABLED_LABEL = i18n.translate(
+  'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.machineLearningSuppressionDisabledLabel',
+  {
+    defaultMessage: 'Machine Learning jobs must be running to enable alert suppression.',
+  }
+);
+
+export const MACHINE_LEARNING_SUPPRESSION_INCOMPLETE_LABEL = i18n.translate(
+  'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.machineLearningSuppressionIncompleteLabel',
+  {
+    defaultMessage:
+      'This list of fields may be incomplete as some Machine Learning jobs are not running.',
+  }
+);
+
 export const GROUP_BY_TECH_PREVIEW_LABEL_APPEND = i18n.translate(
   'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.groupByFieldsTechPreviewLabelAppend',
   {

x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_fields.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { DataViewFieldBase } from '@kbn/es-query';
+
+import { useRuleIndices } from './use_rule_indices';
+import { useFetchIndex } from '../../../common/containers/source';
+
+interface UseRuleFieldParams {
+  machineLearningJobId?: string[];
+  indexPattern?: string[];
+}
+
+interface UseRuleFieldsReturn {
+  loading: boolean;
+  fields: DataViewFieldBase[];
+}
+
+export const useRuleFields = ({
+  machineLearningJobId,
+  indexPattern,
+}: UseRuleFieldParams): UseRuleFieldsReturn => {
+  const { ruleIndices } = useRuleIndices(machineLearningJobId, indexPattern);
+  const [
+    loading,
+    {
+      indexPatterns: { fields },
+    },
+  ] = useFetchIndex(ruleIndices);
+
+  return { loading, fields };
+};

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts

@@ -87,7 +87,7 @@ export default ({ getService }: FtrProviderContext) => {
   };
 
   // FLAKY: https://github.com/elastic/kibana/issues/171426
-  describe.skip('@ess @serverless @serverlessQA Machine learning type rules', () => {
+  describe('@ess @serverless @serverlessQA Machine learning type rules', () => {
     before(async () => {
       // Order is critical here: auditbeat data must be loaded before attempting to start the ML job,
       // as the job looks for certain indices on start
@@ -110,7 +110,7 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     // First test creates a real rule - remaining tests use preview API
-    it('should create 1 alert from ML rule when record meets anomaly_threshold', async () => {
+    it.only('should create 1 alert from ML rule when record meets anomaly_threshold', async () => {
       const createdRule = await createRule(supertest, log, rule);
       const alerts = await getAlerts(supertest, log, es, createdRule);
       expect(alerts.hits.hits.length).toBe(1);

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/machine_learning/machine_learning_setup.ts

@@ -6,6 +6,7 @@
  */
 
 import type SuperTest from 'supertest';
+import { ML_GROUP_ID } from '@kbn/security-solution-plugin/common/constants';
 import { getCommonRequestHeader } from '../../../../../functional/services/ml/common_api';
 
 export const executeSetupModuleRequest = async ({
@@ -22,7 +23,7 @@ export const executeSetupModuleRequest = async ({
     .set(getCommonRequestHeader('1'))
     .send({
       prefix: '',
-      groups: ['auditbeat'],
+      groups: [ML_GROUP_ID],
       indexPatternName: 'auditbeat-*',
       startDatafeed: false,
       useDedicatedIndex: true,