Improvements for field mappings (opensearch-project#432) (opensearch-…

…project#440) * [FEATURE] Detector must have at least one alert set opensearch-project#288 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * [BUG] Create detector | Interval field can be empty opensearch-project#378 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Adjust styling for Finding details flyout opensearch-project#369 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * unit tests Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * detector unit tests Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * detector unit tests Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * detector unit tests Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * detector unit tests Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * detector unit tests Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * detector unit tests Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * detector unit tests Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * detector unit tests Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * detector unit tests Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * unit tests review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * unit tests review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * unit tests review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * unit tests review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * unit tests review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * unit tests review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * unit tests review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * unit tests review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Feature/update vertical domain opensearch-project#372 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Unit tests for public components opensearch-project#383 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Unit tests for public components opensearch-project#383 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Unit tests for public components opensearch-project#383 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Unit tests for public components opensearch-project#383 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Unit tests for public components opensearch-project#383 [BUG] Detector Edit | Custom rule are not selected on update rules opensearch-project#406 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Unit tests for public components opensearch-project#383 [BUG] Detector Edit | Custom rule are not selected on update rules opensearch-project#406 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * PR code review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * PR code review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * PR code review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * [FEATURE] Create detector | Make data source multi-select field opensearch-project#419 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * [FEATURE] Create detector | Make data source multi-select field opensearch-project#419 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * [FEATURE] Create detector | Make data source multi-select field opensearch-project#419 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * [FEATURE] Create detector | Make data source multi-select field opensearch-project#419 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * [FEATURE] Create detector | Make data source multi-select field opensearch-project#419 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * [FEATURE] Create detector | Make data source multi-select field opensearch-project#419 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * [FEATURE] Create detector | Make data source multi-select field opensearch-project#419 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * [FEATURE] Create detector | Make data source multi-select field opensearch-project#419 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * [FEATURE] Create detector | Make data source multi-select field opensearch-project#419 Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * unit tests fix Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Code review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Code review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Code review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Code review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Code review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * Code review Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> * snapshot fix Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> --------- Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> (cherry picked from commit b2994a1) Co-authored-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com> Signed-off-by: AWSHurneyt <hurneyt@amazon.com>
AWSHurneyt · Oct 12, 2023 · 10116b8 · 10116b8
1 parent a439d5b
commit 10116b8
Show file tree

Hide file tree

Showing 9 changed files with 206 additions and 159 deletions.
diff --git a/cypress/integration/1_detectors.spec.js b/cypress/integration/1_detectors.spec.js
@@ -20,14 +20,17 @@ const testMappings = {
 const cypressDNSRule = dns_rule_data.title;
 
 describe('Detectors', () => {
-  const indexName = 'cypress-test-dns';
+  const cypressIndexDns = 'cypress-index-dns';
+  const cypressIndexWindows = 'cypress-index-windows';
   const detectorName = 'test detector';
 
   before(() => {
     cy.cleanUpTests();
 
+    cy.createIndex(cypressIndexWindows, sample_index_settings);
+
     // Create test index
-    cy.createIndex(indexName, sample_index_settings).then(() =>
+    cy.createIndex(cypressIndexDns, sample_index_settings).then(() =>
       cy
         .request('POST', '_plugins/_security_analytics/rules/_search?prePackaged=true', {
           from: 0,
@@ -58,11 +61,6 @@ describe('Detectors', () => {
   });
 
   it('...should show mappings warning', () => {
-    const indexName = 'cypress-index-windows';
-    const dnsName = 'cypress-index-dns';
-    cy.createIndex(indexName, sample_index_settings);
-    cy.createIndex(dnsName, sample_dns_settings);
-
     // Locate Create detector button click to start
     cy.get('.euiButton').filter(':contains("Create detector")').click({ force: true });
 
@@ -71,23 +69,25 @@ describe('Detectors', () => {
       contains: 'Define detector',
     });
 
-    // Select our pre-seeded data source (check indexName)
+    // Select our pre-seeded data source (check cypressIndexDns)
     cy.get(`[data-test-subj="define-detector-select-data-source"]`)
       .find('input')
       .focus()
-      .realType(indexName);
+      .realType(cypressIndexDns);
 
     // Select threat detector type (Windows logs)
     cy.get(`input[id="dns"]`).click({ force: true });
 
-    // Select our pre-seeded data source (check indexName)
+    // Select our pre-seeded data source (check cypressIndexDns)
     cy.get(`[data-test-subj="define-detector-select-data-source"]`)
       .find('input')
       .focus()
-      .realType(dnsName)
+      .realType(cypressIndexWindows)
       .realPress('Enter');
 
-    cy.get('.euiCallOut').should('be.visible').contains('Detector configuration warning');
+    cy.get('.euiCallOut')
+      .should('be.visible')
+      .contains('The selected log sources contain different types of logs');
   });
 
   it('...can be created', () => {
@@ -102,11 +102,11 @@ describe('Detectors', () => {
     // Enter a name for the detector in the appropriate input
     cy.get(`input[placeholder="Enter a name for the detector."]`).focus().realType('test detector');
 
-    // Select our pre-seeded data source (check indexName)
+    // Select our pre-seeded data source (check cypressIndexDns)
     cy.get(`[data-test-subj="define-detector-select-data-source"]`)
       .find('input')
       .focus()
-      .realType(indexName);
+      .realType(cypressIndexDns);
 
     cy.intercept({
       pathname: '/_plugins/_security_analytics/rules/_search',
@@ -198,7 +198,7 @@ describe('Detectors', () => {
     cy.contains('Detector details');
     cy.contains(detectorName);
     cy.contains('dns');
-    cy.contains(indexName);
+    cy.contains(cypressIndexDns);
     cy.contains('Alert on test_trigger');
 
     // Create the detector
@@ -252,8 +252,7 @@ describe('Detectors', () => {
       .find('input')
       .ospClear()
       .focus()
-      .realType('.opensearch-notifications-config')
-      .realPress('Enter');
+      .realType(cypressIndexWindows);
 
     // Change detector scheduling
     cy.get(`[data-test-subj="detector-schedule-number-select"]`).ospClear().focus().realType('10');
@@ -271,7 +270,7 @@ describe('Detectors', () => {
     cy.contains('test detector edited');
     cy.contains('Every 10 hours');
     cy.contains('Edited description');
-    cy.contains('.opensearch-notifications-config');
+    cy.contains(cypressIndexWindows);
   });
 
   it('...rules can be edited', () => {

diff --git a/...r/components/ConfigureFieldMapping/components/RequiredFieldMapping/FieldMappingsTable.tsx b/...r/components/ConfigureFieldMapping/components/RequiredFieldMapping/FieldMappingsTable.tsx
@@ -86,7 +86,7 @@ export default class FieldMappingsTable<T extends MappingViewType> extends Compo
     const columns: EuiBasicTableColumn<FieldMappingsTableItem>[] = [
       {
         field: 'ruleFieldName',
-        name: 'Rule field name',
+        name: 'Detector field name',
         dataType: 'string',
         width: '25%',
         render: (ruleFieldName: string) => ruleFieldName || DEFAULT_EMPTY_DATA,
@@ -100,7 +100,7 @@ export default class FieldMappingsTable<T extends MappingViewType> extends Compo
       },
       {
         field: 'logFieldName',
-        name: 'Log field name',
+        name: 'Log source field name',
         dataType: 'string',
         width: '45%',
         render: (logFieldName: string, entry: FieldMappingsTableItem) => {

diff --git a/...ages/CreateDetector/components/ConfigureFieldMapping/containers/ConfigureFieldMapping.tsx b/...ages/CreateDetector/components/ConfigureFieldMapping/containers/ConfigureFieldMapping.tsx
@@ -180,6 +180,37 @@ export default class ConfigureFieldMapping extends Component<
 
         <EuiSpacer size={'m'} />
 
+        <EuiPanel>
+          <EuiAccordion
+            buttonContent={
+              <div data-test-subj="mapped-fields-btn">
+                <EuiTitle>
+                  <h4>{`Automatically mapped fields (${mappedRuleFields.length})`}</h4>
+                </EuiTitle>
+              </div>
+            }
+            buttonProps={{ style: { paddingLeft: '10px', paddingRight: '10px' } }}
+            id={'mappedFieldsAccordion'}
+            initialIsOpen={false}
+          >
+            <EuiHorizontalRule margin={'xs'} />
+            <FieldMappingsTable<MappingViewType.Edit>
+              {...this.props}
+              loading={loading}
+              ruleFields={mappedRuleFields}
+              indexFields={indexFieldOptions}
+              mappingProps={{
+                type: MappingViewType.Edit,
+                existingMappings,
+                invalidMappingFieldNames,
+                onMappingCreation: this.onMappingCreation,
+              }}
+            />
+          </EuiAccordion>
+        </EuiPanel>
+
+        <EuiSpacer size={'m'} />
+
         {unmappedRuleFields.length > 0 ? (
           <>
             {pendingCount > 0 ? (
@@ -227,34 +258,6 @@ export default class ConfigureFieldMapping extends Component<
           </>
         )}
 
-        <EuiPanel>
-          <EuiAccordion
-            buttonContent={
-              <div data-test-subj="mapped-fields-btn">
-                <EuiTitle>
-                  <h4>{`Default mapped fields (${mappedRuleFields.length})`}</h4>
-                </EuiTitle>
-              </div>
-            }
-            buttonProps={{ style: { paddingLeft: '10px', paddingRight: '10px' } }}
-            id={'mappedFieldsAccordion'}
-            initialIsOpen={false}
-          >
-            <EuiHorizontalRule margin={'xs'} />
-            <FieldMappingsTable<MappingViewType.Edit>
-              {...this.props}
-              loading={loading}
-              ruleFields={mappedRuleFields}
-              indexFields={indexFieldOptions}
-              mappingProps={{
-                type: MappingViewType.Edit,
-                existingMappings,
-                invalidMappingFieldNames,
-                onMappingCreation: this.onMappingCreation,
-              }}
-            />
-          </EuiAccordion>
-        </EuiPanel>
         <EuiSpacer size={'m'} />
       </div>
     );

diff --git a/...teDetector/components/DefineDetector/components/DetectorDataSource/DetectorDataSource.tsx b/...teDetector/components/DefineDetector/components/DetectorDataSource/DetectorDataSource.tsx
@@ -5,39 +5,54 @@
 
 import React, { Component } from 'react';
 import { ContentPanel } from '../../../../../../components/ContentPanel';
-import { EuiComboBox, EuiComboBoxOptionOption, EuiFormRow, EuiSpacer } from '@elastic/eui';
+import {
+  EuiComboBox,
+  EuiComboBoxOptionOption,
+  EuiFormRow,
+  EuiSpacer,
+  EuiCallOut,
+  EuiTextColor,
+} from '@elastic/eui';
 import { FormFieldHeader } from '../../../../../../components/FormFieldHeader/FormFieldHeader';
 import { IndexOption } from '../../../../../Detectors/models/interfaces';
 import { MIN_NUM_DATA_SOURCES } from '../../../../../Detectors/utils/constants';
 import IndexService from '../../../../../../services/IndexService';
 import { NotificationsStart } from 'opensearch-dashboards/public';
 import { errorNotificationToast } from '../../../../../../utils/helpers';
+import _ from 'lodash';
+import { FieldMappingService } from '../../../../../../services';
 
 interface DetectorDataSourceProps {
   detectorIndices: string[];
   indexService: IndexService;
+  filedMappingService: FieldMappingService;
   isEdit: boolean;
   onDetectorInputIndicesChange: (selectedOptions: EuiComboBoxOptionOption<string>[]) => void;
   notifications: NotificationsStart;
+  detector_type: string;
 }
 
 interface DetectorDataSourceState {
   loading: boolean;
   fieldTouched: boolean;
   indexOptions: IndexOption[];
   errorMessage?: string;
+  message: string[];
 }
 
 export default class DetectorDataSource extends Component<
   DetectorDataSourceProps,
   DetectorDataSourceState
 > {
+  private indicesMappings: any = {};
+
   constructor(props: DetectorDataSourceProps) {
     super(props);
     this.state = {
       loading: true,
       fieldTouched: props.isEdit,
       indexOptions: [],
+      message: [],
     };
   }
 
@@ -82,17 +97,75 @@ export default class DetectorDataSource extends Component<
     this.onSelectionChange(parsedOptions);
   };
 
-  onSelectionChange = (options: EuiComboBoxOptionOption<string>[]) => {
+  onSelectionChange = async (options: EuiComboBoxOptionOption<string>[]) => {
+    const allIndices = _.map(options, 'label');
+    for (let indexName in this.indicesMappings) {
+      if (allIndices.indexOf(indexName) === -1) {
+        // cleanup removed indexes
+        delete this.indicesMappings[indexName];
+      }
+    }
+
+    for (const indexName of allIndices) {
+      if (!this.indicesMappings[indexName]) {
+        const detectorType = this.props.detector_type.toLowerCase();
+        const result = await this.props.filedMappingService.getMappingsView(
+          indexName,
+          detectorType
+        );
+        result.ok && (this.indicesMappings[indexName] = result.response.unmapped_field_aliases);
+      }
+    }
+
+    if (!_.isEmpty(this.indicesMappings)) {
+      let firstMapping: string[] = [];
+      let firstMatchMappingIndex: string = '';
+      let message: string[] = [];
+      for (let indexName in this.indicesMappings) {
+        if (this.indicesMappings.hasOwnProperty(indexName)) {
+          if (!firstMapping.length) firstMapping = this.indicesMappings[indexName];
+          !firstMatchMappingIndex.length && (firstMatchMappingIndex = indexName);
+          if (!_.isEqual(firstMapping, this.indicesMappings[indexName])) {
+            message = [
+              `We recommend creating separate detectors for each of the following log sources:`,
+              firstMatchMappingIndex,
+              indexName,
+            ];
+            break;
+          }
+        }
+      }
+
+      this.setState({ message });
+    }
+
     this.props.onDetectorInputIndicesChange(options);
   };
 
   render() {
     const { detectorIndices } = this.props;
-    const { loading, fieldTouched, indexOptions, errorMessage } = this.state;
+    const { loading, fieldTouched, indexOptions, errorMessage, message } = this.state;
     const isInvalid = fieldTouched && detectorIndices.length < MIN_NUM_DATA_SOURCES;
     return (
       <ContentPanel title={'Data source'} titleSize={'m'}>
         <EuiSpacer size={'m'} />
+        {message.length ? (
+          <>
+            <EuiCallOut
+              title="The selected log sources contain different types of logs"
+              color="warning"
+            >
+              {message.map((messageItem: string, index: number) => (
+                <EuiTextColor color={'default'} key={`callout-message-part-${index}`}>
+                  {index === 0 ? '' : 'ㅤ•ㅤ'}
+                  {messageItem}
+                  <br />
+                </EuiTextColor>
+              ))}
+            </EuiCallOut>
+            <EuiSpacer size={'m'} />
+          </>
+        ) : null}
         <EuiFormRow
           label={
             <FormFieldHeader headerTitle={'Select or input source indexes or index patterns'} />