Skip to content

Commit

Permalink
Improvements for field mappings (#432)
Browse files Browse the repository at this point in the history
* [FEATURE] Detector must have at least one alert set #288

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [BUG] Create detector | Interval field can be empty #378

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Adjust styling for Finding details flyout #369

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* unit tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* detector unit tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* detector unit tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* detector unit tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* detector unit tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* detector unit tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* detector unit tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* detector unit tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* detector unit tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* detector unit tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* unit tests review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* unit tests review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* unit tests review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* unit tests review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* unit tests review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* unit tests review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* unit tests review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* unit tests review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Feature/update vertical domain #372

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Unit tests for public components #383

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Unit tests for public components #383

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Unit tests for public components #383

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Unit tests for public components #383

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Unit tests for public components #383
[BUG] Detector Edit | Custom rule are not selected on update rules #406

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Unit tests for public components #383
[BUG] Detector Edit | Custom rule are not selected on update rules #406

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* PR code review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* PR code review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* PR code review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [FEATURE] Create detector | Make data source multi-select field #419

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [FEATURE] Create detector | Make data source multi-select field #419

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [FEATURE] Create detector | Make data source multi-select field #419

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [FEATURE] Create detector | Make data source multi-select field #419

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [FEATURE] Create detector | Make data source multi-select field #419

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [FEATURE] Create detector | Make data source multi-select field #419

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [FEATURE] Create detector | Make data source multi-select field #419

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [FEATURE] Create detector | Make data source multi-select field #419

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [FEATURE] Create detector | Make data source multi-select field #419

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* unit tests fix

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Code review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Code review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Code review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Code review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Code review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Code review

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* snapshot fix

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

---------

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>
(cherry picked from commit b2994a1)
  • Loading branch information
jovancvetkovic3006 authored and github-actions[bot] committed Feb 22, 2023
1 parent ec2e438 commit b38a223
Show file tree
Hide file tree
Showing 9 changed files with 206 additions and 159 deletions.
35 changes: 17 additions & 18 deletions cypress/integration/1_detectors.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,17 @@ const testMappings = {
const cypressDNSRule = dns_rule_data.title;

describe('Detectors', () => {
const indexName = 'cypress-test-dns';
const cypressIndexDns = 'cypress-index-dns';
const cypressIndexWindows = 'cypress-index-windows';
const detectorName = 'test detector';

before(() => {
cy.cleanUpTests();

cy.createIndex(cypressIndexWindows, sample_index_settings);

// Create test index
cy.createIndex(indexName, sample_index_settings).then(() =>
cy.createIndex(cypressIndexDns, sample_index_settings).then(() =>
cy
.request('POST', '_plugins/_security_analytics/rules/_search?prePackaged=true', {
from: 0,
Expand Down Expand Up @@ -58,11 +61,6 @@ describe('Detectors', () => {
});

it('...should show mappings warning', () => {
const indexName = 'cypress-index-windows';
const dnsName = 'cypress-index-dns';
cy.createIndex(indexName, sample_index_settings);
cy.createIndex(dnsName, sample_dns_settings);

// Locate Create detector button click to start
cy.get('.euiButton').filter(':contains("Create detector")').click({ force: true });

Expand All @@ -71,23 +69,25 @@ describe('Detectors', () => {
contains: 'Define detector',
});

// Select our pre-seeded data source (check indexName)
// Select our pre-seeded data source (check cypressIndexDns)
cy.get(`[data-test-subj="define-detector-select-data-source"]`)
.find('input')
.focus()
.realType(indexName);
.realType(cypressIndexDns);

// Select threat detector type (Windows logs)
cy.get(`input[id="dns"]`).click({ force: true });

// Select our pre-seeded data source (check indexName)
// Select our pre-seeded data source (check cypressIndexDns)
cy.get(`[data-test-subj="define-detector-select-data-source"]`)
.find('input')
.focus()
.realType(dnsName)
.realType(cypressIndexWindows)
.realPress('Enter');

cy.get('.euiCallOut').should('be.visible').contains('Detector configuration warning');
cy.get('.euiCallOut')
.should('be.visible')
.contains('The selected log sources contain different types of logs');
});

it('...can be created', () => {
Expand All @@ -102,11 +102,11 @@ describe('Detectors', () => {
// Enter a name for the detector in the appropriate input
cy.get(`input[placeholder="Enter a name for the detector."]`).focus().realType('test detector');

// Select our pre-seeded data source (check indexName)
// Select our pre-seeded data source (check cypressIndexDns)
cy.get(`[data-test-subj="define-detector-select-data-source"]`)
.find('input')
.focus()
.realType(indexName);
.realType(cypressIndexDns);

cy.intercept({
pathname: '/_plugins/_security_analytics/rules/_search',
Expand Down Expand Up @@ -198,7 +198,7 @@ describe('Detectors', () => {
cy.contains('Detector details');
cy.contains(detectorName);
cy.contains('dns');
cy.contains(indexName);
cy.contains(cypressIndexDns);
cy.contains('Alert on test_trigger');

// Create the detector
Expand Down Expand Up @@ -252,8 +252,7 @@ describe('Detectors', () => {
.find('input')
.ospClear()
.focus()
.realType('.opensearch-notifications-config')
.realPress('Enter');
.realType(cypressIndexWindows);

// Change detector scheduling
cy.get(`[data-test-subj="detector-schedule-number-select"]`).ospClear().focus().realType('10');
Expand All @@ -271,7 +270,7 @@ describe('Detectors', () => {
cy.contains('test detector edited');
cy.contains('Every 10 hours');
cy.contains('Edited description');
cy.contains('.opensearch-notifications-config');
cy.contains(cypressIndexWindows);
});

it('...rules can be edited', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ export default class FieldMappingsTable<T extends MappingViewType> extends Compo
const columns: EuiBasicTableColumn<FieldMappingsTableItem>[] = [
{
field: 'ruleFieldName',
name: 'Rule field name',
name: 'Detector field name',
dataType: 'string',
width: '25%',
render: (ruleFieldName: string) => ruleFieldName || DEFAULT_EMPTY_DATA,
Expand All @@ -100,7 +100,7 @@ export default class FieldMappingsTable<T extends MappingViewType> extends Compo
},
{
field: 'logFieldName',
name: 'Log field name',
name: 'Log source field name',
dataType: 'string',
width: '45%',
render: (logFieldName: string, entry: FieldMappingsTableItem) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,37 @@ export default class ConfigureFieldMapping extends Component<

<EuiSpacer size={'m'} />

<EuiPanel>
<EuiAccordion
buttonContent={
<div data-test-subj="mapped-fields-btn">
<EuiTitle>
<h4>{`Automatically mapped fields (${mappedRuleFields.length})`}</h4>
</EuiTitle>
</div>
}
buttonProps={{ style: { paddingLeft: '10px', paddingRight: '10px' } }}
id={'mappedFieldsAccordion'}
initialIsOpen={false}
>
<EuiHorizontalRule margin={'xs'} />
<FieldMappingsTable<MappingViewType.Edit>
{...this.props}
loading={loading}
ruleFields={mappedRuleFields}
indexFields={indexFieldOptions}
mappingProps={{
type: MappingViewType.Edit,
existingMappings,
invalidMappingFieldNames,
onMappingCreation: this.onMappingCreation,
}}
/>
</EuiAccordion>
</EuiPanel>

<EuiSpacer size={'m'} />

{unmappedRuleFields.length > 0 ? (
<>
{pendingCount > 0 ? (
Expand Down Expand Up @@ -227,34 +258,6 @@ export default class ConfigureFieldMapping extends Component<
</>
)}

<EuiPanel>
<EuiAccordion
buttonContent={
<div data-test-subj="mapped-fields-btn">
<EuiTitle>
<h4>{`Default mapped fields (${mappedRuleFields.length})`}</h4>
</EuiTitle>
</div>
}
buttonProps={{ style: { paddingLeft: '10px', paddingRight: '10px' } }}
id={'mappedFieldsAccordion'}
initialIsOpen={false}
>
<EuiHorizontalRule margin={'xs'} />
<FieldMappingsTable<MappingViewType.Edit>
{...this.props}
loading={loading}
ruleFields={mappedRuleFields}
indexFields={indexFieldOptions}
mappingProps={{
type: MappingViewType.Edit,
existingMappings,
invalidMappingFieldNames,
onMappingCreation: this.onMappingCreation,
}}
/>
</EuiAccordion>
</EuiPanel>
<EuiSpacer size={'m'} />
</div>
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,39 +5,54 @@

import React, { Component } from 'react';
import { ContentPanel } from '../../../../../../components/ContentPanel';
import { EuiComboBox, EuiComboBoxOptionOption, EuiFormRow, EuiSpacer } from '@elastic/eui';
import {
EuiComboBox,
EuiComboBoxOptionOption,
EuiFormRow,
EuiSpacer,
EuiCallOut,
EuiTextColor,
} from '@elastic/eui';
import { FormFieldHeader } from '../../../../../../components/FormFieldHeader/FormFieldHeader';
import { IndexOption } from '../../../../../Detectors/models/interfaces';
import { MIN_NUM_DATA_SOURCES } from '../../../../../Detectors/utils/constants';
import IndexService from '../../../../../../services/IndexService';
import { NotificationsStart } from 'opensearch-dashboards/public';
import { errorNotificationToast } from '../../../../../../utils/helpers';
import _ from 'lodash';
import { FieldMappingService } from '../../../../../../services';

interface DetectorDataSourceProps {
detectorIndices: string[];
indexService: IndexService;
filedMappingService: FieldMappingService;
isEdit: boolean;
onDetectorInputIndicesChange: (selectedOptions: EuiComboBoxOptionOption<string>[]) => void;
notifications: NotificationsStart;
detector_type: string;
}

interface DetectorDataSourceState {
loading: boolean;
fieldTouched: boolean;
indexOptions: IndexOption[];
errorMessage?: string;
message: string[];
}

export default class DetectorDataSource extends Component<
DetectorDataSourceProps,
DetectorDataSourceState
> {
private indicesMappings: any = {};

constructor(props: DetectorDataSourceProps) {
super(props);
this.state = {
loading: true,
fieldTouched: props.isEdit,
indexOptions: [],
message: [],
};
}

Expand Down Expand Up @@ -82,17 +97,75 @@ export default class DetectorDataSource extends Component<
this.onSelectionChange(parsedOptions);
};

onSelectionChange = (options: EuiComboBoxOptionOption<string>[]) => {
onSelectionChange = async (options: EuiComboBoxOptionOption<string>[]) => {
const allIndices = _.map(options, 'label');
for (let indexName in this.indicesMappings) {
if (allIndices.indexOf(indexName) === -1) {
// cleanup removed indexes
delete this.indicesMappings[indexName];
}
}

for (const indexName of allIndices) {
if (!this.indicesMappings[indexName]) {
const detectorType = this.props.detector_type.toLowerCase();
const result = await this.props.filedMappingService.getMappingsView(
indexName,
detectorType
);
result.ok && (this.indicesMappings[indexName] = result.response.unmapped_field_aliases);
}
}

if (!_.isEmpty(this.indicesMappings)) {
let firstMapping: string[] = [];
let firstMatchMappingIndex: string = '';
let message: string[] = [];
for (let indexName in this.indicesMappings) {
if (this.indicesMappings.hasOwnProperty(indexName)) {
if (!firstMapping.length) firstMapping = this.indicesMappings[indexName];
!firstMatchMappingIndex.length && (firstMatchMappingIndex = indexName);
if (!_.isEqual(firstMapping, this.indicesMappings[indexName])) {
message = [
`We recommend creating separate detectors for each of the following log sources:`,
firstMatchMappingIndex,
indexName,
];
break;
}
}
}

this.setState({ message });
}

this.props.onDetectorInputIndicesChange(options);
};

render() {
const { detectorIndices } = this.props;
const { loading, fieldTouched, indexOptions, errorMessage } = this.state;
const { loading, fieldTouched, indexOptions, errorMessage, message } = this.state;
const isInvalid = fieldTouched && detectorIndices.length < MIN_NUM_DATA_SOURCES;
return (
<ContentPanel title={'Data source'} titleSize={'m'}>
<EuiSpacer size={'m'} />
{message.length ? (
<>
<EuiCallOut
title="The selected log sources contain different types of logs"
color="warning"
>
{message.map((messageItem: string, index: number) => (
<EuiTextColor color={'default'} key={`callout-message-part-${index}`}>
{index === 0 ? '' : 'ㅤ•ㅤ'}
{messageItem}
<br />
</EuiTextColor>
))}
</EuiCallOut>
<EuiSpacer size={'m'} />
</>
) : null}
<EuiFormRow
label={
<FormFieldHeader headerTitle={'Select or input source indexes or index patterns'} />
Expand Down
Loading

0 comments on commit b38a223

Please sign in to comment.