Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SIEM] [Detections] Import rules #56389

Merged
merged 7 commits into from
Jan 31, 2020
Merged

[SIEM] [Detections] Import rules #56389

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
647e592
refactor import rules to do it by batch
XavierM Jan 30, 2020
f51d0ff
Merge branch 'master' of github.com:elastic/kibana into detections-fi…
XavierM Jan 30, 2020
3d33f14
fix error msg on api to match with json contract
XavierM Jan 30, 2020
3464bb8
Merge branch 'master' of github.com:elastic/kibana into detections-fi…
XavierM Jan 30, 2020
f3fb462
review I
XavierM Jan 31, 2020
a1b8c3e
fix export
XavierM Jan 31, 2020
712a223
Merge branch 'master' of github.com:elastic/kibana into detections-fi…
XavierM Jan 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
310 changes: 168 additions & 142 deletions x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import Boom from 'boom';
import Hapi from 'hapi';
import { extname } from 'path';
import { isFunction } from 'lodash/fp';
import { chunk, isEmpty, isFunction } from 'lodash/fp';
import { createPromiseFromStreams } from '../../../../../../../../../src/legacy/utils/streams';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
import { createRules } from '../../rules/create_rules';
Expand All @@ -18,17 +18,19 @@ import { getIndexExists } from '../../index/get_index_exists';
import {
callWithRequestFactory,
getIndex,
createImportErrorObject,
transformImportError,
ImportSuccessError,
createBulkErrorObject,
ImportRuleResponses,
} from '../utils';
import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
import { ImportRuleAlertRest } from '../../types';
import { transformOrImportError } from './utils';
import { updateRules } from '../../rules/update_rules';
import { importRulesQuerySchema, importRulesPayloadSchema } from '../schemas/import_rules_schema';
import { KibanaRequest } from '../../../../../../../../../src/core/server';

type PromiseFromStreams = ImportRuleAlertRest | Error;

const CHUNK_PARSED_OBJECT_SIZE = 10;
XavierM marked this conversation as resolved.
Show resolved Hide resolved

export const createImportRulesRoute = (server: ServerFacade): Hapi.ServerRoute => {
return {
method: 'POST',
Expand Down Expand Up @@ -67,145 +69,169 @@ export const createImportRulesRoute = (server: ServerFacade): Hapi.ServerRoute =

const objectLimit = server.config().get<number>('savedObjects.maxImportExportSize');
const readStream = createRulesStreamFromNdJson(request.payload.file, objectLimit);
const parsedObjects = await createPromiseFromStreams<[ImportRuleAlertRest | Error]>([
readStream,
]);
const parsedObjects = await createPromiseFromStreams<PromiseFromStreams[]>([readStream]);

const reduced = await parsedObjects.reduce<Promise<ImportSuccessError>>(
async (accum, parsedRule) => {
const existingImportSuccessError = await accum;
if (parsedRule instanceof Error) {
// If the JSON object had a validation or parse error then we return
// early with the error and an (unknown) for the ruleId
return createImportErrorObject({
ruleId: '(unknown)', // TODO: Better handling where we know which ruleId is having issues with imports
statusCode: 400,
message: parsedRule.message,
existingImportSuccessError,
});
}
const chunkParseObjects = chunk(CHUNK_PARSED_OBJECT_SIZE, parsedObjects);
let importRuleResponse: ImportRuleResponses[] = [];

const {
description,
enabled,
false_positives: falsePositives,
from,
immutable,
query,
language,
output_index: outputIndex,
saved_id: savedId,
meta,
filters,
rule_id: ruleId,
index,
interval,
max_signals: maxSignals,
risk_score: riskScore,
name,
severity,
tags,
threat,
to,
type,
references,
timeline_id: timelineId,
timeline_title: timelineTitle,
version,
} = parsedRule;
try {
const finalIndex = outputIndex != null ? outputIndex : getIndex(request, server);
const callWithRequest = callWithRequestFactory(request, server);
const indexExists = await getIndexExists(callWithRequest, finalIndex);
if (!indexExists) {
return createImportErrorObject({
ruleId,
statusCode: 409,
message: `To create a rule, the index must exist first. Index ${finalIndex} does not exist`,
existingImportSuccessError,
});
}
const rule = await readRules({ alertsClient, ruleId });
if (rule == null) {
const createdRule = await createRules({
alertsClient,
actionsClient,
description,
enabled,
falsePositives,
from,
immutable,
query,
language,
outputIndex: finalIndex,
savedId,
timelineId,
timelineTitle,
meta,
filters,
ruleId,
index,
interval,
maxSignals,
riskScore,
name,
severity,
tags,
to,
type,
threat,
references,
version,
});
return transformOrImportError(ruleId, createdRule, existingImportSuccessError);
} else if (rule != null && request.query.overwrite) {
const updatedRule = await updateRules({
alertsClient,
actionsClient,
savedObjectsClient,
description,
enabled,
falsePositives,
from,
immutable,
query,
language,
outputIndex,
savedId,
timelineId,
timelineTitle,
meta,
filters,
id: undefined,
ruleId,
index,
interval,
maxSignals,
riskScore,
name,
severity,
tags,
to,
type,
threat,
references,
version,
});
return transformOrImportError(ruleId, updatedRule, existingImportSuccessError);
} else {
return existingImportSuccessError;
}
} catch (err) {
return transformImportError(ruleId, err, existingImportSuccessError);
}
},
Promise.resolve({
success: true,
success_count: 0,
errors: [],
})
);
return reduced;
while (chunkParseObjects.length) {
const batchParseObjects = chunkParseObjects.shift() ?? [];
const newImportRuleResponse = await Promise.all(
batchParseObjects.reduce<Array<Promise<ImportRuleResponses>>>((accum, parsedRule) => {
const importsWorkerPromise = new Promise<ImportRuleResponses>(
async (resolve, reject) => {
if (parsedRule instanceof Error) {
// If the JSON object had a validation or parse error then we return
// early with the error and an (unknown) for the ruleId
resolve(
createBulkErrorObject({
ruleId: '(unknown)',
statusCode: 400,
message: parsedRule.message,
})
);
return null;
}
const {
description,
enabled = false,
false_positives: falsePositives,
from,
immutable,
query,
language,
output_index: outputIndex,
saved_id: savedId,
meta,
filters,
rule_id: ruleId,
index,
interval,
max_signals: maxSignals,
risk_score: riskScore,
name,
severity,
tags,
threat,
to,
type,
references,
timeline_id: timelineId,
timeline_title: timelineTitle,
version,
} = parsedRule;
try {
const finalIndex = outputIndex != null ? outputIndex : getIndex(request, server);
const callWithRequest = callWithRequestFactory(request, server);
const indexExists = await getIndexExists(callWithRequest, finalIndex);
if (!indexExists) {
resolve(
createBulkErrorObject({
ruleId,
statusCode: 409,
message: `To create a rule, the index must exist first. Index ${finalIndex} does not exist`,
})
);
}
const rule = await readRules({ alertsClient, ruleId });
if (rule == null) {
await createRules({
alertsClient,
actionsClient,
description,
enabled,
falsePositives,
from,
immutable,
query,
language,
outputIndex: finalIndex,
savedId,
timelineId,
timelineTitle,
meta,
filters,
ruleId,
index,
interval,
maxSignals,
riskScore,
name,
severity,
tags,
to,
type,
threat,
references,
version,
});
resolve({ rule_id: ruleId, status_code: 200 });
} else if (rule != null && request.query.overwrite) {
await updateRules({
alertsClient,
actionsClient,
savedObjectsClient,
description,
enabled,
falsePositives,
from,
immutable,
query,
language,
outputIndex,
savedId,
timelineId,
timelineTitle,
meta,
filters,
id: undefined,
ruleId,
index,
interval,
maxSignals,
riskScore,
name,
severity,
tags,
to,
type,
threat,
references,
version,
});
resolve({ rule_id: ruleId, status_code: 200 });
} else if (rule != null) {
resolve(
createBulkErrorObject({
ruleId,
statusCode: 409,
message: `This Rule "${rule.name}" already exists`,
})
);
}
} catch (err) {
resolve(
createBulkErrorObject({
ruleId,
statusCode: 400,
message: err.message,
})
);
}
}
);
return [...accum, importsWorkerPromise];
}, [])
);
importRuleResponse = [...importRuleResponse, ...newImportRuleResponse];
}

const errorsResp = importRuleResponse.filter(resp => !isEmpty(resp.error));
return {
success: errorsResp.length === 0,
success_count: importRuleResponse.filter(resp => resp.status_code === 200).length,
errors: errorsResp,
};
},
};
};
Expand Down
10 changes: 10 additions & 0 deletions x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,16 @@ export const createBulkErrorObject = ({
};
};

export interface ImportRuleResponses {
XavierM marked this conversation as resolved.
Show resolved Hide resolved
rule_id: string;
status_code?: number;
message?: string;
error?: {
status_code: number;
message: string;
};
}

export interface ImportSuccessError {
success: boolean;
success_count: number;
Expand Down