Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Detection Engine] Addresses Flakiness in ML FTR tests #188155

Merged
merged 4 commits into from
Jul 12, 2024
Merged

[Detection Engine] Addresses Flakiness in ML FTR tests #188155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d63a91e
Retry failed ML Job setup in DE FTR tests
rylnd Jul 11, 2024
d2e1ba5
Unskip ML FTR tests
rylnd Jul 11, 2024
3d64719
Merge branch 'main' into rylnd/fix_ml_ftr_flakiness
rylnd Jul 11, 2024
a31a343
Fix type error related to bad merge
rylnd Jul 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions ...gine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,12 @@ import {
importFile,
} from '../../../../../lists_and_exception_lists/utils';
import {
executeSetupModuleRequest,
forceStartDatafeeds,
getAlerts,
getPreviewAlerts,
previewRule,
previewRuleWithExceptionEntries,
setupMlModulesWithRetry,
} from '../../../../utils';
import {
createRule,
Expand Down Expand Up @@ -86,13 +86,12 @@ export default ({ getService }: FtrProviderContext) => {
rule_id: 'ml-rule-id',
};

// FLAKY: https://github.com/elastic/kibana/issues/171426
describe.skip('@ess @serverless @serverlessQA Machine learning type rules', () => {
describe('@ess @serverless @serverlessQA Machine learning type rules', () => {
before(async () => {
// Order is critical here: auditbeat data must be loaded before attempting to start the ML job,
// as the job looks for certain indices on start
await esArchiver.load(auditPath);
await executeSetupModuleRequest({ module: siemModule, rspCode: 200, supertest });
await setupMlModulesWithRetry({ module: siemModule, supertest, retry });
await forceStartDatafeeds({ jobId: mlJobId, rspCode: 200, supertest });
await esArchiver.load('x-pack/test/functional/es_archives/security_solution/anomalies');
});
Expand Down
11 changes: 5 additions & 6 deletions ...n_logic/trial_license_complete_tier/execution_logic/machine_learning_alert_suppression.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import { EsArchivePathBuilder } from '../../../../../../es_archive_path_builder'
import { FtrProviderContext } from '../../../../../../ftr_provider_context';
import {
dataGeneratorFactory,
executeSetupModuleRequest,
forceStartDatafeeds,
getAlerts,
getOpenAlerts,
Expand All @@ -36,6 +35,7 @@ import {
previewRule,
previewRuleWithExceptionEntries,
setAlertStatus,
setupMlModulesWithRetry,
} from '../../../../utils';
import {
createRule,
Expand All @@ -51,6 +51,7 @@ export default ({ getService }: FtrProviderContext) => {
const es = getService('es');
const log = getService('log');
const config = getService('config');
const retry = getService('retry');

const isServerless = config.get('serverless');
const dataPathBuilder = new EsArchivePathBuilder(isServerless);
Expand Down Expand Up @@ -93,7 +94,7 @@ export default ({ getService }: FtrProviderContext) => {
// Order is critical here: auditbeat data must be loaded before attempting to start the ML job,
// as the job looks for certain indices on start
await esArchiver.load(auditbeatArchivePath);
await executeSetupModuleRequest({ module: mlModuleName, rspCode: 200, supertest });
await setupMlModulesWithRetry({ module: mlModuleName, retry, supertest });
await forceStartDatafeeds({ jobId: mlJobId, rspCode: 200, supertest });
await esArchiver.load('x-pack/test/functional/es_archives/security_solution/anomalies');
await deleteAllAnomalies(log, es);
Expand All @@ -112,8 +113,7 @@ export default ({ getService }: FtrProviderContext) => {
await deleteAllAnomalies(log, es);
});

// FLAKY: https://github.com/elastic/kibana/issues/187478
describe.skip('with per-execution suppression duration', () => {
describe('with per-execution suppression duration', () => {
beforeEach(() => {
ruleProps = {
...baseRuleProps,
Expand Down Expand Up @@ -245,8 +245,7 @@ export default ({ getService }: FtrProviderContext) => {
});
});

// FLAKY: https://github.com/elastic/kibana/issues/187614
describe.skip('with interval suppression duration', () => {
describe('with interval suppression duration', () => {
beforeEach(() => {
ruleProps = {
...baseRuleProps,
Expand Down
40 changes: 39 additions & 1 deletion ...egration/test_suites/detections_response/utils/machine_learning/machine_learning_setup.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,18 @@
*/

import type SuperTest from 'supertest';
import { RetryService } from '@kbn/ftr-common-functional-services';
import { ML_GROUP_ID } from '@kbn/security-solution-plugin/common/constants';
import { getCommonRequestHeader } from '../../../../../functional/services/ml/common_api';

interface ModuleJob {
id: string;
success: boolean;
error?: {
status: number;
};
}

export const executeSetupModuleRequest = async ({
module,
rspCode,
Expand All @@ -17,7 +26,7 @@ export const executeSetupModuleRequest = async ({
module: string;
rspCode: number;
supertest: SuperTest.Agent;
}) => {
}): Promise<{ jobs: ModuleJob[] }> => {
const { body } = await supertest
.post(`/internal/ml/modules/setup/${module}`)
.set(getCommonRequestHeader('1'))
Expand All @@ -34,6 +43,35 @@ export const executeSetupModuleRequest = async ({
return body;
};

export const setupMlModulesWithRetry = async ({
module,
retry,
supertest,
}: {
module: string;
retry: RetryService;
supertest: SuperTest.Agent;
}) =>
retry.try(async () => {
const response = await executeSetupModuleRequest({
module,
rspCode: 200,
supertest,
});

const allJobsSucceeded = response?.jobs.every((job) => {
return job.success || (job.error?.status && job.error.status < 500);
});

if (!allJobsSucceeded) {
throw new Error(
`Expected all jobs to set up successfully, but got ${JSON.stringify(response)}`
);
}

return response;
});

export const forceStartDatafeeds = async ({
jobId,
rspCode,
Expand Down