From c060b9f6f76c4971da67c9101e7466db6b727df6 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Mon, 14 Mar 2022 21:48:29 -0600 Subject: [PATCH] [Security Solutions] Increases timeout and unskips detection tests (#126294) ## Summary Increases the timeouts from 2 minutes to now 5 minutes and unskips the detection tests. If any of the tests fail consistently then I will skip just those individual tests instead of the whole suit. See https://github.com/elastic/kibana/issues/125851 and https://github.com/elastic/elasticsearch/issues/84256 This could cause issues with: https://github.com/elastic/kibana/issues/125319 If so, then we will have to deal with the cloud based tests in a different way but in reality we need the extra time as some test cases do take a while to run on CI. This also: * Removes skips around code that have been in there for a while and adds documentation to the parts that are left over. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../basic/tests/open_close_signals.ts | 2 +- .../basic/tests/query_signals.ts | 2 ++ .../security_and_spaces/tests/create_exceptions.ts | 3 +-- .../security_and_spaces/tests/create_index.ts | 1 + .../security_and_spaces/tests/create_ml.ts | 3 +-- .../security_and_spaces/tests/create_rules.ts | 6 ++---- .../security_and_spaces/tests/create_threat_matching.ts | 3 +-- .../tests/finalize_signals_migrations.ts | 1 + .../security_and_spaces/tests/generating_signals.ts | 5 ++--- .../security_and_spaces/tests/index.ts | 3 +-- .../security_and_spaces/tests/open_close_signals.ts | 8 +++++--- x-pack/test/detection_engine_api_integration/utils.ts | 3 ++- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/basic/tests/open_close_signals.ts b/x-pack/test/detection_engine_api_integration/basic/tests/open_close_signals.ts index fc0072de05f5e..f58dee11c48da 100644 --- a/x-pack/test/detection_engine_api_integration/basic/tests/open_close_signals.ts +++ b/x-pack/test/detection_engine_api_integration/basic/tests/open_close_signals.ts @@ -34,7 +34,7 @@ export default ({ getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); const log = getService('log'); - describe.skip('open_close_signals', () => { + describe('open_close_signals', () => { describe('tests with auditbeat data', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); diff --git a/x-pack/test/detection_engine_api_integration/basic/tests/query_signals.ts b/x-pack/test/detection_engine_api_integration/basic/tests/query_signals.ts index 3ef0a12cf70dc..48d13e8b2afd6 100644 --- a/x-pack/test/detection_engine_api_integration/basic/tests/query_signals.ts +++ b/x-pack/test/detection_engine_api_integration/basic/tests/query_signals.ts @@ -39,6 +39,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); + // This fails and should be investigated or removed if it no longer applies it.skip('should not give errors when querying and the signals index does exist and is empty', async () => { await createSignalsIndex(supertest, log); const { body } = await supertest @@ -160,6 +161,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); + // This fails and should be investigated or removed if it no longer applies it.skip('should not give errors when querying and the signals index does exist and is empty', async () => { await createSignalsIndex(supertest, log); const { body } = await supertest diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts index 818ba3b366e40..0dfc753be402c 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts @@ -63,8 +63,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const es = getService('es'); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/125851 - describe.skip('create_rules_with_exceptions', () => { + describe('create_rules_with_exceptions', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts index 17d6b3957637a..6d867877e0dc3 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_index.ts @@ -38,6 +38,7 @@ export default ({ getService }: FtrProviderContext) => { await esArchiver.unload('x-pack/test/functional/es_archives/signals/index_alias_clash'); }); + // This fails and should be investigated or removed if it no longer applies it.skip('should report that signals index does not exist', async () => { const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(404); expect(body).to.eql({ message: 'index for this space does not exist', status_code: 404 }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts index e2ce3922f2d3f..343db03c2ae27 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts @@ -92,8 +92,7 @@ export default ({ getService }: FtrProviderContext) => { return body; } - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/125851 - describe.skip('Generating signals from ml anomalies', () => { + describe('Generating signals from ml anomalies', () => { before(async () => { // Order is critical here: auditbeat data must be loaded before attempting to start the ML job, // as the job looks for certain indices on start diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts index 1075a63742777..82899ff0de8f7 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts @@ -44,8 +44,7 @@ export default ({ getService }: FtrProviderContext) => { const supertestWithoutAuth = getService('supertestWithoutAuth'); const log = getService('log'); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/125851 - describe.skip('create_rules', () => { + describe('create_rules', () => { describe('creating rules', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); @@ -106,8 +105,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccessOrStatus(supertest, log, body.id); }); - // TODO: does the below test work? - it.skip('should create a single rule with a rule_id and an index pattern that does not match anything available and partial failure for the rule', async () => { + it('should create a single rule with a rule_id and an index pattern that does not match anything available and partial failure for the rule', async () => { const simpleRule = getRuleForSignalTesting(['does-not-exist-*']); const { body } = await supertest .post(DETECTION_ENGINE_RULES_URL) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index b5b232f70ec89..346998e7af261 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -69,8 +69,7 @@ export default ({ getService }: FtrProviderContext) => { /** * Specific api integration tests for threat matching rule type */ - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/125851 - describe.skip('create_threat_matching', () => { + describe('create_threat_matching', () => { describe('creating threat match rule', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/finalize_signals_migrations.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/finalize_signals_migrations.ts index df06647c2a8b5..8733c645112d1 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/finalize_signals_migrations.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/finalize_signals_migrations.ts @@ -183,6 +183,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(statusAfter.map((s) => s.is_outdated)).to.eql([false, false]); }); + // This fails and should be investigated or removed if it no longer applies it.skip('deletes the underlying migration task', async () => { await waitFor( async () => { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts index 761792e29ea1d..11c5af219de4c 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -71,8 +71,7 @@ export default ({ getService }: FtrProviderContext) => { const es = getService('es'); const log = getService('log'); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/125851 - describe.skip('Generating signals from source indexes', () => { + describe('Generating signals from source indexes', () => { beforeEach(async () => { await deleteSignalsIndex(supertest, log); await createSignalsIndex(supertest, log); @@ -1160,7 +1159,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe.skip('Signal deduplication', async () => { + describe('Signal deduplication', async () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index d234c70de5240..a9bda19638041 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -9,8 +9,7 @@ import { FtrProviderContext } from '../../common/ftr_provider_context'; // eslint-disable-next-line import/no-default-export export default ({ loadTestFile }: FtrProviderContext): void => { - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/125851 - describe.skip('detection engine api security and spaces enabled', function () { + describe('detection engine api security and spaces enabled', function () { describe('', function () { this.tags('ciGroup11'); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/open_close_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/open_close_signals.ts index f9a5a3283bd2f..6138995bdc02f 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/open_close_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/open_close_signals.ts @@ -39,7 +39,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); describe('open_close_signals', () => { - describe.skip('validation checks', () => { + describe('validation checks', () => { it('should not give errors when querying and the signals index does not exist yet', async () => { const { body } = await supertest .post(DETECTION_ENGINE_SIGNALS_STATUS_URL) @@ -165,7 +165,8 @@ export default ({ getService }: FtrProviderContext) => { expect(everySignalClosed).to.eql(true); }); - it('should be able to close signals with t1 analyst user', async () => { + // This fails and should be investigated or removed if it no longer applies + it.skip('should be able to close signals with t1 analyst user', async () => { const rule = getRuleForSignalTesting(['auditbeat-*']); const { id } = await createRule(supertest, log, rule); await waitForRuleSuccessOrStatus(supertest, log, id); @@ -200,7 +201,8 @@ export default ({ getService }: FtrProviderContext) => { await deleteUserAndRole(getService, ROLES.t1_analyst); }); - it('should be able to close signals with soc_manager user', async () => { + // This fails and should be investigated or removed if it no longer applies + it.skip('should be able to close signals with soc_manager user', async () => { const rule = getRuleForSignalTesting(['auditbeat-*']); const { id } = await createRule(supertest, log, rule); await waitForRuleSuccessOrStatus(supertest, log, id); diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 496faa3c5e421..545bd20fc777b 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -457,6 +457,7 @@ export const getSimpleRulePreviewOutput = ( ) => ({ logs, previewId, + isAborted: false, }); export const resolveSimpleRuleOutput = ( @@ -924,7 +925,7 @@ export const waitFor = async ( functionToTest: () => Promise, functionName: string, log: ToolingLog, - maxTimeout: number = 100000, + maxTimeout: number = 400000, timeoutWait: number = 250 ): Promise => { let found = false;