[#31446] stabilize x-pack-ciGroup6 (#31447)

* rerun x-pack-ciGroup6 40 times * improve reporting tests stability (#31454)
elastic · Feb 26, 2019 · a823c9b · a823c9b
1 parent 256a4f1
commit a823c9b
Show file tree

Hide file tree

Showing 8 changed files with 69 additions and 211 deletions.
diff --git a/.ci/jobs.yml b/.ci/jobs.yml
@@ -1,132 +1,44 @@
 JOB:
-  - kibana-intake
-  - x-pack-intake
-  # make sure all kibana-ciGRoups are listed in tasks/function_test_groups.js
-  - kibana-ciGroup1-1
-  - kibana-ciGroup2-1
-  - kibana-ciGroup3-1
-  - kibana-ciGroup4-1
-  - kibana-ciGroup5-1
-  - kibana-ciGroup6-1
-  - kibana-ciGroup7-1
-  - kibana-ciGroup8-1
-  - kibana-ciGroup9-1
-  - kibana-ciGroup10-1
-  - kibana-ciGroup11-1
-  - kibana-ciGroup12-1
-  # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
-  - x-pack-ciGroup1-1
-  - x-pack-ciGroup2-1
-  - x-pack-ciGroup3-1
-  - x-pack-ciGroup4-1
-  - x-pack-ciGroup5-1
   - x-pack-ciGroup6-1
-  - x-pack-ciGroup7-1
-
-  - kibana-ciGroup1-2
-  - kibana-ciGroup2-2
-  - kibana-ciGroup3-2
-  - kibana-ciGroup4-2
-  - kibana-ciGroup5-2
-  - kibana-ciGroup6-2
-  - kibana-ciGroup7-2
-  - kibana-ciGroup8-2
-  - kibana-ciGroup9-2
-  - kibana-ciGroup10-2
-  - kibana-ciGroup11-2
-  - kibana-ciGroup12-2
-  # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
-  - x-pack-ciGroup1-2
-  - x-pack-ciGroup2-2
-  - x-pack-ciGroup3-2
-  - x-pack-ciGroup4-2
-  - x-pack-ciGroup5-2
   - x-pack-ciGroup6-2
-  - x-pack-ciGroup7-2
-
-  - kibana-ciGroup1-3
-  - kibana-ciGroup2-3
-  - kibana-ciGroup3-3
-  - kibana-ciGroup4-3
-  - kibana-ciGroup5-3
-  - kibana-ciGroup6-3
-  - kibana-ciGroup7-3
-  - kibana-ciGroup8-3
-  - kibana-ciGroup9-3
-  - kibana-ciGroup10-3
-  - kibana-ciGroup11-3
-  - kibana-ciGroup12-3
-  # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
-  - x-pack-ciGroup1-3
-  - x-pack-ciGroup2-3
-  - x-pack-ciGroup3-3
-  - x-pack-ciGroup4-3
-  - x-pack-ciGroup5-3
   - x-pack-ciGroup6-3
-  - x-pack-ciGroup7-3
-
-  - kibana-ciGroup1-4
-  - kibana-ciGroup2-4
-  - kibana-ciGroup3-4
-  - kibana-ciGroup4-4
-  - kibana-ciGroup5-4
-  - kibana-ciGroup6-4
-  - kibana-ciGroup7-4
-  - kibana-ciGroup8-4
-  - kibana-ciGroup9-4
-  - kibana-ciGroup10-4
-  - kibana-ciGroup11-4
-  - kibana-ciGroup12-4
-  # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
-  - x-pack-ciGroup1-4
-  - x-pack-ciGroup2-4
-  - x-pack-ciGroup3-4
-  - x-pack-ciGroup4-4
-  - x-pack-ciGroup5-4
   - x-pack-ciGroup6-4
-  - x-pack-ciGroup7-4
-
-  - kibana-ciGroup1-5
-  - kibana-ciGroup2-5
-  - kibana-ciGroup3-5
-  - kibana-ciGroup4-5
-  - kibana-ciGroup5-5
-  - kibana-ciGroup6-5
-  - kibana-ciGroup7-5
-  - kibana-ciGroup8-5
-  - kibana-ciGroup9-5
-  - kibana-ciGroup10-5
-  - kibana-ciGroup11-5
-  - kibana-ciGroup12-5
-  # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
-  - x-pack-ciGroup1-5
-  - x-pack-ciGroup2-5
-  - x-pack-ciGroup3-5
-  - x-pack-ciGroup4-5
-  - x-pack-ciGroup5-5
   - x-pack-ciGroup6-5
-  - x-pack-ciGroup7-5
-
-  - kibana-ciGroup1-6
-  - kibana-ciGroup2-6
-  - kibana-ciGroup3-6
-  - kibana-ciGroup4-6
-  - kibana-ciGroup5-6
-  - kibana-ciGroup6-6
-  - kibana-ciGroup7-6
-  - kibana-ciGroup8-6
-  - kibana-ciGroup9-6
-  - kibana-ciGroup10-6
-  - kibana-ciGroup11-6
-  - kibana-ciGroup12-6
-  # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
-  - x-pack-ciGroup1-6
-  - x-pack-ciGroup2-6
-  - x-pack-ciGroup3-6
-  - x-pack-ciGroup4-6
-  - x-pack-ciGroup5-6
   - x-pack-ciGroup6-6
-  - x-pack-ciGroup7-6
+  - x-pack-ciGroup6-7
+  - x-pack-ciGroup6-8
+  - x-pack-ciGroup6-9
+  - x-pack-ciGroup6-10
+  - x-pack-ciGroup6-11
+  - x-pack-ciGroup6-12
+  - x-pack-ciGroup6-13
+  - x-pack-ciGroup6-14
+  - x-pack-ciGroup6-15
+  - x-pack-ciGroup6-16
+  - x-pack-ciGroup6-17
+  - x-pack-ciGroup6-18
+  - x-pack-ciGroup6-19
+  - x-pack-ciGroup6-20
+  - x-pack-ciGroup6-21
+  - x-pack-ciGroup6-22
+  - x-pack-ciGroup6-23
+  - x-pack-ciGroup6-24
+  - x-pack-ciGroup6-25
+  - x-pack-ciGroup6-26
+  - x-pack-ciGroup6-27
+  - x-pack-ciGroup6-28
+  - x-pack-ciGroup6-29
+  - x-pack-ciGroup6-30
+  - x-pack-ciGroup6-31
+  - x-pack-ciGroup6-32
+  - x-pack-ciGroup6-33
+  - x-pack-ciGroup6-34
+  - x-pack-ciGroup6-35
+  - x-pack-ciGroup6-36
+  - x-pack-ciGroup6-37
+  - x-pack-ciGroup6-38
+  - x-pack-ciGroup6-39
+  - x-pack-ciGroup6-40
 
 # `~` is yaml for `null`
 exclude: ~
diff --git a/tasks/function_test_groups.js b/tasks/function_test_groups.js
@@ -25,78 +25,18 @@ import grunt from 'grunt';
  * and to the list of jobs in .ci/jobs.yml
  */
 const TEST_TAGS = [
-  'ciGroup1-1',
-  'ciGroup2-1',
-  'ciGroup3-1',
-  'ciGroup4-1',
-  'ciGroup5-1',
-  'ciGroup6-1',
-  'ciGroup7-1',
-  'ciGroup8-1',
-  'ciGroup9-1',
-  'ciGroup10-1',
-  'ciGroup11-1',
-  'ciGroup12-1',
-  'ciGroup1-2',
-  'ciGroup2-2',
-  'ciGroup3-2',
-  'ciGroup4-2',
-  'ciGroup5-2',
-  'ciGroup6-2',
-  'ciGroup7-2',
-  'ciGroup8-2',
-  'ciGroup9-2',
-  'ciGroup10-2',
-  'ciGroup11-2',
-  'ciGroup12-2',
-  'ciGroup1-3',
-  'ciGroup2-3',
-  'ciGroup3-3',
-  'ciGroup4-3',
-  'ciGroup5-3',
-  'ciGroup6-3',
-  'ciGroup7-3',
-  'ciGroup8-3',
-  'ciGroup9-3',
-  'ciGroup10-3',
-  'ciGroup11-3',
-  'ciGroup12-3',
-  'ciGroup1-4',
-  'ciGroup2-4',
-  'ciGroup3-4',
-  'ciGroup4-4',
-  'ciGroup5-4',
-  'ciGroup6-4',
-  'ciGroup7-4',
-  'ciGroup8-4',
-  'ciGroup9-4',
-  'ciGroup10-4',
-  'ciGroup11-4',
-  'ciGroup12-4',
-  'ciGroup1-5',
-  'ciGroup2-5',
-  'ciGroup3-5',
-  'ciGroup4-5',
-  'ciGroup5-5',
-  'ciGroup6-5',
-  'ciGroup7-5',
-  'ciGroup8-5',
-  'ciGroup9-5',
-  'ciGroup10-5',
-  'ciGroup11-5',
-  'ciGroup12-5',
-  'ciGroup1-6',
-  'ciGroup2-6',
-  'ciGroup3-6',
-  'ciGroup4-6',
-  'ciGroup5-6',
-  'ciGroup6-6',
-  'ciGroup7-6',
-  'ciGroup8-6',
-  'ciGroup9-6',
-  'ciGroup10-6',
-  'ciGroup11-6',
-  'ciGroup12-6',
+  'ciGroup1',
+  'ciGroup2',
+  'ciGroup3',
+  'ciGroup4',
+  'ciGroup5',
+  'ciGroup6',
+  'ciGroup7',
+  'ciGroup8',
+  'ciGroup9',
+  'ciGroup10',
+  'ciGroup11',
+  'ciGroup12',
 ];
 
 export function getFunctionalTestGroupRunConfigs({ esFrom, kibanaInstallDir } = {}) {

diff --git a/test/functional/page_objects/dashboard_page.js b/test/functional/page_objects/dashboard_page.js
@@ -356,8 +356,7 @@ export function DashboardPageProvider({ getService, getPageObjects }) {
      */
     async enterDashboardTitleAndClickSave(dashboardTitle, saveOptions = {}) {
       await testSubjects.click('dashboardSaveMenuItem');
-
-      await PageObjects.header.waitUntilLoadingHasFinished();
+      const modalDialog = await testSubjects.find('savedObjectSaveModal');
 
       log.debug('entering new title');
       await testSubjects.setValue('savedObjectTitle', dashboardTitle);
@@ -371,6 +370,7 @@ export function DashboardPageProvider({ getService, getPageObjects }) {
       }
 
       await this.clickSave();
+      await testSubjects.waitForDeleted(modalDialog);
     }
 
     async selectDashboard(dashName) {

diff --git a/test/functional/page_objects/share_page.js b/test/functional/page_objects/share_page.js
@@ -46,7 +46,7 @@ export function SharePageProvider({ getService, getPageObjects }) {
       }
       const menuPanel = await find.byCssSelector('div.euiContextMenuPanel');
       testSubjects.click(`sharePanel-${itemTitle.replace(' ', '')}`);
-      await find.waitForElementStale(menuPanel);
+      await testSubjects.waitForDeleted(menuPanel);
     }
 
     async getSharedUrl() {

diff --git a/test/functional/page_objects/visualize_page.js b/test/functional/page_objects/visualize_page.js
@@ -738,13 +738,11 @@ export function VisualizePageProvider({ getService, getPageObjects }) {
 
     async ensureSavePanelOpen() {
       log.debug('ensureSavePanelOpen');
-      let isOpen = await testSubjects.exists('savedObjectSaveModal');
-      await retry.try(async () => {
-        while (!isOpen) {
-          await testSubjects.click('visualizeSaveButton');
-          isOpen = await testSubjects.exists('savedObjectSaveModal');
-        }
-      });
+      await PageObjects.header.waitUntilLoadingHasFinished();
+      const isOpen = await testSubjects.exists('savedObjectSaveModal', { timeout: 5000 });
+      if (!isOpen) {
+        await testSubjects.click('visualizeSaveButton');
+      }
     }
 
     async saveVisualization(vizName, { saveAsNew = false } = {}) {

diff --git a/test/functional/services/test_subjects.js b/test/functional/services/test_subjects.js
@@ -224,8 +224,12 @@ export function TestSubjectsProvider({ getService }) {
       });
     }
 
-    async waitForDeleted(selector) {
-      await find.waitForDeletedByCssSelector(testSubjSelector(selector));
+    async waitForDeleted(selectorOrElement) {
+      if (typeof (selectorOrElement) === 'string') {
+        await find.waitForDeletedByCssSelector(testSubjSelector(selectorOrElement));
+      } else {
+        await find.waitForElementStale(selectorOrElement);
+      }
     }
 
     async waitForAttributeToChange(selector, attribute, value) {

diff --git a/x-pack/test/functional/page_objects/reporting_page.js b/x-pack/test/functional/page_objects/reporting_page.js
@@ -126,15 +126,15 @@ export function ReportingPageProvider({ getService, getPageObjects }) {
 
     async clearToastNotifications() {
       const toasts = await testSubjects.findAll('toastCloseButton');
-      await Promise.all(toasts.map(t => t.click()));
+      await Promise.all(toasts.map(async t => await t.click()));
     }
 
     async getQueueReportError() {
       return await testSubjects.exists('queueReportError');
     }
 
     async getGenerateReportButton() {
-      return await retry.try(() => testSubjects.find('generateReportButton'));
+      return await retry.try(async () => await testSubjects.find('generateReportButton'));
     }
 
     async checkUsePrintLayout() {
@@ -152,10 +152,14 @@ export function ReportingPageProvider({ getService, getPageObjects }) {
     async checkForReportingToasts() {
       log.debug('Reporting:checkForReportingToasts');
       const isToastPresent = await testSubjects.exists('completeReportSuccess', {
-        timeout: 60000
+        allowHidden: true,
+        timeout: 90000
       });
       // Close toast so it doesn't obscure the UI.
-      await testSubjects.click('completeReportSuccess toastCloseButton');
+      if (isToastPresent) {
+        await testSubjects.click('completeReportSuccess toastCloseButton');
+      }
+
       return isToastPresent;
     }
 

diff --git a/x-pack/test/reporting/functional/reporting.js b/x-pack/test/reporting/functional/reporting.js
@@ -299,7 +299,7 @@ export default function ({ getService, getPageObjects }) {
           await PageObjects.visualize.clickBucket('X-Axis');
           await PageObjects.visualize.selectAggregation('Date Histogram');
           await PageObjects.visualize.clickGo();
-          await PageObjects.visualize.saveVisualizationExpectSuccess('my viz');
+          await PageObjects.visualize.saveVisualization('my viz');
           await PageObjects.reporting.openPdfReportingPanel();
           await expectEnabledGenerateReportButton();
         });