Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

unskipping functional tests to check their flakiness #95349

Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/functional/apps/context/_discover_navigation.js
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ export default function ({ getService, getPageObjects }) {
});

// flaky https://github.com/elastic/kibana/issues/93670
it.skip('navigates to doc view from embeddable', async () => {
it('navigates to doc view from embeddable', async () => {
await PageObjects.common.navigateToApp('discover');
await PageObjects.discover.saveSearch('my search');
await PageObjects.header.waitUntilLoadingHasFinished();
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/dashboard/dashboard_save.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const testSubjects = getService('testSubjects');

// FLAKY: https://github.com/elastic/kibana/issues/89476
describe.skip('dashboard save', function describeIndexTests() {
describe('dashboard save', function describeIndexTests() {
this.tags('includeFirefox');
const dashboardName = 'Dashboard Save Test';
const dashboardNameEnterKey = 'Dashboard Save Test with Enter Key';
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/dashboard/dashboard_unsaved_state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
let unsavedPanelCount = 0;

// FLAKY: https://github.com/elastic/kibana/issues/91191
describe.skip('dashboard unsaved panels', () => {
describe('dashboard unsaved panels', () => {
before(async () => {
await esArchiver.load('dashboard/current/kibana');
await kibanaServer.uiSettings.replace({
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/dashboard/embeddable_rendering.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
};

// Failing: See https://github.com/elastic/kibana/issues/76245
describe.skip('dashboard embeddable rendering', function describeIndexTests() {
describe('dashboard embeddable rendering', function describeIndexTests() {
before(async () => {
await security.testUser.setRoles(['kibana_admin', 'animals', 'test_logstash_reader']);
await esArchiver.load('dashboard/current/kibana');
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/dashboard/url_field_formatter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
};

// FLAKY: https://github.com/elastic/kibana/issues/79463
describe.skip('Changing field formatter to Url', () => {
describe('Changing field formatter to Url', () => {
before(async function () {
await esArchiver.load('dashboard/current/kibana');
await kibanaServer.uiSettings.replace({
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/discover/_data_grid_context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const browser = getService('browser');

// FLAKY: https://github.com/elastic/kibana/issues/94545
describe.skip('discover data grid context tests', () => {
describe('discover data grid context tests', () => {
before(async () => {
await esArchiver.load('discover');
await esArchiver.loadIfNeeded('logstash_functional');
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/discover/_data_grid_doc_table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
});

// flaky https://github.com/elastic/kibana/issues/94889
it.skip('should show popover with expanded cell content by click on expand button', async () => {
it('should show popover with expanded cell content by click on expand button', async () => {
log.debug('open popover with expanded cell content to get json from the editor');
const documentCell = await dataGrid.getCellElement(1, 3);
await documentCell.click();
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/discover/_saved_queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const testSubjects = getService('testSubjects');

// FLAKY: https://github.com/elastic/kibana/issues/89477
describe.skip('saved queries saved objects', function describeIndexTests() {
describe('saved queries saved objects', function describeIndexTests() {
before(async function () {
log.debug('load kibana index with default index pattern');
await esArchiver.load('discover');
Expand Down
4 changes: 2 additions & 2 deletions test/functional/apps/management/_scripted_fields.js
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ export default function ({ getService, getPageObjects }) {

//add a test to sort boolean
//existing bug: https://github.com/elastic/kibana/issues/75519 hence the issue is skipped.
it.skip('should sort scripted field value in Discover', async function () {
it('should sort scripted field value in Discover', async function () {
await testSubjects.click(`docTableHeaderFieldSort_${scriptedPainlessFieldName2}`);
// after the first click on the scripted field, it becomes secondary sort after time.
// click on the timestamp twice to make it be the secondary sort key.
Expand Down Expand Up @@ -424,7 +424,7 @@ export default function ({ getService, getPageObjects }) {

//add a test to sort date scripted field
//https://github.com/elastic/kibana/issues/75711
it.skip('should sort scripted field value in Discover', async function () {
it('should sort scripted field value in Discover', async function () {
await testSubjects.click(`docTableHeaderFieldSort_${scriptedPainlessFieldName2}`);
// after the first click on the scripted field, it becomes secondary sort after time.
// click on the timestamp twice to make it be the secondary sort key.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export default function ({ getService, getPageObjects }) {
const SCRIPTED_FIELD_NAME = 'myScriptedField';

// FLAKY: https://github.com/elastic/kibana/issues/89475
describe.skip('scripted fields preview', () => {
describe('scripted fields preview', () => {
before(async function () {
await browser.setWindowSize(1200, 800);
await PageObjects.settings.createIndexPattern();
Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
};

// Flaky: https://github.com/elastic/kibana/issues/68400
describe.skip('saved objects edition page', () => {
describe('saved objects edition page', () => {
beforeEach(async () => {
await esArchiver.load('saved_objects_management/edit_saved_object');
});
Expand Down
2 changes: 1 addition & 1 deletion test/functional/apps/visualize/_visualize_listing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const listingTable = getService('listingTable');

// FLAKY: https://github.com/elastic/kibana/issues/40912
describe.skip('visualize listing page', function describeIndexTests() {
describe('visualize listing page', function describeIndexTests() {
const vizName = 'Visualize Listing Test';

describe('create and delete', function () {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
const savedQueryManagementComponent = getService('savedQueryManagementComponent');

// FLAKY: https://github.com/elastic/kibana/issues/86950
describe.skip('dashboard feature controls security', () => {
describe('dashboard feature controls security', () => {
before(async () => {
await esArchiver.load('dashboard/feature_controls/security');
await esArchiver.loadIfNeeded('logstash_functional');
Expand Down Expand Up @@ -109,7 +109,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
});

// Can't figure out how to get this test to pass
it.skip(`create new dashboard shows addNew button`, async () => {
it(`create new dashboard shows addNew button`, async () => {
await PageObjects.common.navigateToActualUrl(
'dashboard',
DashboardConstants.CREATE_NEW_DASHBOARD_URL,
Expand Down Expand Up @@ -321,7 +321,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
});

// Has this behavior changed?
it.skip(`create new dashboard redirects to the home page`, async () => {
it(`create new dashboard redirects to the home page`, async () => {
await PageObjects.common.navigateToActualUrl(
'dashboard',
DashboardConstants.CREATE_NEW_DASHBOARD_URL,
Expand Down Expand Up @@ -439,7 +439,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
});

// Has this behavior changed?
it.skip(`create new dashboard redirects to the home page`, async () => {
it(`create new dashboard redirects to the home page`, async () => {
await PageObjects.common.navigateToActualUrl(
'dashboard',
DashboardConstants.CREATE_NEW_DASHBOARD_URL,
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/dashboard/reporting/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@ Every now and then visual changes will be made that will require the snapshots t
3. **Download the reports, and save them into the `reports/baseline` folder.**
Change the names of the PNG/PDF files to overwrite the stored baselines.

The next time functional tests run, the generated reports will be compared to the latest image that you have saved :bowtie:
The next time functional tests run, the generated reports will be compared to the latest image that you have saved :bowtie:
6 changes: 3 additions & 3 deletions x-pack/test/functional/apps/discover/async_scripted_fields.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export default function ({ getService, getPageObjects }) {
const security = getService('security');

// Failing: See https://github.com/elastic/kibana/issues/78553
describe.skip('async search with scripted fields', function () {
describe('async search with scripted fields', function () {
this.tags(['skipFirefox']);

before(async function () {
Expand All @@ -42,7 +42,7 @@ export default function ({ getService, getPageObjects }) {
await security.testUser.restoreDefaults();
});

it.skip('query should show failed shards pop up', async function () {
it('query should show failed shards pop up', async function () {
if (false) {
/* If you had to modify the scripted fields, you could un-comment all this, run it, use es_archiver to update 'kibana_scripted_fields_on_logstash'
*/
Expand Down Expand Up @@ -74,7 +74,7 @@ export default function ({ getService, getPageObjects }) {
});
});

it.skip('query return results with valid scripted field', async function () {
it('query return results with valid scripted field', async function () {
if (false) {
/* the commented-out steps below were used to create the scripted fields in the logstash-* index pattern
which are now saved in the esArchive.
Expand Down
4 changes: 2 additions & 2 deletions x-pack/test/functional/apps/grok_debugger/grok_debugger.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export default function ({ getService, getPageObjects }) {

const PageObjects = getPageObjects(['grokDebugger']);
// FLAKY: https://github.com/elastic/kibana/issues/84440
describe.skip('grok debugger app', function () {
describe('grok debugger app', function () {
this.tags('includeFirefox');
before(async () => {
await esArchiver.load('empty_kibana');
Expand Down Expand Up @@ -50,7 +50,7 @@ export default function ({ getService, getPageObjects }) {
});

describe('syntax highlighting', () => {
it.skip('applies the correct CSS classes', async () => {
it('applies the correct CSS classes', async () => {
const grokPattern = '\\[(?:-|%{NUMBER:bytes:int})\\]';

await grokDebugger.setPatternInput(grokPattern);
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/infra/home_page.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const supertest = getService('supertest');

// FLAKY: https://github.com/elastic/kibana/issues/75724
describe.skip('Home page', function () {
describe('Home page', function () {
this.tags('includeFirefox');
before(async () => {
await esArchiver.load('empty_kibana');
Expand Down
6 changes: 3 additions & 3 deletions x-pack/test/functional/apps/lens/chart_data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,21 +61,21 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {

// Partition chart tests have to be skipped until
// https://github.com/elastic/elastic-charts/issues/917 gets fixed
it.skip('should render pie chart', async () => {
it('should render pie chart', async () => {
await PageObjects.lens.switchToVisualization('pie');
await PageObjects.lens.waitForVisualization();
const data = await PageObjects.lens.getCurrentChartDebugState();
assertMatchesExpectedData(data!);
});

it.skip('should render donut chart', async () => {
it('should render donut chart', async () => {
await PageObjects.lens.switchToVisualization('donut');
await PageObjects.lens.waitForVisualization();
const data = await PageObjects.lens.getCurrentChartDebugState();
assertMatchesExpectedData(data!);
});

it.skip('should render treemap chart', async () => {
it('should render treemap chart', async () => {
await PageObjects.lens.switchToVisualization('treemap');
await PageObjects.lens.waitForVisualization();
const data = await PageObjects.lens.getCurrentChartDebugState();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
const appsMenu = getService('appsMenu');

// FLAKY: https://github.com/elastic/kibana/issues/38414
describe.skip('spaces feature controls', () => {
describe('spaces feature controls', () => {
before(async () => {
PageObjects.maps.setBasePath('/s/custom_space');
});
Expand Down
4 changes: 2 additions & 2 deletions x-pack/test/functional/apps/maps/layer_errors.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import expect from '@kbn/expect';
export default function ({ getPageObjects }) {
const PageObjects = getPageObjects(['maps', 'header']);

describe.skip('layer errors', () => {
describe('layer errors', () => {
before(async () => {
await PageObjects.maps.loadSavedMap('layer with errors');
});
Expand Down Expand Up @@ -90,7 +90,7 @@ export default function ({ getPageObjects }) {
// Flaky test on cloud and windows when run against a snapshot build of 7.11.
// https://github.com/elastic/kibana/issues/91043

it.skip('should diplay error message in layer panel', async () => {
it('should diplay error message in layer panel', async () => {
const errorMsg = await PageObjects.maps.getLayerErrorText(LAYER_NAME);
expect(errorMsg).to.equal(
`Unable to find EMS tile configuration for id: ${MISSING_EMS_ID}. Kibana is unable to access Elastic Maps Service. Contact your system administrator.`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export default function ({ getService }: FtrProviderContext) {
const ml = getService('ml');

// FLAKY: https://github.com/elastic/kibana/issues/93188
describe.skip('total feature importance panel and decision path popover', function () {
describe('total feature importance panel and decision path popover', function () {
Comment on lines 17 to +18
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As you can see from the issues linked in the comment, this has just recently been skipped and we're investigating how to stabilize it. It's failing rather rarely but it's definitely flaky, so chances are that it would pass your PR checks but fail later after re-activation. Please don't remove this skip for now.

const testDataList: Array<{
suiteTitle: string;
archive: string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export default function ({ getService, getPageObjects }) {
});

// Revisit once https://github.com/elastic/eui/issues/1322 is resolved
it.skip('should show indices table with correct rows after sorting by Search Rate Desc', async () => {
it('should show indices table with correct rows after sorting by Search Rate Desc', async () => {
await indicesList.clickSearchCol();
await indicesList.clickSearchCol();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export default function ({ getService, getPageObjects }) {
const retry = getService('retry');

// FLAKY: https://github.com/elastic/kibana/issues/47184
describe.skip('Elasticsearch shard legends', () => {
describe('Elasticsearch shard legends', () => {
const { setup, tearDown } = getLifecycleMethods(getService, getPageObjects);

before(async () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export default function ({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['common', 'console']);

// FLAKY: https://github.com/elastic/kibana/issues/74327
describe.skip('Setup mode metricbeat migration', function () {
describe('Setup mode metricbeat migration', function () {
describe('setup mode btn', () => {
const { setup, tearDown } = getLifecycleMethods(getService, getPageObjects);

Expand Down
4 changes: 2 additions & 2 deletions x-pack/test/functional/apps/monitoring/time_filter.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,15 @@ export default function ({ getService, getPageObjects }) {
});

// FLAKY: https://github.com/elastic/kibana/issues/48910
it.skip('should send another request when clicking Refresh', async () => {
it('should send another request when clicking Refresh', async () => {
await testSubjects.click('querySubmitButton');
const isLoading = await PageObjects.header.isGlobalLoadingIndicatorVisible();
expect(isLoading).to.be(true);
});

// TODO: [cr] I'm not sure this test is any better than the above one, we might need to rely solely on unit tests
// for this functionality
it.skip('should send another request when changing the time picker', async () => {
it('should send another request when changing the time picker', async () => {
await PageObjects.timePicker.setAbsoluteRange(
'Aug 15, 2016 @ 21:00:00.000',
'Aug 16, 2016 @ 00:00:00.000'
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/security/management.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export default function ({ getService, getPageObjects }) {
const CLONE_ROLES_PATH = `${ROLES_PATH}/clone`;

// FLAKY: https://github.com/elastic/kibana/issues/61173
describe.skip('Management', function () {
describe('Management', function () {
this.tags(['skipFirefox']);

before(async () => {
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/spaces/spaces_selection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export default function spaceSelectorFunctonalTests({
]);

// FLAKY: https://github.com/elastic/kibana/issues/51942
describe.skip('Spaces', function () {
describe('Spaces', function () {
this.tags('includeFirefox');
describe('Space Selector', () => {
before(async () => {
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/status_page/status_page.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export default function statusPageFunctonalTests({
const PageObjects = getPageObjects(['security', 'statusPage', 'home']);

// FLAKY: https://github.com/elastic/kibana/issues/50448
describe.skip('Status Page', function () {
describe('Status Page', function () {
this.tags(['skipCloud', 'includeFirefox']);
before(async () => await esArchiver.load('empty_kibana'));
after(async () => await esArchiver.unload('empty_kibana'));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export default function upgradeAssistantFunctionalTests({
const retry = getService('retry');

// Failing: See https://github.com/elastic/kibana/issues/86546
describe.skip('Upgrade Checkup', function () {
describe('Upgrade Checkup', function () {
this.tags('includeFirefox');

before(async () => {
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/uptime/locations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
};

// FLAKY: https://github.com/elastic/kibana/issues/85208
describe.skip('Observer location', () => {
describe('Observer location', () => {
const start = '~ 15 minutes ago';
const end = 'now';

Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/uptime/overview.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const testSubjects = getService('testSubjects');

// FLAKY: https://github.com/elastic/kibana/issues/89072
describe.skip('overview page', function () {
describe('overview page', function () {
const DEFAULT_DATE_START = 'Sep 10, 2019 @ 12:40:08.078';
const DEFAULT_DATE_END = 'Sep 11, 2019 @ 19:40:08.078';

Expand Down