Basic playwright electron support (#12207)

This change allows to write tests against an Electron app by starting the
Electron app first and then attaching Playwright to it. Also, it solves
the workspace problem by creating it upfront and pointing to it on app
start.

Because playwright cannot access native UI elements, we introduce an
optional environment variable THEIA_ELECTRON_DISABLE_NATIVE_ELEMENTS
that, when set to 1, enforces the rendering of HTML-based menus and file
chooser dialogs, which can be accessed by playwright.

Playwright also requires us to disable early rendering of the window,
which we also do via an environment variable
(THEIA_ELECTRON_NO_EARLY_WINDOW).

A unified TheiaAppLoader is introduced that can be used in both browser
and electron tests. All existing tests are adjusted to use the new
AppLoader and can be run in Electron if the environment variable
USE_ELECTRON is set to true. For convenience, a yarn target
ui-tests-electron has been added to the playwright package.json.
Furthermore, the target ui-tests-ci has been added (runs all playwright
tests in the browser and in Electron) and the Github workflow has been
adjusted.

Contributed on behalf of STMicroelectronics

Signed-off-by: Olaf Lessenich <olessenich@eclipsesource.com>
Co-authored-by: Philip Langer <planger@eclipsesource.com>

.github/workflows/playwright.yml

@@ -51,4 +51,4 @@ jobs:
       - name: Test (playwright)
         uses: GabrielBB/xvfb-action@v1
         with:
-          run: yarn test:playwright
+          run: yarn --cwd examples/playwright ui-tests-ci

examples/playwright/configs/playwright.ci.config.ts

@@ -0,0 +1,27 @@
+// *****************************************************************************
+// Copyright (C) 2022 EclipseSource and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { PlaywrightTestConfig } from '@playwright/test';
+import baseConfig from './playwright.config';
+
+const ciConfig: PlaywrightTestConfig = {
+    ...baseConfig,
+    workers: 1,
+    retries: 2,
+    reporter: [['list'], ['allure-playwright'], ['github']]
+};
+
+export default ciConfig;

examples/playwright/playwright.config.ts → examples/playwright/configs/playwright.config.ts

@@ -1,5 +1,5 @@
 // *****************************************************************************
-// Copyright (C) 2021-2023 logi.cals GmbH, EclipseSource and others.
+// Copyright (C) 2021 logi.cals GmbH, EclipseSource and others.
 //
 // This program and the accompanying materials are made available under the
 // terms of the Eclipse Public License v. 2.0 which is available at
@@ -11,41 +11,35 @@
 // with the GNU Classpath Exception which is available at
 // https://www.gnu.org/software/classpath/license.html.
 //
-// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import { defineConfig } from '@playwright/test';
+import { PlaywrightTestConfig } from '@playwright/test';
 
-// Note: process.env.CI is always set to true for GitHub Actions
-
-export default defineConfig({
-    testDir: './lib/tests',
-    testMatch: ['**/*.test.js'],
-    workers: process.env.CI ? 1 : 2,
-    retries: process.env.CI ? 1 : 0,
-    // The number of times to repeat each test, useful for debugging flaky tests
-    repeatEach: 1,
+const config: PlaywrightTestConfig = {
+    testDir: '../lib/tests',
+    testMatch: ['**/*.js'],
+    workers: 1,
+    fullyParallel: false,
     // Timeout for each test in milliseconds.
-    timeout: 30 * 1000,
+    timeout: 60 * 1000,
     use: {
         baseURL: 'http://localhost:3000',
         browserName: 'chromium',
-        screenshot: 'only-on-failure',
         permissions: ['clipboard-read'],
-        viewport: { width: 1920, height: 1080 },
-    },
-    snapshotDir: './src/tests/snapshots',
-    expect: {
-        toMatchSnapshot: { threshold: 0.01 }
+        screenshot: 'only-on-failure'
     },
     preserveOutput: 'failures-only',
-    reporter: process.env.CI
-        ? [['list'], ['allure-playwright'], ['github']]
-        : [['list'], ['allure-playwright']],
+    reporter: [
+        ['list'],
+        ['allure-playwright']
+    ],
     // Reuse Theia backend on port 3000 or start instance before executing the tests
     webServer: {
         command: 'yarn theia:start',
         port: 3000,
         reuseExistingServer: true
     }
-});
+};
+
+export default config;

examples/playwright/src/tests/fixtures/theia-fixture.ts → examples/playwright/configs/playwright.debug.config.ts

@@ -11,15 +11,17 @@
 // with the GNU Classpath Exception which is available at
 // https://www.gnu.org/software/classpath/license.html.
 //
-// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import test, { Page } from '@playwright/test';
+import { PlaywrightTestConfig } from '@playwright/test';
 
-export let page: Page;
+import baseConfig from './playwright.config';
 
-test.beforeAll(async ({ browser }) => {
-    page = await browser.newPage();
-});
+const debugConfig: PlaywrightTestConfig = {
+    ...baseConfig,
+    workers: 1,
+    timeout: 15000000
+};
 
-export default test;
+export default debugConfig;

examples/playwright/configs/playwright.headful.config.ts

@@ -0,0 +1,30 @@
+// *****************************************************************************
+// Copyright (C) 2021 logi.cals GmbH, EclipseSource and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { PlaywrightTestConfig } from '@playwright/test';
+
+import baseConfig from './playwright.config';
+
+const headfulConfig: PlaywrightTestConfig = {
+    ...baseConfig,
+    workers: 1,
+    use: {
+        ...baseConfig.use,
+        headless: false
+    }
+};
+
+export default headfulConfig;

examples/playwright/configs/ui-tests.eslintrc.json

@@ -0,0 +1,7 @@
+{
+  // override existing rules for ui-tests package
+  "rules": {
+    "no-undef": "off", // disabled due to 'browser', '$', '$$'
+    "no-unused-expressions": "off"
+  }
+}

examples/playwright/configs/ui-tests.playwright.eslintrc.json

@@ -0,0 +1,6 @@
+{
+  // override existing rules for ui-tests playwright package
+  "rules": {
+    "no-null/no-null": "off"
+  }
+}

examples/playwright/package.json

@@ -19,8 +19,10 @@
     "lint": "eslint -c ./.eslintrc.js --ext .ts ./src",
     "lint:fix": "eslint -c ./.eslintrc.js --ext .ts ./src --fix",
     "playwright:install": "playwright install chromium",
-    "ui-tests": "yarn build && playwright test",
-    "ui-tests-headful": "yarn build && playwright test --headed",
+    "ui-tests": "yarn build && playwright test --config=./configs/playwright.config.ts",
+    "ui-tests-electron": "yarn build && USE_ELECTRON=true playwright test --config=./configs/playwright.config.ts",
+    "ui-tests-ci": "yarn build && playwright test --config=./configs/playwright.ci.config.ts",
+    "ui-tests-headful": "yarn build && playwright test --config=./configs/playwright.headful.config.ts",
     "ui-tests-report-generate": "allure generate ./allure-results --clean -o allure-results/allure-report",
     "ui-tests-report": "yarn ui-tests-report-generate && allure open allure-results/allure-report"
   },

examples/playwright/src/index.ts

@@ -16,6 +16,7 @@
 
 export * from './theia-about-dialog';
 export * from './theia-app';
+export * from './theia-app-loader';
 export * from './theia-context-menu';
 export * from './theia-dialog';
 export * from './theia-editor';

examples/playwright/src/tests/theia-app.test.ts

@@ -14,20 +14,19 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
 // *****************************************************************************
 
+import { expect, test } from '@playwright/test';
+import { TheiaAppLoader } from '../theia-app-loader';
 import { TheiaApp } from '../theia-app';
 
-import { expect } from '@playwright/test';
-import test, { page } from './fixtures/theia-fixture';
-
-let app: TheiaApp;
-
 test.describe('Theia Application', () => {
+    let app: TheiaApp;
 
-    test('should load', async () => {
-        app = await TheiaApp.load(page);
+    test.afterAll(async () => {
+        await app.page.close();
     });
 
-    test('should show main content panel', async () => {
+    test('should load and should show main content panel', async ({ playwright, browser }) => {
+        app = await TheiaAppLoader.load({ playwright, browser });
         expect(await app.isMainContentPanelVisible()).toBe(true);
     });
 

examples/playwright/src/tests/theia-explorer-view.test.ts

@@ -14,24 +14,37 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import { expect } from '@playwright/test';
+import { expect, test } from '@playwright/test';
+import { TheiaAppLoader } from '../theia-app-loader';
 import { TheiaApp } from '../theia-app';
+import { PreferenceIds, TheiaPreferenceView } from '../theia-preference-view';
 import { DOT_FILES_FILTER, TheiaExplorerView } from '../theia-explorer-view';
 import { TheiaWorkspace } from '../theia-workspace';
-import test, { page } from './fixtures/theia-fixture';
-
-let app: TheiaApp;
-let explorer: TheiaExplorerView;
 
 test.describe('Theia Explorer View', () => {
 
-    test.beforeAll(async () => {
+    let app: TheiaApp;
+    let explorer: TheiaExplorerView;
+
+    test.beforeAll(async ({ playwright, browser }) => {
         const ws = new TheiaWorkspace(['src/tests/resources/sample-files1']);
-        app = await TheiaApp.load(page, ws);
+        app = await TheiaAppLoader.load({ playwright, browser }, ws);
+
+        if (app.isElectron) {
+            // set trash preference to off
+            const preferenceView = await app.openPreferences(TheiaPreferenceView);
+            await preferenceView.setBooleanPreferenceById(PreferenceIds.Files.EnableTrash, false);
+            await preferenceView.close();
+        }
+
         explorer = await app.openView(TheiaExplorerView);
         await explorer.waitForVisibleFileNodes();
     });
 
+    test.afterAll(async () => {
+        await app.page.close();
+    });
+
     test('should be visible and active after being opened', async () => {
         expect(await explorer.isTabVisible()).toBe(true);
         expect(await explorer.isDisplayed()).toBe(true);
@@ -131,7 +144,9 @@ test.describe('Theia Explorer View', () => {
         const menuItems = await menu.visibleMenuItems();
         expect(menuItems).toContain('Open');
         expect(menuItems).toContain('Delete');
-        expect(menuItems).toContain('Download');
+        if (!app.isElectron) {
+            expect(menuItems).toContain('Download');
+        }
 
         await menu.close();
         expect(await menu.isOpen()).toBe(false);
@@ -186,4 +201,11 @@ test.describe('Theia Explorer View', () => {
         expect(updatedFileStatElements.length).toBe(fileStatElements.length - 1);
     });
 
+    test('open "sample.txt" via the context menu', async () => {
+        expect(await explorer.existsFileNode('sample.txt')).toBe(true);
+        await explorer.clickContextMenuItem('sample.txt', ['Open']);
+        const span = await app.page.waitForSelector('span:has-text("content line 2")');
+        expect(await span.isVisible()).toBe(true);
+    });
+
 });

examples/playwright/src/tests/theia-main-menu.test.ts

@@ -14,21 +14,27 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import { expect } from '@playwright/test';
-import { OSUtil } from '../util';
+import { expect, test } from '@playwright/test';
 import { TheiaApp } from '../theia-app';
+import { TheiaAppLoader } from '../theia-app-loader';
+import { TheiaAboutDialog } from '../theia-about-dialog';
 import { TheiaMenuBar } from '../theia-main-menu';
-import test, { page } from './fixtures/theia-fixture';
-
-let menuBar: TheiaMenuBar;
+import { OSUtil } from '../util';
 
 test.describe('Theia Main Menu', () => {
 
-    test.beforeAll(async () => {
-        const app = await TheiaApp.load(page);
+    let app: TheiaApp;
+    let menuBar: TheiaMenuBar;
+
+    test.beforeAll(async ({ playwright, browser }) => {
+        app = await TheiaAppLoader.load({ playwright, browser });
         menuBar = app.menuBar;
     });
 
+    test.afterAll(async () => {
+        await app.page.close();
+    });
+
     test('should show the main menu bar', async () => {
         const menuBarItems = await menuBar.visibleMenuBarItems();
         expect(menuBarItems).toContain('File');
@@ -57,7 +63,7 @@ test.describe('Theia Main Menu', () => {
         expect(label).toBe('New Text File');
 
         const shortCut = await menuItem?.shortCut();
-        expect(shortCut).toBe(OSUtil.isMacOS ? '⌥ N' : 'Alt+N');
+        expect(shortCut).toBe(OSUtil.isMacOS ? '⌥ N' : app.isElectron ? 'Ctrl+N' : 'Alt+N');
 
         const hasSubmenu = await menuItem?.hasSubmenu();
         expect(hasSubmenu).toBe(false);
@@ -86,4 +92,21 @@ test.describe('Theia Main Menu', () => {
         expect(await mainMenu.isOpen()).toBe(false);
     });
 
+    test('open about dialog using menu', async () => {
+        await (await menuBar.openMenu('Help')).clickMenuItem('About');
+        const aboutDialog = new TheiaAboutDialog(app);
+        expect(await aboutDialog.isVisible()).toBe(true);
+        await aboutDialog.page.getByRole('button', { name: 'OK' }).click();
+        expect(await aboutDialog.isVisible()).toBe(false);
+    });
+
+    test('open file via file menu and cancel', async () => {
+        const openFileEntry = app.isElectron ? 'Open File...' : 'Open...';
+        await (await menuBar.openMenu('File')).clickMenuItem(openFileEntry);
+        const fileDialog = await app.page.waitForSelector('div[class="dialogBlock"]');
+        expect(await fileDialog.isVisible()).toBe(true);
+        await app.page.locator('#theia-dialog-shell').getByRole('button', { name: 'Cancel' }).click();
+        expect(await fileDialog.isVisible()).toBe(false);
+    });
+
 });

examples/playwright/src/tests/theia-output-view.test.ts

@@ -14,20 +14,21 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import { expect } from '@playwright/test';
+import { expect, test } from '@playwright/test';
 import { TheiaOutputViewChannel } from '../theia-output-channel';
 import { TheiaApp } from '../theia-app';
+import { TheiaAppLoader } from '../theia-app-loader';
 import { TheiaOutputView } from '../theia-output-view';
-import test, { page } from './fixtures/theia-fixture';
-
-let app: TheiaApp;
-let outputView: TheiaOutputView;
-let testChannel: TheiaOutputViewChannel;
 
+let app: TheiaApp; let outputView: TheiaOutputView; let testChannel: TheiaOutputViewChannel;
 test.describe('Theia Output View', () => {
 
-    test.beforeAll(async () => {
-        app = await TheiaApp.load(page);
+    test.beforeAll(async ({ playwright, browser }) => {
+        app = await TheiaAppLoader.load({ playwright, browser });
+    });
+
+    test.afterAll(async () => {
+        await app.page.close();
     });
 
     test('should open the output view and check if is visible and active', async () => {
@@ -82,4 +83,3 @@ test.describe('Theia Output View', () => {
     });
 
 });
-