Run macOS E2E tests on Buildkite (#207)

* WIP

* Makes onboarding work!

* Make create site work

* Make delete site work

* All E2E tests work 🎉

* Fix E2E tearDown cleanup condition

* Cleanup

* Add a check for consistency

* Only run onboarding when needed

* Improve debugging

* Remove useless comment

* Address lint issues

* Run lint, unit, and E2E tests on Buildkite (issue with caching)

* Remove E2E step – It's failing and don't want to slow the unit tests

* Add step to run E2E tests in Buildkite

Currently failing, see #37

* Add build matrix with Windows and Mac queues for E2E tests

* Call E2E with `bash` explicitly

* Add OS name in E2E tests label

* Add traces to E2E runs

* Fix E2E matrix combinations

* Fix lint issues

* Make paths system agnostic

* Remove no longer valid fixme comment

* Clarify comment

* Use latest CI plugin version

* Use new variables in pipeline

* Fix executable path of Windows build in E2E tests

* Use `throw new Error` instead of undefined `fail` to fail E2E teardown

* Run E2E in Buildkite with `DEBUG=pw:browser`

* Remove a trailing whitespace

* Test - Run unit tests on Windows, too

* Remove Windows test steps in Buildkite

Deferring enabling them till we'll have found a solution for
Windows-specific issues.

In the meantime, I aim to run E2E macOS tests on Buildkite instead of
GitHub Actions in all PRs.

* Remove macOS as a target for E2E tests in GitHub Actions

* Extract CI toolkit version in dedicated var

This should make it easier to locate and update the version if needed

* Use shared var to define NVM CI plugin

* Remove `agents: queue: mac` from the pipeline root

We have a variety of steps and platforms in the pipeline now. It no
longer makes sense to default to macOS.

---------

Co-authored-by: Wojtek Naruniec <wojtek@naruniec.me>
Co-authored-by: Carlos Garcia <fluiddot@gmail.com>

.buildkite/pipeline.yml

@@ -1,14 +1,13 @@
 # Nodes with values to reuse in the pipeline.
 common_params:
   plugins: &common_plugins
-    - automattic/a8c-ci-toolkit#3.4.2
-    - automattic/nvm#0.3.0
+    - $CI_TOOLKIT_PLUGIN
+    - $NVM_PLUGIN
 
-# Most of steps use the same agent and environment; we can define them as top-level
-agents:
-  queue: mac
+# IMAGE_ID is an env var that only macOS agents need.
+# Defining it at the root level propagates it too all agents, which can seem unnecessary but is a the same time convenient and DRY.
 env:
-  IMAGE_ID: xcode-15.2
+  IMAGE_ID: $IMAGE_ID
 
 steps:
   - label: Lint
@@ -17,8 +16,6 @@ steps:
       .buildkite/commands/install-node-dependencies.sh
       echo '--- :eslint: Lint'
       npm run lint
-    agents:
-      queue: default
     plugins: *common_plugins
 
   - label: Unit Tests
@@ -27,14 +24,41 @@ steps:
       .buildkite/commands/install-node-dependencies.sh
       echo '--- :npm: Run Unit Tests'
       npm run test
-    agents:
-      queue: default
     plugins: *common_plugins
 
+  # Notice that we use the queue name (matrix value) in the label.
+  # We could have added values for queue and label, but that would have resulted in 'macOS + windows' and 'Windows + mac' combinations, which we don't want.
+  # Buildkite offers an option to remove those combinations, but the overhead in code readability did not seem worth the visual effect in the build page.
+  # See https://buildkite.com/docs/pipelines/build-matrix#removing-combinations-from-the-build-matrix
+  - label: "E2E Tests on {{ matrix }}"
+    key: e2e_tests
+    command: |
+      # call with bash explicitly because we run this on Windows, too
+      bash .buildkite/commands/install-node-dependencies.sh
+      echo '--- :package: Package app for testing'
+      npm run package
+      echo '--- :playwright: Run End To End Tests'
+      npm run e2e
+    artifact_paths:
+      - test-results/**/*.zip
+    plugins: *common_plugins
+    agents:
+      queue: "{{ matrix }}"
+    env:
+      # See https://playwright.dev/docs/ci#debugging-browser-launches
+      DEBUG: 'pw:browser'
+    matrix:
+      - mac
+      # Windows destination disabled till we find the cause of Windows-specific test failures
+      # See https://github.com/Automattic/studio/pull/76#issuecomment-2146529123
+      # - windows
+
   - group: "📦 Build for Mac"
     key: 'dev-mac'
     steps:
       - label: '🔨 Mac Dev Build - {{matrix}}'
+        agents:
+          queue: mac
         command: |
           .buildkite/commands/prepare-environment.sh
 
@@ -63,7 +87,7 @@ steps:
       - 'out\**\studio-update.nupkg'
       - 'out\**\RELEASES'
     agents:
-      queue: "windows"
+      queue: windows
     if: "build.tag !~ /^v[0-9]+/"
 
   - label: ':rocket: Distribute Dev Builds'
@@ -85,7 +109,7 @@ steps:
     artifact_paths:
       - 'out/releases.json'
     agents:
-      queue: "mac" #
+      queue: mac
     # Using concurrency_group to ensure the CI builds from `trunk` & the git tag, which are likely to run at roughly the
     # same time, don't run into a race condition when downloading+updating+uploading the `releases.json` file from/to S3
     concurrency_group: 'studio/release-manifest-update'
@@ -101,6 +125,8 @@ steps:
     key: 'release-mac'
     steps:
       - label: '🔨 Mac Release Build - {{matrix}}'
+        agents:
+          queue: mac
         command: |
           .buildkite/commands/prepare-environment.sh
 
@@ -125,7 +151,7 @@ steps:
     key: 'release-windows'
     command: .buildkite/commands/build-for-windows-release.ps1
     agents:
-      queue: "windows"
+      queue: windows
     artifact_paths:
       - 'out\**\studio-setup.exe'
       - 'out\**\studio-update.nupkg'
@@ -151,7 +177,7 @@ steps:
     artifact_paths:
       - 'out/releases.json'
     agents:
-      queue: "mac" # 
+      queue: mac
     # Using concurrency_group to ensure the CI builds from `trunk` & the git tag, which are likely to run at roughly the
     # same time, don't run into a race condition when downloading+updating+uploading the `releases.json` file from/to S3
     concurrency_group: 'studio/release-manifest-update'

.buildkite/shared-pipeline-vars

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# This file is `source`'d in CI before calling `buildkite-agent pipeline upload`, and can be used
+# to set up some variables that will be interpolated in the `.yml` pipeline before uploading it.
+
+# The ~> modifier is not currently used, but we check for it just in case
+XCODE_VERSION=$(sed -E -n 's/^(~> )?(.*)/xcode-\2/p' .xcode-version)
+CI_TOOLKIT_VERSION='3.4.2'
+NVM_PLUGIN_VERSION='0.3.0'
+
+export IMAGE_ID="$XCODE_VERSION"
+export CI_TOOLKIT_PLUGIN="automattic/a8c-ci-toolkit#$CI_TOOLKIT_VERSION"
+export NVM_PLUGIN="automattic/nvm#$NVM_PLUGIN_VERSION"

.github/workflows/run-tests.yml

@@ -8,7 +8,6 @@ jobs:
     strategy:
       matrix:
         os:
-          - macos-latest
           - windows-latest
 
     runs-on: ${{ matrix.os }}

.gitignore

@@ -107,3 +107,6 @@ vendor/*
 
 # Build output
 dist
+
+# Playwright traces
+test-results

.xcode-version

@@ -0,0 +1 @@
+15.2

e2e/e2e-helpers.ts

@@ -7,9 +7,15 @@ export async function launchApp( testEnv: NodeJS.ProcessEnv = {} ) {
 
 	// parse the packaged Electron app and find paths and other info
 	const appInfo = parseElectronApp( latestBuild );
+	let executablePath = appInfo.executable;
+	if ( appInfo.platform === 'win32' ) {
+		// `parseElectronApp` function obtains the executable path by finding the first executable from the build folder.
+		// We need to ensure that the executable is the Studio app.
+		executablePath = executablePath.replace( 'Squirrel.exe', 'Studio.exe' );
+	}
 	const electronApp = await electron.launch( {
 		args: [ appInfo.main ], // main file from package.json
-		executablePath: appInfo.executable, // path to the Electron executable
+		executablePath, // path to the Electron executable
 		env: {
 			...process.env,
 			...testEnv,

e2e/page-objects/settings-tab.ts

@@ -1,4 +1,4 @@
-import { type Page, type ElectronApplication, expect } from '@playwright/test';
+import { type Page, type ElectronApplication } from '@playwright/test';
 
 export default class SettingsTab {
 	constructor(

e2e/sites.test.ts

@@ -15,9 +15,12 @@ test.describe( 'Servers', () => {
 	let electronApp: ElectronApplication;
 	let mainWindow: Page;
 	const siteName = `test-site-${ randomUUID() }`;
-	const tmpSiteDir = `${ tmpdir() }/${ siteName }`;
+	const tmpSiteDir = path.join( tmpdir(), siteName );
 	const defaultOnboardingSiteName = 'My WordPress Website';
-	const onboardingTmpSiteDir = `${ tmpdir() }/${ defaultOnboardingSiteName.replace( /\s/g, '-' ) }`;
+	const onboardingTmpSiteDir = path.join(
+		tmpdir(),
+		defaultOnboardingSiteName.replace( /\s/g, '-' )
+	);
 
 	test.beforeAll( async () => {
 		// Print path in error so we can delete it manually if needed.
@@ -65,7 +68,7 @@ test.describe( 'Servers', () => {
 				try {
 					await fs.rm( dir, { recursive: true } );
 				} catch {
-					fail( `Failed to clean up ${ dir }` );
+					throw new Error( `Failed to clean up ${ dir }` );
 				}
 			}
 		} );

src/setup-wp-server-files.ts

@@ -2,9 +2,8 @@ import path from 'path';
 import fs from 'fs-extra';
 import semver from 'semver';
 import { SQLITE_FILENAME } from '../vendor/wp-now/src/constants';
-import { downloadWPCLI, getWordPressVersionPath } from '../vendor/wp-now/src/download';
+import { getWordPressVersionPath } from '../vendor/wp-now/src/download';
 import getSqlitePath from '../vendor/wp-now/src/get-sqlite-path';
-import getWpCliPath from '../vendor/wp-now/src/get-wp-cli-path';
 import { recursiveCopyDirectory } from './lib/fs-utils';
 import { updateLatestSqliteVersion } from './lib/sqlite-versions';
 import {