Jetpack Boost: Split and Parallelize Jetpack Boost Tests (#32686)

* Split Boost tests into directories

* Run Jetpack Boost tests in parallel

* Add changelog entry.

* Invalid colon

* [test] cache artifacts & build artifacts

* Revert "[test] cache artifacts & build artifacts"

This reverts commit f911559.

* Introduce build groups

* Add strategy

* Correct artifact directory

* Cache build output

* Use v3 cache

* Replace artifacts with cache

* Attempt to fix paths

* Use changed files as the cache hash

* Revert "Use changed files as the cache hash"

This reverts commit 492d868.

* Fix cache paths

* Cache the whole matrix path directory

* Skipped jobs are ok

* Fix the key

* Cache the whole directory, not just matrix path

* Always run slack notification and test report

* Rename "Setup" to "Base"

To avoid conflicting naming throughout the workflow runs. "Setup" is kind of confusing.

* Revert "Always run slack notification and test report"

This reverts commit d5d5125.

* Remove always()

* Matrix over concurrency

* Ensure build cache

* Group Jetpack projects into jetpack-core

* Restore repo check

* build-matrix -> matrix

* Fetch the build matrix

* Don't need the project name here

* Clean up job flow

* Create both the test and build matrices in `create-test-matrix`.
* Generate an empty build matrix if no build is needed.
* Don't generate build entries for "atomic" suites.
* Check build matrix for having more than one path.
* Skip `build-projects` job if the matrix is empty.
* Remove suite check in `build-projects` build step since we do it in `create-test-matrix` now.
* We have to run `e2e-tests` even if `build-projects` is skipped, but not it `create-test-matrix` was skipped.
* Skip `e2e-tests` if test matrix is empty, otherwise GH complains.
* Only restore build artifact if we need it.
* Complain if build job wasn't skipped when it should have been.
* Complain if artifact restore didn't when it should have.
* Revert changes to reporting jobs' conditions.

* Try caching only added files

* Source funcs file before the cd

* Remove workflow_run stuff, trigger was removed in #27623 last year

* Need to cd back to GITHUB_WORKSPACE for the after `find` anyway

* Can't pass too long of a list to actions/cache/save. So remove unchanged files insead.

* Can't delete `./.github`, it breaks "Post Setup tools".

* Sigh, xargs

* `path` needs to match, sigh

* Remove the extra dollar 💸

---------

Co-authored-by: Brad Jorsch <brad.jorsch@automattic.com>

.github/files/e2e-tests/e2e-matrix.js

@@ -8,62 +8,87 @@ const projects = [
 		testArgs: [ 'specs/connection', '--retries=1' ],
 		targets: [ 'plugins/jetpack' ],
 		suite: '',
+		buildGroup: 'jetpack-core',
 	},
 	{
 		project: 'Jetpack pre-connection',
 		path: 'projects/plugins/jetpack/tests/e2e',
 		testArgs: [ 'specs/pre-connection', '--retries=1' ],
 		targets: [ 'plugins/jetpack', 'monorepo' ],
 		suite: '',
+		buildGroup: 'jetpack-core',
 	},
 	{
 		project: 'Jetpack post-connection',
 		path: 'projects/plugins/jetpack/tests/e2e',
 		testArgs: [ 'specs/post-connection', '--retries=1' ],
 		targets: [ 'plugins/jetpack' ],
 		suite: '',
+		buildGroup: 'jetpack-core',
 	},
 	{
 		project: 'Jetpack sync',
 		path: 'projects/plugins/jetpack/tests/e2e',
 		testArgs: [ 'specs/sync', '--retries=1' ],
 		targets: [ 'packages/sync' ],
 		suite: '',
+		buildGroup: 'jetpack-sync',
 	},
 	{
 		project: 'Jetpack blocks',
 		path: 'projects/plugins/jetpack/tests/e2e',
 		testArgs: [ 'specs/blocks', '--retries=1' ],
 		targets: [ 'plugins/jetpack' ],
 		suite: '',
+		buildGroup: 'jetpack-core',
 	},
 	{
-		project: 'Boost',
+		project: 'Jetpack Boost - Base',
 		path: 'projects/plugins/boost/tests/e2e',
-		testArgs: [],
+		testArgs: [ 'specs/base', '--retries=1' ],
 		targets: [ 'plugins/boost' ],
 		suite: '',
+		buildGroup: 'jetpack-boost',
+	},
+	{
+		project: 'Jetpack Boost - Modules',
+		path: 'projects/plugins/boost/tests/e2e',
+		testArgs: [ 'specs/modules', '--retries=1' ],
+		targets: [ 'plugins/boost' ],
+		suite: '',
+		buildGroup: 'jetpack-boost',
+	},
+	{
+		project: 'Jetpack Boost - Critical CSS',
+		path: 'projects/plugins/boost/tests/e2e',
+		testArgs: [ 'specs/critical-css', '--retries=1' ],
+		targets: [ 'plugins/boost' ],
+		suite: '',
+		buildGroup: 'jetpack-boost',
 	},
 	{
 		project: 'Search',
 		path: 'projects/plugins/search/tests/e2e',
 		testArgs: [],
 		targets: [ 'plugins/search' ],
 		suite: '',
+		buildGroup: 'jetpack-search',
 	},
 	{
 		project: 'VideoPress',
 		path: 'projects/plugins/videopress/tests/e2e',
 		testArgs: [],
 		targets: [ 'plugins/videopress' ],
 		suite: '',
+		buildGroup: 'jetpack-videopress',
 	},
 	{
 		project: 'Social',
 		path: 'projects/plugins/social/tests/e2e',
 		testArgs: [],
 		targets: [ 'plugins/social' ],
 		suite: '',
+		buildGroup: 'jetpack-social',
 	},
 ];
 
@@ -94,10 +119,6 @@ switch ( process.env.GITHUB_EVENT_NAME ) {
 		}
 		break;
 	}
-	case 'workflow_run': {
-		matrix.push( ...projects );
-		break;
-	}
 	case 'repository_dispatch':
 		if ( process.env.DISPATCH_REPO ) {
 			const repoName = process.env.DISPATCH_REPO.split( '/' )[ 1 ];