E2E perf tests: run each test in a separate page (#47889)

* E2E perf tests: run each test in a separate page * Throw away 1st measurement to avoid caching quirks * Move page closing to `beforeEach` * Move metric collection inside conditional * Revert "Move page closing to `beforeEach`" This reverts commit 2b092e2.
WordPress · Feb 10, 2023 · fab70c5 · fab70c5 · github-actions · Feb 10, 2023
1 parent bb61b4c
commit fab70c5
Show file tree

Hide file tree

Showing 3 changed files with 114 additions and 65 deletions.
diff --git a/packages/e2e-tests/config/setup-performance-test.js b/packages/e2e-tests/config/setup-performance-test.js
@@ -19,9 +19,11 @@ const PUPPETEER_TIMEOUT = process.env.PUPPETEER_TIMEOUT;
 // The Jest timeout is increased because these tests are a bit slow.
 jest.setTimeout( PUPPETEER_TIMEOUT || 100000 );
 
-async function setupBrowser() {
-	await clearLocalStorage();
+async function setupPage() {
 	await setBrowserViewport( 'large' );
+	await page.emulateMediaFeatures( [
+		{ name: 'prefers-reduced-motion', value: 'reduce' },
+	] );
 }
 
 // Before every test suite run, delete all content created by the test. This ensures
@@ -32,13 +34,18 @@ beforeAll( async () => {
 
 	await trashAllPosts();
 	await trashAllPosts( 'wp_block' );
-	await setupBrowser();
+	await clearLocalStorage();
+	await setupPage();
 	await activatePlugin( 'gutenberg-test-plugin-disables-the-css-animations' );
-	await page.emulateMediaFeatures( [
-		{ name: 'prefers-reduced-motion', value: 'reduce' },
-	] );
 } );
 
 afterEach( async () => {
-	await setupBrowser();
+	// Clear localStorage between tests so that the next test starts clean.
+	await clearLocalStorage();
+	// Close the previous page entirely and create a new page, so that the next test
+	// isn't affected by page unload work.
+	await page.close();
+	page = await browser.newPage();
+	// Set up testing config on new page.
+	await setupPage();
 } );
diff --git a/packages/e2e-tests/specs/performance/post-editor.test.js b/packages/e2e-tests/specs/performance/post-editor.test.js
@@ -105,28 +105,43 @@ describe( 'Post Editor Performance', () => {
 			readFile( join( __dirname, '../../assets/large-post.html' ) )
 		);
 		await saveDraft();
-		let i = 5;
+		const draftURL = await page.url();
+
+		// Number of sample measurements to take.
+		const samples = 5;
+		// Number of throwaway measurements to perform before recording samples.
+		// Having at least one helps ensure that caching quirks don't manifest in
+		// the results.
+		const throwaway = 1;
+
+		let i = throwaway + samples;
 		while ( i-- ) {
-			await page.reload();
+			await page.close();
+			page = await browser.newPage();
+
+			await page.goto( draftURL );
 			await page.waitForSelector( '.edit-post-layout', {
 				timeout: 120000,
 			} );
 			await canvas().waitForSelector( '.wp-block', { timeout: 120000 } );
-			const {
-				serverResponse,
-				firstPaint,
-				domContentLoaded,
-				loaded,
-				firstContentfulPaint,
-				firstBlock,
-			} = await getLoadingDurations();
 
-			results.serverResponse.push( serverResponse );
-			results.firstPaint.push( firstPaint );
-			results.domContentLoaded.push( domContentLoaded );
-			results.loaded.push( loaded );
-			results.firstContentfulPaint.push( firstContentfulPaint );
-			results.firstBlock.push( firstBlock );
+			if ( i < samples ) {
+				const {
+					serverResponse,
+					firstPaint,
+					domContentLoaded,
+					loaded,
+					firstContentfulPaint,
+					firstBlock,
+				} = await getLoadingDurations();
+
+				results.serverResponse.push( serverResponse );
+				results.firstPaint.push( firstPaint );
+				results.domContentLoaded.push( domContentLoaded );
+				results.loaded.push( loaded );
+				results.firstContentfulPaint.push( firstContentfulPaint );
+				results.firstBlock.push( firstBlock );
+			}
 		}
 	} );
 

diff --git a/packages/e2e-tests/specs/performance/site-editor.test.js b/packages/e2e-tests/specs/performance/site-editor.test.js
@@ -30,34 +30,29 @@ import {
 
 jest.setTimeout( 1000000 );
 
+const results = {
+	serverResponse: [],
+	firstPaint: [],
+	domContentLoaded: [],
+	loaded: [],
+	firstContentfulPaint: [],
+	firstBlock: [],
+	type: [],
+	typeContainer: [],
+	focus: [],
+	inserterOpen: [],
+	inserterHover: [],
+	inserterSearch: [],
+	listViewOpen: [],
+};
+
+let id;
+
 describe( 'Site Editor Performance', () => {
 	beforeAll( async () => {
 		await activateTheme( 'emptytheme' );
 		await deleteAllTemplates( 'wp_template' );
 		await deleteAllTemplates( 'wp_template_part' );
-	} );
-	afterAll( async () => {
-		await deleteAllTemplates( 'wp_template' );
-		await deleteAllTemplates( 'wp_template_part' );
-		await activateTheme( 'twentytwentyone' );
-	} );
-
-	it( 'Loading', async () => {
-		const results = {
-			serverResponse: [],
-			firstPaint: [],
-			domContentLoaded: [],
-			loaded: [],
-			firstContentfulPaint: [],
-			firstBlock: [],
-			type: [],
-			typeContainer: [],
-			focus: [],
-			inserterOpen: [],
-			inserterHover: [],
-			inserterSearch: [],
-			listViewOpen: [],
-		};
 
 		const html = readFile(
 			join( __dirname, '../../assets/large-post.html' )
@@ -80,37 +75,69 @@ describe( 'Site Editor Performance', () => {
 		}, html );
 		await saveDraft();
 
-		const id = await page.evaluate( () =>
+		id = await page.evaluate( () =>
 			new URL( document.location ).searchParams.get( 'post' )
 		);
+	} );
 
+	afterAll( async () => {
+		await deleteAllTemplates( 'wp_template' );
+		await deleteAllTemplates( 'wp_template_part' );
+		await activateTheme( 'twentytwentyone' );
+	} );
+
+	beforeEach( async () => {
 		await visitSiteEditor( { postId: id, postType: 'page' } );
+	} );
+
+	it( 'Loading', async () => {
+		const editorURL = await page.url();
 
-		let i = 3;
+		// Number of sample measurements to take.
+		const samples = 3;
+		// Number of throwaway measurements to perform before recording samples.
+		// Having at least one helps ensure that caching quirks don't manifest in
+		// the results.
+		const throwaway = 1;
+
+		let i = throwaway + samples;
 
 		// Measuring loading time.
 		while ( i-- ) {
-			await page.reload();
+			await page.close();
+			page = await browser.newPage();
+
+			await page.goto( editorURL );
 			await page.waitForSelector( '.edit-site-visual-editor', {
 				timeout: 120000,
 			} );
 			await canvas().waitForSelector( '.wp-block', { timeout: 120000 } );
-			const {
-				serverResponse,
-				firstPaint,
-				domContentLoaded,
-				loaded,
-				firstContentfulPaint,
-				firstBlock,
-			} = await getLoadingDurations();
-
-			results.serverResponse.push( serverResponse );
-			results.firstPaint.push( firstPaint );
-			results.domContentLoaded.push( domContentLoaded );
-			results.loaded.push( loaded );
-			results.firstContentfulPaint.push( firstContentfulPaint );
-			results.firstBlock.push( firstBlock );
+
+			if ( i < samples ) {
+				const {
+					serverResponse,
+					firstPaint,
+					domContentLoaded,
+					loaded,
+					firstContentfulPaint,
+					firstBlock,
+				} = await getLoadingDurations();
+
+				results.serverResponse.push( serverResponse );
+				results.firstPaint.push( firstPaint );
+				results.domContentLoaded.push( domContentLoaded );
+				results.loaded.push( loaded );
+				results.firstContentfulPaint.push( firstContentfulPaint );
+				results.firstBlock.push( firstBlock );
+			}
 		}
+	} );
+
+	it( 'Typing', async () => {
+		await page.waitForSelector( '.edit-site-visual-editor', {
+			timeout: 120000,
+		} );
+		await canvas().waitForSelector( '.wp-block', { timeout: 120000 } );
 
 		// Measuring typing performance inside the post content.
 		await canvas().waitForSelector(
@@ -121,7 +148,7 @@ describe( 'Site Editor Performance', () => {
 			'[data-type="core/post-content"] [data-type="core/paragraph"]'
 		);
 		await insertBlock( 'Paragraph' );
-		i = 200;
+		let i = 200;
 		const traceFile = __dirname + '/trace.json';
 		await page.tracing.start( {
 			path: traceFile,