E2E tests: Try to reset WP install for every test suite (#13638)

* [not verified] Try to reset WP install for every test suite

* [not verified] use dynamic site URL
- update connect flow to fetch ngrokURL when loading the wp-admin
- get rid of core's `createNewPost` since it relies on static URL that is defined on build-time

* [not verified] Update sidebar page object

* [not verified] Visit block editor directly

* [not verified] add URL logging

* [not verified] fix block editor URL

* [not verified] throw a error if site is not properly connected
- is not connected
- or missing expected plan

* [not verified] fix the cookies URL

* [not verified] use host instead full url

* [not verified] cleanup

* Update gutenblock tests to use Jetpack Start

* [not verified] Add spec name logging

* [not verified] update encrypted config

* clean up

* regenerate config

* reload page to hydrate plan data

* Add ability to log HTML on failure

* Add some dump wait

* log jetpack_active_plan option content

* add more random stuff :fingerscrossed:

* Add Plans page to wait for

* 🤦

* log `jetpack_active_plan` option in s/p block test

* Try to wait for an ad a bit more

* try to properly wait for plan data update

* wait for networkidle on page reload

* add some screenshot logging

* add more logging

* add backend logging

* log into slack instead

* update slack reporter

* more loggig :/

* i don't know what I am doing

* reload frontend page for ads block test

* add back some wait

* add debug lock

* remove debugging info

* try to logout before looking for ads

* visit new post after the logout

* remove delays from `connectThroughJetpackStart`

* add some debugging code for ads test

* add HTML logging

* Add more fancy logging

* remove request logging

* Debug!

* use http url

* trigger jetpack heartbeat

* increase navigation timeout

* try to use classic connection for ads block

* roll back ads test to use classic connection

* cleanup

* remove sandbox cookie

* correctly clean up cookies

* Try to fix mailchimp block test

* fix a passed parameter

* fix styling errors and update changed locator

* add waitForResponse

* add logging to jetpack plan updates

* increase timeout for waitForResponse

* add more wait

* clean-up some debug code

* attempt to decrese waiting time

* try to use jetpack start for wordads block

* revert previous commit

tests/e2e/jest-puppeteer.config.js

@@ -3,21 +3,22 @@
  * https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions
  */
 
+const { CI, E2E_DEBUG, PUPPETEER_HEADLESS, PUPPETEER_SLOWMO } = process.env;
 let executablePath = '';
 let dumpio = false;
-if ( ! process.env.CI ) {
+if ( ! CI ) {
 	executablePath = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
 }
 
-if ( process.env.E2E_DEBUG ) {
+if ( E2E_DEBUG ) {
 	dumpio = true;
 }
 
 module.exports = {
 	launch: {
-		headless: process.env.PUPPETEER_HEADLESS !== 'false',
-		devtools: process.env.PUPPETEER_HEADLESS === 'false',
-		slowMo: parseInt( process.env.PUPPETEER_SLOWMO, 10 ) || 0,
+		headless: PUPPETEER_HEADLESS !== 'false',
+		devtools: PUPPETEER_HEADLESS === 'false',
+		slowMo: parseInt( PUPPETEER_SLOWMO, 10 ) || 0,
 		executablePath,
 		dumpio,
 	},

tests/e2e/lib/flows/jetpack-connect.js

@@ -16,10 +16,15 @@ import WPLoginPage from '../pages/wp-admin/login';
 import CheckoutPage from '../pages/wpcom/checkout';
 import ThankYouPage from '../pages/wpcom/thank-you';
 import MyPlanPage from '../pages/wpcom/my-plan';
+import {
+	getNgrokSiteUrl,
+	provisionJetpackStartConnection,
+	execShellCommand,
+} from '../utils-helper';
+import PlansPage from '../pages/wpcom/plans';
 
 const cookie = config.get( 'storeSandboxCookieValue' );
 const cardCredentials = config.get( 'testCardCredentials' );
-const siteUrl = new URL( process.env.WP_BASE_URL ).host;
 
 /**
  * Connects your site to WPCOM as `wpcomUser`, buys a Professional plan via sandbox cookie
@@ -37,11 +42,14 @@ export async function connectThroughWPAdminIfNeeded( {
 		await login.login( wpcomUser );
 	}
 
-	await ( await WPLoginPage.visit( page ) ).login();
-	await ( await DashboardPage.init( page ) ).setSandboxModeForPayments( cookie, siteUrl );
+	const siteUrl = getNgrokSiteUrl();
+	const host = new URL( siteUrl ).host;
+
+	await ( await WPLoginPage.visit( page, siteUrl + '/wp-login.php' ) ).login();
+	await ( await DashboardPage.init( page ) ).setSandboxModeForPayments( cookie, host );
 	await ( await Sidebar.init( page ) ).selectJetpack();
 
-	const jetpackPage = await JetpackPage.init( page );
+	let jetpackPage = await JetpackPage.init( page );
 	if ( await jetpackPage.isConnected() ) {
 		await jetpackPage.openMyPlan();
 		if ( await jetpackPage.isPlan( plan ) ) {
@@ -68,6 +76,66 @@ export async function connectThroughWPAdminIfNeeded( {
 
 	await ( await MyPlanPage.init( page ) ).returnToWPAdmin();
 
-	await ( await JetpackPage.init( page ) ).waitForPage();
-	await ( await JetpackPage.init( page ) ).setSandboxModeForPayments( cookie, siteUrl );
+	jetpackPage = await JetpackPage.init( page );
+	await jetpackPage.setSandboxModeForPayments( cookie, host );
+
+	// Reload the page to hydrate plans cache
+	await jetpackPage.reload( { waitFor: 'networkidle0' } );
+
+	await page.waitForResponse(
+		response => response.url().includes( 'v4/site?' ) && response.status() === 200
+	);
+
+	await execShellCommand(
+		'wp cron event run jetpack_v2_heartbeat --path="/home/travis/wordpress"'
+	);
+
+	if ( ! ( await jetpackPage.isPlan( plan ) ) ) {
+		throw new Error( `Site does not have ${ plan } plan` );
+	}
+
+	return true;
+}
+
+export async function connectThroughJetpackStart( {
+	wpcomUser = 'defaultUser',
+	plan = 'pro',
+} = {} ) {
+	// remove Sandbox cookie
+	await page.deleteCookie( { name: 'store_sandbox', domain: '.wordpress.com' } );
+
+	// Logs in to WPCOM
+	const login = await LoginPage.visit( page );
+	if ( ! ( await login.isLoggedIn() ) ) {
+		await login.login( wpcomUser );
+	}
+
+	const nextUrl = provisionJetpackStartConnection();
+	await ( await AuthorizePage.visit( page, nextUrl ) ).approve();
+	await ( await PlansPage.init( page ) ).isCurrentPlan( 'business' );
+
+	const siteUrl = getNgrokSiteUrl();
+
+	await ( await WPLoginPage.visit( page, siteUrl + '/wp-login.php' ) ).login();
+	await ( await Sidebar.init( page ) ).selectJetpack();
+
+	const jetpackPage = await JetpackPage.init( page );
+
+	await jetpackPage.openMyPlan();
+
+	await page.waitForResponse(
+		response => response.url().includes( 'v4/site?' ) && response.status() === 200
+	);
+
+	await jetpackPage.reload( { waitFor: 'networkidle0' } );
+
+	await execShellCommand(
+		'wp cron event run jetpack_v2_heartbeat --path="/home/travis/wordpress"'
+	);
+
+	if ( ! ( await jetpackPage.isPlan( plan ) ) ) {
+		throw new Error( `Site does not have ${ plan } plan` );
+	}
+
+	return true;
 }

tests/e2e/lib/jest.test.failure.js

@@ -7,14 +7,14 @@ import { wrap } from 'lodash';
  */
 import { sendFailedTestScreenshotToSlack, sendFailedTestMessageToSlack } from './reporters/slack';
 import { takeScreenshot } from './reporters/screenshot';
-import { readFileSync } from 'fs';
+import { logHTML, logDebugLog } from './page-helper';
 /**
  * Override the test case method so we can take screenshots of assertion failures.
  *
  * See: https://github.com/smooth-code/jest-puppeteer/issues/131#issuecomment-469439666
  */
 let currentBlock;
-const { CI, E2E_DEBUG } = process.env;
+const { CI, E2E_DEBUG, E2E_LOG_HTML } = process.env;
 
 // Use wrap to preserve all previous `wrap`s
 jasmine.getEnv().describe = wrap( jasmine.getEnv().describe, ( func, ...args ) => {
@@ -36,9 +36,11 @@ global.it = async ( name, func ) => {
 				const filePath = await takeScreenshot( currentBlock, name );
 				await sendFailedTestMessageToSlack( { block: currentBlock, name, error } );
 				await sendFailedTestScreenshotToSlack( filePath );
-				const fileContents = readFileSync( '/home/travis/wordpress/wp-content/debug.log' );
-				console.log( '##### WP DEBUG.LOG' );
-				console.log( fileContents.toString() );
+				await logDebugLog();
+			}
+
+			if ( E2E_LOG_HTML ) {
+				logHTML();
 			}
 
 			if ( E2E_DEBUG ) {
@@ -50,3 +52,14 @@ global.it = async ( name, func ) => {
 		}
 	} );
 };
+
+jasmine.getEnv().addReporter( {
+	specStarted( result ) {
+		console.log( `Spec name: ${ result.fullName }, description: ${ result.description }` );
+	},
+} );
+
+jasmine.getEnv().addReporter( {
+	specStarted: result => ( jasmine.currentTest = result ),
+	specDone: result => ( jasmine.currentTest = result ),
+} );

tests/e2e/lib/page-helper.js

@@ -6,9 +6,11 @@ import config from 'config';
  * WordPress dependencies
  */
 import { pressKeyWithModifier } from '@wordpress/e2e-test-utils';
+import { readFileSync } from 'fs';
 /**
  * Internal dependencies
  */
+import { sendSnippetToSlack } from './reporters/slack';
 
 /**
  * Waits for selector to be present in DOM. Throws a `TimeoutError` if element was not found after 30 sec. Behavior can be modified with @param options. Possible keys: `visible`, `hidden`, `timeout`.
@@ -19,26 +21,25 @@ import { pressKeyWithModifier } from '@wordpress/e2e-test-utils';
  * @param {Object} options Custom options to modify function behavior.
  */
 export async function waitForSelector( page, selector, options = {} ) {
+	const startTime = new Date();
+	const { PUPPETEER_HEADLESS } = process.env;
+
 	// set up default options
 	const defaultOptions = { timeout: 30000, logHTML: false };
 	options = Object.assign( defaultOptions, options );
 
-	let el;
-	const startTime = new Date();
 	try {
-		el = await page.waitForSelector( selector, options );
+		const element = await page.waitForSelector( selector, options );
 		const secondsPassed = ( new Date() - startTime ) / 1000;
 		console.log( `Found element by locator: ${ selector }. Waited for: ${ secondsPassed } sec` );
-		return el;
+		return element;
 	} catch ( e ) {
-		if ( options.logHTML && process.env.PUPPETEER_HEADLESS !== 'false' ) {
-			const bodyHTML = await page.evaluate( () => document.body.innerHTML );
-			console.log( page.url() );
-			console.log( bodyHTML );
+		if ( options.logHTML && PUPPETEER_HEADLESS !== 'false' ) {
+			await logHTML();
 		}
 		const secondsPassed = ( new Date() - startTime ) / 1000;
 		console.log(
-			`Failed to locate an element by locator: ${ selector }. Waited for: ${ secondsPassed } sec`
+			`Failed to locate an element by locator: ${ selector }. Waited for: ${ secondsPassed } sec. URL: ${ page.url() }`
 		);
 		throw e;
 	}
@@ -169,3 +170,25 @@ export async function scrollIntoView( page, selector ) {
 	await waitForSelector( page, selector );
 	return await page.evaluate( s => document.querySelector( s ).scrollIntoView(), selector );
 }
+
+export async function logHTML() {
+	const bodyHTML = await page.evaluate( () => document.body.innerHTML );
+	if ( process.env.E2E_DEBUG ) {
+		console.log( '#### PAGE HTML ####' );
+		console.log( page.url() );
+		console.log( bodyHTML );
+	}
+	await sendSnippetToSlack( bodyHTML );
+	return bodyHTML;
+}
+
+export async function logDebugLog() {
+	const log = readFileSync( '/home/travis/wordpress/wp-content/debug.log' ).toString();
+	if ( log.length > 1 ) {
+		if ( process.env.E2E_DEBUG ) {
+			console.log( '#### WP DEBUG.LOG ####' );
+			console.log( log );
+		}
+		await sendSnippetToSlack( log );
+	}
+}

tests/e2e/lib/pages/page.js

@@ -4,13 +4,13 @@
 import { waitForSelector } from '../page-helper';
 
 export default class Page {
-	constructor( page, { expectedSelector, url = null, expectedWaitMC = 25000 } ) {
+	constructor( page, { expectedSelector, url = null, explicitWaitMS = 25000 } ) {
 		this.page = page;
 		this.expectedSelector = expectedSelector;
 		this.visit = false;
 		this.url = url;
 		this.name = this.constructor.name;
-		this.explicitWaitMS = expectedWaitMC;
+		this.explicitWaitMS = explicitWaitMS;
 	}
 
 	/**
@@ -65,6 +65,28 @@ export default class Page {
 			value: sandboxCookieValue,
 			domain,
 		} );
-		return await this.page.reload();
+
+		return await this.reload();
+	}
+
+	/**
+	 * Reloads the page and waits for the expected locator
+	 * @param {Object} options page.reload options object
+	 */
+	async reload( options = {} ) {
+		await this.page.reload( options );
+		return await this.waitForPage();
+	}
+
+	async reloadUntil( callback, options = {} ) {
+		let reloadNeeded = await callback();
+		let count = 1;
+		while ( reloadNeeded || count > 5 ) {
+			console.log( 'Reloading since reloadNeeded is: ', reloadNeeded.toString() );
+
+			await this.reload( options );
+			reloadNeeded = await callback();
+			count++;
+		}
 	}
 }

tests/e2e/lib/pages/postFrontend.js

@@ -2,6 +2,7 @@
  * Internal dependencies
  */
 import Page from './page';
+import { waitAndClick, waitForSelector } from '../page-helper';
 
 export default class PostFrontendPage extends Page {
 	constructor( page ) {
@@ -16,4 +17,12 @@ export default class PostFrontendPage extends Page {
 	async isRenderedBlockPresent( BlockClass ) {
 		return await BlockClass.isRendered( this.page );
 	}
+
+	async logout() {
+		const accountBarSelector = '#wp-admin-bar-my-account';
+		const logoutOptionSelector = '#wp-admin-bar-logout';
+		await waitForSelector( this.page, accountBarSelector );
+		await this.page.hover( accountBarSelector );
+		await waitAndClick( this.page, logoutOptionSelector );
+	}
 }

tests/e2e/lib/pages/wp-admin/block-editor.js

@@ -7,11 +7,13 @@ import Page from '../page';
  */
 import { getAllBlocks, searchForBlock } from '@wordpress/e2e-test-utils';
 import { waitAndClick, waitForSelector, scrollIntoView } from '../../page-helper';
+import { getNgrokSiteUrl } from '../../utils-helper';
 
 export default class BlockEditorPage extends Page {
 	constructor( page ) {
 		const expectedSelector = '.block-editor';
-		super( page, { expectedSelector } );
+		const url = getNgrokSiteUrl() + '/wp-admin/post-new.php';
+		super( page, { expectedSelector, url } );
 	}
 
 	async insertBlock( blockName ) {
@@ -35,7 +37,7 @@ export default class BlockEditorPage extends Page {
 		// Disable reason: Wait for the animation to complete, since otherwise the
 		// click attempt may occur at the wrong point.
 		// Also, for some reason post-publish bar wont show up it we click to fast :/
-		await page.waitFor( 5000 );
+		await page.waitFor( 500 );
 
 		await waitAndClick( this.page, '.editor-post-publish-button' );
 		return await waitForSelector( this.page, '.post-publish-panel__postpublish-buttons a' );

tests/e2e/lib/pages/wpcom/connections.js

@@ -12,8 +12,8 @@ import {
 
 export default class ConnectionsPage extends Page {
 	constructor( page ) {
-		const expectedSelector = '.sharing-settings.sharing-connections';
-		super( page, { expectedSelector } );
+		const expectedSelector = '.connections__sharing-connections';
+		super( page, { expectedSelector, explicitWaitMS: 40000 } );
 	}
 
 	async selectMailchimpList( mailchimpList = 'e2etesting' ) {

tests/e2e/lib/pages/wpcom/login.js

@@ -14,8 +14,7 @@ export default class LoginPage extends Page {
 	constructor( page ) {
 		const expectedSelector = '.wp-login__container';
 		const url = 'https://wordpress.com/log-in';
-		super( page, { expectedSelector, url } );
-		this.explicitWaitMS = 45000;
+		super( page, { expectedSelector, url, explicitWaitMS: 45000 } );
 	}
 
 	async login( wpcomUser ) {

tests/e2e/lib/pages/wpcom/plans.js

@@ -0,0 +1,21 @@
+/**
+ * Internal dependencies
+ */
+import Page from '../page';
+import { waitAndClick, waitForSelector } from '../../page-helper';
+
+export default class PlansPage extends Page {
+	constructor( page ) {
+		const expectedSelector = '.plans-features-main';
+		super( page, { expectedSelector } );
+	}
+
+	async returnToWPAdmin() {
+		return await waitAndClick( this.page, ".jetpack-checklist__footer a[href*='wp-admin']" );
+	}
+
+	async isCurrentPlan( plan = 'business' ) {
+		const currentPlanSelector = `.is-current.is-${ plan }-plan`;
+		return await waitForSelector( this.page, currentPlanSelector );
+	}
+}