chore: Isolate e2e tests and reset them for each run (#8041)

* chore: Isolate e2e tests and reset themn for each run

Fixes #7959
Fixes #8010

bin/install-docker.sh

@@ -1,66 +1,44 @@
 #!/bin/bash
 
-# Exit if any command fails
+# Exit if any command fails.
 set -e
 
-# Include useful functions
+# Include useful functions.
 . "$(dirname "$0")/includes.sh"
 
-# Check that Docker is installed
+# Check that Docker is installed.
 if ! command_exists "docker"; then
 	echo -e $(error_message "Docker doesn't seem to be installed. Please head on over to the Docker site to download it: $(action_format "https://www.docker.com/community-edition#/download")")
 	exit 1
 fi
 
-# Check that Docker is running
+# Check that Docker is running.
 if ! docker info >/dev/null 2>&1; then
 	echo -e $(error_message "Docker isn't running. Please check that you've started your Docker app, and see it in your system tray.")
 	exit 1
 fi
 
-# Stop existing containers
+# Stop existing containers.
 echo -e $(status_message "Stopping Docker containers...")
 docker-compose down --remove-orphans >/dev/null 2>&1
 
-# Download image updates
+# Download image updates.
 echo -e $(status_message "Downloading Docker image updates...")
 docker-compose pull
 
-# Launch the containers
+# Launch the containers.
 echo -e $(status_message "Starting Docker containers...")
 docker-compose up -d >/dev/null
 
-HOST_PORT=$(docker-compose port wordpress 80 | awk -F : '{printf $2}')
+# Set up WordPress Development site.
+# Note: we don't bother installing the test site right now, because that's
+# done on every time `npm run test-e2e` is run.
+. "$(dirname "$0")/install-wordpress.sh"
 
-# Wait until the docker containers are setup properely
-echo -en $(status_message "Attempting to connect to wordpress...")
-until $(curl -L http://localhost:$HOST_PORT -so - 2>&1 | grep -q "WordPress"); do
-    echo -n '.'
-    sleep 5
-done
-echo ''
-
-# Install WordPress
-echo -e $(status_message "Installing WordPress...")
-docker-compose run --rm -u 33 cli core install --url=localhost:$HOST_PORT --title=Gutenberg --admin_user=admin --admin_password=password --admin_email=test@test.com >/dev/null
-# Check for WordPress updates, just in case the WordPress image isn't up to date.
-docker-compose run --rm -u 33 cli core update >/dev/null
-
-# If the 'wordpress' volume wasn't during the down/up earlier, but the post port has changed, we need to update it.
-CURRENT_URL=$(docker-compose run -T --rm cli option get siteurl)
-if [ "$CURRENT_URL" != "http://localhost:$HOST_PORT" ]; then
-	docker-compose run --rm cli option update home "http://localhost:$HOST_PORT" >/dev/null
-	docker-compose run --rm cli option update siteurl "http://localhost:$HOST_PORT" >/dev/null
-fi
-
-# Activate Gutenberg
-echo -e $(status_message "Activating Gutenberg...")
-docker-compose run --rm cli plugin activate gutenberg >/dev/null
-
-# Install the PHPUnit test scaffolding
+# Install the PHPUnit test scaffolding.
 echo -e $(status_message "Installing PHPUnit test scaffolding...")
 docker-compose run --rm wordpress_phpunit /app/bin/install-wp-tests.sh wordpress_test root example mysql latest false >/dev/null
 
-# Install Composer
+# Install Composer. This is only used to run WordPress Coding Standards checks.
 echo -e $(status_message "Installing and updating Composer modules...")
 docker-compose run --rm composer install

bin/install-php-phpunit.sh

@@ -42,7 +42,7 @@ if [[ ${SWITCH_TO_PHP:0:3} == "5.2" ]] || [[ ${SWITCH_TO_PHP:0:3} == "5.3" ]]; t
 
   # php and phpunit3.6 installs should be cached, only build if they're not there.
   if [ ! -f $PHPBREW_BUILT_CHECK ]; then
-    
+
     # init with known --old to get 5.2 and 5.3
     $HOME/php-utils-bin/phpbrew init
     $HOME/php-utils-bin/phpbrew known --old

bin/install-wordpress.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Exit if any command fails.
+set -e
+
+# Gutenberg script includes.
+. "$(dirname "$0")/includes.sh"
+
+# These are the containers and values for the development site.
+CLI='cli'
+CONTAINER='wordpress'
+SITE_TITLE='Gutenberg Dev'
+
+# If we're installing/re-installing the test site, change the containers used.
+if [ "$1" == '--e2e_tests' ]; then
+	CLI="${CLI}_e2e_tests"
+	CONTAINER="${CONTAINER}_e2e_tests"
+	SITE_TITLE='Gutenberg Testing'
+
+	if ! docker ps | grep -q $CONTAINER; then
+		echo -e $(error_message "WordPress e2e tests run in their own Docker container, but that container wasn't found.")
+		echo "Please restart your Docker containers by running 'docker-compose down && docker-compose up -d' or"
+		echo "by running './bin/setup-local-env.sh' again."
+		echo ""
+		exit 1
+	fi
+fi
+
+# Get the host port for the WordPress container.
+HOST_PORT=$(docker-compose port $CONTAINER 80 | awk -F : '{printf $2}')
+
+# Wait until the Docker containers are running and the WordPress site is
+# responding to requests.
+echo -en $(status_message "Attempting to connect to WordPress...")
+until $(curl -L http://localhost:$HOST_PORT -so - 2>&1 | grep -q "WordPress"); do
+    echo -n '.'
+    sleep 5
+done
+echo ''
+
+# If this is the test site, we reset the database so no posts/comments/etc.
+# dirty up the tests.
+if [ "$1" == '--e2e_tests' ]; then
+	echo -e $(status_message "Resetting test database...")
+	docker-compose run --rm $CLI db reset --yes >/dev/null
+fi
+
+# Install WordPress.
+echo -e $(status_message "Installing WordPress...")
+docker-compose run --rm $CLI core install --title="$SITE_TITLE" --admin_user=admin --admin_password=password --admin_email=test@test.com --skip-email --url=http://localhost:$HOST_PORT >/dev/null
+# Check for WordPress updates, just in case the WordPress image isn't up to date.
+docker-compose run --rm $CLI core update >/dev/null
+
+# If the 'wordpress' volume wasn't during the down/up earlier, but the post port has changed, we need to update it.
+CURRENT_URL=$(docker-compose run -T --rm $CLI option get siteurl)
+if [ "$CURRENT_URL" != "http://localhost:$HOST_PORT" ]; then
+	docker-compose run --rm $CLI option update home "http://localhost:$HOST_PORT" >/dev/null
+	docker-compose run --rm $CLI option update siteurl "http://localhost:$HOST_PORT" >/dev/null
+fi
+
+# Activate Gutenberg.
+echo -e $(status_message "Activating Gutenberg...")
+docker-compose run --rm $CLI plugin activate gutenberg >/dev/null

bin/reset-e2e-tests.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Exit if any command fails.
+set -e
+
+# Include useful functions.
+. "$(dirname "$0")/includes.sh"
+
+# Set up WordPress site used for end-to-end (e2e) tests.
+. "$(dirname "$0")/install-wordpress.sh" --e2e_tests

docker-compose.yml

@@ -40,6 +40,27 @@ services:
     volumes:
       - .:/app
 
+  wordpress_e2e_tests:
+    image: wordpress
+    ports:
+      - 8889:80
+    environment:
+      WORDPRESS_DB_NAME: wordpress_e2e_tests
+      WORDPRESS_DB_PASSWORD: example
+      ABSPATH: /usr/src/wordpress/
+    volumes:
+      - wordpress_e2e_tests:/var/www/html
+      - .:/var/www/html/wp-content/plugins/gutenberg
+      - ./test/e2e/test-plugins:/var/www/html/wp-content/plugins/gutenberg-test-plugins
+      - ./test/e2e/test-mu-plugins:/var/www/html/wp-content/mu-plugins
+
+  cli_e2e_tests:
+    image: wordpress:cli
+    volumes:
+      - wordpress_e2e_tests:/var/www/html
+      - .:/var/www/html/wp-content/plugins/gutenberg
+
 volumes:
   testsuite:
   wordpress:
+  wordpress_e2e_tests:

package.json

@@ -172,6 +172,7 @@
 		"publish:dev": "npm run build:packages && lerna publish --npm-tag next",
 		"publish:prod": "npm run build:packages && lerna publish",
 		"test": "npm run lint && npm run test-unit",
+		"pretest-e2e": "./bin/reset-e2e-tests.sh",
 		"test-e2e": "wp-scripts test-unit-js --config test/e2e/jest.config.json --runInBand",
 		"test-e2e:watch": "npm run test-e2e -- --watch",
 		"test-php": "npm run lint-php && npm run test-unit-php",

test/e2e/support/bootstrap.js

@@ -3,10 +3,15 @@
  */
 import puppeteer from 'puppeteer';
 
-const { PUPPETEER_HEADLESS, PUPPETEER_SLOWMO } = process.env;
+/**
+ * Node dependencies
+ */
+import { visitAdmin } from './utils';
+
+const { PUPPETEER_HEADLESS, PUPPETEER_SLOWMO, PUPPETEER_TIMEOUT } = process.env;
 
 // The Jest timeout is increased because these tests are a bit slow
-jest.setTimeout( 100000 );
+jest.setTimeout( PUPPETEER_TIMEOUT || 100000 );
 
 beforeAll( async () => {
 	global.browser = await puppeteer.launch( {
@@ -15,6 +20,43 @@ beforeAll( async () => {
 	} );
 } );
 
+// After every test run, delete all content created by the test. This ensures
+// other posts/comments/etc. aren't dirtying tests and tests don't depend on
+// each other's side-effects.
+//
+// Right now we run the cleanup _after_ each test but we may want to do
+// this in the `beforeAll` call instead, as mentioned here:
+// https://github.com/WordPress/gutenberg/pull/8041#discussion_r203940770
+//
+// We can't do that while we rely on the global page object, but if we ditched
+// the global approach we could use `visitAdmin` before
+// `newDesktopBrowserPage()` is called.
 afterAll( async () => {
+	if ( ! global.page ) {
+		return;
+	}
+
+	// This accepts a page dialog that may appear when navigating away from
+	// Gutenberg to the admin, where we need to go to delete posts.
+	page.on( 'dialog', ( dialog ) => {
+		dialog.accept();
+	} );
+
+	// Visit `/wp-admin/edit.php` so we can see a list of posts and delete them.
+	await visitAdmin( 'edit.php' );
+
+	// If this selector doesn't exist there are no posts for us to delete.
+	const bulkSelector = await page.$( '#bulk-action-selector-top' );
+	if ( bulkSelector ) {
+		// Select all posts.
+		await page.waitForSelector( '#cb-select-all-1' );
+		await page.click( '#cb-select-all-1' );
+		// Select the "bulk actions" > "trash" option.
+		await page.select( '#bulk-action-selector-top', 'trash' );
+		// Submit the form to send all draft/scheduled/published posts to the trash.
+		await page.click( '#doaction' );
+		await page.waitForNavigation();
+	}
+
 	await browser.close();
 } );

test/e2e/support/utils.js

@@ -10,7 +10,7 @@ import { URL } from 'url';
 import { times } from 'lodash';
 
 const {
-	WP_BASE_URL = 'http://localhost:8888',
+	WP_BASE_URL = 'http://localhost:8889',
 	WP_USERNAME = 'admin',
 	WP_PASSWORD = 'password',
 } = process.env;