build: Move "Test plugin upgrades" to a post-build workflow

.github/actions/check-run/action.yml

@@ -0,0 +1,70 @@
+name: "Check Run"
+description: "Create or update a check run. This is a simple wrapper around GitHub's checks API, https://docs.github.com/en/rest/checks/runs."
+inputs:
+  id:
+    description: "Check run ID to update. If not given, a new run will be created"
+  sha:
+    description: "The SHA of the commit."
+  name:
+    description: "The name of the check. For example, \"code-coverage\"."
+  status:
+    description: "The current status. Can be one of: `queued`, `in_progress`, `completed`"
+  conclusion:
+    description: "The final conclusion of the check. Can be one of: `action_required`, `cancelled`, `failure`, `neutral`, `success`, `skipped`, `timed_out`"
+  title:
+    description: "Title of the check. Shown in the PR's checks list."
+  summary:
+    description: "Summary of the check. Can contain Markdown."
+  token:
+    description: "App access token. See https://docs.github.com/en/developers/apps/building-github-apps/authenticating-with-github-apps#authenticating-as-a-github-app for how to get one."
+outputs:
+  id:
+    description: "Check run ID."
+    value: ${{ steps.run.outputs.id }}
+runs:
+  using: composite
+  steps:
+    - id: run
+      shell: bash
+      env:
+        ID: ${{ inputs.id }}
+        SHA: ${{ inputs.sha }}
+        NAME: ${{ inputs.name }}
+        STATUS: ${{ inputs.status }}
+        CONCLUSION: ${{ inputs.conclusion }}
+        TITLE: ${{ inputs.title }}
+        SUMMARY: ${{ inputs.summary }}
+        TOKEN: ${{ inputs.token }}
+      run: |
+        if [[ -n "$ID" ]]; then
+          METHOD=PATCH
+          URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/check-runs/$ID"
+        else
+          METHOD=POST
+          URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/check-runs"
+        fi
+
+        DATA="{}"
+        if [[ -n "$NAME" ]]; then
+          DATA="$(jq --arg v "$NAME" '.name |= $v' <<<"$DATA")"
+        fi
+        if [[ -n "$SHA" ]]; then
+          DATA="$(jq --arg v "$SHA" '.head_sha |= $v' <<<"$DATA")"
+        fi
+        if [[ -n "$STATUS" ]]; then
+          DATA="$(jq --arg v "$STATUS" '.status |= $v' <<<"$DATA")"
+        fi
+        if [[ -n "$CONCLUSION" ]]; then
+          DATA="$(jq --arg v "$CONCLUSION" '.conclusion |= $v' <<<"$DATA")"
+        fi
+        if [[ -n "$TITLE" ]]; then
+          DATA="$(jq --arg v "$TITLE" '.output.title |= $v' <<<"$DATA")"
+        fi
+        if [[ -n "$SUMMARY" ]]; then
+          DATA="$(jq --arg v "$SUMMARY" '.output.summary |= $v' <<<"$DATA")"
+        fi
+
+        echo "Data: $DATA"
+        JSON="$(curl -v -X "$METHOD" --header "authorization: Bearer $TOKEN" --url "$URL" --data "$DATA")"
+        echo "$JSON"
+        echo "id=$(jq -r .id <<<"$JSON")" >> "$GITHUB_OUTPUT"

.github/files/test-plugin-update/prepare-zips.sh

@@ -39,6 +39,7 @@ while IFS=$'\t' read -r SRC MIRROR SLUG; do
 			else
 				echo "::error::Unexpected response from betadownload.jetpack.me for $SLUG"
 				echo "$JSON"
+				echo "info=❌ Unexpected response from betadownload.jetpack.me for $SLUG" >> "$GITHUB_OUTPUT"
 				exit 1
 			fi
 		fi
@@ -59,7 +60,10 @@ while IFS=$'\t' read -r SRC MIRROR SLUG; do
 	else
 		echo "::error::Unexpected response from WordPress.org API for $SLUG"
 		echo "$JSON"
+		echo "info=❌ Unexpected response from WordPress.org API for $SLUG" >> "$GITHUB_OUTPUT"
 		exit 1
 	fi
 	echo "::endgroup::"
 done < build/plugins.tsv
+
+echo 'info=' >> "$GITHUB_OUTPUT"

.github/files/test-plugin-update/test.sh

@@ -2,6 +2,8 @@
 
 set -eo pipefail
 
+source "$GITHUB_WORKSPACE/monorepo/.github/files/gh-funcs.sh"
+
 ZIPDIR="$GITHUB_WORKSPACE/zips"
 EXIT=0
 
@@ -11,20 +13,38 @@ for ZIP in *-dev.zip; do
 	SLUGS+=( "${ZIP%-dev.zip}" )
 done
 
+FINISHED=false
+OUTPUT=()
+
+function onexit {
+	if ! "$FINISHED"; then
+		OUTPUT+=( "💣 The testing script exited unexpectedly." )
+	fi
+	gh_set_output info "$( printf "%s\n" "${OUTPUT[@]}" )"
+}
+trap "onexit" EXIT
+
+function failed {
+	ERRMSG="$1"
+	OUTPUT+=( "❌ $ERRMSG" )
+	FAILED=1
+	EXIT=1
+}
+
 cd /var/www/html
 for SLUG in "${SLUGS[@]}"; do
 	for FROM in stable trunk dev; do
 		for HOW in web cli; do
 			[[ -e "$ZIPDIR/$SLUG-$FROM.zip" ]] || continue
 
+			FAILED=
 			printf '\n\e[1mTest upgrade of %s from %s via %s\e[0m\n' "$SLUG" "$FROM" "$HOW"
 
 			ERRMSG=
 			echo "::group::Installing $SLUG $FROM"
 			: > /var/www/html/wp-content/debug.log
 			if ! wp --allow-root plugin install --activate "$ZIPDIR/$SLUG-$FROM.zip"; then
-				ERRMSG="Plugin install failed for $SLUG $FROM!"
-				EXIT=1
+				failed "Plugin install failed for $SLUG $FROM!"
 			fi
 			echo '== Debug log =='
 			cat /var/www/html/wp-content/debug.log
@@ -46,8 +66,7 @@ for SLUG in "${SLUGS[@]}"; do
 			: > /var/www/html/wp-content/debug.log
 			if [[ "$HOW" == 'cli' ]]; then
 				if ! wp --allow-root plugin upgrade "$SLUG" 2>&1 | tee "$GITHUB_WORKSPACE/out.txt"; then
-					ERRMSG="CLI upgrade of $SLUG from $FROM exited with a non-zero status"
-					EXIT=1
+					failed "CLI upgrade of $SLUG from $FROM exited with a non-zero status"
 				fi
 			else
 				# Everything needs to be owned by www-data for the web upgrade to proceed.
@@ -63,19 +82,18 @@ for SLUG in "${SLUGS[@]}"; do
 			fi
 			ERR="$(grep -i 'Fatal error' /var/www/html/wp-content/debug.log || true)"
 			if [[ -n "$ERR" ]]; then
-				echo "::error::Mid-upgrade fatal detected for $SLUG $HOW update from $FROM!%0A$ERR"
-				EXIT=1
+				failed "Mid-upgrade fatal detected for $SLUG $HOW update from $FROM!%0A$ERR"
+				echo "::error::$ERRMSG"
 			elif [[ ! -e "/var/www/html/wp-content/plugins/$SLUG/ci-flag.txt" ]]; then
-				echo "::error::Plugin $SLUG ($HOW update from $FROM) does not seem to have been updated?"
-				EXIT=1
+				failed "Plugin $SLUG ($HOW update from $FROM) does not seem to have been updated?"
+				echo "::error::$ERRMSG"
 			fi
 
 			ERRMSG=
 			echo "::group::Deactivating $SLUG"
 			: > /var/www/html/wp-content/debug.log
 			if ! wp --allow-root plugin deactivate "$SLUG"; then
-				ERRMSG="Plugin deactivate failed after $SLUG $HOW update from $FROM!"
-				EXIT=1
+				failed "Plugin deactivate failed after $SLUG $HOW update from $FROM!"
 			fi
 			echo '== Debug log =='
 			cat /var/www/html/wp-content/debug.log
@@ -88,8 +106,7 @@ for SLUG in "${SLUGS[@]}"; do
 			echo "::group::Uninstalling $SLUG"
 			: > /var/www/html/wp-content/debug.log
 			if ! wp --allow-root plugin uninstall "$SLUG"; then
-				ERRMSG="Plugin uninstall failed after $SLUG $HOW update from $FROM!"
-				EXIT=1
+				failed "Plugin uninstall failed after $SLUG $HOW update from $FROM!"
 				rm -rf "/var/www/html/wp-content/plugins/$SLUG"
 			fi
 			echo '== Debug log =='
@@ -101,8 +118,13 @@ for SLUG in "${SLUGS[@]}"; do
 
 			wp --allow-root --quiet option delete fake_plugin_update_plugin
 			wp --allow-root --quiet option delete fake_plugin_update_url
+
+			if [[ -z "$FAILED" ]]; then
+				OUTPUT+=( "✅ Upgrade of $SLUG from $FROM via $HOW succeeded!" )
+			fi
 		done
 	done
 done
 
+FINISHED=true
 exit $EXIT

.github/workflows/build.yml

@@ -132,56 +132,6 @@ jobs:
               comment_id: +COMMENT_ID,
             } );
 
-  upgrade_test:
-    name: Test plugin upgrades
-    runs-on: ubuntu-latest
-    needs: build
-    if: needs.build.outputs.any_plugins == 'true'
-    timeout-minutes: 10  # 2022-03-04: Successful runs seem to take about 3 minutes, but give some extra time for the downloads.
-    services:
-      db:
-        image: mariadb:latest
-        env:
-          MARIADB_ROOT_PASSWORD: wordpress
-        ports:
-          - 3306:3306
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
-    container:
-      image: ghcr.io/automattic/jetpack-wordpress-dev:latest
-      env:
-        WP_DOMAIN: localhost
-        WP_ADMIN_USER: wordpress
-        WP_ADMIN_EMAIL: wordpress@example.com
-        WP_ADMIN_PASSWORD: wordpress
-        WP_TITLE: Hello World
-        MYSQL_HOST: db:3306
-        MYSQL_DATABASE: wordpress
-        MYSQL_USER: root
-        MYSQL_PASSWORD: wordpress
-        HOST_PORT: 80
-      ports:
-        - 80:80
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          path: monorepo
-
-      - name: Download build artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: jetpack-build
-      - name: Extract build archive
-        run: tar --xz -xvvf build.tar.xz
-
-      - name: Setup WordPress
-        run: monorepo/.github/files/test-plugin-update/setup.sh
-
-      - name: Prepare plugin zips
-        run: monorepo/.github/files/test-plugin-update/prepare-zips.sh
-
-      - name: Test upgrades
-        run: monorepo/.github/files/test-plugin-update/test.sh
-
   jetpack_beta:
     name: Create artifact for Jetpack Beta plugin
     runs-on: ubuntu-latest