Move post-upgrade test run to workflow (#2078)

Brainicism · Apr 13, 2024 · 214cd28 · 214cd28
1 parent c939a8c
commit 214cd28
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 23 deletions.
diff --git a/.github/workflows/gci-e2e.yml b/.github/workflows/gci-e2e.yml
@@ -108,6 +108,7 @@ jobs:
             - name: Run test runner on live instance (gameplay test)
               run: docker exec --env-file	./.env kmq-gci sh -c 'npx ts-node --swc src/test/end-to-end-tests/test-runner-bot.ts --test-suite=PLAY --debug; exit $?'
             - name: Check for errors
+              if: always()
               run: docker logs kmq-gci 2>&1 | grep -i "\[Error\]" && echo "Errors found in container logs." && exit 1 || echo "No errors found in container logs." && exit 0
             - name: Print logs
               if: always()
@@ -121,7 +122,7 @@ jobs:
                   docker ps
                   echo OLD_CONTAINER_ID=$(sudo docker ps -aqf "name=kmq-gci") > $GITHUB_OUTPUT
             - name: Begin upgrade
-              run: npx ts-node --swc src/scripts/announce-restart.ts --docker-image=ghcr.io/brainicism/kmq_discord:gci --timer=0 --provisioning-timeout=15 --skip-tests
+              run: npx ts-node --swc src/scripts/announce-restart.ts --docker-image=ghcr.io/brainicism/kmq_discord:gci --timer=0 --provisioning-timeout=15
             - name: Check post-upgrade KMQ healthiness
               run: while [[ "$(docker inspect --format='{{json .State.Health.Status}}' kmq-gci)" != "\"healthy\"" ]]; do sleep 1; done
             - name: Check old primary no longer exists

diff --git a/.github/workflows/upgrade-prod.yml b/.github/workflows/upgrade-prod.yml
@@ -19,7 +19,7 @@ jobs:
         env:
             KMQ_DIR: /home/kmq/prod
         runs-on: ubuntu-latest
-        timeout-minutes: 10
+        timeout-minutes: 20
         steps:
             - name: Install SSH Keys
               run: |
@@ -29,3 +29,8 @@ jobs:
                   ssh-keyscan ${{ secrets.PROD_SSH_ADDRESS }} > ~/.ssh/known_hosts
             - name: Redeploy over SSH
               run: ssh ${{ secrets.PROD_SSH_USER }}@${{ secrets.PROD_SSH_ADDRESS }} "source ~/.zshrc; ./.github/workflows/redeploy_prod.sh $KMQ_DIR ${{ inputs.image-name }}"
+           - name: Run basic options test
+              run: ssh ${{ secrets.PROD_SSH_USER }}@${{ secrets.PROD_SSH_ADDRESS }} "docker exec kmq-prod sh -c '. ./.env && npx ts-node --swc src/test/end-to-end-tests/test-runner-bot.ts --test-suite=BASIC_OPTIONS --debug --stage-delay=5'"
+           - name: Run basic gameplay test
+              run: ssh ${{ secrets.PROD_SSH_USER }}@${{ secrets.PROD_SSH_ADDRESS }} "docker exec kmq-prod sh -c '. ./.env && npx ts-node --swc src/test/end-to-end-tests/test-runner-bot.ts --test-suite=PLAY --debug --stage-delay=5'"
+
diff --git a/src/scripts/announce-restart.ts b/src/scripts/announce-restart.ts
@@ -14,7 +14,6 @@ program
         "--no-restart",
         "Automatically restart process when countdown is over",
     )
-    .option("--skip-tests", "Skip test-runner tests")
     .option("--docker-image <docker_image>", "Docker image")
     .option(
         "--timer <minutes>",
@@ -93,7 +92,6 @@ function serverShutdown(
     restart: boolean,
     dockerImage: string,
     provisioningTimeout: number,
-    skipTests: boolean,
 ): Promise<void> {
     return new Promise(async () => {
         // if stopping server, inform immediately
@@ -164,7 +162,7 @@ function serverShutdown(
             await announceRestart(restartMinutes, restartDate, restart);
 
             setTimeout(
-                async () => {
+                () => {
                     // drop old primary
                     console.log("Dropping old primary...");
                     cp.execSync(`docker rm -f ${oldAppName}`);
@@ -174,22 +172,6 @@ function serverShutdown(
                     cp.execSync(
                         `docker exec ${appName} /bin/sh -c "mv standby promoted"`,
                     );
-
-                    // wait for all event listeners to be set-up
-                    await delay(5000);
-
-                    console.log("Running post-upgrade health checks...");
-                    console.log(
-                        "Running post-upgrade test suite: BASIC_OPTIONS...",
-                    );
-
-                    const basicOptionsTestCmd = `docker exec ${appName} sh -c '. ./.env && npx ts-node --swc src/test/end-to-end-tests/test-runner-bot.ts --test-suite=BASIC_OPTIONS --debug --stage-delay=5'`;
-                    const gameplayTestCmd = `docker exec ${appName} sh -c '. ./.env && npx ts-node --swc src/test/end-to-end-tests/test-runner-bot.ts --test-suite=PLAY --debug --stage-delay=5'`;
-                    if (!skipTests) {
-                        cp.exec(
-                            `${basicOptionsTestCmd} && ${gameplayTestCmd}`,
-                        ).unref();
-                    }
                 },
                 restartMinutes * 1000 * 60,
             );
@@ -207,7 +189,6 @@ process.on("SIGINT", async () => {
     const options = program.opts();
     console.log(options);
     const restartMinutes = options.timer;
-    const skipTests = options.skipTests;
     const provisioningTimeout = options.provisioningTimeout;
     const dockerImage = options.dockerImage;
 
@@ -216,6 +197,5 @@ process.on("SIGINT", async () => {
         options.restart,
         dockerImage,
         provisioningTimeout,
-        skipTests,
     );
 })();