Thread dump on test timeout. (#1257)

* Thread dump on test timeout. For both Github CI and local uberjar tests, add a shell signal handler which catches interrupts and resends to the test process as SIGQUIT to trigger a dump.
google · Nov 13, 2024 · 13259a9 · 13259a9
1 parent e1b5d8e
commit 13259a9
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 4 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -347,7 +347,29 @@ jobs:
           DIR="$(find m2/org/conscrypt/conscrypt-openjdk-uber -maxdepth 1 -mindepth 1 -type d -print)"
           VERSION="${DIR##*/}"
           TESTJAR="$(find testjar -name '*-tests.jar')"
-          java -jar junit-platform-console-standalone.jar execute -cp "$DIR/conscrypt-openjdk-uber-$VERSION.jar${{ matrix.separator }}$TESTJAR" -n='org.conscrypt.ConscryptOpenJdkSuite' --scan-classpath --reports-dir=results --fail-if-no-tests
+          # SIGTERM handler, e.g. for when tests hang and time out.
+          # Send SIGQUIT to test process to get thread dump, give it
+          # a few seconds to complete and then kill it.
+          dump_threads() {
+            echo "Generating stack dump."
+            ps -fp "$TESTPID"
+            kill -QUIT "$TESTPID"
+            sleep 3
+            kill -KILL "$TESTPID"
+            exit 1
+          }
+          java -jar junit-platform-console-standalone.jar execute -cp "$DIR/conscrypt-openjdk-uber-$VERSION.jar${{ matrix.separator }}$TESTJAR" -n='org.conscrypt.ConscryptOpenJdkSuite' --scan-classpath --reports-dir=results --fail-if-no-tests &
+          case $(uname -s) in
+            Darwin|Linux)
+              trap dump_threads SIGTERM SIGINT
+              ;;
+            *)
+              # TODO: Probably won't work on Windows but thread dumps
+              # work there already.
+              ;;
+          esac
+          TESTPID=$!
+          wait "$TESTPID"
 
       - name: Archive test results
         if: ${{ always() }}

diff --git a/scripts/testLocalUber.sh b/scripts/testLocalUber.sh
@@ -101,8 +101,33 @@ cd $CONSCRYPT_HOME
 ./gradlew :conscrypt-openjdk:testJar --console=plain
 test -f "$TESTJAR" || die "Test jar not built."
 
+# SIGTERM handler, e.g. for when tests hang and time out.
+# Send SIGQUIT to test process to get thread dump, give it
+# a few seconds to complete and then kill it.
+dump_threads() {
+    echo "Generating stack dump."
+    ps -fp "$TESTPID"
+    kill -QUIT "$TESTPID"
+    sleep 3
+    kill -KILL "$TESTPID"
+    exit 1
+}
+
 echo "Running tests."
 java $JAVADEBUG -jar "$JUNITJAR" execute -cp "${UBERJAR}:${TESTJAR}" \
-	 -n='org.conscrypt.ConscryptOpenJdkSuite' \
-	 --scan-classpath --reports-dir=. \
-	 --fail-if-no-tests $VERBOSE
+     -n='org.conscrypt.ConscryptOpenJdkSuite' \
+     --scan-classpath --reports-dir=. \
+     --fail-if-no-tests $VERBOSE &
+
+case $(uname -s) in
+    Darwin|Linux)
+        trap dump_threads SIGTERM SIGINT
+        ;;
+    *)
+        # TODO: Probably won't work on Windows but thread dumps
+        # work there already.
+        ;;
+esac
+
+TESTPID=$!
+wait "$TESTPID"