diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 42f3d1e..ead6774 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -108,7 +108,7 @@ jobs:
         PR_BRANCH: ${{ github.head_ref }}
         FIXED_BRANCH: ${{ github.head_ref }}-spotless-fixes
 
-    - name: Run server for ${{ inputs.timeout }} seconds
+    - name: Run server for up to ${{ inputs.timeout }} seconds
       if: ${{ !inputs.client-only }}
       run: |
         mkdir -p run
diff --git a/scripts/test_no_error_reports b/scripts/test_no_error_reports
index 1fcc739..dc57a2f 100755
--- a/scripts/test_no_error_reports
+++ b/scripts/test_no_error_reports
@@ -39,6 +39,14 @@ if grep --quiet --fixed-strings 'The state engine was in incorrect state ERRORED
   exit 1
 fi
 
+if grep --quiet --fixed-strings 'Exception stopping the server' "$SERVERLOG"; then
+  {
+    printf "Server didn't shut down cleanly:\n"
+    cat server.log
+  } >&2
+  exit 1
+fi
+
 if ! grep --quiet --perl-regexp --only-matching '.+Done \(.+\)\! For help, type "help" or "\?"' "$SERVERLOG"; then
   {
     printf 'Server did not finish startup:'