From 291c49cc26add0b5fc83830c1a68d1e95e3e82f8 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 6 Nov 2024 15:13:01 +1100
Subject: [PATCH] MDBF-814 for install/upgrade save logs on failure

This applies to Columnstore logs, data

core files also saved.

Also saves the systemd journal of applicable services.

Being an trap ERR it will execute only if an err occurs.

Reviews thanks Faustin and Roman
---
 master-libvirt/master.cfg | 18 ++++++++++++++++
 scripts/bash_lib.sh       | 43 +++++++++++++++++++++++----------------
 scripts/deb-upgrade.sh    |  1 -
 3 files changed, 43 insertions(+), 19 deletions(-)

diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg
index 455a2926..8b126fdc 100644
--- a/master-libvirt/master.cfg
+++ b/master-libvirt/master.cfg
@@ -185,29 +185,47 @@ def getMajorVersionStep():
     )
 
 
+def getSaveArtifactsStep():
+    return steps.DirectoryUpload(
+        name="save mariadb log files",
+        doStepIf=hasFailed,
+        workersrc="/home/buildbot/logs/",
+        masterdest=util.Interpolate(
+            "/srv/buildbot/packages/"
+            + "%(prop:tarbuildnum)s"
+            + "/logs/"
+            + "%(prop:buildername)s"
+        ),
+    )
+
+
 # FACTORY
 
 ## f_deb_install
 f_deb_install = util.BuildFactory()
 f_deb_install.addStep(getScript("deb-install.sh"))
 f_deb_install.addStep(getDebInstallStep())
+f_deb_install.addStep(getSaveArtifactsStep())
 
 ## f_deb_upgrade
 f_deb_upgrade = util.BuildFactory()
 f_deb_upgrade.addStep(getMajorVersionStep())
 f_deb_upgrade.addStep(getScript("deb-upgrade.sh"))
 f_deb_upgrade.addStep(getDebUpgradeStep())
+f_deb_upgrade.addStep(getSaveArtifactsStep())
 
 ## f_rpm_install
 f_rpm_install = util.BuildFactory()
 f_rpm_install.addStep(getScript("rpm-install.sh"))
 f_rpm_install.addStep(getRpmInstallStep())
+f_rpm_install.addStep(getSaveArtifactsStep())
 
 ## f_rpm_upgrade
 f_rpm_upgrade = util.BuildFactory()
 f_rpm_upgrade.addStep(getMajorVersionStep())
 f_rpm_upgrade.addStep(getScript("rpm-upgrade.sh"))
 f_rpm_upgrade.addStep(getRpmUpgradeStep())
+f_rpm_upgrade.addStep(getSaveArtifactsStep())
 
 ####### WORKERS and BUILDERS
 
diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh
index b0f3bf14..4cbf9162 100644
--- a/scripts/bash_lib.sh
+++ b/scripts/bash_lib.sh
@@ -4,6 +4,8 @@
 # Include with:
 # . ./bash_lib.sh
 
+trap save_failure_logs ERR
+
 bb_log_info() {
   set +x
   echo >&1 "INFO: $*"
@@ -437,22 +439,33 @@ upgrade_test_type() {
   esac
 }
 
-get_columnstore_logs() {
+save_failure_logs() {
+  trap "" ERR
+  local logdir=../logs/
+  local logfile=$logdir/$test_mode.log
+  mkdir -p $logdir
   if [[ $test_mode == "columnstore" ]]; then
     bb_log_info "storing Columnstore logs in columnstore_logs"
     set +ex
-    # It is done in such a weird way, because Columnstore currently makes its logs hard to read
-    # //TEMP this is fragile and weird (test that /var/log/mariadb/columnstore exist)
-    for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do
-      f=/var/log/mariadb/columnstore/$f
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" 1>>/home/buildbot/columnstore_logs 2>&1
+    for f in $(sudo find /tmp/columnstore_tmp_files /var/log/mariadb/columnstore -type f); do
+      echo "----------- $f -----------" >>"$logfile"
+      sudo cat "$f" 1>>"$logfile" 2>&1
     done
-    for f in /tmp/columnstore_tmp_files/*; do
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" | sudo tee -a /home/buildbot/columnstore_logs 2>&1
+    for s in mcs-writeengineserver mcs-controllernode.service mcs-ddlproc.service mcs-dmlproc.service \
+            mcs-loadbrm.service mcs-primproc.service mcs-workernode@1.service; do
+        echo "----------- $s -----------" >>"$logfile"
+        sudo journalctl -u "$s" | tee -a "$logfile" 2>&1
     done
+    if [ -d /var/lib/columnstore ]; then
+      tar -Jcvf $logdir/columnstore.tar.bz2 /var/lib/columnstore
+    fi
   fi
+  echo "----------- mariadb.service -----------" >>"$logfile"
+  sudo journalctl -u mariadb.service | tee -a "$logfile" 2>&1
+  if [ -f "$logfile" ]; then
+    bzip2 "$logfile"
+  fi
+  sudo find /var/lib/systemd/coredump/ -type -f -exec mv {} $logdir \;
 }
 
 check_mariadb_server_and_create_structures() {
@@ -468,10 +481,7 @@ check_mariadb_server_and_create_structures() {
   sudo mariadb -e "CREATE PROCEDURE db.p() SELECT * FROM db.v_merge"
   sudo mariadb -e "CREATE FUNCTION db.f() RETURNS INT DETERMINISTIC RETURN 1"
   if [[ $test_mode == "columnstore" ]]; then
-    if ! sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"; then
-      get_columnstore_logs
-      exit 1
-    fi
+    sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"
   fi
   set +e
 }
@@ -495,10 +505,7 @@ check_mariadb_server_and_verify_structures() {
   sudo mariadb -e "SELECT db.f()"
 
   if [[ $test_mode == "columnstore" ]]; then
-    if ! sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"; then
-      get_columnstore_logs
-      exit 1
-    fi
+    sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"
   fi
   set +e
 }
diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh
index 5d6c598a..b8cf92df 100755
--- a/scripts/deb-upgrade.sh
+++ b/scripts/deb-upgrade.sh
@@ -125,7 +125,6 @@ fi
 if [[ $systemdCapability == "yes" ]]; then
   if ! sudo systemctl status mariadb --no-pager; then
     sudo journalctl -xe --no-pager
-    get_columnstore_logs
     bb_log_err "mariadb service didn't start properly after installation"
     exit 1
   fi