From 516c3adcd1f9b6dc7d8a8956428f7a5b9a8be548 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:26:49 +1100
Subject: [PATCH 01/16] master-libvirt: saveArtifactsStep for deb/rpm upgrade

---
 master-libvirt/master.cfg | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg
index 2838d542f..90be25f88 100644
--- a/master-libvirt/master.cfg
+++ b/master-libvirt/master.cfg
@@ -198,12 +198,27 @@ def getPAMTestStep():
         command=["./pam-test.sh"],
     )
 
+def getSaveArtifactsStep():
+    return steps.DirectoryUpload(
+        name="save mariadb log files",
+        doStepIf=hasFailed,
+        workersrc="/home/buildbot/logs/",
+        masterdest=util.Interpolate(
+            "/srv/buildbot/packages/"
+            + "%(prop:tarbuildnum)s"
+            + "/logs/"
+            + "%(prop:buildername)s"
+        ),
+    )
+
+
 # FACTORY
 
 ## f_deb_install
 f_deb_install = util.BuildFactory()
 f_deb_install.addStep(getScript("deb-install.sh"))
 f_deb_install.addStep(getDebInstallStep())
+f_deb_install.addStep(getSaveArtifactsStep())
 f_deb_install.addStep(getScript("pam-test.sh"))
 f_deb_install.addStep(getPAMTestStep())
 
@@ -212,11 +227,13 @@ f_deb_upgrade = util.BuildFactory()
 f_deb_upgrade.addStep(getMajorVersionStep())
 f_deb_upgrade.addStep(getScript("deb-upgrade.sh"))
 f_deb_upgrade.addStep(getDebUpgradeStep())
+f_deb_upgrade.addStep(getSaveArtifactsStep())
 
 ## f_rpm_install
 f_rpm_install = util.BuildFactory()
 f_rpm_install.addStep(getScript("rpm-install.sh"))
 f_rpm_install.addStep(getRpmInstallStep())
+f_rpm_install.addStep(getSaveArtifactsStep())
 f_rpm_install.addStep(getScript("pam-test.sh"))
 f_rpm_install.addStep(getPAMTestStep())
 
@@ -225,6 +242,7 @@ f_rpm_upgrade = util.BuildFactory()
 f_rpm_upgrade.addStep(getMajorVersionStep())
 f_rpm_upgrade.addStep(getScript("rpm-upgrade.sh"))
 f_rpm_upgrade.addStep(getRpmUpgradeStep())
+f_rpm_upgrade.addStep(getSaveArtifactsStep())
 
 ####### WORKERS and BUILDERS
 

From 4ced66e1a6ec150576d3829d0fea85e0d440bec6 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:30:31 +1100
Subject: [PATCH 02/16] bash_lib: change get_columnstore-logs to
 save_failure_logs

This makes it more generic and expands the columnstore saving
to the required information.
* service logs
* columnstore data dir (tar.bz2)
* /tmp/columnstore_tmp_files (appended to log file)

More information like the mariadb.service journal and coredumps are saved.
---
 scripts/bash_lib.sh | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh
index b0f3bf144..0a4a0b33d 100644
--- a/scripts/bash_lib.sh
+++ b/scripts/bash_lib.sh
@@ -437,22 +437,33 @@ upgrade_test_type() {
   esac
 }
 
-get_columnstore_logs() {
+save_failure_logs() {
+  local logdir=../logs/
+  local logfile=$logdir/$test_mode.log
+  mkdir -p $logdir
+  bb_log_err "Previous step failed, saving mariadb logs"
   if [[ $test_mode == "columnstore" ]]; then
     bb_log_info "storing Columnstore logs in columnstore_logs"
     set +ex
-    # It is done in such a weird way, because Columnstore currently makes its logs hard to read
-    # //TEMP this is fragile and weird (test that /var/log/mariadb/columnstore exist)
-    for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do
-      f=/var/log/mariadb/columnstore/$f
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" 1>>/home/buildbot/columnstore_logs 2>&1
+    for f in $(sudo find /tmp/columnstore_tmp_files /var/log/mariadb/columnstore -type f); do
+      echo "----------- $f -----------" >>"$logfile"
+      sudo cat "$f" 1>>"$logfile" 2>&1
     done
-    for f in /tmp/columnstore_tmp_files/*; do
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" | sudo tee -a /home/buildbot/columnstore_logs 2>&1
+    for s in mcs-writeengineserver mcs-controllernode.service mcs-ddlproc.service mcs-dmlproc.service \
+            mcs-loadbrm.service mcs-primproc.service mcs-workernode@1.service; do
+        echo "----------- $s -----------" >>"$logfile"
+        sudo journalctl -u "$s" | tee -a "$logfile" 2>&1
     done
+    if [ -d /var/lib/columnstore ]; then
+      tar -Jcvf $logdir/columnstore.tar.bz2 /var/lib/columnstore
+    fi
+  fi
+  echo "----------- mariadb.service -----------" >>"$logfile"
+  sudo journalctl -u mariadb.service | tee -a "$logfile" 2>&1
+  if [ -f "$logfile" ]; then
+    bzip2 "$logfile"
   fi
+  sudo find /var/lib/systemd/coredump/ -type -f -exec mv {} $logdir \;
 }
 
 check_mariadb_server_and_create_structures() {

From 36ef633afe5473125debbed61e27c60ff957a75a Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 15:13:55 +1100
Subject: [PATCH 03/16] bash_lib: do not change set +-e during
 check_mariadb_server_and_{create,verify}_structures

These are handled by rpm/deb-install/upgrade.
---
 scripts/bash_lib.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh
index 0a4a0b33d..7ee975fdc 100644
--- a/scripts/bash_lib.sh
+++ b/scripts/bash_lib.sh
@@ -468,7 +468,6 @@ save_failure_logs() {
 
 check_mariadb_server_and_create_structures() {
   # All the commands below should succeed
-  set -e
   sudo mariadb -e "CREATE DATABASE db"
   sudo mariadb -e "CREATE TABLE db.t_innodb(a1 SERIAL, c1 CHAR(8)) ENGINE=InnoDB; INSERT INTO db.t_innodb VALUES (1,'foo'),(2,'bar')"
   sudo mariadb -e "CREATE TABLE db.t_myisam(a2 SERIAL, c2 CHAR(8)) ENGINE=MyISAM; INSERT INTO db.t_myisam VALUES (1,'foo'),(2,'bar')"
@@ -484,14 +483,12 @@ check_mariadb_server_and_create_structures() {
       exit 1
     fi
   fi
-  set +e
 }
 
 check_mariadb_server_and_verify_structures() {
   # Print "have_xx" capabilitites for the new server
   sudo mariadb -e "select 'Stat' t, variable_name name, variable_value val from information_schema.global_status where variable_name like '%have%' union select 'Vars' t, variable_name name, variable_value val from information_schema.global_variables where variable_name like '%have%' order by t, name"
   # All the commands below should succeed
-  set -e
   sudo mariadb -e "select @@version, @@version_comment"
   sudo mariadb -e "SHOW TABLES IN db"
   sudo mariadb -e "SELECT * FROM db.t_innodb; INSERT INTO db.t_innodb VALUES (3,'foo'),(4,'bar')"
@@ -511,7 +508,6 @@ check_mariadb_server_and_verify_structures() {
       exit 1
     fi
   fi
-  set +e
 }
 
 control_mariadb_server() {

From 638214dacf8010a22a90ab357d2e163d7801a91b Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:36:02 +1100
Subject: [PATCH 04/16] deb: add trap for save_failure logs when we start to
 process

the new packages in CI.
---
 scripts/deb-install.sh | 6 ++++++
 scripts/deb-upgrade.sh | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/scripts/deb-install.sh b/scripts/deb-install.sh
index 1de387bee..0cc466d99 100755
--- a/scripts/deb-install.sh
+++ b/scripts/deb-install.sh
@@ -69,6 +69,9 @@ fi
 # apt get update may be running in the background (Ubuntu start).
 apt_get_update
 
+# set -e already set at start of script
+trap save_failure_logs ERR
+
 sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
   apt-get install -y $package_list $columnstore_package_list"
 
@@ -120,6 +123,8 @@ sudo mariadb --verbose -e "create database test; \
   grant all on *.* to galera;"
 sudo mariadb -e "select @@version"
 bb_log_info "test for MDEV-18563, MDEV-18526"
+
+# disabling -e so save_failure_logs won't have an effect while having setting up for next test
 set +e
 
 control_mariadb_server stop
@@ -134,6 +139,7 @@ for p in /bin /sbin /usr/bin /usr/sbin /usr/local/bin /usr/local/sbin; do
   fi
 done
 sudo mariadb-install-db --no-defaults --user=mysql --plugin-maturity=unknown
+
 set +e
 ## Install mariadb-test for further use
 # sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 apt-get install -y mariadb-test"
diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh
index f25f348e0..9d204382f 100755
--- a/scripts/deb-upgrade.sh
+++ b/scripts/deb-upgrade.sh
@@ -164,6 +164,9 @@ deb_setup_bb_galera_artifacts_mirror
 deb_setup_bb_artifacts_mirror
 apt_get_update
 
+# now we upgrade, this is what we should save
+trap save_failure_logs ERR
+set -e
 # Install the new packages
 if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
   apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"; then

From 0ee903441b7d2cf0306afda9460ec25120e9fa5f Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:37:47 +1100
Subject: [PATCH 05/16] bash_lib: remove explict call to (remove)
 get_columnstore_logs

As we going be in set -e mode when this is called, the trap will
handle the saving of the error information.
---
 scripts/bash_lib.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh
index 7ee975fdc..c5d2911ac 100644
--- a/scripts/bash_lib.sh
+++ b/scripts/bash_lib.sh
@@ -478,10 +478,7 @@ check_mariadb_server_and_create_structures() {
   sudo mariadb -e "CREATE PROCEDURE db.p() SELECT * FROM db.v_merge"
   sudo mariadb -e "CREATE FUNCTION db.f() RETURNS INT DETERMINISTIC RETURN 1"
   if [[ $test_mode == "columnstore" ]]; then
-    if ! sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"; then
-      get_columnstore_logs
-      exit 1
-    fi
+    sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"
   fi
 }
 
@@ -503,10 +500,7 @@ check_mariadb_server_and_verify_structures() {
   sudo mariadb -e "SELECT db.f()"
 
   if [[ $test_mode == "columnstore" ]]; then
-    if ! sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"; then
-      get_columnstore_logs
-      exit 1
-    fi
+    sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"
   fi
 }
 

From 5d44adae22b37d3cbf965c867f8ba253646755b2 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:39:28 +1100
Subject: [PATCH 06/16] deb-install: remove duplicate columnstore error routine

---
 scripts/deb-install.sh | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/scripts/deb-install.sh b/scripts/deb-install.sh
index 0cc466d99..a86cd3aca 100755
--- a/scripts/deb-install.sh
+++ b/scripts/deb-install.sh
@@ -84,22 +84,6 @@ if [[ $systemdCapability == "yes" ]]; then
   if ! sudo systemctl status mariadb --no-pager; then
     sudo journalctl -xe --no-pager
     bb_log_warn "mariadb service isn't running properly after installation"
-    if echo "$package_list" | grep -q columnstore; then
-      bb_log_info "It is likely to be caused by ColumnStore"
-      bb_log_info "problems upon installation, getting the logs"
-      set +e
-      # It is done in such a weird way, because Columnstore currently makes its
-      # logs hard to read
-      for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do
-        f=/var/log/mariadb/columnstore/$f
-        echo "----------- $f -----------"
-        sudo cat "$f"
-      done
-      for f in /tmp/columnstore_tmp_files/*; do
-        echo "----------- $f -----------"
-        sudo cat "$f"
-      done
-    fi
     bb_log_err "mariadb service didn't start properly after installation"
     exit 1
   fi

From 87ac98902666c376f7167b209b5894071849aa15 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:42:10 +1100
Subject: [PATCH 07/16] deb-upgrade: replace explicit columnstore save logs

---
 scripts/deb-upgrade.sh | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh
index 9d204382f..3ef68860d 100755
--- a/scripts/deb-upgrade.sh
+++ b/scripts/deb-upgrade.sh
@@ -130,12 +130,8 @@ fi
 # To avoid confusing errors in further logic, do an explicit check
 # whether the service is up and running
 if [[ $systemdCapability == "yes" ]]; then
-  if ! sudo systemctl status mariadb --no-pager; then
-    sudo journalctl -xe --no-pager
-    get_columnstore_logs
-    bb_log_err "mariadb service didn't start properly after installation"
-    exit 1
-  fi
+  bb_log_info "Ensure mariadb.service is running"
+  sudo systemctl status mariadb --no-pager
 fi
 
 if [[ $test_mode == "all" ]]; then

From 03821e3c7086b540375f25e567368f826671f482 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:46:03 +1100
Subject: [PATCH 08/16] rpm-upgrade: add trap for error handling

---
 scripts/rpm-upgrade.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh
index 991f1ad7c..c28e7ccd6 100755
--- a/scripts/rpm-upgrade.sh
+++ b/scripts/rpm-upgrade.sh
@@ -194,6 +194,12 @@ fi
 
 rpm_setup_bb_galera_artifacts_mirror
 rpm_setup_bb_artifacts_mirror
+
+# Any of the below steps could fail
+# This is where the new packages are processed from
+trap save_failure_logs ERR
+set -e
+
 if [[ $test_type == "major" ]]; then
   # major upgrade (remove then install)
   echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install

From 59f9a2f448cbac086ec84d1184a5aced4e22e1c8 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:18:59 +1100
Subject: [PATCH 09/16] rpm-install: use save_failure_logs trap

---
 scripts/rpm-install.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scripts/rpm-install.sh b/scripts/rpm-install.sh
index 0c7a3dec2..aef85bd54 100755
--- a/scripts/rpm-install.sh
+++ b/scripts/rpm-install.sh
@@ -58,6 +58,10 @@ set -u
 
 sh -c 'g=/usr/lib*/galera*/libgalera_smm.so; echo -e "[galera]\nwsrep_provider=$g"' |
   sudo tee /etc/my.cnf.d/galera.cnf
+
+# Any of the below steps could fail
+trap save_failure_logs ERR
+set -e
 case "$systemdCapability" in
   yes)
     if ! sudo systemctl start mariadb; then

From 52cd6012f1809815be9303a4defdfdf15917f50d Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:40:42 +1100
Subject: [PATCH 10/16] deb-install: simplify service check

We are under set -e so the error saving routine is there.
---
 scripts/deb-install.sh | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/scripts/deb-install.sh b/scripts/deb-install.sh
index a86cd3aca..c94d345d2 100755
--- a/scripts/deb-install.sh
+++ b/scripts/deb-install.sh
@@ -81,12 +81,8 @@ wait_for_mariadb_upgrade
 # To avoid confusing errors in further logic, do an explicit check whether the
 # service is up and running
 if [[ $systemdCapability == "yes" ]]; then
-  if ! sudo systemctl status mariadb --no-pager; then
-    sudo journalctl -xe --no-pager
-    bb_log_warn "mariadb service isn't running properly after installation"
-    bb_log_err "mariadb service didn't start properly after installation"
-    exit 1
-  fi
+  bb_log_info "test for systemd service for mariadb.service started"
+  sudo systemctl status mariadb.service --no-pager
 fi
 
 # Due to MDEV-14622 and its effect on Spider installation,

From 2ab61c5255ade5b069fe5a5eec3c0a73773bc16a Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:50:47 +1100
Subject: [PATCH 11/16] {deb/rpm}-upgrade: simplify install/upgrade test
 operations

replace pattern:

  # comment about what we are doing that isn't in bb log
  if something_might_go_wrong; then
     log_err - it went wrong
     (missing error capture)
  exit 1

to:

   bb_log_info "where testing this"
   sometime_might_go_wrong

The something_might_go_wrong in the second step will trap
and have all the saving of the failure afterwards leaving
the user with the info of what was being tested to provide
the failure context.
---
 scripts/deb-upgrade.sh | 35 ++++++++++++++---------------------
 scripts/rpm-upgrade.sh |  7 ++-----
 2 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh
index 3ef68860d..932ac7b2c 100755
--- a/scripts/deb-upgrade.sh
+++ b/scripts/deb-upgrade.sh
@@ -110,20 +110,16 @@ apt_get_update
 # We will wait till they finish, to avoid any clashes with SQL we are going to execute
 wait_for_mariadb_upgrade
 
-if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
-  apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"; then
-  bb_log_err "Installation of a previous release failed, see the output above"
-  exit 1
-fi
+bb_log_info "Install previous version"
+sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
+  apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"
 
 wait_for_mariadb_upgrade
 
 if [[ -n $spider_package_list ]]; then
-  if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
-    apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list"; then
-    bb_log_err "Installation of Spider from the previous release failed, see the output above"
-    exit 1
-  fi
+  bb_log_info "Install spider packages"
+  sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
+    apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list"
   wait_for_mariadb_upgrade
 fi
 
@@ -163,20 +159,17 @@ apt_get_update
 # now we upgrade, this is what we should save
 trap save_failure_logs ERR
 set -e
-# Install the new packages
-if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
-  apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"; then
-  bb_log_err "installation of the new packages failed, see the output above"
-  exit 1
-fi
+
+bb_log_info "Install new packages"
+sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
+  apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"
+
 wait_for_mariadb_upgrade
 
 if [[ -n $spider_package_list ]]; then
-  if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
-    apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list"; then
-    bb_log_err "Installation of the new Spider packages failed, see the output above"
-    exit 1
-  fi
+  bb_log_info "Install spider packages $spider_package_list"
+  sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
+    apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list"
   wait_for_mariadb_upgrade
 fi
 if [[ $test_mode == "columnstore" ]]; then
diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh
index c28e7ccd6..23e9e6cac 100755
--- a/scripts/rpm-upgrade.sh
+++ b/scripts/rpm-upgrade.sh
@@ -231,11 +231,8 @@ if [[ $test_type == "major" ]] || [[ $test_mode == "columnstore" ]]; then
   control_mariadb_server restart
 fi
 
-# Make sure that the new server is running
-if sudo mariadb -e "select @@version" | grep "$old_version"; then
-  bb_log_err "the server was not upgraded or was not restarted after upgrade"
-  exit 1
-fi
+bb_log_info "Make sure that the new server is running"
+sudo mariadb -e "select @@version" | grep "$old_version"
 
 # Run mariadb-upgrade for non-GA branches (minor upgrades in GA branches
 # shouldn't need it)

From dbbb89cee20efb82cd849b2b008f392f796faac3 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:15:19 +1100
Subject: [PATCH 12/16] bash_lib: log warning in control_mariadb_server if
 systemdCapability

isn't set
---
 scripts/bash_lib.sh    | 5 +++++
 scripts/rpm-install.sh | 4 ----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh
index c5d2911ac..fc816c88c 100644
--- a/scripts/bash_lib.sh
+++ b/scripts/bash_lib.sh
@@ -513,6 +513,11 @@ control_mariadb_server() {
     no)
       sudo /etc/init.d/mysql "$1"
       ;;
+    *)
+      bb_log_warn "should never happen, check your configuration:"
+      bb_log_warn "(systemdCapability property is not set or is set to a wrong value ($systemdCapability))"
+    ;;
+
   esac
 }
 
diff --git a/scripts/rpm-install.sh b/scripts/rpm-install.sh
index aef85bd54..f7c2bb10c 100755
--- a/scripts/rpm-install.sh
+++ b/scripts/rpm-install.sh
@@ -73,10 +73,6 @@ case "$systemdCapability" in
   no)
     sudo /etc/init.d/mysql restart
     ;;
-  *)
-    bb_log_warn "should never happen, check your configuration:"
-    bb_log_warn "(systemdCapability property is not set or is set to a wrong value)"
-    ;;
 esac
 
 sudo mariadb -e "drop database if exists test; \

From 511c543923588a627906838a779734daef19d51f Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:20:09 +1100
Subject: [PATCH 13/16] rpm-install/upgrade: use control_mariadb_server_start

---
 scripts/rpm-install.sh | 14 ++------------
 scripts/rpm-upgrade.sh | 24 ++----------------------
 2 files changed, 4 insertions(+), 34 deletions(-)

diff --git a/scripts/rpm-install.sh b/scripts/rpm-install.sh
index f7c2bb10c..8fa87b06a 100755
--- a/scripts/rpm-install.sh
+++ b/scripts/rpm-install.sh
@@ -62,18 +62,8 @@ sh -c 'g=/usr/lib*/galera*/libgalera_smm.so; echo -e "[galera]\nwsrep_provider=$
 # Any of the below steps could fail
 trap save_failure_logs ERR
 set -e
-case "$systemdCapability" in
-  yes)
-    if ! sudo systemctl start mariadb; then
-      sudo journalctl -lxn 500 --no-pager -u mariadb.service
-      sudo systemctl -l status mariadb.service --no-pager
-      exit 1
-    fi
-    ;;
-  no)
-    sudo /etc/init.d/mysql restart
-    ;;
-esac
+
+control_mariadb_server start
 
 sudo mariadb -e "drop database if exists test; \
   create database test; \
diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh
index 23e9e6cac..a0419a35b 100755
--- a/scripts/rpm-upgrade.sh
+++ b/scripts/rpm-upgrade.sh
@@ -138,28 +138,8 @@ echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install ||
 #fi
 
 # Start the server, check that it is working and create some structures
-case $(expr "$prev_major_version" '<' "10.1")"$systemdCapability" in
-  0yes)
-    sudo systemctl start mariadb
-    if [[ $distro != *"sles"* ]] && [[ $distro != *"suse"* ]]; then
-      sudo systemctl enable mariadb
-    else
-      bb_log_warn "due to MDEV-23044 mariadb service won't be enabled in the test"
-    fi
-    sudo systemctl status mariadb --no-pager
-    ;;
-  *)
-    sudo /etc/init.d/mysql start
-    ;;
-esac
-
-# shellcheck disable=SC2181
-if (($? != 0)); then
-  bb_log_err "Server startup failed"
-  sudo cat /var/log/messages | grep -iE 'mysqld|mariadb'
-  sudo cat /var/lib/mysql/*.err
-  exit 1
-fi
+#
+control_mariadb_server start
 
 check_mariadb_server_and_create_structures
 

From d3d431822b47fc79e25cd918fe86447d0d0e1762 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 13 Nov 2024 14:56:04 +1100
Subject: [PATCH 14/16] rpm install/upgrade - remove xargs

echo $v | sudo xargs ..
is a antipattern, just use bash arrays.

Replace if test condition; then fail
with just; test condition, and let +e handle it.

Rather than set -u /+u give ID_LIKE
a default value so it won't error
or match suse when its blank.
---
 scripts/rpm-install.sh | 15 ++++++++-------
 scripts/rpm-upgrade.sh | 19 +++++++++----------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/scripts/rpm-install.sh b/scripts/rpm-install.sh
index 8fa87b06a..3f47e100e 100755
--- a/scripts/rpm-install.sh
+++ b/scripts/rpm-install.sh
@@ -44,17 +44,18 @@ rpm_setup_bb_artifacts_mirror
 rpm_pkg_makecache
 
 # install all packages
-pkg_list=$(rpm_repoquery) ||
+read -ra package_array = <<< "$(rpm_repoquery)"
+
+if [ ${#package_array[@]} -eq 0 ]; then
   bb_log_err "Unable to retrieve package list from repository"
+fi
 
-# ID_LIKE may not exist
-set +u
-if [[ $ID_LIKE =~ ^suse* ]]; then
-  echo "$pkg_list" | xargs sudo "$pkg_cmd" -n install
+if [[ "${ID_LIKE:-empty}" =~ ^suse* ]]; then
+  sudo "$pkg_cmd" -n install "${package_array[@]}"
 else
-  echo "$pkg_list" | xargs sudo "$pkg_cmd" -y install
+  sudo "$pkg_cmd" -y install "${package_array[@]}"
 fi
-set -u
+
 
 sh -c 'g=/usr/lib*/galera*/libgalera_smm.so; echo -e "[galera]\nwsrep_provider=$g"' |
   sudo tee /etc/my.cnf.d/galera.cnf
diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh
index a0419a35b..41bf18da4 100755
--- a/scripts/rpm-upgrade.sh
+++ b/scripts/rpm-upgrade.sh
@@ -119,9 +119,7 @@ bb_log_info "Package_list: $package_list"
 # /etc/zypp/repos.d/SUSE_Linux_Enterprise_Server_12_SP3_x86_64:SLES12-SP3-Updates.repo
 # /etc/zypp/repos.d/SUSE_Linux_Enterprise_Server_12_SP3_x86_64:SLES12-SP3-Pool.repo
 
-# ID_LIKE may not exist
-set +u
-if [[ $ID_LIKE =~ ^suse* ]]; then
+if [[ "${ID_LIKE:-empty}" =~ ^suse* ]]; then
   sudo "$pkg_cmd" clean --all
   pkg_cmd_options="-n"
   pkg_cmd_upgrade="update"
@@ -132,10 +130,11 @@ else
 fi
 set -u
 
+read -ra package_array <<< "$package_list"
+
 # Install previous release
-echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install ||
-  bb_log_err "installation of a previous release failed, see the output above"
-#fi
+bb_log_info "Install previous release"
+sudo "$pkg_cmd" "$pkg_cmd_options" install "${package_array[@]}"
 
 # Start the server, check that it is working and create some structures
 #
@@ -167,8 +166,8 @@ fi
 # //TEMP upgrade does not work without this but why? Can't we fix it?
 if [[ $test_type == "major" ]]; then
   bb_log_info "remove old packages for major upgrade"
-  packages_to_remove=$(rpm -qa | grep 'MariaDB-' | awk -F'-' '{print $1"-"$2}')
-  echo "$packages_to_remove" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" remove
+  readarray -t package_array <<< "$(rpm -qa | grep 'MariaDB-' | awk -F'-' '{print $1"-"$2}')"
+  sudo "$pkg_cmd" "$pkg_cmd_options" remove "${package_array[@]}"
   rpm -qa | grep -iE 'maria|mysql' || true
 fi
 
@@ -182,10 +181,10 @@ set -e
 
 if [[ $test_type == "major" ]]; then
   # major upgrade (remove then install)
-  echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install
+  sudo "$pkg_cmd" "$pkg_cmd_options" install "${package_array[@]}"
 else
   # minor upgrade (upgrade works)
-  echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" "$pkg_cmd_upgrade"
+  sudo "$pkg_cmd" "$pkg_cmd_options" "$pkg_cmd_upgrade" "${package_array[@]}"
 fi
 # set +e
 

From 250876c3233401d20ba6ae54b7ee69be9c9d79f3 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 12:55:23 +1100
Subject: [PATCH 15/16] rpm upgrade: info the steps we are doing

As script comments don't show when running make the output more readable.
---
 scripts/rpm-upgrade.sh | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh
index 41bf18da4..812b7ef9c 100755
--- a/scripts/rpm-upgrade.sh
+++ b/scripts/rpm-upgrade.sh
@@ -142,7 +142,7 @@ control_mariadb_server start
 
 check_mariadb_server_and_create_structures
 
-# Store information about the server before upgrade
+bb_log_info "Store information about the server before upgrade"
 collect_dependencies old rpm
 store_mariadb_server_info old
 
@@ -213,19 +213,21 @@ fi
 bb_log_info "Make sure that the new server is running"
 sudo mariadb -e "select @@version" | grep "$old_version"
 
-# Run mariadb-upgrade for non-GA branches (minor upgrades in GA branches
-# shouldn't need it)
 if [[ $major_version == "$development_branch" ]] || [[ $test_type == "major" ]]; then
+  bb_log_info "Run MariaDB upgrade"
   sudo mariadb-upgrade
+else
+  bb_log_info "Skipping mariadb-upgrade for non-GA branches and minor upgrades in GA branchesa - shouldn't need it"
 fi
 
-# Check that the server is functioning and previously created structures are available
+bb_log_info "Check that the server is functioning and previously created structures are available"
 check_mariadb_server_and_verify_structures
 
-# Store information about the server after upgrade
+bb_log_info "Store information about the server after upgrade"
 collect_dependencies new rpm
 store_mariadb_server_info new
 
+bb_log_info "Check upgrade versions"
 check_upgraded_versions
 
 bb_log_ok "all done"

From d32544d05602c7147ea7f7659141b3ee1de934f5 Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Wed, 27 Nov 2024 14:16:37 +1100
Subject: [PATCH 16/16] rpm-install: galera detection with fault tolerance

There was no need for subshell, and to make shellcheck
happy do the glob to arrays.
---
 scripts/rpm-install.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scripts/rpm-install.sh b/scripts/rpm-install.sh
index 3f47e100e..c62076596 100755
--- a/scripts/rpm-install.sh
+++ b/scripts/rpm-install.sh
@@ -57,7 +57,12 @@ else
 fi
 
 
-sh -c 'g=/usr/lib*/galera*/libgalera_smm.so; echo -e "[galera]\nwsrep_provider=$g"' |
+galera=(/usr/lib*/galera*/libgalera_smm.so)
+if [ ${#galera[@]} -ne 1 ]; then
+    bb_log_error "Expected exactly one file, found ${#galera[@]}"
+    exit 1
+fi
+echo -e "[galera]\nwsrep_provider=${galera[0]}" |
   sudo tee /etc/my.cnf.d/galera.cnf
 
 # Any of the below steps could fail