Generating backtrace automatically when ctafrontend dies, collect it for...

Generating backtrace automatically when ctafrontend dies, collect it for artifacts and display it when the namespace gets deleted so that it appears on the Pipeline.

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctafrontend.sh

@@ -58,5 +58,7 @@ echo '/var/log/cta/core_%e.%p' > /proc/sys/kernel/core_pattern
 echo "Launching frontend"
 runuser --shell='/bin/bash' --session-command='cd ~cta; xrootd -n cta -c /etc/xrootd/xrootd-cta.cfg -I v4' cta
 
-echo "frontend died"
+echo "ctafrontend died"
+echo "analysing core file if any"
+/opt/run/bin/ctafrontend_bt.sh
 sleep infinity

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctafrontend_bt.sh

+#!/bin/bash
+
+COREFILE=$(ls /var/log/cta/core* | head -n 1)
+
+test -z ${COREFILE} && (echo "NO COREFILE FOUND, EXITING"; exit 1)
+
+echo "PROCESSING COREFILE: ${COREFILE}"
+
+yum install -y xrootd-debuginfo cta-debuginfo
+
+cat <<EOF > /tmp/ctabt.gdb
+file /usr/bin/xrootd
+core ${COREFILE}
+bt
+quit
+EOF
+
+gdb -x /tmp/ctabt.gdb > ${COREFILE}.bt
+
+echo "BACKTRACE AVAILABLE IN ${COREFILE}.bt"
+exit 0

continuousintegration/orchestration/delete_instance.sh

@@ -27,6 +27,17 @@ if [ ! -z "${error}" ]; then
     exit 1
 fi
 
+###
+# Display all backtraces if any
+###
+
+for backtracefile in $(kubectl -n ssi exec ctacli -- bash -c 'find /mnt/logs | grep core | grep bt$'); do
+  pod=$(echo ${backtracefile} | cut -d/ -f4)
+  echo "Found backtrace in pod ${pod}:"
+  kubectl -n ssi exec ctacli -- cat ${backtracefile}
+done
+
+###
 # Collect the logs
 ###