Collecting /var/log on all containers

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctacli.sh

 #!/bin/bash
 
+/opt/run/bin/init_pod.sh
+
 yum-config-manager --enable cta-artifacts
 yum-config-manager --enable ceph
 

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh

 #!/bin/sh 
 
+/opt/run/bin/init_pod.sh
+
 yum-config-manager --enable cta-artifacts
 yum-config-manager --enable eos-citrine-commit
 yum-config-manager --enable eos-citrine-depend
@@ -8,9 +10,6 @@ yum-config-manager --enable eos-citrine
 # Install missing RPMs
 yum -y install eos-client eos-server xrootd-client xrootd-debuginfo xrootd-server cta-cli cta-debuginfo
 
-# fix reverse DNS for EOS
-/opt/run/bin/init_pod.sh
-
 # copy needed template configuration files (nice to get all lines for logs)
 yes | cp -r /opt/ci/ctaeos/etc /
 

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctafrontend.sh

 #!/bin/sh 
 
+/opt/run/bin/init_pod.sh
+
 yum-config-manager --enable cta-artifacts
 yum-config-manager --enable ceph
 

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init.sh

 #!/bin/sh 
 
+/opt/run/bin/init_pod.sh
+
 # enable cta repository from previously built artifacts
 yum-config-manager --enable cta-artifacts
 yum-config-manager --enable ceph

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init_pod.sh

 #!/bin/bash
 
+LOGMOUNT=/mnt/logs
+
+PV_PATH=""
+
+if [ "-${MY_CONTAINER}-" != "--" ]; then
+  PV_PATH="${LOGMOUNT}/${MY_NAME}/${MY_CONTAINER}"
+else
+  PV_PATH="${LOGMOUNT}/${MY_NAME}"
+fi
+mkdir -p ${PV_PATH}
+
+echo "Copying initial /var/log content to ${PV_PATH}"
+cd /var/log
+tar -c . | tar -C ${PV_PATH} -xv
+
+echo "Mounting logs volume ${PV_PATH} in /var/log"
+mount --bind ${PV_PATH} /var/log
+
 echo -n "Fixing reverse DNS for $(hostname): "
 sed -i -c "s/^\($(hostname -i)\)\s\+.*$/\1 $(hostname -s).$(grep search /etc/resolv.conf | cut -d\  -f2) $(hostname -s)/" /etc/hosts
 echo "DONE"

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/kdc.sh

 #!/bin/bash
 
+/opt/run/bin/init_pod.sh
+
 # Install missing RPMs (kdc)
 yum -y install heimdal-server heimdal-workstation
 

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/rmcd.sh

 #!/bin/sh 
 
+/opt/run/bin/init_pod.sh
+
 yum-config-manager --enable cta-artifacts
 yum-config-manager --enable ceph
 yum-config-manager --enable castor
@@ -7,8 +9,6 @@ yum-config-manager --enable castor
 # Install missing RPMs
 yum -y install mt-st mtx lsscsi sg3_utils cta-taped cta-debuginfo castor-rmc-server
 
-/opt/run/bin/init_pod.sh
-
 # source library configuration file
 echo "Using this configuration for library:"
 /opt/run/bin/init_library.sh

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/taped.sh

 #!/bin/sh 
 
+/opt/run/bin/init_pod.sh
+
 yum-config-manager --enable cta-artifacts
 yum-config-manager --enable ceph
 yum-config-manager --enable castor
@@ -7,8 +9,6 @@ yum-config-manager --enable castor
 # Install missing RPMs
 yum -y install mt-st mtx lsscsi sg3_utils cta-taped cta-debuginfo castor-rmc-server ceph-common
 
-/opt/run/bin/init_pod.sh
-
 echo "Using this configuration for library:"
 /opt/run/bin/init_library.sh
 cat /tmp/library-rc.sh

continuousintegration/orchestration/create_instance.sh

@@ -119,6 +119,9 @@ kubectl --namespace=${instance} create -f /opt/kubernetes/CTA/library/config/lib
 
 echo "Got library: ${LIBRARY_DEVICE}"
 
+echo -n "Requesting an unused log volume"
+kubectl create -f ./pvc_logs.yaml --namespace=${instance}
+
 echo "Creating services in instance"
 
 for service_file in *svc\.yaml; do

continuousintegration/orchestration/delete_instance.sh

@@ -37,6 +37,7 @@ echo "Collecting stdout logs of pods to ${tmpdir}"
 for podcontainer in "init -c ctainit" "ctacli -c ctacli" "ctaeos -c mgm" "ctafrontend -c ctafrontend" "kdc -c kdc" "tpsrv -c taped" "tpsrv -c rmcd"; do
   kubectl --namespace ${instance} logs ${podcontainer} > ${tmpdir}/$(echo ${podcontainer} | sed -e 's/ -c /-/').log
 done
+kubectl --namespace ${instance} exec ctacli -- tar -C /mnt/logs -zcf - . > ${tmpdir}/varlog.tgz
 
 if [ -z "${CI_PIPELINE_ID}" ]; then
 	# we are in the context of a CI run => save artifacts in the directory structure of the build
@@ -55,6 +56,8 @@ done
 
 echo OK
 
+# this now useless as dummy NFS PV were replaced by local volumes
+# those are recycled automatically
 ./recycle_librarydevice_PV.sh
 
 echo "Status of library pool after test:"

continuousintegration/orchestration/pod-ctacli.yaml

@@ -28,6 +28,8 @@ spec:
     volumeMounts:
     - mountPath: /shared
       name: shared
+    - mountPath: /mnt/logs
+      name: logstorage
     securityContext:
       privileged: true
 
@@ -35,4 +37,6 @@ spec:
   - name: shared
     hostPath:
       path: /opt/cta
-
+  - name: logstorage
+    persistentVolumeClaim:
+      claimName: claimlogs

continuousintegration/orchestration/pod-ctaeos.yaml

@@ -30,6 +30,8 @@ spec:
     volumeMounts:
     - mountPath: /shared
       name: shared
+    - mountPath: /mnt/logs
+      name: logstorage
     securityContext:
       privileged: true
 
@@ -37,3 +39,6 @@ spec:
   - name: shared
     hostPath:
       path: /opt/cta
+  - name: logstorage
+    persistentVolumeClaim:
+      claimName: claimlogs

continuousintegration/orchestration/pod-ctafrontend.yaml

@@ -26,8 +26,6 @@ spec:
     command: ['/opt/run/bin/ctafrontend.sh']
     args: ["none"]
     volumeMounts:
-    - mountPath: /dev/log
-      name: dev-log
     - mountPath: /shared
       name: shared
     - mountPath: /etc/config/objectstore
@@ -36,6 +34,8 @@ spec:
       name: mydatabase
     - mountPath: /etc/config/library
       name: mylibrary
+    - mountPath: /mnt/logs
+      name: logstorage
     securityContext:
       # RW access needed to volumes
       privileged: true
@@ -44,9 +44,6 @@ spec:
       name: ctafrontend
       protocol: TCP
   volumes:
-  - name: dev-log
-    hostPath:
-      path: /dev/log
   - name: shared
     hostPath:
       path: /opt/cta
@@ -59,3 +56,6 @@ spec:
   - name: mylibrary
     configMap:
       name: library-config
+  - name: logstorage
+    persistentVolumeClaim:
+      claimName: claimlogs

continuousintegration/orchestration/pod-init.yaml

@@ -42,6 +42,8 @@ spec:
       name: mydatabase
     - mountPath: /etc/config/library
       name: mylibrary
+    - mountPath: /mnt/logs
+      name: logstorage
     securityContext:
       privileged: true
 
@@ -58,3 +60,6 @@ spec:
   - name: mylibrary
     configMap:
       name: library-config
+  - name: logstorage
+    persistentVolumeClaim:
+      claimName: claimlogs

continuousintegration/orchestration/pod-kdc.yaml

@@ -28,6 +28,8 @@ spec:
     volumeMounts:
     - mountPath: /shared
       name: shared
+    - mountPath: /mnt/logs
+      name: logstorage
     securityContext:
       privileged: true
 
@@ -35,3 +37,6 @@ spec:
   - name: shared
     hostPath:
       path: /opt/cta
+  - name: logstorage
+    persistentVolumeClaim:
+      claimName: claimlogs

continuousintegration/orchestration/pod-tpsrv.yaml

@@ -11,6 +11,12 @@ spec:
     image: gitlab-registry.cern.ch/cta/ctageneric:78673git921a9300
     stdin: true
     env:
+    - name: MY_CONTAINER
+      value: "rmcd"
+    - name: MY_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
     - name: MY_NAMESPACE
       valueFrom:
         fieldRef:
@@ -26,6 +32,8 @@ spec:
       name: shared
     - mountPath: /etc/config/library
       name: mylibrary
+    - mountPath: /mnt/logs
+      name: logstorage
     securityContext:
       privileged: true
 
@@ -33,6 +41,8 @@ spec:
     image: gitlab-registry.cern.ch/cta/ctageneric:78673git921a9300
     stdin: true
     env:
+    - name: MY_CONTAINER
+      value: "taped"
     - name: MY_NAME
       valueFrom:
         fieldRef:
@@ -58,6 +68,8 @@ spec:
       name: mydatabase
     - mountPath: /etc/config/library
       name: mylibrary
+    - mountPath: /mnt/logs
+      name: logstorage
     securityContext:
       privileged: true
 
@@ -74,3 +86,6 @@ spec:
   - name: mylibrary
     configMap:
       name: library-config
+  - name: logstorage
+    persistentVolumeClaim:
+      claimName: claimlogs

continuousintegration/orchestration/pvc_logs.yaml

+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: claimlogs
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 2Gi
+  selector:
+    matchLabels:
+      type: logs