From bf03a65ed9edf45853b40ecee31802712e54de25 Mon Sep 17 00:00:00 2001
From: Julien Leduc <julien.leduc@cern.ch>
Date: Tue, 14 Mar 2017 00:27:48 +0100
Subject: [PATCH] Collecting /var/log on all containers
---
.../ctafrontend/cc7/opt/run/bin/ctacli.sh | 2 ++
.../ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh | 5 ++---
.../ctafrontend/cc7/opt/run/bin/ctafrontend.sh | 2 ++
.../docker/ctafrontend/cc7/opt/run/bin/init.sh | 2 ++
.../ctafrontend/cc7/opt/run/bin/init_pod.sh | 18 ++++++++++++++++++
.../docker/ctafrontend/cc7/opt/run/bin/kdc.sh | 2 ++
.../docker/ctafrontend/cc7/opt/run/bin/rmcd.sh | 4 ++--
.../ctafrontend/cc7/opt/run/bin/taped.sh | 4 ++--
.../orchestration/create_instance.sh | 3 +++
.../orchestration/delete_instance.sh | 3 +++
.../orchestration/pod-ctacli.yaml | 6 +++++-
.../orchestration/pod-ctaeos.yaml | 5 +++++
.../orchestration/pod-ctafrontend.yaml | 10 +++++-----
.../orchestration/pod-init.yaml | 5 +++++
.../orchestration/pod-kdc.yaml | 5 +++++
.../orchestration/pod-tpsrv.yaml | 15 +++++++++++++++
.../orchestration/pvc_logs.yaml | 13 +++++++++++++
17 files changed, 91 insertions(+), 13 deletions(-)
create mode 100644 continuousintegration/orchestration/pvc_logs.yaml
diff --git a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctacli.sh b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctacli.sh
index 0bcf2fa849..12089d411a 100755
--- a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctacli.sh
+++ b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctacli.sh
@@ -1,5 +1,7 @@
#!/bin/bash
+/opt/run/bin/init_pod.sh
+
yum-config-manager --enable cta-artifacts
yum-config-manager --enable ceph
diff --git a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh
index 7158391c23..7b8f078378 100755
--- a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh
+++ b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh
@@ -1,5 +1,7 @@
#!/bin/sh
+/opt/run/bin/init_pod.sh
+
yum-config-manager --enable cta-artifacts
yum-config-manager --enable eos-citrine-commit
yum-config-manager --enable eos-citrine-depend
@@ -8,9 +10,6 @@ yum-config-manager --enable eos-citrine
# Install missing RPMs
yum -y install eos-client eos-server xrootd-client xrootd-debuginfo xrootd-server cta-cli cta-debuginfo
-# fix reverse DNS for EOS
-/opt/run/bin/init_pod.sh
-
# copy needed template configuration files (nice to get all lines for logs)
yes | cp -r /opt/ci/ctaeos/etc /
diff --git a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctafrontend.sh b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctafrontend.sh
index f6defbcb9c..9193b44c1c 100755
--- a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctafrontend.sh
+++ b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctafrontend.sh
@@ -1,5 +1,7 @@
#!/bin/sh
+/opt/run/bin/init_pod.sh
+
yum-config-manager --enable cta-artifacts
yum-config-manager --enable ceph
diff --git a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init.sh b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init.sh
index 62cf79b357..08c7b62f8b 100755
--- a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init.sh
+++ b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init.sh
@@ -1,5 +1,7 @@
#!/bin/sh
+/opt/run/bin/init_pod.sh
+
# enable cta repository from previously built artifacts
yum-config-manager --enable cta-artifacts
yum-config-manager --enable ceph
diff --git a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init_pod.sh b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init_pod.sh
index a42de022f7..418e26abc2 100755
--- a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init_pod.sh
+++ b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init_pod.sh
@@ -1,5 +1,23 @@
#!/bin/bash
+LOGMOUNT=/mnt/logs
+
+PV_PATH=""
+
+if [ "-${MY_CONTAINER}-" != "--" ]; then
+ PV_PATH="${LOGMOUNT}/${MY_NAME}/${MY_CONTAINER}"
+else
+ PV_PATH="${LOGMOUNT}/${MY_NAME}"
+fi
+mkdir -p ${PV_PATH}
+
+echo "Copying initial /var/log content to ${PV_PATH}"
+cd /var/log
+tar -c . | tar -C ${PV_PATH} -xv
+
+echo "Mounting logs volume ${PV_PATH} in /var/log"
+mount --bind ${PV_PATH} /var/log
+
echo -n "Fixing reverse DNS for $(hostname): "
sed -i -c "s/^\($(hostname -i)\)\s\+.*$/\1 $(hostname -s).$(grep search /etc/resolv.conf | cut -d\ -f2) $(hostname -s)/" /etc/hosts
echo "DONE"
diff --git a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/kdc.sh b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/kdc.sh
index 69710f983e..a55dc2f172 100755
--- a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/kdc.sh
+++ b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/kdc.sh
@@ -1,5 +1,7 @@
#!/bin/bash
+/opt/run/bin/init_pod.sh
+
# Install missing RPMs (kdc)
yum -y install heimdal-server heimdal-workstation
diff --git a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/rmcd.sh b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/rmcd.sh
index fd2a78d7fd..e3e9f607b5 100755
--- a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/rmcd.sh
+++ b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/rmcd.sh
@@ -1,5 +1,7 @@
#!/bin/sh
+/opt/run/bin/init_pod.sh
+
yum-config-manager --enable cta-artifacts
yum-config-manager --enable ceph
yum-config-manager --enable castor
@@ -7,8 +9,6 @@ yum-config-manager --enable castor
# Install missing RPMs
yum -y install mt-st mtx lsscsi sg3_utils cta-taped cta-debuginfo castor-rmc-server
-/opt/run/bin/init_pod.sh
-
# source library configuration file
echo "Using this configuration for library:"
/opt/run/bin/init_library.sh
diff --git a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/taped.sh b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/taped.sh
index 5424275d29..49769a7f7e 100755
--- a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/taped.sh
+++ b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/taped.sh
@@ -1,5 +1,7 @@
#!/bin/sh
+/opt/run/bin/init_pod.sh
+
yum-config-manager --enable cta-artifacts
yum-config-manager --enable ceph
yum-config-manager --enable castor
@@ -7,8 +9,6 @@ yum-config-manager --enable castor
# Install missing RPMs
yum -y install mt-st mtx lsscsi sg3_utils cta-taped cta-debuginfo castor-rmc-server ceph-common
-/opt/run/bin/init_pod.sh
-
echo "Using this configuration for library:"
/opt/run/bin/init_library.sh
cat /tmp/library-rc.sh
diff --git a/continuousintegration/orchestration/create_instance.sh b/continuousintegration/orchestration/create_instance.sh
index a3f640210e..aeb52817a1 100755
--- a/continuousintegration/orchestration/create_instance.sh
+++ b/continuousintegration/orchestration/create_instance.sh
@@ -119,6 +119,9 @@ kubectl --namespace=${instance} create -f /opt/kubernetes/CTA/library/config/lib
echo "Got library: ${LIBRARY_DEVICE}"
+echo -n "Requesting an unused log volume"
+kubectl create -f ./pvc_logs.yaml --namespace=${instance}
+
echo "Creating services in instance"
for service_file in *svc\.yaml; do
diff --git a/continuousintegration/orchestration/delete_instance.sh b/continuousintegration/orchestration/delete_instance.sh
index ed6ab8225d..dfb6d62a7f 100755
--- a/continuousintegration/orchestration/delete_instance.sh
+++ b/continuousintegration/orchestration/delete_instance.sh
@@ -37,6 +37,7 @@ echo "Collecting stdout logs of pods to ${tmpdir}"
for podcontainer in "init -c ctainit" "ctacli -c ctacli" "ctaeos -c mgm" "ctafrontend -c ctafrontend" "kdc -c kdc" "tpsrv -c taped" "tpsrv -c rmcd"; do
kubectl --namespace ${instance} logs ${podcontainer} > ${tmpdir}/$(echo ${podcontainer} | sed -e 's/ -c /-/').log
done
+kubectl --namespace ${instance} exec ctacli -- tar -C /mnt/logs -zcf - . > ${tmpdir}/varlog.tgz
if [ -z "${CI_PIPELINE_ID}" ]; then
# we are in the context of a CI run => save artifacts in the directory structure of the build
@@ -55,6 +56,8 @@ done
echo OK
+# this now useless as dummy NFS PV were replaced by local volumes
+# those are recycled automatically
./recycle_librarydevice_PV.sh
echo "Status of library pool after test:"
diff --git a/continuousintegration/orchestration/pod-ctacli.yaml b/continuousintegration/orchestration/pod-ctacli.yaml
index 3a51751a31..b0d54509fd 100644
--- a/continuousintegration/orchestration/pod-ctacli.yaml
+++ b/continuousintegration/orchestration/pod-ctacli.yaml
@@ -28,6 +28,8 @@ spec:
volumeMounts:
- mountPath: /shared
name: shared
+ - mountPath: /mnt/logs
+ name: logstorage
securityContext:
privileged: true
@@ -35,4 +37,6 @@ spec:
- name: shared
hostPath:
path: /opt/cta
-
+ - name: logstorage
+ persistentVolumeClaim:
+ claimName: claimlogs
diff --git a/continuousintegration/orchestration/pod-ctaeos.yaml b/continuousintegration/orchestration/pod-ctaeos.yaml
index a57ce017b0..3744bdebf3 100644
--- a/continuousintegration/orchestration/pod-ctaeos.yaml
+++ b/continuousintegration/orchestration/pod-ctaeos.yaml
@@ -30,6 +30,8 @@ spec:
volumeMounts:
- mountPath: /shared
name: shared
+ - mountPath: /mnt/logs
+ name: logstorage
securityContext:
privileged: true
@@ -37,3 +39,6 @@ spec:
- name: shared
hostPath:
path: /opt/cta
+ - name: logstorage
+ persistentVolumeClaim:
+ claimName: claimlogs
diff --git a/continuousintegration/orchestration/pod-ctafrontend.yaml b/continuousintegration/orchestration/pod-ctafrontend.yaml
index 616369d6bb..aedfcd3576 100644
--- a/continuousintegration/orchestration/pod-ctafrontend.yaml
+++ b/continuousintegration/orchestration/pod-ctafrontend.yaml
@@ -26,8 +26,6 @@ spec:
command: ['/opt/run/bin/ctafrontend.sh']
args: ["none"]
volumeMounts:
- - mountPath: /dev/log
- name: dev-log
- mountPath: /shared
name: shared
- mountPath: /etc/config/objectstore
@@ -36,6 +34,8 @@ spec:
name: mydatabase
- mountPath: /etc/config/library
name: mylibrary
+ - mountPath: /mnt/logs
+ name: logstorage
securityContext:
# RW access needed to volumes
privileged: true
@@ -44,9 +44,6 @@ spec:
name: ctafrontend
protocol: TCP
volumes:
- - name: dev-log
- hostPath:
- path: /dev/log
- name: shared
hostPath:
path: /opt/cta
@@ -59,3 +56,6 @@ spec:
- name: mylibrary
configMap:
name: library-config
+ - name: logstorage
+ persistentVolumeClaim:
+ claimName: claimlogs
diff --git a/continuousintegration/orchestration/pod-init.yaml b/continuousintegration/orchestration/pod-init.yaml
index 6229a341a6..36ff9ed56d 100644
--- a/continuousintegration/orchestration/pod-init.yaml
+++ b/continuousintegration/orchestration/pod-init.yaml
@@ -42,6 +42,8 @@ spec:
name: mydatabase
- mountPath: /etc/config/library
name: mylibrary
+ - mountPath: /mnt/logs
+ name: logstorage
securityContext:
privileged: true
@@ -58,3 +60,6 @@ spec:
- name: mylibrary
configMap:
name: library-config
+ - name: logstorage
+ persistentVolumeClaim:
+ claimName: claimlogs
diff --git a/continuousintegration/orchestration/pod-kdc.yaml b/continuousintegration/orchestration/pod-kdc.yaml
index b9b0d5ab9f..079e72c492 100644
--- a/continuousintegration/orchestration/pod-kdc.yaml
+++ b/continuousintegration/orchestration/pod-kdc.yaml
@@ -28,6 +28,8 @@ spec:
volumeMounts:
- mountPath: /shared
name: shared
+ - mountPath: /mnt/logs
+ name: logstorage
securityContext:
privileged: true
@@ -35,3 +37,6 @@ spec:
- name: shared
hostPath:
path: /opt/cta
+ - name: logstorage
+ persistentVolumeClaim:
+ claimName: claimlogs
diff --git a/continuousintegration/orchestration/pod-tpsrv.yaml b/continuousintegration/orchestration/pod-tpsrv.yaml
index bbd3018e12..691f69d5ab 100644
--- a/continuousintegration/orchestration/pod-tpsrv.yaml
+++ b/continuousintegration/orchestration/pod-tpsrv.yaml
@@ -11,6 +11,12 @@ spec:
image: gitlab-registry.cern.ch/cta/ctageneric:78673git921a9300
stdin: true
env:
+ - name: MY_CONTAINER
+ value: "rmcd"
+ - name: MY_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
- name: MY_NAMESPACE
valueFrom:
fieldRef:
@@ -26,6 +32,8 @@ spec:
name: shared
- mountPath: /etc/config/library
name: mylibrary
+ - mountPath: /mnt/logs
+ name: logstorage
securityContext:
privileged: true
@@ -33,6 +41,8 @@ spec:
image: gitlab-registry.cern.ch/cta/ctageneric:78673git921a9300
stdin: true
env:
+ - name: MY_CONTAINER
+ value: "taped"
- name: MY_NAME
valueFrom:
fieldRef:
@@ -58,6 +68,8 @@ spec:
name: mydatabase
- mountPath: /etc/config/library
name: mylibrary
+ - mountPath: /mnt/logs
+ name: logstorage
securityContext:
privileged: true
@@ -74,3 +86,6 @@ spec:
- name: mylibrary
configMap:
name: library-config
+ - name: logstorage
+ persistentVolumeClaim:
+ claimName: claimlogs
diff --git a/continuousintegration/orchestration/pvc_logs.yaml b/continuousintegration/orchestration/pvc_logs.yaml
new file mode 100644
index 0000000000..bdad7aa6df
--- /dev/null
+++ b/continuousintegration/orchestration/pvc_logs.yaml
@@ -0,0 +1,13 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+ name: claimlogs
+spec:
+ accessModes:
+ - ReadWriteMany
+ resources:
+ requests:
+ storage: 2Gi
+ selector:
+ matchLabels:
+ type: logs
--
GitLab