Ci - Testing of DB schema upgrade script

.gitlab/ci/kube-tests.gitlab-ci.yml

@@ -80,6 +80,24 @@ dbunittests-oracle:
     TEST_SCRIPT: "/usr/bin/true"
     EXTENDED_OPTIONS: "-O -D -t 600 -C"
 
+liquibase-update:
+  stage: test
+  extends:
+    - .kubernetes_test
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "web"
+      when: manual
+      allow_failure: true
+    - if: $CI_PIPELINE_SOURCE == "push"
+      when: manual
+      allow_failure: true
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: on_success
+      allow_failure: false
+  variables:
+    TEST_SCRIPT: "tests/update_db_test.sh"
+    EXTENDED_OPTIONS: "-O -D -t 2400 -C -u"
+
 nightly1:
   stage: test
   extends:

ReleaseNotes.md

@@ -5,6 +5,7 @@
 - cta/CTA#89 - Create stubs for Enstore tape label format
 ### Continuous Integration
 - cta/CTA#7  - Use same versionlock.list file for xrootd4 and 5
+- cta/CTA#18 - CI - Testing of DB schema upgrade script
 - cta/CTA#49 - Clean up orchestration test scripts
 ### Building and Packaging
 - cta/CTA#92 - Refactor CTA code so that it can be build without Oracle dependencies
@@ -75,7 +76,6 @@ The following manual pages have been updated:
 - cta/CTA#64       [new repo] - Adds support for the OSM Tape Label format to the CTA
 - cta/CTA-old#1278 [old repo] - Add support for reading multiple tape formats by the ReadtpCmd command
 - cta/CTA-old#1278 [old repo] - Support multiple tape formats in ReadtpCmd command
-
 ### Bug fixes
 - cta/CTA-old#947  [old repo] - cta-taped should log the FST being used for a data transfer
 - cta/CTA-old#1093 [old repo] - The `cta-taped` manpage showed outdated config options

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/dbupdatetest.sh

+#!/bin/bash
+
+# @project      The CERN Tape Archive (CTA)
+# @copyright    Copyright © 2022 CERN
+# @license      This program is free software, distributed under the terms of the GNU General Public
+#               Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can
+#               redistribute it and/or modify it under the terms of the GPL Version 3, or (at your
+#               option) any later version.
+#
+#               This program is distributed in the hope that it will be useful, but WITHOUT ANY
+#               WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+#               PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+#               In applying this licence, CERN does not waive the privileges and immunities
+#               granted to it by virtue of its status as an Intergovernmental Organization or
+#               submit itself to any jurisdiction.
+
+LOGMOUNT=/mnt/logs
+
+PV_PATH=""
+
+PV_PATH="${LOGMOUNT}/${MY_NAME}"
+mkdir -p ${PV_PATH}
+
+echo "Copying initial /var/log content to ${PV_PATH}"
+cd /var/log
+tar -c . | tar -C ${PV_PATH} -xv
+
+echo "Mounting logs volume ${PV_PATH} in /var/log"
+mount --bind ${PV_PATH} /var/log
+
+# all core dumps will go there as all the pods AND kubelet are sharing the same kernel.core_pattern
+mkdir -p /var/log/tmp
+chmod 1777 /var/log/tmp
+echo '/var/log/tmp/%h-%t-%e-%p-%s.core' > /proc/sys/kernel/core_pattern
+
+# This libraries are needed to install oracle-instant-client
+yum install --assumeyes wget libaio;
+
+# Creation of cta-catalogue.con
+/shared/scripts/init_database.sh;
+. /tmp/database-rc.sh;
+mkdir -p /shared/etc_cta;
+echo ${DATABASEURL} &> /shared/etc_cta/cta-catalogue.conf;
+
+echo $CTA_VERSION
+
+if [[ $CTA_VERSION ]]
+then
+  echo "CTA_VERSION"
+  /entrypoint.sh -d -v ${CTA_VERSION} -f ${CATALOGUE_SOURCE_VERSION} -t ${CATALOGUE_DESTINATION_VERSION} -c update;
+else
+  echo "COMMIT_ID"
+  /entrypoint.sh -d -i ${COMMIT_ID} -f ${CATALOGUE_SOURCE_VERSION} -t ${CATALOGUE_DESTINATION_VERSION} -c update;
+fi
+

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/init.sh

@@ -32,18 +32,10 @@ if [ ! -e /etc/buildtreeRunner ]; then
   yum-config-manager --enable ceph
 
   # install needed packages
-  # if we are using the postgres scheduler cta-objectstore-tools will not be available:
-  #   cta-objectstore-tools requires cta-lib, which will create /etc/cta/cta-catalogue.conf.example
-  #   the directory /etc/cta is relied on to exist below, so install cta-lib explicitly
-  yum -y install cta-objectstore-tools cta-lib mt-st mtx lsscsi sg3_utils cta-catalogueutils ceph-common oracle-instantclient19.3-sqlplus oracle-instantclient-tnsnames.ora
+  yum -y install cta-objectstore-tools mt-st mtx lsscsi sg3_utils cta-catalogueutils ceph-common oracle-instantclient19.3-sqlplus oracle-instantclient-tnsnames.ora
   yum clean packages
 fi
 
-pgsched=0
-if rpm -q --qf '%{VERSION}' cta-lib-common | grep -Eq '^[0-9]*pgs'; then
-  pgsched=1
-fi
-
 echo "Using this configuration for library:"
 /opt/run/bin/init_library.sh
 cat /tmp/library-rc.sh
@@ -53,9 +45,7 @@ echo "Configuring objectstore:"
 /opt/run/bin/init_objectstore.sh
 . /tmp/objectstore-rc.sh
 
-if [ $pgsched -ne 0 ]; then
-  echo "Detected that we are using the postgres scheduler. For now ignore the objectstore initialize step and proceed."
-elif [ "$KEEP_OBJECTSTORE" == "0" ]; then
+if [ "$KEEP_OBJECTSTORE" == "0" ]; then
   echo "Wiping objectstore"
   if [ "$OBJECTSTORETYPE" == "file" ]; then
     rm -fr $OBJECTSTOREURL
@@ -96,16 +86,16 @@ if [ "$KEEP_DATABASE" == "0" ]; then
 
   if [ "$DATABASETYPE" == "sqlite" ]; then
     mkdir -p $(dirname $(echo ${DATABASEURL} | cut -d: -f2))
-    cta-catalogue-schema-create /etc/cta/cta-catalogue.conf || die "ERROR: Could not create database schema. cta-catalogue-schema-create /etc/cta/cta-catalogue.conf FAILED"
+    cta-catalogue-schema-create -v $SCHEMA_VERSION /etc/cta/cta-catalogue.conf || die "ERROR: Could not create database schema. cta-catalogue-schema-create /etc/cta/cta-catalogue.conf FAILED"
     chmod -R 777 $(dirname $(echo ${DATABASEURL} | cut -d: -f2)) # needed?
   elif [ "$DATABASETYPE" == "oracle" ]; then
     echo "Purging Oracle recycle bin"
     test -f ${ORACLE_SQLPLUS} || echo "ERROR: ORACLE SQLPLUS client is not present, cannot purge recycle bin: ${ORACLE_SQLPLUS}"
     LD_LIBRARY_PATH=$(readlink ${ORACLE_SQLPLUS} | sed -e 's;/bin/[^/]\+;/lib;') ${ORACLE_SQLPLUS} $(echo $DATABASEURL | sed -e 's/oracle://') @/opt/ci/init/purge_database.ext
     LD_LIBRARY_PATH=$(readlink ${ORACLE_SQLPLUS} | sed -e 's;/bin/[^/]\+;/lib;') ${ORACLE_SQLPLUS} $(echo $DATABASEURL | sed -e 's/oracle://') @/opt/ci/init/purge_recyclebin.ext
-    cta-catalogue-schema-create /etc/cta/cta-catalogue.conf || die "ERROR: Could not create database schema. cta-catalogue-schema-create /etc/cta/cta-catalogue.conf FAILED"
+    cta-catalogue-schema-create -v $SCHEMA_VERSION /etc/cta/cta-catalogue.conf || die "ERROR: Could not create database schema. cta-catalogue-schema-create /etc/cta/cta-catalogue.conf FAILED"
   elif [ "$DATABASETYPE" == "postgres" ]; then
-    cta-catalogue-schema-create /etc/cta/cta-catalogue.conf || die "ERROR: Could not create database schema. cta-catalogue-schema-create /etc/cta/cta-catalogue.conf FAILED"
+    cta-catalogue-schema-create -v $SCHEMA_VERSION /etc/cta/cta-catalogue.conf || die "ERROR: Could not create database schema. cta-catalogue-schema-create /etc/cta/cta-catalogue.conf FAILED"
   else
     die "ERROR: Unsupported database type: ${DATABASETYPE}"
   fi

continuousintegration/orchestration/create_instance.sh

@@ -43,6 +43,9 @@ keepobjectstore=1
 # By default run the standard test no oracle dbunittests
 runoracleunittests=0
 
+# By default doesn't prepare the images with the previous schema version
+updatedatabasetest=0
+
 usage() { cat <<EOF 1>&2
 Usage: $0 -n <namespace> [-o <objectstore_configmap>] [-d <database_configmap>] \
       [-e <eos_configmap>] [-a <additional_k8_resources>]\
@@ -59,13 +62,14 @@ Options:
   -O	wipe objectstore content during initialization phase (objectstore content is kept by default)
   -a    additional kubernetes resources added to the kubernetes namespace
   -U    Run database unit test only
+  -u    Prepare the pods to run the liquibase test
 EOF
 exit 1
 }
 
 die() { echo "$@" 1>&2 ; exit 1; }
 
-while getopts "n:o:d:e:a:p:b:B:E:SDOUm:" o; do
+while getopts "n:o:d:e:a:p:b:B:E:SDOUum" o; do
     case "${o}" in
         o)
             config_objectstore=${OPTARG}
@@ -114,6 +118,9 @@ while getopts "n:o:d:e:a:p:b:B:E:SDOUm:" o; do
         U)
             runoracleunittests=1
             ;;
+        u)
+            updatedatabasetest=1
+            ;;
         *)
             usage
             ;;
@@ -132,6 +139,19 @@ fi
 # everyone needs poddir temporary directory to generate pod yamls
 poddir=$(mktemp -d)
 
+# Get Catalogue Schema version
+MAJOR=$(grep CTA_CATALOGUE_SCHEMA_VERSION_MAJOR ../../cmake/CTAVersions.cmake | sed 's/[^0-9]*//g')
+MINOR=$(grep CTA_CATALOGUE_SCHEMA_VERSION_MINOR ../../cmake/CTAVersions.cmake | sed 's/[^0-9]*//g')
+SCHEMA_VERSION="$MAJOR.$MINOR"
+
+# It sets as schema version the previous to the current one to create a database with that schema version
+if [ "$updatedatabasetest" == "1" ] ; then
+    MIGRATION_FILE=$(find ../../catalogue/ -name "*to${SCHEMA_VERSION}.sql")
+    PREVIOUS_SCHEMA_VERSION=$(echo $MIGRATION_FILE | grep -o -E '[0-9]+\.[0-9]' | head -1)
+    NEW_SCHEMA_VERSION=$SCHEMA_VERSION
+    SCHEMA_VERSION=$PREVIOUS_SCHEMA_VERSION
+fi
+
 if [ ! -z "${buildtree}" ]; then
     # We need to know the subdir as well
     if [ -z "${ctabuildtreesubdir}" ]; then
@@ -197,9 +217,9 @@ else
 fi
 
 if [ $keepobjectstore == 1 ] ; then
-    echo "objecstore content (if used) will be kept"
+    echo "objecstore content will be kept"
 else
-    echo "objectstore content (if used) will be wiped"
+    echo "objectstore content will be wiped"
 fi
 
 
@@ -261,7 +281,7 @@ done
 
 echo "Creating pods in instance"
 
-kubectl	create -f ${poddir}/pod-init.yaml --namespace=${instance}
+sed "s/SCHEMA_VERSION_VALUE/${SCHEMA_VERSION}/g" ${poddir}/pod-init.yaml | kubectl create --namespace=${instance} -f -
 
 echo -n "Waiting for init"
 for ((i=0; i<400; i++)); do
@@ -290,6 +310,7 @@ if [ $runoracleunittests == 1 ] ; then
     kubectl get pod oracleunittests -a --namespace=${instance} | egrep -q 'Completed|Error' && break
     sleep 1
   done
+  echo "\n"
 
   kubectl --namespace=${instance} logs oracleunittests
 
@@ -304,7 +325,6 @@ if [ $runoracleunittests == 1 ] ; then
   exit 0
 fi
 
-
 echo "Launching pods"
 
 for podname in client ctacli tpsrv01 tpsrv02 ctaeos ctafrontend kdc; do
@@ -425,16 +445,20 @@ kubectl --namespace=${instance} exec ctaeos cat /etc/eos.keytab | kubectl --name
 kubectl --namespace=${instance} exec ctaeos cat /etc/eos.keytab | kubectl --namespace=${instance} exec -i client --  bash -c "cat > /etc/eos.keytab; chmod 600 /etc/eos.keytab"
 echo OK
 
-
-echo -n "Waiting for cta-frontend to be Ready"
-for ((i=0; i<300; i++)); do
-  echo -n "."
-  kubectl --namespace=${instance} exec ctafrontend -- bash -c 'test -f /var/log/cta/cta-frontend.log && grep -q "cta-frontend started" /var/log/cta/cta-frontend.log' && break
-  sleep 1
-done
-kubectl --namespace=${instance} exec ctafrontend -- bash -c 'grep -q "cta-frontend started" /var/log/cta/cta-frontend.log' || die "TIMED OUT"
-echo OK
-
+# In case of testing to update the database using liquibase.
+# If the previous and new schema has different major version, the ctafrontend will crash,
+# so it's not necesary to check if it's ready
+NEW_MAJOR=$(echo ${SCHEMA_VERSION} | cut -d. -f1)
+if [ "${MAJOR}" == "${NEW_MAJOR}" ] ; then
+  echo -n "Waiting for cta-frontend to be Ready"
+  for ((i=0; i<300; i++)); do
+    echo -n "."
+    kubectl --namespace=${instance} exec ctafrontend -- bash -c 'test -f /var/log/cta/cta-frontend.log && grep -q "cta-frontend started" /var/log/cta/cta-frontend.log' && break
+    sleep 1
+  done
+  kubectl --namespace=${instance} exec ctafrontend -- bash -c 'grep -q "cta-frontend started" /var/log/cta/cta-frontend.log' || die "TIMED OUT"
+  echo OK
+fi
 
 echo "Instance ${instance} successfully created:"
 kubectl get pods -a --namespace=${instance}

continuousintegration/orchestration/pod-dbupdatetest.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: dbupdatetest
+  labels:
+    k8s-app: dbupdatetest
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dbupdatetest
+    image: gitlab-registry.cern.ch/cta/eoscta-operations/registry/container_registry/cta-catalogue-updater:testing
+    imagePullPolicy: IfNotPresent
+    stdin: true
+    env:
+    - name: MY_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: MY_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+    - name: INSTANCE_NAME
+      value: "$(MY_NAMESPACE)"
+    - name: CATALOGUE_SOURCE_VERSION
+      value: "CATALOGUE_SOURCE_VERSION_VALUE"
+    - name: CATALOGUE_DESTINATION_VERSION
+      value: "CATALOGUE_DESTINATION_VERSION_VALUE"
+    - name: CTA_VERSION
+      value: "CTA_VERSION_VALUE"
+    - name: COMMIT_ID
+      value: "COMMIT_ID_VALUE"
+    - name: TERM
+      value: "xterm"
+    command: ["/shared/scripts/dbupdatetest.sh"]
+    args: ["none"]
+    volumeMounts:
+    - mountPath: /etc/config/database
+      name: mydatabase
+    - mountPath: /shared/etc_yum.repos.d/
+      name: repos
+    - mountPath: /shared/scripts
+      name: scripts
+    - mountPath: /mnt/logs
+      name: logstorage
+    securityContext:
+      privileged: true
+
+  volumes:
+  - name: repos
+    hostPath:
+      path: REPOS_PATH
+  - name: scripts
+    hostPath:
+      path: SCRIPTS_PATH
+  - name: mydatabase
+    configMap:
+      name: database-config
+  - name: logstorage
+    persistentVolumeClaim:
+      claimName: claimlogs
+
+  imagePullSecrets:
+  - name: ctaregsecret

continuousintegration/orchestration/pod-init.yaml

@@ -41,6 +41,8 @@ spec:
         configMapKeyRef:
           name: buildtree
           key: cta_subdir
+    - name: SCHEMA_VERSION
+      value: "SCHEMA_VERSION_VALUE"
     command: ['/opt/run/bin/init.sh']
     args: ["none"]
     volumeMounts:

continuousintegration/orchestration/run_systemtest.sh

@@ -60,6 +60,7 @@ Options:
   -a    additional kubernetes resources added to the kubernetes namespace
   -U    Run database unit test only
   -C    Cleanup leftover kubernetes namespaces
+  -u    Prepare the pods to run the liquibase test
 
 Create a kubernetes instance and launch the system test script specified.
 Makes sure the created instance is cleaned up at the end and return the status of the system test.
@@ -72,7 +73,7 @@ exit 1
 # always delete DB and OBJECTSTORE for tests
 CREATE_OPTS="-D -O"
 
-while getopts "n:d:s:p:b:e:a:B:t:kDOSUC" o; do
+while getopts "n:d:s:p:b:e:a:B:t:ukDOSUC" o; do
     case "${o}" in
         s)
             systemtest_script=${OPTARG}
@@ -122,6 +123,9 @@ while getopts "n:d:s:p:b:e:a:B:t:kDOSUC" o; do
         C)
             cleanup_namespaces=1
             ;;
+        u)
+            CREATE_OPTS="${CREATE_OPTS} -u"
+            ;;
         *)
             usage
             ;;

continuousintegration/orchestration/tests/update_db_test.sh

+#!/bin/bash
+
+# @project      The CERN Tape Archive (CTA)
+# @copyright    Copyright © 2022 CERN
+# @license      This program is free software, distributed under the terms of the GNU General Public
+#               Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can
+#               redistribute it and/or modify it under the terms of the GPL Version 3, or (at your
+#               option) any later version.
+#
+#               This program is distributed in the hope that it will be useful, but WITHOUT ANY
+#               WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+#               PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+#               In applying this licence, CERN does not waive the privileges and immunities
+#               granted to it by virtue of its status as an Intergovernmental Organization or
+#               submit itself to any jurisdiction.
+
+
+usage() { cat <<EOF 1>&2
+Usage: $0 -n <namespace>
+EOF
+exit 1
+}
+
+CTA_VERSION=""
+
+while getopts "n:v:" o; do
+  case "${o}" in
+    n)
+      NAMESPACE=${OPTARG}
+      ;;
+    v)
+      CTA_VERSION=${OPTARG}
+      ;;
+    *)
+      usage
+      ;;
+  esac
+done
+shift $((OPTIND-1))
+
+if [ -z "${NAMESPACE}" ]; then
+  usage
+fi
+
+if [ ! -z "${error}" ]; then
+  echo -e "ERROR:\n${error}"
+  exit 1
+fi
+
+# Get Catalogue Schema version
+MAJOR=$(grep CTA_CATALOGUE_SCHEMA_VERSION_MAJOR ../../../cmake/CTAVersions.cmake | sed 's/[^0-9]*//g')
+MINOR=$(grep CTA_CATALOGUE_SCHEMA_VERSION_MINOR ../../../cmake/CTAVersions.cmake | sed 's/[^0-9]*//g')
+NEW_SCHEMA_VERSION="$MAJOR.$MINOR"
+MIGRATION_FILE=$(find ../../../catalogue/ -name "*to${NEW_SCHEMA_VERSION}.sql")
+PREVIOUS_SCHEMA_VERSION=$(echo $MIGRATION_FILE | grep -o -E '[0-9]+\.[0-9]' | head -1)
+
+YUM_REPOS="$(pwd)/$(find ../../ -name "yum.repos.d")"
+SCRIPTS_DIR="$(pwd)/$(find ../../ -name "dbupdatetest.sh" | xargs dirname)"
+
+# Modify fields of pod yaml with the current data
+tempdir=$(mktemp -d)
+cp ../pod-dbupdatetest.yaml ${tempdir}
+sed -i "s#REPOS_PATH#${YUM_REPOS}#g" ${tempdir}/pod-dbupdatetest.yaml
+sed -i "s#SCRIPTS_PATH#${SCRIPTS_DIR}#g" ${tempdir}/pod-dbupdatetest.yaml
+sed -i "s/CATALOGUE_SOURCE_VERSION_VALUE/${PREVIOUS_SCHEMA_VERSION}/g" ${tempdir}/pod-dbupdatetest.yaml
+sed -i "s/CATALOGUE_DESTINATION_VERSION_VALUE/${NEW_SCHEMA_VERSION}/g" ${tempdir}/pod-dbupdatetest.yaml
+
+COMMITID=$(git log -n1 | grep ^commit | cut -d\  -f2 | sed -e 's/\(........\).*/\1/')
+sed -i "s/COMMIT_ID_VALUE/${COMMITID}/g" ${tempdir}/pod-dbupdatetest.yaml
+sed -i "s/CTA_VERSION_VALUE/${CTA_VERSION}/g" ${tempdir}/pod-dbupdatetest.yaml
+
+# Check if the current schema version is the same as the previous one
+CURRENT_SCHEMA_VERSION=$(kubectl -n ${NAMESPACE} exec ctafrontend -- cta-catalogue-schema-verify /etc/cta/cta-catalogue.conf \
+  | grep -o -E '[0-9]+\.[0-9]')
+
+if [ ${CURRENT_SCHEMA_VERSION} ==  ${PREVIOUS_SCHEMA_VERSION} ]; then
+  echo "The current Catalogue Schema Version is: ${CURRENT_SCHEMA_VERSION}"
+else
+  echo "Error. Unexpected Catalogue Schema Version: ${CURRENT_SCHEMA_VERSION}, it should be: ${PREVIOUS_SCHEMA_VERSION}"
+  exit 1
+fi
+
+kubectl create -f ${tempdir}/pod-dbupdatetest.yaml --namespace=${NAMESPACE}
+
+echo -n "Waiting for dbupdatetest"
+for ((i=0; i<400; i++)); do
+  echo -n "."
+  kubectl get pod dbupdatetest -a --namespace=${NAMESPACE} | egrep -q 'Completed|Error' && break
+  sleep 1
+done
+echo "\n"
+
+# Check if the current schema version is the same as the new one
+CURRENT_SCHEMA_VERSION=$(kubectl -n ${NAMESPACE} exec ctafrontend -- cta-catalogue-schema-verify /etc/cta/cta-catalogue.conf \
+  | grep -o -E '[0-9]+\.[0-9]')
+if [ ${CURRENT_SCHEMA_VERSION} ==  ${NEW_SCHEMA_VERSION} ]; then
+  echo "The current Catalogue Schema Version is: ${CURRENT_SCHEMA_VERSION}"
+else
+  echo "Error. Unexpected Catalogue Schema Version: ${CURRENT_SCHEMA_VERSION}, it should be: ${NEW_SCHEMA_VERSION}"
+  exit 1
+fi
+
+# If the previous and new schema has same major version, we can run a simple archive-retrieve test
+PREVIOUS_MAJOR=$(echo ${PREVIOUS_SCHEMA_VERSION} | cut -d. -f1)
+if [ "${MAJOR}" == "${PREVIOUS_MAJOR}" ] ; then
+  echo
+  echo "Running a simple archive-retrieve test to check if the update is working"
+  echo "Preparing namespace for the tests"
+  ./prepare_tests.sh -n ${NAMESPACE}
+  if [ $? -ne 0 ]; then
+    echo "ERROR: failed to prepare namespace for the tests"
+    exit 1
+  fi
+
+  echo
+  echo "Launching simple_client_ar.sh on client pod"
+  echo " Archiving file: xrdcp as user1"
+  echo " Retrieving it as poweruser1"
+  kubectl -n ${NAMESPACE} cp simple_client_ar.sh client:/root/simple_client_ar.sh
+  kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/client_helper.sh
+  kubectl -n ${NAMESPACE} exec client -- bash /root/simple_client_ar.sh || exit 1
+
+  kubectl -n ${NAMESPACE} cp grep_xrdlog_mgm_for_error.sh ctaeos:/root/grep_xrdlog_mgm_for_error.sh
+  kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh || exit 1
+fi
+
+exit 0