Honoured the "MoveAndAddCopies" repack workflow

Changed repack CI test to test the Add Copies workflow (need reporting though)

continuousintegration/orchestration/tests/repack_systemtest.sh

@@ -4,6 +4,8 @@
 EOSINSTANCE=ctaeos
 #default Repack timeout
 WAIT_FOR_REPACK_TIMEOUT=300
+# default generate report = true
+GENERATE_REPORT=1
 
 die() {
   echo "$@" 1>&2
@@ -12,10 +14,13 @@ die() {
 }
 
 usage() { cat <<EOF 1>&2
-Usage: $0 -v <vid> -b <bufferURL> [-e <eosinstance>] [-t <timeout>]
+Usage: $0 -v <vid> -b <bufferURL> [-e <eosinstance>] [-t <timeout>] [-a] [m]
 (bufferURL example : /eos/ctaeos/repack)
 eosinstance : the name of the ctaeos instance to be used (default ctaeos)
 timeout : the timeout in seconds to wait for the repack to be done
+-a : Launch a repack just add copies workflow
+-m : Launch a repack just move workflow
+-g : Avoid generating repack report at the end of repack
 EOF
 exit 1
 }
@@ -38,7 +43,7 @@ then
   usage
 fi;
 
-while getopts "v:e:b:t:" o; do
+while getopts "v:e:b:t:amg" o; do
   case "${o}" in
     v)
       VID_TO_REPACK=${OPTARG}
@@ -52,6 +57,15 @@ while getopts "v:e:b:t:" o; do
     t)
       WAIT_FOR_REPACK_TIMEOUT=${OPTARG}
       ;;
+    a)
+      ADD_COPIES_ONLY="-a"
+      ;;
+    m)
+      MOVE_ONLY="-m"
+      ;;
+    g)
+      GENERATE_REPORT=0
+      ;;
     *)
       usage
       ;;
@@ -69,6 +83,14 @@ if [ "x${VID_TO_REPACK}" = "x" ]; then
   die "No vid to repack provided."
 fi
 
+REPACK_OPTION=""
+
+if [ "x${ADD_COPIES_ONLY}" != "x" ] && [ "x${MOVE_ONLY}" != "x" ]; then
+  die "-a and -m options are mutually exclusive"
+fi
+
+[[ "x${ADD_COPIES_ONLY}" == "x" ]] && REPACK_OPTION=${MOVE_ONLY} || REPACK_OPTION=${ADD_COPIES_ONLY}
+
 # get some common useful helpers for krb5
 . /root/client_helper.sh
 
@@ -86,7 +108,7 @@ echo "Marking the tape ${VID_TO_REPACK} as full before Repacking it"
 admin_cta tape ch --vid ${VID_TO_REPACK} --full true
 
 echo "Launching repack request for VID ${VID_TO_REPACK}, bufferURL = ${FULL_REPACK_BUFFER_URL}"
-admin_cta re add --vid ${VID_TO_REPACK} --justmove --bufferurl ${FULL_REPACK_BUFFER_URL}
+admin_cta re add --vid ${VID_TO_REPACK} ${REPACK_OPTION} --bufferurl ${FULL_REPACK_BUFFER_URL}
 
 SECONDS_PASSED=0
 while test 0 = `admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r '.[0] | select(.status == "Complete" or .status == "Failed")' | wc -l`; do
@@ -104,4 +126,6 @@ if test 1 = `admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r '.[0] | s
     exit 1
 fi
 
-exec /root/repack_generate_report.sh -v ${VID_TO_REPACK}
\ No newline at end of file
+echo "Repack request on VID ${VID_TO_REPACK} succeeded."
+
+[[ $GENERATE_REPORT == 1 ]] && exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} || exit 0
\ No newline at end of file

continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh

@@ -52,14 +52,14 @@ kubectl -n ${NAMESPACE} cp repack_systemtest.sh client:/root/repack_systemtest.s
 kubectl -n ${NAMESPACE} cp repack_generate_report.sh client:/root/repack_generate_report.sh
 
 echo
-echo "Launching a round trip repack request"
+echo "Launching a round trip repack \"just move\" request"
 
 VID_TO_REPACK=$(getFirstVidContainingFiles)
 if [ "$VID_TO_REPACK" != "null" ] 
 then
 echo
-  echo "Launching the repack test on VID ${VID_TO_REPACK}"
-  kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} || exit 1
+  echo "Launching the repack \"just move\" test on VID ${VID_TO_REPACK}"
+  kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m || exit 1
 else
   echo "No vid found to repack"
   exit 1
@@ -72,8 +72,8 @@ VID_TO_REPACK=$(getFirstVidContainingFiles)
 if [ "$VID_TO_REPACK" != "null" ] 
 then
 echo
-  echo "Launching the repack test on VID ${VID_TO_REPACK}"
-  kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} || exit 1
+  echo "Launching the repack \"just move\" test on VID ${VID_TO_REPACK}"
+  kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m || exit 1
 else
   echo "No vid found to repack"
   exit 1
@@ -82,16 +82,104 @@ fi
 echo "Reclaiming tape ${VID_TO_REPACK}"
 kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}
 
-NB_FILES=1153
+NB_FILES=1152
 kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v -A || exit 1
 
 VID_TO_REPACK=$(getFirstVidContainingFiles)
 if [ "$VID_TO_REPACK" != "null" ] 
 then
 echo
-  echo "Launching the repack test on VID ${VID_TO_REPACK}"
-  kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} || exit 1
+  echo "Launching the repack test \"just move\" on VID ${VID_TO_REPACK}"
+  kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m || exit 1
 else
   echo "No vid found to repack"
   exit 1
 fi
+
+echo "Reclaiming tape ${VID_TO_REPACK}"
+kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}
+
+VID_TO_REPACK=$(getFirstVidContainingFiles)
+if [ "$VID_TO_REPACK" != "null" ] 
+then
+  echo "Launching the repack \"just add copies\" test on VID ${VID_TO_REPACK} with all copies already on CTA"
+  kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -a -g || exit 1
+else
+  echo "No vid found to repack"
+  exit 1
+fi
+
+repackJustAddCopiesResult=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json re ls | jq -r ". [] | select (.vid == \"${VID_TO_REPACK}\")"`
+
+nbRetrievedFiles=`echo ${repackJustAddCopiesResult} | jq -r ".retrievedFiles"`
+nbArchivedFiles=`echo ${repackJustAddCopiesResult} | jq -r ".archivedFiles"`
+
+if [ $nbArchivedFiles == 0 ] && [ $nbRetrievedFiles == 0 ] 
+then
+  echo "Nb retrieved files = 0 and nb archived files = 0. Test OK"
+else
+  echo "Repack \"just add copies\" on VID ${VID_TO_REPACK} failed : nbRetrievedFiles = $nbRetrievedFiles, nbArchivedFiles = $nbArchivedFiles"
+  exit 1
+fi
+
+tapepoolDestination1="ctasystest2"
+tapepoolDestination2="ctasystest3"
+
+echo "Creating two destination tapepool : $tapepoolDestination1 and $tapepoolDestination2"
+kubectl -n cta exec ctacli -- cta-admin tapepool add --name $tapepoolDestination1 --vo vo --partialtapesnumber 2 --encrypted false --comment "$tapepoolDestination1 tapepool"
+kubectl -n cta exec ctacli -- cta-admin tapepool add --name $tapepoolDestination2 --vo vo --partialtapesnumber 2 --encrypted false --comment "$tapepoolDestination2 tapepool"
+echo "OK"
+
+echo "Creating archive routes for adding two copies of the file"
+kubectl -n cta exec ctacli -- cta-admin archiveroute add --instance ctaeos --storageclass ctaStorageClass --copynb 2 --tapepool $tapepoolDestination1 --comment "ArchiveRoute2"
+kubectl -n cta exec ctacli -- cta-admin archiveroute add --instance ctaeos --storageclass ctaStorageClass --copynb 3 --tapepool $tapepoolDestination2 --comment "ArchiveRoute3"
+echo "OK"
+
+echo "Will change the tapepool of the tapes"
+
+allVID=`kubectl -n cta  exec ctacli -- cta-admin --json tape ls --all | jq -r ". [] | .vid"`
+read -a allVIDTable <<< $allVID
+
+nbVid=${#allVIDTable[@]}
+
+allTapepool=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tapepool ls | jq -r ". [] .name"`
+
+read -a allTapepoolTable <<< $allTapepool
+
+nbTapepool=${#allTapepoolTable[@]}
+
+nbTapePerTapepool=$(($nbVid / $nbTapepool))
+
+allTapepool=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tapepool ls | jq -r ". [] .name"`
+read -a allTapepoolTable <<< $allTapepool
+
+
+countChanging=0
+tapepoolIndice=1 #We only change the vid of the remaining other tapes
+
+for ((i=$(($nbTapePerTapepool+$(($nbVid%$nbTapepool)))); i<$nbVid; i++));
+do
+  echo "kubectl -n cta exec ctacli -- cta-admin tape ch --vid ${allVIDTable[$i]} --tapepool ${allTapepoolTable[$tapepoolIndice]}"
+  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape ch --vid ${allVIDTable[$i]} --tapepool ${allTapepoolTable[$tapepoolIndice]}
+  countChanging=$((countChanging + 1))
+  if [ $countChanging != 0 ] && [ $((countChanging % nbTapePerTapepool)) == 0 ]
+  then
+    tapepoolIndice=$((tapepoolIndice + 1))
+  fi
+done
+
+echo "OK"
+
+storageClassName=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json storageclass ls | jq -r ". [0] | .name"`
+instanceName=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json storageclass ls | jq -r ". [0] | .diskInstance"`
+
+echo "Changing the storage class $storageClassName nb copies"
+kubectl -n ${NAMESPACE} exec ctacli -- cta-admin storageclass ch --instance $instanceName --name $storageClassName --copynb 3
+echo "OK"
+
+echo "Putting all drives up"
+kubectl -n ${NAMESPACE} exec ctacli -- cta-admin dr up VD.*
+echo "OK"
+
+echo "Launching the repack \"just add copies\" test on VID ${VID_TO_REPACK}"
+kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -a -g || exit 1