From f82448a1475fce69e3d43d152a229c888c01dd17 Mon Sep 17 00:00:00 2001
From: Cedric CAFFY <cedric.caffy@cern.ch>
Date: Tue, 28 May 2019 16:53:14 +0200
Subject: [PATCH] Added reporting of a repack request
Generated the json files of non repacked files, self repacked files and repacked files for further jq querying
Added a script repack_generate_report.sh that will be executed at the end of each repack_systemtest
---
.../orchestration/tests/repack.sh | 68 -------------
.../tests/repack_generate_report.sh | 97 +++++++++++++++++++
.../orchestration/tests/repack_systemtest.sh | 13 ++-
.../tests/repack_systemtest_wrapper.sh | 1 +
objectstore/RepackRequest.cpp | 4 +-
5 files changed, 106 insertions(+), 77 deletions(-)
delete mode 100755 continuousintegration/orchestration/tests/repack.sh
create mode 100755 continuousintegration/orchestration/tests/repack_generate_report.sh
diff --git a/continuousintegration/orchestration/tests/repack.sh b/continuousintegration/orchestration/tests/repack.sh
deleted file mode 100755
index 6336b64e8d..0000000000
--- a/continuousintegration/orchestration/tests/repack.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/bash
-
-usage() { cat <<EOF 1>&2
-Usage: $0 -n <namespace>
-EOF
-exit 1
-}
-
-while getopts "n:" o; do
- case "${o}" in
- n)
- NAMESPACE=${OPTARG}
- ;;
- *)
- usage
- ;;
- esac
-done
-shift $((OPTIND-1))
-
-if [ -z "${NAMESPACE}" ]; then
- usage
-fi
-
-if [ ! -z "${error}" ]; then
- echo -e "ERROR:\n${error}"
- exit 1
-fi
-
-echo "Preparing namespace for the tests"
-./prepare_tests.sh -n ${NAMESPACE}
-
-kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/client_helper.sh
-
-NB_FILES=1000
-FILE_SIZE_KB=15
-
-echo
-echo "Launching client_ar.sh on client pod"
-echo " Archiving ${NB_FILES} files of ${FILE_SIZE_KB}kB each"
-echo " Archiving files: xrdcp as user1"
-kubectl -n ${NAMESPACE} cp client_ar.sh client:/root/client_ar.sh
-kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v -A || exit 1
-
-#kubectl -n ${NAMESPACE} exec ctaeos -- bash /root/grep_xrdlog_mgm_for_error.sh || exit 1
-
-source ./repack_helper.sh
-
-vidToRepack1=$(getFirstVidContainingFiles)
-if [ "$vidToRepack1" != "null" ]
-then
- echo
- echo "Launching a repack on tape $vidToRepack1"
- echo
- writeTapeSummary $vidToRepack1
- executeRepack $vidToRepack1
- echo
- echo "Reclaiming tape $vidToRepack1"
- executeReclaim $vidToRepack1
- echo
- writeTapeSummary $vidToRepack1
-else
- echo "No vid found to repack"
- exit 1
-fi
-
-vidDestination=$(getFirstVidContainingFiles)
-writeTapeSummary $vidDestination
\ No newline at end of file
diff --git a/continuousintegration/orchestration/tests/repack_generate_report.sh b/continuousintegration/orchestration/tests/repack_generate_report.sh
new file mode 100755
index 0000000000..6669460b20
--- /dev/null
+++ b/continuousintegration/orchestration/tests/repack_generate_report.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+REPORT_DIRECTORY=/var/log
+
+die() {
+ echo "$@" 1>&2
+ test -z $TAILPID || kill ${TAILPID} &> /dev/null
+ exit 1
+}
+
+usage() { cat <<EOF 1>&2
+Usage: $0 -v <vid> [-r <report_directory>]
+Default report_directory = ${REPORT_DIRECTORY}
+EOF
+exit 1
+}
+
+if [ $# -lt 1 ]
+then
+ usage
+fi;
+
+while getopts "v:r:" o; do
+ case "${o}" in
+ v)
+ VID=${OPTARG}
+ ;;
+ r)
+ REPORT_DIRECTORY=${OPTARG}
+ ;;
+ *)
+ usage
+ ;;
+ esac
+done
+shift $((OPTIND-1))
+
+# get some common useful helpers for krb5
+. /root/client_helper.sh
+
+# Get kerberos credentials for user1
+admin_kinit
+admin_klist > /dev/null 2>&1 || die "Cannot get kerberos credentials for user ${USER}"
+
+echo "Generation of a repack report"
+
+DATE=`date +%d-%m-%y-%H:%M:%S`
+
+ARCHIVE_FILE_LS_RESULT_PATH=${REPORT_DIRECTORY}/${VID}_af_ls_${DATE}.json
+NOT_REPACKED_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_not_repacked_${DATE}.json
+SELF_REPACKED_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_self_repacked_${DATE}.json
+REPACKED_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_repacked_${DATE}.json
+
+
+echo "1. Generate archive file ls result into ${ARCHIVE_FILE_LS_RESULT_PATH} file..."
+admin_cta --json archivefile ls --vid ${VID} > ${ARCHIVE_FILE_LS_RESULT_PATH}
+echo "OK"
+
+echo "2. Generate the non-repacked files report into ${NOT_REPACKED_JSON_PATH} file..."
+jq -r '[.[] | select(.tf.supersededByVid == "")]' ${ARCHIVE_FILE_LS_RESULT_PATH} > ${NOT_REPACKED_JSON_PATH}
+echo "OK"
+
+echo "3. Generating the self-repacked files report into ${SELF_REPACKED_JSON_PATH} file..."
+jq -r '[.[] | select((.tf.supersededByVid == .tf.vid) and (.tf.fSeq < .tf.supersededByFSeq))]' ${ARCHIVE_FILE_LS_RESULT_PATH} > ${SELF_REPACKED_JSON_PATH}
+echo "OK"
+
+echo "4. Generate the repacked files report into ${REPACKED_JSON_PATH} file..."
+jq -r '[.[] | select((.tf.supersededByVid != "") and (.tf.supersededByVid != .tf.vid))]' ${ARCHIVE_FILE_LS_RESULT_PATH} > ${REPACKED_JSON_PATH}
+echo "OK"
+
+echo "5. Report of the repacked tape"
+echo
+NB_NON_REPACKED_FILES=$(jq '[.[]] | length' ${NOT_REPACKED_JSON_PATH})
+echo "Number of non repacked files : ${NB_NON_REPACKED_FILES}"
+if [ ${NB_NON_REPACKED_FILES} -ne 0 ]
+then
+ header="ArchiveID\tFSeq\tSize"
+ { echo -e $header; jq -r '.[] | [.af.archiveId,.tf.fSeq, .af.size] | @tsv' ${NOT_REPACKED_JSON_PATH}; } | column -t
+fi;
+echo
+NB_SELF_REPACKED_FILES=$(jq '[.[]] | length' ${SELF_REPACKED_JSON_PATH})
+echo "Number of self-repacked files : ${NB_SELF_REPACKED_FILES}"
+if [ ${NB_SELF_REPACKED_FILES} -ne 0 ]
+then
+ header="ArchiveID\tFSeq\tSize"
+ { echo -e $header; jq -r '.[] | [.af.archiveId, .tf.fSeq, .af.size] | @tsv' ${SELF_REPACKED_JSON_PATH}; } | column -t
+fi;
+echo
+NB_REPACKED_FILES=$(jq '[.[]] | length' ${REPACKED_JSON_PATH})
+echo "Number of repacked files : ${NB_REPACKED_FILES}"
+if [ ${NB_REPACKED_FILES} -ne 0 ]
+then
+ header="DestinationVID\tNbFiles\ttotalSize\n"
+ { echo -e $header; jq -r 'group_by(.tf.supersededByVid)[] | [(.[0].tf.supersededByVid),([.[] | .tf.supersededByFSeq] | length),(reduce [.[] | .af.size | tonumber][] as $currentSize (0; . + $currentSize))] | @tsv' ${REPACKED_JSON_PATH}; } | column -t
+fi;
+
+echo "End of the repack report"
\ No newline at end of file
diff --git a/continuousintegration/orchestration/tests/repack_systemtest.sh b/continuousintegration/orchestration/tests/repack_systemtest.sh
index 306ef4cf00..58818b2eda 100755
--- a/continuousintegration/orchestration/tests/repack_systemtest.sh
+++ b/continuousintegration/orchestration/tests/repack_systemtest.sh
@@ -22,12 +22,15 @@ exit 1
testRepackBufferURL(){
echo "Testing the repack buffer URL at root://${EOSINSTANCE}/${REPACK_BUFFER_BASEDIR}"
- eos root://${EOSINSTANCE} ls -d ${REPACK_BUFFER_BASEDIR} || die "Repack bufferURL directory does not exist"
+ eos root://${EOSINSTANCE} ls -d ${REPACK_BUFFER_BASEDIR} 1> /dev/null || die "Repack bufferURL directory does not exist"
echo "Testing the insertion of a test file in the buffer URL"
tempFilePath=$(mktemp /tmp/testFile.XXXX)
tempFileName=${tempFilePath##*/}
xrdcp ${tempFilePath} ${FULL_REPACK_BUFFER_URL}/${tempFileName} || die "Unable to write a file into the repack buffer directory"
- echo "OK"
+ echo "File ${tempFilePath} written in ${FULL_REPACK_BUFFER_URL}/${tempFileName}"
+ echo "Deleting test file from the test directory"
+ eos root://${EOSINSTANCE} rm ${REPACK_BUFFER_BASEDIR}/${tempFileName} || die "Unable to delete the testing file"
+ echo "Test repack buffer URL OK"
}
if [ $# -lt 2 ]
@@ -82,9 +85,6 @@ admin_cta repack rm --vid ${VID_TO_REPACK}
echo "Marking the tape ${VID_TO_REPACK} as full before Repacking it"
admin_cta tape ch --vid ${VID_TO_REPACK} --full true
-echo "State of the tape VID ${VID_TO_REPACK} BEFORE repack"
-admin_cta --json tape ls --vid ${VID_TO_REPACK} | jq .
-
echo "Launching repack request for VID ${VID_TO_REPACK}, bufferURL = ${FULL_REPACK_BUFFER_URL}"
admin_cta re add --vid ${VID_TO_REPACK} --justmove --bufferurl ${FULL_REPACK_BUFFER_URL}
@@ -104,5 +104,4 @@ if test 1 = `admin_cta repack ls --vid ${VID_TO_REPACK} | grep -E "Failed" | wc
exit 1
fi
-echo "State of the tape VID ${VID_TO_REPACK} AFTER repack"
-admin_cta --json tape ls --vid ${VID_TO_REPACK} | jq .
\ No newline at end of file
+./root/repack_generate_report.sh -v ${VID_TO_REPACK}
\ No newline at end of file
diff --git a/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh b/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh
index fe31c8ce42..dabdec972d 100755
--- a/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh
+++ b/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh
@@ -49,6 +49,7 @@ kubectl -n ${NAMESPACE} exec ctaeos -- eos chmod 1777 ${REPACK_BUFFER_URL}
source ./repack_helper.sh
kubectl -n ${NAMESPACE} cp repack_systemtest.sh client:/root/repack_systemtest.sh
+kubectl -n ${NAMESPACE} cp repack_generate_report.sh client:/root/repack_generate_report.sh
echo
echo "Launching a round trip repack request"
diff --git a/objectstore/RepackRequest.cpp b/objectstore/RepackRequest.cpp
index 25759d7211..a61e58e309 100644
--- a/objectstore/RepackRequest.cpp
+++ b/objectstore/RepackRequest.cpp
@@ -196,9 +196,9 @@ void RepackRequest::setStatus(){
return;
}
}
- //Expand is finished or not, if we have retrieved files, we are in Running,
+ //Expand is finished or not, if we have retrieved files or not (first reporting), we are in Running,
//else we are in starting
- if(m_payload.retrievedfiles()){
+ if(m_payload.retrievedfiles() || m_payload.failedtoretrievefiles()){
setStatus(common::dataStructures::RepackInfo::Status::Running);
} else {
setStatus(common::dataStructures::RepackInfo::Status::Starting);
--
GitLab