diff --git a/continuousintegration/orchestration/tests/repack_generate_report.sh b/continuousintegration/orchestration/tests/repack_generate_report.sh index b21dd1bf5d0e6cef29249e5e992ccb15820c83c1..ad537eff7815828a8ff253449d9a48c9248cd03a 100755 --- a/continuousintegration/orchestration/tests/repack_generate_report.sh +++ b/continuousintegration/orchestration/tests/repack_generate_report.sh @@ -54,6 +54,9 @@ then die "No repack request for this VID." fi; +echo "Creating the report directory ${REPORT_DIRECTORY}" +mkdir -p ${REPORT_DIRECTORY} || die "Unable to create the directory for report generation" + echo "Generation of a repack report of the vid ${VID}" DATE=`date +%d-%m-%y-%H:%M:%S` @@ -64,6 +67,7 @@ NOT_REPACKED_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_not_repacked_${DATE}.js SELF_REPACKED_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_self_repacked_${DATE}.json REPACKED_MOVE_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_repacked_move_${DATE}.json REPACK_ADD_COPIES_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_repack_add_copies_${DATE}.json +STDOUT_REPORT_PATH=${REPORT_DIRECTORY}/${VID}_stdout_report.txt echo "1. Generate archive file ls result into ${ARCHIVE_FILE_LS_RESULT_PATH} file..." @@ -104,7 +108,7 @@ then if [ ${NB_NON_REPACKED_FILES} -ne 0 ] then header="ArchiveID\tFSeq\tSize" - { echo -e $header; jq -r '.[] | [.af.archiveId,.tf.fSeq, .af.size] | @tsv' ${NOT_REPACKED_JSON_PATH}; } | column -t + { echo -e $header; jq -r '.[] | [.af.archiveId,.tf.fSeq, .af.size] | @tsv' ${NOT_REPACKED_JSON_PATH}; } | column -t | tee --append ${STDOUT_REPORT_PATH} fi; fi; @@ -114,7 +118,7 @@ echo "Number of self-repacked files : ${NB_SELF_REPACKED_FILES}" if [ ${NB_SELF_REPACKED_FILES} -ne 0 ] then header="ArchiveID\tFSeq\tSize" - { echo -e $header; jq -r '.[] | [.af.archiveId, .tf.fSeq, .af.size] | @tsv' ${SELF_REPACKED_JSON_PATH}; } | column -t + { echo -e $header; jq -r '.[] | [.af.archiveId, .tf.fSeq, .af.size] | @tsv' ${SELF_REPACKED_JSON_PATH}; } | column -t | tee --append ${STDOUT_REPORT_PATH} fi; echo @@ -123,7 +127,7 @@ echo "Number of repacked (moved) files : ${NB_REPACKED_MOVE_FILES}" if [ ${NB_REPACKED_MOVE_FILES} -ne 0 ] then header="DestinationVID\tNbFiles\ttotalSize\n" - { echo -e $header; jq -r 'group_by(.tf.supersededByVid)[] | [(.[0].tf.supersededByVid),([.[] | .tf.supersededByFSeq] | length),(reduce [.[] | .af.size | tonumber][] as $currentSize (0; . + $currentSize))] | @tsv' ${REPACKED_MOVE_JSON_PATH}; } | column -t + { echo -e $header; jq -r 'group_by(.tf.supersededByVid)[] | [(.[0].tf.supersededByVid),([.[] | .tf.supersededByFSeq] | length),(reduce [.[] | .af.size | tonumber][] as $currentSize (0; . + $currentSize))] | @tsv' ${REPACKED_MOVE_JSON_PATH}; } | column -t | tee --append ${STDOUT_REPORT_PATH} fi; echo @@ -132,7 +136,8 @@ echo "Number of copied files : $NB_COPIED_FILES" if [ ${NB_COPIED_FILES} -ne 0 ] then header="DestinationVID\tNbFiles\ttotalSize\n" - { echo -e $header; jq -r 'group_by(.tf.vid)[] | [(.[0].tf.vid),([.[] | .tf.fSeq] | length),(reduce [.[] | .af.size | tonumber][] as $currentSize (0; . + $currentSize))] | @tsv' ${REPACK_ADD_COPIES_JSON_PATH}; } | column -t + { echo -e $header; jq -r 'group_by(.tf.vid)[] | [(.[0].tf.vid),([.[] | .tf.fSeq] | length),(reduce [.[] | .af.size | tonumber][] as $currentSize (0; . + $currentSize))] | @tsv' ${REPACK_ADD_COPIES_JSON_PATH}; } | column -t | tee --append ${STDOUT_REPORT_PATH} fi; -echo "End of the repack report" \ No newline at end of file +echo "End of the repack report" +exit 0 \ No newline at end of file diff --git a/continuousintegration/orchestration/tests/repack_systemtest.sh b/continuousintegration/orchestration/tests/repack_systemtest.sh index e2b675bfa44dcdcde1e020a865797f1cbbf1a07e..b206f30c7617cbafa1c47ecc290df22e90896dff 100755 --- a/continuousintegration/orchestration/tests/repack_systemtest.sh +++ b/continuousintegration/orchestration/tests/repack_systemtest.sh @@ -5,6 +5,8 @@ EOSINSTANCE=ctaeos #default Repack timeout WAIT_FOR_REPACK_TIMEOUT=300 +REPORT_DIRECTORY=/var/log + die() { echo "$@" 1>&2 test -z $TAILPID || kill ${TAILPID} &> /dev/null @@ -12,10 +14,11 @@ die() { } usage() { cat <<EOF 1>&2 -Usage: $0 -v <vid> -b <bufferURL> [-e <eosinstance>] [-t <timeout>] [-a] [-m] [-d] +Usage: $0 -v <vid> -b <bufferURL> [-e <eosinstance>] [-t <timeout>] [-r <reportDirectory>] [-a] [-m] [-d] (bufferURL example : /eos/ctaeos/repack) -eosinstance : the name of the ctaeos instance to be used (default ctaeos) +eosinstance : the name of the ctaeos instance to be used (default : $EOSINSTANCE) timeout : the timeout in seconds to wait for the repack to be done +reportDirectory : the directory to generate the report of the repack test (default : $REPORT_DIRECTORY) -a : Launch a repack just add copies workflow -m : Launch a repack just move workflow -d : Force a repack on a disabled tape (adds --disabled to the repack add command) @@ -42,7 +45,7 @@ then fi; DISABLED_TAPE_FLAG="" -while getopts "v:e:b:t:amgd" o; do +while getopts "v:e:b:t:r:amd" o; do case "${o}" in v) VID_TO_REPACK=${OPTARG} @@ -62,8 +65,8 @@ while getopts "v:e:b:t:amgd" o; do m) MOVE_ONLY="-m" ;; - g) - GENERATE_REPORT=0 + r) + REPORT_DIRECTORY=${OPTARG} ;; d) DISABLED_TAPE_FLAG="--disabledtape" @@ -121,14 +124,20 @@ while test 0 = `admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r '.[0] if test ${SECONDS_PASSED} == ${WAIT_FOR_REPACK_TIMEOUT}; then echo "Timed out after ${WAIT_FOR_REPACK_TIMEOUT} seconds waiting for tape ${VID_TO_REPACK} to be repacked" + exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} -r ${REPORT_DIRECTORY} ${ADD_COPIES_ONLY} & + wait $! exit 1 fi done if test 1 = `admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r '[.[0] | select (.status == "Failed")] | length'`; then echo "Repack failed for tape ${VID_TO_REPACK}." + exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} -r ${REPORT_DIRECTORY} ${ADD_COPIES_ONLY} & + wait $! exit 1 fi echo "Repack request on VID ${VID_TO_REPACK} succeeded." -exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} ${ADD_COPIES_ONLY} || exit 0 \ No newline at end of file +exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} -r ${REPORT_DIRECTORY} ${ADD_COPIES_ONLY} & +wait $! +exit 0 \ No newline at end of file diff --git a/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh b/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh index 97c8415c600db65ee1ae45431d9f2b538cf5be8b..3c939d7d5f4221444b89bf8e11d9972709659a7b 100755 --- a/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh +++ b/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh @@ -1,5 +1,7 @@ #!/bin/bash +BASE_REPORT_DIRECTORY=/var/log + usage() { cat <<EOF 1>&2 Usage: $0 -n <namespace> EOF @@ -52,14 +54,16 @@ kubectl -n ${NAMESPACE} cp repack_systemtest.sh client:/root/repack_systemtest.s kubectl -n ${NAMESPACE} cp repack_generate_report.sh client:/root/repack_generate_report.sh echo -echo "Launching a round trip repack \"just move\" request" +echo "***********************************************************" +echo "STEP 1. Launching a round trip repack \"just move\" request" +echo "***********************************************************" VID_TO_REPACK=$(getFirstVidContainingFiles) if [ "$VID_TO_REPACK" != "null" ] then echo echo "Launching the repack \"just move\" test on VID ${VID_TO_REPACK}" - kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m || exit 1 + kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step1-RoundTripRepack || exit 1 else echo "No vid found to repack" exit 1 @@ -73,7 +77,7 @@ if [ "$VID_TO_REPACK" != "null" ] then echo echo "Launching the repack \"just move\" test on VID ${VID_TO_REPACK}" - kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m || exit 1 + kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step1-RoundTripRepack || exit 1 else echo "No vid found to repack" exit 1 @@ -81,8 +85,15 @@ fi echo "Reclaiming tape ${VID_TO_REPACK}" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK} +echo +echo "*******************************************************************" +echo "STEP 1. Launching a round trip repack \"just move\" request TEST OK" +echo "*******************************************************************" +echo +echo "*****************************************************" +echo "STEP 2. Launching a Repack Request on a disabled tape" +echo "*****************************************************" -echo "Launching a Repack Request on a disabled tape" VID_TO_REPACK=$(getFirstVidContainingFiles) if [ "$VID_TO_REPACK" != "null" ] @@ -92,7 +103,7 @@ then echo "Waiting 20 seconds so that the RetrieveQueueStatisticsCache is flushed" sleep 20 echo "Launching the repack request test on VID ${VID_TO_REPACK}" - kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} && echo "The repack request is Complete instead of Failed, it should be failed as the tape is disabled" && exit 1 || echo "Test OK" + kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -r ${BASE_REPORT_DIRECTORY}/Step2-RepackDisabledTape && echo "The repack request is Complete instead of Failed, it should be failed as the tape is disabled" && exit 1 || echo "REPACK FAILED, the tape is disabled so, Test OK" else echo "No vid found to repack" exit 1 @@ -100,11 +111,20 @@ fi; echo echo "Launching the repack request test on VID ${VID_TO_REPACK} with the --disabledtape flag" -kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -d || exit 1 +kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -d -r ${BASE_REPORT_DIRECTORY}/Step2-RepackDisabledTape || exit 1 echo "Reclaiming tape ${VID_TO_REPACK}" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK} +echo +echo "*************************************************************" +echo "STEP 2. Launching a Repack Request on a disabled tape TEST OK" +echo "*************************************************************" +echo +echo "*********************************************" +echo "STEP 3. Testing Repack \"Just move\" workflow" +echo "*********************************************" + NB_FILES=1152 kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v -A || exit 1 @@ -113,7 +133,7 @@ if [ "$VID_TO_REPACK" != "null" ] then echo echo "Launching the repack test \"just move\" on VID ${VID_TO_REPACK}" - kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m || exit 1 + kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step3-RepackJustMove || exit 1 else echo "No vid found to repack" exit 1 @@ -121,12 +141,20 @@ fi echo "Reclaiming tape ${VID_TO_REPACK}" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK} +echo +echo "*****************************************************" +echo "STEP 3. Testing Repack \"Just move\" workflow TEST OK" +echo "*****************************************************" +echo +echo "**************************************************************************" +echo "STEP 4. Testing Repack \"Just Add copies\" workflow with all copies on CTA" +echo "**************************************************************************" VID_TO_REPACK=$(getFirstVidContainingFiles) if [ "$VID_TO_REPACK" != "null" ] then echo "Launching the repack \"just add copies\" test on VID ${VID_TO_REPACK} with all copies already on CTA" - kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -a -g || exit 1 + kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -a -r ${BASE_REPORT_DIRECTORY}/Step4-JustAddCopiesAllCopiesInCTA || exit 1 else echo "No vid found to repack" exit 1 @@ -145,6 +173,16 @@ else exit 1 fi +echo +echo "**********************************************************************************" +echo "STEP 4. Testing Repack \"Just Add copies\" workflow with all copies on CTA TEST OK" +echo "**********************************************************************************" + +echo +echo "*******************************************************" +echo "STEP 5. Testing Repack \"Move and Add copies\" workflow" +echo "*******************************************************" + tapepoolDestination1="ctasystest2" tapepoolDestination2="ctasystest3" @@ -205,4 +243,8 @@ kubectl -n ${NAMESPACE} exec ctacli -- cta-admin dr up VD.* echo "OK" echo "Launching the repack \"Move and add copies\" test on VID ${VID_TO_REPACK}" -kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -t 500 || exit 1 +kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -t 600 -r ${BASE_REPORT_DIRECTORY}/Step5-MoveAndAddCopies || exit 1 +echo +echo "***************************************************************" +echo "STEP 5. Testing Repack \"Move and Add copies\" workflow TEST OK" +echo "***************************************************************" diff --git a/scheduler/RetrieveJob.hpp b/scheduler/RetrieveJob.hpp index d85f48bdc13371370cdfe683074a3a70a656e23f..7bb4057195f428fbf6fbfe53e691b4c959b5d34d 100644 --- a/scheduler/RetrieveJob.hpp +++ b/scheduler/RetrieveJob.hpp @@ -27,6 +27,13 @@ #include <limits> #include <memory> +namespace castor { +namespace tape { +namespace tapeserver { +namespace daemon { + class TapeReadTask; +}}}} + namespace cta { class RetrieveMount; @@ -42,7 +49,7 @@ class RetrieveJob { */ friend class RetrieveMount; friend class Scheduler; - + friend class castor::tape::tapeserver::daemon::TapeReadTask; public: /** * Constructor. It is not public as it is generated by the RetrieveMount. diff --git a/scheduler/RetrieveMount.hpp b/scheduler/RetrieveMount.hpp index c2749f44895a9938116cdbfdbecd995989a70d5c..b8a5496777370801ea3040e57e8cf7616869f430 100644 --- a/scheduler/RetrieveMount.hpp +++ b/scheduler/RetrieveMount.hpp @@ -233,7 +233,7 @@ namespace cta { * A pointer to the file catalogue. */ cta::catalogue::Catalogue &m_catalogue; - + }; // class RetrieveMount } // namespace cta diff --git a/tapeserver/castor/tape/tapeserver/daemon/TapeReadTask.hpp b/tapeserver/castor/tape/tapeserver/daemon/TapeReadTask.hpp index 23a830da4b55ec4291b9dd899e99083a1f41bdd1..861bbe2cf6b34a080018163211084b8cb41d87ca 100644 --- a/tapeserver/castor/tape/tapeserver/daemon/TapeReadTask.hpp +++ b/tapeserver/castor/tape/tapeserver/daemon/TapeReadTask.hpp @@ -66,12 +66,14 @@ public: using cta::log::Param; + bool isRepack = m_retrieveJob->m_dbJob->isRepack; // Set the common context for all the coming logs (file info) cta::log::ScopedParamContainer params(lc); params.add("fileId", m_retrieveJob->archiveFile.archiveFileID) .add("BlockId", m_retrieveJob->selectedTapeFile().blockId) .add("fSeq", m_retrieveJob->selectedTapeFile().fSeq) - .add("dstURL", m_retrieveJob->retrieveRequest.dstURL); + .add("dstURL", m_retrieveJob->retrieveRequest.dstURL) + .add("isRepack",isRepack); // We will clock the stats for the file itself, and eventually add those // stats to the session's. @@ -129,6 +131,11 @@ public: localStats.readWriteTime += timer.secs(cta::utils::Timer::resetCounter); auto blockSize = mb->m_payload.size(); localStats.dataVolume += blockSize; + if(isRepack){ + localStats.repackBytesCount += blockSize; + } else { + localStats.userBytesCount += blockSize; + } // Pass the block to the disk write task m_fifo.pushDataBlock(mb); mb=NULL; @@ -143,6 +150,11 @@ public: localStats.headerVolume += TapeSessionStats::trailerVolumePerFile; // We now transmitted one file: localStats.filesCount++; + if(isRepack){ + localStats.repackFilesCount++; + } else { + localStats.userFilesCount++; + } params.add("positionTime", localStats.positionTime) .add("readWriteTime", localStats.readWriteTime) .add("waitFreeMemoryTime",localStats.waitFreeMemoryTime) @@ -156,7 +168,11 @@ public: /1000/1000/localStats.totalTime:0) .add("payloadTransferSpeedMBps", localStats.totalTime?1.0*localStats.dataVolume/1000/1000/localStats.totalTime:0) - .add("LBPMode", LBPMode); + .add("LBPMode", LBPMode) + .add("repackFilesCount",localStats.repackFilesCount) + .add("repackBytesCount",localStats.repackBytesCount) + .add("userFilesCount",localStats.userFilesCount) + .add("userBytesCount",localStats.userBytesCount); lc.log(cta::log::INFO, "File successfully read from tape"); // Add the local counts to the session's stats.add(localStats); diff --git a/tapeserver/castor/tape/tapeserver/daemon/TapeSessionStats.hpp b/tapeserver/castor/tape/tapeserver/daemon/TapeSessionStats.hpp index 05c08dfae930f4844dec3a75dcc9205b29869610..b6715c9cc6d7b12637b6781cacd295f655fce18a 100644 --- a/tapeserver/castor/tape/tapeserver/daemon/TapeSessionStats.hpp +++ b/tapeserver/castor/tape/tapeserver/daemon/TapeSessionStats.hpp @@ -92,6 +92,18 @@ namespace daemon { /** Count of files actually transfered in the session. */ uint64_t filesCount; + /** Count of files coming from repack retrieve request transfered in the session.*/ + uint64_t repackFilesCount; + + /** Count of files coming from user retrieve request transfered in the session.*/ + uint64_t userFilesCount; + + /** Count of bytes coming from repack retrieve request transfered in the session.*/ + uint64_t repackBytesCount; + + /** Count of bytes coming from user retrieve request transfered in the session.*/ + uint64_t userBytesCount; + static const uint64_t headerVolumePerFile = 3*80; static const uint64_t trailerVolumePerFile = 3*80; @@ -100,7 +112,8 @@ namespace daemon { readWriteTime(0.0), flushTime(0.0), unloadTime(0.0), unmountTime(0.0), encryptionControlTime(0.0), waitDataTime(0.0), waitFreeMemoryTime(0.0), waitInstructionsTime(0.0), waitReportingTime(0.0), totalTime(0.0), - deliveryTime(0.0), dataVolume(0), headerVolume(0), filesCount(0) {} + deliveryTime(0.0), dataVolume(0), headerVolume(0), filesCount(0), repackFilesCount(0), + userFilesCount(0), repackBytesCount(0), userBytesCount(0) {} /** Accumulate contents of another stats block */ void add(const TapeSessionStats& other) { @@ -121,6 +134,10 @@ namespace daemon { dataVolume += other.dataVolume; headerVolume += other.headerVolume; filesCount += other.filesCount; + repackFilesCount += other.repackFilesCount; + userFilesCount += other.userFilesCount; + repackBytesCount += other.repackBytesCount; + userBytesCount += other.userBytesCount; } }; diff --git a/tapeserver/castor/tape/tapeserver/daemon/TaskWatchDog.hpp b/tapeserver/castor/tape/tapeserver/daemon/TaskWatchDog.hpp index 16dd263f9680063652b68ee00e5a7426f8db7060..3309d1d5011a11e72b09c6ba1a1958f8b40c1c1e 100644 --- a/tapeserver/castor/tape/tapeserver/daemon/TaskWatchDog.hpp +++ b/tapeserver/castor/tape/tapeserver/daemon/TaskWatchDog.hpp @@ -199,6 +199,12 @@ protected: /1000/1000/totalTime:0.0)); paramList.push_back(Param("driveTransferSpeedMBps", totalTime?1.0*(m_stats.dataVolume+m_stats.headerVolume) /1000/1000/totalTime:0.0)); + if(m_mount.getMountType() == cta::common::dataStructures::MountType::Retrieve){ + paramList.push_back(Param("repackFilesCount",m_stats.repackFilesCount)); + paramList.push_back(Param("userFilesCount",m_stats.userFilesCount)); + paramList.push_back(Param("repackBytesCount",m_stats.repackBytesCount)); + paramList.push_back(Param("userBytesCount",m_stats.userBytesCount)); + } // Ship the logs to the initial process m_initialProcess.addLogParams(m_driveUnitName, paramList); }