Skip to content

GitLab

Commit 1f753892 authored by Cedric CAFFY's avatar Cedric CAFFY
Browse files

Added files and bytes count for Retrieved files from User or Repack in the... 

Added files and bytes count for Retrieved files from User or Repack in the "tape session finished" message
Made the Repack system test clearer by printing steps and by adding folders for the reports
parent e6dbcd53
Hide whitespace changes
Inline Side-by-side
continuousintegration/orchestration/tests/repack_generate_report.sh
View file @ 1f753892
......@@ -54,6 +54,9 @@ then
die "No repack request for this VID."
fi;
echo "Creating the report directory ${REPORT_DIRECTORY}"
mkdir -p ${REPORT_DIRECTORY} || die "Unable to create the directory for report generation"
echo "Generation of a repack report of the vid ${VID}"
DATE=`date +%d-%m-%y-%H:%M:%S`
......@@ -64,6 +67,7 @@ NOT_REPACKED_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_not_repacked_${DATE}.js
SELF_REPACKED_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_self_repacked_${DATE}.json
REPACKED_MOVE_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_repacked_move_${DATE}.json
REPACK_ADD_COPIES_JSON_PATH=${REPORT_DIRECTORY}/${VID}_report_repack_add_copies_${DATE}.json
STDOUT_REPORT_PATH=${REPORT_DIRECTORY}/${VID}_stdout_report.txt
echo "1. Generate archive file ls result into ${ARCHIVE_FILE_LS_RESULT_PATH} file..."
......@@ -104,7 +108,7 @@ then
if [ ${NB_NON_REPACKED_FILES} -ne 0 ]
then
header="ArchiveID\tFSeq\tSize"
{ echo -e $header; jq -r '.[] | [.af.archiveId,.tf.fSeq, .af.size] | @tsv' ${NOT_REPACKED_JSON_PATH}; } | column -t
{ echo -e $header; jq -r '.[] | [.af.archiveId,.tf.fSeq, .af.size] | @tsv' ${NOT_REPACKED_JSON_PATH}; } | column -t | tee --append ${STDOUT_REPORT_PATH}
fi;
fi;
......@@ -114,7 +118,7 @@ echo "Number of self-repacked files : ${NB_SELF_REPACKED_FILES}"
if [ ${NB_SELF_REPACKED_FILES} -ne 0 ]
then
header="ArchiveID\tFSeq\tSize"
{ echo -e $header; jq -r '.[] | [.af.archiveId, .tf.fSeq, .af.size] | @tsv' ${SELF_REPACKED_JSON_PATH}; } | column -t
{ echo -e $header; jq -r '.[] | [.af.archiveId, .tf.fSeq, .af.size] | @tsv' ${SELF_REPACKED_JSON_PATH}; } | column -t | tee --append ${STDOUT_REPORT_PATH}
fi;
echo
......@@ -123,7 +127,7 @@ echo "Number of repacked (moved) files : ${NB_REPACKED_MOVE_FILES}"
if [ ${NB_REPACKED_MOVE_FILES} -ne 0 ]
then
header="DestinationVID\tNbFiles\ttotalSize\n"
{ echo -e $header; jq -r 'group_by(.tf.supersededByVid)[] | [(.[0].tf.supersededByVid),([.[] | .tf.supersededByFSeq] | length),(reduce [.[] | .af.size | tonumber][] as $currentSize (0; . + $currentSize))] | @tsv' ${REPACKED_MOVE_JSON_PATH}; } | column -t
{ echo -e $header; jq -r 'group_by(.tf.supersededByVid)[] | [(.[0].tf.supersededByVid),([.[] | .tf.supersededByFSeq] | length),(reduce [.[] | .af.size | tonumber][] as $currentSize (0; . + $currentSize))] | @tsv' ${REPACKED_MOVE_JSON_PATH}; } | column -t | tee --append ${STDOUT_REPORT_PATH}
fi;
echo
......@@ -132,7 +136,8 @@ echo "Number of copied files : $NB_COPIED_FILES"
if [ ${NB_COPIED_FILES} -ne 0 ]
then
header="DestinationVID\tNbFiles\ttotalSize\n"
{ echo -e $header; jq -r 'group_by(.tf.vid)[] | [(.[0].tf.vid),([.[] | .tf.fSeq] | length),(reduce [.[] | .af.size | tonumber][] as $currentSize (0; . + $currentSize))] | @tsv' ${REPACK_ADD_COPIES_JSON_PATH}; } | column -t
{ echo -e $header; jq -r 'group_by(.tf.vid)[] | [(.[0].tf.vid),([.[] | .tf.fSeq] | length),(reduce [.[] | .af.size | tonumber][] as $currentSize (0; . + $currentSize))] | @tsv' ${REPACK_ADD_COPIES_JSON_PATH}; } | column -t | tee --append ${STDOUT_REPORT_PATH}
fi;
echo "End of the repack report"
\ No newline at end of file
echo "End of the repack report"
exit 0
\ No newline at end of file
continuousintegration/orchestration/tests/repack_systemtest.sh
View file @ 1f753892
......@@ -5,6 +5,8 @@ EOSINSTANCE=ctaeos
#default Repack timeout
WAIT_FOR_REPACK_TIMEOUT=300
REPORT_DIRECTORY=/var/log
die() {
echo "$@" 1>&2
test -z $TAILPID || kill ${TAILPID} &> /dev/null
......@@ -12,10 +14,11 @@ die() {
}
usage() { cat <<EOF 1>&2
Usage: $0 -v <vid> -b <bufferURL> [-e <eosinstance>] [-t <timeout>] [-a] [-m] [-d]
Usage: $0 -v <vid> -b <bufferURL> [-e <eosinstance>] [-t <timeout>] [-r <reportDirectory>] [-a] [-m] [-d]
(bufferURL example : /eos/ctaeos/repack)
eosinstance : the name of the ctaeos instance to be used (default ctaeos)
eosinstance : the name of the ctaeos instance to be used (default : $EOSINSTANCE)
timeout : the timeout in seconds to wait for the repack to be done
reportDirectory : the directory to generate the report of the repack test (default : $REPORT_DIRECTORY)
-a : Launch a repack just add copies workflow
-m : Launch a repack just move workflow
-d : Force a repack on a disabled tape (adds --disabled to the repack add command)
......@@ -42,7 +45,7 @@ then
fi;
DISABLED_TAPE_FLAG=""
while getopts "v:e:b:t:amgd" o; do
while getopts "v:e:b:t:r:amd" o; do
case "${o}" in
v)
VID_TO_REPACK=${OPTARG}
......@@ -62,8 +65,8 @@ while getopts "v:e:b:t:amgd" o; do
m)
MOVE_ONLY="-m"
;;
g)
GENERATE_REPORT=0
r)
REPORT_DIRECTORY=${OPTARG}
;;
d)
DISABLED_TAPE_FLAG="--disabledtape"
......@@ -121,14 +124,20 @@ while test 0 = `admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r '.[0]
if test ${SECONDS_PASSED} == ${WAIT_FOR_REPACK_TIMEOUT}; then
echo "Timed out after ${WAIT_FOR_REPACK_TIMEOUT} seconds waiting for tape ${VID_TO_REPACK} to be repacked"
exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} -r ${REPORT_DIRECTORY} ${ADD_COPIES_ONLY} &
wait $!
exit 1
fi
done
if test 1 = `admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r '[.[0] | select (.status == "Failed")] | length'`; then
echo "Repack failed for tape ${VID_TO_REPACK}."
exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} -r ${REPORT_DIRECTORY} ${ADD_COPIES_ONLY} &
wait $!
exit 1
fi
echo "Repack request on VID ${VID_TO_REPACK} succeeded."
exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} ${ADD_COPIES_ONLY} || exit 0
\ No newline at end of file
exec /root/repack_generate_report.sh -v ${VID_TO_REPACK} -r ${REPORT_DIRECTORY} ${ADD_COPIES_ONLY} &
wait $!
exit 0
\ No newline at end of file
continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh
View file @ 1f753892
#!/bin/bash
BASE_REPORT_DIRECTORY=/var/log
usage() { cat <<EOF 1>&2
Usage: $0 -n <namespace>
EOF
......@@ -52,14 +54,16 @@ kubectl -n ${NAMESPACE} cp repack_systemtest.sh client:/root/repack_systemtest.s
kubectl -n ${NAMESPACE} cp repack_generate_report.sh client:/root/repack_generate_report.sh
echo
echo "Launching a round trip repack \"just move\" request"
echo "***********************************************************"
echo "STEP 1. Launching a round trip repack \"just move\" request"
echo "***********************************************************"
VID_TO_REPACK=$(getFirstVidContainingFiles)
if [ "$VID_TO_REPACK" != "null" ]
then
echo
echo "Launching the repack \"just move\" test on VID ${VID_TO_REPACK}"
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m || exit 1
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step1-RoundTripRepack || exit 1
else
echo "No vid found to repack"
exit 1
......@@ -73,7 +77,7 @@ if [ "$VID_TO_REPACK" != "null" ]
then
echo
echo "Launching the repack \"just move\" test on VID ${VID_TO_REPACK}"
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m || exit 1
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step1-RoundTripRepack || exit 1
else
echo "No vid found to repack"
exit 1
......@@ -81,8 +85,15 @@ fi
echo "Reclaiming tape ${VID_TO_REPACK}"
kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}
echo
echo "*******************************************************************"
echo "STEP 1. Launching a round trip repack \"just move\" request TEST OK"
echo "*******************************************************************"
echo
echo "*****************************************************"
echo "STEP 2. Launching a Repack Request on a disabled tape"
echo "*****************************************************"
echo "Launching a Repack Request on a disabled tape"
VID_TO_REPACK=$(getFirstVidContainingFiles)
if [ "$VID_TO_REPACK" != "null" ]
......@@ -92,7 +103,7 @@ then
echo "Waiting 20 seconds so that the RetrieveQueueStatisticsCache is flushed"
sleep 20
echo "Launching the repack request test on VID ${VID_TO_REPACK}"
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} && echo "The repack request is Complete instead of Failed, it should be failed as the tape is disabled" && exit 1 || echo "Test OK"
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -r ${BASE_REPORT_DIRECTORY}/Step2-RepackDisabledTape && echo "The repack request is Complete instead of Failed, it should be failed as the tape is disabled" && exit 1 || echo "REPACK FAILED, the tape is disabled so, Test OK"
else
echo "No vid found to repack"
exit 1
......@@ -100,11 +111,20 @@ fi;
echo
echo "Launching the repack request test on VID ${VID_TO_REPACK} with the --disabledtape flag"
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -d || exit 1
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -d -r ${BASE_REPORT_DIRECTORY}/Step2-RepackDisabledTape || exit 1
echo "Reclaiming tape ${VID_TO_REPACK}"
kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}
echo
echo "*************************************************************"
echo "STEP 2. Launching a Repack Request on a disabled tape TEST OK"
echo "*************************************************************"
echo
echo "*********************************************"
echo "STEP 3. Testing Repack \"Just move\" workflow"
echo "*********************************************"
NB_FILES=1152
kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v -A || exit 1
......@@ -113,7 +133,7 @@ if [ "$VID_TO_REPACK" != "null" ]
then
echo
echo "Launching the repack test \"just move\" on VID ${VID_TO_REPACK}"
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m || exit 1
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step3-RepackJustMove || exit 1
else
echo "No vid found to repack"
exit 1
......@@ -121,12 +141,20 @@ fi
echo "Reclaiming tape ${VID_TO_REPACK}"
kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}
echo
echo "*****************************************************"
echo "STEP 3. Testing Repack \"Just move\" workflow TEST OK"
echo "*****************************************************"
echo
echo "**************************************************************************"
echo "STEP 4. Testing Repack \"Just Add copies\" workflow with all copies on CTA"
echo "**************************************************************************"
VID_TO_REPACK=$(getFirstVidContainingFiles)
if [ "$VID_TO_REPACK" != "null" ]
then
echo "Launching the repack \"just add copies\" test on VID ${VID_TO_REPACK} with all copies already on CTA"
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -a -g || exit 1
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -a -r ${BASE_REPORT_DIRECTORY}/Step4-JustAddCopiesAllCopiesInCTA || exit 1
else
echo "No vid found to repack"
exit 1
......@@ -145,6 +173,16 @@ else
exit 1
fi
echo
echo "**********************************************************************************"
echo "STEP 4. Testing Repack \"Just Add copies\" workflow with all copies on CTA TEST OK"
echo "**********************************************************************************"
echo
echo "*******************************************************"
echo "STEP 5. Testing Repack \"Move and Add copies\" workflow"
echo "*******************************************************"
tapepoolDestination1="ctasystest2"
tapepoolDestination2="ctasystest3"
......@@ -205,4 +243,8 @@ kubectl -n ${NAMESPACE} exec ctacli -- cta-admin dr up VD.*
echo "OK"
echo "Launching the repack \"Move and add copies\" test on VID ${VID_TO_REPACK}"
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -t 500 || exit 1
kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -t 600 -r ${BASE_REPORT_DIRECTORY}/Step5-MoveAndAddCopies || exit 1
echo
echo "***************************************************************"
echo "STEP 5. Testing Repack \"Move and Add copies\" workflow TEST OK"
echo "***************************************************************"
scheduler/RetrieveJob.hpp
View file @ 1f753892
......@@ -27,6 +27,13 @@
#include <limits>
#include <memory>
namespace castor {
namespace tape {
namespace tapeserver {
namespace daemon {
class TapeReadTask;
}}}}
namespace cta {
class RetrieveMount;
......@@ -42,7 +49,7 @@ class RetrieveJob {
*/
friend class RetrieveMount;
friend class Scheduler;
friend class castor::tape::tapeserver::daemon::TapeReadTask;
public:
/**
* Constructor. It is not public as it is generated by the RetrieveMount.
......
scheduler/RetrieveMount.hpp
View file @ 1f753892
......@@ -233,7 +233,7 @@ namespace cta {
* A pointer to the file catalogue.
*/
cta::catalogue::Catalogue &m_catalogue;
}; // class RetrieveMount
} // namespace cta
tapeserver/castor/tape/tapeserver/daemon/TapeReadTask.hpp
View file @ 1f753892
......@@ -66,12 +66,14 @@ public:
using cta::log::Param;
bool isRepack = m_retrieveJob->m_dbJob->isRepack;
// Set the common context for all the coming logs (file info)
cta::log::ScopedParamContainer params(lc);
params.add("fileId", m_retrieveJob->archiveFile.archiveFileID)
.add("BlockId", m_retrieveJob->selectedTapeFile().blockId)
.add("fSeq", m_retrieveJob->selectedTapeFile().fSeq)
.add("dstURL", m_retrieveJob->retrieveRequest.dstURL);
.add("dstURL", m_retrieveJob->retrieveRequest.dstURL)
.add("isRepack",isRepack);
// We will clock the stats for the file itself, and eventually add those
// stats to the session's.
......@@ -129,6 +131,11 @@ public:
localStats.readWriteTime += timer.secs(cta::utils::Timer::resetCounter);
auto blockSize = mb->m_payload.size();
localStats.dataVolume += blockSize;
if(isRepack){
localStats.repackBytesCount += blockSize;
} else {
localStats.userBytesCount += blockSize;
}
// Pass the block to the disk write task
m_fifo.pushDataBlock(mb);
mb=NULL;
......@@ -143,6 +150,11 @@ public:
localStats.headerVolume += TapeSessionStats::trailerVolumePerFile;
// We now transmitted one file:
localStats.filesCount++;
if(isRepack){
localStats.repackFilesCount++;
} else {
localStats.userFilesCount++;
}
params.add("positionTime", localStats.positionTime)
.add("readWriteTime", localStats.readWriteTime)
.add("waitFreeMemoryTime",localStats.waitFreeMemoryTime)
......@@ -156,7 +168,11 @@ public:
/1000/1000/localStats.totalTime:0)
.add("payloadTransferSpeedMBps",
localStats.totalTime?1.0*localStats.dataVolume/1000/1000/localStats.totalTime:0)
.add("LBPMode", LBPMode);
.add("LBPMode", LBPMode)
.add("repackFilesCount",localStats.repackFilesCount)
.add("repackBytesCount",localStats.repackBytesCount)
.add("userFilesCount",localStats.userFilesCount)
.add("userBytesCount",localStats.userBytesCount);
lc.log(cta::log::INFO, "File successfully read from tape");
// Add the local counts to the session's
stats.add(localStats);
......
tapeserver/castor/tape/tapeserver/daemon/TapeSessionStats.hpp
View file @ 1f753892
......@@ -92,6 +92,18 @@ namespace daemon {
/** Count of files actually transfered in the session. */
uint64_t filesCount;
/** Count of files coming from repack retrieve request transfered in the session.*/
uint64_t repackFilesCount;
/** Count of files coming from user retrieve request transfered in the session.*/
uint64_t userFilesCount;
/** Count of bytes coming from repack retrieve request transfered in the session.*/
uint64_t repackBytesCount;
/** Count of bytes coming from user retrieve request transfered in the session.*/
uint64_t userBytesCount;
static const uint64_t headerVolumePerFile = 3*80;
static const uint64_t trailerVolumePerFile = 3*80;
......@@ -100,7 +112,8 @@ namespace daemon {
readWriteTime(0.0), flushTime(0.0), unloadTime(0.0), unmountTime(0.0),
encryptionControlTime(0.0), waitDataTime(0.0), waitFreeMemoryTime(0.0),
waitInstructionsTime(0.0), waitReportingTime(0.0), totalTime(0.0),
deliveryTime(0.0), dataVolume(0), headerVolume(0), filesCount(0) {}
deliveryTime(0.0), dataVolume(0), headerVolume(0), filesCount(0), repackFilesCount(0),
userFilesCount(0), repackBytesCount(0), userBytesCount(0) {}
/** Accumulate contents of another stats block */
void add(const TapeSessionStats& other) {
......@@ -121,6 +134,10 @@ namespace daemon {
dataVolume += other.dataVolume;
headerVolume += other.headerVolume;
filesCount += other.filesCount;
repackFilesCount += other.repackFilesCount;
userFilesCount += other.userFilesCount;
repackBytesCount += other.repackBytesCount;
userBytesCount += other.userBytesCount;
}
};
......
tapeserver/castor/tape/tapeserver/daemon/TaskWatchDog.hpp
View file @ 1f753892
......@@ -199,6 +199,12 @@ protected:
/1000/1000/totalTime:0.0));
paramList.push_back(Param("driveTransferSpeedMBps", totalTime?1.0*(m_stats.dataVolume+m_stats.headerVolume)
/1000/1000/totalTime:0.0));
if(m_mount.getMountType() == cta::common::dataStructures::MountType::Retrieve){
paramList.push_back(Param("repackFilesCount",m_stats.repackFilesCount));
paramList.push_back(Param("userFilesCount",m_stats.userFilesCount));
paramList.push_back(Param("repackBytesCount",m_stats.repackBytesCount));
paramList.push_back(Param("userBytesCount",m_stats.userBytesCount));
}
// Ship the logs to the initial process
m_initialProcess.addLogParams(m_driveUnitName, paramList);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

imprint | privacy | Conditions of Use