-v Verbose mode: displays live logs of rmcd to see tapes being mounted/dismounted in real time
-r Remove files at the end: launches the delete workflow on the files that were deleted. WARNING: THIS CAN BE FATAL TO THE NAMESPACE IF THERE ARE TOO MANY FILES AND XROOTD STARTS TO TIMEOUT.
curlcmd="curl --connect-timeout 2 -X POST 'https://ctapps-influx02.cern.ch:8086/write?db=annotations&u=annotations&p=annotations&precision=s' --data-binary '${LINE}'"
eval${curlcmd}
}
while getopts"d:e:n:N:s:p:vS:rAPGt:m:" o;do
case"${o}"in
e)
EOSINSTANCE=${OPTARG}
...
...
@@ -39,6 +67,9 @@ while getopts "d:e:n:s:p:vS:r" o; do
n)
NB_FILES=${OPTARG}
;;
N)
NB_DIRS=${OPTARG}
;;
s)
FILE_KB_SIZE=${OPTARG}
;;
...
...
@@ -54,6 +85,21 @@ while getopts "d:e:n:s:p:vS:r" o; do
r)
REMOVE=1
;;
A)
ARCHIVEONLY=1
;;
P)
DONOTARCHIVE=1
;;
G)
TAPEAWAREGC=1
;;
t)
TARGETDIR=${OPTARG}
;;
m)
COMMENT=${OPTARG}
;;
*)
usage
;;
...
...
@@ -66,7 +112,38 @@ if [ ! -z "${error}" ]; then
exit 1
fi
if["x${COMMENT}"="x"];then
echo"No annotation will be pushed to Influxdb"
fi
if[[$DONOTARCHIVE== 1 ]];then
if[["x${TARGETDIR}"="x"]];then
echo"You must provide a target directory to run a test and skip archival"
exit 1
fi
eos root://${EOSINSTANCE}ls-d${EOS_BASEDIR}/${TARGETDIR}|| die "target directory does not exist and there is no archive phase to create it."
fi
if[[$TAPEAWAREGC== 1 ]];then
echo"Enabling tape aware garbage collector"
ssh ${SSH_OPTIONS}-l root ${EOSINSTANCE} eos space config default space.tapeawaregc.minfreebytes=${GC_MINFREEBYTES}|| die "Could not set space.tapeawaregc.minfreebytes to ${GC_MINFREEBYTES}"
ssh ${SSH_OPTIONS}-l root ${EOSINSTANCE} eos space config default space.filearchivedgc=off || die "Could not disable filearchivedgc"
else
echo"Enabling file archived garbage collector"
# no ssh for CI
#ssh ${SSH_OPTIONS} -l root ${EOSINSTANCE} eos space config default space.tapeawaregc.minfreebytes=0 || die "Could not set space.tapeawaregc.minfreebytes to 0"
#ssh ${SSH_OPTIONS} -l root ${EOSINSTANCE} eos space config default space.filearchivedgc=on || die "Could not enable filearchivedgc"
eos root://${EOSINSTANCE}ls-y${EOS_DIR} | sed-e's/^\(d.::t.\).*\(test[0-9]\+\)$/\2 \1/;s/d[^0]::t[^0]/archived/;s/d[^0]::t0/copied/;s/d0::t0/error/;s/d0::t[^0]/tapeonly/'>${STATUS_FILE}
grep-q'archived$'${STATUS_FILE}&&echo"Removing disk replica of $(grep-c'archived$'${STATUS_FILE}) archived files"||echo"$(grep-c'tapeonly$'${STATUS_FILE}) files are on tape with no disk replica"
for TEST_FILE_NAME in$(grep archived$ ${STATUS_FILE} | sed-e's/ .*$//');do
XrdSecPROTOCOL=sss eos -r 0 0 root://${EOSINSTANCE} file drop ${EOS_DIR}/${TEST_FILE_NAME} 1 &> /dev/null ||echo"Could not remove disk replica for ${EOS_DIR}/${TEST_FILE_NAME}"
done
# Updating all files statuses
eos root://${EOSINSTANCE}ls-y${EOS_DIR} | sed-e's/^\(d.::t.\).*\(test[0-9]\+\)$/\2 \1/;s/d[^0]::t[^0]/archived/;s/d[^0]::t0/copied/;s/d0::t0/error/;s/d0::t[^0]/tapeonly/'>${STATUS_FILE}
fi# DONOTARCHIVE
#echo "$(date +%s): Dumping objectstore list"
#ssh root@ctappsfrontend cta-objectstore-list
if[[$ARCHIVEONLY== 1 ]];then
echo"Archiveonly mode: exiting"
exit 0
fi
TAPEONLY=$(grep-c tapeonly$ ${STATUS_FILE})
echo"###"
echo"${TAPEONLY}/${ARCHIVED} on tape only"
...
...
@@ -146,15 +255,36 @@ echo "Sleeping 400 seconds to allow MGM-FST communication to settle after disk c
sleep 400
echo"###"
echo"Trigerring EOS retrieve workflow as poweruser1:powerusers (12001:1200)"
#for TEST_FILE_NAME in $(grep tapeonly$ ${STATUS_FILE} | sed -e 's/ .*$//'); do
# XrdSecPROTOCOL=sss xrdfs ${EOSINSTANCE} prepare -s "${EOS_DIR}/${TEST_FILE_NAME}?eos.ruid=12001&eos.rgid=1200" || echo "Could not trigger retrieve for ${EOS_DIR}/${TEST_FILE_NAME}"
#done
if[[$TAPEAWAREGC== 1 ]];then
echo"Disabling file tape aware garbage collector"
ssh ${SSH_OPTIONS}-l root ${EOSINSTANCE} eos space config default space.tapeawaregc.minfreebytes=0 || die "Could not set space.tapeawaregc.minfreebytes to 0"
# we do not need it for retrieves
# ssh ${SSH_OPTIONS} -l root ${EOSINSTANCE} eos space config default space.filearchivedgc=on || die "Could not enable filearchivedgc"
fi
echo"$(date +%s): Trigerring EOS retrieve workflow as poweruser1:powerusers (12001:1200)"
rm-f${STATUS_FILE}
touch${STATUS_FILE}
for((subdir=0; subdir < ${NB_DIRS}; subdir++));do
eos root://${EOSINSTANCE}ls-y${EOS_DIR}/${subdir} | grep'd0::t1' | sed-e"s%\s\+% %g;s%.* \([^ ]\+\)$%${subdir}/\1%">>${STATUS_FILE}
# sleep 3 # do not hammer eos too hard
done
# We need the -s as we are staging the files from tape (see xrootd prepare definition)
