diff --git a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh index cba83c9f5a39a6d55c8dfd94a25a7a82c0cadffe..a7a9dfe335ebec942cfe91a326b4b0f6a806fc1f 100755 --- a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh +++ b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh @@ -208,6 +208,8 @@ eos attr set sys.workflow.closew.CTA_retrieve="bash:shell:cta eos attr set 'CTA_ # configure preprod directory separately /opt/run/bin/eos_configure_preprod.sh +# configure ssi directory separately +/opt/run/bin/eos_configure_ssi.sh echo "### ctaeos mgm ready ###" diff --git a/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/eos_configure_ssi.sh b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/eos_configure_ssi.sh new file mode 100755 index 0000000000000000000000000000000000000000..fa3844bb7a013e6d13c780582a752b3a61258209 --- /dev/null +++ b/continuousintegration/docker/ctafrontend/cc7/opt/run/bin/eos_configure_ssi.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +###### +# +# Setup an EOS folder with SSI workflows +# until EOS is doing SSI on its own +# +###### + +SSI_DIR=/eos/ctaeos/preprod +CTA_BIN=/usr/bin/eoscta_stub + +mkdir -p /var/eos/wfe/bash +cp /opt/ci/ctaeos/var/eos/wfe/bash/* /var/eos/wfe/bash/ +chmod 755 /var/eos/wfe/bash/* + +eos mkdir ${SSI_DIR} +eos chmod 555 ${SSI_DIR} +eos attr set sys.acl=g:eosusers:rwx!d,g:powerusers:rwx+d ${SSI_DIR} + +eos attr set CTA_StorageClass=ctaStorageClass ${SSI_DIR} + +eos attr set CTA_TapeFsId=65535 ${SSI_DIR} + +eos attr set sys.workflow.closew.default="bash:shell:cta XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/ctafrontend_SSS_c.keytab ${CTA_BIN} archive --user <eos::wfe::rusername> --group <eos::wfe::rgroupname> --diskid <eos::wfe::fid> --instance eoscta --srcurl <eos::wfe::turl> --size <eos::wfe::size> --checksumtype <eos::wfe::checksumtype> --checksumvalue <eos::wfe::checksum> --storageclass <eos::wfe::cxattr:CTA_StorageClass> --diskfilepath <eos::wfe::path> --diskfileowner <eos::wfe::username> --diskfilegroup <eos::wfe::groupname> --recoveryblob:base64 <eos::wfe::base64:metadata> --reportURL 'eosQuery://ctaeos//eos/wfe/passwd?mgm.pcmd=event\&mgm.fid=<eos::wfe::fxid>\&mgm.logid=cta\&mgm.event=archived\&mgm.workflow=default\&mgm.path=/eos/wfe/passwd\&mgm.ruid=0\&mgm.rgid=0' --stderr" ${SSI_DIR} + +#eos attr set sys.workflow.archived.default="bash:shell:cta eos file tag <eos::wfe::path> +<eos::wfe::cxattr:CTA_TapeFsId>" ${SSI_DIR} +eos attr set sys.workflow.archived.default="bash:create_tape_drop_disk_replicas_ssi:cta <eos::wfe::path> <eos::wfe::cxattr:CTA_TapeFsId>" ${SSI_DIR} + + +eos attr set sys.workflow.sync::prepare.default="bash:retrieve_archive_file_ssi:cta <eos::wfe::rusername> <eos::wfe::rgroupname> <eos::wfe::fxattr:sys.archiveFileId> <eos::wfe::turl> <eos::wfe::username> <eos::wfe::groupname> <eos::wfe::base64:metadata> <eos::wfe::path>" ${SSI_DIR} + +eos attr set sys.workflow.closew.CTA_retrieve="bash:shell:cta eos attr set 'CTA_retrieved_timestamp=\"\`date\`\"' <eos::wfe::path>" ${SSI_DIR} + +eos attr set sys.workflow.sync::delete.default="bash:delete_archive_file_ssi:cta <eos::wfe::rusername> <eos::wfe::rgroupname> <eos::wfe::fxattr:sys.archiveFileId> <eos::wfe::path>" ${SSI_DIR} diff --git a/eos_wfe_scripts/create_tape_drop_disk_replicas_ssi b/eos_wfe_scripts/create_tape_drop_disk_replicas_ssi new file mode 100755 index 0000000000000000000000000000000000000000..12acea3666a2c401387a444282d2171d63399038 --- /dev/null +++ b/eos_wfe_scripts/create_tape_drop_disk_replicas_ssi @@ -0,0 +1,52 @@ +#!/bin/bash +if test $# -ne 3; then + echo "Wrong number of command-line arguments" + echo "Usage: create_tape_drop_disk_replicas wf_tag file_path tape_fs_id" + exit -1 +fi + +export XRD_STREAMTIMEOUT=600 # increased from 60s +export XRD_TIMEOUTRESOLUTION=600 # increased from 15s + +WF_TAG="$1" +FILE_PATH="$2" +TAPE_FS_ID="$3" + +LOG_DATE=`/usr/bin/date +%s | /usr/bin/tr '\n' ' ' ; /usr/bin/date` +LOG_SCRIPT_NAME=`/usr/bin/basename $0` +LOG_FILE="/var/log/eos/wfe/${WF_TAG}.log" + +# Creating tape replica +echo "${LOG_DATE} ${LOG_SCRIPT_NAME} creating tape replica with fsid ${TAPE_FS_ID} for ${FILE_PATH}" >> ${LOG_FILE} +OUTPUT=`/usr/bin/eos -r 0 0 file tag "${FILE_PATH}" +${TAPE_FS_ID} 2>&1` +RESULT=$? +if [ 0 -ne ${RESULT} ]; then + echo "${LOG_DATE} ${LOG_SCRIPT_NAME} failed to create tape replica with fsid ${TAPE_FS_ID} for ${FILE_PATH}: ${RESULT} ${OUTPUT}" >> ${LOG_FILE} + exit 1 +fi + +# Checking tape replica +#echo "${LOG_DATE} ${LOG_SCRIPT_NAME} checking tape replica for ${FILE_PATH}" >> ${LOG_FILE} +OUTPUT=`/usr/bin/eos -r 0 0 attr ls "${FILE_PATH}" 2>&1 | /usr/bin/grep -c '^sys.archiveFileId=\"'` +RESULT=$? +if [ 1 -gt ${OUTPUT} ]; then + echo "${LOG_DATE} ${LOG_SCRIPT_NAME} missing tape replica sys.archiveFileId for ${FILE_PATH}: ${RESULT} ${OUTPUT}" >> ${LOG_FILE} + exit 1 +fi +OUTPUT=`/usr/bin/eos -r 0 0 ls -y "${FILE_PATH}" 2>&1 | /usr/bin/grep -c '^d.::t[123456789]'` +RESULT=$? +if [ 1 -gt ${OUTPUT} ]; then + echo "${LOG_DATE} ${LOG_SCRIPT_NAME} tape replica (expecting at least d?::t1) missing in EOS for ${FILE_PATH}: ${RESULT} ${OUTPUT}" >> ${LOG_FILE} + exit 1 +fi + +# Deleting disk replica +for DISK_FSID in `/usr/bin/eos file info "${FILE_PATH}" -m | /usr/bin/sed s/\ /'\n'/g | /usr/bin/grep fsid | /usr/bin/sed s/fsid=// | /usr/bin/grep -v ${TAPE_FS_ID}`; do + echo "${LOG_DATE} ${LOG_SCRIPT_NAME} deleting disk replica with fsid ${DISK_FSID} for ${FILE_PATH}" >> ${LOG_FILE} + OUTPUT=`/usr/bin/eos -r 0 0 file drop "${FILE_PATH}" ${DISK_FSID} 2>&1` + RESULT=$? + if [ 0 -ne ${RESULT} ]; then + echo "${LOG_DATE} ${LOG_SCRIPT_NAME} failed to delete disk replica with fsid ${DISK_FSID} for ${FILE_PATH}: ${RESULT} ${OUTPUT}" >> ${LOG_FILE} + exit 1 + fi +done diff --git a/eos_wfe_scripts/delete_archive_file_ssi b/eos_wfe_scripts/delete_archive_file_ssi new file mode 100755 index 0000000000000000000000000000000000000000..3abcd2089a03b254ebaa3cdd7075b2e9474cbafe --- /dev/null +++ b/eos_wfe_scripts/delete_archive_file_ssi @@ -0,0 +1,42 @@ +#!/bin/bash + +EINVAL=22 +ECANCELED=125 + +export XrdSecPROTOCOL=sss +export XrdSecSSSKT=/etc/cta/cta-cli.sss.keytab # This location is used on EOSCTATAPE +export XrdSecSSSKT=/etc/ctafrontend_SSS_c.keytab # This location is used for CI + +export XRD_STREAMTIMEOUT=600 # increased from 60s +export XRD_TIMEOUTRESOLUTION=600 # increased from 15s + +if test $# -ne 5; then + echo "Wrong number of command-line arguments" + echo "Usage: delete_archive_file wf_tag rusername rgroupname archive_file_id file_path" + exit ${EINVAL} +fi + +CTA_BIN=/usr/bin/eoscta_stub + +WF_TAG="$1" +RUSERNAME="$2" +RGROUPNAME="$3" +ARCHIVE_FILE_ID="$4" +FILE_PATH="$5" + +LOG_DATE=`/usr/bin/date +%s | /usr/bin/tr '\n' ' ' ; /usr/bin/date` +LOG_SCRIPT_NAME=`/usr/bin/basename $0` +LOG_FILE="/var/log/eos/wfe/${WF_TAG}.log" + +if test UNDEF = ${ARCHIVE_FILE_ID}; then + echo "$LOG_DATE $LOG_SCRIPT_NAME ignoring deletion of non-existent tape archive file: rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} path=${FILE_PATH}" >> ${LOG_FILE} + exit 0 +fi + +if RESULT=`2>&1 ${CTA_BIN} deletearchive --user ${RUSERNAME} --group ${RGROUPNAME} --id ${ARCHIVE_FILE_ID}`; then + echo "$LOG_DATE $LOG_SCRIPT_NAME deleted tape archive file: rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} path=${FILE_PATH}" >> ${LOG_FILE} + exit 0 +else + echo "$LOG_DATE $LOG_SCRIPT_NAME failed to delete tape archive file: error=${RESULT} rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} path=${FILE_PATH}" >> ${LOG_FILE} + exit ${ECANCELED} +fi diff --git a/eos_wfe_scripts/retrieve_archive_file_ssi b/eos_wfe_scripts/retrieve_archive_file_ssi new file mode 100755 index 0000000000000000000000000000000000000000..0295b34bc9600a50647c1bb6dbf8f1b8432be3ee --- /dev/null +++ b/eos_wfe_scripts/retrieve_archive_file_ssi @@ -0,0 +1,53 @@ +#!/bin/bash + +EINVAL=22 +ECANCELED=125 + +export XrdSecPROTOCOL=sss +export XrdSecSSSKT=/etc/cta/cta-cli.sss.keytab # This location is used on EOSCTATAPE +export XrdSecSSSKT=/etc/ctafrontend_SSS_c.keytab # This location is used for CI + +export XRD_STREAMTIMEOUT=600 # increased from 60s +export XRD_TIMEOUTRESOLUTION=600 # increased from 15s + +if test $# -ne 9; then + echo "Wrong number of command-line arguments" + echo "Usage: retrieve_archive_file wf_tag rusername rgroupname archive_file_id turl disk_username disk_groupname metadata_base64 file_path" + exit ${EINVAL} +fi + +CTA_BIN=/usr/bin/eoscta_stub + +WF_TAG="$1" +RUSERNAME="$2" +RGROUPNAME="$3" +ARCHIVE_FILE_ID="$4" +TURL="$5" +DISK_USERNAME="$6" +DISK_GROUPNAME="$7" +METADATA_BASE64="$8" +FILE_PATH="$9" + +DST_URL=${TURL}'\&eos.ruid=0&eos.rgid=0\&eos.injection=1\&eos.workflow=CTA_retrieve' + +LOG_DATE=`/usr/bin/date +%s | /usr/bin/tr '\n' ' ' ; /usr/bin/date` +LOG_SCRIPT_NAME=`/usr/bin/basename $0` +LOG_FILE="/var/log/eos/wfe/${WF_TAG}.log" + +if test UNDEF = ${ARCHIVE_FILE_ID}; then + echo "$LOG_DATE $LOG_SCRIPT_NAME cannot retrieve an archive file without the sys.archiveFileId attribute being set: rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} dsturl=${DST_URL} disk_username=${DISK_USERNAME} disk_groupname=${DISK_GROUPNAME} path=${FILE_PATH}" >> ${LOG_FILE} + exit ${ECANCELED} +fi + +if ! RESULT=`unset XrdSecPROTOCOL XrdSecSSSKT ; /usr/bin/eos ls -y ${FILE_PATH} | /usr/bin/grep -s ^d0`; then + echo "$LOG_DATE $LOG_SCRIPT_NAME file already has at least one online replica on disk: rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} dsturl=${DST_URL} disk_username=${DISK_USERNAME} disk_groupname=${DISK_GROUPNAME} path=${FILE_PATH}" >> ${LOG_FILE} + exit 0 +fi + +if RESULT=`2>&1 ${CTA_BIN} retrieve --user ${RUSERNAME} --group ${RGROUPNAME} --id ${ARCHIVE_FILE_ID} --dsturl ${DST_URL} --diskfilepath ${FILE_PATH} --diskfileowner ${DISK_USERNAME} --diskfilegroup ${DISK_GROUPNAME} --recoveryblob:base64 ${METADATA_BASE64}`; then + echo "$LOG_DATE $LOG_SCRIPT_NAME queued retrieve request: rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} dsturl=${DST_URL} disk_username=${DISK_USERNAME} disk_groupname=${DISK_GROUPNAME} path=${FILE_PATH}" >> ${LOG_FILE} + exit 0 +else + echo "$LOG_DATE $LOG_SCRIPT_NAME failed to queue retrieve request: error=${RESULT} rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} dsturl=${DST_URL} disk_username=${DISK_USERNAME} disk_groupname=${DISK_GROUPNAME} path=${FILE_PATH}" >> ${LOG_FILE} + exit ${ECANCELED} +fi