Preparing eos instance and eos workflows for preprod and SSI, so that we can...

Preparing eos instance and eos workflows for preprod and SSI, so that we can test both SSI and non SSI workflows on the same cta instance, just in 2 different EOS folders.

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/ctaeos-mgm.sh

@@ -208,6 +208,8 @@ eos attr set sys.workflow.closew.CTA_retrieve="bash:shell:cta eos attr set 'CTA_
 
 # configure preprod directory separately
 /opt/run/bin/eos_configure_preprod.sh
+# configure ssi directory separately
+/opt/run/bin/eos_configure_ssi.sh
 
 echo "### ctaeos mgm ready ###"
 

continuousintegration/docker/ctafrontend/cc7/opt/run/bin/eos_configure_ssi.sh

+#!/bin/bash
+
+######
+#
+# Setup an EOS folder with SSI workflows
+# until EOS is doing SSI on its own
+#
+######
+
+SSI_DIR=/eos/ctaeos/preprod
+CTA_BIN=/usr/bin/eoscta_stub
+
+mkdir -p /var/eos/wfe/bash
+cp /opt/ci/ctaeos/var/eos/wfe/bash/* /var/eos/wfe/bash/
+chmod 755 /var/eos/wfe/bash/*
+
+eos mkdir ${SSI_DIR}
+eos chmod 555 ${SSI_DIR}
+eos attr set sys.acl=g:eosusers:rwx!d,g:powerusers:rwx+d ${SSI_DIR}
+
+eos attr set CTA_StorageClass=ctaStorageClass ${SSI_DIR}
+
+eos attr set CTA_TapeFsId=65535 ${SSI_DIR}
+
+eos attr set sys.workflow.closew.default="bash:shell:cta XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/ctafrontend_SSS_c.keytab ${CTA_BIN} archive --user <eos::wfe::rusername> --group <eos::wfe::rgroupname> --diskid <eos::wfe::fid> --instance eoscta --srcurl <eos::wfe::turl> --size <eos::wfe::size> --checksumtype <eos::wfe::checksumtype> --checksumvalue <eos::wfe::checksum> --storageclass <eos::wfe::cxattr:CTA_StorageClass> --diskfilepath <eos::wfe::path> --diskfileowner <eos::wfe::username> --diskfilegroup <eos::wfe::groupname> --recoveryblob:base64 <eos::wfe::base64:metadata> --reportURL 'eosQuery://ctaeos//eos/wfe/passwd?mgm.pcmd=event\&mgm.fid=<eos::wfe::fxid>\&mgm.logid=cta\&mgm.event=archived\&mgm.workflow=default\&mgm.path=/eos/wfe/passwd\&mgm.ruid=0\&mgm.rgid=0' --stderr" ${SSI_DIR}
+
+#eos attr set sys.workflow.archived.default="bash:shell:cta eos file tag <eos::wfe::path> +<eos::wfe::cxattr:CTA_TapeFsId>" ${SSI_DIR}
+eos attr set sys.workflow.archived.default="bash:create_tape_drop_disk_replicas_ssi:cta <eos::wfe::path> <eos::wfe::cxattr:CTA_TapeFsId>" ${SSI_DIR}
+
+
+eos attr set sys.workflow.sync::prepare.default="bash:retrieve_archive_file_ssi:cta <eos::wfe::rusername> <eos::wfe::rgroupname> <eos::wfe::fxattr:sys.archiveFileId> <eos::wfe::turl> <eos::wfe::username> <eos::wfe::groupname> <eos::wfe::base64:metadata> <eos::wfe::path>" ${SSI_DIR}
+
+eos attr set sys.workflow.closew.CTA_retrieve="bash:shell:cta eos attr set 'CTA_retrieved_timestamp=\"\`date\`\"' <eos::wfe::path>" ${SSI_DIR}
+
+eos attr set sys.workflow.sync::delete.default="bash:delete_archive_file_ssi:cta <eos::wfe::rusername> <eos::wfe::rgroupname> <eos::wfe::fxattr:sys.archiveFileId> <eos::wfe::path>" ${SSI_DIR}

eos_wfe_scripts/create_tape_drop_disk_replicas_ssi

+#!/bin/bash
+if test $# -ne 3; then
+  echo "Wrong number of command-line arguments"
+  echo "Usage: create_tape_drop_disk_replicas wf_tag file_path tape_fs_id"
+  exit -1
+fi
+
+export XRD_STREAMTIMEOUT=600     # increased from 60s
+export XRD_TIMEOUTRESOLUTION=600 # increased from 15s
+
+WF_TAG="$1"
+FILE_PATH="$2"
+TAPE_FS_ID="$3"
+
+LOG_DATE=`/usr/bin/date +%s | /usr/bin/tr '\n' ' ' ; /usr/bin/date`
+LOG_SCRIPT_NAME=`/usr/bin/basename $0`
+LOG_FILE="/var/log/eos/wfe/${WF_TAG}.log"
+
+# Creating tape replica
+echo "${LOG_DATE} ${LOG_SCRIPT_NAME} creating tape replica with fsid ${TAPE_FS_ID} for ${FILE_PATH}" >> ${LOG_FILE}
+OUTPUT=`/usr/bin/eos -r 0 0 file tag "${FILE_PATH}" +${TAPE_FS_ID} 2>&1`
+RESULT=$?
+if [ 0 -ne ${RESULT} ]; then
+  echo "${LOG_DATE} ${LOG_SCRIPT_NAME} failed to create tape replica with fsid ${TAPE_FS_ID} for ${FILE_PATH}: ${RESULT} ${OUTPUT}" >> ${LOG_FILE}
+  exit 1
+fi
+
+# Checking tape replica
+#echo "${LOG_DATE} ${LOG_SCRIPT_NAME} checking tape replica for ${FILE_PATH}" >> ${LOG_FILE}
+OUTPUT=`/usr/bin/eos -r 0 0 attr ls "${FILE_PATH}" 2>&1 | /usr/bin/grep -c '^sys.archiveFileId=\"'`
+RESULT=$?
+if [ 1 -gt ${OUTPUT} ]; then
+  echo "${LOG_DATE} ${LOG_SCRIPT_NAME} missing tape replica sys.archiveFileId for ${FILE_PATH}: ${RESULT} ${OUTPUT}" >> ${LOG_FILE}
+  exit 1
+fi
+OUTPUT=`/usr/bin/eos -r 0 0 ls -y "${FILE_PATH}" 2>&1 | /usr/bin/grep -c '^d.::t[123456789]'`
+RESULT=$?
+if [ 1 -gt ${OUTPUT} ]; then
+  echo "${LOG_DATE} ${LOG_SCRIPT_NAME} tape replica (expecting at least d?::t1) missing in EOS for ${FILE_PATH}: ${RESULT} ${OUTPUT}" >> ${LOG_FILE}
+  exit 1
+fi
+
+# Deleting disk replica
+for DISK_FSID in `/usr/bin/eos file info "${FILE_PATH}" -m | /usr/bin/sed s/\ /'\n'/g | /usr/bin/grep fsid | /usr/bin/sed s/fsid=// | /usr/bin/grep -v ${TAPE_FS_ID}`; do
+  echo "${LOG_DATE} ${LOG_SCRIPT_NAME} deleting disk replica with fsid ${DISK_FSID} for ${FILE_PATH}" >> ${LOG_FILE}
+  OUTPUT=`/usr/bin/eos -r 0 0 file drop "${FILE_PATH}" ${DISK_FSID} 2>&1`
+  RESULT=$?
+  if [ 0 -ne ${RESULT} ]; then
+    echo "${LOG_DATE} ${LOG_SCRIPT_NAME} failed to delete disk replica with fsid ${DISK_FSID} for ${FILE_PATH}: ${RESULT} ${OUTPUT}" >> ${LOG_FILE}
+    exit 1
+  fi
+done

eos_wfe_scripts/delete_archive_file_ssi

+#!/bin/bash
+
+EINVAL=22
+ECANCELED=125
+
+export XrdSecPROTOCOL=sss
+export XrdSecSSSKT=/etc/cta/cta-cli.sss.keytab   # This location is used on EOSCTATAPE
+export XrdSecSSSKT=/etc/ctafrontend_SSS_c.keytab # This location is used for CI
+
+export XRD_STREAMTIMEOUT=600     # increased from 60s
+export XRD_TIMEOUTRESOLUTION=600 # increased from 15s
+
+if test $# -ne 5; then
+  echo "Wrong number of command-line arguments"
+  echo "Usage: delete_archive_file wf_tag rusername rgroupname archive_file_id file_path"
+  exit ${EINVAL}
+fi
+
+CTA_BIN=/usr/bin/eoscta_stub
+
+WF_TAG="$1"
+RUSERNAME="$2"
+RGROUPNAME="$3"
+ARCHIVE_FILE_ID="$4"
+FILE_PATH="$5"
+
+LOG_DATE=`/usr/bin/date +%s | /usr/bin/tr '\n' ' ' ; /usr/bin/date`
+LOG_SCRIPT_NAME=`/usr/bin/basename $0`
+LOG_FILE="/var/log/eos/wfe/${WF_TAG}.log"
+
+if test UNDEF = ${ARCHIVE_FILE_ID}; then
+  echo "$LOG_DATE $LOG_SCRIPT_NAME ignoring deletion of non-existent tape archive file: rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} path=${FILE_PATH}" >> ${LOG_FILE}
+  exit 0
+fi
+
+if RESULT=`2>&1 ${CTA_BIN} deletearchive --user ${RUSERNAME} --group ${RGROUPNAME} --id ${ARCHIVE_FILE_ID}`; then
+  echo "$LOG_DATE $LOG_SCRIPT_NAME deleted tape archive file: rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} path=${FILE_PATH}" >> ${LOG_FILE}
+  exit 0
+else
+  echo "$LOG_DATE $LOG_SCRIPT_NAME failed to delete tape archive file: error=${RESULT} rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} path=${FILE_PATH}" >> ${LOG_FILE}
+  exit ${ECANCELED}
+fi

eos_wfe_scripts/retrieve_archive_file_ssi

+#!/bin/bash
+
+EINVAL=22
+ECANCELED=125
+
+export XrdSecPROTOCOL=sss
+export XrdSecSSSKT=/etc/cta/cta-cli.sss.keytab   # This location is used on EOSCTATAPE
+export XrdSecSSSKT=/etc/ctafrontend_SSS_c.keytab # This location is used for CI
+
+export XRD_STREAMTIMEOUT=600     # increased from 60s
+export XRD_TIMEOUTRESOLUTION=600 # increased from 15s
+
+if test $# -ne 9; then
+  echo "Wrong number of command-line arguments"
+  echo "Usage: retrieve_archive_file wf_tag rusername rgroupname archive_file_id turl disk_username disk_groupname metadata_base64 file_path"
+  exit ${EINVAL}
+fi
+
+CTA_BIN=/usr/bin/eoscta_stub
+
+WF_TAG="$1"
+RUSERNAME="$2"
+RGROUPNAME="$3"
+ARCHIVE_FILE_ID="$4"
+TURL="$5"
+DISK_USERNAME="$6"
+DISK_GROUPNAME="$7"
+METADATA_BASE64="$8"
+FILE_PATH="$9"
+
+DST_URL=${TURL}'\&eos.ruid=0&eos.rgid=0\&eos.injection=1\&eos.workflow=CTA_retrieve'
+
+LOG_DATE=`/usr/bin/date +%s | /usr/bin/tr '\n' ' ' ; /usr/bin/date`
+LOG_SCRIPT_NAME=`/usr/bin/basename $0`
+LOG_FILE="/var/log/eos/wfe/${WF_TAG}.log"
+
+if test UNDEF = ${ARCHIVE_FILE_ID}; then
+  echo "$LOG_DATE $LOG_SCRIPT_NAME cannot retrieve an archive file without the sys.archiveFileId attribute being set: rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} dsturl=${DST_URL} disk_username=${DISK_USERNAME} disk_groupname=${DISK_GROUPNAME} path=${FILE_PATH}" >> ${LOG_FILE}
+  exit ${ECANCELED}
+fi
+
+if ! RESULT=`unset XrdSecPROTOCOL XrdSecSSSKT ; /usr/bin/eos ls -y ${FILE_PATH} | /usr/bin/grep -s ^d0`; then
+  echo "$LOG_DATE $LOG_SCRIPT_NAME file already has at least one online replica on disk: rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} dsturl=${DST_URL} disk_username=${DISK_USERNAME} disk_groupname=${DISK_GROUPNAME} path=${FILE_PATH}" >> ${LOG_FILE}
+  exit 0
+fi
+
+if RESULT=`2>&1 ${CTA_BIN} retrieve --user ${RUSERNAME} --group ${RGROUPNAME} --id ${ARCHIVE_FILE_ID} --dsturl ${DST_URL} --diskfilepath ${FILE_PATH} --diskfileowner ${DISK_USERNAME} --diskfilegroup ${DISK_GROUPNAME} --recoveryblob:base64 ${METADATA_BASE64}`; then
+  echo "$LOG_DATE $LOG_SCRIPT_NAME queued retrieve request: rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} dsturl=${DST_URL} disk_username=${DISK_USERNAME} disk_groupname=${DISK_GROUPNAME} path=${FILE_PATH}" >> ${LOG_FILE}
+  exit 0
+else
+  echo "$LOG_DATE $LOG_SCRIPT_NAME failed to queue retrieve request: error=${RESULT} rusername=${RUSERNAME} rgroupname=${RGROUPNAME} archiveFileId=${ARCHIVE_FILE_ID} dsturl=${DST_URL} disk_username=${DISK_USERNAME} disk_groupname=${DISK_GROUPNAME} path=${FILE_PATH}" >> ${LOG_FILE}
+  exit ${ECANCELED}
+fi