diff --git a/continuousintegration/docker/ctafrontend/cc7/Dockerfile b/continuousintegration/docker/ctafrontend/cc7/Dockerfile index 5d32796bce53121726bbe1dd2bc03ebc29282e57..551fa88a921e90eb8e0f38b0b325b06010d656ce 100644 --- a/continuousintegration/docker/ctafrontend/cc7/Dockerfile +++ b/continuousintegration/docker/ctafrontend/cc7/Dockerfile @@ -24,6 +24,8 @@ FROM gitlab-registry.cern.ch/linuxsupport/cc7-base # my environment variables +ENV ORCHESTRATIONDIR="continuousintegration/orchestration/pods" +ENV LOCALORCHESTRATIONDIR="/opt/ci" ENV BASEDIR="continuousintegration/docker/ctafrontend/cc7" ENV CTAREPODIR="/tmp/repo" @@ -32,7 +34,7 @@ ADD ${BASEDIR}/etc/yum.repos.d/* /etc/yum.repos.d/ RUN yum install -y yum-plugin-priorities createrepo epel-release # Create local repo for cta artifacts -RUN mkdir -p ${CTAREPODIR} +RUN mkdir -p ${CTAREPODIR} ${LOCALORCHESTRATIONDIR} # Add previously built rpms ADD build_rpm/RPM ${CTAREPODIR} # Populate local repository and enable it @@ -46,6 +48,9 @@ RUN yum -y install cgdb less psmisc lynx strace ltrace && yum clean all # Static configuration files ADD ${BASEDIR}/etc/xrootd/xrootd-cta.cfg /etc/xrootd/xrootd-cta.cfg +# Add orchestration init scripts locally +ADD ${ORCHESTRATIONDIR}/init ${LOCALORCHESTRATIONDIR} + # Docker image run setup ADD ${BASEDIR}/run.sh / ENTRYPOINT ["/run.sh"] diff --git a/continuousintegration/orchestration/pods/.pod-ctainit.yaml.swp b/continuousintegration/orchestration/pods/.pod-ctainit.yaml.swp new file mode 100644 index 0000000000000000000000000000000000000000..26fd35b91e14244635a01332c32c1bdd0c959eea Binary files /dev/null and b/continuousintegration/orchestration/pods/.pod-ctainit.yaml.swp differ diff --git a/continuousintegration/orchestration/pods/init/cli.sh b/continuousintegration/orchestration/pods/init/cli.sh new file mode 100755 index 0000000000000000000000000000000000000000..a827510a6b50c6fbc1b7b33705cde208816d8bb1 --- /dev/null +++ b/continuousintegration/orchestration/pods/init/cli.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +cat <<EOF > /etc/cta/cta-cli.conf +# The CTA frontend address in the form <FQDN>:<TCPPort> +# solved by kubernetes DNS server so KIS... +${frontend} +EOF + +# sleep forever but exit immediately when pod is deleted +exec /bin/bash -c "trap : TERM INT; sleep infinity & wait" diff --git a/continuousintegration/orchestration/pods/init/ctafrontend.sh b/continuousintegration/orchestration/pods/init/ctafrontend.sh new file mode 100755 index 0000000000000000000000000000000000000000..aa8496f9e038674d19821354680c160190ab7683 --- /dev/null +++ b/continuousintegration/orchestration/pods/init/ctafrontend.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +echo "ObjectStore BackendPath $objectstore" > /etc/cta/cta-frontend.conf +echo "Catalogue NumberOfConnections 1" >>/etc/cta/cta-frontend.conf + +echo ${catdb} >/etc/cta/cta_catalogue_db.conf + +useradd cta +# disable kerberos5 check for the admin privileges + sed -i -e "s/krb5/unix/" /usr/lib64/libXrdCtaOfs.so +runuser --shell='/bin/bash' --session-command='cd ~cta; xrootd -n cta -c /etc/xrootd/xrootd-cta.cfg -I v4' cta diff --git a/continuousintegration/orchestration/pods/init/init.sh b/continuousintegration/orchestration/pods/init/init.sh new file mode 100755 index 0000000000000000000000000000000000000000..5d8266eb93095e0ebae24c8c589096ea35e5719b --- /dev/null +++ b/continuousintegration/orchestration/pods/init/init.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +# copy library configuration from persistent volume to shared volume +mkdir -p /shared/${INSTANCE_NAME} +cp -f /library/config ${LIBRARY_CONFIG} + +echo "Using this configuration for library:" +cat ${LIBRARY_CONFIG} +. ${LIBRARY_CONFIG} + +rm -rf ${objectstore} +mkdir -p ${objectstore} + makeMinimalVFS ${objectstore} + chmod -R 777 ${objectstore} + +rm -rf ${catdbdir} +mkdir -p ${catdbdir} + sqlite3 ${catdbdir}/${catdbfile} < `rpm -ql cta-doc|grep sqlite` + chmod -R 777 ${catdbdir} + + +# library management +# BEWARE STORAGE SLOTS START @1 and DRIVE SLOTS START @0!! +echo "Labelling tapes using the first drive in ${LIBRARYNAME}: ${DRIVENAMES[0]} on /dev/${DRIVEDEVICES[0]}:" +for ((i=0; i<${#TAPES[@]}; i++)); do + vid=${TAPES[${i}]} + tapeslot=$((${i}+1)) # tape slot is 1 for tape[0] and so on... + + echo -n "${vid} in slot ${tapeslot} " + mtx -f /dev/${LIBRARYDEVICE} load ${tapeslot} 0 + cd /tmp + echo "VOL1${vid} CASTOR 3">label.file + dd if=label.file of=/dev/${DRIVEDEVICES[0]} bs=80 count=1 + mt -f /dev/${DRIVEDEVICES[0]} rewind + mtx -f /dev/${LIBRARYDEVICE} unload ${tapeslot} 0 + echo "OK" +done + +exit 0 diff --git a/continuousintegration/orchestration/pods/init/init_pod.sh b/continuousintegration/orchestration/pods/init/init_pod.sh new file mode 100755 index 0000000000000000000000000000000000000000..24539f423d5ac5cb482c2cfe9978c3ec7a7693dc --- /dev/null +++ b/continuousintegration/orchestration/pods/init/init_pod.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +echo -n "Fixing reverse DNS for $(hostname): " +sed -i -c "s/^\($(hostname -i)\)\s\+.*$/\1 $(hostname -s).$(grep search /etc/resolv.conf | cut -d\ -f2) $(hostname -s)/" /etc/hosts +echo "DONE" + +echo -n "Yum should resolve names using IPv4 DNS: " +echo "ip_resolve=IPv4" >> /etc/yum.conf +echo "DONE" diff --git a/continuousintegration/orchestration/pods/init/mgm.sh b/continuousintegration/orchestration/pods/init/mgm.sh new file mode 100755 index 0000000000000000000000000000000000000000..1d60f9bafaa3c870eafd5a9a0b96b5a60e92a74c --- /dev/null +++ b/continuousintegration/orchestration/pods/init/mgm.sh @@ -0,0 +1,110 @@ +#!/bin/sh + +# initialize pod +/shared/bin/init_pod.sh + +eoshost=`hostname -f` +eosInstanceName=`hostname -s` + +TAPE_FS_ID=65535 +CTA_BIN=/usr/bin/cta +CTA_KT=/etc/eoscta-cli.keytab +CTA_PROC_DIR=/eos/${eosInstanceName}/proc/cta +CTA_WF_DIR=${CTA_PROC_DIR}/workflow +CTA_TEST_DIR=/eos/${eosInstanceName}/cta + +# prepare CTA cli commands environment + echo ${frontend} > /etc/cta/cta-cli.conf + +# setup eos host and instance name + sed -i -e "s/DUMMY_HOST_TO_REPLACE/${eoshost}/" /etc/sysconfig/eos + sed -i -e "s/DUMMY_INSTANCE_TO_REPLACE/${eosInstanceName}/" /etc/sysconfig/eos + sed -i -e "s/DUMMY_HOST_TO_REPLACE/${eoshost}/" /etc/xrd.cf.mgm + sed -i -e "s/DUMMY_INSTANCE_TO_REPLACE/${eosInstanceName}/" /etc/xrd.cf.mgm + sed -i -e "s/DUMMY_HOST_TO_REPLACE/${eoshost}/" /etc/xrd.cf.mq + sed -i -e "s/DUMMY_HOST_TO_REPLACE/${eoshost}/" /etc/xrd.cf.fst + +# prepare eos startup + # skip systemd for eos initscripts + export SYSTEMCTL_SKIP_REDIRECT=1 + echo y | xrdsssadmin -k ${eosInstanceName} -u daemon -g daemon add /etc/eos.keytab + chmod 400 /etc/eos.keytab + chown daemon:daemon /etc/eos.keytab + mkdir -p /run/lock/subsys + mkdir -p /var/eos/config/${eoshost} + chown daemon:root /var/eos/config/${eoshost} + touch /var/eos/config/${eoshost}/default.eoscf + chown daemon:daemon /var/eos/config/${eoshost}/default.eoscf + + #/etc/init.d/eos master mgm + #/etc/init.d/eos master mq + touch /var/eos/eos.mq.master + touch /var/eos/eos.mgm.rw + echo "Configured mq mgm on localhost as master" + + source /etc/sysconfig/eos + + mkdir -p /fst + chown daemon:daemon /fst/ + +# start and setup eos for xrdcp to the ${CTA_TEST_DIR} + #/etc/init.d/eos start + /usr/bin/xrootd -n mq -c /etc/xrd.cf.mq -l /var/log/eos/xrdlog.mq -b -Rdaemon + /usr/bin/xrootd -n mgm -c /etc/xrd.cf.mgm -m -l /var/log/eos/xrdlog.mgm -b -Rdaemon + /usr/bin/xrootd -n fst -c /etc/xrd.cf.fst -l /var/log/eos/xrdlog.fst -b -Rdaemon + + eos vid enable sss + eos vid enable unix + EOS_MGM_URL="root://${eoshost}" eosfstregister -r /fst default:1 + + eos node set ${eoshost} on + eos space set default on + eos attr -r set default=replica /eos + eos attr -r set sys.forced.nstripes=1 /eos + + eos fs add -m ${TAPE_FS_ID} tape localhost:1234 /does_not_exist tape + eos mkdir ${CTA_PROC_DIR} + eos mkdir ${CTA_WF_DIR} + eos attr set CTA_TapeFsId=${TAPE_FS_ID} ${CTA_WF_DIR} + + eos mkdir ${CTA_TEST_DIR} + eos chmod 777 ${CTA_TEST_DIR} + eos attr set CTA_StorageClass=ctaStorageClass ${CTA_TEST_DIR} + + # hack before it is fixed in EOS + TAPE_FS_ID_TOSET=`eos attr ls ${CTA_WF_DIR} | grep CTA_TapeFsId= | tr '"' ' ' | cut -d ' ' -f 2` + eos attr set CTA_TapeFsId=${TAPE_FS_ID_TOSET} ${CTA_TEST_DIR} + + # Link the attributes of CTA worklow directory to the test directory + eos attr link ${CTA_WF_DIR} ${CTA_TEST_DIR} + +# test EOS +# eos slow behind us and we need to give it time to be ready +# 5 secs is not enough + sleep 10 + eos -b node ls + xrdcp /etc/group root://${eoshost}:/${CTA_TEST_DIR}/testFile + +# prepare EOS workflow + eos space config default space.wfe=on + + # ATTENTION + # for sss authorisation unix has to be replaced by sss + + # Set the worfklow rule for archiving files to tape + eos attr set sys.workflow.closew.default="bash:shell:cta XrdSecPROTOCOL=unix XrdSecSSSKT=${CTA_KT} ${CTA_BIN} archive --user <eos::wfe::rusername> --group <eos::wfe::rgroupname> --diskid <eos::wfe::fid> --instance eoscta --srcurl <eos::wfe::turl> --size <eos::wfe::size> --checksumtype <eos::wfe::checksumtype> --checksumvalue <eos::wfe::checksum> --storageclass <eos::wfe::cxattr:CTA_StorageClass> --diskfilepath <eos::wfe::path> --diskfileowner <eos::wfe::username> --diskfilegroup <eos::wfe::groupname> --recoveryblob:base64 <eos::wfe::base64:metadata> --diskpool default --throughput 10000 --stderr" ${CTA_WF_DIR} + # Set the worflow rule for creating tape file replicas in the EOS namespace + eos attr set sys.workflow.archived.default="bash:shell:cta eos file tag <eos::wfe::path> +<eos::wfe::cxattr:CTA_TapeFsId>" ${CTA_WF_DIR} + # Set the worfklow rule for retrieving file from tape + eos attr set sys.workflow.prepare.default="bash:shell:cta XrdSecPROTOCOL=unix XrdSecSSSKT=${CTA_KT} ${CTA_BIN} retrieve --user <eos::wfe::rusername> --group <eos::wfe::rgroupname> --id <eos::wfe::fxattr:sys.archiveFileId> --dsturl '<eos::wfe::turl>\&eos.injection=1\&eos.workflow=CTA_retrieve' --diskfilepath <eos::wfe::path> --diskfileowner <eos::wfe::username> --diskfilegroup <eos::wfe::groupname> --recoveryblob:base64 <eos::wfe::base64:metadata> --diskpool default --throughput 10001 --stderr" ${CTA_WF_DIR} + # Set the workflow rule for the closew event of the CTA_retrieve workflow. + # Using the CTA_retrieve workflow will prevent the default workflow from + # receiving the closew event. Triggering the default workflow in this way would + # haved causes the unwanted action of copying the disk file to tape again. + # The action of the CTA_retrieve workflow when triggered by the closew event is + # to set the CTA_retrieved_timestamp attribute. + eos attr set sys.workflow.closew.CTA_retrieve="bash:shell:cta eos attr set 'CTA_retrieved_timestamp=\"\`date\`\"' <eos::wfe::path>" ${CTA_WF_DIR} + +touch /shared/${INSTANCE_NAME}/mgm_ready + +/bin/bash diff --git a/continuousintegration/orchestration/pods/init/rmcd.sh b/continuousintegration/orchestration/pods/init/rmcd.sh new file mode 100755 index 0000000000000000000000000000000000000000..550e364c19809dc2bac8c0c16a921716805d9686 --- /dev/null +++ b/continuousintegration/orchestration/pods/init/rmcd.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +/shared/bin/init_pod.sh + +# source library configuration file +echo "Using this configuration for library:" +cat ${LIBRARY_CONFIG} + +. ${LIBRARY_CONFIG} + +# to get rmcd logs to stdout +mkfifo /var/log/castor/rmcd_legacy.log +for ((;;)); do cat </var/log/castor/rmcd_legacy.log; done & +disown + +ln -s /dev/${LIBRARYDEVICE} /dev/smc +/usr/bin/rmcd -f /dev/smc diff --git a/continuousintegration/orchestration/pods/init/taped.sh b/continuousintegration/orchestration/pods/init/taped.sh new file mode 100755 index 0000000000000000000000000000000000000000..df4160acaeaa87bd71b548759427258f33bb2377 --- /dev/null +++ b/continuousintegration/orchestration/pods/init/taped.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +/shared/bin/init_pod.sh + +# source library configuration file +echo "Using this configuration for library:" +cat ${LIBRARY_CONFIG} + +. ${LIBRARY_CONFIG} + + +ln -s /dev/${LIBRARYDEVICE} /dev/smc +#/usr/bin/rmcd -f /dev/smc& + +mkdir -p /etc/castor + +tpconfig="${DRIVENAMES[${driveslot}]} ${LIBRARYNAME} /dev/${DRIVEDEVICES[${driveslot}]} smc0" + +# cta-tapserverd setup +# to be drop later + echo "${tpconfig}" > /etc/castor/TPCONFIG + echo "TapeServer ObjectStoreBackendPath $objectstore" >/etc/castor/castor.conf + echo "TapeServer BufSize 5242880" >>/etc/castor/castor.conf + echo "TapeServer NbBufs 10" >>/etc/castor/castor.conf + echo "TapeServer EOSRemoteHostAndPort ${eoshost}" >>/etc/castor/castor.conf + +# cta-taped setup + echo "taped BufferCount 10" > /etc/cta/cta.conf + echo "general ObjectStoreURL ${objectstore}" >> /etc/cta/cta.conf + echo "${tpconfig}" > /etc/cta/TPCONFIG + +echo ${catdb} >/etc/cta/cta_catalogue_db.conf + +/bin/cta-taped --stdout --foreground diff --git a/continuousintegration/orchestration/pods/pod-ctainit.yaml b/continuousintegration/orchestration/pods/pod-ctainit.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab619b82bd626d7a5bfb2dbb4111dfff0c46579d --- /dev/null +++ b/continuousintegration/orchestration/pods/pod-ctainit.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: Pod +metadata: + name: init + labels: + k8s-app: cta-init +spec: + restartPolicy: Never + containers: + - name: ctainit + image: gitlab-registry.cern.ch/cta/ctageneric:%CTA_BUILD_ID + stdin: true + env: + - name: MY_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: INSTANCE_NAME + value: "$(MY_NAMESPACE)" + - name: objectstore + value: "/shared/$(INSTANCE_NAME)/objectstore" + - name: catdbdir + value: "/shared/$(INSTANCE_NAME)/catdb" + - name: catdbfile + value: "catdb" + - name: LIBRARY_CONFIG + value: "/shared/$(INSTANCE_NAME)/libraryconfig" + command: ['/opt/ci/init/init.sh'] + args: ["none"] + volumeMounts: + - mountPath: /shared + name: shared + - mountPath: /library + name: mylibrary + securityContext: + privileged: true + + volumes: + - name: shared + hostPath: + path: /opt/cta + - name: mylibrary + persistentVolumeClaim: + claimName: claimlibrary