Commit d74b39c7 authored by Gernot Maier's avatar Gernot Maier
Browse files

initial

parents
#!/bin/bash
#
# simple script which allows to download
# all particle types simulatenously
#
# Prod5 NSB 1x
DD="Prod5_Paranal_AdvancedBaseline_NSB1x"
# Prod4b SST production
DD="Prod4b_Paranal"
# Prod4b SST production
DD="Prod3b_Paranal"
# Prod5 NSB 1x
DD="Prod5b_LaPalma_AdvancedBaseline_NSB1x"
# Prod3B SCT DL1
DD="Prod5b_Paranal_AdvancedBaseline_NSB1x"
DD="Prod5_Paranal_AdvancedBaseline_NSB5x"
DD="Prod5b_Paranal_AdvancedBaseline_NSB1x_40deg_DL1"
ZE="40deg"
DL="DL1"
SCT="FALSE"
if [[ $DD == *"Prod3b_Paranal"* ]]; then
DL="HB9"
DL="HB9_SCT_DL1"
SCT="TRUE"
fi
TMDPIR="tmp_log2"
mkdir -p ${TMPDIR}
for P in proton gamma_cone gamma_onSource electron
do
if [[ ${P} == "gamma_cone" ]]; then
PP="gamma-diffuse"
elif [[ ${P} == "gamma_onSource" ]]; then
PP="gamma"
else
PP=${P}
fi
for A in North South
do
rm -f ${TMPDIR}/${P}_${A}.log
# Prod4b SST production
if [[ $DD = *"Prod4"* ]]; then
if [[ $A == "South" ]]; then
continue
fi
for S in "sst-astri" "sst-astri+chec-s"
do
FFLIST="${DD}_${ZE}/${DD}_${PP}_${A}_${ZE}_SSTOnly_${S}_${DL}.GRID.list"
./getRawFilesFromGRID-DIRAC.sh ${FFLIST} ${DD}/${P}/ 100 &> ${TMPDIR}/${P}_${A}_${S}.log &
done
# Prod3b SCT DL1
elif [[ $DD = *"Prod3b"* ]] && [[ $DD = *"SCT_DL1"* ]]; then
FLIST="${DD}/Prod3_Paranal_${PP}_${A}_${ZE}_${DL}.GRID.list"
if [[ ! -e ${FLIST} ]]; then
echo "$FLIST not found"
exit
fi
echo "...found ${FLIST}"
./getRawFilesFromGRID-DIRAC.sh ${FLIST} ${DD}/${P}/ 100 &> ${TMPDIR}/${P}_${A}_${S}.log &
# all other productions
else
FLIST="${DD}_${ZE}_${DL}/${DD}_${PP}_${A}_${ZE}_${DL}.GRID.list"
FLIST="${DD}_${DL}/${DD}_${PP}_${A}_${ZE}_${DL}.GRID.list"
# prod3b needs some special treatment
if [[ ! -e ${FLIST} ]]; then
echo "File list not found; trying prod3b naming ($FLIST); trying prod3b lists:"
FLIST="${DD}_${ZE}_${DL}/Paranal_${PP}_${A}_${ZE}_${DL}.GRID.list"
if [[ ! -e ${FLIST} ]]; then
echo "$FLIST not found"
exit
fi
echo "...found ${FLIST}"
fi
echo "Reading file list $FLIST"
./getRawFilesFromGRID-DIRAC.sh $FLIST ${DD}/${P}/ 100 ${SCT} &> ${TMPDIR}/${P}_${A}.log &
fi
sleep 10
done
done
#!/bin/bash
# Script to prepare and monitor download
#
#
if [ $# -lt 2 ]; then
echo "
Prepare and monitor prod5 download scripts
./getFiles_and_checkifLocal.sh <lists/check> <production>
"
exit
fi
if [[ ! -e ${2}.list ]]; then
echo "Error: list of productions missing"
fi
FF=$(cat ${2}.list)
PDIR=`pwd`
mkdir -p ${2}
for F in $FF
do
if [[ ${F} == *"gamma-diffuse"* ]]; then
P="gamma_cone"
elif [[ ${F} == *"gamma"* ]]; then
P="gamma_onSource"
elif [[ ${F} == *"proton"* ]]; then
P="proton"
elif [[ ${F} == *"electron"* ]]; then
P="electron"
else
echo "unknown particle in $F"
continue
fi
mkdir -p ${2}/${P}
echo "READING $F"
cd ${PDIR}/${2}
if [[ ${1} == "lists" ]]; then
cta-prod-dump-dataset $F
elif [[ ${1} == "check" ]]; then
cd ${PDIR}
./updateFileLists-DESY-SE.sh ${2} ${F} ${P}
fi
cd ${PDIR}
done
cd ${PDIR}
exit
#!/bin/bash
#
# simple script to download raw files from the GRID
# using DIRAC tools
# (adjusted to DESY environment)
#
#
if [ $# -lt 3 ]; then
echo "
./getRawFilesFromGRID-DIRAC.sh <run list> <target directory> <max files (100)> [SCT cut=TRUE]
download using DIRAC tools
(adjusted to DESY environment)
"
exit
fi
echo "RUNLIST ${1}"
echo "Target directory: ${2}"
echo "Max file: ${3}"
[[ "$4" ]] && SCT=$4 || SCT="FALSE"
PDIR=`pwd`
if [ -e ${2} ]
then
mkdir -p ${2}
fi
if [ ! -e ${1} ]; then
echo "error: file list not found: ${1}"
exit
fi
# temporary directory for file lists
FFN=`basename $1`
mkdir -p ${2}/tmplists
# loop over all files in the list
NTMPLIST=`wc -l $1 | awk '{print $1}'`
FILEN=${3}
for ((l = 1; l < $NTMPLIST; l+=$FILEN ))
do
# create file lists with $FILEN files each
let "k = $l + $FILEN - 1"
let "z = $z + 1"
LLIST=$2/tmplists/$FFN.tmplist.d.$z.list
echo $LLIST
# optionally download only SCT files
if [[ $SCT = "TRUE" ]]; then
sed -n "$l,$k p" $1 | grep SCT > $LLIST
else
sed -n "$l,$k p" $1 > $LLIST
fi
# check if files are on disk
FF=`cat $LLIST`
LFIST=$FFN.tmplist.f.$z.list
touch $2/tmplists/$LFIST
for F in $FF
do
FIL=`basename $F`
if [ ! -e $2/$FIL ]
then
echo $F >> $2/tmplists/$LFIST
fi
done
NWC=`wc -l $2/tmplists/$LFIST | awk {'print $1'}`
echo $NWC
if [ "$NWC" -gt 0 ]
then
# run dirac get files
echo "Downloading $NWC files"
cd ${2}
if [[ -e tmplists/$LFIST ]]; then
dirac-dms-get-file tmplists/$LFIST
else
echo "File list not found: tmplists/$LFIST"
fi
fi
cd ${PDIR}
done
cd ${PDIR}
exit
#!/bin/bash
#
# simple script to update list of files produced by dirac tools and
# - prepare a list of all files which are on the DESY dCache
# - prepare a list of all files which are not on the DESY dCache
#
if [ $# -lt 3 ]; then
echo "./updateFileLists-DESY-SE.sh <dset name> <file list> <particle type>"
echo
echo ".e.g for Paranal_proton_South_20deg_HB9 do: "
echo "./CTA.prepareDownloadLists-DIRAC.sh Paranal 20deg_HB9 <data directory>"
exit
fi
# data set
FFN=${1}
# file list
FL=${2}
# particle type
P=${3}
# dcache client
export DCACHE_CLIENT_ACTIVE=1
echo ${FFN}
# data directories
DDIR="${FFN}/${P}/"
echo " Data directory ${DDIR}"
echo " Filelist ${FL}"
#####################
# output file lists:
# list of files on dCache
FDC=${FFN}/${FL}.dCache.list
rm -f $FDC
touch $FDC
# list of files not downloaded yet (one some GRID SE)
FGR=${FFN}/${FL}.GRID.list
rm -f $FGR
touch $FGR
# list of available files (dCache + lustre)
FGA=${FFN}/${FL}.local.list
rm -f $FGA
touch $FGA
# loop over all files in the list
# FILEL=`cat ${FFN}.GRID.list.back`
FILEL=`cat ${FFN}/${FL}.list`
for i in $FILEL
do
OFIL=`basename $i`
if [ -e $DDIR/$OFIL ] && [ -s $DDIR/$OFIL ]
then
echo "FILE EXISTS: $DDIR/$OFIL"
# full path
FFGA=$(readlink -f $DDIR/$OFIL)
echo ${FFGA} >> $FGA
else
# check if it is stored locally on the dcache
DC="/acs/grid/cta/$i"
if [ -e $DC ]
then
echo /acs/grid/cta/$i >> $FDC
echo /acs/grid/cta/$i >> $FGA
else
echo $i >> $FGR
fi
fi
done
exit
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment