Commit a838dc8b authored by Gernot Maier's avatar Gernot Maier
Browse files

initial commit

parent c2d602b0
Prod5_LaPalma_AdvancedBaseline_NSB1x_electron_North_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB1x_electron_South_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma-diffuse_North_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma-diffuse_South_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma_North_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma_South_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB1x_proton_North_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB1x_proton_South_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB5x_electron_North_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB5x_electron_South_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma-diffuse_North_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma-diffuse_South_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma_North_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma_South_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB5x_proton_North_20deg_DL0
Prod5_LaPalma_AdvancedBaseline_NSB5x_proton_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB1x_electron_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB1x_electron_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB1x_gamma_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB1x_gamma_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB1x_proton_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB1x_proton_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_electron_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_electron_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_proton_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_proton_South_20deg_DL0
## Prepare list of available data sets
DIRAC command
e.g.
```
cta-prod-show-dataset | grep Prod5_Paranal_AdvancedBaseline_NSB5x | grep DL0 >| Prod5_Paranal_AdvancedBaseline_NSB5x.list
```
This should result in a list of availabe data sets, e.g.,
```
Prod5_Paranal_AdvancedBaseline_NSB5x_electron_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_electron_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_South_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_proton_North_20deg_DL0
Prod5_Paranal_AdvancedBaseline_NSB5x_proton_South_20deg_DL0
```
## Prepare list of data files
Prepare lists of data files into the directory of the corresponding dataset.
e.g. into Prod5_LaPalma_AdvancedBaseline_NSB1x:
```
./getFiles_and_checkifLocal.sh lists Prod5_LaPalma_AdvancedBaseline_NSB1x
```
This queries the CTA production DB and puts file lists into Prod5_LaPalma_AdvancedBaseline_NSB1x/<particle type>
**Note the difference in naming: gamma-diffuse is renamed for Eventdisplay into gamma_cone**
## Check which files are on DESY dCache, lustre, GRID
The following commands queries both dCache and lustre for files.
This command takes usually a long time as it has to query thousands of files
```
./getFiles_and_checkifLocal.sh check Prod5_LaPalma_AdvancedBaseline_NSB1x
```
#!/bin/bash
# Script to prepare and monitor download
#
#
if [ $# -lt 2 ]; then
echo "
Prepare and monitor prod5 download scripts
./getFiles_and_checkifLocal.sh <lists/check> <production>
"
exit
fi
if [[ ! -e ${2}.list ]]; then
echo "Error: list of productions missing"
fi
FF=$(cat ${2}.list)
PDIR=`pwd`
mkdir -p ${2}
for F in $FF
do
if [[ ${F} == *"gamma-diffuse"* ]]; then
P="gamma_cone"
elif [[ ${F} == *"gamma"* ]]; then
P="gamma"
elif [[ ${F} == *"proton"* ]]; then
P="proton"
elif [[ ${F} == *"electron"* ]]; then
P="electron"
else
echo "unknown particle in $F"
continue
fi
mkdir -p ${2}/${P}
echo "READING $F"
cd ${PDIR}/${2}
if [[ ${1} == "lists" ]]; then
cta-prod-dump-dataset $F
elif [[ ${1} == "check" ]]; then
cd ${PDIR}
./updateFileLists-DESY-SE.sh ${2} ${F} ${P}
fi
cd ${PDIR}
done
cd ${PDIR}
exit
#!/bin/bash
#
# simple script to update list of files produced by dirac tools and
# - prepare a list of all files which are on the DESY dCache
# - prepare a list of all files which are not on the DESY dCache
#
if [ $# -lt 3 ]; then
echo "./updateFileLists-DESY-SE.sh <dset name> <file list> <particle type>"
echo
echo ".e.g for Paranal_proton_South_20deg_HB9 do: "
echo "./CTA.prepareDownloadLists-DIRAC.sh Paranal 20deg_HB9 <data directory>"
exit
fi
# data set
FFN=${1}
# file list
FL=${2}
# particle type
P=${3}
# dcache client
export DCACHE_CLIENT_ACTIVE=1
echo ${FFN}
# data directories
DDIR="${FFN}/${P}/"
echo " Data directory ${DDIR}"
echo " Filelist ${FL}"
#####################
# output file lists:
# list of files on dCache
FDC=${FFN}/${FL}.dCache.list
rm -f $FDC
touch $FDC
# list of files not downloaded yet (one some GRID SE)
FGR=${FFN}/${FL}.GRID.list
rm -f $FGR
touch $FGR
# list of available files (dCache + lustre)
FGA=${FFN}/${FL}.local.list
rm -f $FGA
touch $FGA
# loop over all files in the list
# FILEL=`cat ${FFN}.GRID.list.back`
FILEL=`cat ${FFN}/${FL}.list`
for i in $FILEL
do
OFIL=`basename $i`
if [ -e $DDIR/$OFIL ] && [ -s $DDIR/$OFIL ]
then
echo "FILE EXISTS: $DDIR/$OFIL"
echo $DDIR/$OFIL >> $FGA
else
# check if it is stored locally on the dcache
DC="/acs/grid/cta/$i"
if [ -e $DC ]
then
echo /acs/grid/cta/$i >> $FDC
echo /acs/grid/cta/$i >> $FGA
else
echo $i >> $FGR
fi
fi
done
exit
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment