initial commit

Prod5_LaPalma_AdvancedBaseline_NSB1x.list

+Prod5_LaPalma_AdvancedBaseline_NSB1x_electron_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_electron_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma-diffuse_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma-diffuse_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_proton_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_proton_South_20deg_DL0

Prod5_LaPalma_AdvancedBaseline_NSB5x.list

+Prod5_LaPalma_AdvancedBaseline_NSB5x_electron_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_electron_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma-diffuse_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma-diffuse_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_proton_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_proton_South_20deg_DL0

Prod5_Paranal_AdvancedBaseline_NSB1x.list

+Prod5_Paranal_AdvancedBaseline_NSB1x_electron_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_electron_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_gamma_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_gamma_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_proton_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_proton_South_20deg_DL0

Prod5_Paranal_AdvancedBaseline_NSB5x.list

+Prod5_Paranal_AdvancedBaseline_NSB5x_electron_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_electron_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_proton_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_proton_South_20deg_DL0

README.md

+
+
+## Prepare list of available data sets
+
+DIRAC command
+
+e.g.
+
+```
+cta-prod-show-dataset | grep Prod5_Paranal_AdvancedBaseline_NSB5x  | grep DL0 >| Prod5_Paranal_AdvancedBaseline_NSB5x.list
+```
+
+This should result in a list of availabe data sets, e.g.,
+
+```
+Prod5_Paranal_AdvancedBaseline_NSB5x_electron_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_electron_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_proton_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_proton_South_20deg_DL0
+```
+
+## Prepare list of data files
+
+Prepare lists of data files into the directory of the corresponding dataset.
+
+e.g. into Prod5_LaPalma_AdvancedBaseline_NSB1x:
+
+```
+./getFiles_and_checkifLocal.sh lists Prod5_LaPalma_AdvancedBaseline_NSB1x
+```
+
+This queries the CTA production DB and puts file lists into Prod5_LaPalma_AdvancedBaseline_NSB1x/<particle type>
+
+**Note the difference in naming: gamma-diffuse is renamed for Eventdisplay into gamma_cone**
+
+## Check which files are on DESY dCache, lustre, GRID
+
+The following commands queries both dCache and lustre for files.
+This command takes usually a long time as it has to query thousands of files
+
+
+```
+./getFiles_and_checkifLocal.sh check Prod5_LaPalma_AdvancedBaseline_NSB1x
+```
+
+

getFiles_and_checkifLocal.sh

+#!/bin/bash
+# Script to prepare and monitor download 
+#
+#
+
+if [ $# -lt 2 ]; then
+echo "
+Prepare and monitor prod5 download scripts
+
+./getFiles_and_checkifLocal.sh <lists/check> <production>
+"
+exit
+fi
+
+if [[ ! -e ${2}.list ]]; then
+   echo "Error: list of productions missing"
+fi
+
+FF=$(cat ${2}.list)
+
+PDIR=`pwd`
+
+mkdir -p ${2}
+
+for F in $FF
+do
+    if [[ ${F} == *"gamma-diffuse"* ]]; then
+        P="gamma_cone"
+    elif [[ ${F} == *"gamma"* ]]; then
+        P="gamma"
+    elif [[ ${F} == *"proton"* ]]; then
+        P="proton"
+    elif [[ ${F} == *"electron"* ]]; then
+        P="electron"
+    else
+        echo "unknown particle in $F"
+        continue
+    fi
+    mkdir -p ${2}/${P}
+    echo "READING $F"
+    cd ${PDIR}/${2}
+    if [[ ${1} == "lists" ]]; then
+       cta-prod-dump-dataset $F
+    elif [[ ${1} == "check" ]]; then
+        cd ${PDIR}
+       ./updateFileLists-DESY-SE.sh ${2} ${F} ${P}
+    fi
+    cd ${PDIR}
+done
+
+cd ${PDIR}
+
+exit

updateFileLists-DESY-SE.sh

+#!/bin/bash
+#
+# simple script to update list of files produced by dirac tools and
+# - prepare a list of all files which are on the DESY dCache
+# - prepare a list of all files which are not on the DESY dCache
+#
+
+if [ $# -lt 3 ]; then
+   echo "./updateFileLists-DESY-SE.sh <dset name> <file list> <particle type>"
+   echo
+   echo ".e.g for Paranal_proton_South_20deg_HB9 do: "
+   echo "./CTA.prepareDownloadLists-DIRAC.sh Paranal 20deg_HB9 <data directory>"
+   exit
+fi
+
+# data set
+FFN=${1}
+# file list
+FL=${2}
+# particle type
+P=${3}
+
+# dcache client
+export DCACHE_CLIENT_ACTIVE=1
+
+echo ${FFN}
+
+# data directories
+DDIR="${FFN}/${P}/"
+echo "    Data directory ${DDIR}"
+echo "    Filelist ${FL}"
+
+#####################
+# output file lists:
+
+# list of files on dCache
+FDC=${FFN}/${FL}.dCache.list
+rm -f $FDC
+touch $FDC
+# list of files not downloaded yet (one some GRID SE)
+FGR=${FFN}/${FL}.GRID.list
+rm -f $FGR
+touch $FGR
+# list of available files (dCache + lustre)
+FGA=${FFN}/${FL}.local.list
+rm -f $FGA
+touch $FGA
+
+# loop over all files in the list
+#        FILEL=`cat ${FFN}.GRID.list.back`
+FILEL=`cat ${FFN}/${FL}.list`
+for i in $FILEL
+do
+    OFIL=`basename $i`
+    if [ -e $DDIR/$OFIL ] && [ -s $DDIR/$OFIL ]
+    then
+       echo "FILE EXISTS: $DDIR/$OFIL"
+       echo $DDIR/$OFIL >> $FGA
+    else
+       # check if it is stored locally on the dcache
+       DC="/acs/grid/cta/$i"
+       if [ -e $DC ]
+       then
+          echo /acs/grid/cta/$i >> $FDC
+          echo /acs/grid/cta/$i >> $FGA
+       else
+          echo $i >> $FGR
+       fi
+    fi
+done
+
+exit