initial

LICENSE

+BSD 3-Clause License
+
+Copyright (c) 2020, GernotMaier
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Prod3b_Paranal_20deg_HB9.list

+Paranal_electron_North_20deg_HB9
+Paranal_electron_South_20deg_HB9
+Paranal_gamma-diffuse_North_20deg_HB9
+Paranal_gamma-diffuse_South_20deg_HB9
+Paranal_gamma_North_20deg_HB9
+Paranal_gamma_South_20deg_HB9
+Paranal_proton_North_20deg_HB9
+Paranal_proton_South_20deg_HB9

Prod3b_Paranal_20deg_HB9_SCT_DL1.list

+Prod3_Paranal_electron_North_20deg_HB9_SCT_DL1
+Prod3_Paranal_electron_South_20deg_HB9_SCT_DL1
+Prod3_Paranal_gamma-diffuse_North_20deg_HB9_SCT_DL1
+Prod3_Paranal_gamma-diffuse_South_20deg_HB9_SCT_DL1
+Prod3_Paranal_gamma_North_20deg_HB9_SCT_DL1
+Prod3_Paranal_gamma_South_20deg_HB9_SCT_DL1
+Prod3_Paranal_proton_North_20deg_HB9_SCT_DL1
+Prod3_Paranal_proton_South_20deg_HB9_SCT_DL1

Prod4b_Paranal_20deg.list

+Prod4b_Paranal_electron_North_20deg_SSTOnly_sst-astri+chec-s_DL0
+Prod4b_Paranal_electron_North_20deg_SSTOnly_sst-astri_DL0
+Prod4b_Paranal_gamma-diffuse_North_20deg_SSTOnly_sst-astri+chec-s_DL0
+Prod4b_Paranal_gamma-diffuse_North_20deg_SSTOnly_sst-astri_DL0
+Prod4b_Paranal_gamma_North_20deg_SSTOnly_sst-astri+chec-s_DL0
+Prod4b_Paranal_gamma_North_20deg_SSTOnly_sst-astri_DL0
+Prod4b_Paranal_proton_North_20deg_SSTOnly_sst-astri+chec-s_DL0
+Prod4b_Paranal_proton_North_20deg_SSTOnly_sst-astri_DL0

Prod5_LaPalma_AdvancedBaseline_NSB1x.list

+Prod5_LaPalma_AdvancedBaseline_NSB1x_electron_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_electron_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma-diffuse_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma-diffuse_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_proton_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB1x_proton_South_20deg_DL0

Prod5_LaPalma_AdvancedBaseline_NSB5x.list

+Prod5_LaPalma_AdvancedBaseline_NSB5x_electron_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_electron_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma-diffuse_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma-diffuse_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_gamma_South_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_proton_North_20deg_DL0
+Prod5_LaPalma_AdvancedBaseline_NSB5x_proton_South_20deg_DL0

Prod5_Paranal_AdvancedBaseline_NSB1x.list

+Prod5_Paranal_AdvancedBaseline_NSB1x_electron_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_electron_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_gamma_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_gamma_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_proton_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB1x_proton_South_20deg_DL0

Prod5_Paranal_AdvancedBaseline_NSB5x.list

+Prod5_Paranal_AdvancedBaseline_NSB5x_electron_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_electron_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_proton_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_proton_South_20deg_DL0

Prod5_Paranal_AdvancedBaseline_NSB5x_DL1.list

+Prod5_Paranal_AdvancedBaseline_NSB5x_electron_North_20deg_DL1
+Prod5_Paranal_AdvancedBaseline_NSB5x_electron_South_20deg_DL1
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_North_20deg_DL1
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_South_20deg_DL1
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_North_20deg_DL1
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_South_20deg_DL1
+Prod5_Paranal_AdvancedBaseline_NSB5x_proton_North_20deg_DL1
+Prod5_Paranal_AdvancedBaseline_NSB5x_proton_South_20deg_DL1

Prod5b_LaPalma_AdvancedBaseline_NSB1x.list

+Prod5b_LaPalma_AdvancedBaseline_NSB1x_gamma-diffuse_North_20deg_DL0
+Prod5b_LaPalma_AdvancedBaseline_NSB1x_gamma-diffuse_South_20deg_DL0
+Prod5b_LaPalma_AdvancedBaseline_NSB1x_proton_North_20deg_DL0
+Prod5b_LaPalma_AdvancedBaseline_NSB1x_proton_South_20deg_DL0
+Prod5b_LaPalma_AdvancedBaseline_NSB1x_gamma_North_20deg_DL0
+Prod5b_LaPalma_AdvancedBaseline_NSB1x_gamma_South_20deg_DL0
+Prod5b_LaPalma_AdvancedBaseline_NSB1x_electron_North_20deg_DL0
+Prod5b_LaPalma_AdvancedBaseline_NSB1x_electron_South_20deg_DL0

Prod5b_Paranal_AdvancedBaseline_NSB1x_40deg_DL1.list

+Prod5b_Paranal_AdvancedBaseline_NSB1x_electron_North_40deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_electron_South_40deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_North_40deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_South_40deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma_North_40deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma_South_40deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_proton_North_40deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_proton_South_40deg_DL1

Prod5b_Paranal_AdvancedBaseline_NSB1x_60deg_DL0.list

+Prod5b_Paranal_AdvancedBaseline_NSB1x_electron_North_60deg_DL0
+Prod5b_Paranal_AdvancedBaseline_NSB1x_electron_South_60deg_DL0
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_North_60deg_DL0
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_South_60deg_DL0
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma_North_60deg_DL0
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma_South_60deg_DL0
+Prod5b_Paranal_AdvancedBaseline_NSB1x_proton_North_60deg_DL0
+Prod5b_Paranal_AdvancedBaseline_NSB1x_proton_South_60deg_DL0

Prod5b_Paranal_AdvancedBaseline_NSB1x_60deg_DL1.list

+Prod5b_Paranal_AdvancedBaseline_NSB1x_electron_North_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_electron_South_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_North_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma-diffuse_South_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma_North_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_gamma_South_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_proton_North_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB1x_proton_South_60deg_DL1

Prod5b_Paranal_AdvancedBaseline_NSB5x_60deg_DL1.list

+Prod5b_Paranal_AdvancedBaseline_NSB5x_electron_North_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB5x_electron_South_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_North_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_South_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB5x_gamma_North_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB5x_gamma_South_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB5x_proton_North_60deg_DL1
+Prod5b_Paranal_AdvancedBaseline_NSB5x_proton_South_60deg_DL1

README.md

+# Introduction
+
+Some simple script to prepare and steer file lists from CTA prodx production and compare it with locally (DESY)
+available files
+
+# Usage
+
+## Prepare list of available data sets
+
+uses DIRAC command
+
+e.g.
+
+```
+cta-prod-show-dataset | grep Prod5_Paranal_AdvancedBaseline_NSB5x  | grep DL0 >| Prod5_Paranal_AdvancedBaseline_NSB5x.list
+```
+
+This should result in a list of availabe data sets, e.g.,
+
+```
+Prod5_Paranal_AdvancedBaseline_NSB5x_electron_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_electron_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma-diffuse_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_gamma_South_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_proton_North_20deg_DL0
+Prod5_Paranal_AdvancedBaseline_NSB5x_proton_South_20deg_DL0
+```
+
+## Prepare list of data files
+
+Prepare lists of data files into the directory of the corresponding dataset.
+
+e.g. into Prod5_LaPalma_AdvancedBaseline_NSB1x:
+
+```
+./getFiles_and_checkifLocal.sh lists Prod5_LaPalma_AdvancedBaseline_NSB1x
+```
+
+This queries the CTA production DB and puts file lists into Prod5_LaPalma_AdvancedBaseline_NSB1x/<particle type>
+
+**Note the difference in naming: gamma-diffuse is renamed for Eventdisplay into gamma_cone**
+
+## Check which files are on DESY dCache, lustre, GRID
+
+The following commands queries both dCache and lustre for files.
+This command takes usually a long time as it has to query thousands of files.
+
+```
+./getFiles_and_checkifLocal.sh check Prod5_LaPalma_AdvancedBaseline_NSB1x
+```
+
+This will prepare the following files for each particle type, and azimuth direction:
+
+```
+Prod5_LaPalma_AdvancedBaseline_NSB1x_electron_North_20deg_DL0.dCache.list
+Prod5_LaPalma_AdvancedBaseline_NSB1x_electron_North_20deg_DL0.GRID.list
+Prod5_LaPalma_AdvancedBaseline_NSB1x_electron_North_20deg_DL0.list
+Prod5_LaPalma_AdvancedBaseline_NSB1x_electron_North_20deg_DL0.local.list
+```
+
+## Downloading files
+
+uses DIRAC command
+
+```
+./getRawFilesFromGRID-DIRAC.sh Prod5_LaPalma_AdvancedBaseline_NSB1x/Prod5_LaPalma_AdvancedBaseline_NSB1x_gamma-diffuse_North_20deg_DL0.GRID.list Prod5_LaPalma_AdvancedBaseline_NSB1x/gamma_cone 100
+```
+
+Start downloading in parallel for all particle types (needs adjustment for data set and data level):
+
+```
+./download_paranal.sh
+```
+
+(this obviously takes quite a while)
+
+
+## Copy file lists into productions directories
+
+Rename list to be readable by Eventdisplay scripts
+
+e.g.,
+```
+./copy_file_lists.sh Prod5_LaPalma_AdvancedBaseline_NSB1x $CTA_USER_DATA_DIR/analysis/AnalysisData/FileList_prod5/prod5-LaPalma-20deg/
+```
+Should copy all *local* lists from ./Prod5_LaPalma_AdvancedBaseline_NSB1x/ and rename them to:
+```
+electron_180deg.list
+electron.list
+gamma_cone_0deg.list
+gamma_cone_180deg.list
+gamma_cone.list
+gamma_onSource_0deg.list
+gamma_onSource_180deg.list
+gamma_onSource.list
+proton_0deg.list
+proton_180deg.list
+proton.list
+```
+
+# DL1 files 
+
+## Download DL1 tar packages from GRID
+
+- file lists must be prepared for data sets for DL1 (e.g., Prod5_LaPalma_AdvancedBaseline_NSB1x_DL1)
+- download_parallel.sh must be modified to DL1
+
+## Copy DL1 root files to analysis directory
+
+### Downloaded tar files
+
+Use script *copy_file_lists.sh*, which requires adjustments at the top:
+
+- target directory for analysis
+- list of array types
+- local DL1 data directory
+
+```
+./copy_DL1files.sh copy
+```
+
+Note: will take a while (many files)
+
+### Files from dCache
+
+Use script *copy_DL1files_from_dCache.sh*, which requires adjustments at the top (see previous entry for *copy_file_lists.sh*).
+
+```
+./copy_DL1files_from_dCache.sh
+```
+
+

copy_DL1files.sh

+#!/bin/bash
+#
+# copy DL1 files from tar files to target directory
+# 
+# ./copy_DL1files.sh
+#
+
+if [ $# -eq 0 ]; then
+   echo "./copy_DL1files.sh copy"
+   echo
+   echo "note the hardwired values"
+fi
+
+# hardwired values
+DSET="Prod5_LaPalma_AdvancedBaseline_NSB1x_DL1"
+PLIST="gamma_onSource gamma_cone proton"
+PLIST="gamma_onSource gamma_cone proton electron"
+# targetdir
+OSET="prod5-LaPalma-20deg-EVNDISP"
+########################
+# list of arrays
+# South
+# declare -a ARRAY=("BL-0LSTs15MSTs50SSTs-MSTF" "BL-0LSTs15MSTs50SSTs-MSTN" "BL-4LSTs25MSTs70SSTs-MSTF" "BL-4LSTs25MSTs70SSTs-MSTN" )
+# North
+declare -a ARRAY=("BL-4LSTs05MSTs-MSTF" "BL-4LSTs09MSTs-MSTF" "BL-4LSTs15MSTs-MSTF" ) 
+declare -a ARRAY=("BL-4LSTs15MSTs-MSTN" )
+
+for P in $PLIST
+do
+   for A in "${ARRAY[@]}"
+   do
+       # files are copied here
+       ODIR=${CTA_USER_DATA_DIR}/analysis/AnalysisData/${OSET}/N.${A}/EVNDISP/$P
+       echo $ODIR
+       mkdir -p ${ODIR}
+       find ${DSET}/$P -name "*.tar.gz" -print -exec tar --overwrite -C ${ODIR} -xvzf {} --wildcards "*${A}*" \;
+   done
+done

copy_DL1files_from_dCache.sh

+#!/bin/bash
+#
+# copy DL1 files from tar in dCache (!!) files to target directory
+# 
+# ./copy_DL1files.sh
+#
+
+if [ $# -eq 0 ]; then
+   echo "./copy_DL1files.sh copy"
+   echo
+   echo "note the hardwired values"
+fi
+
+# hardwired values
+DSET="Prod3b_Paranal_20deg_HB9_SCT_DL1"
+PLIST="gamma_onSource gamma_cone proton electron"
+# targetdir
+OSET="Prod3b_Paranal_20deg_HB9_SCT_DL1"
+########################
+# list of arrays
+# South
+# declare -a ARRAY=("BL-0LSTs15MSTs50SSTs-MSTF" "BL-0LSTs15MSTs50SSTs-MSTN" "BL-4LSTs25MSTs70SSTs-MSTF" "BL-4LSTs25MSTs70SSTs-MSTN" )
+# North
+declare -a ARRAY=("BL-4LSTs05MSTs-MSTF" "BL-4LSTs09MSTs-MSTF" "BL-4LSTs15MSTs-MSTF" ) 
+
+for P in $PLIST
+do
+   if [[ ${P} == *"gamma_cone"* ]]; then
+        PP="gamma-diffuse"
+   elif [[ ${P} == "gamma_onSource" ]]; then
+        PP="gamma"
+   elif [[ ${P} == *"proton"* ]]; then
+        PP="proton"
+   elif [[ ${P} == *"electron"* ]]; then
+        PP="electron"
+   else
+        echo "unknown particle in $P"
+        continue
+   fi
+   for A in "${ARRAY[@]}"
+   do
+      # files are copied here
+      ODIR=${CTA_USER_DATA_DIR}/analysis/AnalysisData/${OSET}/N.${A}/EVNDISP/$P
+      echo $ODIR
+      mkdir -p ${ODIR}
+      for D in North South
+      do
+        FLIST=${DSET}/${DSET}_${PP}_${D}_20deg_DL1.dCache.list
+        FF=$(cat ${FLIST})
+        for F in $FF
+        do
+           echo $F ${A}
+           dccp ${F} tt.tar.gz
+           tar --overwrite -C ${ODIR} -xvzf tt.tar.gz --wildcards "*${A}*"
+           rm -f tt.tar.gz
+        done 
+      done
+   done
+done

copy_DLXfiles_from_dCache.sh

+#!/bin/bash
+#
+# copy DLX files from dCache (!!) files to target directory
+# 
+#
+
+if [ $# -eq 0 ]; then
+   echo "./copy_DLXfiles_from_dCache.sh <production>"
+fi
+
+FLIST=$(cat ${1}.list)
+
+for P in $FLIST
+do
+   if [[ ${P} == *"gamma-diffuse"* ]]; then
+        PP="gamma_cone"
+   elif [[ ${P} == *"gamma"* ]]; then
+        PP="gamma_onSource"
+   elif [[ ${P} == *"proton"* ]]; then
+        PP="proton"
+   elif [[ ${P} == *"electron"* ]]; then
+        PP="electron"
+   else
+        echo "unknown particle in $P"
+        continue
+   fi
+
+   # directory per particle type
+   mkdir -p ${1}/${PP}
+
+   FF=$(cat ${1}/${P}.dCache.list)
+#   FF=$(cat ${1}/${P}.list | grep gz.DL1.root)
+
+   for F in $FF
+   do
+      FP=$(basename $F)
+      echo $F
+      if [[ ! -e ${1}/${PP}/${FP} ]]; then
+          #cp -v /pnfs/ifh.de/acs/grid/cta/${F} ${1}/${PP}/
+          echo ${F}
+          dccp ${F} ${1}/${PP}/${FP}
+          #echo /acs/grid/cta/${F} ${1}/${PP}/${FP}
+      fi
+   done
+done

copy_file_lists.sh

+#!/bin/bash
+#
+# copy file list into the analysis directories
+# (lists required for the analysis with Evndisp)
+# 
+
+if [ $# -lt 2 ]; then
+echo "
+copy file lists into the analysis directory
+
+
+./copy_file_lists.sh <production> <target directory>
+"
+exit
+fi
+
+mkdir -p ${2}
+
+for P in gamma_onSource gamma_cone proton electron
+do
+   for D in North South
+   do
+      if [[ ${P} == "gamma_onSource" ]]; then
+         PP="gamma"
+      elif [[ ${P} == "gamma_cone" ]]; then
+         PP="gamma-diffuse"
+      else
+         PP=${P}
+      fi
+      if [[ ${D} == "North" ]]; then
+         DD="0deg"
+      else
+         DD="180deg"
+      fi
+      if [[ -e ${1}/${1}_${PP}_${D}_20deg_DL0.local.list ]]; then
+          OFILE=${P}_${DD}.list
+          cp -v -f ${1}/${1}_${PP}_${D}_20deg_DL0.local.list ${2}/${OFILE}
+      fi
+    done
+    rm -f ${2}/${P}.list
+    touch ${2}/${P}.list
+    cat ${2}/${P}_*.list >> ${2}/${P}.list
+done
+

download_parallel.sh

+#!/bin/bash
+#
+# simple script which allows to download
+# all particle types simulatenously
+#
+
+# Prod5 NSB 1x
+DD="Prod5_Paranal_AdvancedBaseline_NSB1x"
+# Prod4b SST production
+DD="Prod4b_Paranal"
+# Prod4b SST production
+DD="Prod3b_Paranal"
+# Prod3B SCT DL1
+DD="Prod3b_Paranal_20deg_HB9_SCT_DL1"
+# Prod5 NSB 1x
+DD="Prod5b_LaPalma_AdvancedBaseline_NSB1x"
+DD="Prod5b_Paranal_AdvancedBaseline_NSB1x_40deg_DL1"
+
+ZE="40deg"
+DL="DL1"
+SCT="FALSE"
+if [[ $DD == *"Prod3b_Paranal"* ]]; then
+  DL="HB9"
+  DL="HB9_SCT_DL1"
+  SCT="TRUE"
+fi
+
+TMDPIR="tmp_log"
+mkdir -p ${TMPDIR}
+
+for P in proton gamma_cone gamma_onSource electron
+do
+     if [[ ${P} == "gamma_cone" ]]; then
+         PP="gamma-diffuse"
+     elif [[ ${P} == "gamma_onSource" ]]; then
+         PP="gamma"
+     else
+         PP=${P}
+     fi
+
+     for A in North South
+     do
+         rm -f ${TMPDIR}/${P}_${A}.log
+         # Prod4b SST production
+         if [[ $DD = *"Prod4"* ]]; then
+            if [[ $A == "South" ]]; then
+               continue
+            fi
+            for S in "sst-astri" "sst-astri+chec-s" 
+            do
+                FFLIST="${DD}_${ZE}/${DD}_${PP}_${A}_${ZE}_SSTOnly_${S}_${DL}.GRID.list"
+                ./getRawFilesFromGRID-DIRAC.sh ${FFLIST} ${DD}/${P}/ 100 &> ${TMPDIR}/${P}_${A}_${S}.log &
+            done
+         # Prod3b SCT DL1
+         elif [[ $DD = *"Prod3b"* ]] && [[ $DD = *"SCT_DL1"* ]]; then
+             FLIST="${DD}/Prod3_Paranal_${PP}_${A}_${ZE}_${DL}.GRID.list"
+             if [[ ! -e ${FLIST} ]]; then
+               echo "$FLIST not found"
+               exit
+            fi
+            echo "...found ${FLIST}"
+            ./getRawFilesFromGRID-DIRAC.sh ${FLIST} ${DD}/${P}/ 100 &> ${TMPDIR}/${P}_${A}_${S}.log &
+         # all other productions
+         else
+            FLIST="${DD}/${DD}_${PP}_${A}_${ZE}_${DL}.GRID.list"
+            FLIST="${DD}_${DL}/${DD}_${PP}_${A}_${ZE}_${DL}.GRID.list"
+            # prod3b needs some special treatment
+            if [[ ! -e ${FLIST} ]]; then
+                echo "File list not found; trying prod3b naming ($FLIST); trying prod3b lists:"
+                FLIST="${DD}_${ZE}_${DL}/Paranal_${PP}_${A}_${ZE}_${DL}.GRID.list"
+                if [[ ! -e ${FLIST} ]]; then
+                   echo "$FLIST not found"
+                   exit
+                fi
+                echo "...found ${FLIST}"
+            fi
+            echo "Reading file list $FLIST"
+            ./getRawFilesFromGRID-DIRAC.sh $FLIST ${DD}/${P}/ 100 ${SCT} &> ${TMPDIR}/${P}_${A}.log &
+         fi
+         sleep 10
+     done
+done