getRawFilesFromGRID-DIRAC.sh 1.67 KB
Newer Older
Gernot Maier's avatar
Gernot Maier committed
1
2
3
4
5
6
7
8
9
10
11
#!/bin/bash
#
# simple script to download raw files from the GRID
# using DIRAC tools
# (adjusted to DESY environment)
#
#

if [ $# -lt 3 ]; then
echo "

12
./getRawFilesFromGRID-DIRAC.sh <run list> <target directory> <max files (100)> [SCT cut=TRUE]
Gernot Maier's avatar
Gernot Maier committed
13
14
15
16
17
18
19

download using DIRAC tools
(adjusted to DESY environment)
"
exit
fi

20
21
22
23
24
echo "RUNLSIT ${1}"
echo "Target directory: ${2}"
echo "Max file: ${3}"
[[ "$4" ]] && SCT=$4 || SCT="FALSE"

Gernot Maier's avatar
Gernot Maier committed
25
26
27
28
29
30
PDIR=`pwd`

if [ -e ${2} ]
then
   mkdir -p ${2}
fi
31
32
33
34
if [ ! -e ${1} ]; then
  echo "error: file list not found: ${1}"
  exit
fi
Gernot Maier's avatar
Gernot Maier committed
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

# temporary directory for file lists
FFN=`basename $1`
mkdir -p ${2}/tmplists

# loop over all files in the list
NTMPLIST=`wc -l $1 | awk '{print $1}'`
FILEN=${3}
for ((l = 1; l < $NTMPLIST; l+=$FILEN ))
do
# create file lists with $FILEN files each
   let "k = $l + $FILEN - 1"
   let "z = $z + 1"
   LLIST=$2/tmplists/$FFN.tmplist.d.$z.list
   echo $LLIST
50
51
52
53
54
55
   # optionally download only SCT files
   if [[ $SCT = "TRUE" ]]; then
       sed -n "$l,$k p" $1 | grep SCT > $LLIST
   else
       sed -n "$l,$k p" $1 > $LLIST
   fi
Gernot Maier's avatar
Gernot Maier committed
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

   # check if files are on disk
   FF=`cat $LLIST`
   LFIST=$FFN.tmplist.f.$z.list
   touch $2/tmplists/$LFIST
   for F in $FF
   do
      FIL=`basename $F`
      if [ ! -e $2/$FIL ]
      then
          echo $F >> $2/tmplists/$LFIST
      fi
   done
   NWC=`wc -l $2/tmplists/$LFIST | awk {'print $1'}`

   echo $NWC

   if [ "$NWC" -gt 0 ]
   then
      # run dirac get files
      echo "Downloading $NWC files"
      cd ${2}
      if [[ -e tmplists/$LFIST ]]; then
          dirac-dms-get-file tmplists/$LFIST
      else
          echo "File list not found: tmplists/$LFIST"
      fi
   fi
   cd ${PDIR}
done

cd ${PDIR}

exit