repack_systemtest_wrapper.sh 23.5 KB
Newer Older
1
2
#!/bin/bash

3
4
BASE_REPORT_DIRECTORY=/var/log

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
usage() { cat <<EOF 1>&2
Usage: $0 -n <namespace>
EOF
exit 1
}

while getopts "n:" o; do
    case "${o}" in
        n)
            NAMESPACE=${OPTARG}
            ;;
        *)
            usage
            ;;
    esac
done
shift $((OPTIND-1))

if [ -z "${NAMESPACE}" ]; then
    usage
fi

if [ ! -z "${error}" ]; then
    echo -e "ERROR:\n${error}"
    exit 1
fi

echo "Preparing namespace for the tests"
./prepare_tests.sh -n ${NAMESPACE}

kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/client_helper.sh
36
37
38
39
40
kubectl -n ${NAMESPACE} cp client_prepare_file.sh client:/root/client_prepare_file.sh

archiveFiles() {
  NB_FILES=$1
  FILE_SIZE_KB=$2
41
42
43
  echo "Launching client_ar.sh on client pod"
  echo " Archiving ${NB_FILES} files of ${FILE_SIZE_KB}kB each"
  echo " Archiving files: xrdcp as user1"
44
45
  kubectl -n ${NAMESPACE} exec client -- bash /root/client_ar.sh -n ${NB_FILES} -s ${FILE_SIZE_KB} -p 100 -d /eos/ctaeos/preprod -v -A || exit 1
}
46
47
48
49
50
51
52
53
54
55
56
57

echo
kubectl -n ${NAMESPACE} cp client_ar.sh client:/root/client_ar.sh

REPACK_BUFFER_URL=/eos/ctaeos/repack
echo "Creating the repack buffer URL directory (${REPACK_BUFFER_URL})"
kubectl -n ${NAMESPACE} exec ctaeos -- eos mkdir ${REPACK_BUFFER_URL}
kubectl -n ${NAMESPACE} exec ctaeos -- eos chmod 1777 ${REPACK_BUFFER_URL}

source ./repack_helper.sh
kubectl -n ${NAMESPACE} cp repack_systemtest.sh client:/root/repack_systemtest.sh

Cedric CAFFY's avatar
Cedric CAFFY committed
58
59
60
roundTripRepack() {
  echo
  echo "***********************************************************"
61
  echo "STEP $1. Launching a round trip repack \"just move\" request"
Cedric CAFFY's avatar
Cedric CAFFY committed
62
63
64
65
66
67
  echo "***********************************************************"

  VID_TO_REPACK=$(getFirstVidContainingFiles)
  if [ "$VID_TO_REPACK" != "null" ] 
  then
  echo
68
    echo "Launching the repack \"just move\" test on VID ${VID_TO_REPACK} (with backpressure)"
69
    kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step1-RoundTripRepack -p -n ctasystest || exit 1
Cedric CAFFY's avatar
Cedric CAFFY committed
70
71
72
73
  else
    echo "No vid found to repack"
    exit 1
  fi
74

Cedric CAFFY's avatar
Cedric CAFFY committed
75
76
  echo "Reclaiming tape ${VID_TO_REPACK}"
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}
77

Cedric CAFFY's avatar
Cedric CAFFY committed
78
79
80
81
82
  VID_TO_REPACK=$(getFirstVidContainingFiles)
  if [ "$VID_TO_REPACK" != "null" ] 
  then
  echo
    echo "Launching the repack \"just move\" test on VID ${VID_TO_REPACK}"
83
    kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step$1-RoundTripRepack -n ctasystest  || exit 1
Cedric CAFFY's avatar
Cedric CAFFY committed
84
85
86
87
  else
    echo "No vid found to repack"
    exit 1
  fi
88

Cedric CAFFY's avatar
Cedric CAFFY committed
89
90
91
92
  echo "Reclaiming tape ${VID_TO_REPACK}"
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}
  echo 
  echo "*******************************************************************"
93
  echo "STEP $1. Launching a round trip repack \"just move\" request TEST OK"
Cedric CAFFY's avatar
Cedric CAFFY committed
94
95
  echo "*******************************************************************"
}
96

Cedric CAFFY's avatar
Cedric CAFFY committed
97
98
99
repackDisableTape() {
  echo 
  echo "*****************************************************"
100
  echo "STEP $1. Launching a Repack Request on a disabled tape"
Cedric CAFFY's avatar
Cedric CAFFY committed
101
  echo "*****************************************************"
102

Cedric CAFFY's avatar
Cedric CAFFY committed
103
  VID_TO_REPACK=$(getFirstVidContainingFiles)
104

Cedric CAFFY's avatar
Cedric CAFFY committed
105
106
  if [ "$VID_TO_REPACK" != "null" ]
  then
107
108
    echo "Marking the tape ${VID_TO_REPACK} as DISABLED"
    kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape ch --state DISABLED --reason "Repack disabled tape test" --vid ${VID_TO_REPACK}
109
110
    echo "Launching the repack request test on VID ${VID_TO_REPACK} without the disabled flag"
    kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -r ${BASE_REPORT_DIRECTORY}/Step$1-RepackDisabledTape -n ctasystest && echo "The repack command should have failed as the tape is disabled" && exit 1 || echo "The repack submission has failed, test OK"
Cedric CAFFY's avatar
Cedric CAFFY committed
111
112
113
114
  else
    echo "No vid found to repack"
    exit 1
  fi;
115

Cedric CAFFY's avatar
Cedric CAFFY committed
116
117
  echo
  echo "Launching the repack request test on VID ${VID_TO_REPACK} with the --disabledtape flag"
118
  kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -d -r ${BASE_REPORT_DIRECTORY}/Step$1-RepackDisabledTape -n ctasystest  || exit 1
119

Cedric CAFFY's avatar
Cedric CAFFY committed
120
121
  echo "Reclaiming tape ${VID_TO_REPACK}"
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}
122
123
124

  echo "Setting the tape ${VID_TO_REPACK} back to ACTIVE"
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape ch --state ACTIVE --vid ${VID_TO_REPACK}
125

Cedric CAFFY's avatar
Cedric CAFFY committed
126
127
  echo
  echo "*************************************************************"
128
  echo "STEP $1. Launching a Repack Request on a disabled tape TEST OK"
Cedric CAFFY's avatar
Cedric CAFFY committed
129
130
  echo "*************************************************************"
}
131

Cedric CAFFY's avatar
Cedric CAFFY committed
132
133
134
repackJustMove() {
  echo 
  echo "*********************************************"
135
  echo "STEP $1. Testing Repack \"Just move\" workflow"
Cedric CAFFY's avatar
Cedric CAFFY committed
136
  echo "*********************************************"
137

Cedric CAFFY's avatar
Cedric CAFFY committed
138
139
140
141
142
  VID_TO_REPACK=$(getFirstVidContainingFiles)
  if [ "$VID_TO_REPACK" != "null" ] 
  then
  echo
    echo "Launching the repack test \"just move\" on VID ${VID_TO_REPACK}"
143
    kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step$1-RepackJustMove -n ctasystest  || exit 1
Cedric CAFFY's avatar
Cedric CAFFY committed
144
145
146
147
  else
    echo "No vid found to repack"
    exit 1
  fi
148

149
150
  echo "Reclaiming tape ${VID_TO_REPACK}"
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}
Cedric CAFFY's avatar
Cedric CAFFY committed
151
152
  echo 
  echo "*****************************************************"
153
  echo "STEP $1. Testing Repack \"Just move\" workflow TEST OK"
Cedric CAFFY's avatar
Cedric CAFFY committed
154
155
  echo "*****************************************************"
}
156

Cedric CAFFY's avatar
Cedric CAFFY committed
157
158
159
repackJustAddCopies() {
  echo
  echo "**************************************************************************"
160
  echo "STEP $1. Testing Repack \"Just Add copies\" workflow with all copies on CTA"
Cedric CAFFY's avatar
Cedric CAFFY committed
161
  echo "**************************************************************************"
162

Cedric CAFFY's avatar
Cedric CAFFY committed
163
164
165
166
  VID_TO_REPACK=$(getFirstVidContainingFiles)
  if [ "$VID_TO_REPACK" != "null" ] 
  then
    echo "Launching the repack \"just add copies\" test on VID ${VID_TO_REPACK} with all copies already on CTA"
167
    kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -a -r ${BASE_REPORT_DIRECTORY}/Step$1-JustAddCopiesAllCopiesInCTA -n ctasystest 
Cedric CAFFY's avatar
Cedric CAFFY committed
168
169
170
171
  else
    echo "No vid found to repack"
    exit 1
  fi
172

Cedric CAFFY's avatar
Cedric CAFFY committed
173
  repackJustAddCopiesResult=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json re ls | jq -r ". [] | select (.vid == \"${VID_TO_REPACK}\")"`
174

Cedric CAFFY's avatar
Cedric CAFFY committed
175
176
  nbRetrievedFiles=`echo ${repackJustAddCopiesResult} | jq -r ".retrievedFiles"`
  nbArchivedFiles=`echo ${repackJustAddCopiesResult} | jq -r ".archivedFiles"`
177

Cedric CAFFY's avatar
Cedric CAFFY committed
178
179
180
181
182
183
184
  if [ $nbArchivedFiles == 0 ] && [ $nbRetrievedFiles == 0 ] 
  then
    echo "Nb retrieved files = 0 and nb archived files = 0. Test OK"
  else
    echo "Repack \"just add copies\" on VID ${VID_TO_REPACK} failed : nbRetrievedFiles = $nbRetrievedFiles, nbArchivedFiles = $nbArchivedFiles"
    exit 1
  fi
185

Cedric CAFFY's avatar
Cedric CAFFY committed
186
187
  echo
  echo "**********************************************************************************"
188
  echo "STEP $1. Testing Repack \"Just Add copies\" workflow with all copies on CTA TEST OK"
Cedric CAFFY's avatar
Cedric CAFFY committed
189
190
  echo "**********************************************************************************"
}
191

Cedric CAFFY's avatar
Cedric CAFFY committed
192
193
194
repackCancellation() {
  echo
  echo "***********************************"
195
  echo "STEP $1. Testing Repack cancellation"
Cedric CAFFY's avatar
Cedric CAFFY committed
196
  echo "***********************************"
197

Cedric CAFFY's avatar
Cedric CAFFY committed
198
  echo "Putting all drives down"
199
200
  echo 'kubectl -n ${NAMESPACE} exec ctacli -- cta-admin drive down VD.* --reason "Putting drive down for repack test"'
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin drive down VD.* --reason "Putting drive down for repack test"
201

Cedric CAFFY's avatar
Cedric CAFFY committed
202
203
204
205
206
  VID_TO_REPACK=$(getFirstVidContainingFiles)
  if [ "$VID_TO_REPACK" != "null" ] 
  then
  echo
    echo "Launching a repack request on VID ${VID_TO_REPACK}"
207
    kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step$1-RepackCancellation -n ctasystest & 2>/dev/null
Cedric CAFFY's avatar
Cedric CAFFY committed
208
209
210
211
212
    pid=$!
  else
    echo "No vid found to repack"
    exit 1
  fi
213

Cedric CAFFY's avatar
Cedric CAFFY committed
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
  echo "Waiting for the launch of the repack request..."
  returnCode=1
  while [[ $returnCode != 0 ]]
  do
    kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json re ls --vid $VID_TO_REPACK 1> /dev/null 2> /dev/null
    returnCode=$?
  done
  echo "Repack request launched"
  echo
  echo "Waiting for the expansion of the repack request..."

  lastFSeqTapeToRepack=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tape ls --vid ${VID_TO_REPACK} | jq -r ".[0] | .lastFseq"`
  lastExpandedFSeq=0
  while [[ $lastExpandedFSeq != $lastFSeqTapeToRepack ]]
  do
    lastExpandedFSeq=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json repack ls --vid ${VID_TO_REPACK} | jq -r ".[0] | .lastExpandedFseq" || 0`
  done
  
  echo "Expansion finished, deleting the Repack Request"
  kill $pid
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin repack rm --vid ${VID_TO_REPACK} || echo "Error while removing the Repack Request. Test FAILED"

  echo
  echo "Checking that the Retrieve queue of the VID ${VID_TO_REPACK} contains the Retrieve Requests created from the Repack Request expansion"
238
  nbFilesOnTapeToRepack=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tapefile ls --vid ${VID_TO_REPACK} | jq "length"`
Cedric CAFFY's avatar
Cedric CAFFY committed
239
240
241
242
243
244
245
246
247
248
  echo "Nb files on tape = $nbFilesOnTapeToRepack"
  
  nbFilesOnQueue=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json showqueues | jq -r ". [] | select(.vid == \"${VID_TO_REPACK}\") | .queuedFiles"` 
  echo "Nb files on the queue ${VID_TO_REPACK} = $nbFilesOnQueue"

  if [[ $nbFilesOnTapeToRepack != $nbFilesOnQueue ]]
  then
    echo "Nb files on tape != nb files queued, test Failed"
    exit 1
  fi
249

Cedric CAFFY's avatar
Cedric CAFFY committed
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
  echo "Putting all drives up"
  echo 'kubectl -n ${NAMESPACE} exec ctacli -- cta-admin drive up VD.*'
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin drive up VD.*

  WAIT_FOR_EMPTY_QUEUE_TIMEOUT=100
  
  SECONDS_PASSED=0
  while test 0 != `kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json showqueues | jq -r ". [] | select(.vid == \"${VID_TO_REPACK}\")" | wc -l`; do
    echo "Waiting for the Retrieve queue ${VID_TO_REPACK} to be empty: Seconds passed = $SECONDS_PASSED"
    sleep 1
    let SECONDS_PASSED=SECONDS_PASSED+1
    if test ${SECONDS_PASSED} == ${WAIT_FOR_EMPTY_QUEUE_TIMEOUT}; then
      echo "Timed out after ${ WAIT_FOR_EMPTY_QUEUE_TIMEOUT} seconds waiting for retrieve queue ${VID_TO_REPACK} to be emptied. Test failed."
      exit 1
    fi
  done

  echo "Retrieve queue of VID ${VID_TO_REPACK} is empty, test OK"

  echo "*******************************************"
270
  echo "STEP $1. Testing Repack cancellation TEST OK"
Cedric CAFFY's avatar
Cedric CAFFY committed
271
272
  echo "*******************************************"
}
273

Cedric CAFFY's avatar
Cedric CAFFY committed
274
275
276
repackMoveAndAddCopies() {
  echo
  echo "*******************************************************"
277
  echo "STEP $1. Testing Repack \"Move and Add copies\" workflow"
Cedric CAFFY's avatar
Cedric CAFFY committed
278
  echo "*******************************************************"
279

Cedric CAFFY's avatar
Cedric CAFFY committed
280
281
  tapepoolDestination1="ctasystest2"
  tapepoolDestination2="ctasystest3"
282

Cedric CAFFY's avatar
Cedric CAFFY committed
283
284
285
286
  echo "Creating two destination tapepool : $tapepoolDestination1 and $tapepoolDestination2"
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tapepool add --name $tapepoolDestination1 --vo vo --partialtapesnumber 2 --encrypted false --comment "$tapepoolDestination1 tapepool"
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tapepool add --name $tapepoolDestination2 --vo vo --partialtapesnumber 2 --encrypted false --comment "$tapepoolDestination2 tapepool"
  echo "OK"
287

Cedric CAFFY's avatar
Cedric CAFFY committed
288
  echo "Creating archive routes for adding two copies of the file"
289
290
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin archiveroute add --storageclass ctaStorageClass --copynb 2 --tapepool $tapepoolDestination1 --comment "ArchiveRoute2"
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin archiveroute add --storageclass ctaStorageClass --copynb 3 --tapepool $tapepoolDestination2 --comment "ArchiveRoute3"
Cedric CAFFY's avatar
Cedric CAFFY committed
291
  echo "OK"
292

Cedric CAFFY's avatar
Cedric CAFFY committed
293
  echo "Will change the tapepool of the tapes"
294

Cedric CAFFY's avatar
Cedric CAFFY committed
295
296
  allVID=`kubectl -n ${NAMESPACE}  exec ctacli -- cta-admin --json tape ls --all | jq -r ". [] | .vid"`
  read -a allVIDTable <<< $allVID
297

Cedric CAFFY's avatar
Cedric CAFFY committed
298
  nbVid=${#allVIDTable[@]}
299

Cedric CAFFY's avatar
Cedric CAFFY committed
300
  allTapepool=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tapepool ls | jq -r ". [] .name"`
301

Cedric CAFFY's avatar
Cedric CAFFY committed
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
  read -a allTapepoolTable <<< $allTapepool

  nbTapepool=${#allTapepoolTable[@]}

  nbTapePerTapepool=$(($nbVid / $nbTapepool))

  allTapepool=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tapepool ls | jq -r ". [] .name"`
  read -a allTapepoolTable <<< $allTapepool


  countChanging=0
  tapepoolIndice=1 #We only change the vid of the remaining other tapes

  for ((i=$(($nbTapePerTapepool+$(($nbVid%$nbTapepool)))); i<$nbVid; i++));
  do
    echo "kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape ch --vid ${allVIDTable[$i]} --tapepool ${allTapepoolTable[$tapepoolIndice]}"
    kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape ch --vid ${allVIDTable[$i]} --tapepool ${allTapepoolTable[$tapepoolIndice]}
    countChanging=$((countChanging + 1))
    if [ $countChanging != 0 ] && [ $((countChanging % nbTapePerTapepool)) == 0 ]
    then
      tapepoolIndice=$((tapepoolIndice + 1))
    fi
  done

  echo "OK"

  storageClassName=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json storageclass ls | jq -r ". [0] | .name"`

  echo "Changing the storage class $storageClassName nb copies"
331
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin storageclass ch --name $storageClassName --numberofcopies 3
Cedric CAFFY's avatar
Cedric CAFFY committed
332
333
334
335
336
337
  echo "OK"

  echo "Putting all drives up"
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin dr up VD.*
  echo "OK"

338
339
  VID_TO_REPACK=$(getFirstVidContainingFiles)

Cedric CAFFY's avatar
Cedric CAFFY committed
340
  echo "Launching the repack \"Move and add copies\" test on VID ${VID_TO_REPACK}"
341
  kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -t 600 -r ${BASE_REPORT_DIRECTORY}/Step$1-MoveAndAddCopies -n ctasystest  || exit 1
342

343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
  repackLsResult=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json repack ls --vid ${VID_TO_REPACK} | jq ". [0]"`
  totalFilesToRetrieve=`echo $repackLsResult | jq -r ".totalFilesToRetrieve"`
  totalFilesToArchive=`echo $repackLsResult | jq -r ".totalFilesToArchive"`
  retrievedFiles=`echo $repackLsResult | jq -r ".retrievedFiles"`
  archivedFiles=`echo $repackLsResult | jq -r ".archivedFiles"`
  
  if [[ $retrievedFiles != $totalFilesToRetrieve ]]
  then
    echo "RetrievedFiles ($retrievedFiles) != totalFilesToRetrieve ($totalFilesToRetrieve), test FAILED"
    exit 1
  else 
    echo "RetrievedFiles ($retrievedFiles) = totalFilesToRetrieve ($totalFilesToRetrieve), OK"
  fi

  if [[ $archivedFiles != $totalFilesToArchive ]]
  then
    echo "ArchivedFiles ($archivedFiles) != totalFilesToArchive ($totalFilesToArchive), test FAILED"
    exit 1
  else
     echo "ArchivedFiles ($archivedFiles) == totalFilesToArchive ($totalFilesToArchive), OK"
  fi
  
365
366
367
  echo "Reclaimimg tape ${VID_TO_REPACK}"
  kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}

Cedric CAFFY's avatar
Cedric CAFFY committed
368
369
  echo
  echo "***************************************************************"
370
  echo "STEP $1. Testing Repack \"Move and Add copies\" workflow TEST OK"
Cedric CAFFY's avatar
Cedric CAFFY committed
371
372
373
  echo "***************************************************************"
}

374
375
376
repackTapeRepair() {
  echo
  echo "*******************************************************"
377
  echo "STEP $1. Testing Repack \"Tape Repair\" workflow"
378
379
380
381
382
383
384
385
386
387
388
  echo "*******************************************************"  

  VID_TO_REPACK=$(getFirstVidContainingFiles)
  if [ "$VID_TO_REPACK" == "null" ] 
  then
    echo "No vid found to repack"
    exit 1
  fi
  
  echo "Getting files to inject into the repack buffer directory"
  
389
  tfls=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tapefile ls --vid ${VID_TO_REPACK}`
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
  nbFileToInject=10
  
  if [[ $nbFileToInject != 0 ]]
  then
    echo "Will inject $nbFileToInject files into the repack buffer directory"
    bufferDirectory=${REPACK_BUFFER_URL}/${VID_TO_REPACK}
    echo "Creating buffer directory in \"$bufferDirectory\""
    kubectl -n ${NAMESPACE} exec ctaeos -- eos mkdir $bufferDirectory
    kubectl -n ${NAMESPACE} exec ctaeos -- eos chmod 1777 $bufferDirectory

    echo "Retrieving files from the tape"
    allPid=()
    pathOfFilesToInject=()
    diskIds=()

    for i in $(seq 0 $(( nbFileToInject - 1 )) )
    do
407
      diskId=`echo $tfls | jq -r ". [$i] | .df.diskId"` || break
408
409
410
411
412
413
414
415
416
417
418
      diskIds[$i]=$diskId
      pathFileToInject=`kubectl -n ${NAMESPACE} exec ctaeos -- eos fileinfo fid:$diskId --path | cut -d":" -f2 | tr -d " "`
      pathOfFilesToInject[$i]=$pathFileToInject
    done
    
    kubectl -n ${NAMESPACE} exec client -- bash /root/client_prepare_file.sh `for file in ${pathOfFilesToInject[@]}; do echo -n "-f $file "; done`

    echo "Copying the retrieved files into the repack buffer $bufferDirectory"
    
    for i in $(seq 0 $(( nbFileToInject - 1)) )
    do
419
      fseqFile=`echo $tfls | jq -r ". [] | select(.df.diskId == \"${diskIds[$i]}\") | .tf.fSeq"` || break
420
421
422
423
      kubectl -n ${NAMESPACE} exec ctaeos -- eos cp ${pathOfFilesToInject[$i]} $bufferDirectory/`printf "%9d\n" $fseqFile | tr ' ' 0`
    done
   
    echo "Launching a repack request on the vid ${VID_TO_REPACK}"
424
    kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step$1-RepackTapeRepair -n ctasystest  ||      exit 1
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455

    repackLsResult=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json repack ls --vid ${VID_TO_REPACK} | jq -r ". [0]"`
    userProvidedFiles=`echo $repackLsResult | jq -r ".userProvidedFiles"`
    archivedFiles=`echo $repackLsResult | jq -r ".archivedFiles"`
    retrievedFiles=`echo $repackLsResult | jq -r ".retrievedFiles"`
    totalFilesToRetrieve=`echo $repackLsResult | jq -r ".totalFilesToRetrieve"`
    totalFilesToArchive=`echo $repackLsResult | jq -r ".totalFilesToArchive"`

    if [[ $totalFilesToRetrieve != $(( $totalFilesToArchive - $userProvidedFiles )) ]]
    then
      echo "totalFilesToRetrieve ($totalFilesToRetrieve) != totalFilesToArchive ($totalFilesToArchive) - userProvidedFiles ($userProvidedFiles), test FAILED"
      exit 1
    else
      echo "totalFilesToRetrieve ($totalFilesToRetrieve) == totalFilesToArchive ($totalFilesToArchive) - userProvidedFiles ($userProvidedFiles), OK"
    fi  

    if [[ $retrievedFiles != $totalFilesToRetrieve ]]
    then
      echo "retrievedFiles ($retrievedFiles) != totalFilesToRetrieve ($totalFilesToRetrieve) test FAILED"
      exit 1
    else
      echo "retrievedFiles ($retrievedFiles) == totalFilesToRetrieve ($totalFilesToRetrieve), OK"
    fi

    if [[ $archivedFiles != $totalFilesToArchive ]]
    then
      echo "archivedFiles ($archivedFiles) != totalFilesToArchive ($totalFilesToArchive), test FAILED"
      exit 1
    else
       echo "archivedFiles ($archivedFiles) == totalFilesToArchive ($totalFilesToArchive), OK"
    fi
456
457
    echo "Reclaiming tape ${VID_TO_REPACK}"
    kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tape reclaim --vid ${VID_TO_REPACK}
458
459
460
461
462
463
464
465

  else 
    echo "No file to inject, test not OK"
    exit 1
  fi

  echo
  echo "*******************************************************"
466
  echo "STEP $1. Testing Repack \"Tape Repair\" workflow TEST OK"
467
468
469
  echo "*******************************************************"
}

470
471
472
repackTapeRepairNoRecall() {
  echo
  echo "*******************************************************"
473
  echo "STEP $1. Testing Repack \"Tape Repair\" NO RECALL workflow"
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
  echo "*******************************************************"  

  VID_TO_REPACK=$(getFirstVidContainingFiles)
  if [ "$VID_TO_REPACK" == "null" ] 
  then
    echo "No vid found to repack"
    exit 1
  fi
  
  echo "Getting files to inject into the repack buffer directory"
  
  tfls=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tapefile ls --vid ${VID_TO_REPACK}`
  nbFilesOnTape=`echo $tfls | jq length`
  nbFileToInject=10
  
  if [[ $nbFileToInject != 0 ]]
  then
    echo "Will inject $nbFileToInject files into the repack buffer directory"
    bufferDirectory=${REPACK_BUFFER_URL}/${VID_TO_REPACK}
    echo "Creating buffer directory in \"$bufferDirectory\""
    kubectl -n ${NAMESPACE} exec ctaeos -- eos mkdir $bufferDirectory
    kubectl -n ${NAMESPACE} exec ctaeos -- eos chmod 1777 $bufferDirectory

    echo "Retrieving files from the tape"
    allPid=()
    pathOfFilesToInject=()
    diskIds=()
    filesIndices=()
    # Prepare array of indices to pick from the tfls output
    for i in $(seq 0 $(( nbFileToInject - 1 )) )
    do
      filesIndices[$i]=$(( i + 1 )) 
    done

    for i in $(seq 0 $(( nbFileToInject - 1 )) )
    do
      diskId=`echo $tfls | jq -r ". [${filesIndices[$i]}] | .df.diskId"` || break
      diskIds[$i]=$diskId
      pathFileToInject=`kubectl -n ${NAMESPACE} exec ctaeos -- eos fileinfo fid:$diskId --path | cut -d":" -f2 | tr -d " "`
      pathOfFilesToInject[$i]=$pathFileToInject
    done
    
    kubectl -n ${NAMESPACE} exec client -- bash /root/client_prepare_file.sh `for file in ${pathOfFilesToInject[@]}; do echo -n "-f $file "; done`

    echo "Copying the retrieved files into the repack buffer $bufferDirectory"
    
    for i in $(seq 0 $(( nbFileToInject - 1)) )
    do
      fseqFile=`echo $tfls | jq -r ". [] | select(.df.diskId == \"${diskIds[$i]}\") | .tf.fSeq"` || break
      kubectl -n ${NAMESPACE} exec ctaeos -- eos cp ${pathOfFilesToInject[$i]} $bufferDirectory/`printf "%9d\n" $fseqFile | tr ' ' 0`
    done
   
    echo "Launching a repack request on the vid ${VID_TO_REPACK}"
    kubectl -n ${NAMESPACE} exec client -- bash /root/repack_systemtest.sh -v ${VID_TO_REPACK} -b ${REPACK_BUFFER_URL} -m -r ${BASE_REPORT_DIRECTORY}/Step$1-RepackTapeRepairNoRecall -n ctasystest -u ||      exit 1

    repackLsResult=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json repack ls --vid ${VID_TO_REPACK} | jq -r ". [0]"`
    userProvidedFiles=`echo $repackLsResult | jq -r ".userProvidedFiles"`
    archivedFiles=`echo $repackLsResult | jq -r ".archivedFiles"`
    retrievedFiles=`echo $repackLsResult | jq -r ".retrievedFiles"`
    totalFilesToRetrieve=`echo $repackLsResult | jq -r ".totalFilesToRetrieve"`
    totalFilesToArchive=`echo $repackLsResult | jq -r ".totalFilesToArchive"`

    if [[ $totalFilesToRetrieve != $(( $totalFilesToArchive - $userProvidedFiles )) ]]
    then
      echo "totalFilesToRetrieve ($totalFilesToRetrieve) != totalFilesToArchive ($totalFilesToArchive) - userProvidedFiles ($userProvidedFiles), test FAILED"
      exit 1
    else
      echo "totalFilesToRetrieve ($totalFilesToRetrieve) == totalFilesToArchive ($totalFilesToArchive) - userProvidedFiles ($userProvidedFiles), OK"
    fi  

    if [[ $retrievedFiles != $totalFilesToRetrieve ]]
    then
      echo "retrievedFiles ($retrievedFiles) != totalFilesToRetrieve ($totalFilesToRetrieve) test FAILED"
      exit 1
    else
      echo "retrievedFiles ($retrievedFiles) == totalFilesToRetrieve ($totalFilesToRetrieve), OK"
    fi

552
    if [[ $archivedFiles != $totalFilesToArchive || $userProvidedFiles != $archivedFiles ]]
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
    then
      echo "archivedFiles ($archivedFiles) != totalFilesToArchive ($totalFilesToArchive), test FAILED"
      exit 1
    else
       echo "archivedFiles ($archivedFiles) == totalFilesToArchive ($totalFilesToArchive), OK"
    fi

  else 
    echo "No file to inject, test not OK"
    exit 1
  fi

  echo
  echo "*******************************************************"
  echo "STEP $1. Testing Repack \"Tape Repair\" workflow NO RECALL TEST OK"
  echo "*******************************************************"
}

Cedric CAFFY's avatar
Cedric CAFFY committed
571
#Execution of each tests
572
archiveFiles 1 15
573
574
roundTripRepack 1
repackDisableTape 2
575
576
archiveFiles 1152 15
repackJustMove 3
577
578
579
580
581
repackTapeRepair 4
repackJustAddCopies 5
repackCancellation 6
# repackMoveAndAddCopies 7
repackTapeRepairNoRecall 7