create_instance.sh 13.1 KB
Newer Older
1
2
#!/bin/bash

3
4
5
# CTA registry secret name
ctareg_secret='ctaregsecret'

6
7
8
9
# defaults objectstore to file
config_objectstore="./objectstore-file.yaml"
# defaults DB to sqlite
config_database="./database-sqlite.yaml"
10
11
# default library model
model="mhvtl"
12

13
14
15
# EOS short instance name
EOSINSTANCE=ctaeos

16
17
18
# By default to not use systemd to manage services inside the containers
usesystemd=0

19
20
# By default keep Database and keep Objectstore
# default should not make user loose data if he forgot the option
21
22
keepdatabase=1
keepobjectstore=1
23
24

usage() { cat <<EOF 1>&2
Eric Cano's avatar
Eric Cano committed
25
Usage: $0 -n <namespace> [-o <objectstore_configmap>] [-d <database_configmap>] \
26
      [-p <gitlab pipeline ID> | -b <build tree base> -B <build tree subdir> ]  \
27
      [-S] [-D] [-O] [-m [mhvtl|ibm]]
28
29

Options:
30
  -S    Use systemd to manage services inside containers
31
32
33
  -b    The directory containing both the source and the build tree for CTA. It will be mounted RO in the
        containers.
  -B    The subdirectory within the -b directory where the build tree is.
34
35
36
37
38
39
  -D	wipe database content during initialization phase (database content is kept by default)
  -O	wipe objectstore content during initialization phase (objectstore content is kept by default)
EOF
exit 1
}

40
41
die() { echo "$@" 1>&2 ; exit 1; }

42
while getopts "n:o:d:p:b:B:SDOm:" o; do
43
44
45
46
47
48
49
50
51
    case "${o}" in
        o)
            config_objectstore=${OPTARG}
            test -f ${config_objectstore} || error="${error}Objectstore configmap file ${config_objectstore} does not exist\n"
            ;;
        d)
            config_database=${OPTARG}
            test -f ${config_database} || error="${error}Database configmap file ${config_database} does not exist\n"
            ;;
52
53
54
55
        m)
            model=${OPTARG}
            if [ "-${model}-" != "-ibm-" ] && [ "-${model}-" != "-mhvtl-" ] ; then error="${error}Library model ${model} does not exist\n"; fi 
            ;;
56
57
58
        n)
            instance=${OPTARG}
            ;;
59
60
61
	p)
            pipelineid=${OPTARG}
            ;;
62
63
64
        b)
            buildtree=${OPTARG}
            ;;
65
66
67
        S)
            usesystemd=1
            ;;
68
69
70
        B)
            buildtreesubdir=${OPTARG}
            ;;
71
        O)
72
            keepobjectstore=0
73
74
            ;;
        D)
75
            keepdatabase=0
76
77
78
79
80
81
82
83
84
85
86
87
            ;;
        *)
            usage
            ;;
    esac
done
shift $((OPTIND-1))

if [ -z "${instance}" ]; then
    usage
fi

Eric Cano's avatar
Eric Cano committed
88
if [ ! -z "${pipelineid}" -a ! -z "${buildtree}" ]; then
89
90
91
    usage
fi

92
93
94
# everyone needs poddir temporary directory to generate pod yamls
poddir=$(mktemp -d)

Eric Cano's avatar
Eric Cano committed
95
if [ ! -z "${buildtree}" ]; then
96
97
98
99
    # We need to know the subdir as well
    if [ -z "${buildtreesubdir}" ]; then
      usage
    fi
100
101
    # We are going to run with generic images against a build tree.
    echo "Creating instance for build tree in ${buildtree}"
102

103
104
105
    # tag image as otherwise kubernetes will always pull latest and won't find it...
    docker rmi buildtree-runner:v0 &>/dev/null
    docker tag buildtree-runner buildtree-runner:v0
106
    cp pod-* ${poddir}
107
    sed -i ${poddir}/pod-* -e "s/\(^\s\+image\):.*/\1: buildtree-runner:v0\n\1PullPolicy: Never/"
108

109
110
111
112
    # Add the build tree mount point the pods (volume mount then volume).
    sed -i ${poddir}/pod-* -e "s|\(^\s\+\)\(volumeMounts:\)|\1\2\n\1- mountPath: ${buildtree}\n\1  name: buildtree\n\1  readOnly: true|"
    sed -i ${poddir}/pod-* -e "s|\(^\s\+\)\(volumes:\)|\1\2\n\1- name: buildtree\n\1  hostPath:\n\1    path: ${buildtree}|"

113
114
115
116
    if [ ! -z "${error}" ]; then
        echo -e "ERROR:\n${error}"
        exit 1
    fi
117
else
118
119
120
121
122
123
124
125
126
127
128
129
130
131
    # We are going to run with repository based images (they have rpms embedded)
    COMMITID=$(git log -n1 | grep ^commit | cut -d\  -f2 | sed -e 's/\(........\).*/\1/')
    if [ -z "${pipelineid}" ]; then
      echo "Creating instance for latest image built for ${COMMITID} (highest PIPELINEID)"
      imagetag=$(../ci_helpers/list_images.sh 2>/dev/null | grep ${COMMITID} | sort -n | tail -n1)
    else
      echo "Creating instance for image built on commit ${COMMITID} with gitlab pipeline ID ${pipelineid}"
      imagetag=$(../ci_helpers/list_images.sh 2>/dev/null | grep ${COMMITID} | grep ^${pipelineid}git | sort -n | tail -n1)
    fi
    if [ "${imagetag}" == "" ]; then
      echo "commit:${COMMITID} has no docker image available in gitlab registry, please check pipeline status and registry images available."
      exit 1
    fi
    echo "Creating instance using docker image with tag: ${imagetag}"
132

133
134
    cp pod-* ${poddir}
    sed -i ${poddir}/pod-* -e "s/\(^\s\+image:[^:]\+:\).*/\1${imagetag}/"
135

136
137
138
139
    if [ ! -z "${error}" ]; then
        echo -e "ERROR:\n${error}"
        exit 1
    fi
140
141
fi

142
143
144
145
146
147
if [ $usesystemd == 1 ] ; then
    echo "Using systemd to start services on some pods"
    for podname in ctafrontend tpsrv; do
        sed -i "/^\ *command:/d" ${poddir}/pod-${podname}*.yaml
    done
fi
148

149
if [ $keepdatabase == 1 ] ; then
150
151
152
153
154
    echo "DB content will be kept"
else
    echo "DB content will be wiped"
fi

155
if [ $keepobjectstore == 1 ] ; then 
156
157
158
159
160
161
    echo "objecstore content will be kept"
else
    echo "objectstore content will be wiped"
fi


162
echo -n "Creating ${instance} instance "
163

164
kubectl create namespace ${instance} || die "FAILED"
165

166
167
168
169
170
171
172
# The CTA registry secret must be copied in the instance namespace to be usable
kubectl get secret ${ctareg_secret} &> /dev/null
if [ $? -eq 0 ]; then
  echo "Copying ${ctareg_secret} secret in ${instance} namespace"
  kubectl get secret ctaregsecret -o yaml | grep -v '^ *namespace:' | kubectl --namespace ${instance} create -f -
fi

173
kubectl --namespace ${instance} create configmap init --from-literal=keepdatabase=${keepdatabase} --from-literal=keepobjectstore=${keepobjectstore}
174

175
kubectl --namespace ${instance} create configmap buildtree --from-literal=base=${buildtree} --from-literal=subdir=${buildtreesubdir}
176
177
178
179
180
181
182

echo "creating configmaps in instance"

kubectl create -f ${config_objectstore} --namespace=${instance}
kubectl create -f ${config_database} --namespace=${instance}


183
184
echo -n "Requesting an unused ${model} library"
kubectl create -f ./pvc_library_${model}.yaml --namespace=${instance}
185
186
187
188
189
for ((i=0; i<120; i++)); do
  echo -n "."
  kubectl get persistentvolumeclaim claimlibrary --namespace=${instance} | grep -q Bound && break
  sleep 1
done
190
kubectl get persistentvolumeclaim claimlibrary --namespace=${instance} | grep -q Bound || die "TIMED OUT"
191
echo "OK"
192
193
194
195
LIBRARY_DEVICE=$(kubectl get persistentvolumeclaim claimlibrary --namespace=${instance} -o yaml| grep -i volumeName | sed -e 's%.*sg%sg%')

kubectl --namespace=${instance} create -f /opt/kubernetes/CTA/library/config/library-config-${LIBRARY_DEVICE}.yaml

196
echo "Got library: ${LIBRARY_DEVICE}"
197

198
echo "Requesting an unused log volume"
199
200
kubectl create -f ./pvc_logs.yaml --namespace=${instance}

201
202
203
echo "Requesting an unused stg volume"
kubectl create -f ./pvc_stg.yaml --namespace=${instance}

204
echo "Creating services in instance"
205
206
207
208
209
210

for service_file in *svc\.yaml; do
  kubectl create -f ${service_file} --namespace=${instance}
done


211
echo "Creating pods in instance"
212

213
kubectl	create -f ${poddir}/pod-init.yaml --namespace=${instance}
214
215
216
217
218
219
220
221
222

echo -n "Waiting for init"
for ((i=0; i<400; i++)); do
  echo -n "."
  kubectl get pod init -a --namespace=${instance} | grep -q Completed && break
  sleep 1
done

# initialization went wrong => exit now with error
223
kubectl get pod init -a --namespace=${instance} | grep -q Completed || die "TIMED OUT"
224
225
226
227
echo OK

echo "Launching pods"

228
for podname in client ctacli tpsrv01 tpsrv02 ctaeos ctafrontend kdc; do
229
  kubectl create -f ${poddir}/pod-${podname}.yaml --namespace=${instance}
230
231
232
233
234
235
236
237
238
239
240
done

echo -n "Waiting for other pods"
for ((i=0; i<240; i++)); do
  echo -n "."
  # exit loop when all pods are in Running state
  kubectl get pods -a --namespace=${instance} | grep -v init | tail -n+2 | grep -q -v Running || break
  sleep 1
done

if [[ $(kubectl get pods -a --namespace=${instance} | grep -v init | tail -n+2 | grep -q -v Running) ]]; then
241
  echo "TIMED OUT"
242
243
244
245
246
247
248
249
250
251
252
253
254
255
  echo "Some pods have not been initialized properly:"
  kubectl get pods -a --namespace=${instance}
  exit 1
fi
echo OK

echo -n "Waiting for KDC to be configured"
# Kdc logs sometimes get truncated. We rely on a different mechanism to detect completion
for ((i=0; i<300; i++)); do
  echo -n "."
  [ "`kubectl --namespace=${instance} exec kdc -- bash -c "[ -f /root/kdcReady ] && echo -n Ready || echo -n Not ready"`" = "Ready" ] && break
  sleep 1
done

256
[ "`kubectl --namespace=${instance} exec kdc -- bash -c "[ -f /root/kdcReady ] && echo -n Ready || echo -n Not ready"`" = "Ready" ] || die "TIMED OUT"
257
258
echo OK

259
echo -n "Configuring KDC clients (frontend, cli...) "
260
kubectl --namespace=${instance} exec kdc cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i client --  bash -c "cat > /etc/krb5.conf"
261
262
kubectl --namespace=${instance} exec kdc cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i ctacli --  bash -c "cat > /etc/krb5.conf" 
kubectl --namespace=${instance} exec kdc cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i ctafrontend --  bash -c "cat > /etc/krb5.conf"
263
kubectl --namespace=${instance} exec kdc cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i ctaeos --  bash -c "cat > /etc/krb5.conf"
264
kubectl --namespace=${instance} exec kdc cat /root/ctaadmin1.keytab | kubectl --namespace=${instance} exec -i ctacli --  bash -c "cat > /root/ctaadmin1.keytab"
265
kubectl --namespace=${instance} exec kdc cat /root/user1.keytab | kubectl --namespace=${instance} exec -i client --  bash -c "cat > /root/user1.keytab"
266
267
# need to mkdir /etc/cta folder as cta rpm may not already be installed (or put it somewhere else and move it later???)
kubectl --namespace=${instance} exec kdc cat /root/cta-frontend.keytab | kubectl --namespace=${instance} exec -i ctafrontend --  bash -c "mkdir -p /etc/cta; cat > /etc/cta/cta-frontend.krb5.keytab"
268
269
kubectl --namespace=${instance} exec kdc cat /root/eos-server.keytab | kubectl --namespace=${instance} exec -i ctaeos --  bash -c "cat > /etc/eos-server.krb5.keytab"
kubectl --namespace=${instance} exec ctacli -- kinit -kt /root/ctaadmin1.keytab ctaadmin1@TEST.CTA
270
kubectl --namespace=${instance} exec client -- kinit -kt /root/user1.keytab user1@TEST.CTA
Julien Leduc's avatar
Julien Leduc committed
271

Julien Leduc's avatar
Julien Leduc committed
272
273
274
275
276
277


# allow eos to start
kubectl --namespace=${instance} exec ctaeos -- touch /CANSTART


Julien Leduc's avatar
Julien Leduc committed
278
# create users on the mgm
279
# this is done in ctaeos-mgm.sh as the mgm needs this to setup the ACLs
Julien Leduc's avatar
Julien Leduc committed
280
281

# use krb5 and then unix fod xrootd protocol on the client pod for eos, xrdcp and cta everything should be fine!
282
283
284
echo "XrdSecPROTOCOL=krb5,unix" | kubectl --namespace=${instance} exec -i client -- bash -c "cat >> /etc/xrootd/client.conf"
# May be needed for the client to make sure that SSS is not used by default but krb5...
#echo "XrdSecPROTOCOL=krb5,unix" | kubectl --namespace=${instance} exec -i client -- bash -c "cat >> /etc/xrootd/client.conf"
285
echo OK
286

287
288
289
echo "klist for client:"
kubectl --namespace=${instance} exec client klist

290
291
292
293
294
295
296
echo "klist for ctacli:"
kubectl --namespace=${instance} exec ctacli klist


echo -n "Configuring cta SSS for ctafrontend access from ctaeos"
for ((i=0; i<300; i++)); do
  echo -n "."
297
  [ "`kubectl --namespace=${instance} exec ctaeos -- bash -c "[ -f /etc/eos.keytab ] && echo -n Ready || echo -n Not ready"`" = "Ready" ] && break
298
299
  sleep 1
done
300
301
302
[ "`kubectl --namespace=${instance} exec ctaeos -- bash -c "[ -f /etc/eos.keytab ] && echo -n Ready || echo -n Not ready"`" = "Ready" ] || die "TIMED OUT"
kubectl --namespace=${instance} exec ctaeos -- grep ${EOSINSTANCE} /etc/eos.keytab | sed "s/daemon/${EOSINSTANCE}/g" |\
kubectl --namespace=${instance} exec -i ctafrontend -- \
303
bash -c "cat > /etc/cta/eos.sss.keytab.tmp; chmod 400 /etc/cta/eos.sss.keytab.tmp; chown cta:cta /etc/cta/eos.sss.keytab.tmp; mv /etc/cta/eos.sss.keytab.tmp /etc/cta/eos.sss.keytab"
304
echo OK
305
306
307
308
309
310
311

echo -n "Waiting for EOS to be configured"
for ((i=0; i<300; i++)); do
  echo -n "."
  kubectl --namespace=${instance} logs ctaeos | grep -q  "### ctaeos mgm ready ###" && break
  sleep 1
done
312
kubectl --namespace=${instance} logs ctaeos | grep -q  "### ctaeos mgm ready ###" || die "TIMED OUT"
313
314
echo OK

315

316
317
318
# Set the workflow rules for archiving, creating tape file replicas in the EOS namespace, retrieving
# files from tape and deleting files.
#
319
320
321
322
323
324
# The FQDN can be set as follows:
# CTA_ENDPOINT=ctafrontend.${instance}.svc.cluster.local:10955
#
# however the simple hostname should be sufficient:
CTA_ENDPOINT=ctafrontend:10955

325
echo "Setting workflows in namespace ${instance} pod ctaeos:"
326
CTA_WF_DIR=/eos/${EOSINSTANCE}/proc/cta/workflow
327
for WORKFLOW in sync::create.default closew.default archived.default sync::prepare.default sync::delete.default
328
do
Steven Murray's avatar
Steven Murray committed
329
330
  echo "eos attr set sys.workflow.${WORKFLOW}=\"proto:${CTA_ENDPOINT} ctafrontend\" ${CTA_WF_DIR}"
  kubectl --namespace=${instance} exec ctaeos -- bash -c "eos attr set sys.workflow.${WORKFLOW}=\"proto:${CTA_ENDPOINT} ctafrontend\" ${CTA_WF_DIR}"
331
332
333
done


334
echo -n "Copying eos SSS on ctacli and client pods to allow recalls"
335
336
kubectl --namespace=${instance} exec ctaeos cat /etc/eos.keytab | kubectl --namespace=${instance} exec -i ctacli --  bash -c "cat > /etc/eos.keytab; chmod 600 /etc/eos.keytab"
kubectl --namespace=${instance} exec ctaeos cat /etc/eos.keytab | kubectl --namespace=${instance} exec -i client --  bash -c "cat > /etc/eos.keytab; chmod 600 /etc/eos.keytab"
337
338
339
echo OK


340
341
342
343
echo "Instance ${instance} successfully created:"
kubectl get pods -a --namespace=${instance}

exit 0