From e01f61606bfc821dc7ef21f4b894cc0e579e638e Mon Sep 17 00:00:00 2001
From: Julien Leduc <julien.leduc@cern.ch>
Date: Mon, 9 Jan 2023 16:08:26 +0100
Subject: [PATCH] Updating CI to run with latest kubernetes versions as well as
older kubernetes version on our historical runners.
---
.../orchestration/create_instance.sh | 59 +++++++++++--------
.../orchestration/run_systemtest.sh | 3 +
.../orchestration/tests/prepare_tests.sh | 6 +-
3 files changed, 39 insertions(+), 29 deletions(-)
diff --git a/continuousintegration/orchestration/create_instance.sh b/continuousintegration/orchestration/create_instance.sh
index 757db5abf7..64adf8b05d 100755
--- a/continuousintegration/orchestration/create_instance.sh
+++ b/continuousintegration/orchestration/create_instance.sh
@@ -194,6 +194,8 @@ else
else
echo "Creating instance for image built on commit ${COMMITID} with gitlab pipeline ID ${pipelineid}"
imagetag=$(../ci_helpers/list_images.sh 2>/dev/null | grep ${COMMITID} | grep ^${pipelineid}git | sort -n | tail -n1)
+ # just a shortcut to avoid time lost checking against the docker registry...
+ #imagetag=${pipelineid}git${COMMITID}
fi
if [ "${imagetag}" == "" ]; then
echo "commit:${COMMITID} has no docker image available in gitlab registry, please check pipeline status and registry images available."
@@ -234,6 +236,11 @@ echo -n "Creating ${instance} instance "
kubectl create namespace ${instance} || die "FAILED"
+# Does kubernetes version supports `get pod --show-all=true`?
+# use it for older versions and this is not needed for new versions of kubernetes
+KUBECTL_DEPRECATED_SHOWALL=$(kubectl get pod --show-all=true >/dev/null 2>&1 && echo "--show-all=true")
+test -z ${KUBECTL_DEPRECATED_SHOWALL} || echo "WARNING: you are running a old version of kubernetes and should think about updating it."
+
# The CTA registry secret must be copied in the instance namespace to be usable
kubectl get secret ${ctareg_secret} &> /dev/null
if [ $? -eq 0 ]; then
@@ -248,7 +255,7 @@ kubectl --namespace ${instance} create configmap buildtree --from-literal=base=$
if [ ! -z "${additional_resources}" ]; then
kubectl --namespace ${instance} create -f ${additional_resources} || die "Could not create additional resources described in ${additional_resources}"
- kubectl --namespace ${instance} get pods -a
+ kubectl --namespace ${instance} get pod ${KUBECTL_DEPRECATED_SHOWALL}
fi
echo "creating configmaps in instance"
@@ -267,7 +274,7 @@ for ((i=0; i<120; i++)); do
done
kubectl get persistentvolumeclaim claimlibrary --namespace=${instance} | grep -q Bound || die "TIMED OUT"
echo "OK"
-LIBRARY_DEVICE=$(kubectl get persistentvolumeclaim claimlibrary --namespace=${instance} -o yaml| grep -i volumeName | sed -e 's%.*sg%sg%')
+LIBRARY_DEVICE=$(kubectl get persistentvolumeclaim claimlibrary --namespace=${instance} -o json | jq -r '.spec.volumeName')
kubectl --namespace=${instance} create -f /opt/kubernetes/CTA/library/config/library-config-${LIBRARY_DEVICE}.yaml
@@ -293,18 +300,18 @@ sed "s/SCHEMA_VERSION_VALUE/${SCHEMA_VERSION}/g" ${poddir}/pod-init.yaml | kubec
echo -n "Waiting for init"
for ((i=0; i<400; i++)); do
echo -n "."
- kubectl get pod init -a --namespace=${instance} | egrep -q 'Completed|Error' && break
+ kubectl --namespace=${instance} get pod init ${KUBECTL_DEPRECATED_SHOWALL} -o json | jq -r .status.phase | egrep -q 'Succeeded|Failed' && break
sleep 1
done
# initialization went wrong => exit now with error
-if $(kubectl get pod init -a --namespace=${instance} | grep -q Error); then
+if $(kubectl --namespace=${instance} get pod init ${KUBECTL_DEPRECATED_SHOWALL} -o json | jq -r .status.phase | grep -q Failed); then
echo "init pod in Error status here are its last log lines:"
kubectl --namespace=${instance} logs init --tail 10
die "ERROR: init pod in ErERROR: init pod in Error state. Initialization failed."
fi
-kubectl get pod init -a --namespace=${instance} | grep -q Completed || die "TIMED OUT"
+kubectl --namespace=${instance} get pod init ${KUBECTL_DEPRECATED_SHOWALL} -o json | jq -r .status.phase | grep -q Succeeded || die "TIMED OUT"
echo OK
if [ $runoracleunittests == 1 ] ; then
@@ -314,7 +321,7 @@ if [ $runoracleunittests == 1 ] ; then
echo -n "Waiting for oracleunittests"
for ((i=0; i<400; i++)); do
echo -n "."
- kubectl get pod oracleunittests -a --namespace=${instance} | egrep -q 'Completed|Error' && break
+ kubectl --namespace=${instance} get pod oracleunittests ${KUBECTL_DEPRECATED_SHOWALL} -o json | jq -r .status.phase | egrep -q 'Succeeded|Failed' && break
sleep 1
done
echo "\n"
@@ -322,8 +329,8 @@ if [ $runoracleunittests == 1 ] ; then
kubectl --namespace=${instance} logs oracleunittests
# database unit-tests went wrong => exit now with error
- if $(kubectl get pod oracleunittests -a --namespace=${instance} | grep -q Error); then
- echo "init pod in Error status here are its last log lines:"
+ if $(kubectl --namespace=${instance} get pod oracleunittests ${KUBECTL_DEPRECATED_SHOWALL} -o json | jq -r .status.phase | grep -q Failed); then
+ echo "oracleunittests pod in Error status here are its last log lines:"
kubectl --namespace=${instance} logs oracleunittests --tail 10
die "ERROR: oracleunittests pod in Error state. Initialization failed."
fi
@@ -342,14 +349,14 @@ echo -n "Waiting for other pods"
for ((i=0; i<240; i++)); do
echo -n "."
# exit loop when all pods are in Running state
- kubectl get pods -a --namespace=${instance} | grep -v init | grep -v oracleunittests | tail -n+2 | grep -q -v Running || break
+ kubectl -n ${instance} get pod ${KUBECTL_DEPRECATED_SHOWALL} -o json | jq -r '.items[] | select(.metadata.name != "init") | select(.metadata.name != "oracleunittests") | .status.phase'| grep -q -v Running || break
sleep 1
done
-if [[ $(kubectl get pods -a --namespace=${instance} | grep -v init | grep -v oracleunittests | tail -n+2 | grep -q -v Running) ]]; then
+if [[ $(kubectl -n toto get pod ${KUBECTL_DEPRECATED_SHOWALL} -o json | jq -r '.items[] | select(.metadata.name != "init") | select(.metadata.name != "oracleunittests") | .status.phase'| grep -q -v Running) ]]; then
echo "TIMED OUT"
echo "Some pods have not been initialized properly:"
- kubectl get pods -a --namespace=${instance}
+ kubectl --namespace=${instance} get pod ${KUBECTL_DEPRECATED_SHOWALL}
exit 1
fi
echo OK
@@ -366,15 +373,15 @@ done
echo OK
echo -n "Configuring KDC clients (frontend, cli...) "
-kubectl --namespace=${instance} exec kdc cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i client -- bash -c "cat > /etc/krb5.conf"
-kubectl --namespace=${instance} exec kdc cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i ctacli -- bash -c "cat > /etc/krb5.conf"
-kubectl --namespace=${instance} exec kdc cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i ctafrontend -- bash -c "cat > /etc/krb5.conf"
-kubectl --namespace=${instance} exec kdc cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i ctaeos -- bash -c "cat > /etc/krb5.conf"
-kubectl --namespace=${instance} exec kdc cat /root/ctaadmin1.keytab | kubectl --namespace=${instance} exec -i ctacli -- bash -c "cat > /root/ctaadmin1.keytab"
-kubectl --namespace=${instance} exec kdc cat /root/user1.keytab | kubectl --namespace=${instance} exec -i client -- bash -c "cat > /root/user1.keytab"
+kubectl --namespace=${instance} exec kdc -- cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i client -- bash -c "cat > /etc/krb5.conf"
+kubectl --namespace=${instance} exec kdc -- cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i ctacli -- bash -c "cat > /etc/krb5.conf"
+kubectl --namespace=${instance} exec kdc -- cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i ctafrontend -- bash -c "cat > /etc/krb5.conf"
+kubectl --namespace=${instance} exec kdc -- cat /etc/krb5.conf | kubectl --namespace=${instance} exec -i ctaeos -- bash -c "cat > /etc/krb5.conf"
+kubectl --namespace=${instance} exec kdc -- cat /root/ctaadmin1.keytab | kubectl --namespace=${instance} exec -i ctacli -- bash -c "cat > /root/ctaadmin1.keytab"
+kubectl --namespace=${instance} exec kdc -- cat /root/user1.keytab | kubectl --namespace=${instance} exec -i client -- bash -c "cat > /root/user1.keytab"
# need to mkdir /etc/cta folder as cta rpm may not already be installed (or put it somewhere else and move it later???)
-kubectl --namespace=${instance} exec kdc cat /root/cta-frontend.keytab | kubectl --namespace=${instance} exec -i ctafrontend -- bash -c "mkdir -p /etc/cta; cat > /etc/cta/cta-frontend.krb5.keytab"
-kubectl --namespace=${instance} exec kdc cat /root/eos-server.keytab | kubectl --namespace=${instance} exec -i ctaeos -- bash -c "cat > /etc/eos-server.krb5.keytab"
+kubectl --namespace=${instance} exec kdc -- cat /root/cta-frontend.keytab | kubectl --namespace=${instance} exec -i ctafrontend -- bash -c "mkdir -p /etc/cta; cat > /etc/cta/cta-frontend.krb5.keytab"
+kubectl --namespace=${instance} exec kdc -- cat /root/eos-server.keytab | kubectl --namespace=${instance} exec -i ctaeos -- bash -c "cat > /etc/eos-server.krb5.keytab"
kubectl --namespace=${instance} exec ctacli -- kinit -kt /root/ctaadmin1.keytab ctaadmin1@TEST.CTA
kubectl --namespace=${instance} exec client -- kinit -kt /root/user1.keytab user1@TEST.CTA
@@ -401,10 +408,10 @@ echo "XrdSecPROTOCOL=krb5,unix" | kubectl --namespace=${instance} exec -i client
echo OK
echo "klist for client:"
-kubectl --namespace=${instance} exec client klist
+kubectl --namespace=${instance} exec client -- klist
echo "klist for ctacli:"
-kubectl --namespace=${instance} exec ctacli klist
+kubectl --namespace=${instance} exec ctacli -- klist
echo -n "Configuring cta SSS for ctafrontend access from ctaeos"
@@ -448,8 +455,8 @@ done
echo -n "Copying eos SSS on ctacli and client pods to allow recalls"
-kubectl --namespace=${instance} exec ctaeos cat /etc/eos.keytab | kubectl --namespace=${instance} exec -i ctacli -- bash -c "cat > /etc/eos.keytab; chmod 600 /etc/eos.keytab"
-kubectl --namespace=${instance} exec ctaeos cat /etc/eos.keytab | kubectl --namespace=${instance} exec -i client -- bash -c "cat > /etc/eos.keytab; chmod 600 /etc/eos.keytab"
+kubectl --namespace=${instance} exec ctaeos -- cat /etc/eos.keytab | kubectl --namespace=${instance} exec -i ctacli -- bash -c "cat > /etc/eos.keytab; chmod 600 /etc/eos.keytab"
+kubectl --namespace=${instance} exec ctaeos -- cat /etc/eos.keytab | kubectl --namespace=${instance} exec -i client -- bash -c "cat > /etc/eos.keytab; chmod 600 /etc/eos.keytab"
echo OK
# In case of testing to update the database using liquibase.
@@ -468,7 +475,7 @@ if [ "${MAJOR}" == "${NEW_MAJOR}" ] ; then
fi
echo "Instance ${instance} successfully created:"
-kubectl get pods -a --namespace=${instance}
+kubectl --namespace=${instance} get pod ${KUBECTL_DEPRECATED_SHOWALL}
if [ $runexternaltapetests == 1 ] ; then
echo "Running database unit-tests"
@@ -477,8 +484,8 @@ if [ $runexternaltapetests == 1 ] ; then
kubectl --namespace=${instance} logs externaltapetests
# database unit-tests went wrong => exit now with error
- if $(kubectl get pod externaltapetests -a --namespace=${instance} | grep -q Error); then
- echo "init pod in Error status here are its last log lines:"
+ if $(kubectl --namespace=${instance} get pod externaltapetests ${KUBECTL_DEPRECATED_SHOWALL} -o json | jq -r .status.phase | egrep -q 'Failed'); then
+ echo "externaltapetests pod in Failed status here are its last log lines:"
kubectl --namespace=${instance} logs externaltapetests --tail 10
die "ERROR: externaltapetests pod in Error state. Initialization failed."
fi
diff --git a/continuousintegration/orchestration/run_systemtest.sh b/continuousintegration/orchestration/run_systemtest.sh
index bd031a85d3..dd51af94d3 100755
--- a/continuousintegration/orchestration/run_systemtest.sh
+++ b/continuousintegration/orchestration/run_systemtest.sh
@@ -43,6 +43,9 @@ PREFLIGHTTEST_TIMEOUT=60
SYSTEMTEST_TIMEOUT=3600
# by default do not cleanup leftover namespaces
cleanup_namespaces=0
+# By default assume that ORACLE_SUPPORT is ON
+# if this script is running outside of gitlab CI
+test -z ${ORACLE_SUPPORT+x} && ORACLE_SUPPORT="ON"
die() { echo "$@" 1>&2 ; exit 1; }
diff --git a/continuousintegration/orchestration/tests/prepare_tests.sh b/continuousintegration/orchestration/tests/prepare_tests.sh
index 8fbae32920..7e87507b08 100755
--- a/continuousintegration/orchestration/tests/prepare_tests.sh
+++ b/continuousintegration/orchestration/tests/prepare_tests.sh
@@ -350,9 +350,9 @@ echo "Adding super client capabilities"
clientIP=`kubectl --namespace ${NAMESPACE} describe pod client | grep IP | sed -E 's/IP:[[:space:]]+//'`
kubectl --namespace ${NAMESPACE} exec ctacli -- cta-admin admin add --username ctaadmin2 --comment "ctaadmin2"
-kubectl --namespace=${NAMESPACE} exec kdc cat /root/ctaadmin2.keytab | kubectl --namespace=${NAMESPACE} exec -i client -- bash -c "cat > /root/ctaadmin2.keytab; mkdir -p /tmp/ctaadmin2"
-kubectl --namespace=${NAMESPACE} exec kdc cat /root/poweruser1.keytab | kubectl --namespace=${NAMESPACE} exec -i client -- bash -c "cat > /root/poweruser1.keytab; mkdir -p /tmp/poweruser1"
-kubectl --namespace=${NAMESPACE} exec kdc cat /root/eosadmin1.keytab | kubectl --namespace=${NAMESPACE} exec -i client -- bash -c "cat > /root/eosadmin1.keytab; mkdir -p /tmp/eosadmin1"
+kubectl --namespace=${NAMESPACE} exec kdc -- cat /root/ctaadmin2.keytab | kubectl --namespace=${NAMESPACE} exec -i client -- bash -c "cat > /root/ctaadmin2.keytab; mkdir -p /tmp/ctaadmin2"
+kubectl --namespace=${NAMESPACE} exec kdc -- cat /root/poweruser1.keytab | kubectl --namespace=${NAMESPACE} exec -i client -- bash -c "cat > /root/poweruser1.keytab; mkdir -p /tmp/poweruser1"
+kubectl --namespace=${NAMESPACE} exec kdc -- cat /root/eosadmin1.keytab | kubectl --namespace=${NAMESPACE} exec -i client -- bash -c "cat > /root/eosadmin1.keytab; mkdir -p /tmp/eosadmin1"
###
# Filling services in DNS on all pods
--
GitLab