.gitlab-ci.yml 11.2 KB
Newer Older
1
stages:
Julien Leduc's avatar
Julien Leduc committed
2
3
  - build:srpm
  - build:rpm
4
  - build:dockerimage
5
  - rename:dockerimage
6
  - test
7
  - regressions
8
  - shouldfail
9
  - publish_rpm
10

11
before_script:
12
  - export CTA_BUILD_ID=${CI_PIPELINE_ID}git${CI_COMMIT_SHA:0:8}
13
  - echo "Exporting CTA_BUILD_ID=${CTA_BUILD_ID}"
14
15
  - test -n "${CI_COMMIT_TAG}" && export TAG_VERSION=$(echo ${CI_COMMIT_TAG} | sed -e 's/^v//;s/-.*$//')
  - test -n "${CI_COMMIT_TAG}" && export TAG_RELEASE=$(echo ${CI_COMMIT_TAG} | sed -e 's/^[^-]*-//')
16
17
  - echo "Removing protectbase from all repos (same as Puppet)"
  - sed -i '/^protect=/d' /etc/yum.repos.d/* 
18

19
20
cta_srpm:
  stage: build:srpm
21
  retry: 1
Julien Leduc's avatar
Julien Leduc committed
22
23
24
  image: gitlab-registry.cern.ch/linuxsupport/cc7-base
  script:
    - yum install -y gcc-c++ cmake make rpm-build
Julien Leduc's avatar
Julien Leduc committed
25
26
    - mkdir build_srpm
    - cd build_srpm
27
    - cmake -DPackageOnly:Bool=true -DVCS_VERSION=${CTA_BUILD_ID} ..
Julien Leduc's avatar
Julien Leduc committed
28
    - make cta_srpm
29

Julien Leduc's avatar
Julien Leduc committed
30
  artifacts:
31
    expire_in: 30 days
Julien Leduc's avatar
Julien Leduc committed
32
    paths:
Julien Leduc's avatar
Julien Leduc committed
33
34
    - build_srpm/RPM/SRPMS/

35
  tags:
Julien Leduc's avatar
Julien Leduc committed
36
37
    - docker

38
cta_rpm:
39
40
  except:
    - tags
41
  stage: build:rpm
42
  retry: 1
Julien Leduc's avatar
Julien Leduc committed
43
44
  image: gitlab-registry.cern.ch/linuxsupport/cc7-base
  script:
45
    - cp -f continuousintegration/docker/ctafrontend/cc7/etc/yum.repos.d/*.repo /etc/yum.repos.d/
Julien Leduc's avatar
Julien Leduc committed
46
    - yum install -y gcc-c++ cmake make rpm-build
47
    - yum -y install yum-plugin-priorities yum-plugin-versionlock
Michael Davis's avatar
Michael Davis committed
48
49
    - yum install -y git
    - git submodule update --init --recursive
50
    - cp -f continuousintegration/docker/ctafrontend/cc7/etc/yum/pluginconf.d/versionlock.list /etc/yum/pluginconf.d/
Julien Leduc's avatar
Julien Leduc committed
51
52
53
    - yum-builddep --nogpgcheck -y build_srpm/RPM/SRPMS/*
    - mkdir build_rpm
    - cd build_rpm
54
    - cmake -DVCS_VERSION=${CTA_BUILD_ID} ..
Julien Leduc's avatar
Julien Leduc committed
55
56
57
    - make cta_rpm

  artifacts:
58
    expire_in: 30 days
Julien Leduc's avatar
Julien Leduc committed
59
60
    paths:
    - build_rpm/RPM/RPMS
61
62
63

  tags:
    - docker
Julien Leduc's avatar
Julien Leduc committed
64

65
cta_tagged_rpm:
66
67
  only:
    - tags
68
  stage: build:rpm
69
  retry: 1
70
71
72
  image: gitlab-registry.cern.ch/linuxsupport/cc7-base
  script:
    - if [ -z "${CI_COMMIT_TAG}" ]; then echo "This is not a tagged commit, exiting"; exit 0; fi
Julien Leduc's avatar
Julien Leduc committed
73
    - echo "Building package for tag ${CI_COMMIT_TAG}. CTA_VERSION=${TAG_VERSION} - CTA_RELEASE=${TAG_RELEASE}"
74
75
    - cp -f continuousintegration/docker/ctafrontend/cc7/etc/yum.repos.d/*.repo /etc/yum.repos.d/
    - yum install -y gcc-c++ cmake make rpm-build
76
    - yum -y install yum-plugin-priorities yum-plugin-versionlock
Michael Davis's avatar
Michael Davis committed
77
78
    - yum install -y git
    - git submodule update --init --recursive
79
    - cp -f continuousintegration/docker/ctafrontend/cc7/etc/yum/pluginconf.d/versionlock.list /etc/yum/pluginconf.d/
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    - yum-builddep --nogpgcheck -y build_srpm/RPM/SRPMS/*
    - mkdir build_tagged_rpm
    - cd build_tagged_rpm
    - CTA_VERSION=${TAG_VERSION} cmake -DVCS_VERSION=${TAG_RELEASE} ..
    - make cta_rpm

  artifacts:
    expire_in: 30 days
    paths:
    - build_tagged_rpm/RPM/RPMS

  tags:
    - docker

Julien Leduc's avatar
Julien Leduc committed
94
publish_tagged_rpm:
95
  stage: publish_rpm
96
97
  only:
    - tags
98
99
100
  image: gitlab-registry.cern.ch/linuxsupport/cc7-base
  script:
    - if [ -z "${CI_COMMIT_TAG}" ]; then echo "This is not a tagged commit, exiting"; exit 0; fi
Julien Leduc's avatar
Julien Leduc committed
101
    - EOS_ACCOUNT_USERNAME=${DOCKER_LOGIN_USERNAME} EOS_ACCOUNT_PASSWORD=${DOCKER_LOGIN_PASSWORD} CI_OUTPUT_DIR=build_tagged_rpm/RPM/RPMS EOS_PATH=/eos/user/c/ctareg/www/cta-repo HOOK=/eos/user/c/ctareg/www/cta-repo/update_repos.sh continuousintegration/ci_helpers/deploy-eos.sh
102
103
104
  tags:
    - docker

105
cta_valgrind:
106
107
  except:
    - tags
108
  stage: test
109
  retry: 1
110
111
  variables:
    CTAREPODIR: /tmp/repo
112
113
114
  image: gitlab-registry.cern.ch/linuxsupport/cc7-base
  script:
    - cp -f continuousintegration/docker/ctafrontend/cc7/etc/yum.repos.d/* /etc/yum.repos.d/
115
    - yum -y install yum-plugin-priorities createrepo
116
    - cp -f continuousintegration/docker/ctafrontend/cc7/etc/yum/pluginconf.d/versionlock.list /etc/yum/pluginconf.d/
117
118
119
    - mkdir -p ${CTAREPODIR}; cp -r build_rpm/RPM ${CTAREPODIR}
    - createrepo ${CTAREPODIR}; echo -e "[cta-artifacts]\nname=CTA artifacts\nbaseurl=file://${CTAREPODIR}\ngpgcheck=0\nenabled=1\npriority=2" > /etc/yum.repos.d/cta-artifacts.repo
    - yum -y --nogpgcheck install cta-systemtests cta-debuginfo sqlite-debuginfo --enablerepo=debug
120
121
122
123
124
    - /usr/bin/cta-unitPlusSystemTests.sh

  tags:
    - docker

125
dbunittests_postgresql:
126
127
  except:
    - tags
128
  stage: test
129
130
131
132
133
134
135
136
137
138
139
  variables:
    CTAREPODIR: /tmp/repo
  image: gitlab-registry.cern.ch/linuxsupport/cc7-base
  script:
    - cp -f continuousintegration/docker/ctafrontend/cc7/etc/yum.repos.d/* /etc/yum.repos.d/
    - yum -y install yum-plugin-priorities createrepo
    - cp -f continuousintegration/docker/ctafrontend/cc7/etc/yum/pluginconf.d/versionlock.list /etc/yum/pluginconf.d/
    - mkdir -p ${CTAREPODIR}; cp -r build_rpm/RPM ${CTAREPODIR}
    - createrepo ${CTAREPODIR}; echo -e "[cta-artifacts]\nname=CTA artifacts\nbaseurl=file://${CTAREPODIR}\ngpgcheck=0\nenabled=1\npriority=2" > /etc/yum.repos.d/cta-artifacts.repo
    - yum -y --nogpgcheck install cta-systemtests cta-debuginfo sqlite-debuginfo --enablerepo=debug
    - yum -y install cta-catalogueutils
140
    - yum -y install rh-postgresql12-postgresql-server
141
142
143
144
145
146
147
148
    - POSTGRESQL_DATA_DIR=/usr/local/cta_test_pgsql_data
    - POSTGRESQL_LOG_DIR=/var/log/postgres
    - echo POSTGRESQL_DATA_DIR=${POSTGRESQL_DATA_DIR}
    - echo POSTGRESQL_LOG_DIR=${POSTGRESQL_LOG_DIR}
    - mkdir -p ${POSTGRESQL_DATA_DIR}
    - chown -R postgres:postgres ${POSTGRESQL_DATA_DIR}
    - mkdir -p ${POSTGRESQL_LOG_DIR}
    - chown -R postgres:postgres ${POSTGRESQL_LOG_DIR}
149
150
    - export LD_LIBRARY_PATH=/opt/rh/rh-postgresql12/root/usr/lib64
    - POSTGRES_BIN=/opt/rh/rh-postgresql12/root/usr/bin
151
152
153
154
155
156
157
158
159
160
    - echo POSTGRES_BIN=${POSTGRES_BIN}
    - runuser -u postgres -- ${POSTGRES_BIN}/initdb -D ${POSTGRESQL_DATA_DIR}
    - runuser -u postgres -- ${POSTGRES_BIN}/pg_ctl start -w -t 10 -D ${POSTGRESQL_DATA_DIR} -l ${POSTGRESQL_LOG_DIR}/cta_test_postgres.log
    - runuser -u postgres -- ${POSTGRES_BIN}/createdb cta
    - runuser -u postgres -- ${POSTGRES_BIN}/createuser -E cta
    - CTA_CATALOGUE_CONF=/etc/cta/cta-catalogue.conf
    - echo CTA_CATALOGUE_CONF=${CTA_CATALOGUE_CONF}
    - echo 'postgresql:postgresql://cta@localhost/cta' > ${CTA_CATALOGUE_CONF}
    - /usr/bin/cta-catalogue-schema-create ${CTA_CATALOGUE_CONF}
    - /usr/bin/cta-rdbmsUnitTests ${CTA_CATALOGUE_CONF}
161
    - valgrind /usr/bin/cta-rdbmsUnitTests ${CTA_CATALOGUE_CONF}
162
163
164
165
166
    - runuser -u postgres -- ${POSTGRES_BIN}/pg_ctl stop -D ${POSTGRESQL_DATA_DIR}

  tags:
    - docker

167
168
169
dbunittests_mysql:
  except:
    - tags
170
  stage: test
171
172
173
174
175
176
177
178
179
180
181
  variables:
    CTAREPODIR: /tmp/repo
  image: gitlab-registry.cern.ch/linuxsupport/cc7-base
  script:
    - cp -f continuousintegration/docker/ctafrontend/cc7/etc/yum.repos.d/* /etc/yum.repos.d/
    - yum -y install yum-plugin-priorities createrepo
    - cp -f continuousintegration/docker/ctafrontend/cc7/etc/yum/pluginconf.d/versionlock.list /etc/yum/pluginconf.d/
    - mkdir -p ${CTAREPODIR}; cp -r build_rpm/RPM ${CTAREPODIR}
    - createrepo ${CTAREPODIR}; echo -e "[cta-artifacts]\nname=CTA artifacts\nbaseurl=file://${CTAREPODIR}\ngpgcheck=0\nenabled=1\npriority=2" > /etc/yum.repos.d/cta-artifacts.repo
    - yum -y --nogpgcheck install cta-systemtests cta-debuginfo sqlite-debuginfo --enablerepo=debug
    - yum -y install cta-catalogueutils
182
183
184
185
186
187
188
    - yum -y install lsof rh-mariadb103-mariadb-server
    - echo '[mariadb]' > /etc/opt/rh/rh-mariadb103/my.cnf.d/cta_server_listen.cnf
    - echo 'bind-address=localhost' >> /etc/opt/rh/rh-mariadb103/my.cnf.d/cta_server_listen.cnf
    - scl enable rh-mariadb103 -- mysql_install_db --user=mysql
    - cd '/opt/rh/rh-mariadb103/root/usr'
    - scl enable rh-mariadb103 -- /opt/rh/rh-mariadb103/root/usr/bin/mysqld_safe --user=mysql --datadir=/var/opt/rh/rh-mariadb103/lib/mysql < /dev/null &
    - echo "Sleeping 5 seconds"
189
    - sleep 5
190
191
192
193
194
    - echo "Finished sleeping"
    - echo 'create database cta;' | scl enable rh-mariadb103 -- mysql
    - echo 'create user cta@localhost;' | scl enable rh-mariadb103 -- mysql
    - echo 'grant all privileges on *.* to cta@localhost' | scl enable rh-mariadb103 -- mysql
    - echo 'flush privileges;' | scl enable rh-mariadb103 -- mysql
195
196
197
198
199
    - CTA_CATALOGUE_CONF=/etc/cta/cta-catalogue.conf
    - echo CTA_CATALOGUE_CONF=${CTA_CATALOGUE_CONF}
    - echo 'mysql://cta@localhost:3306/cta' > ${CTA_CATALOGUE_CONF}
    - /usr/bin/cta-catalogue-schema-create ${CTA_CATALOGUE_CONF}
    - /usr/bin/cta-rdbmsUnitTests ${CTA_CATALOGUE_CONF}
200
    - valgrind /usr/bin/cta-rdbmsUnitTests ${CTA_CATALOGUE_CONF}
201
202
203
204
205
    - kill `ps -ef | egrep '^mysql.*/usr/libexec/mysqld' | awk '{print $2;}'`

  tags:
    - docker

206
ctageneric_docker:
207
208
  except:
    - tags
Julien Leduc's avatar
Julien Leduc committed
209
  stage: build:dockerimage
210
  retry: 1
Julien Leduc's avatar
Julien Leduc committed
211
212
  tags:
    - docker-image-build
213
  script:
214
    - ""
Julien Leduc's avatar
Julien Leduc committed
215
  variables:
216
    TO: gitlab-registry.cern.ch/cta/ctageneric:${CI_PIPELINE_ID}
217
    DOCKER_FILE: continuousintegration/docker/ctafrontend/cc7/ci_runner/Dockerfile
218

219
ctageneric_docker_rename:
220
221
  except:
    - tags
222
223
  stage: rename:dockerimage
  variables:
224
225
    TO: gitlab-registry.cern.ch/cta/ctageneric
    REPOSITORY: cta/ctageneric
226
227
    GITLAB_HOST: gitlab.cern.ch
  script:
228
229
230
231
    - export OLDTAG=${CI_PIPELINE_ID}
    - export NEWTAG=${CTA_BUILD_ID}
    - cd continuousintegration/ci_helpers/
    - ./rename_tag.sh
232
233

archiveretrieve:
234
235
  except:
    - tags
236
  stage: test
237
  script:
238
    - export NAMESPACE="archiveretrieve-${CTA_BUILD_ID}-$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 4 | head -n 1)"
239
    - cd continuousintegration/orchestration/; ./run_systemtest.sh -n ${NAMESPACE} -p ${CI_PIPELINE_ID} -s tests/archive_retrieve.sh -O -D -t 2400 -C
240
  artifacts:
241
    when: always
242
    expire_in: 30 days
243
244
    paths:
    - pod_logs
245
246
247
  tags:
    - kubernetes
    - mhvtl
Cedric CAFFY's avatar
Cedric CAFFY committed
248
249
250
251
252
253
254
    
repack:
  except:
    - tags
  stage: test
  script:
    - export NAMESPACE="repack-${CTA_BUILD_ID}-$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 4 | head -n 1)"
255
    - cd continuousintegration/orchestration/; ./run_systemtest.sh -n ${NAMESPACE} -p ${CI_PIPELINE_ID} -s tests/repack_systemtest_wrapper.sh -O -D -t 2400 -C
Cedric CAFFY's avatar
Cedric CAFFY committed
256
257
258
259
260
261
262
263
  artifacts:
    when: always
    expire_in: 30 days
    paths:
    - pod_logs
  tags:
    - kubernetes
    - mhvtl
264

265
266
267
268
269
dbunittests_oracle:
  except:
    - tags
  stage: test
  script:
270
    - export NAMESPACE="dbunittestsoracle-${CTA_BUILD_ID}-$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 4 | head -n 1)"
271
    - cd continuousintegration/orchestration/; ./run_systemtest.sh -n ${NAMESPACE} -p ${CI_PIPELINE_ID} -s /usr/bin/true -O -D -U -t 600 -C
272
273
274
275
276
277
278
279
  artifacts:
    when: always
    expire_in: 30 days
    paths:
    - pod_logs
  tags:
    - kubernetes

280
nightly1:
281
  stage: shouldfail
282
  only:
283
    - triggers
284
  script:
285
    - echo "Running nightly tests"
286
    - export NAMESPACE="archiveretrieve-${CTA_BUILD_ID}-$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 4 | head -n 1)"
287
    - cd continuousintegration/orchestration/; ./run_systemtest.sh -n ${NAMESPACE} -p ${CI_PIPELINE_ID} -s tests/test_nightly1.sh -O -D
288
289
290
291
292
293
294
295
  artifacts:
    when: always
    expire_in: 30 days
    paths:
    - pod_logs
  tags:
    - kubernetes
    - mhvtl
296
    - xlarge
297
298

regressioneos:
299
  stage: regressions
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
  only:
    - triggers
    - schedules
  script:
    - echo "Running nightly eos regression tests"
    - export NAMESPACE="archiveretrieve-${CTA_BUILD_ID}-$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 4 | head -n 1)"
    - cd continuousintegration/orchestration/; ./run_systemtest.sh -n ${NAMESPACE} -p ${CI_PIPELINE_ID} -s tests/upgrade-eos.sh -O -D -S
  artifacts:
    when: always
    expire_in: 30 days
    paths:
    - pod_logs
  tags:
    - kubernetes
    - mhvtl
    - xlarge