From a065953e1bb3ddaa4b2a5669b57bb0af36fa24f2 Mon Sep 17 00:00:00 2001
From: George Sedov <george.sedov@desy.de>
Date: Thu, 9 Mar 2023 17:54:01 +0100
Subject: [PATCH] Update .gitlab-ci.yml
---
.gitlab-ci.yml | 411 ++-----------------------------------------------
1 file changed, 16 insertions(+), 395 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 43f76f4b7..e85ca11d0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,401 +1,22 @@
-include:
- - local: deploy/build_env/services-linux/gitlab-ci-build-docker.yml
- - local: deploy/build_env/services-windows/gitlab-ci-build-docker.yml
- - local: deploy/build_env/centos/gitlab-ci-build-docker.yml
- - local: deploy/build_env/debians/gitlab-ci-build-docker.yml
- - local: deploy/build_env/ubuntu/gitlab-ci-build-docker.yml
- - local: deploy/build_env/manylinux2014/gitlab-ci-build-docker.yml
- - local: deploy/build_env/site/gitlab-ci-build-docker.yml
- - local: deploy/secondary_services/fluentd_elastic/gitlab-ci-build-docker.yml
-
-.go-cache:
- variables:
- GOPATH: $CI_PROJECT_DIR/.go
- before_script:
- - mkdir -p .go
- cache:
- paths:
- - .go/pkg/mod/
-
-.client-packages:
- stage: build
- script:
- - mkdir $CI_PROJECT_DIR/build
- - cd $CI_PROJECT_DIR/build
- - PACKAGE_UP=${PACKAGE_TYPE^^}
- - PACKAGE_LOW=${PACKAGE_TYPE,,}
- - >
- if [ -z "$WITH_LIBFABRIC" ]; then
- ENABLE_LIBFABRIC="OFF"
- else
- ENABLE_LIBFABRIC="ON"
- fi
- - >
- if [ -z "$WITH_PYTHON2" ]; then
- BUILD_PYTHON2="OFF"
- else
- BUILD_PYTHON2="ON"
- fi
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=$ENABLE_LIBFABRIC -DINSTALL_EXAMPLES=ON -DBUILD_CLIENTS_ONLY=ON -DPACKAGE_RELEASE_SUFFIX=$OS_SUFFIX -DBUILD_PYTHON=OFF -DCPACK_PACKAGE_NAME="asapo-devel" -DCPACK_GENERATOR=$PACKAGE_UP ..
- - make -j 4
- - make package
- - rm -f CMakeCache.txt
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=$ENABLE_LIBFABRIC -DBUILD_CLIENTS_ONLY=ON -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DPACKAGE_RELEASE_SUFFIX=$OS_SUFFIX -DBUILD_PYTHON_PACKAGES="source;${PACKAGE_LOW}" -DBUILD_PYTHON2_PACKAGES=$BUILD_PYTHON2 -DBUILD_PYTHON_DOCS=OFF ..
- - make -j 1
- - mkdir -p $CI_PROJECT_DIR/linux_packages/$PACKAGE_FOLDER
- - cp $CI_PROJECT_DIR/build/asapo-devel* $CI_PROJECT_DIR/linux_packages/$PACKAGE_FOLDER
- - cp $CI_PROJECT_DIR/build/*/api/python/dist_linux/dist/* $CI_PROJECT_DIR/linux_packages/$PACKAGE_FOLDER
- tags:
- - kubernetes-executor
- rules:
- - when: on_success
- artifacts:
- paths:
- - linux_packages
- expire_in: 3 hrs
-
-.client-packages-manylinux:
- image:
- name: $CI_REGISTRY_IMAGE/asapo-packages-manylinux-build-env:latest
- entrypoint: [""]
- stage: build
+# The Docker image that will be used to build your app
+image: $CI_REGISTRY_IMAGE/asapo-site-build-env:latest
+# Functions that should be executed before the build script is run
+pages:
script:
- - python_bin=$PYTHON_PATH/bin/python
- - pip_bin=$PYTHON_PATH/bin/pip
- - $pip_bin install cython numpy==$NUMPY_VERSION
- mkdir $CI_PROJECT_DIR/build
- cd $CI_PROJECT_DIR/build
- - cmake -DENABLE_LIBFABRIC=ON -DCMAKE_BUILD_TYPE="Release" -DBUILD_CLIENTS_ONLY=ON -DPython_EXECUTABLE=$python_bin -DBUILD_PYTHON_PACKAGES=source -DBUILD_PYTHON2_PACKAGES=OFF -DNUMPY_VERSION=$NUMPY_VERSION ..
- - cd $CI_PROJECT_DIR/build/consumer/api/python/dist_linux
- - make python-dist-consumer
- - $pip_bin wheel dist/*.tar.gz --no-deps
- - cd $CI_PROJECT_DIR/build/producer/api/python/dist_linux
- - make python-dist-producer
- - $pip_bin wheel dist/*.tar.gz --no-deps
- - mkdir -p $CI_PROJECT_DIR/wheel_packages/$WHEEL_FOLDER
- - cp $CI_PROJECT_DIR/build/*/api/python/dist_linux/*.whl $CI_PROJECT_DIR/wheel_packages/$WHEEL_FOLDER
- tags:
- - kubernetes-executor
- rules:
- - when: on_success
+ - cmake -DCMAKE_BUILD_TYPE="Release" -DBUILD_CLIENTS_ONLY=ON
+ -DBUILD_PYTHON=OFF -DBUILD_ASAPO_SITE=ON -DLIBCURL_DIR=/curl
+ -DPython_EXECUTABLE="noop" ..
+ - cd docs/site
+ - npm install
+ - make site
+ - mv ./build /public
artifacts:
paths:
- - wheel_packages
- expire_in: 3 hrs
-
-build-services-linux-debug:
- extends: .go-cache
- image:
- name: $CI_REGISTRY_IMAGE/asapo-services-linux-build-env:latest
- entrypoint: [""]
- stage: build
- script:
- - mkdir $CI_PROJECT_DIR/build
- - cd $CI_PROJECT_DIR/build
- - cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DBUILD_INTEGRATION_TESTS=ON -DBUILD_EXAMPLES=ON -DBUILD_CONSUMER_TOOLS=ON -DENABLE_LIBFABRIC=ON -DENABLE_LIBFABRIC_LOCALHOST=ON -DBUILD_EVENT_MONITOR_PRODUCER=ON -DSTOP_BUILD_ON_WARNINGS=ON -DENABLE_NEW_RECEIVER_MONITORING=ON ..
- - make -j 4
- tags:
- - kubernetes-executor
- rules:
- - when: always
- artifacts:
- paths:
- - build
- exclude:
- - build/**/CMakeFiles/**/*.o
- - build/**/CMakeFiles/**/*.gcno
- - build/**/CMakeFiles/**/*.gcda
- - build/**/CMakeFiles/**/*.objlib
- - build/**/CMakeFiles/**/*.includecache
- - build/**/*.a
- expire_in: 3 hrs
-
-test-services-linux-debug:
- extends: .go-cache
- image:
- name: $CI_REGISTRY_IMAGE/asapo-services-linux-build-env:latest
- stage: test
- script:
- - bash /services_start.sh
- # - cd $CI_PROJECT_DIR/build/tests/automatic/common_scripts
- # - bash start_services.sh
- - cd $CI_PROJECT_DIR/build
- - ctest --no-compress-output -T Test -L "unit|memcheck_unit|coverage" --output-on-failure --output-junit testResult.xml
- tags:
- - kubernetes-executor
- rules:
- - when: on_success
- dependencies:
- - build-services-linux-debug
- artifacts:
- reports:
- junit: build/testResult.xml
-
-build-services-linux-release:
- extends: .go-cache
- image:
- name: $CI_REGISTRY_IMAGE/asapo-services-linux-build-env:latest
- entrypoint: [""]
- variables:
- CI_REGISTRY_IMAGE_USERNAME: "asapo-dev-registry"
- CI_REGISTRY_IMAGE_PASSWORD: ${asapo_dev_registry_password}
- stage: build
- script:
- - mkdir $CI_PROJECT_DIR/build
- - cd $CI_PROJECT_DIR/build
- - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_EXAMPLES=ON -DBUILD_CONSUMER_TOOLS=ON -DBUILD_BROKER=ON -DBUILD_INTEGRATION_TESTS=ON -DENABLE_LIBFABRIC=ON -DBUILD_CPP_DOCS=ON -DBUILD_EVENT_MONITOR_PRODUCER=ON -DSTOP_BUILD_ON_WARNINGS=ON -DENABLE_NEW_RECEIVER_MONITORING=ON ..
- - make -j 4
- tags:
- - kubernetes-executor
- rules:
- - when: on_success
- artifacts:
- paths:
- - build
- exclude:
- - build/**/CMakeFiles/**/*
- - build/**/*.a
- expire_in: 3 hrs
-
-build-packages-centos7:
- extends: .client-packages
- image:
- name: $CI_REGISTRY_IMAGE/asapo-packages-centos7-build-env:latest
- entrypoint: [""]
- variables:
- PACKAGE_TYPE: "RPM"
- WITH_LIBFABRIC: "YES"
- WITH_PYTHON2: "YES"
- OS_SUFFIX: "1.el7"
- PACKAGE_FOLDER: "centos7.9.2009"
-
-build-packages-centos8:
- extends: .client-packages
- image:
- name: $CI_REGISTRY_IMAGE/asapo-packages-centos8-build-env:latest
- entrypoint: [""]
- variables:
- PACKAGE_TYPE: "RPM"
- WITH_LIBFABRIC: "YES"
- OS_SUFFIX: "1.el8"
- PACKAGE_FOLDER: "centos8.3.2011"
-
-build-packages-debian9:
- extends: .client-packages
- image:
- name: $CI_REGISTRY_IMAGE/asapo-packages-debian9-build-env:latest
- entrypoint: [""]
- variables:
- PACKAGE_TYPE: "DEB"
- WITH_PYTHON2: "YES"
- OS_SUFFIX: "debian9.13"
- PACKAGE_FOLDER: "debian9.13"
-
-build-packages-debian10:
- extends: .client-packages
- image:
- name: $CI_REGISTRY_IMAGE/asapo-packages-debian10-build-env:latest
- entrypoint: [""]
- variables:
- PACKAGE_TYPE: "DEB"
- WITH_LIBFABRIC: "YES"
- WITH_PYTHON2: "YES"
- OS_SUFFIX: "debian10.11"
- PACKAGE_FOLDER: "debian10.11"
-
-build-packages-debian11:
- extends: .client-packages
- image:
- name: $CI_REGISTRY_IMAGE/asapo-packages-debian11-build-env:latest
- entrypoint: [""]
- variables:
- PACKAGE_TYPE: "DEB"
- WITH_LIBFABRIC: "YES"
- OS_SUFFIX: "debian11.3"
- PACKAGE_FOLDER: "debian11.3"
-
-build-packages-ubuntu1804:
- extends: .client-packages
- image:
- name: $CI_REGISTRY_IMAGE/asapo-packages-ubuntu1804-build-env:latest
- entrypoint: [""]
- variables:
- PACKAGE_TYPE: "DEB"
- WITH_LIBFABRIC: "YES"
- OS_SUFFIX: "ubuntu18.04"
- PACKAGE_FOLDER: "ubuntu18.04"
-
-build-packages-ubuntu2004:
- extends: .client-packages
- image:
- name: $CI_REGISTRY_IMAGE/asapo-packages-ubuntu2004-build-env:latest
- entrypoint: [""]
- variables:
- PACKAGE_TYPE: "DEB"
- WITH_LIBFABRIC: "YES"
- OS_SUFFIX: "ubuntu20.04"
- PACKAGE_FOLDER: "ubuntu20.04"
-
-build-packages-python36:
- extends: .client-packages-manylinux
- variables:
- PYTHON_PATH: /opt/python/cp36-cp36m
- NUMPY_VERSION: 1.12.1
- WHEEL_FOLDER: python36
-
-build-packages-python37:
- extends: .client-packages-manylinux
- variables:
- PYTHON_PATH: /opt/python/cp37-cp37m
- NUMPY_VERSION: 1.14.5
- WHEEL_FOLDER: python37
-
-build-packages-python38:
- extends: .client-packages-manylinux
- variables:
- PYTHON_PATH: /opt/python/cp38-cp38
- NUMPY_VERSION: 1.17.3
- WHEEL_FOLDER: python38
-
-build-packages-python39:
- extends: .client-packages-manylinux
- variables:
- PYTHON_PATH: /opt/python/cp39-cp39
- NUMPY_VERSION: 1.19.3
- WHEEL_FOLDER: python39
-
-build-packages-python310:
- extends: .client-packages-manylinux
- variables:
- PYTHON_PATH: /opt/python/cp310-cp310
- NUMPY_VERSION: 1.23.4
- WHEEL_FOLDER: python310
-
-.build-services-docker-template:
- image:
- name: gcr.io/kaniko-project/executor:debug
- entrypoint: [""]
- stage: deploy
- script:
- - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
- - services="broker authorizer discovery receiver file-transfer"
- - >
- for service in $services; do
- service_dir=${service//"-"/"_"}
- /kaniko/executor --cache=false --context $CI_PROJECT_DIR/build/${service_dir} --dockerfile $CI_PROJECT_DIR/build/${service_dir}/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-${service}${CONTAINER_TAG}
- done
- - /kaniko/executor --cache=false --context $CI_PROJECT_DIR/build/monitoring/monitoring_server --dockerfile $CI_PROJECT_DIR/monitoring/monitoring_server/docker/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-monitoring-server${CONTAINER_TAG}
- - /kaniko/executor --cache=false --context $CI_PROJECT_DIR/monitoring/monitoring_ui --dockerfile $CI_PROJECT_DIR/monitoring/monitoring_ui/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-monitoring-ui${CONTAINER_TAG}
- - /kaniko/executor --cache=false --context $CI_PROJECT_DIR/build/deploy/asapo_services --dockerfile $CI_PROJECT_DIR/build/deploy/asapo_services/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-cluster${CONTAINER_TAG} --destination $CI_REGISTRY_IMAGE/asapo-cluster${CONTAINER_TAG_LATEST}
- tags:
- - kubernetes-executor
- dependencies:
- - build-services-linux-release
-
-build-services-docker-dev:
- extends: .build-services-docker-template
- rules:
- - if: $CI_COMMIT_REF_NAME == "develop"
- when: on_success
- - when: never
- variables:
- CONTAINER_TAG: "-dev:${CI_COMMIT_SHORT_SHA}"
- CONTAINER_TAG_LATEST: "-dev:latest"
-
-build-services-docker-release:
- extends: .build-services-docker-template
- only:
- - tags
- variables:
- CONTAINER_TAG: ":${CI_COMMIT_TAG}"
- CONTAINER_TAG_LATEST: ":latest"
-
-.upload-client-packages-template:
- image:
- name: $CI_REGISTRY_IMAGE/asapo-services-linux-build-env:latest
- entrypoint: [""]
- stage: deploy
- variables:
- GIT_STRATEGY: none
- script:
- - >
- for dist in $(ls $CI_PROJECT_DIR/linux_packages); do
- files="asapo-devel python-asapo python3-asapo"
- for file in $files; do
- if ls $CI_PROJECT_DIR/linux_packages/${dist}/${file}* 1> /dev/null 2>&1; then
- PACKAGE_ID=$(curl -s -G --header "JOB-TOKEN: $CI_JOB_TOKEN" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages -d package_name=$file-$dist -d package_type=generic | jq -c ".[] | select(.pipeline.ref | contains(\"$CI_COMMIT_REF_NAME\")) | (.id)")
- if ! [ -z "$PACKAGE_ID" ]; then
- echo Removing Package ${PACKAGE_ID}
- curl --request DELETE --header "JOB-TOKEN: $CI_JOB_TOKEN" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${PACKAGE_ID}"
- fi
- for distfile in $(ls $CI_PROJECT_DIR/linux_packages/$dist/${file}*); do
- DIST_FILE=${distfile//"~"/"_"}
- curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file $distfile "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/$file-$dist/$DIST_VERSION/$(basename $DIST_FILE)"
- done
- fi
- done
- done
- tags:
- - kubernetes-executor
- dependencies:
- - build-packages-centos7
- - build-packages-centos8
- - build-packages-debian9
- - build-packages-debian10
- - build-packages-debian11
- - build-packages-ubuntu1804
- - build-packages-ubuntu2004
-
-upload-client-packages-dev:
- extends: .upload-client-packages-template
- rules:
- - if: $CI_COMMIT_REF_NAME == "develop"
- when: on_success
- - when: never
- variables:
- DIST_VERSION: "100.0.dev"
-
-upload-client-packages-release:
- extends: .upload-client-packages-template
- only:
- - tags
- variables:
- DIST_VERSION: ${CI_COMMIT_TAG}
-
-upload-python-packages:
- image:
- name: python:latest
- entrypoint: [""]
- stage: deploy
- variables:
- GIT_STRATEGY: none
- script:
- - mkdir -p ~/.ssh
- - echo "$NIMS_SSH_KNOWN_HOST" >> ~/.ssh/known_hosts
- - chmod 600 $NIMS_SSH_PRIVATE_KEY
- - pip install twine
- - >
- for pac_id in $(curl -s -G --header "JOB-TOKEN: $CI_JOB_TOKEN" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages -d package_type=pypi | python -c "import sys, json; [print(x['id']) for x in json.load(sys.stdin) if ('pipeline' in x) and ('ref' in x['pipeline']) and (x['pipeline']['ref'] == '$CI_COMMIT_REF_NAME')]"); do
- echo Removing package ${pac_id}
- curl --request DELETE --header "JOB-TOKEN: $CI_JOB_TOKEN" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pac_id}"
- done
- - >
- for dist in $(ls $CI_PROJECT_DIR/wheel_packages); do
- TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi $CI_PROJECT_DIR/wheel_packages/${dist}/*
- echo "put $CI_PROJECT_DIR/wheel_packages/${dist}/* asapo/linux_wheels/" | sftp -vvvv -i $NIMS_SSH_PRIVATE_KEY -P 2022 asapo@it-fs5.desy.de
- done
- - wget -q -O- http://it-fs5.desy.de/cgi-bin/asapoSync.cgi
- tags:
- - DESY-intern
+ # The folder that contains the files to be exposed at the Page URL
+ - public
rules:
- - if: $CI_COMMIT_REF_NAME == "develop"
- when: on_success
- - if: $CI_COMMIT_TAG
- when: on_success
- - when: never
- dependencies:
- - build-packages-python36
- - build-packages-python37
- - build-packages-python38
- - build-packages-python39
- - build-packages-python310
+ # This ensures that only pushes to the default branch will trigger
+ # a pages deploy
+ - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
--
GitLab