From 1de95ae0d42af7aa8b4cbddd1d4799c5798785c5 Mon Sep 17 00:00:00 2001
From: George Sedov <george.sedov@desy.de>
Date: Mon, 14 Nov 2022 16:51:06 +0100
Subject: [PATCH] test build python wheels
---
.gitlab-ci.yml | 228 ++++++++++--------
deploy/build_env/centos/Dockerfile.gitlab.7 | 1 -
deploy/build_env/manylinux2014/Dockerfile | 3 +
.../manylinux2014/gitlab-ci-build-docker.yml | 15 ++
.../src/store/groupDelayStore.ts | 2 +-
5 files changed, 148 insertions(+), 101 deletions(-)
create mode 100644 deploy/build_env/manylinux2014/Dockerfile
create mode 100644 deploy/build_env/manylinux2014/gitlab-ci-build-docker.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 51e66e95a..bdb6eb0f2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,6 +4,7 @@ include:
- local: deploy/build_env/centos/gitlab-ci-build-docker.yml
- local: deploy/build_env/debians/gitlab-ci-build-docker.yml
- local: deploy/build_env/ubuntu/gitlab-ci-build-docker.yml
+ - local: deploy/build_env/manylinux2014/gitlab-ci-build-docker.yml
- local: deploy/secondary_services/fluentd_elastic/gitlab-ci-build-docker.yml
.go-cache:
@@ -16,10 +17,54 @@ include:
- .go/pkg/mod/
.client-package:
- after_script:
- - mkdir -p $CI_PROJECT_DIR/linux_packages/$OS_FOLDER
- - cp $CI_PROJECT_DIR/build/asapo-devel* $CI_PROJECT_DIR/linux_packages/$OS_FOLDER
- - cp $CI_PROJECT_DIR/build/*/api/python/dist_linux/dist/* $CI_PROJECT_DIR/linux_packages/$OS_FOLDER
+ stage: build
+ script:
+ - mkdir $CI_PROJECT_DIR/build
+ - cd $CI_PROJECT_DIR/build
+ - PACKAGE_UP=${PACKAGE_TYPE^^}
+ - PACKAGE_LOW=${PACKAGE_TYPE,,}
+ - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DINSTALL_EXAMPLES=ON -DBUILD_CLIENTS_ONLY=ON -DPACKAGE_RELEASE_SUFFIX=$OS_SUFFIX -DBUILD_PYTHON=OFF -DCPACK_PACKAGE_NAME="asapo-devel" -DCPACK_GENERATOR=$PACKAGE_UP ..
+ - make -j 4
+ - make package
+ - rm -f CMakeCache.txt
+ - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DBUILD_CLIENTS_ONLY=ON -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DPACKAGE_RELEASE_SUFFIX=$OS_SUFFIX -DBUILD_PYTHON_PACKAGES="source;${PACKAGE_LOW}" -DBUILD_PYTHON2_PACKAGES=$BUILD_PYTHON2 -DBUILD_PYTHON_DOCS=OFF ..
+ - make -j 1
+ - mkdir -p $CI_PROJECT_DIR/linux_packages/$PACKAGE_FOLDER
+ - cp $CI_PROJECT_DIR/build/asapo-devel* $CI_PROJECT_DIR/linux_packages/$PACKAGE_FOLDER
+ - cp $CI_PROJECT_DIR/build/*/api/python/dist_linux/dist/* $CI_PROJECT_DIR/linux_packages/$PACKAGE_FOLDER
+ tags:
+ - kubernetes-executor
+ rules:
+ - when: on_success
+ artifacts:
+ paths:
+ - linux_packages
+ expire_in: 3 hrs
+
+.client-packages-manylinux:
+ image:
+ name: $CI_REGISTRY_IMAGE/asapo-packages-manylinux-build-env:latest
+ entrypoint: [""]
+ stage: build
+ script:
+ - python_bin=$PYTHON_PATH/bin/python
+ - pip_bin=$PYTHON_PATH/bin/pip
+ - $pip install cython numpy==$NUMPY_VERSION
+ - cd $CI_PROJECT_DIR/build
+ - cmake -DENABLE_LIBFABRIC=on -DCMAKE_BUILD_TYPE="Release" -DBUILD_CLIENTS_ONLY=ON -DPython_EXECUTABLE=$python -DBUILD_PYTHON_PACKAGES=source -DBUILD_PYTHON2_PACKAGES=OFF -DNUMPY_VERSION=$NUMPY_VERSION ..
+ - cd $CI_PROJECT_DIR/build/consumer/api/python/dist_linux
+ - make python-dist-consumer
+ - $pip wheel dist/*.tar.gz --no-deps
+ - cd $CI_PROJECT_DIR/build/producer/api/python/dist_linux
+ - make python-dist-producer
+ - $pip wheel dist/*.tar.gz --no-deps
+ - mkdir -p $CI_PROJECT_DIR/wheel_packages/$WHEEL_FOLDER
+ - cp $CI_PROJECT_DIR/build/*/api/python/dist_linux/*.whl $CI_PROJECT_DIR/wheel_packages/$WHEEL_FOLDER
+ - cp $CI_PROJECT_DIR/build/*/api/python/dist_linux/dist/*.tar.gz $CI_PROJECT_DIR/linux_packages/$WHEEL_FOLDER
+ tags:
+ - kubernetes-executor
+ rules:
+ - when: on_success
artifacts:
paths:
- linux_packages
@@ -104,140 +149,125 @@ build-packages-centos7:
image:
name: $CI_REGISTRY_IMAGE/asapo-packages-centos7-build-env:latest
entrypoint: [""]
- stage: build
- script:
- - mkdir $CI_PROJECT_DIR/build
- - cd $CI_PROJECT_DIR/build
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DINSTALL_EXAMPLES=ON -DBUILD_CLIENTS_ONLY=ON -DPACKAGE_RELEASE_SUFFIX=1.el7 -DBUILD_PYTHON=OFF -DCPACK_PACKAGE_NAME="asapo-devel" -DCPACK_GENERATOR="RPM" ..
- - make -j 4
- - make package
- - rm -f CMakeCache.txt
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DBUILD_CLIENTS_ONLY=ON -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DPACKAGE_RELEASE_SUFFIX=1.el7 -DBUILD_PYTHON_PACKAGES="source;rpm" -DBUILD_PYTHON2_PACKAGES=ON -DBUILD_PYTHON_DOCS=OFF ..
- - make -j 1
- tags:
- - kubernetes-executor
- rules:
- - when: on_success
+ variables:
+ PACKAGE_TYPE: "RPM"
+ BUILD_PYTHON2: "ON"
+ OS_SUFFIX: "1.el7"
+ PACKAGE_FOLDER: "centos7.9.2009"
build-packages-centos8:
extends: .client-package
image:
name: $CI_REGISTRY_IMAGE/asapo-packages-centos8-build-env:latest
entrypoint: [""]
- stage: build
- script:
- - mkdir $CI_PROJECT_DIR/build
- - cd $CI_PROJECT_DIR/build
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DINSTALL_EXAMPLES=ON -DBUILD_CLIENTS_ONLY=ON -DPACKAGE_RELEASE_SUFFIX=1.el8 -DBUILD_PYTHON=OFF -DCPACK_PACKAGE_NAME="asapo-devel" -DCPACK_GENERATOR="RPM" ..
- - make -j 4
- - make package
- - rm -f CMakeCache.txt
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DBUILD_CLIENTS_ONLY=ON -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DPACKAGE_RELEASE_SUFFIX=1.el8 -DBUILD_PYTHON_PACKAGES="source;rpm" -DBUILD_PYTHON2_PACKAGES=OFF -DBUILD_PYTHON_DOCS=OFF ..
- - make -j 1
- tags:
- - kubernetes-executor
- rules:
- - when: on_success
+ variables:
+ PACKAGE_TYPE: "RPM"
+ BUILD_PYTHON2: "OFF"
+ OS_SUFFIX: "1.el8"
+ PACKAGE_FOLDER: "centos8.3.2011"
build-packages-debian9:
extends: .client-package
image:
name: $CI_REGISTRY_IMAGE/asapo-packages-debian9-build-env:latest
entrypoint: [""]
- stage: build
- script:
- - mkdir $CI_PROJECT_DIR/build
- - cd $CI_PROJECT_DIR/build
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=OFF -DINSTALL_EXAMPLES=ON -DBUILD_CLIENTS_ONLY=ON -DPACKAGE_RELEASE_SUFFIX=debian9.13 -DBUILD_PYTHON=OFF -DCPACK_PACKAGE_NAME="asapo-devel" -DCPACK_GENERATOR="DEB" ..
- - make -j 4
- - make package
- - rm -f CMakeCache.txt
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=OFF -DBUILD_CLIENTS_ONLY=ON -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DPACKAGE_RELEASE_SUFFIX=debian9.13 -DBUILD_PYTHON_PACKAGES="source;deb" -DBUILD_PYTHON2_PACKAGES=ON -DBUILD_PYTHON_DOCS=OFF ..
- - make -j 1
- tags:
- - kubernetes-executor
- rules:
- - when: on_success
+ variables:
+ PACKAGE_TYPE: "DEB"
+ BUILD_PYTHON2: "ON"
+ OS_SUFFIX: "debian9.13"
+ PACKAGE_FOLDER: "debian9.13"
build-packages-debian10:
extends: .client-package
image:
name: $CI_REGISTRY_IMAGE/asapo-packages-debian10-build-env:latest
entrypoint: [""]
- stage: build
- script:
- - mkdir $CI_PROJECT_DIR/build
- - cd $CI_PROJECT_DIR/build
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DINSTALL_EXAMPLES=ON -DBUILD_CLIENTS_ONLY=ON -DPACKAGE_RELEASE_SUFFIX=debian10.11 -DBUILD_PYTHON=OFF -DCPACK_PACKAGE_NAME="asapo-devel" -DCPACK_GENERATOR="DEB" ..
- - make -j 4
- - make package
- - rm -f CMakeCache.txt
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DBUILD_CLIENTS_ONLY=ON -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DPACKAGE_RELEASE_SUFFIX=debian10.11 -DBUILD_PYTHON_PACKAGES="source;deb" -DBUILD_PYTHON2_PACKAGES=ON -DBUILD_PYTHON_DOCS=OFF ..
- - make -j 1
- tags:
- - kubernetes-executor
- rules:
- - when: on_success
+ variables:
+ PACKAGE_TYPE: "DEB"
+ BUILD_PYTHON2: "ON"
+ OS_SUFFIX: "debian10.11"
+ PACKAGE_FOLDER: "debian10.11"
build-packages-debian11:
extends: .client-package
image:
name: $CI_REGISTRY_IMAGE/asapo-packages-debian11-build-env:latest
entrypoint: [""]
- stage: build
- script:
- - mkdir $CI_PROJECT_DIR/build
- - cd $CI_PROJECT_DIR/build
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DINSTALL_EXAMPLES=ON -DBUILD_CLIENTS_ONLY=ON -DPACKAGE_RELEASE_SUFFIX=debian11.2 -DBUILD_PYTHON=OFF -DCPACK_PACKAGE_NAME="asapo-devel" -DCPACK_GENERATOR="DEB" ..
- - make -j 4
- - make package
- - rm -f CMakeCache.txt
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DBUILD_CLIENTS_ONLY=ON -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DPACKAGE_RELEASE_SUFFIX=debian11.2 -DBUILD_PYTHON_PACKAGES="source;deb" -DBUILD_PYTHON2_PACKAGES=OFF -DBUILD_PYTHON_DOCS=OFF ..
- - make -j 1
- tags:
- - kubernetes-executor
- rules:
- - when: on_success
+ variables:
+ PACKAGE_TYPE: "DEB"
+ BUILD_PYTHON2: "OFF"
+ OS_SUFFIX: "debian11.2"
+ PACKAGE_FOLDER: "debian11.2"
build-packages-ubuntu1804:
extends: .client-package
image:
name: $CI_REGISTRY_IMAGE/asapo-packages-ubuntu1804-build-env:latest
entrypoint: [""]
- stage: build
- script:
- - mkdir $CI_PROJECT_DIR/build
- - cd $CI_PROJECT_DIR/build
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DINSTALL_EXAMPLES=ON -DBUILD_CLIENTS_ONLY=ON -DPACKAGE_RELEASE_SUFFIX=ubuntu18.04 -DBUILD_PYTHON=OFF -DCPACK_PACKAGE_NAME="asapo-devel" -DCPACK_GENERATOR="DEB" ..
- - make -j 4
- - make package
- - rm -f CMakeCache.txt
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DBUILD_CLIENTS_ONLY=ON -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DPACKAGE_RELEASE_SUFFIX=ubuntu18.04 -DBUILD_PYTHON_PACKAGES="source;deb" -DBUILD_PYTHON2_PACKAGES=ON -DBUILD_PYTHON_DOCS=OFF ..
- - make -j 1
- tags:
- - kubernetes-executor
- rules:
- - when: on_success
+ variables:
+ PACKAGE_TYPE: "DEB"
+ BUILD_PYTHON2: "ON"
+ OS_SUFFIX: "ubuntu18.04"
+ PACKAGE_FOLDER: "ubuntu18.04"
build-packages-ubuntu2004:
extends: .client-package
image:
name: $CI_REGISTRY_IMAGE/asapo-packages-ubuntu2004-build-env:latest
entrypoint: [""]
- stage: build
+ variables:
+ PACKAGE_TYPE: "DEB"
+ BUILD_PYTHON2: "OFF"
+ OS_SUFFIX: "ubuntu20.04"
+ PACKAGE_FOLDER: "ubuntu20.04"
+
+build-package-python36:
+ extends: .client-packages-manylinux
+ variables:
+ PYTHON_PATH: /opt/python/cp36-cp36m
+ NUMPY_VERSION: 1.12.1
+ WHEEL_FOLDER: python36
+
+build-package-python37:
+ extends: .client-packages-manylinux
+ variables:
+ PYTHON_PATH: /opt/python/cp37-cp37m
+ NUMPY_VERSION: 1.14.5
+ WHEEL_FOLDER: python37
+
+build-package-python38:
+ extends: .client-packages-manylinux
+ variables:
+ PYTHON_PATH: /opt/python/cp38-cp38
+ NUMPY_VERSION: 1.17.3
+ WHEEL_FOLDER: python38
+
+build-package-python39:
+ extends: .client-packages-manylinux
+ variables:
+ PYTHON_PATH: /opt/python/cp39-cp39
+ NUMPY_VERSION: 1.19.3
+ WHEEL_FOLDER: python39
+
+upload-python-packages:
+ image:
+ name: python:latest
+ entrypoint: [""]
+ stage: deploy
script:
- - mkdir $CI_PROJECT_DIR/build
- - cd $CI_PROJECT_DIR/build
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DINSTALL_EXAMPLES=ON -DBUILD_CLIENTS_ONLY=ON -DPACKAGE_RELEASE_SUFFIX=ubuntu20.04 -DBUILD_PYTHON=OFF -DCPACK_PACKAGE_NAME="asapo-devel" -DCPACK_GENERATOR="DEB" ..
- - make -j 4
- - make package
- - rm -f CMakeCache.txt
- - cmake -DCMAKE_BUILD_TYPE="Release" -DENABLE_LIBFABRIC=ON -DBUILD_CLIENTS_ONLY=ON -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DPACKAGE_RELEASE_SUFFIX=ubuntu20.04 -DBUILD_PYTHON_PACKAGES="source;deb" -DBUILD_PYTHON2_PACKAGES=OFF -DBUILD_PYTHON_DOCS=OFF ..
- - make -j 1
+ - pip install build twine
+ - python -m build
+ - >
+ for dist in $(ls $CI_PROJECT_DIR/linux_packages); do
+ TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi $CI_PROJECT_DIR/linux_packages/${dist}/*
+ done
tags:
- kubernetes-executor
- rules:
- - when: on_success
+ dependencies:
+ - build-package-python36
+ - build-package-python37
+ - build-package-python38
+ - build-package-python39
.build-services-docker-template:
image:
diff --git a/deploy/build_env/centos/Dockerfile.gitlab.7 b/deploy/build_env/centos/Dockerfile.gitlab.7
index c988c2148..d3c9500e4 100644
--- a/deploy/build_env/centos/Dockerfile.gitlab.7
+++ b/deploy/build_env/centos/Dockerfile.gitlab.7
@@ -19,7 +19,6 @@ RUN yum install -y epel-release; \
python3-devel \
numpy \
python3-numpy \
- libcurl-devel \
libfabric-devel \
libibverbs \
librdmacm \
diff --git a/deploy/build_env/manylinux2014/Dockerfile b/deploy/build_env/manylinux2014/Dockerfile
new file mode 100644
index 000000000..8584bdbb1
--- /dev/null
+++ b/deploy/build_env/manylinux2014/Dockerfile
@@ -0,0 +1,3 @@
+FROM quay.io/pypa/manylinux2014_x86_64
+
+RUN yum install -y libcurl-devel libfabric-devel
diff --git a/deploy/build_env/manylinux2014/gitlab-ci-build-docker.yml b/deploy/build_env/manylinux2014/gitlab-ci-build-docker.yml
new file mode 100644
index 000000000..93ada4c34
--- /dev/null
+++ b/deploy/build_env/manylinux2014/gitlab-ci-build-docker.yml
@@ -0,0 +1,15 @@
+docker-create-asapo-packages-manylinux-build-env:
+ image:
+ name: gcr.io/kaniko-project/executor:debug
+ entrypoint: [""]
+ stage: .pre
+ script:
+ - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+ - /kaniko/executor --cache=false --context $CI_PROJECT_DIR/deploy/build_env/manylinux2014 --dockerfile $CI_PROJECT_DIR/deploy/build_env/manylinux2014/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-packages-manylinux-build-env:${CI_COMMIT_SHORT_SHA} --destination $CI_REGISTRY_IMAGE/asapo-packages-manylinux-build-env:latest
+ tags:
+ - kubernetes-executor
+ rules:
+ - changes:
+ - deploy/build_env/manylinux2014/Dockerfile
+ when: always
+ - when: never
diff --git a/monitoring/monitoring_ui/src/store/groupDelayStore.ts b/monitoring/monitoring_ui/src/store/groupDelayStore.ts
index 96e68b540..739f273a8 100644
--- a/monitoring/monitoring_ui/src/store/groupDelayStore.ts
+++ b/monitoring/monitoring_ui/src/store/groupDelayStore.ts
@@ -29,7 +29,7 @@ class GroupDelayStore {
} else if (delayMs > 1000) {
return Math.floor(delayMs / 1000).toString() + " secs"
}
- return "0."+delayMs.toString().padStart(4, '0')+" secs"
+ return "0."+delayMs.toString().padStart(3, '0')+" secs"
}
public parseServerResponse(groupDelayData: GroupDelayResponse): void {
--
GitLab