From 391caf4bf49ee8460970085eda425ea477038760 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Tue, 15 Dec 2020 11:59:23 +0100
Subject: [PATCH] update deploys
---
CMakeLists.txt | 4 +--
CMakeModules/prepare_version_tag.cmake | 5 ++--
consumer/api/python/dist_linux/CMakeLists.txt | 30 +++++++++++++++++--
deploy/build_env/Ubuntu16.04/Dockerfile | 19 ------------
deploy/build_env/Ubuntu16.04/build.sh | 18 -----------
deploy/build_env/Ubuntu16.04/build_image.sh | 4 ---
deploy/build_env/centos/build.sh | 4 ++-
deploy/build_env/centos/build_el7.sh | 1 +
deploy/build_env/centos/build_el8.sh | 1 +
.../build_env/debians/Dockerfile_debian10.7 | 20 +++++++++++++
.../build_env/debians/Dockerfile_debian9.13 | 20 +++++++++++++
.../build_env/debians/Dockerfile_ubuntu16.04 | 20 +++++++++++++
.../build_env/debians/Dockerfile_ubuntu18.04 | 20 +++++++++++++
deploy/build_env/debians/build.sh | 22 ++++++++++++++
deploy/build_env/debians/build_images.sh | 13 ++++++++
.../{Ubuntu16.04 => debians}/install_cmake.sh | 4 +--
.../{Ubuntu16.04 => debians}/install_curl.sh | 0
.../install_libfabric.sh | 0
deploy/build_env/manylinux2010/build_image.sh | 3 +-
producer/api/python/dist_linux/CMakeLists.txt | 28 +++++++++++++++--
20 files changed, 180 insertions(+), 56 deletions(-)
delete mode 100644 deploy/build_env/Ubuntu16.04/Dockerfile
delete mode 100755 deploy/build_env/Ubuntu16.04/build.sh
delete mode 100755 deploy/build_env/Ubuntu16.04/build_image.sh
create mode 100644 deploy/build_env/debians/Dockerfile_debian10.7
create mode 100644 deploy/build_env/debians/Dockerfile_debian9.13
create mode 100644 deploy/build_env/debians/Dockerfile_ubuntu16.04
create mode 100644 deploy/build_env/debians/Dockerfile_ubuntu18.04
create mode 100755 deploy/build_env/debians/build.sh
create mode 100755 deploy/build_env/debians/build_images.sh
rename deploy/build_env/{Ubuntu16.04 => debians}/install_cmake.sh (82%)
rename deploy/build_env/{Ubuntu16.04 => debians}/install_curl.sh (100%)
rename deploy/build_env/{Ubuntu16.04 => debians}/install_libfabric.sh (100%)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index f56f10d2a..c7c2703a2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -135,9 +135,7 @@ endif()
set(CPACK_PACKAGE_VERSION ${ASAPO_VERSION})
-set(CPACK_GENERATOR "RPM")
-set(CPACK_PACKAGE_NAME "asapo")
-set(CPACK_PACKAGE_RELEASE 1.${PACKAGE_RELEASE_SUFFIX})
+set(CPACK_PACKAGE_RELEASE ${PACKAGE_RELEASE_SUFFIX})
set(CPACK_PACKAGE_CONTACT "IT")
set(CPACK_PACKAGE_VENDOR "DESY")
set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
diff --git a/CMakeModules/prepare_version_tag.cmake b/CMakeModules/prepare_version_tag.cmake
index 3a862a5c1..308beae3a 100644
--- a/CMakeModules/prepare_version_tag.cmake
+++ b/CMakeModules/prepare_version_tag.cmake
@@ -28,8 +28,9 @@ else()
SET (ASAPO_VERSION ${BRANCH})
SET (ASAPO_VERSION_COMMIT ", build ${ASAPO_VERSION_COMMIT}")
SET (ASAPO_VERSION_DOCKER_SUFFIX "-dev")
- SET (PYTHON_ASAPO_VERSION 1.0.${ASAPO_VERSION})
-
+ string(REPLACE "_" "-" ASAPO_VERSION ${ASAPO_VERSION})
+ SET (ASAPO_VERSION 1.0.${ASAPO_VERSION})
+ SET (PYTHON_ASAPO_VERSION ${ASAPO_VERSION})
endif()
message("Asapo Version: " ${ASAPO_VERSION})
diff --git a/consumer/api/python/dist_linux/CMakeLists.txt b/consumer/api/python/dist_linux/CMakeLists.txt
index 67e5a6e9e..a5290d802 100644
--- a/consumer/api/python/dist_linux/CMakeLists.txt
+++ b/consumer/api/python/dist_linux/CMakeLists.txt
@@ -10,14 +10,38 @@ endif()
if ("rpm" IN_LIST BUILD_PYTHON_PACKAGES)
ADD_CUSTOM_TARGET(python-rpm-consumer ALL
- COMMAND PACKAGE_PREFIX=python- python setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX} --requires=numpy
+ COMMAND PACKAGE_PREFIX=python- python setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX}
+ --requires=numpy --binary-only
+ COMMAND rm dist/python*.gz dist/*debuginfo*
)
ADD_CUSTOM_TARGET(python3-rpm-consumer ALL
- COMMAND PACKAGE_PREFIX=python3- python3 setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX} --requires=python3-numpy
+ COMMAND PACKAGE_PREFIX=python3- python3 setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX}
+ --requires=python3-numpy --binary-only
+ COMMAND rm dist/python3*.gz dist/*debuginfo*
)
- ADD_DEPENDENCIES(python-rpm-consumer copy_python_dist-consumer)
ADD_DEPENDENCIES(python3-rpm-consumer copy_python_dist-consumer)
+ ADD_DEPENDENCIES(python-rpm-consumer copy_python_dist-consumer)
+endif()
+
+if ("deb" IN_LIST BUILD_PYTHON_PACKAGES)
+ ADD_CUSTOM_TARGET(python-deb-consumer ALL
+ COMMAND rm -rf deb_dist/*/
+ COMMAND PACKAGE_PREFIX= python setup.py --command-packages=stdeb.command
+ sdist_dsc --debian-version=${PACKAGE_RELEASE_SUFFIX} --depends=python-numpy bdist_deb
+ COMMAND rm deb_dist/*dbgsym*
+ COMMAND cp deb_dist/*.deb dist/
+ )
+
+ ADD_CUSTOM_TARGET(python3-deb-consumer ALL
+ COMMAND rm -rf deb_dist/*/
+ COMMAND PACKAGE_PREFIX= python3 setup.py --command-packages=stdeb.command
+ sdist_dsc --debian-version=${PACKAGE_RELEASE_SUFFIX} --depends3=python3-numpy bdist_deb
+ COMMAND rm deb_dist/*dbgsym*
+ COMMAND cp deb_dist/*.deb dist/
+ )
+ ADD_DEPENDENCIES(python3-deb-consumer copy_python_dist-consumer)
+ ADD_DEPENDENCIES(python-deb-consumer copy_python_dist-consumer)
endif()
ADD_CUSTOM_TARGET(copy_python_dist-consumer ALL
diff --git a/deploy/build_env/Ubuntu16.04/Dockerfile b/deploy/build_env/Ubuntu16.04/Dockerfile
deleted file mode 100644
index 943b0329a..000000000
--- a/deploy/build_env/Ubuntu16.04/Dockerfile
+++ /dev/null
@@ -1,19 +0,0 @@
-from ubuntu:16.04
-
-ENV GOPATH /tmp
-
-ADD install_curl.sh install_curl.sh
-ADD install_cmake.sh install_cmake.sh
-
-RUN apt update && apt install -y g++ golang zlib1g-dev python cython python-numpy python3 cython3 python3-numpy \
-git wget python-pip python3-pip && \
-pip --no-cache-dir install --upgrade cython && pip3 --no-cache-dir install --upgrade cython && pip3 --no-cache-dir install sphinx && \
-./install_curl.sh /curl && ./install_cmake.sh
-
-ADD install_libfabric.sh install_libfabric.sh
-RUN ./install_libfabric.sh
-
-RUN apt-get purge -y --auto-remove
-
-ADD build.sh /bin/build.sh
-
diff --git a/deploy/build_env/Ubuntu16.04/build.sh b/deploy/build_env/Ubuntu16.04/build.sh
deleted file mode 100755
index c0031edf8..000000000
--- a/deploy/build_env/Ubuntu16.04/build.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-cd /asapo/build
-cmake \
- -DCMAKE_BUILD_TYPE="Release" \
- -DENABLE_LIBFABRIC=on \
- -DLIBCURL_DIR=/curl \
- -DBUILD_PYTHON_DOCS=ON \
- -DBUILD_PYTHON_PACKAGES=source \
- -DBUILD_CLIENTS_ONLY=ON \
- ..
-make
-#cd consumer/api/python/dist_linux && make python-dist-consumer
-#cd ../producer/api/python/dist_linux && make python-dist-producer
-#cd ../docs/sphinx && make
-
diff --git a/deploy/build_env/Ubuntu16.04/build_image.sh b/deploy/build_env/Ubuntu16.04/build_image.sh
deleted file mode 100755
index 415b215af..000000000
--- a/deploy/build_env/Ubuntu16.04/build_image.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-set -e
-docker build -t yakser/asapo-env:ubuntu16.04 .
-docker push yakser/asapo-env:ubuntu16.04
diff --git a/deploy/build_env/centos/build.sh b/deploy/build_env/centos/build.sh
index 9ca1c74d7..112f6b292 100755
--- a/deploy/build_env/centos/build.sh
+++ b/deploy/build_env/centos/build.sh
@@ -6,9 +6,11 @@ cmake \
-DENABLE_LIBFABRIC=ON \
-DINSTALL_EXAMPLES=ON \
-DBUILD_CLIENTS_ONLY=ON \
- -DPACKAGE_RELEASE_SUFFIX=$OS \
+ -DPACKAGE_RELEASE_SUFFIX=1.$OS \
-DBUILD_PYTHON=OFF \
-DLIBCURL_DIR=/curl \
+ -DCPACK_PACKAGE_NAME="asapo-devel" \
+ -DCPACK_GENERATOR="RPM" \
..
make
make package
diff --git a/deploy/build_env/centos/build_el7.sh b/deploy/build_env/centos/build_el7.sh
index 407078a61..94db9e958 100755
--- a/deploy/build_env/centos/build_el7.sh
+++ b/deploy/build_env/centos/build_el7.sh
@@ -1,2 +1,3 @@
#!/usr/bin/env bash
docker build -t yakser/asapo-env:centos7.9.2009 -f Dockerfile.7.9.2009 .
+docker push yakser/asapo-env:centos7.9.2009
\ No newline at end of file
diff --git a/deploy/build_env/centos/build_el8.sh b/deploy/build_env/centos/build_el8.sh
index 44d5e7dcf..b6474aa27 100755
--- a/deploy/build_env/centos/build_el8.sh
+++ b/deploy/build_env/centos/build_el8.sh
@@ -1,2 +1,3 @@
#!/usr/bin/env bash
docker build -t yakser/asapo-env:centos8.3.2011 -f Dockerfile.8.3.2011 .
+docker push yakser/asapo-env:centos8.3.2011
diff --git a/deploy/build_env/debians/Dockerfile_debian10.7 b/deploy/build_env/debians/Dockerfile_debian10.7
new file mode 100644
index 000000000..5c669bd91
--- /dev/null
+++ b/deploy/build_env/debians/Dockerfile_debian10.7
@@ -0,0 +1,20 @@
+from debian:10.7
+
+ENV GOPATH /tmp
+
+ADD install_curl.sh install_curl.sh
+ADD install_cmake.sh install_cmake.sh
+
+RUN apt update && apt install -y g++ git wget python python3 python-numpy python3-numpy python-pip python3-pip \
+zlib1g-dev python3-all-dev python-all-dev python-stdeb python3-stdeb
+
+RUN pip --no-cache-dir install cython && pip3 --no-cache-dir install cython
+
+RUN ./install_curl.sh /curl && ./install_cmake.sh
+
+ADD install_libfabric.sh install_libfabric.sh
+RUN ./install_libfabric.sh
+
+ARG OS
+ENV OS=${OS}
+ADD build.sh /bin/build.sh
\ No newline at end of file
diff --git a/deploy/build_env/debians/Dockerfile_debian9.13 b/deploy/build_env/debians/Dockerfile_debian9.13
new file mode 100644
index 000000000..c59953f71
--- /dev/null
+++ b/deploy/build_env/debians/Dockerfile_debian9.13
@@ -0,0 +1,20 @@
+from debian:9.13
+
+ENV GOPATH /tmp
+
+ADD install_curl.sh install_curl.sh
+ADD install_cmake.sh install_cmake.sh
+
+RUN apt update && apt install -y g++ git wget python python3 python-numpy python3-numpy python-pip python3-pip \
+zlib1g-dev python3-all-dev python-all-dev python-stdeb python3-stdeb
+
+RUN pip --no-cache-dir install cython && pip3 --no-cache-dir install cython
+
+RUN ./install_curl.sh /curl && ./install_cmake.sh
+
+ADD install_libfabric.sh install_libfabric.sh
+RUN ./install_libfabric.sh
+
+ARG OS
+ENV OS=${OS}
+ADD build.sh /bin/build.sh
\ No newline at end of file
diff --git a/deploy/build_env/debians/Dockerfile_ubuntu16.04 b/deploy/build_env/debians/Dockerfile_ubuntu16.04
new file mode 100644
index 000000000..9b672c32f
--- /dev/null
+++ b/deploy/build_env/debians/Dockerfile_ubuntu16.04
@@ -0,0 +1,20 @@
+from ubuntu:16.04
+
+ENV GOPATH /tmp
+
+ADD install_curl.sh install_curl.sh
+ADD install_cmake.sh install_cmake.sh
+
+RUN apt update && apt install -y g++ git wget python python3 python-numpy python3-numpy python-pip python3-pip \
+zlib1g-dev python3-all-dev python-all-dev python-stdeb python3-stdeb
+
+RUN pip --no-cache-dir install cython && pip3 --no-cache-dir install cython
+
+RUN ./install_curl.sh /curl && ./install_cmake.sh
+
+ADD install_libfabric.sh install_libfabric.sh
+RUN ./install_libfabric.sh
+
+ARG OS
+ENV OS=${OS}
+ADD build.sh /bin/build.sh
\ No newline at end of file
diff --git a/deploy/build_env/debians/Dockerfile_ubuntu18.04 b/deploy/build_env/debians/Dockerfile_ubuntu18.04
new file mode 100644
index 000000000..f90bc71d2
--- /dev/null
+++ b/deploy/build_env/debians/Dockerfile_ubuntu18.04
@@ -0,0 +1,20 @@
+from ubuntu:18.04
+
+ENV GOPATH /tmp
+
+ADD install_curl.sh install_curl.sh
+ADD install_cmake.sh install_cmake.sh
+
+RUN apt update && apt install -y g++ git wget python python3 python-numpy python3-numpy python-pip python3-pip \
+zlib1g-dev python3-all-dev python-all-dev python-stdeb python3-stdeb
+
+RUN pip --no-cache-dir install cython && pip3 --no-cache-dir install cython
+
+RUN ./install_curl.sh /curl && ./install_cmake.sh
+
+ADD install_libfabric.sh install_libfabric.sh
+RUN ./install_libfabric.sh
+
+ARG OS
+ENV OS=${OS}
+ADD build.sh /bin/build.sh
\ No newline at end of file
diff --git a/deploy/build_env/debians/build.sh b/deploy/build_env/debians/build.sh
new file mode 100755
index 000000000..5037380f7
--- /dev/null
+++ b/deploy/build_env/debians/build.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd /asapo/build
+cmake \
+ -DCMAKE_BUILD_TYPE="Release" \
+ -DENABLE_LIBFABRIC=ON \
+ -DINSTALL_EXAMPLES=ON \
+ -DBUILD_CLIENTS_ONLY=ON \
+ -DPACKAGE_RELEASE_SUFFIX=$OS \
+ -DBUILD_PYTHON=OFF \
+ -DLIBCURL_DIR=/curl \
+ -DCPACK_PACKAGE_NAME="asapo-dev" \
+ -DCPACK_GENERATOR="DEB" \
+ ..
+cmake .. #second time for to correctly build deb packages
+make
+make package
+
+cmake -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DBUILD_PYTHON_PACKAGES="source;deb" ..
+make
diff --git a/deploy/build_env/debians/build_images.sh b/deploy/build_env/debians/build_images.sh
new file mode 100755
index 000000000..cdf50e6a9
--- /dev/null
+++ b/deploy/build_env/debians/build_images.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+
+vers="ubuntu16.04 ubuntu18.04 debian9.13 debian10.7"
+
+for ver in $vers
+do
+ docker build -t yakser/asapo-env:${ver} -f Dockerfile_${ver} --build-arg OS=${ver} .
+ docker push yakser/asapo-env:${ver}
+done
+
+
+
diff --git a/deploy/build_env/Ubuntu16.04/install_cmake.sh b/deploy/build_env/debians/install_cmake.sh
similarity index 82%
rename from deploy/build_env/Ubuntu16.04/install_cmake.sh
rename to deploy/build_env/debians/install_cmake.sh
index 3e399d4ec..25b7917fc 100755
--- a/deploy/build_env/Ubuntu16.04/install_cmake.sh
+++ b/deploy/build_env/debians/install_cmake.sh
@@ -2,8 +2,8 @@
wget https://cmake.org/files/v3.10/cmake-3.10.0.tar.gz
-tar zxvf cmake-3.*
-cd cmake-3.*
+tar zxvf cmake-3.10.0.tar.gz
+cd cmake-3.10.0
./bootstrap --prefix=/usr/local
make -j$(nproc)
make install
diff --git a/deploy/build_env/Ubuntu16.04/install_curl.sh b/deploy/build_env/debians/install_curl.sh
similarity index 100%
rename from deploy/build_env/Ubuntu16.04/install_curl.sh
rename to deploy/build_env/debians/install_curl.sh
diff --git a/deploy/build_env/Ubuntu16.04/install_libfabric.sh b/deploy/build_env/debians/install_libfabric.sh
similarity index 100%
rename from deploy/build_env/Ubuntu16.04/install_libfabric.sh
rename to deploy/build_env/debians/install_libfabric.sh
diff --git a/deploy/build_env/manylinux2010/build_image.sh b/deploy/build_env/manylinux2010/build_image.sh
index ed3fc8061..9a2bfe886 100755
--- a/deploy/build_env/manylinux2010/build_image.sh
+++ b/deploy/build_env/manylinux2010/build_image.sh
@@ -4,8 +4,7 @@
#./docker-squash yakser/asapo-env:manylinux2010_ -t yakser/asapo-env:manylinux2010
docker build -t yakser/asapo-env:manylinux2010 .
-#docker push yakser/asapo-env:manylinux2010
-#docker push yakser/asapo-env:manylinux2010
+docker push yakser/asapo-env:manylinux2010
diff --git a/producer/api/python/dist_linux/CMakeLists.txt b/producer/api/python/dist_linux/CMakeLists.txt
index 654c4a611..a76ce7c22 100644
--- a/producer/api/python/dist_linux/CMakeLists.txt
+++ b/producer/api/python/dist_linux/CMakeLists.txt
@@ -5,16 +5,40 @@ file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/setup.py INPUT ${CMAKE_CURRENT_
if ("rpm" IN_LIST BUILD_PYTHON_PACKAGES)
ADD_CUSTOM_TARGET(python-rpm-producer ALL
- COMMAND PACKAGE_PREFIX=python- python setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX} --requires=numpy
+ COMMAND PACKAGE_PREFIX=python- python setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX}
+ --requires=numpy --binary-only
+ COMMAND rm dist/python*.gz dist/*debuginfo*
)
ADD_CUSTOM_TARGET(python3-rpm-producer ALL
- COMMAND PACKAGE_PREFIX=python3- python3 setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX} --requires=python3-numpy
+ COMMAND PACKAGE_PREFIX=python3- python3 setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX}
+ --requires=python3-numpy --binary-only
+ COMMAND rm dist/python3*.gz dist/*debuginfo*
)
ADD_DEPENDENCIES(python3-rpm-producer copy_python_dist-producer)
ADD_DEPENDENCIES(python-rpm-producer copy_python_dist-producer)
endif()
+if ("deb" IN_LIST BUILD_PYTHON_PACKAGES)
+ ADD_CUSTOM_TARGET(python-deb-producer ALL
+ COMMAND rm -rf deb_dist/*/
+ COMMAND PACKAGE_PREFIX= python setup.py --command-packages=stdeb.command
+ sdist_dsc --debian-version=${PACKAGE_RELEASE_SUFFIX} --depends=python-numpy bdist_deb
+ COMMAND rm deb_dist/*dbgsym*
+ COMMAND cp deb_dist/*.deb dist/
+ )
+
+ ADD_CUSTOM_TARGET(python3-deb-producer ALL
+ COMMAND rm -rf deb_dist/*/
+ COMMAND PACKAGE_PREFIX= python3 setup.py --command-packages=stdeb.command
+ sdist_dsc --debian-version=${PACKAGE_RELEASE_SUFFIX} --depends3=python3-numpy bdist_deb
+ COMMAND rm deb_dist/*dbgsym*
+ COMMAND cp deb_dist/*.deb dist/
+ )
+ ADD_DEPENDENCIES(python3-deb-producer copy_python_dist-producer)
+ ADD_DEPENDENCIES(python-deb-producer copy_python_dist-producer)
+endif()
+
if ("source" IN_LIST BUILD_PYTHON_PACKAGES )
ADD_CUSTOM_TARGET(python-dist-producer ALL
COMMAND ${Python_EXECUTABLE} setup.py sdist
--
GitLab