From 602d5df1ae2146052c25bb575edf4d71cc7a1e89 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon, 6 Dec 2021 14:08:48 +0100
Subject: [PATCH] refactor python packaging, added python3.9

---
 CHANGELOG.md                                     |  2 ++
 CMakeIncludes/prepare_version_tag.cmake          | 12 +++++++++++-
 CMakeLists.txt                                   |  2 +-
 deploy/build_env/centos/build.sh                 |  8 ++++++++
 deploy/build_env/centos/install_curl.sh          |  2 +-
 deploy/build_env/debians/build.sh                |  7 +++----
 deploy/build_env/debians/install_curl.sh         |  2 +-
 deploy/build_env/manylinux2010/Dockerfile        |  2 +-
 deploy/build_env/manylinux2010/build.sh          |  6 +++---
 deploy/build_env/manylinux2010/build_image.sh    |  4 ++--
 deploy/build_env/manylinux2010/install_curl.sh   |  2 +-
 docs/site/examples/install_python_clients_pip.sh | 12 ++++++------
 docs/site/examples/install_python_clients_pkg.sh |  8 ++++----
 docs/site/freeze_version.sh                      |  3 ++-
 14 files changed, 46 insertions(+), 26 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 95d828b26..e7dff6d3d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,8 @@
 FEATURES
 * Consumer API: Get last within consumer group returns message only once
 * Producer API: An option to write raw data to core filesystem directly
+* Consumer/Producer API - packages for Debian 11.1
+* Consumer/Producer API - dropped Python 2 support for wheels and packages for new Debian/CentOS versions
 
 ## 21.09.0
 
diff --git a/CMakeIncludes/prepare_version_tag.cmake b/CMakeIncludes/prepare_version_tag.cmake
index 59de5c91e..30826e765 100644
--- a/CMakeIncludes/prepare_version_tag.cmake
+++ b/CMakeIncludes/prepare_version_tag.cmake
@@ -33,7 +33,17 @@ else()
     SET (ASAPO_VERSION_DOCKER_SUFFIX "-dev")
     string(REPLACE "_" "-" ASAPO_VERSION ${ASAPO_VERSION})
     SET (ASAPO_VERSION 100.0.${ASAPO_VERSION})
-    SET (PYTHON_ASAPO_VERSION ${ASAPO_VERSION})
+    if (${BRANCH} STREQUAL "develop")
+        SET (PYTHON_ASAPO_VERSION 100.0.dev0)
+    else()
+        string(SUBSTRING ${ASAPO_VERSION} 20 -1 TMP)
+        string(REGEX MATCH "^([0-9]+)|.+$" ISSUE_NUM "${TMP}")
+        if (ISSUE_NUM STREQUAL "")
+            SET (PYTHON_ASAPO_VERSION 100.0.dev1)
+        else()
+            SET (PYTHON_ASAPO_VERSION 100.0.dev${ISSUE_NUM})
+        endif()
+    endif()
     SET (ASAPO_WHEEL_VERSION ${ASAPO_VERSION})
 endif()
 
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4b3cd7593..4e5c19ed4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,7 +22,7 @@ set(ConfigPackageLocation lib/cmake/Asapo)
 # options
 
 option(BUILD_PYTHON "Build python libs" ON)
-option(BUILD_PYTHON2_PACKAGES "Build python2 packages" ON)
+option(BUILD_PYTHON2_PACKAGES "Build python2 packages" OFF)
 
 option(BUILD_CLIENTS_ONLY "Build clients only" OFF)
 
diff --git a/deploy/build_env/centos/build.sh b/deploy/build_env/centos/build.sh
index 103ae9e77..4b0cb05be 100755
--- a/deploy/build_env/centos/build.sh
+++ b/deploy/build_env/centos/build.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 cd /asapo/build
+
 cmake \
     -DCMAKE_BUILD_TYPE="Release" \
     -DENABLE_LIBFABRIC=ON \
@@ -15,6 +16,12 @@ cmake \
 make -j 4
 make package
 
+if [ $OS == "centos7.9.2009" ]; then
+  BUILD_PYTHON2_PACKAGES=ON
+else
+  BUILD_PYTHON2_PACKAGES=OFF
+fi
+
 #switch to static curl for Python packages
 rm CMakeCache.txt
 cmake \
@@ -26,6 +33,7 @@ cmake \
     -DBUILD_PYTHON=ON   \
     -DPACKAGE_RELEASE_SUFFIX=1.$OS \
     -DBUILD_PYTHON_PACKAGES="source;rpm"   \
+    -DBUILD_PYTHON2_PACKAGES=$BUILD_PYTHON2_PACKAGES \
     -DBUILD_PYTHON_DOCS=$BUILD_PYTHON_DOCS \
     ..
 make -j 1
diff --git a/deploy/build_env/centos/install_curl.sh b/deploy/build_env/centos/install_curl.sh
index d8017177b..ec291cfda 100755
--- a/deploy/build_env/centos/install_curl.sh
+++ b/deploy/build_env/centos/install_curl.sh
@@ -2,7 +2,7 @@
 
 mkdir -p $1
 cd $1
-wget https://curl.haxx.se/download/curl-7.58.0.tar.gz
+wget --no-check-certificate https://curl.haxx.se/download/curl-7.58.0.tar.gz
 tar xzf curl-7.58.0.tar.gz
 cd curl-7.58.0
 ./configure --without-ssl --disable-shared --disable-manual --disable-ares \
diff --git a/deploy/build_env/debians/build.sh b/deploy/build_env/debians/build.sh
index 6f1dabce1..844a4f2c3 100755
--- a/deploy/build_env/debians/build.sh
+++ b/deploy/build_env/debians/build.sh
@@ -25,13 +25,12 @@ else
   BUILD_PYTHON_DOCS=OFF
 fi
 
-if [ "$OS" = "debian11.1" ]; then
-  BUILD_PYTHON2_PACKAGES=OFF
-else
+if [ $OS == "debian9.13" -o $OS == "debian10.7" -o $OS == "ubuntu16.04" -o $OS == "ubuntu18.04" ]; then
   BUILD_PYTHON2_PACKAGES=ON
+else
+  BUILD_PYTHON2_PACKAGES=OFF
 fi
 
-
 #switch to static curl for Python packages
 rm CMakeCache.txt
 cmake \
diff --git a/deploy/build_env/debians/install_curl.sh b/deploy/build_env/debians/install_curl.sh
index d400f5235..12c6fb561 100755
--- a/deploy/build_env/debians/install_curl.sh
+++ b/deploy/build_env/debians/install_curl.sh
@@ -2,7 +2,7 @@
 
 mkdir -p $1
 cd $1
-wget https://curl.haxx.se/download/curl-7.58.0.tar.gz
+wget --no-check-certificate https://curl.haxx.se/download/curl-7.58.0.tar.gz
 tar xzf curl-7.58.0.tar.gz
 cd curl-7.58.0
 ./configure --without-ssl --disable-shared --disable-manual --disable-ares  \
diff --git a/deploy/build_env/manylinux2010/Dockerfile b/deploy/build_env/manylinux2010/Dockerfile
index cc73faaa3..31dde5c59 100644
--- a/deploy/build_env/manylinux2010/Dockerfile
+++ b/deploy/build_env/manylinux2010/Dockerfile
@@ -1,4 +1,4 @@
-FROM quay.io/pypa/manylinux2010_x86_64
+FROM quay.io/pypa/manylinux2010_x86_64:2021-04-05-a6ea1ab
 
 ENV GOPATH /tmp
 
diff --git a/deploy/build_env/manylinux2010/build.sh b/deploy/build_env/manylinux2010/build.sh
index 0fee6f1fa..c39149562 100755
--- a/deploy/build_env/manylinux2010/build.sh
+++ b/deploy/build_env/manylinux2010/build.sh
@@ -2,14 +2,13 @@
 set -e
 
 declare -A numpy_versions
-numpy_versions[cp27mu]=1.12.1
-numpy_versions[cp27m]=1.12.1
 numpy_versions[cp35m]=1.12.1
 numpy_versions[cp36m]=1.12.1
 numpy_versions[cp37m]=1.14.5
 numpy_versions[cp38]=1.17.3
+numpy_versions[cp39]=1.19.3
 
-for python_path in /opt/python/cp{27,35,36,37,38}*; do
+for python_path in /opt/python/cp{35,36,37,38,39}*; do
     python_version=$(basename $python_path)
     python_version=${python_version#*-}
     python=$python_path/bin/python
@@ -23,6 +22,7 @@ for python_path in /opt/python/cp{27,35,36,37,38}*; do
           -DBUILD_CLIENTS_ONLY=ON \
           -DLIBCURL_DIR=/curl -DPython_EXECUTABLE=$python \
           -DBUILD_PYTHON_PACKAGES=source \
+          -DBUILD_PYTHON2_PACKAGES=OFF \
           -DNUMPY_VERSION=$numpy_version ..
     cd /asapo/build/consumer/api/python/dist_linux \
         && $pip install -r ../dev-requirements.txt \
diff --git a/deploy/build_env/manylinux2010/build_image.sh b/deploy/build_env/manylinux2010/build_image.sh
index 9a2bfe886..cddfddad4 100755
--- a/deploy/build_env/manylinux2010/build_image.sh
+++ b/deploy/build_env/manylinux2010/build_image.sh
@@ -3,8 +3,8 @@
 #docker build -t yakser/asapo-env:manylinux2010_ .
 #./docker-squash yakser/asapo-env:manylinux2010_ -t yakser/asapo-env:manylinux2010
 
-docker build -t yakser/asapo-env:manylinux2010 .
-docker push yakser/asapo-env:manylinux2010
+docker build -t yakser/asapo-env:manylinux2010-2021-04-05-a6ea1ab .
+docker push yakser/asapo-env:manylinux2010-2021-04-05-a6ea1ab
 
 
 
diff --git a/deploy/build_env/manylinux2010/install_curl.sh b/deploy/build_env/manylinux2010/install_curl.sh
index d8017177b..ec291cfda 100755
--- a/deploy/build_env/manylinux2010/install_curl.sh
+++ b/deploy/build_env/manylinux2010/install_curl.sh
@@ -2,7 +2,7 @@
 
 mkdir -p $1
 cd $1
-wget https://curl.haxx.se/download/curl-7.58.0.tar.gz
+wget --no-check-certificate https://curl.haxx.se/download/curl-7.58.0.tar.gz
 tar xzf curl-7.58.0.tar.gz
 cd curl-7.58.0
 ./configure --without-ssl --disable-shared --disable-manual --disable-ares \
diff --git a/docs/site/examples/install_python_clients_pip.sh b/docs/site/examples/install_python_clients_pip.sh
index 3a9d1faf5..2acfcaa0c 100644
--- a/docs/site/examples/install_python_clients_pip.sh
+++ b/docs/site/examples/install_python_clients_pip.sh
@@ -1,13 +1,13 @@
 #!/usr/bin/env bash
 
-pip3 install --user --trusted-host nims.desy.de --find-links=http://nims.desy.de/extra/asapo/linux_wheels asapo_producer==100.0.develop
-pip3 install --user --trusted-host nims.desy.de --find-links=http://nims.desy.de/extra/asapo/linux_wheels asapo_consumer==100.0.develop
+pip3 install --user --trusted-host nims.desy.de --find-links=http://nims.desy.de/extra/asapo/linux_wheels asapo_producer==100.0.dev0
+pip3 install --user --trusted-host nims.desy.de --find-links=http://nims.desy.de/extra/asapo/linux_wheels asapo_consumer==100.0.dev0
 # you might need to update pip if the above commands error: pip3 install --upgrade pip
 
 # if that does not work (abi incompatibility, etc) you may try to install source packages
 # take a look at http://nims.desy.de/extra/asapo/linux_packages/ or http://nims.desy.de/extra/asapo/windows10 for your OS. E.g. for Debian 10.7
-# wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo_producer-100.0.develop.tar.gz
-# wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo_consumer-100.0.develop.tar.gz
+# wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo_producer-100.0.dev0.tar.gz
+# wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo_consumer-100.0.dev0.tar.gz
 
-# pip3 install asapo_producer-100.0.develop.tar.gz
-# pip3 install asapo_consumer-100.0.develop.tar.gz
+# pip3 install asapo_producer-100.0.dev0.tar.gz
+# pip3 install asapo_consumer-100.0.dev0.tar.gz
diff --git a/docs/site/examples/install_python_clients_pkg.sh b/docs/site/examples/install_python_clients_pkg.sh
index 4542fcc48..2dabe2c13 100644
--- a/docs/site/examples/install_python_clients_pkg.sh
+++ b/docs/site/examples/install_python_clients_pkg.sh
@@ -2,8 +2,8 @@
 
 # you can also install Linux/Windows packages if you have root access (or install locally).
 # take a look at http://nims.desy.de/extra/asapo/linux_packages/ or http://nims.desy.de/extra/asapo/windows10 for your OS. E.g. for Debian 10.7
-wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/python-asapo-producer_100.0~develop-debian10.7_amd64.deb
-wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/python-asapo-consumer_100.0~develop-debian10.7_amd64.deb
+wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/python-asapo-producer_100.0~dev0-debian10.7_amd64.deb
+wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/python-asapo-consumer_100.0~dev0-debian10.7_amd64.deb
 
-sudo apt install ./python3-asapo-producer_100.0~develop-debian10.7_amd64.deb
-sudo apt install ./python3-asapo_consumer_100.0~develop-debian10.7_amd64.deb
+sudo apt install ./python3-asapo-producer_100.0~dev0-debian10.7_amd64.deb
+sudo apt install ./python3-asapo_consumer_100.0~dev0-debian10.7_amd64.deb
diff --git a/docs/site/freeze_version.sh b/docs/site/freeze_version.sh
index 1e88279cb..9f2980af3 100755
--- a/docs/site/freeze_version.sh
+++ b/docs/site/freeze_version.sh
@@ -38,8 +38,9 @@ for file in $(find ./${VERSIONED_EXAMPLES} -type f)
 do
 ed -s $file <<ED_COMMANDS > /dev/null 2>&1
 ,s/asapo-cluster-dev:100\.0\.develop/asapo-cluster:${DOCS_VERSION}/g
-,s/==100\.0\.develop/==${VERSION_FOR_PIP}/g
+,s/==100\.0\.dev0/==${VERSION_FOR_PIP}/g
 ,s/100\.0[~.]develop/${DOCS_VERSION}/g
+,s/100\.0[~.]dev0/${DOCS_VERSION}/g
 w
 ED_COMMANDS
 done
-- 
GitLab