refactor cmake files

CMakeLists.txt

@@ -34,6 +34,13 @@ option(BUILD_CLIENTS_ONLY "Build clients only" OFF)
 option(BUILD_CONSUMER_TOOLS "Build consumer tools" OFF)
 option(BUILD_EVENT_MONITOR_PRODUCER "Build the event monitor producer" OFF)
 
+option(BUILD_PYTHON "Build python libs" ON)
+if(BUILD_PYTHON)
+    set(BUILD_PYTHON_PACKAGES "" CACHE STRING "which python packages to build")
+    set_property(CACHE BUILD_PYTHON_PACKAGES PROPERTY STRINGS source rpm deb win)
+endif()
+
+
 option(BUILD_TESTS "Build unit tests" OFF)
 option(BUILD_INTEGRATION_TESTS "Include integration tests" OFF)
 
@@ -46,6 +53,7 @@ option(BUILD_PYTHON_DOCS "Build the Python documentaion with sphinx" OFF)
 option(ENABLE_LIBFABRIC "Enables LibFabric support for RDMA transfers" OFF)
 option(ENABLE_LIBFABRIC_LOCALHOST "Emulates LibFabric stack over TCP. Only for localhost and testing purposes." OFF)
 
+
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMakeModules/)
 
 set (ASAPO_CXX_COMMON_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/common/cpp/include)

consumer/CMakeLists.txt

 add_subdirectory(api/cpp)
-add_subdirectory(api/python)
+
+if (BUILD_PYTHON)
+    add_subdirectory(api/python)
+endif()
 
 
 if(BUILD_CONSUMER_TOOLS)

consumer/api/python/CMakeLists.txt

@@ -2,7 +2,7 @@
 if (UNIX)
     include(CMakeLists_Linux.cmake)
     set (suf "so")
-ELSEIF(CMAKE_BUILD_TYPE STREQUAL "Release")
+ELSE()
     set (suf "pyd")
     include(CMakeLists_Windows.cmake)
 ENDIF()
\ No newline at end of file

consumer/api/python/CMakeLists_Linux.cmake

@@ -19,8 +19,10 @@ set (ASAPO_CONSUMER_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../cpp/include)
 
 configure_files(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} @ONLY)
 
-ADD_CUSTOM_TARGET(python-lib COMMAND ${Python_EXECUTABLE} setup.py build_ext --inplace --force)
+ADD_CUSTOM_TARGET(python-lib-consumer ALL COMMAND ${Python_EXECUTABLE} setup.py build_ext --inplace --force)
 
-ADD_DEPENDENCIES(python-lib asapo-consumer)
+ADD_DEPENDENCIES(python-lib-consumer asapo-consumer)
 
-add_subdirectory(source_dist_linux)
+if (BUILD_PYTHON_PACKAGES)
+    add_subdirectory(dist_linux)
+endif()

consumer/api/python/source_dist_linux/CMakeLists.txt → consumer/api/python/dist_linux/CMakeLists.txt

 configure_file(setup.py.in setup.py @ONLY)
 configure_file(MANIFEST.in MANIFEST.in @ONLY)
 
-ADD_CUSTOM_TARGET(python-dist
+if ("source" IN_LIST BUILD_PYTHON_PACKAGES )
+    ADD_CUSTOM_TARGET(python-dist-consumer ALL
         COMMAND ${Python_EXECUTABLE} setup.py sdist
         )
-
-ADD_CUSTOM_TARGET(python-rpm-consumer
-        COMMAND PACKAGE_PREFIX=python- python setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX} --requires=numpy
-        )
-
-ADD_CUSTOM_TARGET(python3-rpm-consumer
-        COMMAND PACKAGE_PREFIX=python3- python3 setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX} --requires=python3-numpy
-        )
-
-#get_property(ASAPO_CONSUMER_LIB TARGET asapo-consumer PROPERTY LOCATION)
-
-ADD_CUSTOM_TARGET(copy_python_dist
+    ADD_DEPENDENCIES(python-dist-consumer copy_python_dist-consumer)
+endif()
+
+if ("rpm" IN_LIST BUILD_PYTHON_PACKAGES)
+    ADD_CUSTOM_TARGET(python-rpm-consumer ALL
+            COMMAND PACKAGE_PREFIX=python- python setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX} --requires=numpy
+            )
+
+    ADD_CUSTOM_TARGET(python3-rpm-consumer ALL
+            COMMAND PACKAGE_PREFIX=python3- python3 setup.py bdist_rpm --release=1.${PACKAGE_RELEASE_SUFFIX} --requires=python3-numpy
+            )
+    ADD_DEPENDENCIES(python-rpm-consumer copy_python_dist-consumer)
+    ADD_DEPENDENCIES(python3-rpm-consumer copy_python_dist-consumer)
+endif()
+
+ADD_CUSTOM_TARGET(copy_python_dist-consumer ALL
         COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/../asapo_wrappers.h ${CMAKE_CURRENT_BINARY_DIR}/.
         COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/../asapo_consumer.cpp ${CMAKE_CURRENT_BINARY_DIR}/.
         COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/consumer/api/cpp/include ${CMAKE_CURRENT_BINARY_DIR}/include
@@ -28,9 +33,4 @@ ADD_CUSTOM_TARGET(copy_python_dist
 
         )
 
-
-ADD_DEPENDENCIES(copy_python_dist python-lib)
-
-ADD_DEPENDENCIES(python-dist copy_python_dist)
-ADD_DEPENDENCIES(python-rpm-consumer python-dist)
-ADD_DEPENDENCIES(python3-rpm-consumer python-dist)
+ADD_DEPENDENCIES(copy_python_dist-consumer python-lib-consumer)

deploy/build_env/Ubuntu16.04/Dockerfile

@@ -4,13 +4,16 @@ ENV GOPATH /tmp
 
 ADD install_curl.sh install_curl.sh
 ADD install_cmake.sh install_cmake.sh
-ADD install_libfabric.sh install_libfabric.sh
 
 RUN apt update && apt install -y g++ golang zlib1g-dev python cython python-numpy python3 cython3 python3-numpy  \
 git wget python-pip python3-pip && \
 pip  --no-cache-dir install --upgrade cython && pip3  --no-cache-dir install --upgrade cython && pip3  --no-cache-dir install sphinx && \
-./install_curl.sh /curl &&  ./install_cmake.sh && ./install_libfabric.sh \
-apt-get purge -y --auto-remove
+./install_curl.sh /curl &&  ./install_cmake.sh
+
+ADD install_libfabric.sh install_libfabric.sh
+RUN ./install_libfabric.sh
+
+RUN apt-get purge -y --auto-remove
 
 ADD build.sh /bin/build.sh
 

deploy/build_env/Ubuntu16.04/build.sh

@@ -8,9 +8,11 @@ cmake \
     -DENABLE_LIBFABRIC=on \
     -DLIBCURL_DIR=/curl \
     -DBUILD_PYTHON_DOCS=ON \
-    -DBUILD_EVENT_MONITOR_PRODUCER=ON \
+    -DBUILD_PYTHON_PACKAGES=source \
+    -DBUILD_CLIENTS_ONLY=ON \
     ..
-cd consumer/api/python/source_dist_linux && make python-dist
-cd ../producer/api/python/source_dist_linux && make python-dist-producer
-cd ../docs/sphinx && make
+make
+#cd consumer/api/python/dist_linux && make python-dist-consumer
+#cd ../producer/api/python/dist_linux && make python-dist-producer
+#cd ../docs/sphinx && make
 

deploy/build_env/Ubuntu16.04/install_libfabric.sh

 #!/usr/bin/env bash
 
-apt install -y wget autoconf libtool make librdmacm-dev rdma-core
 wget https://github.com/ofiwg/libfabric/archive/v1.11.0.tar.gz
 tar xzf v1.11.0.tar.gz
+
+apt install -y autoconf libtool make librdmacm-dev mc
+
 cd libfabric-1.11.0
 ./autogen.sh
 ./configure

deploy/build_env/centos/build.sh

@@ -7,12 +7,12 @@ cmake \
     -DINSTALL_EXAMPLES=ON \
     -DBUILD_CLIENTS_ONLY=ON \
     -DPACKAGE_RELEASE_SUFFIX=$OS \
-    -DNUMPY_VERSION=0   \
+    -DBUILD_PYTHON=OFF   \
     -DLIBCURL_DIR=/curl \
     ..
 make
 make package
-cd consumer/api/python/source_dist_linux && make python-rpm-consumer python3-rpm-consumer && cd -
-cd producer/api/python/source_dist_linux && make python-rpm-producer python3-rpm-producer
 
+cmake -DNUMPY_VERSION=0 -DBUILD_PYTHON=ON -DBUILD_PYTHON_PACKAGES="source;rpm" ..
+make
 

deploy/build_env/manylinux2010/build.sh

@@ -18,12 +18,17 @@ for python_path in /opt/python/cp{27,35,36,37,38}*; do
     echo "building wheel for python_version=$python_version with numpy_version=$numpy_version"
 
     cd /asapo/build
-    cmake -DENABLE_LIBFABRIC=on -DCMAKE_BUILD_TYPE="Release" -DLIBCURL_DIR=/curl -DPython_EXECUTABLE=$python -DNUMPY_VERSION=$numpy_version ..
-    cd /asapo/build/consumer/api/python/source_dist_linux \
+    cmake -DENABLE_LIBFABRIC=on \
+          -DCMAKE_BUILD_TYPE="Release" \
+          -DBUILD_CLIENTS_ONLY=ON \
+          -DLIBCURL_DIR=/curl -DPython_EXECUTABLE=$python \
+          -DBUILD_PYTHON_PACKAGES=source \
+          -DNUMPY_VERSION=$numpy_version ..
+    cd /asapo/build/consumer/api/python/dist_linux \
         && $pip install -r ../dev-requirements.txt \
-        && make python-dist \
+        && make python-dist-consumer \
         && $pip wheel dist/*.tar.gz -w /asapo/build/consumer/wheelhouse --no-deps
-    cd  /asapo/build/producer/api/python/source_dist_linux \
+    cd  /asapo/build/producer/api/python/dist_linux \
         && $pip install -r ../dev-requirements.txt \
         && make python-dist-producer \
         && $pip wheel dist/*.tar.gz -w /asapo/build/producer/wheelhouse --no-deps

deploy/build_env/manylinux2010/build_image.sh

 #!/usr/bin/env bash
 
-docker build -t yakser/asapo-env:manylinux2010_ .
-./docker-squash yakser/asapo-env:manylinux2010_ -t yakser/asapo-env:manylinux2010
-docker push yakser/asapo-env:manylinux2010
+#docker build -t yakser/asapo-env:manylinux2010_ .
+#./docker-squash yakser/asapo-env:manylinux2010_ -t yakser/asapo-env:manylinux2010
+
+docker build -t yakser/asapo-env:manylinux2010 .
+#docker push yakser/asapo-env:manylinux2010
+#docker push yakser/asapo-env:manylinux2010
+
+
 

docs/sphinx/CMakeLists.txt

@@ -9,4 +9,4 @@ add_custom_target(
 )
 
 add_dependencies(docs-py python-lib-producer)
-add_dependencies(docs-py python-lib)
+add_dependencies(docs-py python-lib-consumer)

examples/consumer/CMakeLists.txt

@@ -2,10 +2,8 @@ find_package(Threads)
 
 add_subdirectory(getnext_broker)
 
-if(BUILD_EXAMPLES)
-    if (UNIX OR CMAKE_BUILD_TYPE STREQUAL "Release")
+if(BUILD_EXAMPLES AND BUILD_PYTHON)
         add_subdirectory(getnext_broker_python)
-    endif()
 endif()
 
 

examples/consumer/getnext_broker_python/CMakeLists.txt

@@ -4,7 +4,7 @@ set(TARGET_NAME getnext_broker_python)
 prepare_asapo()
 
 if (UNIX)
-    get_target_property(PYTHON_LIBS python-lib BINARY_DIR)
+    get_target_property(PYTHON_LIBS python-lib-consumer BINARY_DIR)
 else()
     get_target_property(PYTHON_LIBS asapo_consumer BINARY_DIR)
 endif()

examples/pipeline/CMakeLists.txt

 
 add_subdirectory(in_to_out)
 
-if (UNIX OR CMAKE_BUILD_TYPE STREQUAL "Release")
+if (BUILD_PYTHON)
     add_subdirectory(in_to_out_python)
 endif()

examples/pipeline/in_to_out_python/CMakeLists.txt

@@ -3,7 +3,7 @@ set(TARGET_NAME pipeline_inout_python)
 prepare_asapo()
 
 if (UNIX)
-    get_target_property(PYTHON_LIBS_CONSUMER python-lib BINARY_DIR)
+    get_target_property(PYTHON_LIBS_CONSUMER python-lib-consumer BINARY_DIR)
     get_target_property(PYTHON_LIBS_PRODUCER python-lib-producer BINARY_DIR)
 else()
     get_target_property(PYTHON_LIBS_CONSUMER asapo_consumer BINARY_DIR)

producer/CMakeLists.txt

 add_subdirectory(api/cpp)
-add_subdirectory(api/python)
+
+if (BUILD_PYTHON)
+    add_subdirectory(api/python)
+endif()
 
 if(BUILD_EVENT_MONITOR_PRODUCER)
     add_subdirectory(event_monitor_producer)

producer/api/python/CMakeLists.txt

@@ -2,7 +2,7 @@
 if (UNIX)
     include(CMakeLists_Linux.cmake)
     set (suf "so")
-ELSEIF(CMAKE_BUILD_TYPE STREQUAL "Release")
+ELSE()
     set (suf "pyd")
     include(CMakeLists_Windows.cmake)
 ENDIF()
\ No newline at end of file