diff --git a/.gitignore b/.gitignore
index 005f5fbce704c564ffd17f71971fd02faa6a99e1..46ab8a011f7afe328452bc4f49a077f2cb5a23a5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -135,4 +135,7 @@ asapo_tools/pkg
#version files
common/cpp/include/common/version.h
-common/go/src/asapo_common/version/version_lib.go
\ No newline at end of file
+common/go/src/asapo_common/version/version_lib.go
+
+
+!sphinx/*
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 84299a75a24e093527305f41c39c973e92cd7377..8fba9c3a691810e151bfe10dbe06b20d5abab0ea 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -31,6 +31,7 @@ option(BUILD_TESTS "Uses googletest to build tests" OFF)
option(BUILD_INTEGRATION_TESTS "Include integration tests (CMAKE >3.7 is needed)" OFF)
option(BUILD_DOCS "Uses doxygen to build the documentaion" OFF)
option(BUILD_BROKER "Build broker" OFF)
+option(BUILD_PYTHON_DOCS "Uses sphinx to build the Python documentaion" OFF)
option(BUILD_WORKER_TOOLS "Build worker tools" OFF)
option(BUILD_EXAMPLES "Build examples" OFF)
@@ -93,4 +94,13 @@ if(BUILD_DOCS)
include(doxygen)
endif()
+
+if(BUILD_PYTHON_DOCS)
+ add_subdirectory (sphinx)
+endif()
+
+
+
+
+
include(prepare_version)
\ No newline at end of file
diff --git a/deploy/docker_buildenv/Ubuntu16.04/Dockerfile b/deploy/docker_buildenv/Ubuntu16.04/Dockerfile
index 7cdc53109b0248001fdfdd72dc7857fef7ba1954..45fe5432d1ac7b1a3ad4a0df0dee6a2010ccbd92 100644
--- a/deploy/docker_buildenv/Ubuntu16.04/Dockerfile
+++ b/deploy/docker_buildenv/Ubuntu16.04/Dockerfile
@@ -9,4 +9,7 @@ ADD install_curl.sh install_curl.sh
RUN ./install_curl.sh /curl
RUN apt install -y python-pip python3-pip && pip install --upgrade cython && pip3 install --upgrade cython
+RUN pip3 install sphinx
+
ADD build.sh /bin/build.sh
+
diff --git a/deploy/docker_buildenv/Ubuntu16.04/build.sh b/deploy/docker_buildenv/Ubuntu16.04/build.sh
index 48e7d972c0f0c7af47d1f4977f8e767f5ccdbeb4..edebc5038a65d919b7dbb10f9db3973c97d12c8f 100755
--- a/deploy/docker_buildenv/Ubuntu16.04/build.sh
+++ b/deploy/docker_buildenv/Ubuntu16.04/build.sh
@@ -1,7 +1,8 @@
#!/usr/bin/env bash
cd /asapo/build
-cmake -DCMAKE_BUILD_TYPE="Release" -DLIBCURL_DIR=/curl ..
+cmake -DCMAKE_BUILD_TYPE="Release" -DLIBCURL_DIR=/curl -DBUILD_PYTHON_DOCS=ON ..
cd worker && make
cd ../producer && make
+cd ../sphinx && make
diff --git a/producer/api/python/asapo_producer.pyx.in b/producer/api/python/asapo_producer.pyx.in
index a5987b464c5ce159774d1d8270515d58e0ad5ef0..d3c884afff427085c70f0b9ebdd1e8345f94709d 100644
--- a/producer/api/python/asapo_producer.pyx.in
+++ b/producer/api/python/asapo_producer.pyx.in
@@ -15,7 +15,6 @@ INGEST_MODE_TRANSFER_DATA = kTransferData
INGEST_MODE_TRANSFER_METADATA_ONLY = kTransferMetaDataOnly
INGEST_MODE_STORE_IN_FILESYSTEM = kStoreInFilesystem
-
cdef extern from "numpy/ndarraytypes.h":
void PyArray_ENABLEFLAGS(np.ndarray arr, int flags)
@@ -57,6 +56,8 @@ cdef class PyProducer:
log_level = LogLevel_Debug
elif level == "info" :
log_level = LogLevel_Info
+ elif level == "error" :
+ log_level = LogLevel_Error
elif level == "none" :
log_level = LogLevel_None
elif level == "warn" :
@@ -66,7 +67,7 @@ cdef class PyProducer:
return
self.c_producer.get().SetLogLevel(log_level)
- def send_np_array(self,int id, exposed_path,data, user_meta=None,subset=None,ingest_mode = DEFAULT_INGEST_MODE,callback=None):
+ def __send_np_array(self,int id, exposed_path,data, user_meta=None,subset=None,ingest_mode = DEFAULT_INGEST_MODE,callback=None):
cdef EventHeader event_header = self.create_event_header(id,exposed_path,user_meta,subset,ingest_mode)
event_header.file_id = id
if data is None:
@@ -99,7 +100,7 @@ cdef class PyProducer:
event_header.subset_size = subset[1]
return event_header
- def send_bytes(self,int id, exposed_path,data, user_meta=None,subset=None,ingest_mode = DEFAULT_INGEST_MODE,callback=None):
+ def __send_bytes(self,int id, exposed_path,data, user_meta=None,subset=None,ingest_mode = DEFAULT_INGEST_MODE,callback=None):
cdef EventHeader event_header = self.create_event_header(id,exposed_path,user_meta,subset,ingest_mode)
event_header.file_size = len(data)
err = self.c_producer.get().SendData_(event_header, data_pointer_bytes(data), ingest_mode,
@@ -112,15 +113,53 @@ cdef class PyProducer:
Py_XINCREF(<PyObject*>data)
return None
- def send_data(self,int id, exposed_path,data, user_meta=None,subset=None,ingest_mode = DEFAULT_INGEST_MODE,callback=None):
+ def send_data(self, id, exposed_path,data, user_meta=None,subset=None,ingest_mode = DEFAULT_INGEST_MODE,callback=None):
+ """
+ :param id: unique data id
+ :type id: int
+ :param exposed_path: Path which will be exposed to consumers
+ :type exposed_path: string
+ :param data: data to send
+ :type data: contiguous numpy or bytes array, can be None for INGEST_MODE_TRANSFER_METADATA_ONLY ingest mode
+ :param user_meta: user metadata, default None
+ :type user_meta: JSON string
+ :param subset: a tuple with two int values (subset id, subset size), default None
+ :type subset: tuple
+ :param ingest_mode: ingest mode flag
+ :type ingest_mode: int
+ :param callback: callback function, default None
+ :type callback: callback(info,err), where info - json string with event header that was used to send data, err - error string or None
+ :return: error, None if success.
+ :rtype: string
+ """
+
if type(data) == np.ndarray or data == None:
- return self.send_np_array(id,exposed_path,data,user_meta,subset,ingest_mode,callback)
+ return self.__send_np_array(id,exposed_path,data,user_meta,subset,ingest_mode,callback)
elif type(data) == bytes:
- return self.send_bytes(id,exposed_path,data,user_meta,subset,ingest_mode,callback)
+ return self.__send_bytes(id,exposed_path,data,user_meta,subset,ingest_mode,callback)
else:
return "wrong data type: " + str(type(data))
- def send_file(self,int id, local_path, exposed_path,user_meta=None,subset=None,ingest_mode = DEFAULT_INGEST_MODE,callback=None):
+ def send_file(self, id, local_path, exposed_path,user_meta=None,subset=None,ingest_mode = DEFAULT_INGEST_MODE,callback=None):
+ """
+ :param id: unique data id
+ :type id: int
+ :param local_path: Path to file to send
+ :type local_path: string
+ :param exposed_path: Path which will be exposed to consumers
+ :type exposed_path: string
+ :param user_meta: user metadata, default None
+ :type user_meta: JSON string
+ :param subset: a tuple with two int values (subset id, subset size), default None
+ :type subset: tuple
+ :param ingest_mode: ingest mode flag
+ :type ingest_mode: int
+ :param callback: callback function, default None
+ :type callback: callback(info,err), where info - json string with event header that was used to send data, err - error string or None
+ :return: error, None if success.
+ :rtype: string
+ """
+
cdef EventHeader event_header = self.create_event_header(id,exposed_path,user_meta,subset,ingest_mode)
event_header.file_size = 0
err = self.c_producer.get().SendFile(event_header, _bytes(local_path), ingest_mode,
@@ -158,7 +197,7 @@ cdef class PyProducer:
self.c_callback_python(py_callback,header,GetErrorString(&err))
@staticmethod
- def create_producer(endpoint,beamtime_id,stream,token,nthreads):
+ def __create_producer(endpoint,beamtime_id,stream,token,nthreads):
pyProd = PyProducer()
cdef Error err
cdef SourceCredentials source
@@ -173,7 +212,7 @@ cdef class PyProducer:
return pyProd,None
def create_producer(endpoint,beamtime_id,stream,token,nthreads):
- return PyProducer.create_producer(_bytes(endpoint),_bytes(beamtime_id),_bytes(stream),_bytes(token),nthreads)
+ return PyProducer.__create_producer(_bytes(endpoint),_bytes(beamtime_id),_bytes(stream),_bytes(token),nthreads)
__version__ = "@ASAPO_VERSION_PYTHON@"
diff --git a/producer/api/python/setup.py.in b/producer/api/python/setup.py.in
index 81fa3e5a0289f328993058039d895132e241b7e1..180039026505919394e2739dbb3e070a4e4c97c1 100644
--- a/producer/api/python/setup.py.in
+++ b/producer/api/python/setup.py.in
@@ -14,7 +14,7 @@ module = Extension("asapo_producer", ["asapo_producer.pyx"],
language="c++",
)
-ext_modules = cythonize([module])
+ext_modules = cythonize([module],compiler_directives={'embedsignature': True})
setup(ext_modules = ext_modules,
setup_requires=["numpy","cython>=0.28"],
diff --git a/sphinx/CMakeLists.txt b/sphinx/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7c5bab3f11c1cb145a4c790a974fdf825d7192f4
--- /dev/null
+++ b/sphinx/CMakeLists.txt
@@ -0,0 +1,12 @@
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
+
+add_custom_target(
+ pydoc ALL
+ COMMAND
+ make html VERBATIM
+ WORKING_DIRECTORY sphinx
+)
+
+add_dependencies(pydoc python-lib2-producer)
+add_dependencies(pydoc python-lib2)
diff --git a/sphinx/Makefile b/sphinx/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..70276370f6e9e85e88c5d7085b11b3161287a9a0
--- /dev/null
+++ b/sphinx/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+SPHINXPROJ = ASAPO
+SOURCEDIR = source
+BUILDDIR = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
diff --git a/sphinx/source/conf.py b/sphinx/source/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..7bcf43111ca77b773438b90d1857a8826c1660dc
--- /dev/null
+++ b/sphinx/source/conf.py
@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+#
+# ASAP::O documentation build configuration file, created by
+# sphinx-quickstart on Fri Aug 30 13:31:57 2019.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('../../../worker/api/python'))
+sys.path.insert(0, os.path.abspath('../../../producer/api/python'))
+
+
+import asapo_producer
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.autodoc',
+ 'sphinx.ext.doctest',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.todo',
+ 'sphinx.ext.ifconfig']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'ASAP::O'
+copyright = u'2019, DESY IT'
+author = u'DESY IT'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u''
+# The full version, including alpha/beta/rc tags.
+release = asapo_producer.__version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'classic'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+html_show_sourcelink = False
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'ASAPOdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'ASAPO.tex', u'ASAP::O Documentation',
+ u'DESY IT', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'asapo', u'ASAP::O Documentation',
+ [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'ASAPO', u'ASAP::O Documentation',
+ author, 'ASAPO', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
diff --git a/sphinx/source/consumer.rst b/sphinx/source/consumer.rst
new file mode 100644
index 0000000000000000000000000000000000000000..4bda1c1a0f323091a120ed025b110a4790c7d53b
--- /dev/null
+++ b/sphinx/source/consumer.rst
@@ -0,0 +1,9 @@
+##################
+Consumer
+##################
+
+
+.. automodule:: asapo_worker
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/sphinx/source/index.rst b/sphinx/source/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..f1f36e7b36a75a74106ed51381dfd58f913773e1
--- /dev/null
+++ b/sphinx/source/index.rst
@@ -0,0 +1,17 @@
+.. ASAP::O documentation
+
+ASAP::O documentation
+===================================
+
+ASAPO::O
+
+
+Python API
+===================================
+
+.. toctree::
+ consumer
+
+.. toctree::
+ producer
+
diff --git a/sphinx/source/producer.rst b/sphinx/source/producer.rst
new file mode 100644
index 0000000000000000000000000000000000000000..b3985141cffca4fd1de08ef40976933fad739c5b
--- /dev/null
+++ b/sphinx/source/producer.rst
@@ -0,0 +1,29 @@
+##################
+Producer
+##################
+
+
+.. automodule:: asapo_producer
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Injest modes:
+-------------
+.. data:: INGEST_MODE_TRANSFER_DATA
+.. data:: INGEST_MODE_TRANSFER_METADATA_ONLY
+.. data:: INGEST_MODE_STORE_IN_FILESYSTEM
+.. data:: DEFAULT_INGEST_MODE = INGEST_MODE_TRANSFER_DATA | INGEST_MODE_STORE_IN_FILESYSTEM
+
+
+Logger levels:
+--------------
+info (default)
+
+error
+
+warn
+
+debug
+
+none
\ No newline at end of file
diff --git a/worker/api/python/CMakeLists_Linux.cmake b/worker/api/python/CMakeLists_Linux.cmake
index ffe11d7ae277da561c4f578f256536b680a46167..9c2d447fc2b6fdc6e82d6937427b1eeed4daf164 100644
--- a/worker/api/python/CMakeLists_Linux.cmake
+++ b/worker/api/python/CMakeLists_Linux.cmake
@@ -16,10 +16,10 @@ set (ASAPO_WORKER_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../cpp/include)
configure_files(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} @ONLY)
ADD_CUSTOM_TARGET(python-lib2 ALL
- COMMAND python setup.py build_ext --inplace)
+ COMMAND python setup.py build_ext --inplace --force)
ADD_CUSTOM_TARGET(python-lib3 ALL
- COMMAND python3 setup.py build_ext --inplace)
+ COMMAND python3 setup.py build_ext --inplace --force)
ADD_DEPENDENCIES(python-lib2 asapo-worker)
ADD_DEPENDENCIES(python-lib3 asapo-worker)
diff --git a/worker/api/python/asapo_worker.pyx.in b/worker/api/python/asapo_worker.pyx.in
index 059307c2b4227be19dc05227e8cd021d90d72937..cd19e16d5304de8fceef47e88edced7c12041bbb 100644
--- a/worker/api/python/asapo_worker.pyx.in
+++ b/worker/api/python/asapo_worker.pyx.in
@@ -172,7 +172,7 @@ cdef class PyDataBroker:
del meta['_id']
return meta, None
-cdef class PyDataBrokerFactory:
+cdef class __PyDataBrokerFactory:
cdef DataBrokerFactory c_factory
def __cinit__(self):
with nogil:
@@ -197,9 +197,17 @@ cdef class PyDataBrokerFactory:
else:
return broker,None
-def create_server_broker(server_name,source_path,beamtime_id,stream,token,timeout):
- factory = PyDataBrokerFactory()
- return factory.create_server_broker(_bytes(server_name),_bytes(source_path),_bytes(beamtime_id),_bytes(stream),_bytes(token),timeout)
+def create_server_broker(server_name,source_path,beamtime_id,stream,token,timeout_ms):
+ """
+ :param server_name: Server endpoint (hostname:port)
+ :type server_name: string
+ :param source_path: Path to the folder to read data from
+ :type source_path: string
+ :return: Broker object and error. (None,err) if case of error, (broker, None) if success
+ :rtype: Tuple with broker object and error.
+ """
+ factory = __PyDataBrokerFactory()
+ return factory.create_server_broker(_bytes(server_name),_bytes(source_path),_bytes(beamtime_id),_bytes(stream),_bytes(token),timeout_ms)
__version__ = "@ASAPO_VERSION_PYTHON@"
diff --git a/worker/api/python/setup.py.in b/worker/api/python/setup.py.in
index 6d61825cfe2fd12b06616132409605cd204ffcfd..b2815861f2983e18f31171082368afaf5bb51383 100644
--- a/worker/api/python/setup.py.in
+++ b/worker/api/python/setup.py.in
@@ -14,6 +14,7 @@ module = Extension("asapo_worker", ["asapo_worker.pyx"],
language="c++",
)
-ext_modules = cythonize([module])
+
+ext_modules = cythonize([module],compiler_directives={'embedsignature': True})
setup(ext_modules = ext_modules)