diff --git a/CMakeIncludes/prepare_version_tag.cmake b/CMakeIncludes/prepare_version_tag.cmake
index 7ce10de4822f4076c90ed12f1da61006cb1be1b6..e2a41293f288770013d83c7664018edbea285a6e 100644
--- a/CMakeIncludes/prepare_version_tag.cmake
+++ b/CMakeIncludes/prepare_version_tag.cmake
@@ -3,7 +3,7 @@ function(cleanup varname)
SET( ${varname} ${out} PARENT_SCOPE)
endfunction()
-execute_process(COMMAND git describe --tags --abbrev=0
+execute_process(COMMAND git describe --tags --abbrev=0
OUTPUT_VARIABLE ASAPO_TAG
WORKING_DIRECTORY ..)
string(STRIP ${ASAPO_TAG} ASAPO_TAG)
@@ -20,6 +20,7 @@ execute_process(COMMAND git rev-parse --short=10 HEAD
string(STRIP ${ASAPO_VERSION_COMMIT} ASAPO_VERSION_COMMIT)
if (${BRANCH} STREQUAL "master")
+ SET (ASAPO_VERSION_IN_DOCS ${ASAPO_TAG})
SET (ASAPO_VERSION ${ASAPO_TAG})
SET (ASAPO_VERSION_COMMIT "")
SET (ASAPO_VERSION_DOCKER_SUFFIX "")
@@ -37,6 +38,10 @@ else()
SET (ASAPO_WHEEL_VERSION ${ASAPO_VERSION})
endif()
+string(REGEX REPLACE "\\.0([0-9]+)\\."
+ ".\\1." ASAPO_WHEEL_VERSION_IN_DOCS
+ ${ASAPO_VERSION_IN_DOCS})
+
message("Asapo Version: " ${ASAPO_VERSION})
message("Python Asapo Version: " ${PYTHON_ASAPO_VERSION})
message("Asapo commit: " ${ASAPO_VERSION_COMMIT})
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ec51ea4ee739c7682f2f5c0c42255b20b5c52f1e..fcdc271fbc47de17adc767cf2939b496155a054a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,6 +3,10 @@ project(ASAPO)
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMakeModules/ ${PROJECT_SOURCE_DIR}/CMakeIncludes/)
+set (ASAPO_VERSION_IN_DOCS 21.06.0) # is overwritten in master branch
+set (ASAPO_EXAMPLES_DIR .)
+#set (ASAPO_EXAMPLES_DIR ./frozen_versions/${ASAPO_VERSION_IN_DOCS})
+
#protocol version changes if one of the microservice API's change
set (ASAPO_CONSUMER_PROTOCOL "v0.4")
set (ASAPO_PRODUCER_PROTOCOL "v0.3")
diff --git a/docs/site/CMakeLists.txt b/docs/site/CMakeLists.txt
index 76a2e398c9fdd564715a511abea8ee40e004f5cb..7f16530b4a7da2043b369fab818e4ea5f9d025b8 100644
--- a/docs/site/CMakeLists.txt
+++ b/docs/site/CMakeLists.txt
@@ -2,9 +2,18 @@ configure_files(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} true)
configure_files(${CMAKE_CURRENT_SOURCE_DIR}/blog ${CMAKE_CURRENT_BINARY_DIR}/blog)
configure_files(${CMAKE_CURRENT_SOURCE_DIR}/docs ${CMAKE_CURRENT_BINARY_DIR}/docs)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/static DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/versioned_docs DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/versioned_sidebars DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/src DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/plugins DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/examples DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
+configure_files(${CMAKE_CURRENT_SOURCE_DIR}/examples/getting_started ${CMAKE_CURRENT_BINARY_DIR}/examples/getting_started)
+
+
add_custom_target(
site ALL
COMMAND
diff --git a/docs/site/docs/compare-to-others.md b/docs/site/docs/compare-to-others.md
new file mode 100644
index 0000000000000000000000000000000000000000..d8bf7df7292e35c13cd32a13f63525890f252c1e
--- /dev/null
+++ b/docs/site/docs/compare-to-others.md
@@ -0,0 +1,49 @@
+---
+title: Comparison to Other Solutions
+---
+
+Here we consider how ASAPO is different from other workflows practiced at DESY. The possible candidates are:
+
+### Filesystem
+Probably the most often used approach for now. Files are written to the beamline filesystem directly via NFS/SMB mount or by HiDRA and copied to the core filesystem by a copy daemon. A user (software) then reads the files from the filesystem.
+
+### Filesystem + Kafka
+Previous workflow + there is a Kafka instance that produces messages when a file appears in the core filesystem. These messages can then be consumed by user software.
+
+### HiDRA
+HiDRA can work in two modes - wether data is transferred via it or data is written over NFS/SMB mounts and HiDRA monitors a folder in a beamline filesystem. In both case one can subscribe to HiDRA's data queue to be informed about a new file.
+
+### ASAPO
+
+ASAPO does not work with files, rather with data streams. Well, behind the scene it does use files, but in principle what a user see is a stream of messages, where a message typically consists of metadata and data blobs. Data blob can be a single image, a file with arbitrary content or whatever else, even null. Important is - what goes to one end, appears on the other. And that each message must have a consecutive index. These messages must be ingested to an ASAPO data stream in some way (e.g. using ASAPO Producer API or HiDRA) and data, if not yet, will be transferred and stored in the data center. A user can then read the messages from the stream and process it in a way he likes.
+
+### Compare by categories
+
+In the table below we compare the approaches from different points of view and in the [next table](#compare-by-features) we compare the approaches by available features.
+
+| Category | Filesystem | Filesystem+Kafka | HiDRA | ASAPO |
+|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Data ingest | traditional way - write to disk | same as Filesystem, additionally a message is send to a Kafka topic with basic file metadata (name, size, creation date, ...) | outperforms data ingest via NFS/SMB, uses ZeroMQ(TCP), saturates 10GE bandwidth. To be tested with 100GE. Same metadata as with Kafka | uses parallel TCP connections & in the future RDMA. Can add arbitrary metadata (JSON format). Can send data to various streams to create arbitrary data hierarchy (e.g. stream per scan). Saves data to a memory buffer and on disk. |
+| Offline analysis | traditional way - read from disk. Need to know the filename to read, usually not a problem. | same as Filesystem, there is no benefits reading Kafka queue after all data is arrived | not possible, need to fallback to Filesystem | same as online analysis (see below) |
+| Online analysis | no efficient way to recognise that new data is available - periodically read folder content and compare with previous state?, periodically check file appeared? | using a subscription to a "new files" topic, a user software can be made aware of new data quite soon and react correspondingly. | one can subscribe to a HiDRA's ZeroMQ stream and consume data. If data arrives faster than it can be processed or e.g. user software crashes - data might be skipped. | one can get data from various streams in different ways - get next unprocessed message ordered by index, get last, get by id. Since everything is stored in persistent storage, processing is possible with arbitrary slow (but also fast) consumers. Resilient to connections loss or consumer crashes. |
+| Performance | as good as read/write to disk. Can be an issue, especially with new detectors 100GE+ networks. | as good as read/write to disk + some latency for the file to be written to the beamline filesystem, copied to the core filesystem and a message to go through Kafka. | data is available as soon as it is received by HiDRA. If multiple consumer groups want to read same data (one consumer is known - the process that writes the file), data will be transferred multiple times, which influences the network performance. | data is available to be consumed as soon as it is arrived and saved to beamline filesystem (later can be optimised by using persistent memory instead). No need to read from disk since it also remains in memory buffer. |
+| Parallelisation | Parallelisation is easily possible e.g. with an MPI library. | Parallelisation is possible if Kafka's topics are partitioned (which is not the case in the moment) | Not out of the box, possible with some changes from user's side | Parallelisation is easily possible, one can consume data concurrently with different consumers from the same stream. Normally, synchronisation between consumers is not needed, but this might depend on a use case. When configured, data can be resent if not acknowledged during a specified time period. |
+| Search/filter | hardly possible, manually parsing some metadata file, using POSIX commands? | same as Filesystem. There is Kafka SQL query language which could be used if there would be metadata in messages, which is not the case (yet?). | not possible | one can use a set of SQL queries to ingested metadata. |
+| General comments | Might be ok for slow detectors or/and a use case without online analysis requirements. Might be the only way to work with legacy applications | Fits well for the cases where a software just need a trigger that some new data has arrived, when processing order, extra metadata, parallelisation is not that important or implemented by other means. Some delay between an image is generated and the event is emitted by Kafka is there, but probably not that significant (maximum a couple of seconds). Might be not that appropriate for very fast detectors since still using filesystem to write/read data. | Works quite well to transfer files from detector to the data center. Also a good candidate for live viewers, where the last available "image" should be displayed. Does not work for offline analysis or for near real-time analysis where image processing can take longer than image taking. | Tries to be a general solution which improves in areas where other approaches not suffice: single code for offline/near real-time/online analysis, parallelisation, extended metadata, efficient memory/storage management, getting data without access to filesystem (e.g. from detector pc without core filesystem mounted), computational pipelines, ... Everything has its own price: user software must be modified to use ASAPO, a wrapper might be needed for legacy software that cannot be modified, user/beamtime scientist should better structure the data - e.g. consecutive indexes must be available for each image, one has to define to which stream write/read data, what is the format of the data, ... |
+
+
+### Compare by features
+
+| Feature | Filesystem | Filesystem+Kafka | HiDRA | ASAPO |
+|--------------------------------------------------------------------------------------------------------------------|------------------------------------|------------------------------------|------------------------------------|------------------------------------|
+| send metadata with image | No | No | No | Yes |
+| get last image | No | No | Yes | Yes |
+| get image by id | No | No | No | Yes |
+| get image in order | No | No | No | Yes |
+| Immediately get informed that a new image is arrived | No | Yes | Yes | Yes |
+| access image remotely, without reading filesystem | No | No | Yes, if it is still in buffer | Yes |
+| access past images | Yes | Yes | No | Yes |
+| need to change user code | No | Yes | Yes | Yes |
+| parallelisation | Yes (if user software allows that) | Not out of the box | Not out of the box | Yes |
+| legacy applications | Yes | No (wrapper could be a workaround) | No (wrapper could be a workaround) | No (wrapper could be a workaround) |
+| transparent restart/continuation of simulations in case e.g. worker process crashes, also for parallel simulations | Not out of the box | Yes | No | Yes |
\ No newline at end of file
diff --git a/docs/site/docs/consumer-clients.md b/docs/site/docs/consumer-clients.md
new file mode 100644
index 0000000000000000000000000000000000000000..d04c14034585b2ef69f335b0104b2bfafddf69d1
--- /dev/null
+++ b/docs/site/docs/consumer-clients.md
@@ -0,0 +1,33 @@
+---
+title: Consumer Clients
+---
+
+Consumer client (or consumer) is a part of a distributed streaming system that is responsible for processing streams of data that were created by producer. It is usually a user (beamline scientist, detector developer, physicist, ... ) responsibility to develop a client for specific beamline, detector or experiment using ASAPO Consumer API and ASAPO responsibility to make sure data is delivered to consumers in an efficient and reliable way.
+
+
+
+Consumer API is available for C++ and Python and has the following main functionality:
+
+- Create a consumer instance and bind it to a specific beamtime and data source
+ - multiple instances can be created (also within a single application) to receive data from different sources
+ - a beamtime token is used for access control
+- If needed (mainly for get_next_XX operations), create a consumer group that allows to process messages independently from other groups
+- Receive messages from a specific stream (you can read more [here](data-in-asapo) about data in ASAPO)
+ - GetNext to receive process messages one after another without need to know message indexes
+ - Consumer API returns a message with index 1, then 2, ... as they were set by producer.
+ - This also works in parallel so that payload is distributed within multiple consumers within same consumer group or between threads of a single consumer instance. In parallel case order of indexes of the messages is not determined.
+ - GetLast to receive last available message - for e.g. live visualisation
+ - GetById - get message by index - provides random access
+- Make queries based on metadata contained in a message - returns all messages in a stream with specific metadata. A subset of SQL language is used
+
+
+All of the above functions can return only metadata part of the message, so that an application can e.g. extract the filename and pass it to a 3rd party tool for processing. Alternative, a function may return the complete message with metadata and data so that consumer can directly process it. An access to the filesystem where data is actually stored is not required in this case.
+
+:::note
+In case of dataset family of functions, only list of dataset messages is returned, the data can be retrieved in a separate call.
+:::
+
+Please refer to [C++](http://asapo.desy.de/cpp/) and [Python](http://asapo.desy.de/python/) documentation for specific details (available from DESY intranet only).
+
+
+
diff --git a/docs/site/docs/core-architecture.md b/docs/site/docs/core-architecture.md
new file mode 100644
index 0000000000000000000000000000000000000000..a02048e96d33c687f38622661b735974541a36e4
--- /dev/null
+++ b/docs/site/docs/core-architecture.md
@@ -0,0 +1,29 @@
+---
+title: Core Architecture
+---
+
+For those who are curious about ASAPO architecture, the diagram shows some details. Here arrows with numbers is an example of data workflow explained below.
+
+
+
+## Data Workflow (example)
+the workflow can be split into two more or less independent tasks - data ingestion and data retrieval
+
+### Data ingestion (numbers with i on the diagram)
+1i) As we [know](producer-clients.md), producer client is responsible for ingesting data in the system. Therefore the first step is to detect that the new message is available. This can be done using another tool developed at DESY named [HiDRA](https://confluence.desy.de/display/FSEC/HiDRA). This tool monitors the source of data (e.g. by monitoring a filesystem or using HTTP request or ZeroMQ streams, depending on detector type)
+
+2i) HiDRA (or other user application) then uses ASAPO Producer API to send messages (M1 and M2 in our case) in parallel to ASAPO Receiver. TCP/IP or RDMA protocols are used to send data most efficiently. ASAPO Receiver receives data in a memory cache
+
+3i) - 4i) ASAPO saves data to a filesystem and adds a metadata record to a database
+
+5i) A feedback is send to the producer client with success or error message (in case of error, some of the step above may not happen)
+
+### Data retrieval (numbers with r on the diagram)
+
+[Consumer client](consumer-clients.md)) is usually a user application that retrieves data from the system to analyse/process it.
+
+The first step to retrieve a message via Consumer API is to pass the request to the Data Broker (1r). The Data Broker retrieves the metadata information about the message from the database (2r) and returns it to the Consumer Client. The Consumer Client analyses the metadata information and decides how to get the data. It the data is still in the Receiver memory cache, the client requests data from there via a Data Server (which is a part of ASAPO Receiver). Otherwise, client gets the data from the filesystem - directly if the filesystem is accessible on the machine where the client is running or via File Transfer Service if not.
+
+
+
+
diff --git a/docs/site/docs/create-a-blog-post.md b/docs/site/docs/create-a-blog-post.md
deleted file mode 100644
index 4485a8af102e0ca5370230578b390379e8069f60..0000000000000000000000000000000000000000
--- a/docs/site/docs/create-a-blog-post.md
+++ /dev/null
@@ -1,25 +0,0 @@
----
-title: Create a Blog Post
----
-
-This page will help you on how to create blog posts in Docusaurus.
-
-## Create a Blog Post
-
-Create a file at `blog/2021-02-28-greetings.md`:
-
-```md title="blog/2021-02-28-greetings.md"
----
-title: Greetings!
-author: Steven Hansel
-author_title: Docusaurus Contributor
-author_url: https://github.com/ShinteiMai
-author_image_url: https://github.com/ShinteiMai.png
----
-
-Congratulations, you have made your first post!
-
-Feel free to play around and edit this post as much you like.
-```
-
-A new blog post is now available at `http://localhost:3000/blog/greetings`.
diff --git a/docs/site/docs/create-a-document.md b/docs/site/docs/create-a-document.md
deleted file mode 100644
index 7922129312687394beb2a1e0a4b5d7a28305f70a..0000000000000000000000000000000000000000
--- a/docs/site/docs/create-a-document.md
+++ /dev/null
@@ -1,38 +0,0 @@
----
-title: Create a Document
----
-
-Documents are pages with a **sidebar**, a **previous/next navigation** and many other useful features.
-
-## Create a Document
-
-Create a markdown file at `docs/my-doc.md`:
-
-```mdx title="docs/hello.md"
----
-title: Hello, World!
----
-
-## Hello, World!
-
-This is your first document in **Docusaurus**, Congratulations!
-```
-
-A new document is now available at `http://localhost:3000/docs/hello`.
-
-## Add your document to the sidebar
-
-Add `hello` to the `sidebars.js` file:
-
-```diff title="sidebars.js"
-module.exports = {
- docs: [
- {
- type: 'category',
- label: 'Docusaurus Tutorial',
-- items: ['getting-started', 'create-a-doc', ...],
-+ items: ['getting-started', 'create-a-doc', 'hello', ...],
- },
- ],
-};
-```
diff --git a/docs/site/docs/create-a-page.md b/docs/site/docs/create-a-page.md
deleted file mode 100644
index 1056090453a7af5c5bb96a859357aedd19d72218..0000000000000000000000000000000000000000
--- a/docs/site/docs/create-a-page.md
+++ /dev/null
@@ -1,45 +0,0 @@
----
-title: Create a Page
----
-
-Any React or Markdown file created under `src/pages` directory is converted into a website page:
-
-- `src/pages/index.js` -> `localhost:3000/`
-- `src/pages/foo.md` -> `localhost:3000/foo`
-- `src/pages/foo/bar.js` -> `localhost:3000/foo/bar`
-
-## Create a React Page
-
-Create a file at `src/pages/my-react-page.js`:
-
-```jsx title="src/pages/my-react-page.js"
-import React from 'react';
-import Layout from '@theme/Layout';
-
-function HelloWorld() {
- return (
- <Layout>
- <h1>My React page</h1>
- <p>This is a React page</p>
- </Layout>
- );
-}
-```
-
-A new page is now available at `http://localhost:3000/my-react-page`.
-
-## Create a Markdown Page
-
-Create a file at `src/pages/my-markdown-page.md`:
-
-```mdx title="src/pages/my-markdown-page.md"
----
-title: My Markdown page
----
-
-# My Markdown page
-
-This is a Markdown page
-```
-
-A new page is now available at `http://localhost:3000/my-markdown-page`.
diff --git a/docs/site/docs/data-in-asapo.md b/docs/site/docs/data-in-asapo.md
new file mode 100644
index 0000000000000000000000000000000000000000..96abf43c86c13873641ea767b7766e3145baa5d3
--- /dev/null
+++ b/docs/site/docs/data-in-asapo.md
@@ -0,0 +1,29 @@
+---
+title: Data in ASAPO
+---
+All data that is produced, stored and consumed via ASAPO is structured on several levels.
+
+#### Beamtime
+This is the top level. Contains all data collected/produced during a single beamtime (Beamtime is the term used at DESY. Can also be Run, Experiment, Proposal, ...). Each beamtime has its own unique ID.
+
+#### Data Source
+During a beamtime, data can be produced by different sources. For example, a detector is a data source, if multiple detectors are used during an experiment, they can be different data sources or the same data source (more details below in datasets section). A user application that simulates or analyses data can also act as an ASAPO data source. Each data source has its own unique name within a beamtime.
+
+#### Data Stream
+Each data source can emit multiple data streams. Each stream has a unique within a specific data source name.
+
+#### Message
+Data streams consist of smaller entities - messages. The content of a message is quite flexible, to be able to cover a broad amount of use cases. Usually it is a metadata and some binary data (e.g. a detector image, or an hdf5 file with multiple images). At the moment ASAPO itself is agnostic to the data and sees it as a binary array. Later some specific cases might be handled as well (the most prominent use case - an hdf5 file with multiple images).
+
+An important aspect is that each message within a data stream must be assigned a consecutive integer index. Therefore, a streams always contain messages with index = 1,2,3 ... . This is different to traditional messaging systems where messages have timestamps or arbitrary unique hash IDs. The reason is that with timestamps the order of messages saved in the system might differ from the order the were generated by the data source (e.g. detector). And keeping correct order is required in many cases. Second reason is that it makes a random access to a specific message quite straightforward.
+
+#### Datasets/Dataset substreams
+In some cases multiple detectors are using during an experiment. E.g. a 3D image is composed from multiple 2D images created by different detectors. In this case these 2D images can be composed to a dataset so that it a be processed later as a whole. One would then use a single data source (which would mean a set of detectors or "multi-detector" in this case), single data stream and, to compose a dataset, for each of it's components (each 2D image in our example) the corresponding detector would send a message with same id but to a different dataset substream.
+
+So, for the case without datasets (single detector) the data hierarchy is Beamtime→Data Source → Data Stream → Message:
+
+
+
+And with datasets (multi-detector) the data hierarchy is Beamtime→Data Source → Data Stream → Dataset→ Message in Dataset Substream:
+
+
diff --git a/docs/site/docs/getting-started.mdx b/docs/site/docs/getting-started.mdx
index 72eb44949edfe6301eac61060fef0d39c8df1c78..05d757e34097a6625cfcb5a2f4ac0d2c956edebd 100644
--- a/docs/site/docs/getting-started.mdx
+++ b/docs/site/docs/getting-started.mdx
@@ -14,10 +14,10 @@ If you already have running ASAPO services and know the endpoint, you can skip t
Otherwise, for testing purposes one can start ASAPO services in a standalone mode (this is not recommended for production deployment).
-The easiest way is to use a Docker container.
+The easiest way is to use a Docker container.
So, make sure Docker is installed and you have necessary permissions to use it.
Please note that this will only work on a Linux machine. Also please note that ASAPO needs some ports to be available. You can check the list
-[here](https://stash.desy.de/projects/ASAPO/repos/asapo/browse/deploy/asapo_services/scripts/asapo.auto.tfvars.in?at={ASAPO_VERSION}#37).
+[here](https://stash.desy.de/projects/ASAPO/repos/asapo/browse/deploy/asapo_services/scripts/asapo.auto.tfvars.in?at=@ASAPO_VERSION_IN_DOCS@#37).
Now, depending on how your Docker daemon is configured (if it uses a
unix socket or a tcp port for communications)
@@ -32,14 +32,14 @@ unix socket or a tcp port for communications)
}>
<TabItem value="unix">
-```shell content="getting_started/start_asapo_socket.sh"
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/start_asapo_socket.sh"
```
</TabItem>
<TabItem value="tcp">
-```shell content="getting_started/start_asapo_tcp.sh"
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/start_asapo_tcp.sh"
```
</TabItem>
@@ -64,9 +64,9 @@ mkdir -p $ASAPO_HOST_DIR/global_shared/online_data/test/current/raw
mkdir -p $ASAPO_HOST_DIR/global_shared/data/test_facility/gpfs/test/2019/data/asapo_test
```
-:::note ASAP::O in production mode
+:::note ASAP::O in production mode
-We have a running instance for processing data collected during beamtimes. Please get in touch with FS-SC group for more information.
+We have a running instance for processing data collected during experiments. Please get in touch with FS-SC group for more information.
:::
@@ -84,31 +84,161 @@ Now you can install Python packages or C++ libraries for ASAPO Producer and Cons
}>
<TabItem value="python-pip">
-```shell content="getting_started/install_python_clients_pip.sh" snippetTag="#snippet1"
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/install_python_clients_pip.sh" snippetTag="#snippet1"
```
</TabItem>
<TabItem value="python-packages">
-```shell content="getting_started/install_python_clients_pkg.sh"
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/install_python_clients_pkg.sh"
```
</TabItem>
<TabItem value="cpp">
-```shell content="getting_started/install_cpp_clients.sh"
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/install_cpp_clients.sh"
```
</TabItem>
</Tabs>
+## Step 3: Produce a message {#step-3}
-Run the development server in the newly created `my-website` folder:
+<Tabs
+ groupId="language"
+ defaultValue="python"
+ values={[
+ { label: 'Python', value: 'python', },
+ { label: 'C++', value: 'cpp', },
+ ]
+}>
+<TabItem value="python">
-Open `docs/getting-started.md` and edit some lines. The site reloads automatically and display your changes.
+Now you can write a Producer client (API documentation [here](http://asapo.desy.de/python/producer.html)).
+
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/python/produce.py"
+```
+
+Execute it with python3
+
+```
+$ python3 produce.py
+```
+
+</TabItem>
+
+<TabItem value="cpp">
+
+Now you can write a Producer client (API documentation [here](http://asapo.desy.de/cpp/producer)).
+
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/cpp/produce.cpp"
+```
+
+Compile e.g. using CMake and execute. You might need to point cmake (with CMAKE_PREFIX_PATH) to asapo installation and curl library if installed to non-standard location.
+
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/cpp/CMakeLists.txt" snippetTag="#producer"
+```
+
+```
+$ cmake . && make
+$ ./asapo-produce
+```
+
+
+</TabItem>
+</Tabs>
+
+the output should look like
+
+```
+{"time":"***** *****","source":"producer_api","level":"info","message":"authorized connection to receiver at ****:****"}
+successfuly sent: {"id": 1, "buffer": "test_file"}
+```
-## Step 3: Produce a message {#step-3}
## Step 4: Consume a message {#step-4}
+A consumer data that reads the message ingested during step 3. Note that a token is needed to work with data. In production, the token is provided during start of the beamtime.
+
+<Tabs
+ groupId="language"
+ defaultValue="python"
+ values={[
+ { label: 'Python', value: 'python', },
+ { label: 'C++', value: 'cpp', },
+ { label: 'C', value: 'c', },
+ ]
+}>
+<TabItem value="python">
+
+Complete API documentation [here](http://asapo.desy.de/python/consumer.html)
+
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/python/consume.py"
+```
+
+Execute it with python3
+
+```
+$ python3 consumer.py
+```
+
+</TabItem>
+
+<TabItem value="cpp">
+
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/cpp/consume.cpp"
+```
+
+Compile e.g. using CMake and execute. You might need to point cmake (with CMAKE_PREFIX_PATH) to asapo installation and curl library if installed to non-standard location.
+
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/cpp/CMakeLists.txt" snippetTag="#consumer"
+```
+
+```
+$ cmake . && make
+$ ./asapo-consume
+```
+
+</TabItem>
+
+<TabItem value="c">
+
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/c/consume.c"
+```
+
+Compile e.g. using Makefile and pkg-config (although we recommend CMake - see C++ section) and execute. This example assumes asapo is installed to /opt/asapo. Adjust correspondingly.
+
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/c/Makefile" snippetTag="#consumer"
+```
+
+```
+$ make
+$ ./asapo-consume
+```
+
+
+</TabItem>
+
+</Tabs>
+
+the output should look like
+
+```
+id: 1
+file name: processed/test_file
+file content: hello
+stream deleted
+```
+
## Step 5: Clean-up
+
+Optionally, last step is to stop ASAPO services and remove files:
+
+```shell content="@ASAPO_EXAMPLES_DIR@/getting_started/cleanup.sh"
+```
+
+<br/><br/>
+
+:::tip
+You can see more examples in ASAPO [source code](https://stash.desy.de/projects/ASAPO/repos/asapo/browse/examples?at=@ASAPO_VERSION_IN_DOCS@)
+:::
diff --git a/docs/site/docs/markdown-features.mdx b/docs/site/docs/markdown-features.mdx
deleted file mode 100644
index 622391df57f6d39a2e5931b42852e42cfee8c552..0000000000000000000000000000000000000000
--- a/docs/site/docs/markdown-features.mdx
+++ /dev/null
@@ -1,128 +0,0 @@
----
-title: Markdown Features
----
-
-Docusaurus supports the [Markdown](https://daringfireball.net/projects/markdown/syntax) syntax and has some additional features.
-
-## Front Matter
-
-Markdown documents can have associated metadata at the top called [Front Matter](https://jekyllrb.com/docs/front-matter/):
-
-```md
----
-id: my-doc
-title: My document title
-description: My document description
-sidebar_label: My doc
----
-
-Markdown content
-```
-
-## Markdown links
-
-Regular Markdown links are supported using url paths or relative file paths.
-
-```md
-Let's see how to [Create a page](/create-a-page).
-```
-
-```md
-Let's see how to [Create a page](./create-a-page.md).
-```
-
-Let's see how to [Create a page](./create-a-page.md).
-
-## Markdown images
-
-Regular Markdown images are supported.
-
-Add an image at `static/img/docusaurus.png` and use this Markdown declaration:
-
-```md
-
-```
-
-
-
-## Code Blocks
-
-Markdown code blocks are supported with Syntax highlighting.
-
- ```jsx title="src/components/HelloDocusaurus.js"
- function HelloDocusaurus() {
- return (
- <h1>Hello, Docusaurus!</h1>
- )
- }
- ```
-
-```jsx title="src/components/HelloDocusaurus.js"
-function HelloDocusaurus() {
- return <h1>Hello, Docusaurus!</h1>;
-}
-```
-
-## Admonitions
-
-Docusaurus has a special syntax to create admonitions and callouts:
-
- :::tip My tip
-
- Use this awesome feature option
-
- :::
-
- :::danger Take care
-
- This action is dangerous
-
- :::
-
-:::tip My tip
-
-Use this awesome feature option
-
-:::
-
-:::danger Take care
-
-This action is dangerous
-
-:::
-
-## React components
-
-Thanks to [MDX](https://mdxjs.com/), you can make your doc more interactive and use React components inside Markdown:
-
-```jsx
-export const Highlight = ({children, color}) => (
- <span
- style={{
- backgroundColor: color,
- borderRadius: '2px',
- color: 'red',
- padding: '0.2rem',
- }}>
- {children}
- </span>
-);
-
-<Highlight color="#25c2a0">Docusaurus green</Highlight> and <Highlight color="#1877F2">Facebook blue</Highlight> are my favorite colors.
-```
-
-export const Highlight = ({children, color}) => (
- <span
- style={{
- backgroundColor: color,
- borderRadius: '2px',
- color: '#fff',
- padding: '0.2rem',
- }}>
- {children}
- </span>
-);
-
-<Highlight color="#25c2a0">Docusaurus green</Highlight> and <Highlight color="#1877F2">
- Facebook blue
-</Highlight> are my favorite colors.
diff --git a/docs/site/docs/overview.md b/docs/site/docs/overview.md
new file mode 100644
index 0000000000000000000000000000000000000000..7af1f0471cfe9deb243d8d5de447c76ebcf9b30a
--- /dev/null
+++ b/docs/site/docs/overview.md
@@ -0,0 +1,40 @@
+---
+title: Overview
+---
+
+
+
+ASAP::O (or ASAPO) is a high performance distributed streaming platform. It is being developed at DESY and is mainly aimed to support online/offline analysis of experimental data produced at its facilities. The ideas behind are quite similar to that of Apache Kafka and similar messaging solutions, but ASAPO is developed and tuned for scientific use cases with their specific workflows and where the size of the messages is much large (MBs to GBs as compared to KBs in traditional systems).
+
+
+
+ASAPO has the following key capabilities:
+
+- Deliver data produced by an experimental facility (e.g. detector) to a data center in a high-performant fault-tolerant way
+- Consume this data in various modes (as soon as new data occurs, random access, latest available data, in parallel, ...)
+- Ingest own data/ create computational pipelines
+
+
+ASAPO consists of the following three components:
+
+- Core services (run in background on a single node or a cluster and provide ASAPO functionality)
+- Producer API to ingest data into the system
+- Consumer API to retrieve data from the system
+
+### Bird's eye view
+
+A workflow when using ASAPO can be represented as follows:
+
+
+
+
+Usually, an end user can see ASAPO core services as a black box. But some more details are given [here](core-architecture).
+
+Next, one can learn more about following concepts:
+
+- [Data in ASAPO](data-in-asapo)
+- [Producer clients](producer-clients)
+- [Consumer clients](consumer-clients)
+
+You can also compare with other solutions, jump directly to [Getting Started](getting-started.mdx) or have a look in use cases section
+
diff --git a/docs/site/docs/p02.1.md b/docs/site/docs/p02.1.md
new file mode 100644
index 0000000000000000000000000000000000000000..1605ec707f4e7a647ae1a2d601b5e2aa72a827d3
--- /dev/null
+++ b/docs/site/docs/p02.1.md
@@ -0,0 +1,43 @@
+---
+title: ASAP::O at P02.1
+---
+
+Online analysis at P02.1 has two main goals:
+
+- Doing as much beamline specific data analysis as possible for the user, so that they can concentrate on analyzing the experiment specific details. This will lead to a comprehensive support for the user from beamline side and therefore lead to a higher user satisfaction. Automatization of the analysis is essential to achieve the necessary high throughput, which is mandatory for current and future diffraction applications.
+- Enabling timely decisions through a "live" view of raw images and analyzed data. Problems with the measurement can often be more easily detected in the analyzed data, which should be made available to the user as early as possible to avoid wasting valuable measurement time on suboptimal experimental conditions.
+
+## Description of a typical beamtime at P02.1
+
+- A beamtime consists of a number of scans
+- Each scan consists of one or more steps
+- At each step, an image is taken by the detectors, as well as several other scalar sensors values are gathered, e.g., temperature, electric current, position, etc.
+- The parameters for the analysis are fixed during one scan but might need to change from one scan to the next
+
+## Analysis Pipeline
+
+- Images are taken by one or two detectors
+- Optionally, a number of consecutive images of a single detector are merged into one averaged image to reduce the noise
+- The (averaged) images are stored into one NeXus file per detector per scan
+- Each (averaged) image is analyzed independently
+- The analyzed data is written to one NeXus file per detector per scan
+- All scalar sensor data and additional metadata is written to one NeXus file per scan that links to the other NeXus files with the (averaged) images and analyzed data
+- A viewer displays the live and history output of all relevant processing steps
+
+
+
+## Use of ASAPO
+
+In the following, ASAPO specific details for the pipeline of a single detector are given. For multiple detectors, all stream names are suffixed by the detector ID.
+
+1. The data acquisition software stores the parameters for the analysis in a "scan-metadata" stream with one substream per scan and one metadata entry per substream
+2. Images are ingested into ASAPO
+3. The images taken by the detectors are written to the beamline filesystem by HiDRA (one file per image)
+4. HiDRA inserts the files into ASAPO. It assigns the files to the correct "detector" stream based on the file name. Each stream uses one substream per scan, its name is also extracted from the filename by HiDRA. This applies to the index within a substream as well.
+5. If enabled, one "averager" worker per detector stream reads the files from the "detector" stream and emits the averaged images into the "averaged" stream. The name of the substream of the input is used for the name of the output substream. The indices within a substream are chosen consecutively.
+6. One "nexus-writer" worker per detector reads the images either from the "detector" or the "averaged" stream. All images of a single substream are stored into one file. The filename is constructed from the name of the stream and substream the image belongs to. The index within a substream corresponds to the index within the HDF5 dataset.
+7. Multiple "asapo-dawn" worker read their parameters from the "scan-metadata" stream at the start of each substream. The images are read from the "detector" or "averaged" stream. The worker emit the resulting data into an "analyzed" stream with the same substream name as the input and the same index.
+8. One "nexus-write" worker per detector reads the analyzed data from the "analyzed" stream and writes it into one NeXus file per substream. The filename is constructed from the name of the stream and substream the data belongs to. The index within a substream corresponds to the index within the HDF5 dataset.
+9. The data acquisition software stores all scalar data and all additional scan-metadata in a master NeXus file that links to the NeXus files produced by the ASAPO workers.
+10. The viewer listens to all streams and parses the metadata to create a continuously updated tree view of all available data. Clicking on an item uses get_by_id to retrieve the actual data. A "live" mode automatically retrieves the latest data.
+
\ No newline at end of file
diff --git a/docs/site/docs/producer-clients.md b/docs/site/docs/producer-clients.md
new file mode 100644
index 0000000000000000000000000000000000000000..d74adb0b60e04ae6cf0ab5b9c2b264a25c46419a
--- /dev/null
+++ b/docs/site/docs/producer-clients.md
@@ -0,0 +1,23 @@
+---
+title: Producer Clients
+---
+
+Producer client (or producer) is a part of a distributed streaming system that is responsible for creating data streams (i.e. ingesting data in the system). It is usually a user (beamline scientist, detector developer, physicist, ... ) responsibility to develop a client for specific beamline, detector or experiment using ASAPO Producer API and ASAPO responsibility to make sure data is transferred and saved an in efficient and reliable way.
+
+
+
+Producer API is available for C++ and Python and has the following main functionality:
+
+- Create a producer instance and bind it to a specific beamtime and data source
+multiple instances can be created (also within a single application) to send data from different sources
+- Send messages to a specific stream (you can read more [here](data-in-asapo) about data in ASAPO)
+ - each message must have a consecutive integer index, ASAPO does not create indexes automatically
+ - to compose datasets, dataset substream (and dataset size) should be send along with each message
+ - messages are sent asynchronously, in parallel using multiple threads
+ - retransfer will be attempted in case of system failure
+ - a callback function can be provided to react after data was sent/process error
+
+Please refer to [C++](http://asapo.desy.de/cpp/) and [Python](http://asapo.desy.de/python/) documentation for specific details (available from DESY intranet only).
+
+
+
diff --git a/docs/site/docs/thank-you.md b/docs/site/docs/thank-you.md
deleted file mode 100644
index 808847e61fe381f150949444617abb33fe906d1f..0000000000000000000000000000000000000000
--- a/docs/site/docs/thank-you.md
+++ /dev/null
@@ -1,17 +0,0 @@
----
-title: Thank you!
----
-
-Congratulations on making it this far!
-
-You have learned the **basics of Docusaurus** and made some changes to the **initial template**.
-
-But Docusaurus has **much more to offer**!
-
-## What's next?
-
-- [Read the official documentation](https://v2.docusaurus.io/).
-- [Design and Layout your Docusaurus site](https://v2.docusaurus.io/docs/styling-layout)
-- [Integrate a search bar into your site](https://v2.docusaurus.io/docs/search)
-- [Find inspirations in Docusaurus showcase](https://v2.docusaurus.io/showcase)
-- [Get involved in the Docusaurus Community](https://v2.docusaurus.io/community/support)
diff --git a/docs/site/docusaurus.config.js b/docs/site/docusaurus.config.js
index 735c7727e2614764da8bbd594aca21e6755c3ff4..fb16fb6207c0235d2059335406743fe75410f4ed 100644
--- a/docs/site/docusaurus.config.js
+++ b/docs/site/docusaurus.config.js
@@ -14,7 +14,7 @@ module.exports = {
organizationName: 'DESY', // Usually your GitHub org/user name.
projectName: 'ASAPO', // Usually your repo name.
customFields: {
- version: '@ASAPO_VERSION@',
+ version: '@ASAPO_VERSION_IN_DOCS@',
},
plugins: [path.resolve(__dirname, 'plugins/webpackconf/src/index.js')],
themeConfig: {
@@ -47,7 +47,17 @@ module.exports = {
],
},
{
- href: 'https://stash.desy.de/projects/ASAPO/repos/asapo/browse?at=@ASAPO_VERSION@/',
+ type: 'docsVersionDropdown',
+ //// Optional
+ position: 'right',
+ // Add additional dropdown items at the beginning/end of the dropdown.
+ dropdownItemsBefore: [],
+ dropdownItemsAfter: [],
+ dropdownActiveClassDisabled: true,
+ docsPluginId: 'default',
+ },
+ {
+ href: 'https://stash.desy.de/projects/ASAPO/repos/asapo/browse?at=@ASAPO_VERSION_IN_DOCS@/',
label: 'BitBucket',
title: 'BitBucket',
position: 'right',
@@ -65,6 +75,12 @@ module.exports = {
{
docs: {
sidebarPath: require.resolve('./sidebars.js'),
+ versions: {
+ current: {
+ "label": "Develop",
+ "path": "next"
+ },
+ },
},
blog: {
showReadingTime: true,
diff --git a/docs/site/examples/.gitignore b/docs/site/examples/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..3d758890d349a004d946898e11906fe95d43b2d8
--- /dev/null
+++ b/docs/site/examples/.gitignore
@@ -0,0 +1 @@
+!Makefile
\ No newline at end of file
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/c/Makefile b/docs/site/examples/frozen_versions/21.06.0/getting_started/c/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..ba3d4a872835ae3d20e6dfe43452a6ede2771332
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/c/Makefile
@@ -0,0 +1,29 @@
+PROGRAM=asapo-consume
+
+LDFLAGS = "-Wl,-rpath,/opt/asapo/lib"
+CFLAGS += `PKG_CONFIG_PATH=/opt/asapo/lib/pkgconfig pkg-config --cflags libasapo-consumer`
+LIBS = `PKG_CONFIG_PATH=/opt/asapo/lib/pkgconfig pkg-config --libs libasapo-consumer`
+
+# for default installation
+#LDFLAGS =
+#CFLAGS += `pkg-config --cflags libasapo-consumer`
+#LIBS = `pkg-config --libs libasapo-consumer`
+
+RM=rm -f
+
+SRCS=consume.c
+OBJS=$(subst .c,.o,$(SRCS))
+
+all: $(PROGRAM)
+
+$(PROGRAM): $(OBJS)
+ $(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
+
+%.o: %.cpp
+ $(CC) $(CFLAGS) $(INCLUDE) -c -o $@ $<
+
+clean:
+ $(RM) $(OBJS)
+
+distclean: clean
+ $(RM) $(PROGRAM)
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/c/consume.c b/docs/site/examples/frozen_versions/21.06.0/getting_started/c/consume.c
new file mode 100644
index 0000000000000000000000000000000000000000..3038c22bf37ad44e84cd25a2a512f713472271cd
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/c/consume.c
@@ -0,0 +1,62 @@
+#include "asapo/consumer_c.h"
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+void exit_if_error(const char *error_string, const AsapoErrorHandle err) {
+ if (asapo_is_error(err)) {
+ char buf[1024];
+ asapo_error_explain(err, buf, sizeof(buf));
+ printf("%s %s\n", error_string, buf);
+ exit(EXIT_FAILURE);
+ }
+}
+
+int main(int argc, char* argv[]) {
+ AsapoErrorHandle err = asapo_new_handle();
+ AsapoMessageMetaHandle mm = asapo_new_handle();
+ AsapoMessageDataHandle data = asapo_new_handle();
+
+ const char *endpoint = "localhost:8400";
+ const char *beamtime = "asapo_test";
+ const char *token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjk1NzE3MTAyMTYsImp0aSI6ImMzaXFhbGpmNDNhbGZwOHJua20wIiwic3ViIjoiYnRfYXNhcG9fdGVzdCIsIkV4dHJhQ2xhaW1zIjp7IkFjY2Vzc1R5cGVzIjpbIndyaXRlIiwicmVhZCJdfX0.dkWupPO-ysI4t-jtWiaElAzDyJF6T7hu_Wz_Au54mYU";
+
+ const char * path_to_files = "/var/tmp/asapo/global_shared/data/test_facility/gpfs/test/2019/data/asapo_test"; //set it according to your configuration.
+
+ AsapoSourceCredentialsHandle cred = asapo_create_source_credentials(kProcessed,
+ beamtime,
+ "", "test_source", token);
+ AsapoConsumerHandle consumer = asapo_create_consumer(endpoint,
+ path_to_files, 1,
+ cred,
+ &err);
+ asapo_free_handle(&cred);
+
+ exit_if_error("Cannot create consumer", err);
+ asapo_consumer_set_timeout(consumer, 5000ull);
+
+ AsapoStringHandle group_id = asapo_consumer_generate_new_group_id(consumer, &err);
+ exit_if_error("Cannot create group id", err);
+
+ asapo_consumer_get_next(consumer, group_id, &mm, &data, "default",&err);
+ exit_if_error("Cannot get next record", err);
+
+ printf("id: %llu\n", (unsigned long long)asapo_message_meta_get_id(mm));
+ printf("file name: %s\n", asapo_message_meta_get_name(mm));
+ printf("file content: %s\n", asapo_message_data_get_as_chars(data));
+
+
+// delete stream
+ asapo_consumer_delete_stream(consumer,"default", 1,1,&err);
+ exit_if_error("Cannot delete stream", err);
+ printf("stream deleted\n");
+
+ asapo_free_handle(&err);
+ asapo_free_handle(&mm);
+ asapo_free_handle(&data);
+ asapo_free_handle(&consumer);
+ asapo_free_handle(&group_id);
+ return EXIT_SUCCESS;
+}
+
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/cleanup.sh b/docs/site/examples/frozen_versions/21.06.0/getting_started/cleanup.sh
new file mode 100644
index 0000000000000000000000000000000000000000..7344a690f3905218aa423a7f6feec4b7b0e0e394
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/cleanup.sh
@@ -0,0 +1,5 @@
+ASAPO_HOST_DIR=/var/tmp/asapo # you can change this if needed
+
+docker exec asapo jobs-stop
+docker stop asapo
+rm -rf $ASAPO_HOST_DIR
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/cpp/CMakeLists.txt b/docs/site/examples/frozen_versions/21.06.0/getting_started/cpp/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d7e53c78e14292c2b9dc2e5dadf91100021a969
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/cpp/CMakeLists.txt
@@ -0,0 +1,31 @@
+cmake_minimum_required(VERSION 3.3)
+
+project(asapo-client)
+
+set(CMAKE_CXX_STANDARD 11)
+
+# optionally use some other curl lib (e.g. static)
+# set (CURL_LIBRARY /usr/local/lib/libasapo-curl.a)
+# optionally linh gcc and stdc++ statically
+# set (ASAPO_STATIC_CXX_LIBS ON)
+# optionally link asapo as shared libs (ASAPO_STATIC_CXX_LIBS not used then)
+# set (ASAPO_SHARED_LIBS ON)
+
+#consumer snippet_start_remove
+find_package (Asapo REQUIRED COMPONENTS Producer)
+
+set(TARGET_NAME asapo-produce)
+set(SOURCE_FILES produce.cpp)
+
+add_executable(${TARGET_NAME} ${SOURCE_FILES})
+target_link_libraries(${TARGET_NAME} imported::asapo-producer)
+#consumer snippet_end_remove
+#producer snippet_start_remove
+find_package (Asapo REQUIRED COMPONENTS Consumer)
+
+set(TARGET_NAME asapo-consume)
+set(SOURCE_FILES consume.cpp)
+
+add_executable(${TARGET_NAME} ${SOURCE_FILES})
+target_link_libraries(${TARGET_NAME} imported::asapo-consumer)
+#producer snippet_end_remove
\ No newline at end of file
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/cpp/consume.cpp b/docs/site/examples/frozen_versions/21.06.0/getting_started/cpp/consume.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ce4d34f8a00d2686cde5fdaee8f7220b8d45d89b
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/cpp/consume.cpp
@@ -0,0 +1,43 @@
+#include "asapo/asapo_consumer.h"
+#include <iostream>
+
+
+void exit_if_error(std::string error_string, const asapo::Error& err) {
+ if (err) {
+ std::cerr << error_string << err << std::endl;
+ exit(EXIT_FAILURE);
+ }
+}
+
+int main(int argc, char* argv[]) {
+ asapo::Error err;
+
+ auto endpoint = "localhost:8400"; // // or your endpoint
+ auto beamtime = "asapo_test";
+ auto token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjk1NzE3MTAyMTYsImp0aSI6ImMzaXFhbGpmNDNhbGZwOHJua20wIiwic3ViIjoiYnRfYXNhcG9fdGVzdCIsIkV4dHJhQ2xhaW1zIjp7IkFjY2Vzc1R5cGVzIjpbIndyaXRlIiwicmVhZCJdfX0.dkWupPO-ysI4t-jtWiaElAzDyJF6T7hu_Wz_Au54mYU";
+
+ auto path_to_files = "/var/tmp/asapo/global_shared/data/test_facility/gpfs/test/2019/data/asapo_test"; //set it according to your configuration.
+
+ auto consumer = asapo::ConsumerFactory::CreateConsumer(endpoint, path_to_files, true, asapo::SourceCredentials{asapo::SourceType::kProcessed,beamtime, "", "test_source", token}, &err);
+ exit_if_error("Cannot create consumer", err);
+ consumer->SetTimeout((uint64_t) 5000);
+
+ auto group_id = consumer->GenerateNewGroupId(&err);
+ exit_if_error("Cannot create group id", err);
+
+ asapo::MessageMeta mm;
+ asapo::MessageData data;
+ err = consumer->GetNext(group_id, &mm, &data,"default");
+ exit_if_error("Cannot get next record", err);
+
+ std::cout << "id: " << mm.id << std::endl;
+ std::cout << "file name: " << mm.name << std::endl;
+ std::cout << "message content: " << reinterpret_cast<char const*>(data.get()) << std::endl;
+
+// delete stream
+ err = consumer->DeleteStream("default", asapo::DeleteStreamOptions{true, true});
+ exit_if_error("Cannot delete stream", err);
+ std::cout << "stream deleted";
+
+ return EXIT_SUCCESS;
+}
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/cpp/produce.cpp b/docs/site/examples/frozen_versions/21.06.0/getting_started/cpp/produce.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b8261ace167f28cb807ebbb4b81696cd3554cd75
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/cpp/produce.cpp
@@ -0,0 +1,45 @@
+#include "asapo/asapo_producer.h"
+
+#include <iostream>
+
+void ProcessAfterSend(asapo::RequestCallbackPayload payload, asapo::Error err) {
+ if (err) {
+ std::cerr << "error/warning during send: " << err << std::endl;
+ return;
+ } else {
+ std::cout << "successfuly send " << payload.original_header.Json() << std::endl;
+ return;
+ }
+}
+
+void exit_if_error(std::string error_string, const asapo::Error& err) {
+ if (err) {
+ std::cerr << error_string << err << std::endl;
+ exit(EXIT_FAILURE);
+ }
+}
+
+int main(int argc, char* argv[]) {
+ asapo::Error err;
+
+ auto endpoint = "localhost:8400"; // or your endpoint
+ auto beamtime = "asapo_test";
+
+ auto producer = asapo::Producer::Create(endpoint, 1,asapo::RequestHandlerType::kTcp,
+ asapo::SourceCredentials{asapo::SourceType::kProcessed,beamtime, "", "test_source", ""}, 60000, &err);
+ exit_if_error("Cannot start producer", err);
+
+ std::string to_send = "hello";
+ auto send_size = to_send.size() + 1;
+ auto buffer = asapo::MessageData(new uint8_t[send_size]);
+ memcpy(buffer.get(),to_send.c_str(),send_size);
+
+ asapo::MessageHeader message_header{1, send_size, "processed/test_file"};
+ err = producer->Send(message_header, std::move(buffer), asapo::kDefaultIngestMode, "default", &ProcessAfterSend);
+ exit_if_error("Cannot send message", err);
+
+ err = producer->WaitRequestsFinished(2000);
+ exit_if_error("Producer exit on timeout", err);
+
+ return EXIT_SUCCESS;
+}
diff --git a/examples/for_site/getting_started/install_cpp_clients.sh b/docs/site/examples/frozen_versions/21.06.0/getting_started/install_cpp_clients.sh
similarity index 73%
rename from examples/for_site/getting_started/install_cpp_clients.sh
rename to docs/site/examples/frozen_versions/21.06.0/getting_started/install_cpp_clients.sh
index 0c5ee3d9ecb7b98328dda36f4bcf44a967805569..b31d13b1d27dce5452c018fe0fa31467fb2719ac 100644
--- a/examples/for_site/getting_started/install_cpp_clients.sh
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/install_cpp_clients.sh
@@ -2,7 +2,7 @@
# you can also install Linux/Windows packages if you have root access (or install locally).
# take a look at http://nims.desy.de/extra/asapo/linux_packages/ or http://nims.desy.de/extra/asapo/windows10 for your OS. E.g. for Debian 10.7
-wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo-dev-@ASAPO_VERSION@-debian10.7.x86_64.deb
-sudo apt install ./asapo-dev-@ASAPO_VERSION@-debian10.7.x86_64.deb
+wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo-dev-21.06.0-debian10.7.x86_64.deb
+sudo apt install ./asapo-dev-21.06.0-debian10.7.x86_64.deb
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/install_python_clients_pip.sh b/docs/site/examples/frozen_versions/21.06.0/getting_started/install_python_clients_pip.sh
new file mode 100644
index 0000000000000000000000000000000000000000..00254d51310cafa44cfedd3ee94b5792521ff7a1
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/install_python_clients_pip.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+pip3 install --user --trusted-host nims.desy.de --find-links=http://nims.desy.de/extra/asapo/linux_wheels asapo_producer=21.6.0
+pip3 install --user --trusted-host nims.desy.de --find-links=http://nims.desy.de/extra/asapo/linux_wheels asapo_consumer==21.6.0
+# you might need to update pip if the above commands error: pip3 install --upgrade pip
+
+# if that does not work (abi incompatibility, etc) you may try to install source packages
+# take a look at http://nims.desy.de/extra/asapo/linux_packages/ or http://nims.desy.de/extra/asapo/windows10 for your OS. E.g. for Debian 10.7
+# wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo_producer-21.06.0.tar.gz
+# wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo_consumer-21.06.0.tar.gz
+
+# pip3 install asapo_producer-21.06.0.tar.gz
+# pip3 install asapo_consumer-21.06.0.tar.gz
diff --git a/examples/for_site/getting_started/install_python_clients_pkg.sh b/docs/site/examples/frozen_versions/21.06.0/getting_started/install_python_clients_pkg.sh
similarity index 58%
rename from examples/for_site/getting_started/install_python_clients_pkg.sh
rename to docs/site/examples/frozen_versions/21.06.0/getting_started/install_python_clients_pkg.sh
index 7abe5674efbf4740f0d203a6f04cfa660a44719e..aefe94611237b8c2c843b5772e3b15e3b96eeb61 100644
--- a/examples/for_site/getting_started/install_python_clients_pkg.sh
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/install_python_clients_pkg.sh
@@ -2,7 +2,7 @@
# you can also install Linux/Windows packages if you have root access (or install locally).
# take a look at http://nims.desy.de/extra/asapo/linux_packages/ or http://nims.desy.de/extra/asapo/windows10 for your OS. E.g. for Debian 10.7
-wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/python-asapo-producer_@ASAPO_VERSION@-debian10.7_amd64.deb
-wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/python-asapo-consumer_@ASAPO_VERSION@-debian10.7_amd64.deb
-sudo apt install ./python3-asapo-producer_@ASAPO_VERSION@-debian10.7_amd64.deb
-sudo apt install ./python3-asapo_consumer_@ASAPO_VERSION@-debian10.7_amd64.deb
\ No newline at end of file
+wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/python-asapo-producer_21.06.0-debian10.7_amd64.deb
+wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/python-asapo-consumer_21.06.0-debian10.7_amd64.deb
+sudo apt install ./python3-asapo-producer_21.06.0-debian10.7_amd64.deb
+sudo apt install ./python3-asapo_consumer_21.06.0-debian10.7_amd64.deb
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/python/consume.py b/docs/site/examples/frozen_versions/21.06.0/getting_started/python/consume.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2fae5d3498ab85cb7dec589ccbdb45778e4989d
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/python/consume.py
@@ -0,0 +1,19 @@
+import asapo_consumer
+
+endpoint = "localhost:8400"
+beamtime = "asapo_test"
+token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjk1NzE3MTAyMTYsImp0aSI6ImMzaXFhbGpmNDNhbGZwOHJua20wIiwic3ViIjoiYnRfYXNhcG9fdGVzdCIsIkV4dHJhQ2xhaW1zIjp7IkFjY2Vzc1R5cGVzIjpbIndyaXRlIiwicmVhZCJdfX0.dkWupPO-ysI4t-jtWiaElAzDyJF6T7hu_Wz_Au54mYU"
+
+path_to_files = "/var/tmp/asapo/global_shared/data/test_facility/gpfs/test/2019/data/asapo_test" #set it according to your configuration.
+consumer = asapo_consumer.create_consumer(endpoint,path_to_files,False, beamtime,"test_source",token,5000)
+group_id = consumer.generate_group_id()
+
+data, meta = consumer.get_next(group_id, meta_only = False)
+
+print ('id:',meta['_id'])
+print ('file name:',meta['name'])
+print ('file content:',data.tobytes().decode("utf-8"))
+
+#delete stream
+consumer.delete_stream(error_on_not_exist = True)
+print ('stream deleted')
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/python/produce.py b/docs/site/examples/frozen_versions/21.06.0/getting_started/python/produce.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8e9efd86dc089c6e023f201e3ac19501a20e5a4
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/python/produce.py
@@ -0,0 +1,20 @@
+import asapo_producer
+
+def callback(payload,err):
+ if err is not None:
+ print("could not sent: ",payload,err)
+ else:
+ print ("successfuly sent: ",payload)
+
+endpoint = "localhost:8400"
+beamtime = "asapo_test"
+
+# source type 'processed' to write to the core filesystem
+producer = asapo_producer.create_producer(endpoint,'processed',
+ beamtime,'auto','test_source','', 1,60000)
+
+# we are sending a message with with index 1 to the default stream. Filename must start with processed/
+producer.send(1, "processed/test_file",b"hello",
+ callback = callback)
+
+producer.wait_requests_finished(2000)
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/start_asapo_socket.sh b/docs/site/examples/frozen_versions/21.06.0/getting_started/start_asapo_socket.sh
new file mode 100644
index 0000000000000000000000000000000000000000..9d899f035c998daa6af5033c2b59fc1c66b18a84
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/start_asapo_socket.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -e
+
+ASAPO_HOST_DIR=/var/tmp/asapo # you can change this if needed, make sure there is enough space ( >3GB on disk)
+
+NOMAD_ALLOC_HOST_SHARED=$ASAPO_HOST_DIR/container_host_shared/nomad_alloc
+SERVICE_DATA_CLUSTER_SHARED=$ASAPO_HOST_DIR/asapo_cluster_shared/service_data
+DATA_GLOBAL_SHARED=$ASAPO_HOST_DIR/global_shared/data
+DATA_GLOBAL_SHARED_ONLINE=$ASAPO_HOST_DIR/global_shared/online_data
+MONGO_DIR=$SERVICE_DATA_CLUSTER_SHARED/mongodb
+
+ASAPO_USER=`id -u`:`id -g`
+
+mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED $DATA_GLOBAL_SHARED_ONLINE
+chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED $DATA_GLOBAL_SHARED_ONLINE
+
+cd $SERVICE_DATA_CLUSTER_SHARED
+mkdir -p fluentd grafana influxdb influxdb2 mongodb
+chmod 777 *
+
+docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \
+ -u $ASAPO_USER \
+ --group-add `getent group docker | cut -d: -f3` \
+ -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \
+ -v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \
+ -v $DATA_GLOBAL_SHARED:$DATA_GLOBAL_SHARED \
+ -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \
+ -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \
+ -e TF_VAR_online_dir=$DATA_GLOBAL_SHARED_ONLINE \
+ -e TF_VAR_offline_dir=$DATA_GLOBAL_SHARED \
+ -e TF_VAR_mongo_dir=$MONGO_DIR \
+ -e TF_VAR_asapo_user=$ASAPO_USER \
+ -e ACL_ENABLED=true \
+ --name asapo --net=host -d yakser/asapo-cluster:21.06.0
+
+sleep 15
+docker exec asapo jobs-start -var elk_logs=false -var influxdb_version=1.8.4
diff --git a/docs/site/examples/frozen_versions/21.06.0/getting_started/start_asapo_tcp.sh b/docs/site/examples/frozen_versions/21.06.0/getting_started/start_asapo_tcp.sh
new file mode 100644
index 0000000000000000000000000000000000000000..83439b18dc1e5b26d30fc43e4fcaaa4fd0e6d198
--- /dev/null
+++ b/docs/site/examples/frozen_versions/21.06.0/getting_started/start_asapo_tcp.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -e
+
+ASAPO_HOST_DIR=/var/tmp/asapo # you can change this if needed, make sure there is enough space ( >3GB on disk)
+# change this according to your Docker configuration
+DOCKER_ENDPOINT="127.0.0.1:2376"
+DOCKER_TLS_CA=/usr/local/docker/certs/$USER/ca.pem
+DOCKER_TLS_KEY=/usr/local/docker/certs/$USER/key.pem
+DOCKER_TLS_CERT=/usr/local/docker/certs/$USER/cert.pem
+
+
+NOMAD_ALLOC_HOST_SHARED=$ASAPO_HOST_DIR/container_host_shared/nomad_alloc
+SERVICE_DATA_CLUSTER_SHARED=$ASAPO_HOST_DIR/asapo_cluster_shared/service_data
+DATA_GLOBAL_SHARED=$ASAPO_HOST_DIR/global_shared/data
+DATA_GLOBAL_SHARED_ONLINE=$ASAPO_HOST_DIR/global_shared/online_data
+MONGO_DIR=$SERVICE_DATA_CLUSTER_SHARED/mongodb
+
+ASAPO_USER=`id -u`:`id -g`
+
+mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED $DATA_GLOBAL_SHARED_ONLINE
+chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED $DATA_GLOBAL_SHARED_ONLINE
+
+cd $SERVICE_DATA_CLUSTER_SHARED
+mkdir -p fluentd grafana influxdb2 mongodb
+chmod 777 *
+
+docker run --privileged --userns=host --security-opt no-new-privileges --rm \
+ -u $ASAPO_USER \
+ -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \
+ -v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \
+ -v $DATA_GLOBAL_SHARED:$DATA_GLOBAL_SHARED \
+ -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \
+ -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \
+ -e TF_VAR_online_dir=$DATA_GLOBAL_SHARED_ONLINE \
+ -e TF_VAR_offline_dir=$DATA_GLOBAL_SHARED \
+ -e TF_VAR_mongo_dir=$MONGO_DIR \
+ -e TF_VAR_asapo_user=$ASAPO_USER \
+ -e ACL_ENABLED=true \
+ -v $DOCKER_TLS_CA:/etc/nomad/ca.pem \
+ -v $DOCKER_TLS_KEY:/etc/nomad/key.pem \
+ -v $DOCKER_TLS_CERT:/etc/nomad/cert.pem \
+ -e DOCKER_ENDPOINT=$DOCKER_ENDPOINT \
+ --name asapo --net=host -d yakser/asapo-cluster:21.06.0
+
+sleep 15
+docker exec asapo jobs-start -var elk_logs=false
diff --git a/docs/site/examples/getting_started/c/Makefile b/docs/site/examples/getting_started/c/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..ba3d4a872835ae3d20e6dfe43452a6ede2771332
--- /dev/null
+++ b/docs/site/examples/getting_started/c/Makefile
@@ -0,0 +1,29 @@
+PROGRAM=asapo-consume
+
+LDFLAGS = "-Wl,-rpath,/opt/asapo/lib"
+CFLAGS += `PKG_CONFIG_PATH=/opt/asapo/lib/pkgconfig pkg-config --cflags libasapo-consumer`
+LIBS = `PKG_CONFIG_PATH=/opt/asapo/lib/pkgconfig pkg-config --libs libasapo-consumer`
+
+# for default installation
+#LDFLAGS =
+#CFLAGS += `pkg-config --cflags libasapo-consumer`
+#LIBS = `pkg-config --libs libasapo-consumer`
+
+RM=rm -f
+
+SRCS=consume.c
+OBJS=$(subst .c,.o,$(SRCS))
+
+all: $(PROGRAM)
+
+$(PROGRAM): $(OBJS)
+ $(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
+
+%.o: %.cpp
+ $(CC) $(CFLAGS) $(INCLUDE) -c -o $@ $<
+
+clean:
+ $(RM) $(OBJS)
+
+distclean: clean
+ $(RM) $(PROGRAM)
diff --git a/docs/site/examples/getting_started/c/consume.c b/docs/site/examples/getting_started/c/consume.c
new file mode 100644
index 0000000000000000000000000000000000000000..3038c22bf37ad44e84cd25a2a512f713472271cd
--- /dev/null
+++ b/docs/site/examples/getting_started/c/consume.c
@@ -0,0 +1,62 @@
+#include "asapo/consumer_c.h"
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+void exit_if_error(const char *error_string, const AsapoErrorHandle err) {
+ if (asapo_is_error(err)) {
+ char buf[1024];
+ asapo_error_explain(err, buf, sizeof(buf));
+ printf("%s %s\n", error_string, buf);
+ exit(EXIT_FAILURE);
+ }
+}
+
+int main(int argc, char* argv[]) {
+ AsapoErrorHandle err = asapo_new_handle();
+ AsapoMessageMetaHandle mm = asapo_new_handle();
+ AsapoMessageDataHandle data = asapo_new_handle();
+
+ const char *endpoint = "localhost:8400";
+ const char *beamtime = "asapo_test";
+ const char *token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjk1NzE3MTAyMTYsImp0aSI6ImMzaXFhbGpmNDNhbGZwOHJua20wIiwic3ViIjoiYnRfYXNhcG9fdGVzdCIsIkV4dHJhQ2xhaW1zIjp7IkFjY2Vzc1R5cGVzIjpbIndyaXRlIiwicmVhZCJdfX0.dkWupPO-ysI4t-jtWiaElAzDyJF6T7hu_Wz_Au54mYU";
+
+ const char * path_to_files = "/var/tmp/asapo/global_shared/data/test_facility/gpfs/test/2019/data/asapo_test"; //set it according to your configuration.
+
+ AsapoSourceCredentialsHandle cred = asapo_create_source_credentials(kProcessed,
+ beamtime,
+ "", "test_source", token);
+ AsapoConsumerHandle consumer = asapo_create_consumer(endpoint,
+ path_to_files, 1,
+ cred,
+ &err);
+ asapo_free_handle(&cred);
+
+ exit_if_error("Cannot create consumer", err);
+ asapo_consumer_set_timeout(consumer, 5000ull);
+
+ AsapoStringHandle group_id = asapo_consumer_generate_new_group_id(consumer, &err);
+ exit_if_error("Cannot create group id", err);
+
+ asapo_consumer_get_next(consumer, group_id, &mm, &data, "default",&err);
+ exit_if_error("Cannot get next record", err);
+
+ printf("id: %llu\n", (unsigned long long)asapo_message_meta_get_id(mm));
+ printf("file name: %s\n", asapo_message_meta_get_name(mm));
+ printf("file content: %s\n", asapo_message_data_get_as_chars(data));
+
+
+// delete stream
+ asapo_consumer_delete_stream(consumer,"default", 1,1,&err);
+ exit_if_error("Cannot delete stream", err);
+ printf("stream deleted\n");
+
+ asapo_free_handle(&err);
+ asapo_free_handle(&mm);
+ asapo_free_handle(&data);
+ asapo_free_handle(&consumer);
+ asapo_free_handle(&group_id);
+ return EXIT_SUCCESS;
+}
+
diff --git a/docs/site/examples/getting_started/cleanup.sh b/docs/site/examples/getting_started/cleanup.sh
new file mode 100644
index 0000000000000000000000000000000000000000..7344a690f3905218aa423a7f6feec4b7b0e0e394
--- /dev/null
+++ b/docs/site/examples/getting_started/cleanup.sh
@@ -0,0 +1,5 @@
+ASAPO_HOST_DIR=/var/tmp/asapo # you can change this if needed
+
+docker exec asapo jobs-stop
+docker stop asapo
+rm -rf $ASAPO_HOST_DIR
diff --git a/docs/site/examples/getting_started/cpp/CMakeLists.txt b/docs/site/examples/getting_started/cpp/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d7e53c78e14292c2b9dc2e5dadf91100021a969
--- /dev/null
+++ b/docs/site/examples/getting_started/cpp/CMakeLists.txt
@@ -0,0 +1,31 @@
+cmake_minimum_required(VERSION 3.3)
+
+project(asapo-client)
+
+set(CMAKE_CXX_STANDARD 11)
+
+# optionally use some other curl lib (e.g. static)
+# set (CURL_LIBRARY /usr/local/lib/libasapo-curl.a)
+# optionally linh gcc and stdc++ statically
+# set (ASAPO_STATIC_CXX_LIBS ON)
+# optionally link asapo as shared libs (ASAPO_STATIC_CXX_LIBS not used then)
+# set (ASAPO_SHARED_LIBS ON)
+
+#consumer snippet_start_remove
+find_package (Asapo REQUIRED COMPONENTS Producer)
+
+set(TARGET_NAME asapo-produce)
+set(SOURCE_FILES produce.cpp)
+
+add_executable(${TARGET_NAME} ${SOURCE_FILES})
+target_link_libraries(${TARGET_NAME} imported::asapo-producer)
+#consumer snippet_end_remove
+#producer snippet_start_remove
+find_package (Asapo REQUIRED COMPONENTS Consumer)
+
+set(TARGET_NAME asapo-consume)
+set(SOURCE_FILES consume.cpp)
+
+add_executable(${TARGET_NAME} ${SOURCE_FILES})
+target_link_libraries(${TARGET_NAME} imported::asapo-consumer)
+#producer snippet_end_remove
\ No newline at end of file
diff --git a/docs/site/examples/getting_started/cpp/consume.cpp b/docs/site/examples/getting_started/cpp/consume.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ce4d34f8a00d2686cde5fdaee8f7220b8d45d89b
--- /dev/null
+++ b/docs/site/examples/getting_started/cpp/consume.cpp
@@ -0,0 +1,43 @@
+#include "asapo/asapo_consumer.h"
+#include <iostream>
+
+
+void exit_if_error(std::string error_string, const asapo::Error& err) {
+ if (err) {
+ std::cerr << error_string << err << std::endl;
+ exit(EXIT_FAILURE);
+ }
+}
+
+int main(int argc, char* argv[]) {
+ asapo::Error err;
+
+ auto endpoint = "localhost:8400"; // // or your endpoint
+ auto beamtime = "asapo_test";
+ auto token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjk1NzE3MTAyMTYsImp0aSI6ImMzaXFhbGpmNDNhbGZwOHJua20wIiwic3ViIjoiYnRfYXNhcG9fdGVzdCIsIkV4dHJhQ2xhaW1zIjp7IkFjY2Vzc1R5cGVzIjpbIndyaXRlIiwicmVhZCJdfX0.dkWupPO-ysI4t-jtWiaElAzDyJF6T7hu_Wz_Au54mYU";
+
+ auto path_to_files = "/var/tmp/asapo/global_shared/data/test_facility/gpfs/test/2019/data/asapo_test"; //set it according to your configuration.
+
+ auto consumer = asapo::ConsumerFactory::CreateConsumer(endpoint, path_to_files, true, asapo::SourceCredentials{asapo::SourceType::kProcessed,beamtime, "", "test_source", token}, &err);
+ exit_if_error("Cannot create consumer", err);
+ consumer->SetTimeout((uint64_t) 5000);
+
+ auto group_id = consumer->GenerateNewGroupId(&err);
+ exit_if_error("Cannot create group id", err);
+
+ asapo::MessageMeta mm;
+ asapo::MessageData data;
+ err = consumer->GetNext(group_id, &mm, &data,"default");
+ exit_if_error("Cannot get next record", err);
+
+ std::cout << "id: " << mm.id << std::endl;
+ std::cout << "file name: " << mm.name << std::endl;
+ std::cout << "message content: " << reinterpret_cast<char const*>(data.get()) << std::endl;
+
+// delete stream
+ err = consumer->DeleteStream("default", asapo::DeleteStreamOptions{true, true});
+ exit_if_error("Cannot delete stream", err);
+ std::cout << "stream deleted";
+
+ return EXIT_SUCCESS;
+}
diff --git a/docs/site/examples/getting_started/cpp/produce.cpp b/docs/site/examples/getting_started/cpp/produce.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b8261ace167f28cb807ebbb4b81696cd3554cd75
--- /dev/null
+++ b/docs/site/examples/getting_started/cpp/produce.cpp
@@ -0,0 +1,45 @@
+#include "asapo/asapo_producer.h"
+
+#include <iostream>
+
+void ProcessAfterSend(asapo::RequestCallbackPayload payload, asapo::Error err) {
+ if (err) {
+ std::cerr << "error/warning during send: " << err << std::endl;
+ return;
+ } else {
+ std::cout << "successfuly send " << payload.original_header.Json() << std::endl;
+ return;
+ }
+}
+
+void exit_if_error(std::string error_string, const asapo::Error& err) {
+ if (err) {
+ std::cerr << error_string << err << std::endl;
+ exit(EXIT_FAILURE);
+ }
+}
+
+int main(int argc, char* argv[]) {
+ asapo::Error err;
+
+ auto endpoint = "localhost:8400"; // or your endpoint
+ auto beamtime = "asapo_test";
+
+ auto producer = asapo::Producer::Create(endpoint, 1,asapo::RequestHandlerType::kTcp,
+ asapo::SourceCredentials{asapo::SourceType::kProcessed,beamtime, "", "test_source", ""}, 60000, &err);
+ exit_if_error("Cannot start producer", err);
+
+ std::string to_send = "hello";
+ auto send_size = to_send.size() + 1;
+ auto buffer = asapo::MessageData(new uint8_t[send_size]);
+ memcpy(buffer.get(),to_send.c_str(),send_size);
+
+ asapo::MessageHeader message_header{1, send_size, "processed/test_file"};
+ err = producer->Send(message_header, std::move(buffer), asapo::kDefaultIngestMode, "default", &ProcessAfterSend);
+ exit_if_error("Cannot send message", err);
+
+ err = producer->WaitRequestsFinished(2000);
+ exit_if_error("Producer exit on timeout", err);
+
+ return EXIT_SUCCESS;
+}
diff --git a/docs/site/examples/getting_started/install_cpp_clients.sh b/docs/site/examples/getting_started/install_cpp_clients.sh
new file mode 100644
index 0000000000000000000000000000000000000000..db4190c738615551228e7968950f3286cb1c1a6f
--- /dev/null
+++ b/docs/site/examples/getting_started/install_cpp_clients.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# you can also install Linux/Windows packages if you have root access (or install locally).
+# take a look at http://nims.desy.de/extra/asapo/linux_packages/ or http://nims.desy.de/extra/asapo/windows10 for your OS. E.g. for Debian 10.7
+wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo-dev-@ASAPO_VERSION_IN_DOCS@-debian10.7.x86_64.deb
+sudo apt install ./asapo-dev-@ASAPO_VERSION_IN_DOCS@-debian10.7.x86_64.deb
+
+
diff --git a/examples/for_site/getting_started/install_python_clients_pip.sh b/docs/site/examples/getting_started/install_python_clients_pip.sh
similarity index 73%
rename from examples/for_site/getting_started/install_python_clients_pip.sh
rename to docs/site/examples/getting_started/install_python_clients_pip.sh
index ea6c27511d108c5bed4b434b5de1b3c8a4fe04f8..338bd3a27a500fad8c535b25061979124007c94e 100644
--- a/examples/for_site/getting_started/install_python_clients_pip.sh
+++ b/docs/site/examples/getting_started/install_python_clients_pip.sh
@@ -1,13 +1,13 @@
#!/usr/bin/env bash
-pip3 install --user --trusted-host nims.desy.de --find-links=http://nims.desy.de/extra/asapo/linux_wheels asapo_producer=@ASAPO_WHEEL_VERSION@
-pip3 install --user --trusted-host nims.desy.de --find-links=http://nims.desy.de/extra/asapo/linux_wheels asapo_consumer==@ASAPO_WHEEL_VERSION@
+pip3 install --user --trusted-host nims.desy.de --find-links=http://nims.desy.de/extra/asapo/linux_wheels asapo_producer=@ASAPO_WHEEL_VERSION_IN_DOCS@
+pip3 install --user --trusted-host nims.desy.de --find-links=http://nims.desy.de/extra/asapo/linux_wheels asapo_consumer==@ASAPO_WHEEL_VERSION_IN_DOCS@
# you might need to update pip if the above commands error: pip3 install --upgrade pip
# if that does not work (abi incompatibility, etc) you may try to install source packages
# take a look at http://nims.desy.de/extra/asapo/linux_packages/ or http://nims.desy.de/extra/asapo/windows10 for your OS. E.g. for Debian 10.7
-# wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo_producer-@ASAPO_VERSION@.tar.gz
-# wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo_consumer-@ASAPO_VERSION@.tar.gz
+# wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo_producer-@ASAPO_VERSION_IN_DOCS@.tar.gz
+# wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/asapo_consumer-@ASAPO_VERSION_IN_DOCS@.tar.gz
-# pip3 install asapo_producer-@ASAPO_VERSION@.tar.gz
-# pip3 install asapo_consumer-@ASAPO_VERSION@.tar.gz
\ No newline at end of file
+# pip3 install asapo_producer-@ASAPO_VERSION_IN_DOCS@.tar.gz
+# pip3 install asapo_consumer-@ASAPO_VERSION_IN_DOCS@.tar.gz
\ No newline at end of file
diff --git a/docs/site/examples/getting_started/install_python_clients_pkg.sh b/docs/site/examples/getting_started/install_python_clients_pkg.sh
new file mode 100644
index 0000000000000000000000000000000000000000..5c13804b7381097511a0778068c098f9f8c0eec5
--- /dev/null
+++ b/docs/site/examples/getting_started/install_python_clients_pkg.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# you can also install Linux/Windows packages if you have root access (or install locally).
+# take a look at http://nims.desy.de/extra/asapo/linux_packages/ or http://nims.desy.de/extra/asapo/windows10 for your OS. E.g. for Debian 10.7
+wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/python-asapo-producer_@ASAPO_VERSION_IN_DOCS@-debian10.7_amd64.deb
+wget http://nims.desy.de/extra/asapo/linux_packages/debian10.7/python-asapo-consumer_@ASAPO_VERSION_IN_DOCS@-debian10.7_amd64.deb
+sudo apt install ./python3-asapo-producer_@ASAPO_VERSION_IN_DOCS@-debian10.7_amd64.deb
+sudo apt install ./python3-asapo_consumer_@ASAPO_VERSION_IN_DOCS@-debian10.7_amd64.deb
\ No newline at end of file
diff --git a/docs/site/examples/getting_started/python/consume.py b/docs/site/examples/getting_started/python/consume.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2fae5d3498ab85cb7dec589ccbdb45778e4989d
--- /dev/null
+++ b/docs/site/examples/getting_started/python/consume.py
@@ -0,0 +1,19 @@
+import asapo_consumer
+
+endpoint = "localhost:8400"
+beamtime = "asapo_test"
+token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjk1NzE3MTAyMTYsImp0aSI6ImMzaXFhbGpmNDNhbGZwOHJua20wIiwic3ViIjoiYnRfYXNhcG9fdGVzdCIsIkV4dHJhQ2xhaW1zIjp7IkFjY2Vzc1R5cGVzIjpbIndyaXRlIiwicmVhZCJdfX0.dkWupPO-ysI4t-jtWiaElAzDyJF6T7hu_Wz_Au54mYU"
+
+path_to_files = "/var/tmp/asapo/global_shared/data/test_facility/gpfs/test/2019/data/asapo_test" #set it according to your configuration.
+consumer = asapo_consumer.create_consumer(endpoint,path_to_files,False, beamtime,"test_source",token,5000)
+group_id = consumer.generate_group_id()
+
+data, meta = consumer.get_next(group_id, meta_only = False)
+
+print ('id:',meta['_id'])
+print ('file name:',meta['name'])
+print ('file content:',data.tobytes().decode("utf-8"))
+
+#delete stream
+consumer.delete_stream(error_on_not_exist = True)
+print ('stream deleted')
diff --git a/docs/site/examples/getting_started/python/produce.py b/docs/site/examples/getting_started/python/produce.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8e9efd86dc089c6e023f201e3ac19501a20e5a4
--- /dev/null
+++ b/docs/site/examples/getting_started/python/produce.py
@@ -0,0 +1,20 @@
+import asapo_producer
+
+def callback(payload,err):
+ if err is not None:
+ print("could not sent: ",payload,err)
+ else:
+ print ("successfuly sent: ",payload)
+
+endpoint = "localhost:8400"
+beamtime = "asapo_test"
+
+# source type 'processed' to write to the core filesystem
+producer = asapo_producer.create_producer(endpoint,'processed',
+ beamtime,'auto','test_source','', 1,60000)
+
+# we are sending a message with with index 1 to the default stream. Filename must start with processed/
+producer.send(1, "processed/test_file",b"hello",
+ callback = callback)
+
+producer.wait_requests_finished(2000)
diff --git a/examples/for_site/getting_started/start_asapo_socket.sh b/docs/site/examples/getting_started/start_asapo_socket.sh
similarity index 99%
rename from examples/for_site/getting_started/start_asapo_socket.sh
rename to docs/site/examples/getting_started/start_asapo_socket.sh
index 909b4cec88f0c2a151d7584bf624ec8368d8b0b1..d8f14743f8b732213ce6973b3348cc3e10d3d88e 100644
--- a/examples/for_site/getting_started/start_asapo_socket.sh
+++ b/docs/site/examples/getting_started/start_asapo_socket.sh
@@ -32,7 +32,7 @@ docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \
-e TF_VAR_mongo_dir=$MONGO_DIR \
-e TF_VAR_asapo_user=$ASAPO_USER \
-e ACL_ENABLED=true \
- --name asapo --net=host -d yakser/asapo-cluster:@ASAPO_VERSION@
+ --name asapo --net=host -d yakser/asapo-cluster:@ASAPO_VERSION_IN_DOCS@
sleep 15
docker exec asapo jobs-start -var elk_logs=false -var influxdb_version=1.8.4
\ No newline at end of file
diff --git a/examples/for_site/getting_started/start_asapo_tcp.sh b/docs/site/examples/getting_started/start_asapo_tcp.sh
similarity index 99%
rename from examples/for_site/getting_started/start_asapo_tcp.sh
rename to docs/site/examples/getting_started/start_asapo_tcp.sh
index 3bd68d1bb402b5121304d0591037bf3aae1b1db8..5dd30bd2512f2d518dab08b260414154a808ed94 100644
--- a/examples/for_site/getting_started/start_asapo_tcp.sh
+++ b/docs/site/examples/getting_started/start_asapo_tcp.sh
@@ -41,7 +41,7 @@ docker run --privileged --userns=host --security-opt no-new-privileges --rm \
-v $DOCKER_TLS_KEY:/etc/nomad/key.pem \
-v $DOCKER_TLS_CERT:/etc/nomad/cert.pem \
-e DOCKER_ENDPOINT=$DOCKER_ENDPOINT \
- --name asapo --net=host -d yakser/asapo-cluster:@ASAPO_VERSION@
+ --name asapo --net=host -d yakser/asapo-cluster:@ASAPO_VERSION_IN_DOCS@
sleep 15
docker exec asapo jobs-start -var elk_logs=false
diff --git a/docs/site/plugins/webpackconf/src/index.js b/docs/site/plugins/webpackconf/src/index.js
index dfff0071b586fa02942dca6e1b1effe06f972135..327e359755de81be15707d039aa9532ccbf0cfac 100644
--- a/docs/site/plugins/webpackconf/src/index.js
+++ b/docs/site/plugins/webpackconf/src/index.js
@@ -14,6 +14,18 @@ module.exports = function (context, options) {
test: /\.cpp$/i,
use: 'raw-loader',
},
+ {
+ test: /\.c$/i,
+ use: 'raw-loader',
+ },
+ {
+ test: /Makefile/i,
+ use: 'raw-loader',
+ },
+ {
+ test: /\.txt$/i,
+ use: 'raw-loader',
+ },
{
test: /\.py$/i,
use: 'raw-loader',
diff --git a/docs/site/sidebars.js b/docs/site/sidebars.js
index a40242a6de8d39d5e8965299a75e91dfcbedd5dc..a1ff08e60ca70bd76e6fa83d3cfd9fba75859783 100644
--- a/docs/site/sidebars.js
+++ b/docs/site/sidebars.js
@@ -1,15 +1,23 @@
module.exports = {
docs: [
'getting-started',
+ 'overview',
+ 'compare-to-others',
{
type: 'category',
- label: 'Docusaurus Tutorial',
+ label: 'Concepts And Architecture',
items: [
- 'create-a-page',
- 'create-a-document',
- 'create-a-blog-post',
- 'markdown-features',
- 'thank-you',
+ 'data-in-asapo',
+ 'producer-clients',
+ 'consumer-clients',
+ 'core-architecture',
+ ],
+ },
+ {
+ type: 'category',
+ label: 'Use Cases',
+ items: [
+ 'p02.1',
],
},
],
diff --git a/docs/site/src/pages/index.js b/docs/site/src/pages/index.js
index c8219405f9ee058c18cae8167ccdd018a5fe15f7..49f49adaa4bbc209dc0197318e6c1d325afb73d2 100644
--- a/docs/site/src/pages/index.js
+++ b/docs/site/src/pages/index.js
@@ -8,32 +8,29 @@ import styles from './styles.module.css';
const features = [
{
- title: 'Easy to Use',
- imageUrl: 'img/undraw_docusaurus_mountain.svg',
+ title: 'Designed to be Fast',
+ imageUrl: 'img/high-performance.png',
description: (
<>
- Docusaurus was designed from the ground up to be easily installed and
- used to get your website up and running quickly.
+ ASAP::O was designed to be able to keep up with huge data volumes and frame rates of next generation high-speed detectors.
</>
),
},
{
title: 'Focus on What Matters',
- imageUrl: 'img/undraw_docusaurus_tree.svg',
+ imageUrl: 'img/science.png',
description: (
<>
- Docusaurus lets you focus on your docs, and we'll do the chores. Go
- ahead and move your docs into the <code>docs</code> directory.
+ ASAP::O lets you focus on science, while we'll take care of nasty details like storage and network and deliver your data right where you need it.
</>
),
},
{
- title: 'Powered by React',
- imageUrl: 'img/undraw_docusaurus_react.svg',
+ title: 'Easy to Use',
+ imageUrl: 'img/user-friendly.png',
description: (
<>
- Extend or customize your website layout by reusing React. Docusaurus can
- be extended while reusing the same header and footer.
+ ASAP::O API is avaiable in Python or C/C++ and is quite simple. Just couple lines of code and you can start using your data.
</>
),
},
diff --git a/docs/site/src/pages/styles.module.css b/docs/site/src/pages/styles.module.css
index c1aa85121c9142f2b50045ff499e97e01c661dd3..da015c6d0bcc6149affcddb54779eccb3ef09eb5 100644
--- a/docs/site/src/pages/styles.module.css
+++ b/docs/site/src/pages/styles.module.css
@@ -31,7 +31,10 @@
width: 100%;
}
+
+
.featureImage {
- height: 200px;
- width: 200px;
+ height: 150px;
+ wmargin-bottom: 1rem;
}
+
diff --git a/docs/site/src/theme/CodeBlock.tsx b/docs/site/src/theme/CodeBlock.tsx
index 24cc6b1859a626b77fca2f9944fd75887a5cf1a8..d806f1954181aac5af3f7abc6ab0d43b0ab30658 100644
--- a/docs/site/src/theme/CodeBlock.tsx
+++ b/docs/site/src/theme/CodeBlock.tsx
@@ -2,7 +2,9 @@ import React from 'react'
import InitCodeBlock from '@theme-init/CodeBlock'
import useDocusaurusContext from '@docusaurus/useDocusaurusContext';
-const requireContext = require.context('../../../../examples/for_site/', true, /\.(sh|py|cpp)$/);
+
+
+const requireContext = require.context('../../examples/', true, /(\.sh|\.py|\.cpp|\.c|\.txt|Makefile)$/);
const noteStyle: React.CSSProperties = {
textAlign: 'right',
@@ -14,48 +16,6 @@ export interface State {
isCancelled: boolean
}
-async function fetchCode(url: string, snippetTag: string, state: State, setFetchResultState: React.Dispatch<React.SetStateAction<string>>) {
- let res: Response
- try {
- if (!state.isCancelled) {
- res = await fetch(url);
- }
- } catch (err) {
- if (!state.isCancelled) {
- setFetchResultState("cannot fetch code: " + err.toString());
- }
- }
-
- if (state.isCancelled) {
- return;
- }
-
- if (res.status !== 200) {
- const error = await res.text()
- setFetchResultState("cannot fetch code: " + error);
- }
-
- let body = (await res.text()).split('\n')
- const fromLine = body.indexOf(snippetTag + " start") + 1 || 0;
- const toLine = body.indexOf(snippetTag + " end", fromLine) - 1 || undefined;
- body = body.slice(fromLine, (toLine || fromLine) + 1)
-
- const preceedingSpace = body.reduce((prev: number, line: string) => {
- if (line.length === 0) {
- return prev
- }
-
- const spaces = line.match(/^\s+/)
- if (spaces) {
- return Math.min(prev, spaces[0].length)
- }
-
- return 0
- }, Infinity)
-
- setFetchResultState(body.map((line) => line.slice(preceedingSpace)).join('\n'));
-}
-
function getVal(name: string, props: any) {
const codeRegex = new RegExp("(?:" + name + "=\")(.*?)(\")")
@@ -76,15 +36,27 @@ function ReferenceCode(props: any) {
}
const {siteConfig} = useDocusaurusContext();
const version = siteConfig.customFields.version;
+ console.log(siteConfig);
const urlLink = "https://stash.desy.de/projects/ASAPO/repos/asapo/browse/examples/for_site/" + codeBlockContent + "?at=" + version
const snippetTag = getVal("snippetTag", props)
if (codeBlockContent) {
- const res = requireContext('./'+codeBlockContent)
+ const c = codeBlockContent.replace('@ASAPO_EXAMPLES_DIR@', '.')
+ const res = requireContext(c)
let body = res.default.split('\n')
- const fromLine = body.indexOf(snippetTag + " start") + 1 || 0;
- const toLine = body.indexOf(snippetTag + " end", fromLine) - 1 || undefined;
- body = body.slice(fromLine, (toLine || fromLine) + 1).join('\n')
+ const fromLine = body.indexOf(snippetTag + " snippet_start") + 1;
+ const toLine = body.indexOf(snippetTag + " snippet_end", fromLine) - 1;
+ if (fromLine > 0) {
+ body = body.slice(fromLine, (toLine>-1?toLine:fromLine) + 1)
+ }
+ const fromLineRemove = body.indexOf(snippetTag + " snippet_start_remove");
+ const toLineRemove = body.indexOf(snippetTag + " snippet_end_remove", fromLineRemove);
+ if (fromLineRemove>-1) {
+ body.splice(fromLineRemove, toLineRemove>-1?toLineRemove-fromLineRemove + 1:2)
+ }
+ body = body.filter(a => !a.includes("snippet_start_remove") && !a.includes("snippet_end_remove"))
+ body = body.join('\n')
+
const customProps = {
...props,
diff --git a/docs/site/static/img/Asapo_Analysis_Pipeline_P02-1.png b/docs/site/static/img/Asapo_Analysis_Pipeline_P02-1.png
new file mode 100644
index 0000000000000000000000000000000000000000..a6192d8133fe26ca16f936be448e8ccfb315dd9c
Binary files /dev/null and b/docs/site/static/img/Asapo_Analysis_Pipeline_P02-1.png differ
diff --git a/docs/site/static/img/asapo_bird_eye.png b/docs/site/static/img/asapo_bird_eye.png
new file mode 100644
index 0000000000000000000000000000000000000000..2ce8907e94cdb20361b99292cad48d4432df7809
Binary files /dev/null and b/docs/site/static/img/asapo_bird_eye.png differ
diff --git a/docs/site/static/img/consumer-clients.png b/docs/site/static/img/consumer-clients.png
new file mode 100644
index 0000000000000000000000000000000000000000..80230f69fc20fee181a7215f1c73208214ae596a
Binary files /dev/null and b/docs/site/static/img/consumer-clients.png differ
diff --git a/docs/site/static/img/core-architecture.png b/docs/site/static/img/core-architecture.png
new file mode 100644
index 0000000000000000000000000000000000000000..67efd4644ccd338032d264886df5f40e45be2ca0
Binary files /dev/null and b/docs/site/static/img/core-architecture.png differ
diff --git a/docs/site/static/img/data-in-asapo-workflow.png b/docs/site/static/img/data-in-asapo-workflow.png
new file mode 100644
index 0000000000000000000000000000000000000000..cdc18928cfc657c16d7d527979b6965277b502ef
Binary files /dev/null and b/docs/site/static/img/data-in-asapo-workflow.png differ
diff --git a/docs/site/static/img/data-in-asapo-workflow2.png b/docs/site/static/img/data-in-asapo-workflow2.png
new file mode 100644
index 0000000000000000000000000000000000000000..3189508803408485e3b9aa322ddda317c8e03a69
Binary files /dev/null and b/docs/site/static/img/data-in-asapo-workflow2.png differ
diff --git a/docs/site/static/img/high-performance.png b/docs/site/static/img/high-performance.png
new file mode 100644
index 0000000000000000000000000000000000000000..56b5e7784d003aa7c6ccda0fa301845f3ae28a75
Binary files /dev/null and b/docs/site/static/img/high-performance.png differ
diff --git a/docs/site/static/img/producer-clients.png b/docs/site/static/img/producer-clients.png
new file mode 100644
index 0000000000000000000000000000000000000000..afe3142fcdc79cb2e914dad130dfaae91ea7a32e
Binary files /dev/null and b/docs/site/static/img/producer-clients.png differ
diff --git a/docs/site/static/img/science.png b/docs/site/static/img/science.png
new file mode 100644
index 0000000000000000000000000000000000000000..d7939086cdcdc208b5430a111893a705cf2b3102
Binary files /dev/null and b/docs/site/static/img/science.png differ
diff --git a/docs/site/static/img/user-friendly.png b/docs/site/static/img/user-friendly.png
new file mode 100644
index 0000000000000000000000000000000000000000..eb6a179349cdf4bff52b963fcacf4f60c857345f
Binary files /dev/null and b/docs/site/static/img/user-friendly.png differ
diff --git a/docs/site/versioned_docs/version-21.06.0/compare-to-others.md b/docs/site/versioned_docs/version-21.06.0/compare-to-others.md
new file mode 100644
index 0000000000000000000000000000000000000000..97e0bd7fc30ae8166e3493742164a3bb6044fc31
--- /dev/null
+++ b/docs/site/versioned_docs/version-21.06.0/compare-to-others.md
@@ -0,0 +1,49 @@
+---
+title: Comparison to Other Solutions
+---
+
+Here we consider how ASAPO is different from other workflows practiced at DESY. The possible candidates are:
+
+### Filesystem
+Probably the most often used approach for now. Files are written to the beamline filesystem directly via NFS/SMB mount or by HiDRA and copied to the core filesystem by a copy daemon. A user (software) then reads the files from the filesystem.
+
+### Filesystem + Kafka
+Previous workflow + there is a Kafka instance that produces messages when a file appears in the core filesystem. These messages can then be consumed by user software.
+
+### HiDRA
+HiDRA can work in two modes - wether data is transferred via it or data is written over NFS/SMB mounts and HiDRA monitors a folder in a beamline filesystem. In both case one can subscribe to HiDRA's data queue to be informed about a new file.
+
+### ASAPO
+
+ASAPO does not work with files, rather with data streams. Well, behind the scene it does use files, but in principle what a user see is a stream of messages, where a message typically consists of metadata and data blobs. Data blob can be a single image, a file with arbitrary content or whatever else, even null. Important is - what goes to one end, appears on the other. And that each message must have a consecutive index. These messages must be ingested to an ASAPO data stream in some way (e.g. using ASAPO Producer API or HiDRA) and data, if not yet, will be transferred and stored in the data center. A user can then read the messages from the stream and process it in a way he likes.
+
+### Compare by categories
+
+In the table below we compare the approaches from different points of view and in the [next table](#compare-by-features) we compare the approaches by available features.
+
+| Category | Filesystem | Filesystem+Kafka | HiDRA | ASAPO |
+|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Data ingest | traditional way - write to disk | same as Filesystem, additionally a message is send to a Kafka topic with basic file metadata (name, size, creation date, ...) | outperforms data ingest via NFS/SMB, uses ZeroMQ(TCP), saturates 10GE bandwidth. To be tested with 100GE. Same metadata as with Kafka | uses parallel TCP connections & in the future RDMA. Can add arbitrary metadata (JSON format). Can send data to various streams to create arbitrary data hierarchy (e.g. stream per scan). Saves data to a memory buffer and on disk. |
+| Offline analysis | traditional way - read from disk. Need to know the filename to read, usually not a problem. | same as Filesystem, there is no benefits reading Kafka queue after all data is arrived | not possible, need to fallback to Filesystem | same as online analysis (see below) |
+| Online analysis | no efficient way to recognise that new data is available - periodically read folder content and compare with previous state?, periodically check file appeared? | using a subscription to a "new files" topic, a user software can be made aware of new data quite soon and react correspondingly. | one can subscribe to a HiDRA's ZeroMQ stream and consume data. If data arrives faster than it can be processed or e.g. user software crashes - data might be skipped. | one can get data from various streams in different ways - get next unprocessed message ordered by index, get last, get by id. Since everything is stored in persistent storage, processing is possible with arbitrary slow (but also fast) consumers. Resilient to connections loss or consumer crashes. |
+| Performance | as good as read/write to disk. Can be an issue, especially with new detectors 100GE+ networks. | as good as read/write to disk + some latency for the file to be written to the beamline filesystem, copied to the core filesystem and a message to go through Kafka. | data is available as soon as it is received by HiDRA. If multiple consumer groups want to read same data (one consumer is known - the process that writes the file), data will be transferred multiple times, which influences the network performance. | data is available to be consumed as soon as it is arrived and saved to beamline filesystem (later can be optimised by using persistent memory instead). No need to read from disk since it also remains in memory buffer. |
+| Parallelisation | Parallelisation is easily possible e.g. with an MPI library. | Parallelisation is possible if Kafka's topics are partitioned (which is not the case in the moment) | Not out of the box, possible with some changes from user's side | Parallelisation is easily possible, one can consume data concurrently with different consumers from the same stream. Normally, synchronisation between consumers is not needed, but this might depend on a use case. When configured, data can be resent if not acknowledged during a specified time period. |
+| Search/filter | hardly possible, manually parsing some metadata file, using POSIX commands? | same as Filesystem. There is Kafka SQL query language which could be used if there would be metadata in messages, which is not the case (yet?). | not possible | one can use a set of SQL queries to ingested metadata. |
+| General comments | Might be ok for slow detectors or/and a use case without online analysis requirements. Might be the only way to work with legacy applications | Fits well for the cases where a software just need a trigger that some new data has arrived, when processing order, extra metadata, parallelisation is not that important or implemented by other means. Some delay between an image is generated and the event is emitted by Kafka is there, but probably not that significant (maximum a couple of seconds). Might be not that appropriate for very fast detectors since still using filesystem to write/read data. | Works quite well to transfer files from detector to the data center. Also a good candidate for live viewers, where the last available "image" should be displayed. Does not work for offline analysis or for near real-time analysis where image processing can take longer than image taking. | Tries to be a general solution which improves in areas where other approaches not suffice: single code for offline/near real-time/online analysis, parallelisation, extended metadata, efficient memory/storage management, getting data without access to filesystem (e.g. from detector pc without core filesystem mounted), computational pipelines, ... Everything has its own price: user software must be modified to use ASAPO, a wrapper might be needed for legacy software that cannot be modified, user/beamtime scientist should better structure the data - e.g. consecutive indexes must be available for each image, one has to define to which stream write/read data, what is the format of the data, ... |
+
+
+### Compare by features
+
+| Feature | Filesystem | Filesystem+Kafka | HiDRA | ASAPO |
+|--------------------------------------------------------------------------------------------------------------------|------------------------------------|------------------------------------|------------------------------------|------------------------------------|
+| send metadata with image | No | No | No | Yes |
+| get last image | No | No | Yes | Yes |
+| get image by id | No | No | No | Yes |
+| get image in order | No | No | No | Yes |
+| Immediately get informed that a new image is arrived | No | Yes | Yes | Yes |
+| access image remotely, without reading filesystem | No | No | Yes, if it is still in buffer | Yes |
+| access past images | Yes | Yes | No | Yes |
+| need to change user code | No | Yes | Yes | Yes |
+| parallelisation | Yes (if user software allows that) | Not out of the box | Not out of the box | Yes |
+| legacy applications | Yes | No (wrapper could be a workaround) | No (wrapper could be a workaround) | No (wrapper could be a workaround) |
+| transparent restart/continuation of simulations in case e.g. worker process crashes, also for parallel simulations | Not out of the box | Yes | No | Yes |
diff --git a/docs/site/versioned_docs/version-21.06.0/consumer-clients.md b/docs/site/versioned_docs/version-21.06.0/consumer-clients.md
new file mode 100644
index 0000000000000000000000000000000000000000..d04c14034585b2ef69f335b0104b2bfafddf69d1
--- /dev/null
+++ b/docs/site/versioned_docs/version-21.06.0/consumer-clients.md
@@ -0,0 +1,33 @@
+---
+title: Consumer Clients
+---
+
+Consumer client (or consumer) is a part of a distributed streaming system that is responsible for processing streams of data that were created by producer. It is usually a user (beamline scientist, detector developer, physicist, ... ) responsibility to develop a client for specific beamline, detector or experiment using ASAPO Consumer API and ASAPO responsibility to make sure data is delivered to consumers in an efficient and reliable way.
+
+
+
+Consumer API is available for C++ and Python and has the following main functionality:
+
+- Create a consumer instance and bind it to a specific beamtime and data source
+ - multiple instances can be created (also within a single application) to receive data from different sources
+ - a beamtime token is used for access control
+- If needed (mainly for get_next_XX operations), create a consumer group that allows to process messages independently from other groups
+- Receive messages from a specific stream (you can read more [here](data-in-asapo) about data in ASAPO)
+ - GetNext to receive process messages one after another without need to know message indexes
+ - Consumer API returns a message with index 1, then 2, ... as they were set by producer.
+ - This also works in parallel so that payload is distributed within multiple consumers within same consumer group or between threads of a single consumer instance. In parallel case order of indexes of the messages is not determined.
+ - GetLast to receive last available message - for e.g. live visualisation
+ - GetById - get message by index - provides random access
+- Make queries based on metadata contained in a message - returns all messages in a stream with specific metadata. A subset of SQL language is used
+
+
+All of the above functions can return only metadata part of the message, so that an application can e.g. extract the filename and pass it to a 3rd party tool for processing. Alternative, a function may return the complete message with metadata and data so that consumer can directly process it. An access to the filesystem where data is actually stored is not required in this case.
+
+:::note
+In case of dataset family of functions, only list of dataset messages is returned, the data can be retrieved in a separate call.
+:::
+
+Please refer to [C++](http://asapo.desy.de/cpp/) and [Python](http://asapo.desy.de/python/) documentation for specific details (available from DESY intranet only).
+
+
+
diff --git a/docs/site/versioned_docs/version-21.06.0/core-architecture.md b/docs/site/versioned_docs/version-21.06.0/core-architecture.md
new file mode 100644
index 0000000000000000000000000000000000000000..a02048e96d33c687f38622661b735974541a36e4
--- /dev/null
+++ b/docs/site/versioned_docs/version-21.06.0/core-architecture.md
@@ -0,0 +1,29 @@
+---
+title: Core Architecture
+---
+
+For those who are curious about ASAPO architecture, the diagram shows some details. Here arrows with numbers is an example of data workflow explained below.
+
+
+
+## Data Workflow (example)
+the workflow can be split into two more or less independent tasks - data ingestion and data retrieval
+
+### Data ingestion (numbers with i on the diagram)
+1i) As we [know](producer-clients.md), producer client is responsible for ingesting data in the system. Therefore the first step is to detect that the new message is available. This can be done using another tool developed at DESY named [HiDRA](https://confluence.desy.de/display/FSEC/HiDRA). This tool monitors the source of data (e.g. by monitoring a filesystem or using HTTP request or ZeroMQ streams, depending on detector type)
+
+2i) HiDRA (or other user application) then uses ASAPO Producer API to send messages (M1 and M2 in our case) in parallel to ASAPO Receiver. TCP/IP or RDMA protocols are used to send data most efficiently. ASAPO Receiver receives data in a memory cache
+
+3i) - 4i) ASAPO saves data to a filesystem and adds a metadata record to a database
+
+5i) A feedback is send to the producer client with success or error message (in case of error, some of the step above may not happen)
+
+### Data retrieval (numbers with r on the diagram)
+
+[Consumer client](consumer-clients.md)) is usually a user application that retrieves data from the system to analyse/process it.
+
+The first step to retrieve a message via Consumer API is to pass the request to the Data Broker (1r). The Data Broker retrieves the metadata information about the message from the database (2r) and returns it to the Consumer Client. The Consumer Client analyses the metadata information and decides how to get the data. It the data is still in the Receiver memory cache, the client requests data from there via a Data Server (which is a part of ASAPO Receiver). Otherwise, client gets the data from the filesystem - directly if the filesystem is accessible on the machine where the client is running or via File Transfer Service if not.
+
+
+
+
diff --git a/docs/site/versioned_docs/version-21.06.0/data-in-asapo.md b/docs/site/versioned_docs/version-21.06.0/data-in-asapo.md
new file mode 100644
index 0000000000000000000000000000000000000000..96abf43c86c13873641ea767b7766e3145baa5d3
--- /dev/null
+++ b/docs/site/versioned_docs/version-21.06.0/data-in-asapo.md
@@ -0,0 +1,29 @@
+---
+title: Data in ASAPO
+---
+All data that is produced, stored and consumed via ASAPO is structured on several levels.
+
+#### Beamtime
+This is the top level. Contains all data collected/produced during a single beamtime (Beamtime is the term used at DESY. Can also be Run, Experiment, Proposal, ...). Each beamtime has its own unique ID.
+
+#### Data Source
+During a beamtime, data can be produced by different sources. For example, a detector is a data source, if multiple detectors are used during an experiment, they can be different data sources or the same data source (more details below in datasets section). A user application that simulates or analyses data can also act as an ASAPO data source. Each data source has its own unique name within a beamtime.
+
+#### Data Stream
+Each data source can emit multiple data streams. Each stream has a unique within a specific data source name.
+
+#### Message
+Data streams consist of smaller entities - messages. The content of a message is quite flexible, to be able to cover a broad amount of use cases. Usually it is a metadata and some binary data (e.g. a detector image, or an hdf5 file with multiple images). At the moment ASAPO itself is agnostic to the data and sees it as a binary array. Later some specific cases might be handled as well (the most prominent use case - an hdf5 file with multiple images).
+
+An important aspect is that each message within a data stream must be assigned a consecutive integer index. Therefore, a streams always contain messages with index = 1,2,3 ... . This is different to traditional messaging systems where messages have timestamps or arbitrary unique hash IDs. The reason is that with timestamps the order of messages saved in the system might differ from the order the were generated by the data source (e.g. detector). And keeping correct order is required in many cases. Second reason is that it makes a random access to a specific message quite straightforward.
+
+#### Datasets/Dataset substreams
+In some cases multiple detectors are using during an experiment. E.g. a 3D image is composed from multiple 2D images created by different detectors. In this case these 2D images can be composed to a dataset so that it a be processed later as a whole. One would then use a single data source (which would mean a set of detectors or "multi-detector" in this case), single data stream and, to compose a dataset, for each of it's components (each 2D image in our example) the corresponding detector would send a message with same id but to a different dataset substream.
+
+So, for the case without datasets (single detector) the data hierarchy is Beamtime→Data Source → Data Stream → Message:
+
+
+
+And with datasets (multi-detector) the data hierarchy is Beamtime→Data Source → Data Stream → Dataset→ Message in Dataset Substream:
+
+
diff --git a/docs/site/versioned_docs/version-21.06.0/getting-started.mdx b/docs/site/versioned_docs/version-21.06.0/getting-started.mdx
new file mode 100644
index 0000000000000000000000000000000000000000..2ba2462b4040589357a2d0f0a2cf470677d0f559
--- /dev/null
+++ b/docs/site/versioned_docs/version-21.06.0/getting-started.mdx
@@ -0,0 +1,244 @@
+---
+title: Getting Started
+slug: /
+---
+
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## Step 1: Start ASAPO services {#step-1}
+
+If you already have running ASAPO services and know the endpoint, you can skip to the [step 2](#step-2).
+
+Otherwise, for testing purposes one can start ASAPO services in a standalone mode (this is not recommended for production deployment).
+
+
+The easiest way is to use a Docker container.
+So, make sure Docker is installed and you have necessary permissions to use it.
+Please note that this will only work on a Linux machine. Also please note that ASAPO needs some ports to be available. You can check the list
+[here](https://stash.desy.de/projects/ASAPO/repos/asapo/browse/deploy/asapo_services/scripts/asapo.auto.tfvars.in?at=100.0.docs#37).
+
+Now, depending on how your Docker daemon is configured (if it uses a
+unix socket or a tcp port for communications)
+ you can use pick corresponding script below, adjust and execute it to start ASAPO services.
+
+<Tabs
+ defaultValue="unix"
+ values={[
+ { label: 'Docker with unix socket (default)', value: 'unix', },
+ { label: 'Docker with tcp (used on FS machines)', value: 'tcp', },
+ ]
+}>
+<TabItem value="unix">
+
+```shell content="./frozen_versions/21.06.0/getting_started/start_asapo_socket.sh"
+```
+
+</TabItem>
+
+<TabItem value="tcp">
+
+```shell content="./frozen_versions/21.06.0/getting_started/start_asapo_tcp.sh"
+```
+
+</TabItem>
+</Tabs>
+
+at the end you should see
+
+<p className="green-text"><strong>Apply complete! Resources: 19 added, 0 changed, 0 destroyed.</strong></p>
+
+which means ASAPO services successfully started. Your ASAPO endpoint for API calls will be **localhost:8400**.
+
+### Create data directories
+
+Next, you need to create directories where ASAPO will store the data
+(the structure matches the one used at DESY experiments).
+Since we are going to use beamline `test` and beamtime `asapo_test` in following examples,
+we must create two folders, one for the beamline filesystem and one for the core file system:
+
+```shell
+ASAPO_HOST_DIR=/var/tmp/asapo # the folder used in step 1
+mkdir -p $ASAPO_HOST_DIR/global_shared/online_data/test/current/raw
+mkdir -p $ASAPO_HOST_DIR/global_shared/data/test_facility/gpfs/test/2019/data/asapo_test
+```
+
+:::note ASAP::O in production mode
+
+We have a running instance for processing data collected during experiments. Please get in touch with FS-SC group for more information.
+
+:::
+
+## Step 2: Install client libraries {#step-2}
+
+Now you can install Python packages or C++ libraries for ASAPO Producer and Consumer API (you need to be in DESY intranet to access files).
+
+<Tabs
+ defaultValue="python-pip"
+ values={[
+ { label: 'Python - pip', value: 'python-pip', },
+ { label: 'Python - packages', value: 'python-packages', },
+ { label: 'C++ packages', value: 'cpp', },
+ ]
+}>
+<TabItem value="python-pip">
+
+```shell content="./frozen_versions/21.06.0/getting_started/install_python_clients_pip.sh" snippetTag="#snippet1"
+```
+
+</TabItem>
+<TabItem value="python-packages">
+
+```shell content="./frozen_versions/21.06.0/getting_started/install_python_clients_pkg.sh"
+```
+
+</TabItem>
+<TabItem value="cpp">
+
+```shell content="./frozen_versions/21.06.0/getting_started/install_cpp_clients.sh"
+```
+
+</TabItem>
+</Tabs>
+
+## Step 3: Produce a message {#step-3}
+
+<Tabs
+ groupId="language"
+ defaultValue="python"
+ values={[
+ { label: 'Python', value: 'python', },
+ { label: 'C++', value: 'cpp', },
+ ]
+}>
+<TabItem value="python">
+
+Now you can write a Producer client (API documentation [here](http://asapo.desy.de/python/producer.html)).
+
+```shell content="./frozen_versions/21.06.0/getting_started/python/produce.py"
+```
+
+Execute it with python3
+
+```
+$ python3 produce.py
+```
+
+</TabItem>
+
+<TabItem value="cpp">
+
+Now you can write a Producer client (API documentation [here](http://asapo.desy.de/cpp/producer)).
+
+```shell content="./frozen_versions/21.06.0/getting_started/cpp/produce.cpp"
+```
+
+Compile e.g. using CMake and execute. You might need to point cmake (with CMAKE_PREFIX_PATH) to asapo installation and curl library if installed to non-standard location.
+
+```shell content="./frozen_versions/21.06.0/getting_started/cpp/CMakeLists.txt" snippetTag="#producer"
+```
+
+```
+$ cmake . && make
+$ ./asapo-produce
+```
+
+
+</TabItem>
+</Tabs>
+
+the output should look like
+
+```
+{"time":"***** *****","source":"producer_api","level":"info","message":"authorized connection to receiver at ****:****"}
+successfuly sent: {"id": 1, "buffer": "test_file"}
+```
+
+
+## Step 4: Consume a message {#step-4}
+
+A consumer data that reads the message ingested during step 3. Note that a token is needed to work with data. In production, the token is provided during start of the beamtime.
+
+<Tabs
+ groupId="language"
+ defaultValue="python"
+ values={[
+ { label: 'Python', value: 'python', },
+ { label: 'C++', value: 'cpp', },
+ { label: 'C', value: 'c', },
+ ]
+}>
+<TabItem value="python">
+
+Complete API documentation [here](http://asapo.desy.de/python/consumer.html)
+
+```shell content="./frozen_versions/21.06.0/getting_started/python/consume.py"
+```
+
+Execute it with python3
+
+```
+$ python3 consumer.py
+```
+
+</TabItem>
+
+<TabItem value="cpp">
+
+```shell content="./frozen_versions/21.06.0/getting_started/cpp/consume.cpp"
+```
+
+Compile e.g. using CMake and execute. You might need to point cmake (with CMAKE_PREFIX_PATH) to asapo installation and curl library if installed to non-standard location.
+
+```shell content="./frozen_versions/21.06.0/getting_started/cpp/CMakeLists.txt" snippetTag="#consumer"
+```
+
+```
+$ cmake . && make
+$ ./asapo-consume
+```
+
+</TabItem>
+
+<TabItem value="c">
+
+```shell content="./frozen_versions/21.06.0/getting_started/c/consume.c"
+```
+
+Compile e.g. using Makefile and pkg-config (although we recommend CMake - see C++ section) and execute. This example assumes asapo is installed to /opt/asapo. Adjust correspondingly.
+
+```shell content="./frozen_versions/21.06.0/getting_started/c/Makefile" snippetTag="#consumer"
+```
+
+```
+$ make
+$ ./asapo-consume
+```
+
+
+</TabItem>
+
+</Tabs>
+
+the output should look like
+
+```
+id: 1
+file name: processed/test_file
+file content: hello
+stream deleted
+```
+
+## Step 5: Clean-up
+
+Optionally, last step is to stop ASAPO services and remove files:
+
+```shell content="./frozen_versions/21.06.0/getting_started/cleanup.sh"
+```
+
+<br/><br/>
+
+:::tip
+You can see more examples in ASAPO [source code](https://stash.desy.de/projects/ASAPO/repos/asapo/browse/examples?at=21.06.0)
+:::
diff --git a/docs/site/versioned_docs/version-21.06.0/overview.md b/docs/site/versioned_docs/version-21.06.0/overview.md
new file mode 100644
index 0000000000000000000000000000000000000000..7af1f0471cfe9deb243d8d5de447c76ebcf9b30a
--- /dev/null
+++ b/docs/site/versioned_docs/version-21.06.0/overview.md
@@ -0,0 +1,40 @@
+---
+title: Overview
+---
+
+
+
+ASAP::O (or ASAPO) is a high performance distributed streaming platform. It is being developed at DESY and is mainly aimed to support online/offline analysis of experimental data produced at its facilities. The ideas behind are quite similar to that of Apache Kafka and similar messaging solutions, but ASAPO is developed and tuned for scientific use cases with their specific workflows and where the size of the messages is much large (MBs to GBs as compared to KBs in traditional systems).
+
+
+
+ASAPO has the following key capabilities:
+
+- Deliver data produced by an experimental facility (e.g. detector) to a data center in a high-performant fault-tolerant way
+- Consume this data in various modes (as soon as new data occurs, random access, latest available data, in parallel, ...)
+- Ingest own data/ create computational pipelines
+
+
+ASAPO consists of the following three components:
+
+- Core services (run in background on a single node or a cluster and provide ASAPO functionality)
+- Producer API to ingest data into the system
+- Consumer API to retrieve data from the system
+
+### Bird's eye view
+
+A workflow when using ASAPO can be represented as follows:
+
+
+
+
+Usually, an end user can see ASAPO core services as a black box. But some more details are given [here](core-architecture).
+
+Next, one can learn more about following concepts:
+
+- [Data in ASAPO](data-in-asapo)
+- [Producer clients](producer-clients)
+- [Consumer clients](consumer-clients)
+
+You can also compare with other solutions, jump directly to [Getting Started](getting-started.mdx) or have a look in use cases section
+
diff --git a/docs/site/versioned_docs/version-21.06.0/p02.1.md b/docs/site/versioned_docs/version-21.06.0/p02.1.md
new file mode 100644
index 0000000000000000000000000000000000000000..271b2bf10a91d2ecf398e2a571b49b0df874bbe7
--- /dev/null
+++ b/docs/site/versioned_docs/version-21.06.0/p02.1.md
@@ -0,0 +1,43 @@
+---
+title: ASAP::O at P02.1
+---
+
+Online analysis at P02.1 has two main goals:
+
+- Doing as much beamline specific data analysis as possible for the user, so that they can concentrate on analyzing the experiment specific details. This will lead to a comprehensive support for the user from beamline side and therefore lead to a higher user satisfaction. Automatization of the analysis is essential to achieve the necessary high throughput, which is mandatory for current and future diffraction applications.
+- Enabling timely decisions through a "live" view of raw images and analyzed data. Problems with the measurement can often be more easily detected in the analyzed data, which should be made available to the user as early as possible to avoid wasting valuable measurement time on suboptimal experimental conditions.
+
+## Description of a typical beamtime at P02.1
+
+- A beamtime consists of a number of scans
+- Each scan consists of one or more steps
+- At each step, an image is taken by the detectors, as well as several other scalar sensors values are gathered, e.g., temperature, electric current, position, etc.
+- The parameters for the analysis are fixed during one scan but might need to change from one scan to the next
+
+## Analysis Pipeline
+
+- Images are taken by one or two detectors
+- Optionally, a number of consecutive images of a single detector are merged into one averaged image to reduce the noise
+- The (averaged) images are stored into one NeXus file per detector per scan
+- Each (averaged) image is analyzed independently
+- The analyzed data is written to one NeXus file per detector per scan
+- All scalar sensor data and additional metadata is written to one NeXus file per scan that links to the other NeXus files with the (averaged) images and analyzed data
+- A viewer displays the live and history output of all relevant processing steps
+
+
+
+## Use of ASAPO
+
+In the following, ASAPO specific details for the pipeline of a single detector are given. For multiple detectors, all stream names are suffixed by the detector ID.
+
+1. The data acquisition software stores the parameters for the analysis in a "scan-metadata" stream with one substream per scan and one metadata entry per substream
+2. Images are ingested into ASAPO
+3. The images taken by the detectors are written to the beamline filesystem by HiDRA (one file per image)
+4. HiDRA inserts the files into ASAPO. It assigns the files to the correct "detector" stream based on the file name. Each stream uses one substream per scan, its name is also extracted from the filename by HiDRA. This applies to the index within a substream as well.
+5. If enabled, one "averager" worker per detector stream reads the files from the "detector" stream and emits the averaged images into the "averaged" stream. The name of the substream of the input is used for the name of the output substream. The indices within a substream are chosen consecutively.
+6. One "nexus-writer" worker per detector reads the images either from the "detector" or the "averaged" stream. All images of a single substream are stored into one file. The filename is constructed from the name of the stream and substream the image belongs to. The index within a substream corresponds to the index within the HDF5 dataset.
+7. Multiple "asapo-dawn" worker read their parameters from the "scan-metadata" stream at the start of each substream. The images are read from the "detector" or "averaged" stream. The worker emit the resulting data into an "analyzed" stream with the same substream name as the input and the same index.
+8. One "nexus-write" worker per detector reads the analyzed data from the "analyzed" stream and writes it into one NeXus file per substream. The filename is constructed from the name of the stream and substream the data belongs to. The index within a substream corresponds to the index within the HDF5 dataset.
+9. The data acquisition software stores all scalar data and all additional scan-metadata in a master NeXus file that links to the NeXus files produced by the ASAPO workers.
+10. The viewer listens to all streams and parses the metadata to create a continuously updated tree view of all available data. Clicking on an item uses get_by_id to retrieve the actual data. A "live" mode automatically retrieves the latest data.
+
diff --git a/docs/site/versioned_docs/version-21.06.0/producer-clients.md b/docs/site/versioned_docs/version-21.06.0/producer-clients.md
new file mode 100644
index 0000000000000000000000000000000000000000..d74adb0b60e04ae6cf0ab5b9c2b264a25c46419a
--- /dev/null
+++ b/docs/site/versioned_docs/version-21.06.0/producer-clients.md
@@ -0,0 +1,23 @@
+---
+title: Producer Clients
+---
+
+Producer client (or producer) is a part of a distributed streaming system that is responsible for creating data streams (i.e. ingesting data in the system). It is usually a user (beamline scientist, detector developer, physicist, ... ) responsibility to develop a client for specific beamline, detector or experiment using ASAPO Producer API and ASAPO responsibility to make sure data is transferred and saved an in efficient and reliable way.
+
+
+
+Producer API is available for C++ and Python and has the following main functionality:
+
+- Create a producer instance and bind it to a specific beamtime and data source
+multiple instances can be created (also within a single application) to send data from different sources
+- Send messages to a specific stream (you can read more [here](data-in-asapo) about data in ASAPO)
+ - each message must have a consecutive integer index, ASAPO does not create indexes automatically
+ - to compose datasets, dataset substream (and dataset size) should be send along with each message
+ - messages are sent asynchronously, in parallel using multiple threads
+ - retransfer will be attempted in case of system failure
+ - a callback function can be provided to react after data was sent/process error
+
+Please refer to [C++](http://asapo.desy.de/cpp/) and [Python](http://asapo.desy.de/python/) documentation for specific details (available from DESY intranet only).
+
+
+
diff --git a/docs/site/versioned_sidebars/version-21.06.0-sidebars.json b/docs/site/versioned_sidebars/version-21.06.0-sidebars.json
new file mode 100644
index 0000000000000000000000000000000000000000..8e72d0a1a48b4e4436eca58c8403f001ef3181a2
--- /dev/null
+++ b/docs/site/versioned_sidebars/version-21.06.0-sidebars.json
@@ -0,0 +1,50 @@
+{
+ "version-21.06.0/docs": [
+ {
+ "type": "doc",
+ "id": "version-21.06.0/getting-started"
+ },
+ {
+ "type": "doc",
+ "id": "version-21.06.0/overview"
+ },
+ {
+ "type": "doc",
+ "id": "version-21.06.0/compare-to-others"
+ },
+ {
+ "collapsed": true,
+ "type": "category",
+ "label": "Concepts And Architecture",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-21.06.0/data-in-asapo"
+ },
+ {
+ "type": "doc",
+ "id": "version-21.06.0/producer-clients"
+ },
+ {
+ "type": "doc",
+ "id": "version-21.06.0/consumer-clients"
+ },
+ {
+ "type": "doc",
+ "id": "version-21.06.0/core-architecture"
+ }
+ ]
+ },
+ {
+ "collapsed": true,
+ "type": "category",
+ "label": "Use Cases",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-21.06.0/p02.1"
+ }
+ ]
+ }
+ ]
+}
diff --git a/docs/site/versions.json b/docs/site/versions.json
new file mode 100644
index 0000000000000000000000000000000000000000..e0f897a4fca0b405528a418161b4f9be396b42d2
--- /dev/null
+++ b/docs/site/versions.json
@@ -0,0 +1,3 @@
+[
+ "21.06.0"
+]
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 8950f7f63c1f91ae1172d1c0ebcceb48ad3b8d68..70a3f64fdf4b577b7d1ab10b2fb8fd5f58cd0332 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -6,9 +6,3 @@ if(BUILD_EXAMPLES OR INSTALL_EXAMPLES)
add_subdirectory(producer)
add_subdirectory(consumer)
endif()
-
-if(BUILD_ASAPO_SITE)
- add_subdirectory(for_site)
-endif()
-
-
diff --git a/examples/for_site/CMakeLists.txt b/examples/for_site/CMakeLists.txt
deleted file mode 100644
index 1d2595b7e83a86a6d4b6660037fa4303b033f2d7..0000000000000000000000000000000000000000
--- a/examples/for_site/CMakeLists.txt
+++ /dev/null
@@ -1 +0,0 @@
-configure_files(${CMAKE_CURRENT_SOURCE_DIR}/getting_started ${CMAKE_CURRENT_BINARY_DIR}/getting_started)