Merge pull request #138 in ASAPO/asapo from fix-ldap-auth to develop

* commit 'ac91d78f':
  update changelogs
  fix getting list of hosts from LDAP
  update docs
  update docs
  update docs
  update
  add asapo icons
  update deployment
  fix build
  fix build
  fix build
  make asapo site with docusaurius

3d_party/ldap/record.ldif

@@ -11,11 +11,15 @@ objectClass :top
 objectClass :nisNetgroup
 cn:a3p00-hosts
 description: Netgroup for nodes on PETRA III Beamline P00
+nisNetgroupTriple: (blabla,-,)
 nisNetgroupTriple: (localhost,-,)
+nisNetgroupTriple: (blabla2,-,)
 
 dn: cn=a3p07-hosts,ou=netgroup,ou=rgy,o=desy,c=de
 objectClass :top
 objectClass :nisNetgroup
 cn:a3p07-hosts
 description: Netgroup for nodes on PETRA III Beamline P07
+nisNetgroupTriple: (blabla,-,)
 nisNetgroupTriple: (localhost,-,)
+nisNetgroupTriple: (blabla2,-,)
\ No newline at end of file

CHANGELOG.md

+## 21.03.1
+
+BUG FIXES
+* fix LDAP authorization for raw data type Producers
+
+
 ## 21.03.0
 
  IMPROVEMENTS

CMakeLists.txt

@@ -46,6 +46,8 @@ option(BUILD_CLIENTS_ONLY "Build clients only" OFF)
 option(BUILD_CONSUMER_TOOLS "Build consumer tools" OFF)
 option(BUILD_EVENT_MONITOR_PRODUCER "Build the event monitor producer" OFF)
 
+
+
 option(BUILD_PYTHON "Build python libs" ON)
 if(BUILD_PYTHON)
     set(BUILD_PYTHON_PACKAGES "" CACHE STRING "which python packages to build")
@@ -61,6 +63,8 @@ option(INSTALL_EXAMPLES "Install examples" OFF)
 
 option(BUILD_CPP_DOCS "Build the C++ documentaion with doxygen" OFF)
 option(BUILD_PYTHON_DOCS "Build the Python documentaion with sphinx" OFF)
+option(BUILD_ASAPO_SITE "Build the asapo web site" OFF)
+
 
 option(ENABLE_LIBFABRIC "Enables LibFabric support for RDMA transfers" OFF)
 option(ENABLE_LIBFABRIC_LOCALHOST "Emulates LibFabric stack over TCP. Only for localhost and testing purposes." OFF)
@@ -133,7 +137,7 @@ if(BUILD_INTEGRATION_TESTS)
     add_subdirectory(tests)
 endif()
 
-if(BUILD_EXAMPLES OR INSTALL_EXAMPLES)
+if(BUILD_EXAMPLES OR INSTALL_EXAMPLES OR BUILD_ASAPO_SITE)
     add_subdirectory(examples)
 endif()
 

CMakeModules/prepare_version_tag.cmake

@@ -24,6 +24,10 @@ if (${BRANCH} STREQUAL "master")
     SET (ASAPO_VERSION_COMMIT "")
     SET (ASAPO_VERSION_DOCKER_SUFFIX "")
     SET (PYTHON_ASAPO_VERSION ${ASAPO_VERSION})
+    SET (ASAPO_VERSION "20.01.1")
+    string(REGEX REPLACE "\\.0([0-9]+)\\."
+            ".\\1." ASAPO_WHEEL_VERSION
+            ${ASAPO_VERSION})
 else()
     SET (ASAPO_VERSION ${BRANCH})
     SET (ASAPO_VERSION_COMMIT ", build ${ASAPO_VERSION_COMMIT}")
@@ -31,6 +35,7 @@ else()
     string(REPLACE "_" "-" ASAPO_VERSION ${ASAPO_VERSION})
     SET (ASAPO_VERSION 1.0.${ASAPO_VERSION})
     SET (PYTHON_ASAPO_VERSION ${ASAPO_VERSION})
+    SET (ASAPO_WHEEL_VERSION ${ASAPO_VERSION})
 endif()
 
 message("Asapo Version: " ${ASAPO_VERSION})

PROTOCOL-VERSIONS.md

 ### Producer Protocol
 | Release      | Supported by client | Supported by server  | Status           |
 | ------------ | ------------------- | -------------------- | ---------------- |
-| v0.1         | 21.03.0 - 21.03.0   | 21.03.0  - 21.03.0   | Current version  |
+| v0.1         | 21.03.0 - 21.03.1   | 21.03.0  - 21.03.1   | Current version  |
 
 
 ### Consumer Protocol
 | Release      | Supported by client | Supported by server  | Status           |
 | ------------ | ------------------- | -------------------- | ---------------- |
-| v0.1         | 21.03.0 - 21.03.0   | 21.03.0  - 21.03.0   | Current version  |
+| v0.1         | 21.03.0 - 21.03.1   | 21.03.0  - 21.03.1   | Current version  |

VERSIONS.md

 ### Producer API
 
-| Release      | API changed | Breaking changes | Protocol | Supported by server up to | Status |
-| ------------ | ----------- |----------------- | -------- | ------------------------- | ------- |
-| 21.03.0      | Yes         | No               | v0.1     | 21.03.0                   | Current version  |
+| Release      | API changed | Breaking changes | Protocol | Supported by server from/to | Status          |Comment|
+| ------------ | ----------- |----------------- | -------- | ------------------------- | ----------------- | ------- |
+| 21.03.1      | No          | No               | v0.1     | 21.03.0/21.03.1           | current version  |bugfix in server|
+| 21.03.0      | Yes         | Yes              | v0.1     | 21.03.0/21.03.1            |                 |          |
 
 ### Consumer API
 
-| Release      | API changed | Breaking changes | Protocol | Supported by server up to | Status |
-| ------------ | ----------- |----------------- | -------- | ------------------------- | ------- |
-| 21.03.0      | Yes         | Yes              | v0.1     | 21.03.0                   | Current version  |
+| Release      | API changed | Breaking changes | Protocol | Supported by server from/to | Status |Comment|
+| ------------ | ----------- |----------------- | -------- | ------------------------- | ------- | ------- |
+| 21.03.1      | No          | No               | v0.1     | 21.03.0/21.03.1           | current version  |bugfix in server|
+| 21.03.0      | Yes         | Yes              | v0.1     | 21.03.0/21.03.1           |         |        |

authorizer/src/asapo_authorizer/ldap_client/openldap_client.go

@@ -2,6 +2,7 @@ package ldap_client
 
 import (
 	"asapo_authorizer/common"
+	log "asapo_common/logger"
 	"asapo_common/utils"
 	"net"
 	"strings"
@@ -11,10 +12,10 @@ import "github.com/go-ldap/ldap"
 type OpenLdapClient struct {
 }
 
-func (c *OpenLdapClient) GetAllowedIpsForBeamline(url string,base string,filter string) ([]string, error) {
+func (c *OpenLdapClient) GetAllowedIpsForBeamline(url string, base string, filter string) ([]string, error) {
 	l, err := ldap.DialURL(url)
 	if err != nil {
-		return []string{},&common.ServerError{utils.StatusServiceUnavailable, err.Error()}
+		return []string{}, &common.ServerError{utils.StatusServiceUnavailable, err.Error()}
 	}
 	defer l.Close()
 
@@ -28,28 +29,35 @@ func (c *OpenLdapClient) GetAllowedIpsForBeamline(url string,base string,filter
 
 	sr, err := l.Search(searchRequest)
 	if err != nil {
-		if ldap.IsErrorWithCode(err,ldap.LDAPResultNoSuchObject) {
-			return []string{},nil
+		if ldap.IsErrorWithCode(err, ldap.LDAPResultNoSuchObject) {
+			return []string{}, nil
 		} else {
-			return []string{},err
+			return []string{}, err
 		}
 	}
 
-	res := make([]string,0)
+	res := make([]string, 0)
+	var lasterr error = nil
 	for _, entry := range sr.Entries {
-		host := entry.GetAttributeValue("nisNetgroupTriple")
-		host = strings.TrimPrefix(host,"(")
-		host = strings.Split(host, ",")[0]
-		addrs,err := net.LookupIP(host)
-
-		if err != nil {
-			return []string{},err
-		}
-		for _, addr := range addrs {
-			if ipv4 := addr.To4(); ipv4 != nil {
-				res = append(res,ipv4.String())
+		hosts := entry.GetAttributeValues("nisNetgroupTriple")
+		for _, host := range hosts {
+			host = strings.TrimPrefix(host, "(")
+			host = strings.Split(host, ",")[0]
+			addrs, err := net.LookupIP(host)
+			if err != nil {
+				lasterr = err
+				log.Warning("cannot lookup ip for " + host)
+				continue
+			}
+			for _, addr := range addrs {
+				if ipv4 := addr.To4(); ipv4 != nil {
+					res = append(res, ipv4.String())
+				}
 			}
 		}
 	}
-	return res,nil
+	if len(res) == 0 {
+		return res, lasterr
+	}
+	return res, nil
 }

deploy/build_env/site/Dockerfile

+FROM node:14.4 AS base
+
+
+RUN apt-get update && apt-get -y install cmake
+
+ADD build.sh /bin/build.sh

deploy/build_env/site/build.sh

+#!/usr/bin/env bash
+
+cd /asapo/build
+cmake \
+    -DCMAKE_BUILD_TYPE="Release" \
+    -DBUILD_CLIENTS_ONLY=ON \
+    -DBUILD_PYTHON=OFF   \
+    -DBUILD_ASAPO_SITE=ON   \
+    -DLIBCURL_DIR=/curl \
+    -DPython_EXECUTABLE="noop" \
+    ..
+export HOME="/asapo/build"
+cd docs/site
+npm install
+make site
+
+

deploy/build_env/site/build_image.sh

+#!/usr/bin/env bash
+docker build -t yakser/asapo-site-env -f Dockerfile .
+docker push yakser/asapo-site-env
\ No newline at end of file

docs/CMakeLists.txt

@@ -6,3 +6,8 @@ if(BUILD_PYTHON_DOCS)
     add_subdirectory (sphinx)
 endif()
 
+
+if(BUILD_ASAPO_SITE)
+    add_subdirectory (site)
+endif()
+

docs/site/.gitignore

+# Dependencies
+/node_modules
+
+# Production
+/build
+
+# Generated files
+.docusaurus
+.cache-loader
+
+# Misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

docs/site/CMakeLists.txt

+configure_files(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} true)
+configure_files(${CMAKE_CURRENT_SOURCE_DIR}/blog ${CMAKE_CURRENT_BINARY_DIR}/blog)
+configure_files(${CMAKE_CURRENT_SOURCE_DIR}/docs ${CMAKE_CURRENT_BINARY_DIR}/docs)
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/static DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/src DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/plugins DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
+add_custom_target(
+        site ALL
+        COMMAND
+        npm run build VERBATIM
+)

docs/site/README.md

+# Website
+
+This website is built using [Docusaurus 2](https://v2.docusaurus.io/), a modern static website generator.
+
+## Installation
+
+```console
+yarn install
+```
+
+## Local Development
+
+```console
+yarn start
+```
+
+This command starts a local development server and open up a browser window. Most changes are reflected live without having to restart the server.
+
+## Build
+
+```console
+yarn build
+```
+
+This command generates static content into the `build` directory and can be served using any static contents hosting service.
+
+## Deployment
+
+```console
+GIT_USER=<Your GitHub username> USE_SSH=true yarn deploy
+```
+
+If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch.

docs/site/babel.config.js

+module.exports = {
+  presets: [require.resolve('@docusaurus/core/lib/babel/preset')],
+};

docs/site/blog/2021-03-28-hi.md

+---
+slug: Hi
+title: Hello
+author: Sergey Yakubov
+author_title: DESY IT
+<!-- author_url: https://github.com/wgao19 -->
+<!-- author_image_url: https://avatars1.githubusercontent.com/u/2055384?v=4 -->
+tags: [asapo]
+---
+
+Welcome to the new ASAP::O Project site
\ No newline at end of file

docs/site/docs/create-a-blog-post.md

+---
+title: Create a Blog Post
+---
+
+This page will help you on how to create blog posts in Docusaurus.
+
+## Create a Blog Post
+
+Create a file at `blog/2021-02-28-greetings.md`:
+
+```md title="blog/2021-02-28-greetings.md"
+---
+title: Greetings!
+author: Steven Hansel
+author_title: Docusaurus Contributor
+author_url: https://github.com/ShinteiMai
+author_image_url: https://github.com/ShinteiMai.png
+---
+
+Congratulations, you have made your first post!
+
+Feel free to play around and edit this post as much you like.
+```
+
+A new blog post is now available at `http://localhost:3000/blog/greetings`.

docs/site/docs/create-a-document.md

+---
+title: Create a Document
+---
+
+Documents are pages with a **sidebar**, a **previous/next navigation** and many other useful features.
+
+## Create a Document
+
+Create a markdown file at `docs/my-doc.md`:
+
+```mdx title="docs/hello.md"
+---
+title: Hello, World!
+---
+
+## Hello, World!
+
+This is your first document in **Docusaurus**, Congratulations!
+```
+
+A new document is now available at `http://localhost:3000/docs/hello`.
+
+## Add your document to the sidebar
+
+Add `hello` to the `sidebars.js` file:
+
+```diff title="sidebars.js"
+module.exports = {
+  docs: [
+    {
+      type: 'category',
+      label: 'Docusaurus Tutorial',
+-     items: ['getting-started', 'create-a-doc', ...],
++     items: ['getting-started', 'create-a-doc', 'hello', ...],
+    },
+  ],
+};
+```

docs/site/docs/create-a-page.md

+---
+title: Create a Page
+---
+
+Any React or Markdown file created under `src/pages` directory is converted into a website page:
+
+- `src/pages/index.js` -> `localhost:3000/`
+- `src/pages/foo.md` -> `localhost:3000/foo`
+- `src/pages/foo/bar.js` -> `localhost:3000/foo/bar`
+
+## Create a React Page
+
+Create a file at `src/pages/my-react-page.js`:
+
+```jsx title="src/pages/my-react-page.js"
+import React from 'react';
+import Layout from '@theme/Layout';
+
+function HelloWorld() {
+  return (
+    <Layout>
+      <h1>My React page</h1>
+      <p>This is a React page</p>
+    </Layout>
+  );
+}
+```
+
+A new page is now available at `http://localhost:3000/my-react-page`.
+
+## Create a Markdown Page
+
+Create a file at `src/pages/my-markdown-page.md`:
+
+```mdx title="src/pages/my-markdown-page.md"
+---
+title: My Markdown page
+---
+
+# My Markdown page
+
+This is a Markdown page
+```
+
+A new page is now available at `http://localhost:3000/my-markdown-page`.

docs/site/docs/getting-started.mdx

+---
+title: Getting Started
+slug: /
+---
+
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## Step 1: Start ASAPO services {#step-1}
+
+If you already have running ASAPO services and know the endpoint, you can skip to the [step 2](#step-2).
+
+Otherwise, for testing purposes one can start ASAPO services in a standalone mode (this is not recommended for  production deployment).
+
+
+The easiest way is to use a Docker container. 
+So, make sure Docker is installed and you have necessary permissions to use it.
+Please note that this will only work on a Linux machine. Also please note that ASAPO needs some ports to be available. You can check the list
+[here](https://stash.desy.de/projects/ASAPO/repos/asapo/browse/deploy/asapo_services/scripts/asapo.auto.tfvars.in?at={ASAPO_VERSION}#37).
+
+Now, depending on how your Docker daemon is configured (if it uses a
+unix socket or a tcp port for communications)
+ you can use pick corresponding script below, adjust and execute it to start ASAPO services.
+
+<Tabs
+  defaultValue="unix"
+  values={[
+    { label: 'Docker with unix socket (default)', value: 'unix', },
+    { label: 'Docker with tcp (used on FS machines)', value: 'tcp', },
+  ]
+}>
+<TabItem value="unix">
+
+```shell content="getting_started/start_asapo_socket.sh"
+```
+
+</TabItem>
+
+<TabItem value="tcp">
+
+```shell content="getting_started/start_asapo_tcp.sh"
+```
+
+</TabItem>
+</Tabs>
+
+at the end you should see
+
+<p className="green-text"><strong>Apply complete! Resources: 19 added, 0 changed, 0 destroyed.</strong></p>
+
+which means ASAPO services successfully  started. Your ASAPO endpoint for API calls will be **localhost:8400**.
+
+### Create data directories
+
+Next, you need to create directories where ASAPO will store the data
+(the structure matches the one used at DESY experiments).
+Since we are going to use beamline `test` and beamtime `asapo_test` in following examples,
+we must create two folders, one for the beamline filesystem and one for the core file system:
+
+```shell
+ASAPO_HOST_DIR=/var/tmp/asapo # the folder used in step 1
+mkdir -p $ASAPO_HOST_DIR/global_shared/online_data/test/current/raw
+mkdir -p $ASAPO_HOST_DIR/global_shared/data/test_facility/gpfs/test/2019/data/asapo_test
+```
+
+:::note ASAP::O in production mode 
+
+We have a running instance for processing data collected during beamtimes. Please get in touch with FS-SC group for more information.
+
+:::
+
+## Step 2: Install client libraries {#step-2}
+
+Now you can install Python packages or C++ libraries for ASAPO Producer and Consumer API (you need to be in DESY intranet to access files).
+
+<Tabs
+  defaultValue="python-pip"
+  values={[
+    { label: 'Python - pip', value: 'python-pip', },
+    { label: 'Python - packages', value: 'python-packages', },
+    { label: 'C++ packages', value: 'cpp', },
+  ]
+}>
+<TabItem value="python-pip">
+
+```shell content="getting_started/install_python_clients_pip.sh" snippetTag="#snippet1"
+```
+
+</TabItem>
+<TabItem value="python-packages">
+
+```shell content="getting_started/install_python_clients_pkg.sh"
+```
+
+</TabItem>
+<TabItem value="cpp">
+
+```shell content="getting_started/install_cpp_clients.sh"
+```
+
+</TabItem>
+</Tabs>
+
+
+Run the development server in the newly created `my-website` folder:
+
+Open `docs/getting-started.md` and edit some lines. The site reloads automatically and display your changes.
+
+## Step 3: Produce a message {#step-3}
+
+## Step 4: Consume a message {#step-4}
+
+## Step 5: Clean-up