From 967dbe1612a7ab17ad134e99390d240d9a53b89a Mon Sep 17 00:00:00 2001
From: George Sedov <george.sedov@desy.de>
Date: Thu, 8 Jun 2023 19:31:16 +0200
Subject: [PATCH] add asapo standalone package
---
.gitlab-ci.yml | 25 ++++++
deploy/asapo_services_light/Dockerfile | 47 +++++++++++
deploy/asapo_services_light/auth_secret.key | 1 +
.../auth_secret_admin.key | 1 +
deploy/asapo_services_light/authorizer.json | 22 ++++++
deploy/asapo_services_light/broker.json | 12 +++
deploy/asapo_services_light/discovery.json | 21 +++++
.../asapo_services_light/file-transfer.json | 8 ++
deploy/asapo_services_light/monitoring.json | 7 ++
deploy/asapo_services_light/nginx.conf | 77 +++++++++++++++++++
deploy/asapo_services_light/prepare-docker.sh | 7 ++
deploy/asapo_services_light/receiver.json | 33 ++++++++
deploy/asapo_services_light/supervisord.conf | 72 +++++++++++++++++
13 files changed, 333 insertions(+)
create mode 100644 deploy/asapo_services_light/Dockerfile
create mode 100644 deploy/asapo_services_light/auth_secret.key
create mode 100644 deploy/asapo_services_light/auth_secret_admin.key
create mode 100644 deploy/asapo_services_light/authorizer.json
create mode 100644 deploy/asapo_services_light/broker.json
create mode 100644 deploy/asapo_services_light/discovery.json
create mode 100644 deploy/asapo_services_light/file-transfer.json
create mode 100644 deploy/asapo_services_light/monitoring.json
create mode 100644 deploy/asapo_services_light/nginx.conf
create mode 100755 deploy/asapo_services_light/prepare-docker.sh
create mode 100644 deploy/asapo_services_light/receiver.json
create mode 100644 deploy/asapo_services_light/supervisord.conf
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 44100c23b..d524ba410 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -302,6 +302,27 @@ build-docs:
paths:
- public
expire_in: 3 hrs
+
+build-front:
+ image:
+ name: node:16
+ entrypoint: [""]
+ stage: build
+ script:
+ - mkdir -p $CI_PROJECT_DIR/build/monitoring_ui
+ - cd $CI_PROJECT_DIR/build/monitoring_ui
+ - cp -r $CI_PROJECT_DIR/monitoring/monitoring_ui .
+ - npm install
+ - npm run build
+ tags:
+ - kubernetes-executor
+ rules:
+ - when: on_success
+ artifacts:
+ paths:
+ - build
+ expire_in: 3 hrs
+
.build-services-docker-template:
image:
name: gcr.io/kaniko-project/executor:debug
@@ -318,10 +339,14 @@ build-docs:
- /kaniko/executor --cache=false --context $CI_PROJECT_DIR/build/monitoring/monitoring_server --dockerfile $CI_PROJECT_DIR/monitoring/monitoring_server/docker/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-monitoring-server${CONTAINER_TAG}
- /kaniko/executor --cache=false --context $CI_PROJECT_DIR/monitoring/monitoring_ui --dockerfile $CI_PROJECT_DIR/monitoring/monitoring_ui/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-monitoring-ui${CONTAINER_TAG}
- /kaniko/executor --cache=false --context $CI_PROJECT_DIR/build/deploy/asapo_services --dockerfile $CI_PROJECT_DIR/build/deploy/asapo_services/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-cluster${CONTAINER_TAG} --destination $CI_REGISTRY_IMAGE/asapo-cluster${CONTAINER_TAG_LATEST}
+ - cd $CI_PROJECT_DIR/build/deploy/asapo_services_light
+ - ./prepare-docker.sh
+ - /kaniko/executor --cache=false --context $CI_PROJECT_DIR/build/deploy/asapo_services_light --dockerfile $CI_PROJECT_DIR/build/deploy/asapo_services_light/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-standalone${CONTAINER_TAG} --destination $CI_REGISTRY_IMAGE/asapo-standalone${CONTAINER_TAG_LATEST}
tags:
- kubernetes-executor
dependencies:
- build-services-linux-release
+ - build-front
build-services-docker-dev:
extends: .build-services-docker-template
diff --git a/deploy/asapo_services_light/Dockerfile b/deploy/asapo_services_light/Dockerfile
new file mode 100644
index 000000000..1e8863bb1
--- /dev/null
+++ b/deploy/asapo_services_light/Dockerfile
@@ -0,0 +1,47 @@
+FROM debian:bullseye
+
+RUN set -eux; \
+ apt-get update; \
+ apt-get install -y --no-install-recommends \
+ ca-certificates \
+ supervisor \
+ librdkafka++1 \
+ libgrpc++1 \
+ libfabric1 \
+ libmongoc-1.0-0 \
+ nginx \
+ curl \
+ gnupg
+
+RUN curl -fsSL https://pgp.mongodb.com/server-6.0.pub | gpg --dearmor > /usr/share/keyrings/mongodb-server-6.0.gpg
+RUN curl -fsSL https://repos.influxdata.com/influxdata-archive_compat.key | gpg --dearmor > /usr/share/keyrings/influxdb.gpg
+
+RUN echo "deb [signed-by=/usr/share/keyrings/mongodb-server-6.0.gpg] http://repo.mongodb.org/apt/debian bullseye/mongodb-org/6.0 main" > /etc/apt/sources.list.d/mongodb-org-6.0.list
+RUN echo "deb [signed-by=/usr/share/keyrings/influxdb.gpg] https://repos.influxdata.com/debian bullseye stable" > /etc/apt/sources.list.d/influxdb.list
+RUN set -eux; \
+ apt-get update; \
+ apt-get install -y --no-install-recommends \
+ influxdb \
+ mongodb-org; \
+ rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /var/log/supervisord/
+
+COPY asapo-discovery /asapo-discovery
+COPY asapo-authorizer /asapo-authorizer
+COPY asapo-broker /asapo-broker
+COPY asapo-file-transfer /asapo-file-transfer
+COPY receiver /receiver
+COPY asapo-monitoring-server /asapo-monitoring-server
+
+COPY *.json /
+COPY *.key /
+
+COPY nginx.conf /etc/nginx/nginx.conf
+COPY supervisord.conf /etc/
+
+COPY front-dist /usr/share/nginx/html
+
+RUN mkdir -p /tmp/asapo/receiver/files/test_facility/gpfs/test/2019/data/asapo_test/
+
+CMD ["supervisord", "--configuration", "/etc/supervisord.conf"]
diff --git a/deploy/asapo_services_light/auth_secret.key b/deploy/asapo_services_light/auth_secret.key
new file mode 100644
index 000000000..a953ac63b
--- /dev/null
+++ b/deploy/asapo_services_light/auth_secret.key
@@ -0,0 +1 @@
+sadhfi334yxan
\ No newline at end of file
diff --git a/deploy/asapo_services_light/auth_secret_admin.key b/deploy/asapo_services_light/auth_secret_admin.key
new file mode 100644
index 000000000..571ba2fe5
--- /dev/null
+++ b/deploy/asapo_services_light/auth_secret_admin.key
@@ -0,0 +1 @@
+veryverysecretkeyadmin
diff --git a/deploy/asapo_services_light/authorizer.json b/deploy/asapo_services_light/authorizer.json
new file mode 100644
index 000000000..17e3d044d
--- /dev/null
+++ b/deploy/asapo_services_light/authorizer.json
@@ -0,0 +1,22 @@
+{
+ "Port": 8412,
+ "LogLevel":"debug",
+ "RootBeamtimesFolder":"/tmp/asapo/asap3",
+ "CurrentBeamlinesFolder":"/tmp/asapo/beamline",
+ "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","corePath":"/tmp/asapo/receiver/files/test_facility/gpfs/test/2019/data/asapo_test",
+ "beamline-path":"/tmp/asapo/receiver/files/beamline/test/current"},
+ {"beamtimeId":"asapo_test1","beamline":"test1","corePath":"/tmp/asapo/receiver/files/test_facility/gpfs/test1/2019/data/asapo_test1"},
+ {"beamtimeId":"asapo_test2","beamline":"test2","corePath":"/tmp/asapo/receiver/files/test_facility/gpfs/test2/2019/data/asapo_test2"}],
+ "UserSecretFile":"auth_secret.key",
+ "AdminSecretFile":"auth_secret_admin.key",
+ "TokenDurationMin":600,
+ "Ldap":
+ {
+ "Uri" : "ldap://localhost:389",
+ "BaseDn" : "ou=rgy,o=desy,c=de",
+ "FilterTemplate" : "(cn=a3__BEAMLINE__-hosts)"
+ },
+ "DatabaseServer":"auto",
+ "DiscoveryServer": "localhost:8410",
+ "UpdateRevokedTokensIntervalSec": 60
+}
diff --git a/deploy/asapo_services_light/broker.json b/deploy/asapo_services_light/broker.json
new file mode 100644
index 000000000..212e193c1
--- /dev/null
+++ b/deploy/asapo_services_light/broker.json
@@ -0,0 +1,12 @@
+{
+ "DatabaseServer":"auto",
+ "DiscoveryServer": "localhost:8410",
+ "AuthorizationServer": "localhost:8412",
+ "PerformanceDbServer": "localhost:8086",
+ "MonitorPerformance": true,
+ "MonitoringServerUrl":"auto",
+ "CheckResendInterval":0,
+ "PerformanceDbName": "db_test",
+ "Port": 8413,
+ "LogLevel":"debug"
+}
diff --git a/deploy/asapo_services_light/discovery.json b/deploy/asapo_services_light/discovery.json
new file mode 100644
index 000000000..d249f418f
--- /dev/null
+++ b/deploy/asapo_services_light/discovery.json
@@ -0,0 +1,21 @@
+{
+ "Mode": "static",
+ "Receiver": {
+ "MaxConnections": 32,
+ "StaticEndpoints": [ "localhost:8411" ]
+ },
+ "Broker": {
+ "StaticEndpoint": "localhost:8413"
+ },
+ "Mongo": {
+ "StaticEndpoint": "localhost:27017"
+ },
+ "FileTransferService": {
+ "StaticEndpoint": "localhost:8414"
+ },
+ "Monitoring": {
+ "StaticEndpoint": "localhost:8420"
+ },
+ "Port": 8410,
+ "LogLevel": "info"
+}
diff --git a/deploy/asapo_services_light/file-transfer.json b/deploy/asapo_services_light/file-transfer.json
new file mode 100644
index 000000000..5e0d2a204
--- /dev/null
+++ b/deploy/asapo_services_light/file-transfer.json
@@ -0,0 +1,8 @@
+{
+ "Port": 8414,
+ "LogLevel":"debug",
+ "SecretFile":"auth_secret.key",
+ "DiscoveryServer": "localhost:8410",
+ "MonitorPerformance": true,
+ "MonitoringServerUrl": "auto"
+}
diff --git a/deploy/asapo_services_light/monitoring.json b/deploy/asapo_services_light/monitoring.json
new file mode 100644
index 000000000..6149979e8
--- /dev/null
+++ b/deploy/asapo_services_light/monitoring.json
@@ -0,0 +1,7 @@
+{
+ "ThisClusterName": "asapo",
+ "ServerPort": 8420,
+ "LogLevel": "debug",
+ "InfluxDbUrl":"http://localhost:8400/influxdb",
+ "InfluxDbDatabase": "asapo_monitoring"
+}
diff --git a/deploy/asapo_services_light/nginx.conf b/deploy/asapo_services_light/nginx.conf
new file mode 100644
index 000000000..cd752244f
--- /dev/null
+++ b/deploy/asapo_services_light/nginx.conf
@@ -0,0 +1,77 @@
+worker_processes 1;
+
+events {
+ worker_connections 1024;
+}
+
+error_log "/tmp/nginx_error.log";
+pid "/tmp/nginx.pid";
+
+http {
+ client_body_temp_path "/tmp/client_body" 1 2;
+ client_max_body_size 10M;
+ proxy_temp_path "/tmp/proxy" 1 2;
+ fastcgi_temp_path "/tmp/fastcgi" 1 2;
+ scgi_temp_path "/tmp/scgi" 1 2;
+ uwsgi_temp_path "/tmp/uwsgi" 1 2;
+
+ server {
+ listen 8400;
+
+ location /influxdb/ {
+ rewrite ^/influxdb(/.*) $1 break;
+ proxy_pass http://127.0.0.1:8086$uri$is_args$args;
+ }
+
+ location /asapo-discovery/ {
+ rewrite ^/asapo-discovery(/.*) $1 break;
+ proxy_pass http://127.0.0.1:8410$uri$is_args$args;
+ }
+
+ location /performance/ {
+ rewrite ^/performance(/.*) $1 break;
+ proxy_pass http://127.0.0.1:8440$uri$is_args$args;
+ }
+
+ location /tv/ {
+ rewrite ^/tv(/.*) $1 break;
+ proxy_pass http://127.0.0.1:8080$uri$is_args$args;
+ }
+
+ location /tv-api/ {
+ rewrite ^/tv-api(/.*) $1 break;
+ proxy_http_version 1.1;
+ proxy_pass http://127.0.0.1:8421$uri$is_args$args;
+ }
+
+ location /asapo-authorizer/ {
+ rewrite ^/asapo-authorizer(/.*) $1 break;
+ proxy_pass http://127.0.0.1:8412$uri$is_args$args;
+ }
+ }
+
+ server {
+ listen 8080;
+ root /usr/share/nginx/html;
+
+ include /etc/nginx/default.d/*.conf;
+ include mime.types;
+
+ index index.html;
+
+ etag on;
+
+ location /js/ {
+ add_header Cache-Control max-age=31536000;
+ }
+
+ location / {
+ try_files $uri $uri/ /index.html;
+ }
+ location /index.html {
+ add_header Cache-Control no-cache;
+ }
+
+ }
+}
+
diff --git a/deploy/asapo_services_light/prepare-docker.sh b/deploy/asapo_services_light/prepare-docker.sh
new file mode 100755
index 000000000..b218a80a3
--- /dev/null
+++ b/deploy/asapo_services_light/prepare-docker.sh
@@ -0,0 +1,7 @@
+cp ../../discovery/asapo-discovery ./asapo-discovery
+cp ../../authorizer/asapo-authorizer ./asapo-authorizer
+cp ../../broker/asapo-broker ./asapo-broker
+cp ../../file_transfer/asapo-file-transfer ./asapo-file-transfer
+cp ../../receiver/receiver ./receiver
+cp ../../monitoring/monitoring_server/asapo-monitoring-server ./asapo-monitoring-server
+cp -r ../../monitoring_ui/dist ./front-dist
diff --git a/deploy/asapo_services_light/receiver.json b/deploy/asapo_services_light/receiver.json
new file mode 100644
index 000000000..0c2ce7589
--- /dev/null
+++ b/deploy/asapo_services_light/receiver.json
@@ -0,0 +1,33 @@
+{
+ "MonitoringServer": "auto",
+ "PerformanceDbServer":"localhost:8086",
+ "MonitorPerformance": true,
+ "PerformanceDbName": "db_test",
+ "DatabaseServer":"auto",
+ "DiscoveryServer": "localhost:8410",
+ "DataServer": {
+ "AdvertiseURI": "127.0.0.1:8431",
+ "NThreads": 2,
+ "ListenPort": 8431,
+ "NetworkMode": ["tcp"]
+ },
+ "Metrics": {
+ "Expose": true,
+ "ListenPort": 8430
+ },
+ "DataCache": {
+ "Use": true,
+ "SizeGB": 1,
+ "ReservedShare": 10
+ },
+ "AuthorizationServer": "localhost:8412",
+ "AuthorizationInterval": 1000,
+ "ListenPort": 8411,
+ "Tag": "localhost",
+ "ReceiveToDiskThresholdMB":50,
+ "MaxNumPersistedStreams":100,
+ "LogLevel" : "debug",
+ "Kafka" : {
+ "Enabled" : false
+ }
+ }
diff --git a/deploy/asapo_services_light/supervisord.conf b/deploy/asapo_services_light/supervisord.conf
new file mode 100644
index 000000000..9a8b3c548
--- /dev/null
+++ b/deploy/asapo_services_light/supervisord.conf
@@ -0,0 +1,72 @@
+[supervisord]
+nodaemon = true
+logfile=/var/log/supervisord/supervisord.log ; supervisord log file
+logfile_maxbytes=50MB ; maximum size of logfile before rotation
+logfile_backups=10 ; number of backed up logfiles
+loglevel=info ; info, debug, warn, trace
+childlogdir=/var/log/supervisord/ ; where child log files will live
+use=root
+
+[program:mongo]
+command=/usr/bin/mongod --bind_ip_all --dbpath /var/lib/mongodb/
+stderr_logfile=/tmp/mongo_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/mongo_out.log
+stdout_logfile_maxbytes = 0
+
+[program:influxdb]
+command = /usr/bin/influxd -config /etc/influxdb/influxdb.conf
+environment=INFLUXDB_HTTP_FLUX_ENABLED="true", PRE_CREATE_DB="asapo_receivers;asapo_brokers"
+stderr_logfile=/tmp/influx_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/influx_out.log
+stdout_logfile_maxbytes = 0
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+stderr_logfile=/tmp/nginx_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/nginx_out.log
+stdout_logfile_maxbytes = 0
+
+[program:discovery]
+command=/asapo-discovery -config discovery.json
+stderr_logfile=/tmp/discovery_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/discovery_out.log
+stdout_logfile_maxbytes = 0
+
+[program:authorizer]
+command=/asapo-authorizer -config authorizer.json
+stderr_logfile=/tmp/authorizer_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/authorizer_out.log
+stdout_logfile_maxbytes = 0
+
+[program:broker]
+command=/asapo-broker -config broker.json
+stderr_logfile=/tmp/broker_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/broker_out.log
+stdout_logfile_maxbytes = 0
+
+[program:fts]
+command=/asapo-file-transfer -config file-transfer.json
+stderr_logfile=/tmp/fts_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/fts_out.log
+stdout_logfile_maxbytes = 0
+
+[program:receiver]
+command=/receiver receiver.json
+stderr_logfile=/tmp/receiver_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/receiver_out.log
+stdout_logfile_maxbytes = 0
+
+[program:monitoring]
+command=/asapo-monitoring-server monitoring.json
+stderr_logfile=/tmp/monitoring_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/monitoring_out.log
+stdout_logfile_maxbytes = 0
--
GitLab