add asapo standalone package

.gitlab-ci.yml

@@ -302,6 +302,27 @@ build-docs:
     paths:
       - public
     expire_in: 3 hrs
+
+build-front:
+  image:
+    name: node:16
+    entrypoint: [""]
+  stage: build
+  script:
+    - mkdir -p $CI_PROJECT_DIR/build/monitoring_ui
+    - cd $CI_PROJECT_DIR/build/monitoring_ui
+    - cp -r $CI_PROJECT_DIR/monitoring/monitoring_ui .
+    - npm install
+    - npm run build
+  tags:
+    - kubernetes-executor
+  rules:
+    - when: on_success
+  artifacts:
+    paths:
+      - build
+    expire_in: 3 hrs
+    
 .build-services-docker-template:
   image:
     name: gcr.io/kaniko-project/executor:debug
@@ -318,10 +339,14 @@ build-docs:
     - /kaniko/executor --cache=false --context $CI_PROJECT_DIR/build/monitoring/monitoring_server --dockerfile $CI_PROJECT_DIR/monitoring/monitoring_server/docker/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-monitoring-server${CONTAINER_TAG}
     - /kaniko/executor --cache=false --context $CI_PROJECT_DIR/monitoring/monitoring_ui --dockerfile $CI_PROJECT_DIR/monitoring/monitoring_ui/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-monitoring-ui${CONTAINER_TAG}
     - /kaniko/executor --cache=false --context $CI_PROJECT_DIR/build/deploy/asapo_services --dockerfile $CI_PROJECT_DIR/build/deploy/asapo_services/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-cluster${CONTAINER_TAG} --destination $CI_REGISTRY_IMAGE/asapo-cluster${CONTAINER_TAG_LATEST}
+    - cd $CI_PROJECT_DIR/build/deploy/asapo_services_light
+    - ./prepare-docker.sh
+    - /kaniko/executor --cache=false --context $CI_PROJECT_DIR/build/deploy/asapo_services_light --dockerfile $CI_PROJECT_DIR/build/deploy/asapo_services_light/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-standalone${CONTAINER_TAG} --destination $CI_REGISTRY_IMAGE/asapo-standalone${CONTAINER_TAG_LATEST}
   tags:
     - kubernetes-executor
   dependencies:
     - build-services-linux-release
+    - build-front
 
 build-services-docker-dev:
   extends: .build-services-docker-template

deploy/asapo_services_light/Dockerfile

+FROM debian:bullseye
+
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+	    ca-certificates \
+    	supervisor \
+	    librdkafka++1 \
+	    libgrpc++1 \
+		libfabric1 \
+		libmongoc-1.0-0 \
+		nginx \
+		curl \
+		gnupg
+
+RUN curl -fsSL https://pgp.mongodb.com/server-6.0.pub | gpg --dearmor > /usr/share/keyrings/mongodb-server-6.0.gpg
+RUN curl -fsSL https://repos.influxdata.com/influxdata-archive_compat.key | gpg --dearmor > /usr/share/keyrings/influxdb.gpg
+
+RUN echo "deb [signed-by=/usr/share/keyrings/mongodb-server-6.0.gpg] http://repo.mongodb.org/apt/debian bullseye/mongodb-org/6.0 main" > /etc/apt/sources.list.d/mongodb-org-6.0.list
+RUN echo "deb [signed-by=/usr/share/keyrings/influxdb.gpg] https://repos.influxdata.com/debian bullseye stable" > /etc/apt/sources.list.d/influxdb.list
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+	    influxdb \
+	    mongodb-org; \
+	rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /var/log/supervisord/
+
+COPY asapo-discovery /asapo-discovery
+COPY asapo-authorizer /asapo-authorizer
+COPY asapo-broker /asapo-broker
+COPY asapo-file-transfer /asapo-file-transfer
+COPY receiver /receiver
+COPY asapo-monitoring-server /asapo-monitoring-server
+
+COPY *.json /
+COPY *.key /
+
+COPY nginx.conf /etc/nginx/nginx.conf
+COPY supervisord.conf /etc/
+
+COPY front-dist /usr/share/nginx/html
+
+RUN mkdir -p /tmp/asapo/receiver/files/test_facility/gpfs/test/2019/data/asapo_test/
+
+CMD ["supervisord", "--configuration", "/etc/supervisord.conf"]

deploy/asapo_services_light/auth_secret.key

+sadhfi334yxan
\ No newline at end of file

deploy/asapo_services_light/auth_secret_admin.key

+veryverysecretkeyadmin

deploy/asapo_services_light/authorizer.json

+{
+  "Port": 8412,
+  "LogLevel":"debug",
+  "RootBeamtimesFolder":"/tmp/asapo/asap3",
+  "CurrentBeamlinesFolder":"/tmp/asapo/beamline",
+  "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","corePath":"/tmp/asapo/receiver/files/test_facility/gpfs/test/2019/data/asapo_test",
+  "beamline-path":"/tmp/asapo/receiver/files/beamline/test/current"},
+  {"beamtimeId":"asapo_test1","beamline":"test1","corePath":"/tmp/asapo/receiver/files/test_facility/gpfs/test1/2019/data/asapo_test1"},
+  {"beamtimeId":"asapo_test2","beamline":"test2","corePath":"/tmp/asapo/receiver/files/test_facility/gpfs/test2/2019/data/asapo_test2"}],
+  "UserSecretFile":"auth_secret.key",
+  "AdminSecretFile":"auth_secret_admin.key",
+  "TokenDurationMin":600,
+  "Ldap":
+  {
+     "Uri" : "ldap://localhost:389",
+     "BaseDn" : "ou=rgy,o=desy,c=de",
+     "FilterTemplate" : "(cn=a3__BEAMLINE__-hosts)"
+  },
+  "DatabaseServer":"auto",
+  "DiscoveryServer": "localhost:8410",
+  "UpdateRevokedTokensIntervalSec": 60
+}

deploy/asapo_services_light/broker.json

+{
+  "DatabaseServer":"auto",
+  "DiscoveryServer": "localhost:8410",
+  "AuthorizationServer": "localhost:8412",
+  "PerformanceDbServer": "localhost:8086",
+  "MonitorPerformance": true,
+  "MonitoringServerUrl":"auto",
+  "CheckResendInterval":0,
+  "PerformanceDbName": "db_test",
+  "Port": 8413,
+  "LogLevel":"debug"
+}

deploy/asapo_services_light/discovery.json

+{
+  "Mode": "static",
+  "Receiver": {
+    "MaxConnections": 32,
+    "StaticEndpoints": [ "localhost:8411" ]
+  },
+  "Broker": {
+    "StaticEndpoint": "localhost:8413"
+  },
+  "Mongo": {
+    "StaticEndpoint": "localhost:27017"
+  },
+  "FileTransferService": {
+    "StaticEndpoint": "localhost:8414"
+  },
+  "Monitoring": {
+    "StaticEndpoint": "localhost:8420"
+  },
+  "Port": 8410,
+  "LogLevel": "info"
+}

deploy/asapo_services_light/file-transfer.json

+{
+  "Port": 8414,
+  "LogLevel":"debug",
+  "SecretFile":"auth_secret.key",
+  "DiscoveryServer": "localhost:8410",
+  "MonitorPerformance": true,
+  "MonitoringServerUrl": "auto"
+}

deploy/asapo_services_light/monitoring.json

+{
+    "ThisClusterName": "asapo",
+    "ServerPort": 8420,
+    "LogLevel": "debug",
+    "InfluxDbUrl":"http://localhost:8400/influxdb",
+    "InfluxDbDatabase": "asapo_monitoring"
+}

deploy/asapo_services_light/nginx.conf

+worker_processes  1;
+
+events {
+    worker_connections  1024;
+}
+
+error_log         "/tmp/nginx_error.log";
+pid               "/tmp/nginx.pid";
+
+http {
+    client_body_temp_path  "/tmp/client_body" 1 2;
+    client_max_body_size   10M;
+    proxy_temp_path        "/tmp/proxy" 1 2;
+    fastcgi_temp_path      "/tmp/fastcgi" 1 2;
+    scgi_temp_path         "/tmp/scgi" 1 2;
+    uwsgi_temp_path        "/tmp/uwsgi" 1 2;
+
+    server {
+          listen 8400;
+
+          location /influxdb/ {
+            rewrite ^/influxdb(/.*) $1 break;
+            proxy_pass http://127.0.0.1:8086$uri$is_args$args;
+          }
+
+          location /asapo-discovery/ {
+            rewrite ^/asapo-discovery(/.*) $1 break;
+            proxy_pass http://127.0.0.1:8410$uri$is_args$args;
+          }
+
+          location /performance/ {
+            rewrite ^/performance(/.*) $1 break;
+            proxy_pass http://127.0.0.1:8440$uri$is_args$args;
+          }
+
+          location /tv/ {
+            rewrite ^/tv(/.*) $1 break;
+            proxy_pass http://127.0.0.1:8080$uri$is_args$args;
+          }
+
+          location /tv-api/ {
+            rewrite ^/tv-api(/.*) $1 break;
+            proxy_http_version 1.1;
+            proxy_pass http://127.0.0.1:8421$uri$is_args$args;
+          }
+
+          location /asapo-authorizer/ {
+             rewrite ^/asapo-authorizer(/.*) $1 break;
+             proxy_pass http://127.0.0.1:8412$uri$is_args$args;
+          }
+    }
+
+    server {
+        listen       8080;
+        root         /usr/share/nginx/html;
+
+        include /etc/nginx/default.d/*.conf;
+        include       mime.types;
+
+        index index.html;
+
+        etag on;
+
+        location /js/ {
+          add_header Cache-Control max-age=31536000;
+        }
+
+        location / {
+          try_files $uri $uri/ /index.html;
+        }
+        location /index.html {
+          add_header Cache-Control no-cache;
+        }
+
+    }
+}
+

deploy/asapo_services_light/prepare-docker.sh

+cp ../../discovery/asapo-discovery ./asapo-discovery
+cp ../../authorizer/asapo-authorizer ./asapo-authorizer
+cp ../../broker/asapo-broker ./asapo-broker
+cp ../../file_transfer/asapo-file-transfer ./asapo-file-transfer
+cp ../../receiver/receiver ./receiver
+cp ../../monitoring/monitoring_server/asapo-monitoring-server ./asapo-monitoring-server
+cp -r ../../monitoring_ui/dist ./front-dist

deploy/asapo_services_light/receiver.json

+{
+  "MonitoringServer": "auto",
+  "PerformanceDbServer":"localhost:8086",
+  "MonitorPerformance": true,
+  "PerformanceDbName": "db_test",
+  "DatabaseServer":"auto",
+  "DiscoveryServer": "localhost:8410",
+  "DataServer": {
+    "AdvertiseURI": "127.0.0.1:8431",
+    "NThreads": 2,
+    "ListenPort": 8431,
+    "NetworkMode": ["tcp"]
+  },
+  "Metrics": {
+    "Expose": true,
+    "ListenPort": 8430
+  },
+  "DataCache": {
+    "Use": true,
+    "SizeGB": 1,
+    "ReservedShare": 10
+  },
+  "AuthorizationServer": "localhost:8412",
+  "AuthorizationInterval": 1000,
+  "ListenPort": 8411,
+  "Tag": "localhost",
+  "ReceiveToDiskThresholdMB":50,
+  "MaxNumPersistedStreams":100,
+  "LogLevel" : "debug",
+  "Kafka" : {
+    "Enabled" : false
+  }
+  }

deploy/asapo_services_light/supervisord.conf

+[supervisord]
+nodaemon = true
+logfile=/var/log/supervisord/supervisord.log    ; supervisord log file
+logfile_maxbytes=50MB                           ; maximum size of logfile before rotation
+logfile_backups=10                              ; number of backed up logfiles
+loglevel=info                                   ; info, debug, warn, trace
+childlogdir=/var/log/supervisord/               ; where child log files will live
+use=root
+
+[program:mongo]
+command=/usr/bin/mongod --bind_ip_all --dbpath /var/lib/mongodb/
+stderr_logfile=/tmp/mongo_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/mongo_out.log
+stdout_logfile_maxbytes = 0
+
+[program:influxdb]
+command = /usr/bin/influxd -config /etc/influxdb/influxdb.conf
+environment=INFLUXDB_HTTP_FLUX_ENABLED="true", PRE_CREATE_DB="asapo_receivers;asapo_brokers"
+stderr_logfile=/tmp/influx_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/influx_out.log
+stdout_logfile_maxbytes = 0
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+stderr_logfile=/tmp/nginx_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/nginx_out.log
+stdout_logfile_maxbytes = 0
+
+[program:discovery]
+command=/asapo-discovery -config discovery.json
+stderr_logfile=/tmp/discovery_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/discovery_out.log
+stdout_logfile_maxbytes = 0
+
+[program:authorizer]
+command=/asapo-authorizer -config authorizer.json
+stderr_logfile=/tmp/authorizer_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/authorizer_out.log
+stdout_logfile_maxbytes = 0
+
+[program:broker]
+command=/asapo-broker -config broker.json
+stderr_logfile=/tmp/broker_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/broker_out.log
+stdout_logfile_maxbytes = 0
+
+[program:fts]
+command=/asapo-file-transfer -config file-transfer.json
+stderr_logfile=/tmp/fts_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/fts_out.log
+stdout_logfile_maxbytes = 0
+
+[program:receiver]
+command=/receiver receiver.json
+stderr_logfile=/tmp/receiver_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/receiver_out.log
+stdout_logfile_maxbytes = 0
+
+[program:monitoring]
+command=/asapo-monitoring-server monitoring.json
+stderr_logfile=/tmp/monitoring_err.log
+stderr_logfile_maxbytes = 0
+stdout_logfile=/tmp/monitoring_out.log
+stdout_logfile_maxbytes = 0