From dd28b0f86ea8bcaa2228b94e5722354528ab46dc Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Tue, 31 Mar 2020 15:40:00 +0200
Subject: [PATCH] discovery
---
.../asapo/configs/asapo-broker.json | 2 +-
.../asapo/configs/asapo-discovery.json | 23 +++++++++
.../asapo/configs/asapo-file-transfer.json | 5 ++
.../asapo/configs/asapo-fluentd.conf | 1 +
.../asapo/configs/asapo-receiver.json | 24 +++++++++
.../asapo/templates/_service.tpl | 2 +-
.../asapo/templates/authorizer-ingress.yaml | 2 -
.../asapo/templates/configs.yaml | 4 --
.../asapo/templates/discovery-deployment.yaml | 35 +++++++++++++
.../templates/file-transfer-deployment.yaml | 50 +++++++++++++++++++
.../asapo/templates/ingress.yaml | 7 +++
.../asapo/templates/receiver-deployment.yaml | 48 ++++++++++++++++++
.../asapo/templates/services.yaml | 6 +--
deploy/asapo_helm_chart/asapo/values.yaml | 23 +++++++++
14 files changed, 220 insertions(+), 12 deletions(-)
create mode 100644 deploy/asapo_helm_chart/asapo/configs/asapo-discovery.json
create mode 100644 deploy/asapo_helm_chart/asapo/configs/asapo-file-transfer.json
create mode 100644 deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json
delete mode 100644 deploy/asapo_helm_chart/asapo/templates/authorizer-ingress.yaml
create mode 100644 deploy/asapo_helm_chart/asapo/templates/discovery-deployment.yaml
create mode 100644 deploy/asapo_helm_chart/asapo/templates/file-transfer-deployment.yaml
create mode 100644 deploy/asapo_helm_chart/asapo/templates/ingress.yaml
create mode 100644 deploy/asapo_helm_chart/asapo/templates/receiver-deployment.yaml
diff --git a/deploy/asapo_helm_chart/asapo/configs/asapo-broker.json b/deploy/asapo_helm_chart/asapo/configs/asapo-broker.json
index 10441cd9f..d59aaed6d 100644
--- a/deploy/asapo_helm_chart/asapo/configs/asapo-broker.json
+++ b/deploy/asapo_helm_chart/asapo/configs/asapo-broker.json
@@ -1,6 +1,6 @@
{
"DatabaseServer":"auto",
- "DiscoveryServer": "discovery:{{ .Values.ownServices.discovery.port }}",
+ "DiscoveryServer": "asapo-discovery:{{ .Values.ownServices.discovery.port }}",
"PerformanceDbServer":"{{ .Chart.Name }}-influxdb:{{ .Values.influxdb.influxdb.service.port }}",
"PerformanceDbName": "asapo_brokers",
"Port": {{ .Values.ownServices.broker.port }},
diff --git a/deploy/asapo_helm_chart/asapo/configs/asapo-discovery.json b/deploy/asapo_helm_chart/asapo/configs/asapo-discovery.json
new file mode 100644
index 000000000..fdd032094
--- /dev/null
+++ b/deploy/asapo_helm_chart/asapo/configs/asapo-discovery.json
@@ -0,0 +1,23 @@
+{
+ "Mode": "static",
+ "Receiver": {
+ "StaticEndpoints": [
+ "127.0.0.1:22001"
+ ],
+ "MaxConnections": 32,
+ "UseIBAddress": false
+ },
+ "Broker": {
+ "StaticEndpoint": "127.0.0.1:5005"
+ },
+ "Mongo": {
+ "StaticEndpoint": "127.0.0.1:27017"
+ },
+ "FileTransferService": {
+ "StaticEndpoint": "127.0.0.1:27017"
+ },
+ "Port": {{ .Values.ownServices.discovery.port }},
+ "LogLevel": "debug"
+}
+
+
diff --git a/deploy/asapo_helm_chart/asapo/configs/asapo-file-transfer.json b/deploy/asapo_helm_chart/asapo/configs/asapo-file-transfer.json
new file mode 100644
index 000000000..49cc9bfc3
--- /dev/null
+++ b/deploy/asapo_helm_chart/asapo/configs/asapo-file-transfer.json
@@ -0,0 +1,5 @@
+{
+ "Port": {{ .Values.ownServices.fileTransfer.port }},
+ "LogLevel":"debug",
+ "SecretFile":"/etc/file-transfer/auth_secret.key"
+}
diff --git a/deploy/asapo_helm_chart/asapo/configs/asapo-fluentd.conf b/deploy/asapo_helm_chart/asapo/configs/asapo-fluentd.conf
index 516f52d8a..2ba55b940 100644
--- a/deploy/asapo_helm_chart/asapo/configs/asapo-fluentd.conf
+++ b/deploy/asapo_helm_chart/asapo/configs/asapo-fluentd.conf
@@ -2,6 +2,7 @@
@type tail
path /var/log/containers/{{ .service.serviceName }}*{{ .service.serviceName }}*.log
pos_file /tmp/{{ .service.serviceName }}.log.pos
+read_from_head true
tag asapo
<parse>
@type json
diff --git a/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json b/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json
new file mode 100644
index 000000000..1c0d4439c
--- /dev/null
+++ b/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json
@@ -0,0 +1,24 @@
+{
+ "AdvertiseIP": "auto",
+ "PerformanceDbServer": "{{ .Chart.Name }}-influxdb:{{ .Values.influxdb.influxdb.service.port }}",
+ "PerformanceDbName": "asapo_receivers",
+ "DatabaseServer": "auto",
+ "DiscoveryServer": "asapo-discovery:{{ .Values.ownServices.discovery.port }}",
+ "AuthorizationServer": "asapo-authorizer:{{ .Values.ownServices.authorizer.port }}",
+ "AuthorizationInterval": 10000,
+ "ListenPort": {{ .Values.ownServices.receiver.port }},
+ "DataServer": {
+ "NThreads": {{ .Values.ownServices.receiver.dataServer.nThreads }},
+ "ListenPort": {{ .Values.ownServices.receiver.dataServer.port }}
+ },
+ "DataCache": {
+ "Use": {{ .Values.ownServices.receiver.dataCache.enable }},
+ "SizeGB": {{ .Values.ownServices.receiver.dataCache.sizeGb }},
+ "ReservedShare": 10
+ },
+ "Tag": "receiver",
+ "WriteToDisk":true,
+ "ReceiveToDiskThresholdMB": {{ .Values.ownServices.receiver.receiveToDiskThresholdMB }},
+ "WriteToDb":true,
+ "LogLevel": "info"
+}
diff --git a/deploy/asapo_helm_chart/asapo/templates/_service.tpl b/deploy/asapo_helm_chart/asapo/templates/_service.tpl
index c766111db..09cb95b02 100644
--- a/deploy/asapo_helm_chart/asapo/templates/_service.tpl
+++ b/deploy/asapo_helm_chart/asapo/templates/_service.tpl
@@ -10,5 +10,5 @@ spec:
- protocol: TCP
port: {{ .service.port }}
selector:
- app: {{ .service.serviceName }}
+ app: {{ .service.appName | default .service.serviceName }}
{{- end }}
diff --git a/deploy/asapo_helm_chart/asapo/templates/authorizer-ingress.yaml b/deploy/asapo_helm_chart/asapo/templates/authorizer-ingress.yaml
deleted file mode 100644
index 08bc7107b..000000000
--- a/deploy/asapo_helm_chart/asapo/templates/authorizer-ingress.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-{{ $data := dict "Release" .Release "Values" .Values "service" .Values.ownServices.authorizer }}
-{{- template "asapo.ingress" $data }}
diff --git a/deploy/asapo_helm_chart/asapo/templates/configs.yaml b/deploy/asapo_helm_chart/asapo/templates/configs.yaml
index 4c6fcd30b..6dd1c5c49 100644
--- a/deploy/asapo_helm_chart/asapo/templates/configs.yaml
+++ b/deploy/asapo_helm_chart/asapo/templates/configs.yaml
@@ -1,9 +1,5 @@
-{{ $servicesToConfig := list .Values.ownServices.authorizer.serviceName .Values.ownServices.broker.serviceName }}
-
{{- range .Values.ownServices }}
{{ $data := dict "Chart" $.Chart "Template" $.Template "Values" $.Values "Files" $.Files "service" . }}
- {{- if has .serviceName $servicesToConfig }}
{{- template "asapo.configmap-fromfile" $data }}
---
- {{- end }}
{{- end }}
diff --git a/deploy/asapo_helm_chart/asapo/templates/discovery-deployment.yaml b/deploy/asapo_helm_chart/asapo/templates/discovery-deployment.yaml
new file mode 100644
index 000000000..7cee3709d
--- /dev/null
+++ b/deploy/asapo_helm_chart/asapo/templates/discovery-deployment.yaml
@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: asapo-discovery
+ labels:
+ app: asapo-discovery
+spec:
+ replicas: {{ .Values.ownServices.discovery.replicaCount }}
+ selector:
+ matchLabels:
+ app: asapo-discovery
+ template:
+ metadata:
+ labels:
+ app: asapo-discovery
+ annotations:
+ checksum/config: {{ .Files.Get "configs/asapo-discovery.json" | sha256sum }}
+ checksum/fluentd-config: {{ .Files.Get "configs/asapo-fluentd.conf" | sha256sum }}
+ spec:
+ volumes:
+ - name: asapo-discovery-config
+ configMap:
+ name: asapo-discovery-config
+ {{- include "asapo.fluentd.volumes" .Values.ownServices.discovery | indent 8 }}
+ containers:
+ - name: asapo-discovery
+ image: "yakser/asapo-discovery-dev:{{ .Values.common.asapoVersionTag }}"
+ command: ["/asapo-discovery"]
+ args: ["-config", "/etc/discovery/asapo-discovery.json"]
+ ports:
+ - containerPort: {{ .Values.ownServices.discovery.port }}
+ volumeMounts:
+ - mountPath: "/etc/discovery"
+ name: asapo-discovery-config
+ {{- include "asapo.fluentd.container" . | indent 8 }}
\ No newline at end of file
diff --git a/deploy/asapo_helm_chart/asapo/templates/file-transfer-deployment.yaml b/deploy/asapo_helm_chart/asapo/templates/file-transfer-deployment.yaml
new file mode 100644
index 000000000..04e5d502f
--- /dev/null
+++ b/deploy/asapo_helm_chart/asapo/templates/file-transfer-deployment.yaml
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: asapo-file-transfer
+ labels:
+ app: asapo-file-transfer
+spec:
+ replicas: {{ .Values.ownServices.fileTransfer.replicaCount }}
+ selector:
+ matchLabels:
+ app: asapo-file-transfer
+ template:
+ metadata:
+ labels:
+ app: asapo-file-transfer
+ annotations:
+ checksum/config: {{ .Files.Get "configs/asapo-file-transfer.json" | sha256sum }}
+ checksum/secret: {{ include (print $.Template.BasePath "/auth-secret.yaml") . | sha256sum }}
+ checksum/fluentd-config: {{ .Files.Get "configs/asapo-fluentd.conf" | sha256sum }}
+ spec:
+ volumes:
+ - name: all-in-one
+ projected:
+ sources:
+ - configMap:
+ name: asapo-file-transfer-config
+ - secret:
+ name: auth-secret
+ - name: shared-volume-offline
+ persistentVolumeClaim:
+ claimName: asapo-offline-pv
+ - name: shared-volume-online
+ persistentVolumeClaim:
+ claimName: asapo-online-pv
+ {{- include "asapo.fluentd.volumes" .Values.ownServices.fileTransfer | indent 8 }}
+ containers:
+ - name: asapo-file-transfer
+ image: "yakser/asapo-file-transfer-dev:{{ .Values.common.asapoVersionTag }}"
+ command: ["/asapo-file-transfer"]
+ args: ["-config", "/etc/file-transfer/asapo-file-transfer.json"]
+ ports:
+ - containerPort: {{ .Values.ownServices.fileTransfer.port }}
+ volumeMounts:
+ - mountPath: "/etc/file-transfer"
+ name: all-in-one
+ - mountPath: {{ .Values.common.offlineDir }}
+ name: shared-volume-offline
+ - mountPath: {{ .Values.common.onlineDir }}
+ name: shared-volume-online
+ {{- include "asapo.fluentd.container" . | indent 8 }}
\ No newline at end of file
diff --git a/deploy/asapo_helm_chart/asapo/templates/ingress.yaml b/deploy/asapo_helm_chart/asapo/templates/ingress.yaml
new file mode 100644
index 000000000..9efb8c0ac
--- /dev/null
+++ b/deploy/asapo_helm_chart/asapo/templates/ingress.yaml
@@ -0,0 +1,7 @@
+{{- range .Values.ownServices }}
+ {{- if not ._exposeServiceExtrernally }}
+ {{- $data := dict "Release" $.Release "Values" $.Values "service" . }}
+ {{- template "asapo.ingress" $data }}
+ {{- end }}
+---
+{{- end }}
diff --git a/deploy/asapo_helm_chart/asapo/templates/receiver-deployment.yaml b/deploy/asapo_helm_chart/asapo/templates/receiver-deployment.yaml
new file mode 100644
index 000000000..bcf8c2e10
--- /dev/null
+++ b/deploy/asapo_helm_chart/asapo/templates/receiver-deployment.yaml
@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: asapo-receiver
+ labels:
+ app: asapo-receiver
+spec:
+ replicas: {{ .Values.ownServices.receiver.replicaCount }}
+ selector:
+ matchLabels:
+ app: asapo-receiver
+ template:
+ metadata:
+ labels:
+ app: asapo-receiver
+ annotations:
+ checksum/config: {{ .Files.Get "configs/asapo-receiver.json" | sha256sum }}
+ checksum/fluentd-config: {{ .Files.Get "configs/asapo-fluentd.conf" | sha256sum }}
+ spec:
+ volumes:
+ - name: asapo-receiver-config
+ configMap:
+ name: asapo-receiver-config
+ - name: shared-volume-offline
+ persistentVolumeClaim:
+ claimName: asapo-offline-pv
+ - name: shared-volume-online
+ persistentVolumeClaim:
+ claimName: asapo-online-pv
+ {{- include "asapo.fluentd.volumes" .Values.ownServices.receiver | indent 8 }}
+ containers:
+ - name: asapo-receiver
+ image: "yakser/asapo-receiver-dev:{{ .Values.common.asapoVersionTag }}"
+ command: ["/receiver"]
+ args: ["/etc/receiver/asapo-receiver.json"]
+ ports:
+ - containerPort: {{ .Values.ownServices.receiver.port }}
+ name: http
+ - containerPort: {{ .Values.ownServices.receiver.dataServer.port }}
+ name: http-ds
+ volumeMounts:
+ - mountPath: "/etc/receiver"
+ name: asapo-receiver-config
+ - mountPath: {{ .Values.common.offlineDir }}
+ name: shared-volume-offline
+ - mountPath: {{ .Values.common.onlineDir }}
+ name: shared-volume-online
+ {{- include "asapo.fluentd.container" . | indent 8 }}
\ No newline at end of file
diff --git a/deploy/asapo_helm_chart/asapo/templates/services.yaml b/deploy/asapo_helm_chart/asapo/templates/services.yaml
index c40f4a530..fb7f80661 100644
--- a/deploy/asapo_helm_chart/asapo/templates/services.yaml
+++ b/deploy/asapo_helm_chart/asapo/templates/services.yaml
@@ -1,9 +1,7 @@
-{{ $servicesToConfig := list .Values.ownServices.authorizer.serviceName .Values.ownServices.broker.serviceName }}
-
{{- range .Values.ownServices }}
{{ $data := dict "Values" $.Values "service" . }}
- {{- if has .serviceName $servicesToConfig }}
{{- template "asapo.service" $data }}
---
- {{- end }}
{{- end }}
+{{ $data := dict "Values" $.Values "service" .Values.ownServices.receiver.dataServer }}
+{{- template "asapo.service" $data }}
diff --git a/deploy/asapo_helm_chart/asapo/values.yaml b/deploy/asapo_helm_chart/asapo/values.yaml
index d65c1ee1d..890ecf098 100644
--- a/deploy/asapo_helm_chart/asapo/values.yaml
+++ b/deploy/asapo_helm_chart/asapo/values.yaml
@@ -16,6 +16,29 @@ ownServices:
replicaCount: 1
port: 5008
_exposeServiceExtrernally: false
+ receiver:
+ serviceName: asapo-receiver
+ replicaCount: 1
+ port: 5009
+ sidecarLogs: true
+ _exposeServiceExtrernally: true
+ dataServer:
+ appName: asapo-receiver
+ serviceName: asapo-receiver-dataserver
+ port: 5010
+ nThreads: 1
+ _exposeServiceExtrernally: true
+ dataCache:
+ enable: false
+ sizeGb: 1
+ receiveToDiskThresholdMB: 200
+ fileTransfer:
+ serviceName: asapo-file-transfer
+ replicaCount: 1
+ port: 5011
+ sidecarLogs: true
+ _exposeServiceExtrernally: true
+
common:
authSecret: "12ljzgneasfd"
--
GitLab