From 8d6fdd33ed4d0d57c63127df4ff4e5051da35a55 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon, 6 Apr 2020 18:49:35 +0200
Subject: [PATCH] change AdvertiseIP->AdvertiseURI, update helm chart to set
dataservice AdvertiseURI
---
.../asapo/configs/asapo-receiver.json | 3 +-
.../asapo/templates/receiver-deployment.yaml | 38 +++++++++++++++++++
.../asapo/templates/roles.yaml | 26 +++++++++++++
deploy/asapo_helm_chart/asapo/values.yaml | 2 +-
.../asapo_services/scripts/receiver.json.tpl | 2 +-
receiver/src/receiver_config.cpp | 2 +-
receiver/src/receiver_config.h | 1 -
.../receiver_datacenter_config.h | 1 +
receiver/src/request_handler_db_write.cpp | 3 +-
receiver/unittests/mock_receiver_config.cpp | 2 +-
receiver/unittests/test_config.cpp | 7 ++--
.../test_request_handler_db_check_request.cpp | 6 +--
.../test_request_handler_db_writer.cpp | 2 +-
.../settings/receiver.json.tpl.lin.in | 2 +-
.../settings/receiver.json.tpl.win.in | 2 +-
tests/manual/broker_debug_local/receiver.json | 2 +-
.../broker_debug_local/receiver.json.tpl | 2 +-
.../python_tests/producer/receiver.json.tpl | 2 +-
.../producer_wait_bug_mongo/receiver.json.tpl | 2 +-
.../manual/receiver_debug_local/receiver.json | 2 +-
20 files changed, 86 insertions(+), 23 deletions(-)
diff --git a/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json b/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json
index 990fe7465..4e13a308b 100644
--- a/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json
+++ b/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json
@@ -1,5 +1,4 @@
{
- "AdvertiseIP": "auto",
"PerformanceDbServer": "{{ .Chart.Name }}-influxdb:{{ .Values.influxdb.influxdb.service.port }}",
"PerformanceDbName": "asapo_receivers",
"DatabaseServer": "asapo-mongodb:{{ .Values.ownServices.mongodb.port }}",
@@ -7,7 +6,9 @@
"AuthorizationServer": "asapo-authorizer:{{ .Values.ownServices.authorizer.port }}",
"AuthorizationInterval": 10000,
"ListenPort": {{ .Values.ownServices.receiver.port }},
+ "AdvertiseIP": "auto",
"DataServer": {
+ "AdvertiseURI": "auto",
"NThreads": {{ .Values.ownServices.receiver.dataServer.nThreads }},
"ListenPort": {{ .Values.ownServices.receiver.dataServer.port }}
},
diff --git a/deploy/asapo_helm_chart/asapo/templates/receiver-deployment.yaml b/deploy/asapo_helm_chart/asapo/templates/receiver-deployment.yaml
index bcf8c2e10..144feeaf7 100644
--- a/deploy/asapo_helm_chart/asapo/templates/receiver-deployment.yaml
+++ b/deploy/asapo_helm_chart/asapo/templates/receiver-deployment.yaml
@@ -1,3 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: config-script
+data:
+ entrypoint.sh: |-
+ #!/bin/bash
+ export TOKEN=`cat /var/run/secrets/kubernetes.io/serviceaccount/token`
+ export NODE_PORT=`curl https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/{{ .Release.Namespace }}/services/asapo-receiver-dataserver --insecure --header "Authorization: Bearer $TOKEN" --silent | jq .spec.ports[0].nodePort`
+ jq ".DataServer.AdvertiseURI = \"$NODE_IP:$NODE_PORT\"" /etc/receiver/init/asapo-receiver.json > /etc/receiver/asapo-receiver.json
+---
+
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -18,7 +30,13 @@ spec:
checksum/fluentd-config: {{ .Files.Get "configs/asapo-fluentd.conf" | sha256sum }}
spec:
volumes:
+ - name: config-script
+ configMap:
+ defaultMode: 0700
+ name: config-script
- name: asapo-receiver-config
+ emptyDir: {}
+ - name: asapo-receiver-config-init
configMap:
name: asapo-receiver-config
- name: shared-volume-offline
@@ -28,6 +46,25 @@ spec:
persistentVolumeClaim:
claimName: asapo-online-pv
{{- include "asapo.fluentd.volumes" .Values.ownServices.receiver | indent 8 }}
+ initContainers:
+ - name: prepare-config
+ image: yakser/alpine_curl_jq
+ command: ['/bin/entrypoint.sh']
+ env:
+ - name: NODE_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.hostIP
+ volumeMounts:
+ - mountPath: "/etc/receiver"
+ name: asapo-receiver-config
+ - mountPath: "/etc/receiver/init"
+ name: asapo-receiver-config-init
+ - mountPath: /bin/entrypoint.sh
+ name: config-script
+ readOnly: true
+ subPath: entrypoint.sh
+
containers:
- name: asapo-receiver
image: "yakser/asapo-receiver-dev:{{ .Values.common.asapoVersionTag }}"
@@ -45,4 +82,5 @@ spec:
name: shared-volume-offline
- mountPath: {{ .Values.common.onlineDir }}
name: shared-volume-online
+
{{- include "asapo.fluentd.container" . | indent 8 }}
\ No newline at end of file
diff --git a/deploy/asapo_helm_chart/asapo/templates/roles.yaml b/deploy/asapo_helm_chart/asapo/templates/roles.yaml
index e69de29bb..e412266d9 100644
--- a/deploy/asapo_helm_chart/asapo/templates/roles.yaml
+++ b/deploy/asapo_helm_chart/asapo/templates/roles.yaml
@@ -0,0 +1,26 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ creationTimestamp: null
+ name: role
+rules:
+ - apiGroups:
+ - ""
+ resources:
+ - services
+ verbs:
+ - get
+ - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ creationTimestamp: null
+ name: role-bind
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: role
+subjects:
+ - kind: ServiceAccount
+ name: default
diff --git a/deploy/asapo_helm_chart/asapo/values.yaml b/deploy/asapo_helm_chart/asapo/values.yaml
index 8e6d17564..1729e975a 100644
--- a/deploy/asapo_helm_chart/asapo/values.yaml
+++ b/deploy/asapo_helm_chart/asapo/values.yaml
@@ -30,7 +30,7 @@ ownServices:
nThreads: 1
_exposeServiceExtrernally: true
dataCache:
- enable: false
+ enable: true
sizeGb: 1
receiveToDiskThresholdMB: 200
fileTransfer:
diff --git a/deploy/asapo_services/scripts/receiver.json.tpl b/deploy/asapo_services/scripts/receiver.json.tpl
index ace0671e8..a5f819235 100644
--- a/deploy/asapo_services/scripts/receiver.json.tpl
+++ b/deploy/asapo_services/scripts/receiver.json.tpl
@@ -1,5 +1,4 @@
{
- "AdvertiseIP": "{{ if or (env "meta.ib_address") "none" | regexMatch "none" }}{{ env "NOMAD_IP_recv" }}{{ else }}{{ env "meta.ib_address" }}{{ end }}",
"PerformanceDbServer":"localhost:8400/influxdb",
"PerformanceDbName": "asapo_receivers",
"DatabaseServer":"auto",
@@ -8,6 +7,7 @@
"AuthorizationInterval": 10000,
"ListenPort": {{ env "NOMAD_PORT_recv" }},
"DataServer": {
+ "AdvertiseURI": "{{ if or (env "meta.ib_address") "none" | regexMatch "none" }}{{ env "NOMAD_IP_recv" }}{{ else }}{{ env "meta.ib_address" }}{{ end }}:{{ env "NOMAD_PORT_recv_ds" }}",
"NThreads": {{ env "NOMAD_META_receiver_dataserver_nthreads" }},
"ListenPort": {{ env "NOMAD_PORT_recv_ds" }}
},
diff --git a/receiver/src/receiver_config.cpp b/receiver/src/receiver_config.cpp
index f82f44afe..d919cfd0e 100644
--- a/receiver/src/receiver_config.cpp
+++ b/receiver/src/receiver_config.cpp
@@ -34,7 +34,7 @@ Error ReceiverConfigFactory::SetConfig(std::string file_name) {
(err = parser.GetString("AuthorizationServer", &config.authorization_server)) ||
(err = parser.GetUInt64("AuthorizationInterval", &config.authorization_interval_ms)) ||
(err = parser.GetString("PerformanceDbName", &config.performance_db_name)) ||
- (err = parser.GetString("AdvertiseIP", &config.advertise_ip)) ||
+ (err = parser.Embedded("DataServer").GetString("AdvertiseURI", &config.dataserver.advertise_uri)) ||
(err = parser.GetString("LogLevel", &log_level));
if (err) {
diff --git a/receiver/src/receiver_config.h b/receiver/src/receiver_config.h
index c69ff7990..514905eac 100644
--- a/receiver/src/receiver_config.h
+++ b/receiver/src/receiver_config.h
@@ -23,7 +23,6 @@ struct ReceiverConfig {
uint64_t receive_to_disk_threshold_mb = 0;
LogLevel log_level = LogLevel::Info;
std::string tag;
- std::string advertise_ip;
ReceiverDataCenterConfig dataserver;
std::string discovery_server;
};
diff --git a/receiver/src/receiver_data_server/receiver_datacenter_config.h b/receiver/src/receiver_data_server/receiver_datacenter_config.h
index ddef216fd..27b4c401c 100644
--- a/receiver/src/receiver_data_server/receiver_datacenter_config.h
+++ b/receiver/src/receiver_data_server/receiver_datacenter_config.h
@@ -9,6 +9,7 @@ struct ReceiverDataCenterConfig {
uint64_t listen_port = 0;
uint64_t nthreads = 0;
std::string tag;
+ std::string advertise_uri;
};
}
diff --git a/receiver/src/request_handler_db_write.cpp b/receiver/src/request_handler_db_write.cpp
index 6a0e31bcb..ad0dbf902 100644
--- a/receiver/src/request_handler_db_write.cpp
+++ b/receiver/src/request_handler_db_write.cpp
@@ -80,8 +80,7 @@ FileInfo RequestHandlerDbWrite::PrepareFileInfo(const Request* request) const {
file_info.size = request->GetDataSize();
file_info.id = request->GetDataID();
file_info.buf_id = request->GetSlotId();
- file_info.source = GetReceiverConfig()->advertise_ip + ":" + string_format("%ld",
- GetReceiverConfig()->dataserver.listen_port);
+ file_info.source = GetReceiverConfig()->dataserver.advertise_uri;
file_info.metadata = request->GetMetaData();
return file_info;
}
diff --git a/receiver/unittests/mock_receiver_config.cpp b/receiver/unittests/mock_receiver_config.cpp
index 99d2873ce..aacffd49c 100644
--- a/receiver/unittests/mock_receiver_config.cpp
+++ b/receiver/unittests/mock_receiver_config.cpp
@@ -49,6 +49,7 @@ Error SetReceiverConfig (const ReceiverConfig& config, std::string error_field)
config_string += "," + Key("ListenPort", error_field) + std::to_string(config.listen_port);
config_string += "," + Key("DataServer", error_field) + "{";
config_string += Key("ListenPort", error_field) + std::to_string(config.dataserver.listen_port);
+ config_string += "," + Key("AdvertiseURI", error_field) + "\"" + config.dataserver.advertise_uri + "\"";
config_string += "," + Key("NThreads", error_field) + std::to_string(config.dataserver.nthreads);
config_string += "}";
config_string += "," + Key("DataCache", error_field) + "{";
@@ -63,7 +64,6 @@ Error SetReceiverConfig (const ReceiverConfig& config, std::string error_field)
config_string += "," + Key("WriteToDisk", error_field) + (config.write_to_disk ? "true" : "false");
config_string += "," + Key("WriteToDb", error_field) + (config.write_to_db ? "true" : "false");
config_string += "," + Key("LogLevel", error_field) + "\"" + log_level + "\"";
- config_string += "," + Key("AdvertiseIP", error_field) + "\"" + config.advertise_ip + "\"";
config_string += "," + Key("Tag", error_field) + "\"" + config.tag + "\"";
config_string += "}";
diff --git a/receiver/unittests/test_config.cpp b/receiver/unittests/test_config.cpp
index 538f6f9dc..d3c81a500 100644
--- a/receiver/unittests/test_config.cpp
+++ b/receiver/unittests/test_config.cpp
@@ -58,10 +58,9 @@ class ConfigTests : public Test {
test_config.use_datacache = false;
test_config.datacache_reserved_share = 10;
test_config.datacache_size_gb = 2;
- test_config.advertise_ip = "host";
test_config.dataserver.nthreads = 5;
test_config.discovery_server = "discovery";
- test_config.advertise_ip = "0.0.0.1";
+ test_config.dataserver.advertise_uri = "0.0.0.1:4201";
test_config.receive_to_disk_threshold_mb = 50;
}
@@ -94,7 +93,7 @@ TEST_F(ConfigTests, ReadSettings) {
ASSERT_THAT(config->dataserver.nthreads, Eq(5));
ASSERT_THAT(config->dataserver.tag, Eq("receiver1_ds"));
ASSERT_THAT(config->discovery_server, Eq("discovery"));
- ASSERT_THAT(config->advertise_ip, Eq("0.0.0.1"));
+ ASSERT_THAT(config->dataserver.advertise_uri, Eq("0.0.0.1:4201"));
ASSERT_THAT(config->receive_to_disk_threshold_mb, Eq(50));
}
@@ -105,7 +104,7 @@ TEST_F(ConfigTests, ErrorReadSettings) {
std::vector<std::string>fields {"PerformanceDbServer", "ListenPort", "DataServer", "ListenPort", "WriteToDisk",
"WriteToDb", "DataCache", "Use", "SizeGB", "ReservedShare", "DatabaseServer", "Tag",
"AuthorizationServer", "AuthorizationInterval", "PerformanceDbName", "LogLevel",
- "NThreads", "DiscoveryServer", "AdvertiseIP", "ReceiveToDiskThresholdMB"};
+ "NThreads", "DiscoveryServer", "AdvertiseURI", "ReceiveToDiskThresholdMB"};
for (const auto& field : fields) {
auto err = asapo::SetReceiverConfig(test_config, field);
ASSERT_THAT(err, Ne(nullptr));
diff --git a/receiver/unittests/test_request_handler_db_check_request.cpp b/receiver/unittests/test_request_handler_db_check_request.cpp
index e40cbfbb4..4675c3fdd 100644
--- a/receiver/unittests/test_request_handler_db_check_request.cpp
+++ b/receiver/unittests/test_request_handler_db_check_request.cpp
@@ -75,7 +75,7 @@ class DbCheckRequestHandlerTests : public Test {
std::string expected_beamtime_id = "beamtime_id";
std::string expected_default_stream = "detector";
std::string expected_stream = "stream";
- std::string expected_host_ip = "127.0.0.1";
+ std::string expected_host_uri = "127.0.0.1:1234";
uint64_t expected_port = 1234;
uint64_t expected_buf_id = 18446744073709551615ull;
std::string expected_file_name = "2";
@@ -96,7 +96,7 @@ class DbCheckRequestHandlerTests : public Test {
handler.log__ = &mock_logger;
mock_request.reset(new NiceMock<MockRequest> {request_header, 1, "", nullptr});
config.database_uri = "127.0.0.1:27017";
- config.advertise_ip = expected_host_ip;
+ config.dataserver.advertise_uri = expected_host_uri;
config.dataserver.listen_port = expected_port;
SetReceiverConfig(config, "none");
expected_file_info = PrepareFileInfo();
@@ -196,7 +196,7 @@ FileInfo DbCheckRequestHandlerTests::PrepareFileInfo() {
file_info.name = expected_file_name;
file_info.id = expected_id;
file_info.buf_id = expected_buf_id;
- file_info.source = expected_host_ip + ":" + std::to_string(expected_port);
+ file_info.source = expected_host_uri;
file_info.metadata = expected_metadata;
return file_info;
}
diff --git a/receiver/unittests/test_request_handler_db_writer.cpp b/receiver/unittests/test_request_handler_db_writer.cpp
index 4f82656c7..ec2c75ae3 100644
--- a/receiver/unittests/test_request_handler_db_writer.cpp
+++ b/receiver/unittests/test_request_handler_db_writer.cpp
@@ -94,7 +94,7 @@ class DbWriterHandlerTests : public Test {
handler.log__ = &mock_logger;
mock_request.reset(new NiceMock<MockRequest> {request_header, 1, "", &mock_db_check_handler});
config.database_uri = "127.0.0.1:27017";
- config.advertise_ip = expected_host_ip;
+ config.dataserver.advertise_uri = expected_host_ip+":"+std::to_string(expected_port);
config.dataserver.listen_port = expected_port;
SetReceiverConfig(config, "none");
diff --git a/tests/automatic/settings/receiver.json.tpl.lin.in b/tests/automatic/settings/receiver.json.tpl.lin.in
index baa9b4b6c..1177861e1 100644
--- a/tests/automatic/settings/receiver.json.tpl.lin.in
+++ b/tests/automatic/settings/receiver.json.tpl.lin.in
@@ -1,10 +1,10 @@
{
- "AdvertiseIP": "127.0.0.1",
"PerformanceDbServer":"localhost:8086",
"PerformanceDbName": "db_test",
"DatabaseServer":"auto",
"DiscoveryServer": "localhost:8400/asapo-discovery",
"DataServer": {
+ "AdvertiseURI": "127.0.0.1:{{ env "NOMAD_PORT_recv_ds" }}",
"NThreads": 2,
"ListenPort": {{ env "NOMAD_PORT_recv_ds" }}
},
diff --git a/tests/automatic/settings/receiver.json.tpl.win.in b/tests/automatic/settings/receiver.json.tpl.win.in
index d18f3b98d..02fdd657b 100644
--- a/tests/automatic/settings/receiver.json.tpl.win.in
+++ b/tests/automatic/settings/receiver.json.tpl.win.in
@@ -1,5 +1,4 @@
{
- "AdvertiseIP": "127.0.0.1",
"PerformanceDbServer":"localhost:8086",
"PerformanceDbName": "db_test",
"DatabaseServer":"auto",
@@ -8,6 +7,7 @@
"AuthorizationInterval": 1000,
"ListenPort": {{ env "NOMAD_PORT_recv" }},
"DataServer": {
+ "AdvertiseURI": "127.0.0.1:{{ env "NOMAD_PORT_recv_ds" }}",
"NThreads": 2,
"ListenPort": {{ env "NOMAD_PORT_recv_ds" }}
},
diff --git a/tests/manual/broker_debug_local/receiver.json b/tests/manual/broker_debug_local/receiver.json
index 6a5d12f59..5567105bf 100644
--- a/tests/manual/broker_debug_local/receiver.json
+++ b/tests/manual/broker_debug_local/receiver.json
@@ -3,8 +3,8 @@
"PerformanceDbName": "db_test",
"DatabaseServer":"localhost:27017",
"DiscoveryServer": "localhost:8400/discovery",
- "AdvertiseIP":"127.0.0.1",
"DataServer": {
+ "AdvertiseURI":"127.0.0.1",
"NThreads": 2,
"ListenPort": 22000
},
diff --git a/tests/manual/broker_debug_local/receiver.json.tpl b/tests/manual/broker_debug_local/receiver.json.tpl
index 1f7cbc7c1..a6c0887d9 100644
--- a/tests/manual/broker_debug_local/receiver.json.tpl
+++ b/tests/manual/broker_debug_local/receiver.json.tpl
@@ -1,10 +1,10 @@
{
- "AdvertiseIP": "127.0.0.1",
"PerformanceDbServer":"localhost:8086",
"PerformanceDbName": "db_test",
"DatabaseServer":"auto",
"DiscoveryServer": "localhost:8400/discovery",
"DataServer": {
+ "AdvertiseURI": "127.0.0.1",
"NThreads": 2,
"ListenPort": {{ env "NOMAD_PORT_recv_ds" }}
},
diff --git a/tests/manual/python_tests/producer/receiver.json.tpl b/tests/manual/python_tests/producer/receiver.json.tpl
index a235fec0d..6f80d44bb 100644
--- a/tests/manual/python_tests/producer/receiver.json.tpl
+++ b/tests/manual/python_tests/producer/receiver.json.tpl
@@ -1,10 +1,10 @@
{
- "AdvertiseIP": "127.0.0.1",
"PerformanceDbServer":"localhost:8086",
"PerformanceDbName": "db_test",
"DatabaseServer":"localhost:27017",
"DiscoveryServer": "localhost:8400/discovery",
"DataServer": {
+ "AdvertiseURI": "127.0.0.1",
"NThreads": 2,
"ListenPort": {{ env "NOMAD_PORT_recv_ds" }}
},
diff --git a/tests/manual/python_tests/producer_wait_bug_mongo/receiver.json.tpl b/tests/manual/python_tests/producer_wait_bug_mongo/receiver.json.tpl
index a235fec0d..6f80d44bb 100644
--- a/tests/manual/python_tests/producer_wait_bug_mongo/receiver.json.tpl
+++ b/tests/manual/python_tests/producer_wait_bug_mongo/receiver.json.tpl
@@ -1,10 +1,10 @@
{
- "AdvertiseIP": "127.0.0.1",
"PerformanceDbServer":"localhost:8086",
"PerformanceDbName": "db_test",
"DatabaseServer":"localhost:27017",
"DiscoveryServer": "localhost:8400/discovery",
"DataServer": {
+ "AdvertiseURI": "127.0.0.1",
"NThreads": 2,
"ListenPort": {{ env "NOMAD_PORT_recv_ds" }}
},
diff --git a/tests/manual/receiver_debug_local/receiver.json b/tests/manual/receiver_debug_local/receiver.json
index 6a5d12f59..5567105bf 100644
--- a/tests/manual/receiver_debug_local/receiver.json
+++ b/tests/manual/receiver_debug_local/receiver.json
@@ -3,8 +3,8 @@
"PerformanceDbName": "db_test",
"DatabaseServer":"localhost:27017",
"DiscoveryServer": "localhost:8400/discovery",
- "AdvertiseIP":"127.0.0.1",
"DataServer": {
+ "AdvertiseURI":"127.0.0.1",
"NThreads": 2,
"ListenPort": 22000
},
--
GitLab