From 9126fa10a4553b269447a7029f70d87bc60fc6c1 Mon Sep 17 00:00:00 2001
From: George Sedov <george.sedov@desy.de>
Date: Thu, 2 Dec 2021 15:47:32 +0100
Subject: [PATCH] Add kafka config to deploy configuration
---
deploy/asapo_services/asap3.tfvars | 1 +
deploy/asapo_services/scripts/asapo-receivers.nmd.tpl | 2 ++
deploy/asapo_services/scripts/receiver.json.tpl | 11 ++++++++++-
deploy/asapo_services/scripts/templates.tf | 2 ++
deploy/asapo_services/scripts/vars.tf | 6 +++++-
tests/automatic/settings/receiver_tcp.json.tpl.lin.in | 5 ++++-
tests/automatic/settings/receiver_tcp.json.tpl.win.in | 5 ++++-
7 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/deploy/asapo_services/asap3.tfvars b/deploy/asapo_services/asap3.tfvars
index 1070bcc40..991b42e2f 100644
--- a/deploy/asapo_services/asap3.tfvars
+++ b/deploy/asapo_services/asap3.tfvars
@@ -21,6 +21,7 @@ receiver_dataserver_cache_size = 30 #gb
receiver_receive_to_disk_threshold = 50 # mb
receiver_dataserver_nthreads = 8
receiver_network_modes = "tcp"
+receiver_kafka_enabled = true
grafana_total_memory_size = 2000
influxdb_total_memory_size = 2000
diff --git a/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl b/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl
index fee0524f4..5a48f8a93 100644
--- a/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl
+++ b/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl
@@ -100,6 +100,8 @@ job "asapo-receivers" {
receiver_network_modes = "${receiver_network_modes}"
perf_monitor = "${perf_monitor}"
receiver_expose_metrics = "${receiver_expose_metrics}"
+ receiver_kafka_enabled = "${receiver_kafka_enabled}"
+ receiver_kafka_metadata_broker_list = "${receiver_kafka_metadata_broker_list}"
}
template {
diff --git a/deploy/asapo_services/scripts/receiver.json.tpl b/deploy/asapo_services/scripts/receiver.json.tpl
index da0b30cd3..3183e72ab 100644
--- a/deploy/asapo_services/scripts/receiver.json.tpl
+++ b/deploy/asapo_services/scripts/receiver.json.tpl
@@ -24,5 +24,14 @@
},
"Tag": "{{ env "attr.unique.hostname" }}",
"ReceiveToDiskThresholdMB": {{ env "NOMAD_META_receiver_receive_to_disk_threshold" }},
- "LogLevel": "{{ keyOrDefault "receiver_log_level" "info" }}"
+ "LogLevel": "{{ keyOrDefault "receiver_log_level" "info" }}",
+ "Kafka": {
+ "Enabled": {{ env "NOMAD_META_receiver_kafka_enabled" }},
+ "KafkaClient": {
+ "metadata.broker.list": "{{ env "NOMAD_META_receiver_kafka_metadata_broker_list" }}"
+ },
+ "KafkaTopics": {
+ "asapo": {}
+ }
+ }
}
diff --git a/deploy/asapo_services/scripts/templates.tf b/deploy/asapo_services/scripts/templates.tf
index 5cc67dc9e..664505890 100644
--- a/deploy/asapo_services/scripts/templates.tf
+++ b/deploy/asapo_services/scripts/templates.tf
@@ -56,6 +56,8 @@ data "template_file" "asapo_receivers" {
force_pull_images = "${var.force_pull_images}"
perf_monitor = "${var.perf_monitor}"
receiver_expose_metrics = "${var.receiver_expose_metrics}"
+ receiver_kafka_enabled = "${var.receiver_kafka_enabled}"
+ receiver_kafka_metadata_broker_list = "${var.receiver_kafka_metadata_broker_list}"
}
}
diff --git a/deploy/asapo_services/scripts/vars.tf b/deploy/asapo_services/scripts/vars.tf
index c7d16e5f1..13068f32a 100644
--- a/deploy/asapo_services/scripts/vars.tf
+++ b/deploy/asapo_services/scripts/vars.tf
@@ -58,6 +58,10 @@ variable "receiver_network_modes" {}
variable "receiver_expose_metrics" {}
+variable "receiver_kafka_enabled" {}
+
+variable "receiver_kafka_metadata_broker_list" {}
+
variable "grafana_total_memory_size" {}
variable "influxdb_total_memory_size" {}
@@ -111,4 +115,4 @@ variable "n_brokers" {}
variable "n_fts" {}
-variable "ldap_uri" {}
\ No newline at end of file
+variable "ldap_uri" {}
diff --git a/tests/automatic/settings/receiver_tcp.json.tpl.lin.in b/tests/automatic/settings/receiver_tcp.json.tpl.lin.in
index 498d9fdcc..ebaa3aea2 100644
--- a/tests/automatic/settings/receiver_tcp.json.tpl.lin.in
+++ b/tests/automatic/settings/receiver_tcp.json.tpl.lin.in
@@ -24,5 +24,8 @@
"ListenPort": {{ env "NOMAD_PORT_recv" }},
"Tag": "{{ env "NOMAD_ADDR_recv" }}",
"ReceiveToDiskThresholdMB":50,
- "LogLevel" : "debug"
+ "LogLevel" : "debug",
+ "Kafka" : {
+ "Enabled" : false
+ }
}
diff --git a/tests/automatic/settings/receiver_tcp.json.tpl.win.in b/tests/automatic/settings/receiver_tcp.json.tpl.win.in
index 446ebcbf2..1c08e1b84 100644
--- a/tests/automatic/settings/receiver_tcp.json.tpl.win.in
+++ b/tests/automatic/settings/receiver_tcp.json.tpl.win.in
@@ -24,5 +24,8 @@
},
"Tag": "{{ env "NOMAD_ADDR_recv" }}",
"ReceiveToDiskThresholdMB":50,
- "LogLevel" : "debug"
+ "LogLevel" : "debug",
+ "Kafka" : {
+ "Enabled" : false
+ }
}
--
GitLab