From 4546343159b2cb7b78985e984759c63cb48c4d66 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Wed, 9 Feb 2022 14:42:39 +0100
Subject: [PATCH] added monitoring server to nomad deployment
---
deploy/asapo_services/local.tfvars | 26 +++++++++
.../scripts/asapo-perfmetrics.nmd.tpl | 58 ++++++++++++++++++-
.../scripts/asapo.auto.tfvars.in | 7 ++-
.../scripts/file-transfer.json.tpl | 5 +-
.../scripts/monitoring_server.json.tpl | 7 +++
.../asapo_services/scripts/receiver.json.tpl | 3 +-
deploy/asapo_services/scripts/resources.tf | 7 +--
.../scripts/resources_services.tf | 8 +++
deploy/asapo_services/scripts/templates.tf | 8 ++-
deploy/asapo_services/scripts/vars.tf | 3 +
10 files changed, 123 insertions(+), 9 deletions(-)
create mode 100644 deploy/asapo_services/local.tfvars
create mode 100644 deploy/asapo_services/scripts/monitoring_server.json.tpl
diff --git a/deploy/asapo_services/local.tfvars b/deploy/asapo_services/local.tfvars
new file mode 100644
index 000000000..4e61d66f8
--- /dev/null
+++ b/deploy/asapo_services/local.tfvars
@@ -0,0 +1,26 @@
+elk_logs = false
+perf_monitor = true
+
+asapo_docker_repository = "yakser"
+#asapo_imagename_suffix = ""
+#asapo_image_tag = ""
+
+influxdb_version="1.8.4"
+
+service_dir="/var/tmp/asapo"
+online_dir="/var/tmp/asapo/bl"
+offline_dir="/var/tmp/asapo/core"
+mongo_dir="/var/tmp/asapo/mongo"
+asapo_user="26655:1000"
+job_scripts_dir="/home/yakubov/projects/asapo/build/deploy/asapo_services/scripts"
+
+
+ldap_uri="ldap://it-ldap-slave.desy.de:1389"
+
+receiver_network_modes = "tcp"
+receiver_kafka_enabled = false
+receiver_kafka_metadata_broker_list=""
+
+n_receivers = 1
+n_brokers = 1
+n_fts = 1
diff --git a/deploy/asapo_services/scripts/asapo-perfmetrics.nmd.tpl b/deploy/asapo_services/scripts/asapo-perfmetrics.nmd.tpl
index 22b832bd7..991f0cb04 100644
--- a/deploy/asapo_services/scripts/asapo-perfmetrics.nmd.tpl
+++ b/deploy/asapo_services/scripts/asapo-perfmetrics.nmd.tpl
@@ -23,13 +23,14 @@ job "asapo-perfmetrics" {
security_opt = ["no-new-privileges"]
userns_mode = "host"
image = "influxdb:${influxdb_version}"
- volumes = ["/${service_dir}/influxdb2:/var/lib/influxdb2"]
+ volumes = ["/${service_dir}/influxdb:/var/lib/influxdb"]
}
env {
PRE_CREATE_DB="asapo_receivers;asapo_brokers"
INFLUXDB_BIND_ADDRESS="127.0.0.1:$${NOMAD_PORT_influxdb_rpc}"
INFLUXDB_HTTP_BIND_ADDRESS=":$${NOMAD_PORT_influxdb}"
+ INFLUXDB_HTTP_FLUX_ENABLED="true"
}
resources {
@@ -110,6 +111,61 @@ job "asapo-perfmetrics" {
} #grafana
+ task "monitoring-server" {
+ driver = "docker"
+ user = "${asapo_user}"
+
+ config {
+ ulimit {
+ memlock = "-1:-1"
+ }
+ network_mode = "host"
+ security_opt = ["no-new-privileges"]
+ userns_mode = "host"
+ privileged = true
+ image = "${docker_repository}/asapo-monitoring-server${image_suffix}"
+ force_pull = ${force_pull_images}
+ volumes = ["local/config.json:/var/lib/monitoring_server/config.json"]
+ %{ if ! nomad_logs }
+ logging {
+ type = "fluentd"
+ config {
+ fluentd-address = "localhost:9881"
+ fluentd-async-connect = true
+ tag = "asapo.docker"
+ }
+ }
+ %{endif}
+ }
+
+ resources {
+ memory = "${monitoring_server_total_memory_size}"
+ network {
+ port "monitoring_server" {}
+ }
+ }
+
+ service {
+ name = "asapo-monitoring"
+ port = "monitoring_server"
+ check {
+ name = "alive"
+ port = "monitoring_server"
+ type = "tcp"
+ interval = "10s"
+ timeout = "2s"
+ initial_status = "passing"
+ }
+ }
+
+ template {
+ source = "${scripts_dir}/monitoring_server.json.tpl"
+ destination = "local/config.json"
+ change_mode = "restart"
+ }
+ } # monitoring server
+
+
}
}
diff --git a/deploy/asapo_services/scripts/asapo.auto.tfvars.in b/deploy/asapo_services/scripts/asapo.auto.tfvars.in
index 4f8133d36..64f10eaaf 100644
--- a/deploy/asapo_services/scripts/asapo.auto.tfvars.in
+++ b/deploy/asapo_services/scripts/asapo.auto.tfvars.in
@@ -1,12 +1,14 @@
asapo_imagename_suffix="@ASAPO_VERSION_DOCKER_SUFFIX@"
asapo_image_tag = "@ASAPO_VERSION@"
+asapo_docker_repository = "yakser"
+
nginx_version = "1.14"
elasticsearch_version = "7.3.2"
kibana_version = "7.3.2"
mongo_version = "4.0.0"
grafana_version="7.5.0"
-influxdb_version = "2.0.4"
+influxdb_version = "1.8.4"
elk_logs = false
perf_monitor = false
nomad_logs = false
@@ -40,6 +42,7 @@ authorizer_total_memory_size = 256
discovery_total_memory_size = 256
prometheus_total_memory_size = 256
alertmanager_total_memory_size = 256
+monitoring_server_total_memory_size = 256
ldap_uri = "ldap://localhost:389"
@@ -51,8 +54,10 @@ fluentd_port = 9880
fluentd_port_stream = 24224
elasticsearch_port = 9200
kibana_port = 5601
+
discovery_port = 5006
authorizer_port = 5007
+
consul_dns_port = 8600
prometheus_port = 9090
alertmanager_port = 9093
diff --git a/deploy/asapo_services/scripts/file-transfer.json.tpl b/deploy/asapo_services/scripts/file-transfer.json.tpl
index 5f0f028a0..5f6a28549 100644
--- a/deploy/asapo_services/scripts/file-transfer.json.tpl
+++ b/deploy/asapo_services/scripts/file-transfer.json.tpl
@@ -1,5 +1,8 @@
{
"Port": {{ env "NOMAD_PORT_fts" }},
"LogLevel":"debug",
- "SecretFile":"/local/secret.key"
+ "SecretFile":"/local/secret.key",
+ "DiscoveryServer": "localhost:8400/asapo-discovery",
+ "MonitorPerformance": true,
+ "MonitoringServerUrl": "auto"
}
diff --git a/deploy/asapo_services/scripts/monitoring_server.json.tpl b/deploy/asapo_services/scripts/monitoring_server.json.tpl
new file mode 100644
index 000000000..9ac042419
--- /dev/null
+++ b/deploy/asapo_services/scripts/monitoring_server.json.tpl
@@ -0,0 +1,7 @@
+{
+ "ThisClusterName": "asapo",
+ "ServerPort": {{ env "NOMAD_PORT_monitoring_server" }},
+ "LogLevel": "{{ keyOrDefault "log_level" "debug" }}",
+ "InfluxDbUrl":"http://localhost:8400/influxdb",
+ "InfluxDbDatabase": "asapo_monitoring"
+}
diff --git a/deploy/asapo_services/scripts/receiver.json.tpl b/deploy/asapo_services/scripts/receiver.json.tpl
index 3183e72ab..5ca82a26d 100644
--- a/deploy/asapo_services/scripts/receiver.json.tpl
+++ b/deploy/asapo_services/scripts/receiver.json.tpl
@@ -1,6 +1,7 @@
{
- "PerformanceDbServer":"localhost:8400/influxdb",
"MonitorPerformance": {{ env "NOMAD_META_perf_monitor" }},
+ "MonitoringServer": "auto",
+ "PerformanceDbServer":"localhost:8400/influxdb",
"PerformanceDbName": "asapo_receivers",
"DatabaseServer":"auto",
"DiscoveryServer": "localhost:8400/asapo-discovery",
diff --git a/deploy/asapo_services/scripts/resources.tf b/deploy/asapo_services/scripts/resources.tf
index 928a5a762..1a5d81783 100644
--- a/deploy/asapo_services/scripts/resources.tf
+++ b/deploy/asapo_services/scripts/resources.tf
@@ -13,7 +13,6 @@ resource "nomad_job" "asapo-perfmetrics" {
resource "nomad_job" "asapo-monitoring" {
jobspec = data.template_file.asapo_monitoring.rendered
- depends_on = [null_resource.asapo-broker,null_resource.asapo-receiver]
}
@@ -29,16 +28,16 @@ resource "nomad_job" "asapo-services" {
resource "nomad_job" "asapo-receivers" {
jobspec = data.template_file.asapo_receivers.rendered
- depends_on = [nomad_job.asapo-services,null_resource.asapo-authorizer,null_resource.asapo-discovery]
+ depends_on = [nomad_job.asapo-services,null_resource.asapo-authorizer,null_resource.asapo-discovery,null_resource.asapo-monitoring-server]
}
resource "nomad_job" "asapo-brokers" {
jobspec = data.template_file.asapo_brokers.rendered
- depends_on = [nomad_job.asapo-services,null_resource.asapo-authorizer,null_resource.asapo-discovery]
+ depends_on = [nomad_job.asapo-services,null_resource.asapo-authorizer,null_resource.asapo-discovery,null_resource.asapo-monitoring-server]
}
resource "nomad_job" "asapo-fts" {
jobspec = data.template_file.asapo_fts.rendered
- depends_on = [nomad_job.asapo-services,null_resource.asapo-authorizer,null_resource.asapo-discovery]
+ depends_on = [nomad_job.asapo-services,null_resource.asapo-authorizer,null_resource.asapo-discovery,null_resource.asapo-monitoring-server]
}
diff --git a/deploy/asapo_services/scripts/resources_services.tf b/deploy/asapo_services/scripts/resources_services.tf
index 0ad672ea2..08332309d 100644
--- a/deploy/asapo_services/scripts/resources_services.tf
+++ b/deploy/asapo_services/scripts/resources_services.tf
@@ -67,6 +67,14 @@ resource "null_resource" "asapo-broker" {
depends_on = [nomad_job.asapo-brokers]
}
+resource "null_resource" "asapo-monitoring-server" {
+ provisioner "local-exec" {
+ command = "asapo-wait-service asapo-monitoring"
+ }
+ depends_on = [nomad_job.asapo-monitoring]
+}
+
+
resource "null_resource" "asapo-fts" {
provisioner "local-exec" {
command = "asapo-wait-service asapo-file-transfer"
diff --git a/deploy/asapo_services/scripts/templates.tf b/deploy/asapo_services/scripts/templates.tf
index ddd5656dc..12b905c63 100644
--- a/deploy/asapo_services/scripts/templates.tf
+++ b/deploy/asapo_services/scripts/templates.tf
@@ -96,6 +96,8 @@ data "template_file" "asapo_perfmetrics" {
template = "${file("${var.job_scripts_dir}/asapo-perfmetrics.nmd.tpl")}"
vars = {
service_dir = "${var.service_dir}"
+ image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}"
+ docker_repository = "${var.asapo_docker_repository}"
influxdb_version = "${var.influxdb_version}"
grafana_version = "${var.grafana_version}"
grafana_total_memory_size = "${var.grafana_total_memory_size}"
@@ -105,7 +107,11 @@ data "template_file" "asapo_perfmetrics" {
asapo_user = "${var.asapo_user}"
influxdb_rpc_port = "${var.influxdb_rpc_port}"
perf_monitor = "${var.perf_monitor}"
- }
+ monitoring_server_total_memory_size = "${var.monitoring_server_total_memory_size}"
+ force_pull_images = "${var.force_pull_images}"
+ nomad_logs = "${var.nomad_logs}"
+ scripts_dir = "${var.job_scripts_dir}"
+ }
}
diff --git a/deploy/asapo_services/scripts/vars.tf b/deploy/asapo_services/scripts/vars.tf
index 13068f32a..559b4fd24 100644
--- a/deploy/asapo_services/scripts/vars.tf
+++ b/deploy/asapo_services/scripts/vars.tf
@@ -1,3 +1,5 @@
+variable "asapo_docker_repository" {}
+
variable "elk_logs" {}
variable "perf_monitor" {}
@@ -82,6 +84,7 @@ variable "prometheus_total_memory_size" {}
variable "alertmanager_total_memory_size" {}
+variable "monitoring_server_total_memory_size" {}
variable "grafana_port" {}
--
GitLab