From 333837b0062078449feb48c94ef136c2f7d1d0a1 Mon Sep 17 00:00:00 2001
From: George Sedov <george.sedov@desy.de>
Date: Thu, 15 Sep 2022 19:21:07 +0200
Subject: [PATCH] update config and fluent docker
---
.../scripts/asapo-logging.nmd.tpl | 12 ++++++---
.../scripts/asapo-mongo.nmd.tpl | 4 +--
.../scripts/asapo-monitoring.nmd.tpl | 5 ++--
.../scripts/asapo-perfmetrics.nmd.tpl | 8 ++++++
.../scripts/asapo-services.nmd.tpl | 2 +-
deploy/asapo_services/scripts/templates.tf | 3 +++
deploy/asapo_services/scripts/vars.tf | 12 ++++-----
.../fluentd_elastic/Dockerfile | 26 ++++++-------------
8 files changed, 39 insertions(+), 33 deletions(-)
diff --git a/deploy/asapo_services/scripts/asapo-logging.nmd.tpl b/deploy/asapo_services/scripts/asapo-logging.nmd.tpl
index a033d9bb8..1cd0a6d4e 100644
--- a/deploy/asapo_services/scripts/asapo-logging.nmd.tpl
+++ b/deploy/asapo_services/scripts/asapo-logging.nmd.tpl
@@ -36,9 +36,15 @@ job "asapo-logging" {
network_mode = "host"
security_opt = ["no-new-privileges"]
userns_mode = "host"
- image = "yakser/fluentd_elastic"
+ image = "${docker_repository}/asapo-secondary-fluentd-elasticsearch"
volumes = ["local/fluentd.conf:/fluentd/etc/fluent.conf",
"/${service_dir}/fluentd:/shared"]
+ %{ if docker_repository_username != "" }
+ auth {
+ username = "${docker_repository_username}"
+ password = "${docker_repository_password}"
+ }
+ %{endif}
}
resources {
@@ -106,7 +112,7 @@ job "asapo-logging" {
network_mode = "host"
security_opt = ["no-new-privileges"]
userns_mode = "host"
- image = "yakser/elasticsearch:${elasticsearch_version}"
+ image = "elasticsearch:${elasticsearch_version}"
volumes = ["/${service_dir}/esdatadir:/usr/share/elasticsearch/data"]
}
@@ -144,7 +150,7 @@ job "asapo-logging" {
network_mode = "host"
security_opt = ["no-new-privileges"]
userns_mode = "host"
- image = "yakser/kibana:${kibana_version}"
+ image = "kibana:${kibana_version}"
volumes = ["local/kibana.yml:/usr/share/kibana/config/kibana.yml"]
}
diff --git a/deploy/asapo_services/scripts/asapo-mongo.nmd.tpl b/deploy/asapo_services/scripts/asapo-mongo.nmd.tpl
index 6b18019fd..d3bba6057 100644
--- a/deploy/asapo_services/scripts/asapo-mongo.nmd.tpl
+++ b/deploy/asapo_services/scripts/asapo-mongo.nmd.tpl
@@ -43,7 +43,7 @@ job "asapo-mongo" {
config {
security_opt = ["no-new-privileges"]
userns_mode = "host"
- image = "yakser/mongodb-exporter"
+ image = "percona/mongodb_exporter"
args = [
"--mongodb.uri=mongodb://$${NOMAD_ADDR_mongo}"
]
@@ -97,7 +97,7 @@ job "asapo-mongo" {
}
check_restart {
limit = 2
- grace = "1800s"
+ grace = "3h"
ignore_warnings = false
}
}
diff --git a/deploy/asapo_services/scripts/asapo-monitoring.nmd.tpl b/deploy/asapo_services/scripts/asapo-monitoring.nmd.tpl
index 0f1a15493..af52918d5 100644
--- a/deploy/asapo_services/scripts/asapo-monitoring.nmd.tpl
+++ b/deploy/asapo_services/scripts/asapo-monitoring.nmd.tpl
@@ -1,6 +1,5 @@
job "asapo-monitoring" {
- datacenters = [
- "dc1"]
+ datacenters = ["dc1"]
affinity {
attribute = "$${meta.node_group}"
value = "utl"
@@ -141,4 +140,4 @@ job "asapo-monitoring" {
}
}
}
-}
\ No newline at end of file
+}
diff --git a/deploy/asapo_services/scripts/asapo-perfmetrics.nmd.tpl b/deploy/asapo_services/scripts/asapo-perfmetrics.nmd.tpl
index 44364c591..45404438f 100644
--- a/deploy/asapo_services/scripts/asapo-perfmetrics.nmd.tpl
+++ b/deploy/asapo_services/scripts/asapo-perfmetrics.nmd.tpl
@@ -127,6 +127,10 @@ job "asapo-perfmetrics" {
image = "${docker_repository}/asapo-monitoring-server${image_suffix}"
force_pull = ${force_pull_images}
volumes = ["local/config.json:/var/lib/monitoring_server/config.json"]
+ auth {
+ username = "${docker_repository_username}"
+ password = "${docker_repository_password}"
+ }
%{ if ! nomad_logs }
logging {
type = "fluentd"
@@ -227,6 +231,10 @@ job "asapo-perfmetrics" {
image = "${docker_repository}/asapo-monitoring-ui${image_suffix}"
force_pull = ${force_pull_images}
volumes = ["local/nginx.conf:/etc/nginx/nginx.conf"]
+ auth {
+ username = "${docker_repository_username}"
+ password = "${docker_repository_password}"
+ }
}
resources {
diff --git a/deploy/asapo_services/scripts/asapo-services.nmd.tpl b/deploy/asapo_services/scripts/asapo-services.nmd.tpl
index c43f08f42..d9b5b5d4d 100644
--- a/deploy/asapo_services/scripts/asapo-services.nmd.tpl
+++ b/deploy/asapo_services/scripts/asapo-services.nmd.tpl
@@ -31,7 +31,7 @@ job "asapo-services" {
}
%{endif}
- %{ if ! nomad_logs }
+ %{ if ! nomad_logs }
logging {
type = "fluentd"
config {
diff --git a/deploy/asapo_services/scripts/templates.tf b/deploy/asapo_services/scripts/templates.tf
index 3c4b25c29..f036ed4c1 100644
--- a/deploy/asapo_services/scripts/templates.tf
+++ b/deploy/asapo_services/scripts/templates.tf
@@ -170,6 +170,9 @@ data "template_file" "asapo_logging" {
vars = {
service_dir = "${var.service_dir}"
scripts_dir = "${var.job_scripts_dir}"
+ docker_repository = "${var.asapo_docker_repository}"
+ docker_repository_username = "${var.asapo_docker_repository_username}"
+ docker_repository_password = "${var.asapo_docker_repository_password}"
elk_logs = "${var.elk_logs}"
nomad_logs = "${var.nomad_logs}"
fluentd_total_memory_size = "${var.fluentd_total_memory_size}"
diff --git a/deploy/asapo_services/scripts/vars.tf b/deploy/asapo_services/scripts/vars.tf
index 0eba6a158..0f618fb8e 100644
--- a/deploy/asapo_services/scripts/vars.tf
+++ b/deploy/asapo_services/scripts/vars.tf
@@ -1,3 +1,9 @@
+variable "asapo_docker_repository" {}
+
+variable "asapo_docker_repository_username" {}
+
+variable "asapo_docker_repository_password" {}
+
variable "elk_logs" {}
variable "perf_monitor" {}
@@ -32,12 +38,6 @@ variable "prometheus_version" {}
variable "alertmanager_version" {}
-variable "asapo_docker_repository" {}
-
-variable "asapo_docker_repository_username" {}
-
-variable "asapo_docker_repository_password" {}
-
variable "asapo_imagename_suffix" {}
variable "asapo_image_tag" {}
diff --git a/deploy/secondary_services/fluentd_elastic/Dockerfile b/deploy/secondary_services/fluentd_elastic/Dockerfile
index a62a8ad0d..4655a59bb 100644
--- a/deploy/secondary_services/fluentd_elastic/Dockerfile
+++ b/deploy/secondary_services/fluentd_elastic/Dockerfile
@@ -1,20 +1,10 @@
-FROM fluent/fluentd:v1.7.3-debian-1.0
-
-# Use root account to use apt
+FROM fluent/fluentd:v1.15.2-debian-1.0
USER root
-
-# below RUN includes plugin as examples elasticsearch is not required
-# you may customize including plugins as you wish
-RUN buildDeps="sudo make gcc g++ libc-dev" \
- && apt-get update \
- && apt-get install -y --no-install-recommends $buildDeps \
- && sudo gem install fluent-plugin-elasticsearch \
- && sudo gem sources --clear-all \
- && SUDO_FORCE_REMOVE=yes \
- apt-get purge -y --auto-remove \
- -o APT::AutoRemove::RecommendsImportant=false \
- $buildDeps \
- && rm -rf /var/lib/apt/lists/* \
- && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
-
+RUN gem install fluent-plugin-elasticsearch --no-document --version 5.2.3 \
+ && gem uninstall --ignore-dependencies faraday \
+ && gem install faraday --no-document --version 1.10.0 \
+ && gem sources --clear-all \
+ && apt-get purge -y --auto-remove \
+ && rm -rf /var/lib/apt/lists/* \
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
USER fluent
--
GitLab