From 847bcbe09aa9c582043bac69df65d556f0bd49e3 Mon Sep 17 00:00:00 2001
From: Tim Schoof <tim.schoof@desy.de>
Date: Fri, 6 Mar 2020 18:26:05 +0100
Subject: [PATCH] Create a fluent-bit task for cluster logs
Parsing timestamps leads to unordered logs because for lines without matching timestamps the
current time is used. By always using the current time, the line order is conserved.
---
.../fluent-bit-cluster-custom-parsers.conf | 5 ++
.../scripts/fluent-bit-cluster.conf.tpl | 68 +++++++++++++++++++
.../scripts/monitoring.nmd.tpl | 14 ++++
3 files changed, 87 insertions(+)
create mode 100644 deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster-custom-parsers.conf
create mode 100644 deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster.conf.tpl
diff --git a/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster-custom-parsers.conf b/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster-custom-parsers.conf
new file mode 100644
index 000000000..e06d8994d
--- /dev/null
+++ b/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster-custom-parsers.conf
@@ -0,0 +1,5 @@
+[PARSER]
+ Name level
+ Format regex
+ # from https://github.com/grafana/grafana/blob/917860c4d396cf9ce22127fb39159fe2a1e09a5e/packages/grafana-data/src/types/logs.ts
+ Regex (?i)^(?<log>.*?\b(?<level>trace|dbug|debug|notice|info|warn|warning|err|eror|error|crit|critical|emerg|alert|exception)\b.*)
diff --git a/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster.conf.tpl b/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster.conf.tpl
new file mode 100644
index 000000000..8cd64e714
--- /dev/null
+++ b/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster.conf.tpl
@@ -0,0 +1,68 @@
+[SERVICE]
+ Parsers_File /etc/td-agent-bit/parsers.conf
+ Parsers_File /var/run/asapo/fluent-bit-cluster-custom-parsers.conf
+
+{{ range ("nomad consul supervisord" | split " ") }}
+[INPUT]
+ Name tail
+ Path /var/log/supervisord/{{.}}*.log
+ Tag asapo.{{.}}
+ # Tag_Regex .*/(?<service>[^-.]*)(-(?<channel>stderr|stdout))?.*
+ DB ${NOMAD_TASK_DIR}/logs.db
+ Parser level
+{{ end }}
+
+{{ range ("influxdb grafana" | split " ") }}
+[INPUT]
+ Name tail
+ Path ${NOMAD_ALLOC_DIR}/${NOMAD_ALLOC_ID}/alloc/logs/{{.}}*.0
+ Tag asapo.{{.}}
+ # Tag_Regex .*/(?<service>.*).(?<channel>stderr|stdout).*
+ DB ${NOMAD_TASK_DIR}/logs.db
+ Parser level
+{{ end }}
+
+[FILTER]
+ Name modify
+ Match *
+ Condition Key_value_matches level (?i)^d.*
+ Set level DEBUG
+
+[FILTER]
+ Name modify
+ Match *
+ Condition Key_value_matches level (?i)^(i.*|no.*)
+ Set level INFO
+
+[FILTER]
+ Name modify
+ Match *
+ Condition Key_value_matches level (?i)^w.*
+ Set level WARNING
+
+[FILTER]
+ Name modify
+ Match *
+ Condition Key_value_matches level (?i)^er.*
+ Set level ERROR
+
+[FILTER]
+ Name modify
+ Match *
+ Condition Key_value_matches level (?i)^(c.*|a.*|emerg|exception)
+ Set level CRITICAL
+
+[FILTER]
+ Name modify
+ Match *
+ Add level UNKNOWN
+
+{{ range service "influxdb" }}
+[OUTPUT]
+ Name influxdb
+ Match *
+ Host {{ .Address }}
+ Port {{ .Port }}
+ Database logs
+ Tag_Keys level
+{{ end }}
diff --git a/deploy/asapo_orchestration_docker/scripts/monitoring.nmd.tpl b/deploy/asapo_orchestration_docker/scripts/monitoring.nmd.tpl
index 33dd52f41..a86149f7d 100644
--- a/deploy/asapo_orchestration_docker/scripts/monitoring.nmd.tpl
+++ b/deploy/asapo_orchestration_docker/scripts/monitoring.nmd.tpl
@@ -196,5 +196,19 @@ job "monitoring" {
} #telegraf
+ task "fluent-bit-cluster" {
+ driver = "raw_exec"
+
+ config {
+ command = "/opt/td-agent-bit/bin/td-agent-bit"
+ args = ["-c", "local/fluent-bit.conf"]
+ }
+
+ template {
+ source = "${scripts_dir}/fluent-bit-cluster.conf.tpl"
+ destination = "local/fluent-bit.conf"
+ change_mode = "restart"
+ }
+ } # fluent-bit-cluster
}
}
--
GitLab