diff --git a/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster-custom-parsers.conf b/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster-custom-parsers.conf new file mode 100644 index 0000000000000000000000000000000000000000..e06d8994dfb0d7adb5c61c0c6a413731eb31ee60 --- /dev/null +++ b/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster-custom-parsers.conf @@ -0,0 +1,5 @@ +[PARSER] + Name level + Format regex + # from https://github.com/grafana/grafana/blob/917860c4d396cf9ce22127fb39159fe2a1e09a5e/packages/grafana-data/src/types/logs.ts + Regex (?i)^(?<log>.*?\b(?<level>trace|dbug|debug|notice|info|warn|warning|err|eror|error|crit|critical|emerg|alert|exception)\b.*) diff --git a/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster.conf.tpl b/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster.conf.tpl new file mode 100644 index 0000000000000000000000000000000000000000..8cd64e714b4dbd89e24a43ca6d2e1bd4bcac253b --- /dev/null +++ b/deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster.conf.tpl @@ -0,0 +1,68 @@ +[SERVICE] + Parsers_File /etc/td-agent-bit/parsers.conf + Parsers_File /var/run/asapo/fluent-bit-cluster-custom-parsers.conf + +{{ range ("nomad consul supervisord" | split " ") }} +[INPUT] + Name tail + Path /var/log/supervisord/{{.}}*.log + Tag asapo.{{.}} + # Tag_Regex .*/(?<service>[^-.]*)(-(?<channel>stderr|stdout))?.* + DB ${NOMAD_TASK_DIR}/logs.db + Parser level +{{ end }} + +{{ range ("influxdb grafana" | split " ") }} +[INPUT] + Name tail + Path ${NOMAD_ALLOC_DIR}/${NOMAD_ALLOC_ID}/alloc/logs/{{.}}*.0 + Tag asapo.{{.}} + # Tag_Regex .*/(?<service>.*).(?<channel>stderr|stdout).* + DB ${NOMAD_TASK_DIR}/logs.db + Parser level +{{ end }} + +[FILTER] + Name modify + Match * + Condition Key_value_matches level (?i)^d.* + Set level DEBUG + +[FILTER] + Name modify + Match * + Condition Key_value_matches level (?i)^(i.*|no.*) + Set level INFO + +[FILTER] + Name modify + Match * + Condition Key_value_matches level (?i)^w.* + Set level WARNING + +[FILTER] + Name modify + Match * + Condition Key_value_matches level (?i)^er.* + Set level ERROR + +[FILTER] + Name modify + Match * + Condition Key_value_matches level (?i)^(c.*|a.*|emerg|exception) + Set level CRITICAL + +[FILTER] + Name modify + Match * + Add level UNKNOWN + +{{ range service "influxdb" }} +[OUTPUT] + Name influxdb + Match * + Host {{ .Address }} + Port {{ .Port }} + Database logs + Tag_Keys level +{{ end }} diff --git a/deploy/asapo_orchestration_docker/scripts/monitoring.nmd.tpl b/deploy/asapo_orchestration_docker/scripts/monitoring.nmd.tpl index 33dd52f41145a042576508fcbd33d4f0b6e1651c..a86149f7da62c3ff68816eac8df86165fa716a01 100644 --- a/deploy/asapo_orchestration_docker/scripts/monitoring.nmd.tpl +++ b/deploy/asapo_orchestration_docker/scripts/monitoring.nmd.tpl @@ -196,5 +196,19 @@ job "monitoring" { } #telegraf + task "fluent-bit-cluster" { + driver = "raw_exec" + + config { + command = "/opt/td-agent-bit/bin/td-agent-bit" + args = ["-c", "local/fluent-bit.conf"] + } + + template { + source = "${scripts_dir}/fluent-bit-cluster.conf.tpl" + destination = "local/fluent-bit.conf" + change_mode = "restart" + } + } # fluent-bit-cluster } }