Create a fluent-bit task for cluster logs

Parsing timestamps leads to unordered logs because for lines without matching timestamps the
current time is used. By always using the current time, the line order is conserved.

deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster-custom-parsers.conf

+[PARSER]
+    Name level
+    Format regex
+    # from https://github.com/grafana/grafana/blob/917860c4d396cf9ce22127fb39159fe2a1e09a5e/packages/grafana-data/src/types/logs.ts
+    Regex (?i)^(?<log>.*?\b(?<level>trace|dbug|debug|notice|info|warn|warning|err|eror|error|crit|critical|emerg|alert|exception)\b.*)

deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster.conf.tpl

+[SERVICE]
+    Parsers_File /etc/td-agent-bit/parsers.conf
+    Parsers_File /var/run/asapo/fluent-bit-cluster-custom-parsers.conf
+
+{{ range ("nomad consul supervisord" | split " ") }}
+[INPUT]
+    Name tail
+    Path /var/log/supervisord/{{.}}*.log
+    Tag asapo.{{.}}
+    # Tag_Regex .*/(?<service>[^-.]*)(-(?<channel>stderr|stdout))?.*
+    DB ${NOMAD_TASK_DIR}/logs.db
+    Parser level
+{{ end }}
+
+{{ range ("influxdb grafana" | split " ") }}
+[INPUT]
+    Name tail
+    Path ${NOMAD_ALLOC_DIR}/${NOMAD_ALLOC_ID}/alloc/logs/{{.}}*.0
+    Tag asapo.{{.}}
+    # Tag_Regex .*/(?<service>.*).(?<channel>stderr|stdout).*
+    DB ${NOMAD_TASK_DIR}/logs.db
+    Parser level
+{{ end }}
+
+[FILTER]
+    Name modify
+    Match *
+    Condition Key_value_matches level (?i)^d.*
+    Set level DEBUG
+
+[FILTER]
+    Name modify
+    Match *
+    Condition Key_value_matches level (?i)^(i.*|no.*)
+    Set level INFO
+
+[FILTER]
+    Name modify
+    Match *
+    Condition Key_value_matches level (?i)^w.*
+    Set level WARNING
+
+[FILTER]
+    Name modify
+    Match *
+    Condition Key_value_matches level (?i)^er.*
+    Set level ERROR
+
+[FILTER]
+    Name modify
+    Match *
+    Condition Key_value_matches level (?i)^(c.*|a.*|emerg|exception)
+    Set level CRITICAL
+
+[FILTER]
+    Name modify
+    Match *
+    Add level UNKNOWN
+
+{{ range service "influxdb" }}
+[OUTPUT]
+    Name influxdb
+    Match *
+    Host {{ .Address }}
+    Port {{ .Port }}
+    Database logs
+    Tag_Keys level
+{{ end }}

deploy/asapo_orchestration_docker/scripts/monitoring.nmd.tpl

@@ -196,5 +196,19 @@ job "monitoring" {
 
    } #telegraf
 
+    task "fluent-bit-cluster" {
+      driver = "raw_exec"
+
+      config {
+        command = "/opt/td-agent-bit/bin/td-agent-bit"
+        args    = ["-c", "local/fluent-bit.conf"]
+      }
+
+      template {
+        source        = "${scripts_dir}/fluent-bit-cluster.conf.tpl"
+        destination   = "local/fluent-bit.conf"
+        change_mode   = "restart"
+      }
+    } # fluent-bit-cluster
   }
 }