add nginx and co to the image

deploy/asapo_clients/run.sh

@@ -12,13 +12,17 @@ ASAPO_USER=`id -u`:`id -g`
 #SERVER_ADRESSES=
 #N_SERVERS=
 
+NOMAD_TELEMETRY=true
+NGINX_PORT_STREAM=8402
+TELEGRAF_ADDRESS="127.0.0.1:${NGINX_PORT_STREAM}"
+
 ASAPO_VAR_FILE=`pwd`/asapo_client_overwrite_vars.tfvars
 
 mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED
 chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED
 
 cd $SERVICE_DATA_CLUSTER_SHARED
-mkdir grafana
+mkdir grafana influxdb
 chmod 777 *
 
 if [ -f $ASAPO_VAR_FILE ]; then
@@ -36,12 +40,15 @@ docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \
   -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \
   -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \
   -e TF_VAR_data_dir=$DATA_GLOBAL_SHARED \
+  -e TF_VAR_nginx_port_stream=$NGINX_PORT_STREAM \
    $MOUNT_VAR_FILE \
   -e ADVERTISE_IP=$ADVERTISE_IP \
   -e RECURSORS=$RECURSORS \
   -e TF_VAR_asapo_user=$ASAPO_USER \
   -e IB_ADDRESS=$IB_ADDRESS \
   -e SERVER_ADRESSES=$SERVER_ADRESSES \
+  -e TELEGRAF_ADDRESS=$TELEGRAF_ADDRESS \
+  -e NOMAD_TELEMETRY=$NOMAD_TELEMETRY \
   -e N_SERVERS=$N_SERVERS \
   --name asapo-client --net=host -d yakser/asapo-client
 

deploy/asapo_clients/scripts/asapo.auto.tfvars

-grafana_version="latest"
-
 job_scripts_dir = "/var/run/asapo"
 
+consul_dns_port = 8600
+
 grafana_total_memory_size = "256"
 grafana_port = 3000
+grafana_version="latest"
+
+nginx_total_memory_size = 256
+nginx_port = 8401
+nginx_port_stream = 8402
+nginx_version = "1.14"
+
+influxdb_total_memory_size = "256"
+influxdb_port = 8086
+influxdb_version = "latest"
+
+telegraf_total_memory_size = "256"
+telegraf_port_stream = 8125
+telegraf_version = "latest"
+
+monitoring = true

deploy/asapo_clients/scripts/grafana.nmd.tpl

-job "grafana" {
-  datacenters = ["dc1"]
-  affinity {
-    attribute = "$${meta.asapo_service}"
-    value     = "true"
-    weight    = 100
-  }
-
-  group "grafana" {
-    count = 1
-    restart {
-      attempts = 2
-      interval = "3m"
-      delay = "15s"
-      mode = "delay"
-    }
-
-    task "grafana" {
-      driver = "docker"
-      user = "${asapo_user}"
-      config {
-        network_mode = "host"
-	    security_opt = ["no-new-privileges"]
-	    userns_mode = "host"
-        image = "grafana/grafana:${grafana_version}"
-        volumes = ["/${service_dir}/grafana:/var/lib/grafana"]
-      }
-
-      resources {
-        memory = "${grafana_total_memory_size}"
-        network {
-          port "grafana" {
-          static = "${grafana_port}"
-          }
-        }
-      }
-
-     service {
-       port = "grafana"
-       name = "grafana"
-       check {
-           name = "alive"
-           type     = "http"
-           path     = "/api/health"
-           interval = "10s"
-           timeout  = "1s"
-       }
-       check_restart {
-         limit = 2
-         grace = "90s"
-         ignore_warnings = false
-       }
-     }
-
-   } #grafana
-  }
-}

deploy/asapo_clients/scripts/grafana/grafana.ini.tpl

+[server]
+http_port = {{ env "NOMAD_PORT_grafana" }}
+
+[auth.anonymous]
+enabled = true
+org_name = Main Org.
+org_role = Admin
+
+#[auth]
+#disable_login_form = true

deploy/asapo_clients/scripts/grafana/provisioning/dashboards/dashboards.yaml

+apiVersion: 1
+
+providers:
+- name: 'client monitoring'
+  folder: ''
+  type: file
+  disableDeletion: false
+  editable: true
+  updateIntervalSeconds: 10000
+  allowUiUpdates: true
+  options:
+    path: /var/lib/grafana_config/dashboards
\ No newline at end of file

deploy/asapo_clients/scripts/grafana/provisioning/datasources/datasources.yaml.tpl

+apiVersion: 1
+
+datasources:
+  - name: InfluxDB
+    type: influxdb
+    access: proxy
+    database: telegraf
+    url: http://localhost:{{ env "NOMAD_META_nginx_port" }}/influxdb
+    jsonData:
+      httpMode: GET

deploy/asapo_clients/scripts/monitoring.nmd.tpl

+job "monitoring" {
+  datacenters = ["dc1"]
+  affinity {
+    attribute = "$${meta.asapo_service}"
+    value     = "true"
+    weight    = 100
+  }
+
+#  update {
+#    max_parallel = 1
+#    min_healthy_time = "10s"
+#    healthy_deadline = "3m"
+#    auto_revert = false
+#  }
+
+  group "monitoring" {
+    count = "%{ if monitoring }1%{ else }0%{ endif }"
+    restart {
+      attempts = 2
+      interval = "3m"
+      delay = "15s"
+      mode = "delay"
+    }
+
+     meta {
+          nginx_port = "${nginx_port}"
+      }
+
+    task "influxdb" {
+      driver = "docker"
+      user = "${asapo_user}"
+      config {
+        network_mode = "host"
+	    security_opt = ["no-new-privileges"]
+	    userns_mode = "host"
+        image = "influxdb:${influxdb_version}"
+        volumes = ["/${service_dir}/influxdb:/var/lib/influxdb"]
+      }
+
+      env {
+        PRE_CREATE_DB="asapo_receivers;asapo_brokers"
+      }
+
+      resources {
+        memory = "${influxdb_total_memory_size}"
+        network {
+          port "influxdb" {
+          static = "${influxdb_port}"
+          }
+        }
+      }
+
+     service {
+       port = "influxdb"
+       name = "influxdb"
+       check {
+           name = "alive"
+           type     = "http"
+           path     = "/ping"
+           interval = "10s"
+           timeout  = "1s"
+       }
+       check_restart {
+         limit = 2
+         grace = "90s"
+         ignore_warnings = false
+       }
+     }
+
+   } #influxdb
+
+
+    task "grafana" {
+      driver = "docker"
+      user = "${asapo_user}"
+      env {
+        GF_SERVER_DOMAIN = "$${attr.unique.hostname}"
+        GF_SERVER_ROOT_URL = "%(protocol)s://%(domain)s/monitoring/"
+        GF_PATHS_PROVISIONING = "/var/lib/grafana_config/provisioning"
+      }
+
+      config {
+        network_mode = "host"
+	    security_opt = ["no-new-privileges"]
+	    userns_mode = "host"
+        image = "grafana/grafana:${grafana_version}"
+        volumes = ["/${service_dir}/grafana:/var/lib/grafana",
+        "local/grafana:/var/lib/grafana_config",
+        "local/grafana/grafana.ini:/etc/grafana/grafana.ini"
+        ]
+      }
+
+      resources {
+        memory = "${grafana_total_memory_size}"
+        network {
+          port "grafana" {
+          static = "${grafana_port}"
+          }
+        }
+      }
+
+     service {
+       port = "grafana"
+       name = "grafana"
+       check {
+           name = "alive"
+           type     = "http"
+           path     = "/api/health"
+           interval = "10s"
+           timeout  = "1s"
+       }
+       check_restart {
+         limit = 2
+         grace = "90s"
+         ignore_warnings = false
+       }
+     }
+
+      template {
+         source        = "${scripts_dir}/grafana/grafana.ini.tpl"
+         destination   = "local/grafana/grafana.ini"
+         change_mode   = "restart"
+      }
+      template {
+         source        = "${scripts_dir}/grafana/provisioning/dashboards/dashboards.yaml"
+         destination   = "local/grafana/provisioning/dashboards/dashboards.yaml"
+         change_mode   = "restart"
+      }
+      template {
+         source        = "${scripts_dir}/grafana/provisioning/datasources/datasources.yaml.tpl"
+         destination   = "local/grafana/provisioning/datasources/datasources.yaml"
+         change_mode   = "restart"
+      }
+      template {
+         source        = "${scripts_dir}/grafana/dashboards/grafana.json"
+         destination   = "local/grafana/dashboards/grafana.json"
+         change_mode   = "restart"
+      }
+
+   } #grafana
+
+    task "telegraf" {
+      driver = "docker"
+      user = "${asapo_user}"
+       config {
+        network_mode = "host"
+	    security_opt = ["no-new-privileges"]
+	    userns_mode = "host"
+        image = "telegraf:${telegraf_version}"
+        volumes = [
+          "local/telegraf.conf:/etc/telegraf/telegraf.conf"
+        ]
+      }
+
+      resources {
+        memory = "${telegraf_total_memory_size}"
+        network {
+          port "telegraf_stream" {
+          static = "${telegraf_port_stream}"
+          }
+        }
+      }
+
+	  service {
+        name = "telegraf"
+        port = "telegraf_stream"
+        check {
+          name     = "telegraf-alive"
+          type     = "script"
+          command  = "/bin/ps"
+          args     = ["-fC","telegraf"]
+          interval = "10s"
+          timeout  = "2s"
+        }
+        check_restart {
+          limit = 2
+          grace = "15s"
+          ignore_warnings = false
+        }
+      }
+
+      template {
+         source        = "${scripts_dir}/telegraf.conf.tpl"
+         destination   = "local/telegraf.conf"
+         change_mode   = "restart"
+      }
+
+   } #telegraf
+
+  }
+}

deploy/asapo_clients/scripts/nginx.conf.tpl

+worker_processes  1;
+
+events {
+    worker_connections  1024;
+}
+
+error_log         "/tmp/nginx_error.log";
+pid               "/tmp/nginx.pid";
+
+http {
+#    include       mime.types;
+#    default_type  application/octet-stream;
+
+#    sendfile        on;
+#    tcp_nopush     on;
+
+#    keepalive_timeout  0;
+#    keepalive_timeout  65;
+
+
+    client_body_temp_path  "/tmp/client_body" 1 2;
+    proxy_temp_path        "/tmp/proxy" 1 2;
+    fastcgi_temp_path      "/tmp/fastcgi" 1 2;
+    scgi_temp_path         "/tmp/scgi" 1 2;
+    uwsgi_temp_path        "/tmp/uwsgi" 1 2;
+
+
+    resolver 127.0.0.1:{{ env "NOMAD_META_consul_dns_port" }} valid=1s;
+    server {
+          listen {{ env "NOMAD_PORT_nginx" }};
+          set $grafana_endpoint grafana.service.asapo;
+          set $influxdb_endpoint influxdb.service.asapo;
+
+   		  location /influxdb/ {
+            rewrite ^/influxdb(/.*) $1 break;
+            proxy_pass http://$influxdb_endpoint:{{ env "NOMAD_META_influxdb_port" }}$uri$is_args$args;
+          }
+
+           location /monitoring/ {
+            rewrite ^/monitoring(/.*) $1 break;
+            proxy_pass http://$grafana_endpoint:{{ env "NOMAD_META_grafana_port" }}$uri$is_args$args;
+          }
+
+      	  location /nginx-health {
+  	        return 200 "healthy\n";
+	      }
+    }
+}
+
+stream {
+    log_format  basic   '$time_iso8601 $remote_addr '
+                        '$protocol $status $bytes_sent $bytes_received '
+                        '$session_time $upstream_addr '
+                        '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
+
+    access_log      /tmp/nginx_stream.log  basic buffer=10k flush=1s;
+
+    resolver 127.0.0.1:{{ env "NOMAD_META_consul_dns_port" }} valid=1s;
+
+    map $remote_addr $upstream {
+        default telegraf.service.asapo;
+    }
+
+    server {
+        listen {{ env "NOMAD_PORT_nginx_stream" }} udp;
+        proxy_pass $upstream:{{ env "NOMAD_META_telegraf_port_stream" }};
+    }
+}

deploy/asapo_clients/scripts/nginx.nmd.tpl

+job "asapo-nginx" {
+  datacenters = ["dc1"]
+
+  type = "system"
+
+  group "nginx" {
+    count = 1
+
+    restart {
+      attempts = 2
+      interval = "3m"
+      delay = "16s"
+      mode = "delay"
+    }
+
+    task "nginx" {
+      driver = "docker"
+
+      user = "${asapo_user}"
+
+      meta {
+        telegraf_port_stream = "${telegraf_port_stream}"
+        grafana_port = "${grafana_port}"
+        influxdb_port = "${influxdb_port}"
+        consul_dns_port = "${consul_dns_port}"
+      }
+
+      config {
+        network_mode = "host"
+	    security_opt = ["no-new-privileges"]
+	    userns_mode = "host"
+        image = "nginx:${nginx_version}"
+        volumes = [
+          "local/nginx.conf:/etc/nginx/nginx.conf"
+        ]
+      }
+
+      resources {
+        cpu    = 500
+        memory = ${nginx_total_memory_size}
+        network {
+          mbits = 10
+          port "nginx" {
+          	static = ${nginx_port}
+          }
+          port "nginx_stream" {
+          	static = ${nginx_port_stream}
+          }
+        }
+      }
+
+      service {
+        port = "nginx"
+        name = "nginx"
+        check {
+          name     = "alive"
+          type     = "http"
+	      path     = "/nginx-health"
+          timeout  = "2s"
+	      interval = "10s"
+        }
+
+        check_restart {
+          limit = 2
+          grace = "15s"
+          ignore_warnings = false
+        }
+      }
+
+      template {
+         source        = "${scripts_dir}/nginx.conf.tpl"
+         destination   = "local/nginx.conf"
+         change_mode   = "restart"
+      }
+   }
+  }
+}

deploy/asapo_clients/scripts/resources.tf

-resource "nomad_job" "grafana" {
-  jobspec = "${data.template_file.grafana_template.rendered}"
+resource "nomad_job" "monitoring" {
+  jobspec = "${data.template_file.monitoring_template.rendered}"
 }
 
-
+resource "nomad_job" "nginx" {
+  jobspec = "${data.template_file.nginx_template.rendered}"
+}

deploy/asapo_clients/scripts/telegraf.conf.tpl

+[[inputs.statsd]]
+  protocol = "udp"
+  service_address = ":{{ env "NOMAD_PORT_telegraf_stream" }}"
+  namepass= ["nomad_client_allocs*","nomad_nomad_job_summary*"]
+  templates = [
+    "nomad.client.allocs.* measurement.measurement.measurement.field.field.job.task-group.alloc-id.task.hostname",
+    "nomad.nomad.job_summary.* measurement.measurement.measurement.field.job.task-group.hostname"
+  ]
+
+[[inputs.consul]]
+
+[[outputs.influxdb]]
+    urls = ["http://localhost:{{ env "NOMAD_META_nginx_port" }}/influxdb"]
+

deploy/asapo_clients/scripts/templates.tf

-data "template_file" "grafana_template" {
-  template = "${file("${var.job_scripts_dir}/grafana.nmd.tpl")}"
+data "template_file" "monitoring_template" {
+  template = "${file("${var.job_scripts_dir}/monitoring.nmd.tpl")}"
   vars = {
+  	monitoring = "${var.monitoring}"
     service_dir = "${var.service_dir}"
     grafana_version = "${var.grafana_version}"
     grafana_total_memory_size = "${var.grafana_total_memory_size}"
     grafana_port = "${var.grafana_port}"
+    grafana_version = "${var.grafana_version}"
+    grafana_total_memory_size = "${var.grafana_total_memory_size}"
+    grafana_port = "${var.grafana_port}"
+    influxdb_version = "${var.influxdb_version}"
+    influxdb_total_memory_size = "${var.influxdb_total_memory_size}"
+    influxdb_port = "${var.influxdb_port}"
+    telegraf_version = "${var.telegraf_version}"
+    telegraf_total_memory_size = "${var.telegraf_total_memory_size}"
+    telegraf_port_stream = "${var.telegraf_port_stream}"
+    nginx_port = "${var.nginx_port}"
     asapo_user = "${var.asapo_user}"
+    scripts_dir = "${var.job_scripts_dir}"
     }
 }
 
+data "template_file" "nginx_template" {
+  template = "${file("${var.job_scripts_dir}/nginx.nmd.tpl")}"
+  vars = {
+    scripts_dir = "${var.job_scripts_dir}"
+    nginx_total_memory_size = "${var.nginx_total_memory_size}"
+    nginx_version = "${var.nginx_version}"
+    nginx_port = "${var.nginx_port}"
+    nginx_port_stream = "${var.nginx_port_stream}"
+    telegraf_port_stream = "${var.telegraf_port_stream}"
+    grafana_port = "${var.grafana_port}"
+    influxdb_port = "${var.influxdb_port}"
+    asapo_user = "${var.asapo_user}"
+    consul_dns_port = "${var.consul_dns_port}"
+  }
+}
 

deploy/asapo_clients/scripts/vars.tf

 variable "asapo_user" {}
+
+variable "monitoring" {}
+
 variable "grafana_version" {}
+variable "telegraf_version" {}
+variable "influxdb_version" {}
+variable "nginx_version" {}
 
 variable "job_scripts_dir" {}
 variable "service_dir" {}
 variable "data_dir" {}
 
-
 variable "grafana_total_memory_size" {}
 variable "grafana_port" {}
+
+variable "telegraf_port_stream" {}
+variable "telegraf_total_memory_size" {}
+
+variable "influxdb_total_memory_size" {}
+variable "influxdb_port" {}
+
+variable "nginx_total_memory_size" {}
+variable "nginx_port" {}
+variable "nginx_port_stream" {}
+
+variable "consul_dns_port" {}
+

deploy/nomad_consul_docker/nomad.hcl.tpl

@@ -39,3 +39,12 @@ plugin "docker" {
   }
 }
 
+telemetry {
+  publish_allocation_metrics = $use_telemetry
+  publish_node_metrics       = $use_telemetry
+  statsd_address = "$telegraf_address"
+  collection_interval = "10s"
+}
+
+
+

deploy/nomad_consul_docker/orchestr_config.py

@@ -49,6 +49,9 @@ def set_parameters():
     d['recursors']=my_get_env('RECURSORS','["8.8.8.8"]')
     lightweight_service_nodes=my_get_env('ASAPO_LIGHTWEIGHT_SERVICE_NODES','[]')
     d['is_asapo_lightweight_service_node']=in_server_list(d['advertise_ip'],lightweight_service_nodes, True)
+    d['use_telemetry'] = my_get_env('NOMAD_TELEMETRY','false')
+    d['telegraf_address'] = my_get_env('TELEGRAF_ADDRESS','localhost:8125')
+
     return d
 
 def process_file(file_in,file_out):