more nomad jobs, start logging

common/cpp/src/logger/spd_logger.cpp

@@ -26,7 +26,7 @@ void SpdLogger::SetLogLevel(LogLevel level) {
     }
 }
 std::string EncloseMsg(std::string msg) {
-    if (msg.find(":") == std::string::npos) {
+    if (msg.find("\"") != 0) {
         return std::string(R"("message":")") + msg + "\"";
     } else {
         return msg;

config/nomad/broker.nmd.in

@@ -25,7 +25,7 @@ job "broker" {
       }
 
       service {
-        name = "broker"
+        name = "asapo-broker"
         port = "broker"
         check {
           name     = "alive"

config/nomad/receiver.nmd.in

@@ -23,7 +23,7 @@ job "receiver" {
       }
 
       service {
-        name = "receiver"
+        name = "asapo-receiver"
         port = "recv"
         check {
           name     = "alive"

deploy/nomad_jobs/asapo-brokers.nmd.in

+job "asapo-brokers" {
+  datacenters = ["dc1"]
+
+  update {
+    max_parallel = 1
+    min_healthy_time = "10s"
+    healthy_deadline = "3m"
+    auto_revert = false
+  }
+
+  group "brokers" {
+    count = 1
+
+    restart {
+      attempts = 2
+      interval = "3m"
+      delay = "15s"
+      mode = "fail"
+    }
+
+    task "brokers" {
+      driver = "docker"
+
+      config {
+        network_mode = "host"
+        dns_servers = ["127.0.0.1"]
+        image = "yakser/asapo-broker:feature_ha"
+	    force_pull = true
+        volumes = ["local/config.json:/var/lib/broker/config.json"]
+        logging {
+            type = "fluentd"
+            config {
+                fluentd-address = "localhost:9881"
+                tag = "asapo.docker"
+            }
+        }
+      }
+
+      resources {
+        network {
+          port "broker" {}
+        }
+      }
+
+      service {
+        port = "broker"
+        name = "asapo-broker"
+        check {
+          name     = "asapo-broker-alive"
+          type     = "http"
+          path     = "/health"
+          interval = "10s"
+          timeout  = "2s"
+        }
+        check_restart {
+          limit = 2
+          grace = "15s"
+          ignore_warnings = false
+        }
+      }
+
+      template {
+         source        = "@NOMAD_INSTALL@/broker.json.tpl"
+         destination   = "local/config.json"
+         change_mode   = "restart"
+      }
+
+      template {
+        source        = "@NOMAD_INSTALL@/broker_secret.key"
+        destination   = "secrets/secret.key"
+        change_mode   = "restart"
+      }
+   }
+  }
+}

deploy/nomad_jobs/asapo-logging.nmd.in

@@ -20,6 +20,10 @@ job "asapo-logging" {
     task "fluentd" {
       driver = "docker"
 
+      meta {
+        change_me_to_restart = 1
+      }
+
       config {
         dns_servers = ["127.0.0.1"]
         network_mode = "host"

deploy/nomad_jobs/asapo-mongo.nmd.in

+job "asapo-mongo" {
+  datacenters = ["dc1"]
+
+  update {
+    max_parallel = 1
+    min_healthy_time = "10s"
+    healthy_deadline = "3m"
+    auto_revert = false
+  }
+
+  group "mongo" {
+    count = 1
+
+    restart {
+      attempts = 2
+      interval = "3m"
+      delay = "15s"
+      mode = "delay"
+    }
+
+    task "mongo" {
+      driver = "docker"
+
+      config {
+        network_mode = "host"
+        image = "mongo:4.0.0"
+        volumes = ["/${meta.shared_storage}/mongodb:/data/db"]
+      }
+
+      resources {
+        network {
+          port "mongo" {
+          static = 27017
+          }
+        }
+      }
+
+      service {
+        port = "mongo"
+        name = "mongo"
+        check {
+          type     = "script"
+          name     = "alive"
+          command  = "mongo"
+          args     = ["--eval","db.version()"]
+          interval = "10s"
+          timeout  = "5s"
+        }
+        check_restart {
+          limit = 2
+          grace = "90s"
+          ignore_warnings = false
+        }
+      }
+    }
+  }
+}

deploy/nomad_jobs/asapo-nginx.nmd.in

-job "nginx" {
+job "asapo-nginx" {
   datacenters = ["dc1"]
 
   type = "system"

deploy/nomad_jobs/asapo-perfmetrics.nmd.in

@@ -28,8 +28,8 @@ job "asapo-perfmetrics" {
       }
 
       resources {
-        cpu    = 500
-        memory = 256
+        cpu    = 1500
+        memory = 32000
         network {
           mbits = 10
           port "influxdb" {
@@ -74,8 +74,8 @@ job "asapo-perfmetrics" {
       }
 
       resources {
-        cpu    = 500
-        memory = 256
+        cpu    = 1500
+        memory = 2560
         network {
           mbits = 10
           port "grafana" {

deploy/nomad_jobs/asapo-receivers.nmd.in

-job "receivers" {
+job "asapo-receivers" {
   datacenters = ["dc1"]
 
   update {
@@ -25,9 +25,16 @@ job "receivers" {
         network_mode = "host"
         dns_servers = ["127.0.0.1"]
         image = "yakser/asapo-receiver:feature_ha"
-	force_pull = true
+	    force_pull = true
         volumes = ["local/config.json:/var/lib/receiver/config.json",
                    "/bldocuments/support/asapo/data:/var/lib/receiver/data"]
+        logging {
+            type = "fluentd"
+            config {
+                fluentd-address = "localhost:9881"
+                tag = "asapo.docker"
+            }
+        }
       }
 
       resources {
@@ -38,13 +45,12 @@ job "receivers" {
 
       service {
         port = "recv"
-        name = "recv"
+        name = "asapo-receiver"
         check {
-          name     = "alive"
+          name     = "asapo-receiver-alive"
           type     = "tcp"
           interval = "10s"
           timeout  = "2s"
-          initial_status =   "passing"
         }
         check_restart {
           limit = 2
@@ -61,3 +67,4 @@ job "receivers" {
    }
   }
 }
+

deploy/nomad_jobs/asapo-services.nmd.in

+job "asapo-services" {
+  datacenters = ["dc1"]
+
+  type = "service"
+
+  group "asapo-authorizer" {
+    count = 1
+
+    task "asapo-authorizer" {
+      driver = "docker"
+
+      config {
+        network_mode = "host"
+        dns_servers = ["127.0.0.1"]
+        image = "yakser/asapo-authorizer:feature_ha"
+	    force_pull = true
+        volumes = ["local/config.json:/var/lib/authorizer/config.json",
+                   "/bldocuments/support/asapo/beamtime_beamline_mapping.txt:/var/lib/authorizer/beamtime_beamline_mapping.txt",
+                   "/bldocuments/support/asapo/ip_beamtime_mapping:/var/lib/authorizer/ip_beamtime_mapping"]
+        logging {
+            type = "fluentd"
+            config {
+                fluentd-address = "localhost:9881"
+                tag = "asapo.docker"
+            }
+        }
+      }
+
+      resources {
+        cpu    = 500 # 500 MHz
+        memory = 256 # 256MB
+        network {
+          port "authorizer" {
+            static = "5007"
+          }
+        }
+      }
+
+      service {
+        name = "asapo-authorizer"
+        port = "authorizer"
+        check {
+          name     = "alive"
+          type     = "http"
+          path     = "/health-check"
+          interval = "10s"
+          timeout  = "2s"
+          initial_status =   "passing"
+        }
+        check_restart {
+          limit = 2
+          grace = "15s"
+          ignore_warnings = false
+        }
+      }
+
+      template {
+         source        = "@NOMAD_INSTALL@/authorizer.json.tpl"
+         destination   = "local/config.json"
+         change_mode   = "restart"
+      }
+   }
+  } #authorizer
+  group "asapo-discovery" {
+    count = 1
+
+    task "asapo-discovery" {
+      driver = "docker"
+
+      config {
+        network_mode = "host"
+        dns_servers = ["127.0.0.1"]
+        image = "yakser/asapo-discovery:feature_ha"
+	    force_pull = true
+        volumes = ["local/config.json:/var/lib/discovery/config.json"]
+        logging {
+            type = "fluentd"
+            config {
+                fluentd-address = "localhost:9881"
+                tag = "asapo.docker"
+            }
+        }
+      }
+
+      resources {
+        cpu    = 500 # 500 MHz
+        memory = 256 # 256MB
+        network {
+          port "discovery" {
+            static = "5006"
+          }
+        }
+      }
+
+      service {
+        name = "asapo-discovery"
+        port = "discovery"
+        check {
+          name     = "alive"
+          type     = "http"
+          path     = "/receivers"
+          interval = "10s"
+          timeout  = "2s"
+          initial_status =   "passing"
+        }
+        check_restart {
+          limit = 2
+          grace = "15s"
+          ignore_warnings = false
+        }
+
+      }
+
+      template {
+         source        = "@NOMAD_INSTALL@/discovery.json.tpl"
+         destination   = "local/config.json"
+         change_mode   = "restart"
+      }
+   }
+  }
+}

deploy/nomad_jobs/authorizer.json.tpl

+{
+  "Port": {{ env "NOMAD_PORT_authorizer" }},
+  "LogLevel":"debug",
+  "AlwaysAllowedBeamtimes":[{"BeamtimeId":"asapo_test","Beamline":"test"},
+  {"BeamtimeId":"asapo_test1","Beamline":"test1"},
+  {"BeamtimeId":"asapo_test2","Beamline":"test2"}],
+  "BeamtimeBeamlineMappingFile":"//var//lib//authorizer//beamtime_beamline_mapping.txt",
+  "IpBeamlineMappingFolder":"//var//lib//authorizer//ip_beamtime_mapping"
+}
+
+

deploy/nomad_jobs/broker.json.tpl

+{
+  "BrokerDbAddress":"mongo.service.asapo:27017",
+  "MonitorDbAddress":"influxdb.service.asapo:8086",
+  "MonitorDbName": "asapo_brokers",
+  "port":{{ env "NOMAD_PORT_broker" }},
+  "LogLevel":"info",
+  "SecretFile":"/secrets/secret.key"
+}
\ No newline at end of file

deploy/nomad_jobs/discovery.json.tpl

+{
+  "Mode": "consul",
+  "Receiver": {
+    "MaxConnections": 32
+  },
+  "Port": {{ env "NOMAD_PORT_discovery" }},
+  "LogLevel": "{{ keyOrDefault "log_level" "info" }}"
+}
+
+

deploy/nomad_jobs/fluentd.conf

+<source>
+  @type forward
+  port 24224
+  source_hostname_key source_addr
+  bind 0.0.0.0
+</source>
+
 <source>
  @type http
  port 9880
@@ -7,6 +14,23 @@
  time_format %Y-%m-%d %H:%M:%S.%N
 </source>
 
+<filter asapo.docker>
+  @type parser
+  key_name log
+  format json
+  time_format %Y-%m-%d %H:%M:%S.%N
+  reserve_data true
+</filter>
+
+<filter asapo.docker>
+  @type record_transformer
+  enable_ruby
+  remove_keys ["log","container_id","container_name"]
+  <record>
+   source_addr ${record["source_addr"].split('.')[0]}
+ </record>
+</filter>
+
 <match asapo.**>
 @type copy
 <store>
@@ -28,3 +52,4 @@
   path /shared/asapo-logs
   </store>
 </match>
+

deploy/nomad_jobs/init_influx.sh

 #!/usr/bin/env bash
 
-influx=`dig +short @127.0.0.1  influxdb.service.asapo | head -1`
+influx=`dig +short @127.0.0.1 influxdb.service.asapo | head -1`
 
-curl -i -XPOST http://${influx}:8086/query --data-urlencode "q=CREATE DATABASE asapo_receivers"
+databases="asapo_receivers asapo_brokers"
\ No newline at end of file
+
+for database in $databases
+do
+    curl -i -XPOST http://${influx}:8086/query --data-urlencode "q=CREATE DATABASE $database"
+done
\ No newline at end of file

deploy/nomad_jobs/nginx.conf.tpl

@@ -17,8 +17,8 @@ http {
     resolver 127.0.0.1:53 valid=1s;
     server {
           listen {{ env "NOMAD_PORT_nginx" }};
-          set $discovery_endpoint discovery.service.asapo;
-          set $authorizer_endpoint authorizer.service.asapo;
+          set $discovery_endpoint asapo-discovery.service.asapo;
+#          set $authorizer_endpoint asapo-authorizer.service.asapo;
           set $fluentd_endpoint fluentd.service.asapo;
           set $kibana_endpoint kibana.service.asapo;
           set $grafana_endpoint grafana.service.asapo;
@@ -35,7 +35,6 @@ http {
 
           location /logsview/ {
             proxy_pass http://$kibana_endpoint:5601$uri$is_args$args;
-            proxy_ignore_client_abort on;
             proxy_set_header  X-Real-IP  $remote_addr;
             proxy_set_header  X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header  Host $http_host;
@@ -46,13 +45,29 @@ http {
             proxy_pass http://$grafana_endpoint:3000$uri$is_args$args;
           }
 
-          location /authorizer/ {
-             rewrite ^/authorizer(/.*) $1 break;
-             proxy_pass http://$authorizer_endpoint:5007$uri$is_args$args;
-          }
+#          location /authorizer/ {
+#             rewrite ^/authorizer(/.*) $1 break;
+#             proxy_pass http://$authorizer_endpoint:5007$uri$is_args$args;
+#          }
 
       	  location /nginx-health {
   	        return 200 "healthy\n";
 	      }
     }
 }
+
+stream {
+    resolver 127.0.0.1:53 valid=1s;
+
+    map $remote_addr $upstream {
+        default fluentd.service.asapo;
+    }
+
+
+    server {
+        listen     9881;
+        proxy_pass $upstream:24224;
+    }
+}
+
+

deploy/nomad_jobs/receiver.json.tpl

 {
   "MonitorDbAddress":"influxdb.service.asapo:8086",
   "MonitorDbName": "asapo_receivers",
-  "BrokerDbAddress":"localhost:27017",
-  "AuthorizationServer": "authorizer.service.asapo:8400",
+  "BrokerDbAddress":"mongo.service.asapo:27017",
+  "AuthorizationServer": "asapo-authorizer.service.asapo:5007",
   "AuthorizationInterval": 10000,
   "ListenPort": {{ env "NOMAD_PORT_recv" }},
   "Tag": "{{ env "NOMAD_ADDR_recv" }}",

deploy/nomad_jobs/start_asapo.sh

+#!/usr/bin/env bash
+
+nomad run asapo-nginx.nmd
+nomad run asapo-logging.nmd
+nomad run asapo-mongo.nmd
+nomad run asapo-services.nmd
+nomad run asapo-perfmetrics.nmd
+nomad run asapo-receivers.nmd
+nomad run asapo-brokers.nmd

deploy/nomad_jobs/stop_asapo.sh

+#!/usr/bin/env bash
+
+nomad stop asapo-nginx
+nomad stop asapo-logging
+nomad stop asapo-mongo
+nomad stop asapo-services
+nomad stop asapo-perfmetrics
+nomad stop asapo-receivers
+nomad stop asapo-brokers

discovery/src/asapo_discovery/request_handler/request_handler_consul.go

@@ -53,7 +53,7 @@ func (rh *ConsulRequestHandler) GetReceivers() ([]byte, error) {
 	}
 	var response Responce
 	var err error
-	response.Uris, err = rh.GetServices("receiver")
+	response.Uris, err = rh.GetServices("asapo-receiver")
 	if err != nil {
 		return nil, err
 	}
@@ -65,7 +65,7 @@ func (rh *ConsulRequestHandler) GetBroker() ([]byte, error) {
 	if (rh.client == nil) {
 		return nil, errors.New("consul client not connected")
 	}
-	response, err := rh.GetServices("broker")
+	response, err := rh.GetServices("asapo-broker")
 	if err != nil {
 		return nil, err
 	}