discover mongodb via discovery service, not dns

deploy/docker/cluster/jobs/asapo-services.nmd.tpl → deploy/docker/cluster/scripts/asapo-services.nmd.tpl

@@ -56,12 +56,12 @@ job "asapo-services" {
       }
 
       template {
-         source        = "/usr/local/nomad_jobs/authorizer.json.tpl"
+         source        = "${scripts_dir}/authorizer.json.tpl"
          destination   = "local/config.json"
          change_mode   = "restart"
       }
       template {
-        source        = "/usr/local/nomad_jobs/auth_secret.key"
+        source        = "${scripts_dir}/auth_secret.key"
         destination   = "local/secret.key"
         change_mode   = "restart"
       }
@@ -79,14 +79,16 @@ job "asapo-services" {
         image = "yakser/asapo-discovery${image_suffix}"
 	    force_pull = true
         volumes = ["local/config.json:/var/lib/discovery/config.json"]
+        %{ if fluentd_logs }
         logging {
-            type = "fluentd"
-            config {
-                fluentd-address = "localhost:9881"
-                fluentd-async-connect = true
-                tag = "asapo.docker"
-            }
+        type = "fluentd"
+        config {
+        fluentd-address = "localhost:9881"
+        fluentd-async-connect = true
+        tag = "asapo.docker"
+        }
         }
+        %{endif}
       }
 
       resources {
@@ -119,7 +121,7 @@ job "asapo-services" {
       }
 
       template {
-         source        = "/usr/local/nomad_jobs/discovery.json.tpl"
+         source        = "${scripts_dir}/discovery.json.tpl"
          destination   = "local/config.json"
          change_mode   = "restart"
       }

deploy/docker/cluster/jobs/asapo.auto.tfvars → deploy/docker/cluster/scripts/asapo.auto.tfvars

 nginx_version = "1.14"
+
+
+
 asapo_imagename_suffix="-dev"
 asapo_image_tag = "feature_virtualized-deployment.latest"
 

deploy/docker/cluster/scripts/broker.json.tpl

+{
+  "DatabaseServer":"auto",
+  "DiscoveryServer": "localhost:8400/discovery",
+  "PerformanceDbServer":"localhost:8400/influxdb",
+  "PerformanceDbName": "asapo_brokers",
+  "port":{{ env "NOMAD_PORT_broker" }},
+  "LogLevel":"info",
+  "SecretFile":"/local/secret.key"
+}

deploy/docker/cluster/jobs/nginx.conf.tpl → deploy/docker/cluster/scripts/nginx.conf.tpl

@@ -22,15 +22,9 @@ http {
           set $fluentd_endpoint fluentd.service.asapo;
           set $kibana_endpoint kibana.service.asapo;
           set $grafana_endpoint grafana.service.asapo;
-          set $mongo_endpoint mongo.service.asapo;
           set $influxdb_endpoint influxdb.service.asapo;
           set $elasticsearch_endpoint elasticsearch.service.asapo;
 
-   		  location /mongo/ {
-            rewrite ^/mongo(/.*) $1 break;
-            proxy_pass http://$mongo_endpoint:27017$uri$is_args$args;
-          }
-
    		  location /influxdb/ {
             rewrite ^/influxdb(/.*) $1 break;
             proxy_pass http://$influxdb_endpoint:8086$uri$is_args$args;
@@ -81,7 +75,6 @@ stream {
         default fluentd.service.asapo;
     }
 
-
     server {
         listen     9881;
         proxy_pass $upstream:24224;

deploy/docker/cluster/scripts/provider.tf

+provider "nomad" {
+  address = "http://localhost:4646"
+}
+

deploy/docker/cluster/jobs/receiver.json.tpl → deploy/docker/cluster/scripts/receiver.json.tpl

 {
-  "MonitorDbAddress":"localhost:8400/influxdb",
-  "MonitorDbName": "asapo_receivers",
-  "BrokerDbAddress":"localhost:8400/mongo",
+  "PerformanceDbServer":"localhost:8400/influxdb",
+  "PerformanceDbName": "asapo_receivers",
+  "DatabaseServer":"auto",
+  "DiscoveryServer": "localhost:8400/discovery",
   "AuthorizationServer": "localhost:8400/authorizer",
   "AuthorizationInterval": 10000,
   "ListenPort": {{ env "NOMAD_PORT_recv" }},
@@ -11,7 +12,7 @@
   },
   "DataCache": {
     "Use": true,
-    "SizeGB": 30,
+    "SizeGB": {{ env "NOMAD_META_receiver_dataserver_cache_size" }},
     "ReservedShare": 10
   },
   "Tag": "{{ env "NOMAD_ADDR_recv" }}",

deploy/docker/cluster/scripts/resources.tf

+resource "nomad_job" "asapo-perfmetrics" {
+  jobspec = "${data.template_file.asapo_perfmetrics.rendered}"
+}
+
+resource "nomad_job" "asapo-mongo" {
+  jobspec = "${data.template_file.asapo_mongo.rendered}"
+}
+
+resource "nomad_job" "asapo-nginx" {
+  jobspec = "${data.template_file.nginx.rendered}"
+}
+
+resource "nomad_job" "asapo-services" {
+  jobspec = "${data.template_file.asapo_services.rendered}"
+}
+
+resource "nomad_job" "asapo-receivers" {
+  jobspec = "${data.template_file.asapo_receivers.rendered}"
+}
+
+resource "nomad_job" "asapo-brokers" {
+  jobspec = "${data.template_file.asapo_brokers.rendered}"
+}

deploy/docker/cluster/scripts/templates.tf

+data "template_file" "nginx" {
+  template = "${file("${var.job_scripts_dir}/asapo-nginx.nmd.tpl")}"
+  vars = {
+    scripts_dir = "${var.job_scripts_dir}"
+    nginx_version = "${var.nginx_version}"
+  }
+}
+
+data "template_file" "asapo_services" {
+  template = "${file("${var.job_scripts_dir}/asapo-services.nmd.tpl")}"
+  vars = {
+    scripts_dir = "${var.job_scripts_dir}"
+    image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}"
+    fluentd_logs = "${var.fluentd_logs}"
+  }
+}
+
+data "template_file" "asapo_receivers" {
+  template = "${file("${var.job_scripts_dir}/asapo-receivers.nmd.tpl")}"
+  vars = {
+    scripts_dir = "${var.job_scripts_dir}"
+    data_dir = "${var.data_dir}"
+    image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}"
+    fluentd_logs = "${var.fluentd_logs}"
+    receiver_total_memory_size = "${var.receiver_total_memory_size}"
+    receiver_dataserver_cache_size = "${var.receiver_dataserver_cache_size}"
+  }
+}
+
+data "template_file" "asapo_brokers" {
+  template = "${file("${var.job_scripts_dir}/asapo-brokers.nmd.tpl")}"
+  vars = {
+    scripts_dir = "${var.job_scripts_dir}"
+    image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}"
+    fluentd_logs = "${var.fluentd_logs}"
+  }
+}
+
+
+data "template_file" "asapo_perfmetrics" {
+  template = "${file("${var.job_scripts_dir}/asapo-perfmetrics.nmd.tpl")}"
+  vars = {
+    service_dir = "${var.service_dir}"
+    influxdb_version = "${var.influxdb_version}"
+    grafana_version = "${var.grafana_version}"
+    grafana_total_memory_size = "${var.grafana_total_memory_size}"
+    grafana_port = "${var.grafana_port}"
+    influxdb_total_memory_size = "${var.influxdb_total_memory_size}"
+    influxdb_port = "${var.influxdb_port}"
+
+    }
+}
+
+
+data "template_file" "asapo_mongo" {
+  template = "${file("${var.job_scripts_dir}/asapo-mongo.nmd.tpl")}"
+  vars = {
+    service_dir = "${var.service_dir}"
+    mongo_version = "${var.mongo_version}"
+    mongo_total_memory_size = "${var.mongo_total_memory_size}"
+    mongo_port = "${var.mongo_port}"
+  }
+}

deploy/docker/cluster/scripts/vars.tf

+variable "fluentd_logs" {
+ default = true
+}
+
+variable "nginx_version" {
+  default = "latest"
+}
+
+variable "grafana_version" {
+  default = "latest"
+}
+
+
+variable "influxdb_version" {
+  default = "latest"
+}
+
+variable "asapo_imagename_suffix" {
+  default = ""
+}
+
+variable "asapo_image_tag" {
+#  default = "latest"
+}
+
+variable "job_scripts_dir" {
+  default = "/var/run/asapo"
+}
+
+variable "service_dir" {
+}
+
+variable "data_dir" {
+}
+
+
+variable "receiver_total_memory_size" {
+  default = "2000" #mb
+}
+
+variable "receiver_dataserver_cache_size" {
+  default = "1"
+}
+
+variable "grafana_total_memory_size" {
+  default = "2000" #mb
+}
+
+variable "influxdb_total_memory_size" {
+  default = "2000" #mb
+}
+
+variable "grafana_port" {
+  default = "3000"
+}
+
+variable "influxdb_port" {
+  default = "8086"
+}
+
+
+variable "mongo_port" {
+  default = "27017"
+}
+
+
+variable "mongo_version" {
+  default = "4.0.0"
+}
+
+variable "mongo_total_memory_size" {
+  default = "300"
+}
+

deploy/docker/cluster/supervisord.conf

@@ -12,5 +12,5 @@ command=/bin/consul agent -dev -client 0.0.0.0 -domain asapo -recursor=8.8.8.8
 #-config-dir=/etc/consul.d
 
 [program:nomad]
-command=/bin/nomad agent -data-dir=/tmp/nomad -dev -client -bind 0.0.0.0
+command=/bin/nomad agent -dev -client -bind 0.0.0.0 -alloc-dir=%(ENV_NOMAD_ALLOC)s
 # -config=/etc/nomad.d

deploy/docker/cluster/tf_run.sh

+#!/usr/bin/env bash
+
+docker exec -w /var/run/asapo asapo terraform init
+
+docker exec -w /var/run/asapo asapo terraform apply -auto-approve -var fluentd_logs=false
\ No newline at end of file

deploy/nomad_jobs/broker.json.tpl

 {
-  "BrokerDbAddress":"localhost:8400/mongo",
-  "MonitorDbAddress":"localhost:8400/influxdb",
-  "MonitorDbName": "asapo_brokers",
+  "DatabaseServer":"auto",
+  "DiscoveryServer": "localhost:8400/discovery",
+  "PerformanceDbServer":"localhost:8400/influxdb",
+  "PerformanceDbName": "asapo_brokers",
   "port":{{ env "NOMAD_PORT_broker" }},
   "LogLevel":"info",
   "SecretFile":"/secrets/secret.key"

deploy/nomad_jobs/nginx.conf.tpl

@@ -22,15 +22,9 @@ http {
           set $fluentd_endpoint fluentd.service.asapo;
           set $kibana_endpoint kibana.service.asapo;
           set $grafana_endpoint grafana.service.asapo;
-          set $mongo_endpoint mongo.service.asapo;
           set $influxdb_endpoint influxdb.service.asapo;
           set $elasticsearch_endpoint elasticsearch.service.asapo;
 
-   		  location /mongo/ {
-            rewrite ^/mongo(/.*) $1 break;
-            proxy_pass http://$mongo_endpoint:27017$uri$is_args$args;
-          }
-
    		  location /influxdb/ {
             rewrite ^/influxdb(/.*) $1 break;
             proxy_pass http://$influxdb_endpoint:8086$uri$is_args$args;

deploy/nomad_jobs/receiver.json.tpl

 {
-  "MonitorDbAddress":"localhost:8400/influxdb",
-  "MonitorDbName": "asapo_receivers",
-  "BrokerDbAddress":"localhost:8400/mongo",
+  "PerformanceDbServer":"localhost:8400/influxdb",
+  "PerformanceDbName": "asapo_receivers",
+  "DatabaseServer":"auto",
+  "DiscoveryServer": "localhost:8400/discovery",
   "AuthorizationServer": "localhost:8400/authorizer",
   "AuthorizationInterval": 10000,
   "ListenPort": {{ env "NOMAD_PORT_recv" }},

discovery/src/asapo_discovery/request_handler/request_handler.go

@@ -5,6 +5,7 @@ import "asapo_common/utils"
 type Agent interface {
 	GetReceivers() ([]byte, error)
 	GetBroker() ([]byte, error)
+	GetMongo() ([]byte, error)
 	Init(settings utils.Settings) error
 }