From 29bb06db8a9016a720acb735c6cb2e295dbb515d Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon, 2 Sep 2019 15:14:25 +0200
Subject: [PATCH] all discovery via nginx
---
deploy/nomad_jobs/broker.json.tpl | 4 +-
deploy/nomad_jobs/fluentd.conf | 3 +-
deploy/nomad_jobs/kibana.yml | 2 +-
deploy/nomad_jobs/nginx.conf.tpl | 28 +++++--
deploy/nomad_jobs/receiver.json.tpl | 6 +-
.../authorizer/check_authorize/check_linux.sh | 4 +-
tests/automatic/settings/nginx.conf.tpl | 60 +++++++++++----
.../python_tests/producer/authorizer.nmd | 2 +-
.../python_tests/producer/nginx.conf.tpl | 75 +++++++++++++++----
.../receiver_debug_local/nginx.conf.tpl | 75 +++++++++++++++----
10 files changed, 204 insertions(+), 55 deletions(-)
diff --git a/deploy/nomad_jobs/broker.json.tpl b/deploy/nomad_jobs/broker.json.tpl
index fe73cff79..ddaeb1486 100644
--- a/deploy/nomad_jobs/broker.json.tpl
+++ b/deploy/nomad_jobs/broker.json.tpl
@@ -1,6 +1,6 @@
{
- "BrokerDbAddress":"mongo.service.asapo:27017",
- "MonitorDbAddress":"influxdb.service.asapo:8086",
+ "BrokerDbAddress":"localhost:8400/mongo",
+ "MonitorDbAddress":"localhost:8400/influxdb",
"MonitorDbName": "asapo_brokers",
"port":{{ env "NOMAD_PORT_broker" }},
"LogLevel":"info",
diff --git a/deploy/nomad_jobs/fluentd.conf b/deploy/nomad_jobs/fluentd.conf
index 5be45afa2..948c5109d 100644
--- a/deploy/nomad_jobs/fluentd.conf
+++ b/deploy/nomad_jobs/fluentd.conf
@@ -35,8 +35,7 @@
@type copy
<store>
@type elasticsearch
- host elasticsearch.service.asapo
- port 9200
+ hosts localhost:8400/elasticsearch
flush_interval 5s
logstash_format true
time_key_format %Y-%m-%dT%H:%M:%S.%N
diff --git a/deploy/nomad_jobs/kibana.yml b/deploy/nomad_jobs/kibana.yml
index 4b8e272b4..10675706c 100644
--- a/deploy/nomad_jobs/kibana.yml
+++ b/deploy/nomad_jobs/kibana.yml
@@ -1,5 +1,5 @@
elasticsearch:
- url: "http://elasticsearch.service.asapo:9200"
+ url: "http://localhost:8400/elasticsearch"
server:
basePath: "/logsview"
rewriteBasePath: true
diff --git a/deploy/nomad_jobs/nginx.conf.tpl b/deploy/nomad_jobs/nginx.conf.tpl
index 854cd7a18..c53d8c503 100644
--- a/deploy/nomad_jobs/nginx.conf.tpl
+++ b/deploy/nomad_jobs/nginx.conf.tpl
@@ -18,10 +18,28 @@ http {
server {
listen {{ env "NOMAD_PORT_nginx" }};
set $discovery_endpoint asapo-discovery.service.asapo;
-# set $authorizer_endpoint asapo-authorizer.service.asapo;
+ set $authorizer_endpoint asapo-authorizer.service.asapo;
set $fluentd_endpoint fluentd.service.asapo;
set $kibana_endpoint kibana.service.asapo;
set $grafana_endpoint grafana.service.asapo;
+ set $mongo_endpoint mongo.service.asapo;
+ set $influxdb_endpoint influxdb.service.asapo;
+ set $elasticsearch_endpoint elasticsearch.service.asapo;
+
+ location /mongo/ {
+ rewrite ^/mongo(/.*) $1 break;
+ proxy_pass http://$mongo_endpoint:27017$uri$is_args$args;
+ }
+
+ location /influxdb/ {
+ rewrite ^/influxdb(/.*) $1 break;
+ proxy_pass http://$influxdb_endpoint:8086$uri$is_args$args;
+ }
+
+ location /elasticsearch/ {
+ rewrite ^/elasticsearch(/.*) $1 break;
+ proxy_pass http://$elasticsearch_endpoint:9200$uri$is_args$args;
+ }
location /discovery/ {
rewrite ^/discovery(/.*) $1 break;
@@ -45,10 +63,10 @@ http {
proxy_pass http://$grafana_endpoint:3000$uri$is_args$args;
}
-# location /authorizer/ {
-# rewrite ^/authorizer(/.*) $1 break;
-# proxy_pass http://$authorizer_endpoint:5007$uri$is_args$args;
-# }
+ location /authorizer/ {
+ rewrite ^/authorizer(/.*) $1 break;
+ proxy_pass http://$authorizer_endpoint:5007$uri$is_args$args;
+ }
location /nginx-health {
return 200 "healthy\n";
diff --git a/deploy/nomad_jobs/receiver.json.tpl b/deploy/nomad_jobs/receiver.json.tpl
index 1d669fc6f..4410725d8 100644
--- a/deploy/nomad_jobs/receiver.json.tpl
+++ b/deploy/nomad_jobs/receiver.json.tpl
@@ -1,8 +1,8 @@
{
- "MonitorDbAddress":"influxdb.service.asapo:8086",
+ "MonitorDbAddress":"localhost:8400/influxdb",
"MonitorDbName": "asapo_receivers",
- "BrokerDbAddress":"mongo.service.asapo:27017",
- "AuthorizationServer": "asapo-authorizer.service.asapo:5007",
+ "BrokerDbAddress":"localhost:8400/mongo",
+ "AuthorizationServer": "localhost:8400/authorizer",
"AuthorizationInterval": 10000,
"ListenPort": {{ env "NOMAD_PORT_recv" }},
"DataServer": {
diff --git a/tests/automatic/authorizer/check_authorize/check_linux.sh b/tests/automatic/authorizer/check_authorize/check_linux.sh
index 83f157b28..4ff8c44ce 100644
--- a/tests/automatic/authorizer/check_authorize/check_linux.sh
+++ b/tests/automatic/authorizer/check_authorize/check_linux.sh
@@ -7,11 +7,11 @@ trap Cleanup EXIT
Cleanup() {
echo cleanup
kill -9 $authorizeid
+ sleep 1
}
$@ -config settings.json &
-
-sleep 0.3
+sleep 1
authorizeid=`echo $!`
curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%stream%","OriginHost":"127.0.0.1:5555"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep c20180508-000-COM20181
diff --git a/tests/automatic/settings/nginx.conf.tpl b/tests/automatic/settings/nginx.conf.tpl
index a545307b3..8f9608e57 100644
--- a/tests/automatic/settings/nginx.conf.tpl
+++ b/tests/automatic/settings/nginx.conf.tpl
@@ -1,5 +1,4 @@
worker_processes 1;
-daemon off;
events {
worker_connections 1024;
@@ -17,26 +16,61 @@ http {
resolver 127.0.0.1:8600 valid=1s;
server {
- listen {{ env "NOMAD_PORT_nginx" }};
+ listen {{ env "NOMAD_PORT_nginx" }};
set $discovery_endpoint discovery.service.asapo;
set $authorizer_endpoint authorizer.service.asapo;
- # set $fluentd_endpoint localhost;
+ set $fluentd_endpoint fluentd.service.asapo;
+ set $kibana_endpoint kibana.service.asapo;
+ set $grafana_endpoint grafana.service.asapo;
+ set $mongo_endpoint mongo.service.asapo;
+ set $influxdb_endpoint influxdb.service.asapo;
+ set $elasticsearch_endpoint elasticsearch.service.asapo;
+
+ location /mongo/ {
+ rewrite ^/mongo(/.*) $1 break;
+ proxy_pass http://$mongo_endpoint:27017$uri$is_args$args;
+ }
+
+ location /influxdb/ {
+ rewrite ^/influxdb(/.*) $1 break;
+ proxy_pass http://$influxdb_endpoint:8086$uri$is_args$args;
+ }
+
+ location /elasticsearch/ {
+ rewrite ^/elasticsearch(/.*) $1 break;
+ proxy_pass http://$elasticsearch_endpoint:9200$uri$is_args$args;
+ }
+
location /discovery/ {
rewrite ^/discovery(/.*) $1 break;
- proxy_pass http://$discovery_endpoint:5006$uri;
+ proxy_pass http://$discovery_endpoint:5006$uri$is_args$args;
}
+
location /logs/ {
- # rewrite ^/logs(/.*) $1 break;
- proxy_pass http://localhost:9880/asapo;
+ rewrite ^/logs(/.*) $1 break;
+ proxy_pass http://$fluentd_endpoint:9880$uri$is_args$args;
+ }
+
+ location /logsview/ {
+ proxy_pass http://$kibana_endpoint:5601$uri$is_args$args;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $http_host;
}
- location /authorizer/ {
- rewrite ^/authorizer(/.*) $1 break;
- proxy_pass http://$authorizer_endpoint:5007$uri;
- }
+ location /performance/ {
+ rewrite ^/performance(/.*) $1 break;
+ proxy_pass http://$grafana_endpoint:3000$uri$is_args$args;
+ }
- location /nginx-health {
- return 200 "healthy\n";
- }
+ location /authorizer/ {
+ rewrite ^/authorizer(/.*) $1 break;
+ proxy_pass http://$authorizer_endpoint:5007$uri$is_args$args;
+ }
+
+ location /nginx-health {
+ return 200 "healthy\n";
+ }
}
}
+
diff --git a/tests/manual/python_tests/producer/authorizer.nmd b/tests/manual/python_tests/producer/authorizer.nmd
index 8b32105cf..e03b1538f 100644
--- a/tests/manual/python_tests/producer/authorizer.nmd
+++ b/tests/manual/python_tests/producer/authorizer.nmd
@@ -6,7 +6,7 @@ job "authorizer" {
group "group" {
count = 1
- task "authorizer" {
+ task "asapo-authorizer" {
driver = "raw_exec"
config {
diff --git a/tests/manual/python_tests/producer/nginx.conf.tpl b/tests/manual/python_tests/producer/nginx.conf.tpl
index a545307b3..b784d0725 100644
--- a/tests/manual/python_tests/producer/nginx.conf.tpl
+++ b/tests/manual/python_tests/producer/nginx.conf.tpl
@@ -1,5 +1,4 @@
worker_processes 1;
-daemon off;
events {
worker_connections 1024;
@@ -17,26 +16,76 @@ http {
resolver 127.0.0.1:8600 valid=1s;
server {
- listen {{ env "NOMAD_PORT_nginx" }};
+ listen {{ env "NOMAD_PORT_nginx" }};
set $discovery_endpoint discovery.service.asapo;
set $authorizer_endpoint authorizer.service.asapo;
- # set $fluentd_endpoint localhost;
+ set $fluentd_endpoint fluentd.service.asapo;
+ set $kibana_endpoint kibana.service.asapo;
+ set $grafana_endpoint grafana.service.asapo;
+ set $mongo_endpoint mongo.service.asapo;
+ set $influxdb_endpoint influxdb.service.asapo;
+ set $elasticsearch_endpoint elasticsearch.service.asapo;
+
+ location /mongo/ {
+ rewrite ^/mongo(/.*) $1 break;
+ proxy_pass http://$mongo_endpoint:27017$uri$is_args$args;
+ }
+
+ location /influxdb/ {
+ rewrite ^/influxdb(/.*) $1 break;
+ proxy_pass http://$influxdb_endpoint:8086$uri$is_args$args;
+ }
+
+ location /elasticsearch/ {
+ rewrite ^/elasticsearch(/.*) $1 break;
+ proxy_pass http://$elasticsearch_endpoint:9200$uri$is_args$args;
+ }
+
location /discovery/ {
rewrite ^/discovery(/.*) $1 break;
- proxy_pass http://$discovery_endpoint:5006$uri;
+ proxy_pass http://$discovery_endpoint:5006$uri$is_args$args;
}
+
location /logs/ {
- # rewrite ^/logs(/.*) $1 break;
- proxy_pass http://localhost:9880/asapo;
+ rewrite ^/logs(/.*) $1 break;
+ proxy_pass http://$fluentd_endpoint:9880$uri$is_args$args;
+ }
+
+ location /logsview/ {
+ proxy_pass http://$kibana_endpoint:5601$uri$is_args$args;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $http_host;
+ }
+
+ location /performance/ {
+ rewrite ^/performance(/.*) $1 break;
+ proxy_pass http://$grafana_endpoint:3000$uri$is_args$args;
}
- location /authorizer/ {
- rewrite ^/authorizer(/.*) $1 break;
- proxy_pass http://$authorizer_endpoint:5007$uri;
- }
+ location /authorizer/ {
+ rewrite ^/authorizer(/.*) $1 break;
+ proxy_pass http://$authorizer_endpoint:5007$uri$is_args$args;
+ }
+
+ location /nginx-health {
+ return 200 "healthy\n";
+ }
+ }
+}
+
+stream {
+ resolver 127.0.0.1:8600 valid=1s;
- location /nginx-health {
- return 200 "healthy\n";
- }
+ map $remote_addr $upstream {
+ default fluentd.service.asapo;
+ }
+
+
+ server {
+ listen 9881;
+ proxy_pass $upstream:24224;
}
}
+
+
diff --git a/tests/manual/receiver_debug_local/nginx.conf.tpl b/tests/manual/receiver_debug_local/nginx.conf.tpl
index a545307b3..b784d0725 100644
--- a/tests/manual/receiver_debug_local/nginx.conf.tpl
+++ b/tests/manual/receiver_debug_local/nginx.conf.tpl
@@ -1,5 +1,4 @@
worker_processes 1;
-daemon off;
events {
worker_connections 1024;
@@ -17,26 +16,76 @@ http {
resolver 127.0.0.1:8600 valid=1s;
server {
- listen {{ env "NOMAD_PORT_nginx" }};
+ listen {{ env "NOMAD_PORT_nginx" }};
set $discovery_endpoint discovery.service.asapo;
set $authorizer_endpoint authorizer.service.asapo;
- # set $fluentd_endpoint localhost;
+ set $fluentd_endpoint fluentd.service.asapo;
+ set $kibana_endpoint kibana.service.asapo;
+ set $grafana_endpoint grafana.service.asapo;
+ set $mongo_endpoint mongo.service.asapo;
+ set $influxdb_endpoint influxdb.service.asapo;
+ set $elasticsearch_endpoint elasticsearch.service.asapo;
+
+ location /mongo/ {
+ rewrite ^/mongo(/.*) $1 break;
+ proxy_pass http://$mongo_endpoint:27017$uri$is_args$args;
+ }
+
+ location /influxdb/ {
+ rewrite ^/influxdb(/.*) $1 break;
+ proxy_pass http://$influxdb_endpoint:8086$uri$is_args$args;
+ }
+
+ location /elasticsearch/ {
+ rewrite ^/elasticsearch(/.*) $1 break;
+ proxy_pass http://$elasticsearch_endpoint:9200$uri$is_args$args;
+ }
+
location /discovery/ {
rewrite ^/discovery(/.*) $1 break;
- proxy_pass http://$discovery_endpoint:5006$uri;
+ proxy_pass http://$discovery_endpoint:5006$uri$is_args$args;
}
+
location /logs/ {
- # rewrite ^/logs(/.*) $1 break;
- proxy_pass http://localhost:9880/asapo;
+ rewrite ^/logs(/.*) $1 break;
+ proxy_pass http://$fluentd_endpoint:9880$uri$is_args$args;
+ }
+
+ location /logsview/ {
+ proxy_pass http://$kibana_endpoint:5601$uri$is_args$args;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $http_host;
+ }
+
+ location /performance/ {
+ rewrite ^/performance(/.*) $1 break;
+ proxy_pass http://$grafana_endpoint:3000$uri$is_args$args;
}
- location /authorizer/ {
- rewrite ^/authorizer(/.*) $1 break;
- proxy_pass http://$authorizer_endpoint:5007$uri;
- }
+ location /authorizer/ {
+ rewrite ^/authorizer(/.*) $1 break;
+ proxy_pass http://$authorizer_endpoint:5007$uri$is_args$args;
+ }
+
+ location /nginx-health {
+ return 200 "healthy\n";
+ }
+ }
+}
+
+stream {
+ resolver 127.0.0.1:8600 valid=1s;
- location /nginx-health {
- return 200 "healthy\n";
- }
+ map $remote_addr $upstream {
+ default fluentd.service.asapo;
+ }
+
+
+ server {
+ listen 9881;
+ proxy_pass $upstream:24224;
}
}
+
+
--
GitLab