From f9a1201dd352823a43659939cf94c7bcd95b3df9 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon, 11 Oct 2021 19:00:15 +0200
Subject: [PATCH] metrics for receiver
---
config/nomad/discovery.nmd.in | 11 +++---
config/nomad/nginx.nmd.in | 17 +++++-----
config/nomad/receiver_fabric.nmd.in | 1 +
config/nomad/receiver_tcp.nmd.in | 15 +++++++-
.../asapo/configs/asapo-receiver.json | 4 +++
.../scripts/asapo-receivers.nmd.tpl | 20 +++++++++--
.../scripts/asapo.auto.tfvars.in | 1 +
.../asapo_services/scripts/prometheus.yml.tpl | 17 ++++++----
.../asapo_services/scripts/receiver.json.tpl | 4 +++
deploy/asapo_services/scripts/templates.tf | 1 +
deploy/asapo_services/scripts/vars.tf | 2 ++
receiver/src/main.cpp | 34 ++++++++++++-------
.../src/metrics/receiver_metrics_config.h | 15 ++++++++
.../src/metrics/receiver_metrics_server.h | 6 +++-
.../src/metrics/receiver_mongoose_server.cpp | 29 ++++++++++------
.../src/metrics/receiver_mongoose_server.h | 8 +++--
.../metrics/receiver_prometheus_metrics.cpp | 2 +-
receiver/src/receiver_config.cpp | 2 ++
receiver/src/receiver_config.h | 3 ++
receiver/unittests/mock_receiver_config.cpp | 5 +++
receiver/unittests/test_config.cpp | 8 ++++-
.../common_scripts/start_services.sh | 12 +++----
.../common_scripts/stop_services.bat | 12 +++----
.../automatic/common_scripts/stop_services.sh | 12 +++----
.../settings/receiver_fabric.json.tpl.lin.in | 4 +++
.../settings/receiver_tcp.json.tpl.lin.in | 4 +++
.../settings/receiver_tcp.json.tpl.win.in | 4 +++
27 files changed, 184 insertions(+), 69 deletions(-)
create mode 100644 receiver/src/metrics/receiver_metrics_config.h
diff --git a/config/nomad/discovery.nmd.in b/config/nomad/discovery.nmd.in
index 621403fc2..c12361721 100644
--- a/config/nomad/discovery.nmd.in
+++ b/config/nomad/discovery.nmd.in
@@ -6,6 +6,12 @@ job "discovery" {
group "group" {
count = 1
+ network {
+ port "discovery" {
+ static = "5006"
+ }
+ }
+
task "discovery" {
driver = "raw_exec"
@@ -17,11 +23,6 @@ job "discovery" {
resources {
cpu = 500 # 500 MHz
memory = 256 # 256MB
- network {
- port "discovery" {
- static = "5006"
- }
- }
}
service {
diff --git a/config/nomad/nginx.nmd.in b/config/nomad/nginx.nmd.in
index e908b7fee..97d35ab68 100644
--- a/config/nomad/nginx.nmd.in
+++ b/config/nomad/nginx.nmd.in
@@ -20,6 +20,13 @@ job "nginx" {
mode = "fail"
}
+ network {
+ port "nginx" {
+ static = 8400
+ }
+ }
+
+
task "nginx" {
driver = "raw_exec"
@@ -31,12 +38,6 @@ job "nginx" {
resources {
cpu = 500 # 500 MHz
memory = 256 # 256MB
- network {
- mbits = 10
- port "nginx" {
- static = 8400
- }
- }
}
service {
@@ -45,9 +46,9 @@ job "nginx" {
check {
name = "alive"
type = "http"
- path = "/nginx-health"
+ path = "/nginx-health"
timeout = "2s"
- interval = "10s"
+ interval = "10s"
}
}
diff --git a/config/nomad/receiver_fabric.nmd.in b/config/nomad/receiver_fabric.nmd.in
index 3be448ba6..938f4cd60 100644
--- a/config/nomad/receiver_fabric.nmd.in
+++ b/config/nomad/receiver_fabric.nmd.in
@@ -20,6 +20,7 @@ job "receiver" {
network {
port "recv" {}
port "recv_ds" {}
+ port "recv_metrics" {}
}
}
diff --git a/config/nomad/receiver_tcp.nmd.in b/config/nomad/receiver_tcp.nmd.in
index fe74ca93f..2d7b7c843 100644
--- a/config/nomad/receiver_tcp.nmd.in
+++ b/config/nomad/receiver_tcp.nmd.in
@@ -20,19 +20,32 @@ job "receiver" {
network {
port "recv" {}
port "recv_ds" {}
+ port "recv_metrics" {}
}
}
service {
name = "asapo-receiver"
- port = "recv"
check {
name = "alive"
type = "tcp"
+ port = "recv"
interval = "10s"
timeout = "2s"
initial_status = "passing"
}
+ check {
+ name = "metrics"
+ type = "http"
+ port = "recv_metrics"
+ path = "/metrics"
+ interval = "10s"
+ timeout = "2s"
+ initial_status = "passing"
+ }
+ meta {
+ metrics-port = "${NOMAD_PORT_recv_metrics}"
+ }
}
template {
diff --git a/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json b/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json
index 6dde5c8ad..c32c6703a 100644
--- a/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json
+++ b/deploy/asapo_helm_chart/asapo/configs/asapo-receiver.json
@@ -13,6 +13,10 @@
"ListenPort": {{ .Values.ownServices.receiver.dataServer.port }},
"NetworkMode": ["tcp"]
},
+ "Metrics": {
+ "Expose": {{ .Values.ownServices.receiver.metrics.expose },
+ "ListenPort": {{ .Values.ownServices.receiver.metrics.port }}
+ },
"DataCache": {
"Use": {{ .Values.ownServices.receiver.dataCache.enable }},
"SizeGB": {{ .Values.ownServices.receiver.dataCache.sizeGb }},
diff --git a/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl b/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl
index 2096021f5..a50a5e664 100644
--- a/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl
+++ b/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl
@@ -55,16 +55,28 @@ job "asapo-receivers" {
network {
port "recv" {}
port "recv_ds" {}
+ port "recv_metrics" {}
}
memory = "${receiver_total_memory_size}"
}
-
-
service {
name = "asapo-receiver"
- port = "recv"
+ %{ if receiver_expose_metrics }
check {
+ name = "metrics"
+ type = "http"
+ port = "recv_metrics"
+ path = "/metrics"
+ interval = "10s"
+ timeout = "2s"
+ initial_status = "passing"
+ }
+ meta {
+ metrics-port = "${NOMAD_PORT_recv_metrics}"
+ }
+ %{ else }
+ check {
name = "asapo-receiver-alive"
type = "script"
command = "/bin/ps"
@@ -72,6 +84,7 @@ job "asapo-receivers" {
interval = "10s"
timeout = "2s"
}
+ %{ endif }
check_restart {
limit = 2
grace = "15s"
@@ -85,6 +98,7 @@ job "asapo-receivers" {
receiver_receive_to_disk_threshold = "${receiver_receive_to_disk_threshold}"
receiver_network_modes = "${receiver_network_modes}"
perf_monitor = "${perf_monitor}"
+ receiver_expose_metrics = "${receiver_expose_metrics}"
}
template {
diff --git a/deploy/asapo_services/scripts/asapo.auto.tfvars.in b/deploy/asapo_services/scripts/asapo.auto.tfvars.in
index 39fc875d1..716e24900 100644
--- a/deploy/asapo_services/scripts/asapo.auto.tfvars.in
+++ b/deploy/asapo_services/scripts/asapo.auto.tfvars.in
@@ -24,6 +24,7 @@ receiver_dataserver_cache_size = 1 #gb
receiver_dataserver_nthreads = 4
receiver_receive_to_disk_threshold = 50 #mb
receiver_network_modes = "tcp"
+receiver_expose_metrics = true
grafana_total_memory_size = "256"
diff --git a/deploy/asapo_services/scripts/prometheus.yml.tpl b/deploy/asapo_services/scripts/prometheus.yml.tpl
index 89ee5478b..812d6158d 100644
--- a/deploy/asapo_services/scripts/prometheus.yml.tpl
+++ b/deploy/asapo_services/scripts/prometheus.yml.tpl
@@ -42,14 +42,19 @@ scrape_configs:
relabel_configs:
- source_labels: [__meta_consul_service]
target_label: job
-
- # metrics_path: /health
- - job_name: dummy
+ - job_name: receiver
consul_sd_configs:
- server: '{{ env "NOMAD_IP_prometheus_ui" }}:8500'
+ services:
+ - 'asapo-receiver'
relabel_configs:
- - source_labels: [__meta_consul_tags]
- regex: .*,asapo,.*
- action: keep
- source_labels: [__meta_consul_service]
target_label: job
+ - source_labels: [__meta_consul_service_address,__meta_consul_service_metadata_metrics_port]
+ separator: ';'
+ regex: (.*);(\d{4,5})
+ target_label: '__address__'
+ replacement: '$1:$2'
+ action: 'replace'
+
+
diff --git a/deploy/asapo_services/scripts/receiver.json.tpl b/deploy/asapo_services/scripts/receiver.json.tpl
index 2a69b27e9..da0b30cd3 100644
--- a/deploy/asapo_services/scripts/receiver.json.tpl
+++ b/deploy/asapo_services/scripts/receiver.json.tpl
@@ -13,6 +13,10 @@
"ListenPort": {{ env "NOMAD_PORT_recv_ds" }},
"NetworkMode": ["{{ if or (env "meta.ib_address") "none" | regexMatch "none" }}{{ printf "%s" "tcp" }}{{ else }}{{ env "NOMAD_META_receiver_network_modes" | split "," | join "\",\"" }}{{ end }}"]
},
+ "Metrics": {
+ "Expose": {{ env "NOMAD_META_receiver_expose_metrics" }},
+ "ListenPort": {{ env "NOMAD_PORT_recv_metrics" }}
+ },
"DataCache": {
"Use": true,
"SizeGB": {{ env "NOMAD_META_receiver_dataserver_cache_size" }},
diff --git a/deploy/asapo_services/scripts/templates.tf b/deploy/asapo_services/scripts/templates.tf
index 2c7fa07d0..8914c75e2 100644
--- a/deploy/asapo_services/scripts/templates.tf
+++ b/deploy/asapo_services/scripts/templates.tf
@@ -55,6 +55,7 @@ data "template_file" "asapo_receivers" {
n_receivers = "${var.n_receivers}"
force_pull_images = "${var.force_pull_images}"
perf_monitor = "${var.perf_monitor}"
+ receiver_expose_metrics = "${var.receiver_expose_metrics}"
}
}
diff --git a/deploy/asapo_services/scripts/vars.tf b/deploy/asapo_services/scripts/vars.tf
index fab7c9a18..16d7da4a6 100644
--- a/deploy/asapo_services/scripts/vars.tf
+++ b/deploy/asapo_services/scripts/vars.tf
@@ -52,6 +52,8 @@ variable "receiver_receive_to_disk_threshold" {}
variable "receiver_network_modes" {}
+variable "receiver_expose_metrics" {}
+
variable "grafana_total_memory_size" {}
variable "influxdb_total_memory_size" {}
diff --git a/receiver/src/main.cpp b/receiver/src/main.cpp
index f88186cbe..4ac76015f 100644
--- a/receiver/src/main.cpp
+++ b/receiver/src/main.cpp
@@ -77,32 +77,40 @@ int StartReceiver(const asapo::ReceiverConfig* config, asapo::SharedCache cache,
asapo::AbstractLogger* logger) {
static const std::string address = "0.0.0.0:" + std::to_string(config->listen_port);
-
logger->Info(std::string("starting receiver, version ") + asapo::kVersion);
auto* receiver = new asapo::Receiver(cache);
logger->Info("listening on " + address);
asapo::Error err;
receiver->Listen(address, &err);
- if(err) {
+ if (err) {
logger->Error("failed to start receiver: " + err->Explain());
return 1;
}
return 0;
}
-asapo::Error StartMetricsServer() {
- auto srv = std::unique_ptr<asapo::ReceiverMetricsServer>(new asapo::ReceiverMongooseServer());
- auto* provider = new asapo::ReceiverPrometheusMetrics();
- srv->ListenAndServe("7009", std::unique_ptr<asapo::ReceiverMetricsProvider>(provider));
- return nullptr;
+std::unique_ptr<std::thread> StartMetricsServer(const asapo::ReceiverMetricsConfig& config,
+ const asapo::AbstractLogger* logger) {
+ if (!config.expose) {
+ return nullptr;
+ }
+ return std::unique_ptr<std::thread> {
+ new std::thread{
+ [config, logger] {
+ auto srv = std::unique_ptr<asapo::ReceiverMetricsServer>(new asapo::ReceiverMongooseServer());
+ auto* provider = new asapo::ReceiverPrometheusMetrics();
+ logger->Debug("metrics server listening on " + std::to_string(config.listen_port));
+ srv->ListenAndServe(std::to_string(config.listen_port),
+ std::unique_ptr<asapo::ReceiverMetricsProvider>(provider));
+ }
+ }
+ };
}
-int main (int argc, char* argv[]) {
+int main(int argc, char* argv[]) {
asapo::ExitAfterPrintVersionIfNeeded("ASAPO Receiver", argc, argv);
-// StartMetricsServer();
-
auto err = ReadConfigFile(argc, argv);
const auto& logger = asapo::GetDefaultReceiverLogger();
if (err) {
@@ -116,15 +124,17 @@ int main (int argc, char* argv[]) {
asapo::SharedCache cache = nullptr;
if (config->use_datacache) {
- cache.reset(new asapo::DataCache{config->datacache_size_gb * 1024 * 1024 * 1024, (float)config->datacache_reserved_share / 100});
+ cache.reset(new asapo::DataCache{config->datacache_size_gb * 1024 * 1024 * 1024,
+ (float) config->datacache_reserved_share / 100});
}
auto dataServerThreads = StartDataServers(config, cache, &err);
if (err) {
- logger->Error("Cannot start data server: " + err->Explain());
+ logger->Error("cannot start data server: " + err->Explain());
return 1;
}
+ auto t = StartMetricsServer(config->metrics, logger);
auto exit_code = StartReceiver(config, cache, logger);
return exit_code;
}
diff --git a/receiver/src/metrics/receiver_metrics_config.h b/receiver/src/metrics/receiver_metrics_config.h
new file mode 100644
index 000000000..d3c17ebc9
--- /dev/null
+++ b/receiver/src/metrics/receiver_metrics_config.h
@@ -0,0 +1,15 @@
+#ifndef ASAPO_RECEIVER_SRC_METRICS_RECEIVER_METRICS_CONFIG_H_
+#define ASAPO_RECEIVER_SRC_METRICS_RECEIVER_METRICS_CONFIG_H_
+
+#include <string>
+
+namespace asapo {
+
+struct ReceiverMetricsConfig {
+ bool expose = false;
+ uint64_t listen_port = 0;
+};
+
+}
+
+#endif //ASAPO_RECEIVER_SRC_METRICS_RECEIVER_METRICS_CONFIG_H_
diff --git a/receiver/src/metrics/receiver_metrics_server.h b/receiver/src/metrics/receiver_metrics_server.h
index d2d668757..32e186c05 100644
--- a/receiver/src/metrics/receiver_metrics_server.h
+++ b/receiver/src/metrics/receiver_metrics_server.h
@@ -2,13 +2,17 @@
#define ASAPO_RECEIVER_SRC_METRICS_RECEIVER_METRICS_SERVER_H_
#include "asapo/common/error.h"
+
+#include <thread>
+#include <memory>
+
#include "receiver_metrics_provider.h"
namespace asapo {
class ReceiverMetricsServer {
public:
- virtual Error ListenAndServe(std::string port, std::unique_ptr<ReceiverMetricsProvider> provider) = 0;
+ virtual void ListenAndServe(std::string port, std::unique_ptr<ReceiverMetricsProvider> provider) = 0;
virtual ~ReceiverMetricsServer() = default;
};
diff --git a/receiver/src/metrics/receiver_mongoose_server.cpp b/receiver/src/metrics/receiver_mongoose_server.cpp
index 72cefbd8e..fa601ed71 100644
--- a/receiver/src/metrics/receiver_mongoose_server.cpp
+++ b/receiver/src/metrics/receiver_mongoose_server.cpp
@@ -12,33 +12,40 @@
#pragma GCC diagnostic pop
#endif
+#include "../receiver_logger.h"
+#include "asapo/common/io_error.h"
+
namespace asapo {
-static void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
- auto provider = static_cast<const ReceiverMetricsProvider *>(fn_data);
+static void fn(struct mg_connection* c, int ev, void* ev_data, void* fn_data) {
+ auto provider = static_cast<const ReceiverMetricsProvider*>(fn_data);
if (ev == MG_EV_HTTP_MSG) {
- struct mg_http_message *hm = (struct mg_http_message *) ev_data;
+ struct mg_http_message* hm = (struct mg_http_message*) ev_data;
if (mg_http_match_uri(hm, "/metrics")) {
- mg_http_reply(c, 200, "", "{\"result\": %s}\n", provider->Metrics().c_str());
+ mg_http_reply(c, 200, "", "%s", provider->Metrics().c_str());
} else {
mg_http_reply(c, 404, "", "%s", "Not found\n");
}
}
}
-asapo::Error asapo::ReceiverMongooseServer::ListenAndServe(std::string port,
- std::unique_ptr<ReceiverMetricsProvider> provider) {
+void asapo::ReceiverMongooseServer::ListenAndServe(std::string port,
+ std::unique_ptr<ReceiverMetricsProvider> provider) {
struct mg_mgr mgr; // Event manager
mg_mgr_init(&mgr); // Initialise event manager
auto uri = "0.0.0.0:" + port;
- if (mg_http_listen(&mgr, uri.c_str(), fn, (void *) provider.get()) == NULL) {
- LOG(LL_ERROR, ("Cannot listen on %s. Use http://ADDR:PORT or :PORT",
- port.c_str()));
- exit(EXIT_FAILURE);
+ if (mg_http_listen(&mgr, uri.c_str(), fn, (void*) provider.get()) == NULL) {
+ log__->Error("cannot listen on port " + port);
+ mg_mgr_free(&mgr);
+ return;
}
for (;;) mg_mgr_poll(&mgr, 1000); // Infinite event loop
mg_mgr_free(&mgr);
- return nullptr;
+ return;
+}
+
+ReceiverMongooseServer::ReceiverMongooseServer() : log__{GetDefaultReceiverLogger()} {
+
}
}
diff --git a/receiver/src/metrics/receiver_mongoose_server.h b/receiver/src/metrics/receiver_mongoose_server.h
index 88587a7f6..da7e48b20 100644
--- a/receiver/src/metrics/receiver_mongoose_server.h
+++ b/receiver/src/metrics/receiver_mongoose_server.h
@@ -4,10 +4,14 @@
#include "receiver_metrics_server.h"
namespace asapo {
+class AbstractLogger;
+
class ReceiverMongooseServer final : public ReceiverMetricsServer {
public:
- Error ListenAndServe(std::string port, std::unique_ptr<ReceiverMetricsProvider> provider) override;
-
+ ReceiverMongooseServer();
+ void ListenAndServe(std::string port, std::unique_ptr<ReceiverMetricsProvider> provider) override;
+ private:
+ const AbstractLogger* log__;
};
}
diff --git a/receiver/src/metrics/receiver_prometheus_metrics.cpp b/receiver/src/metrics/receiver_prometheus_metrics.cpp
index a0a671a76..7ebb2156b 100644
--- a/receiver/src/metrics/receiver_prometheus_metrics.cpp
+++ b/receiver/src/metrics/receiver_prometheus_metrics.cpp
@@ -3,7 +3,7 @@
namespace asapo {
std::string ReceiverPrometheusMetrics::Metrics() const {
- return "hello";
+ return "alive 1\n";
}
}
diff --git a/receiver/src/receiver_config.cpp b/receiver/src/receiver_config.cpp
index 78052168b..e78004925 100644
--- a/receiver/src/receiver_config.cpp
+++ b/receiver/src/receiver_config.cpp
@@ -35,6 +35,8 @@ Error ReceiverConfigFactory::SetConfig(std::string file_name) {
(err = parser.GetString("PerformanceDbName", &config.performance_db_name)) ||
(err = parser.Embedded("DataServer").GetString("AdvertiseURI", &config.dataserver.advertise_uri)) ||
(err = parser.Embedded("DataServer").GetArrayString("NetworkMode", &config.dataserver.network_mode)) ||
+ (err = parser.Embedded("Metrics").GetBool("Expose", &config.metrics.expose)) ||
+ (err = parser.Embedded("Metrics").GetUInt64("ListenPort", &config.metrics.listen_port)) ||
(err = parser.GetString("LogLevel", &log_level));
if (err) {
diff --git a/receiver/src/receiver_config.h b/receiver/src/receiver_config.h
index c8128362a..89c0d918f 100644
--- a/receiver/src/receiver_config.h
+++ b/receiver/src/receiver_config.h
@@ -6,6 +6,8 @@
#include "asapo/logger/logger.h"
#include "receiver_data_server/receiver_data_server_config.h"
+#include "metrics/receiver_metrics_config.h"
+
namespace asapo {
struct ReceiverConfig {
@@ -23,6 +25,7 @@ struct ReceiverConfig {
LogLevel log_level = LogLevel::Info;
std::string tag;
ReceiverDataServerConfig dataserver;
+ ReceiverMetricsConfig metrics;
std::string discovery_server;
};
diff --git a/receiver/unittests/mock_receiver_config.cpp b/receiver/unittests/mock_receiver_config.cpp
index 5577cbe9c..489fe8b96 100644
--- a/receiver/unittests/mock_receiver_config.cpp
+++ b/receiver/unittests/mock_receiver_config.cpp
@@ -64,6 +64,11 @@ Error SetReceiverConfig (const ReceiverConfig& config, std::string error_field)
config_string += "," + Key("NThreads", error_field) + std::to_string(config.dataserver.nthreads);
config_string += "}";
+ config_string += "," + Key("Metrics", error_field) + "{";
+ config_string += Key("ListenPort", error_field) + std::to_string(config.metrics.listen_port);
+ config_string += "," + Key("Expose", error_field) + (config.metrics.expose ? "true" : "false");
+ config_string += "}";
+
config_string += "," + Key("DataCache", error_field) + "{";
config_string += Key("Use", error_field) + (config.use_datacache ? "true" : "false") ;
config_string += "," + Key("SizeGB", error_field) + std::to_string(config.datacache_size_gb);
diff --git a/receiver/unittests/test_config.cpp b/receiver/unittests/test_config.cpp
index 40289b3d7..6e656529e 100644
--- a/receiver/unittests/test_config.cpp
+++ b/receiver/unittests/test_config.cpp
@@ -62,6 +62,9 @@ class ConfigTests : public Test {
test_config.dataserver.advertise_uri = "0.0.0.1:4201";
test_config.dataserver.network_mode = {"tcp", "fabric"};
test_config.receive_to_disk_threshold_mb = 50;
+ test_config.metrics.expose = true;
+ test_config.metrics.listen_port = 123;
+
}
@@ -97,9 +100,12 @@ TEST_F(ConfigTests, ReadSettings) {
ASSERT_THAT(config->dataserver.network_mode[0], Eq("tcp"));
ASSERT_THAT(config->dataserver.network_mode[1], Eq("fabric"));
ASSERT_THAT(config->receive_to_disk_threshold_mb, Eq(50));
+ ASSERT_THAT(config->metrics.expose, Eq(true));
+ ASSERT_THAT(config->metrics.listen_port, Eq(123));
}
+
TEST_F(ConfigTests, ErrorReadSettings) {
PrepareConfig();
@@ -107,7 +113,7 @@ TEST_F(ConfigTests, ErrorReadSettings) {
"DataCache", "Use", "SizeGB", "ReservedShare", "DatabaseServer", "Tag",
"AuthorizationServer", "AuthorizationInterval", "PerformanceDbName", "LogLevel",
"NThreads", "DiscoveryServer", "AdvertiseURI", "NetworkMode", "MonitorPerformance",
- "ReceiveToDiskThresholdMB"};
+ "ReceiveToDiskThresholdMB", "Metrics", "Expose"};
for (const auto& field : fields) {
auto err = asapo::SetReceiverConfig(test_config, field);
ASSERT_THAT(err, Ne(nullptr));
diff --git a/tests/automatic/common_scripts/start_services.sh b/tests/automatic/common_scripts/start_services.sh
index 944f3ed0b..e59a39cb8 100755
--- a/tests/automatic/common_scripts/start_services.sh
+++ b/tests/automatic/common_scripts/start_services.sh
@@ -1,5 +1,5 @@
-nomad run discovery.nmd
-nomad run nginx.nmd
+nomad run -detach discovery.nmd
+nomad run -detach nginx.nmd
while true
do
@@ -10,10 +10,10 @@ do
break
done
-nomad run authorizer.nmd
-nomad run file_transfer.nmd
-nomad run receiver_tcp.nmd
-nomad run broker.nmd
+nomad run -detach authorizer.nmd
+nomad run -detach file_transfer.nmd
+nomad run -detach receiver_tcp.nmd
+nomad run -detach broker.nmd
while true
do
diff --git a/tests/automatic/common_scripts/stop_services.bat b/tests/automatic/common_scripts/stop_services.bat
index 1f75f6970..8a5846133 100644
--- a/tests/automatic/common_scripts/stop_services.bat
+++ b/tests/automatic/common_scripts/stop_services.bat
@@ -1,7 +1,7 @@
-c:\opt\consul\nomad stop receiver
-c:\opt\consul\nomad stop discovery
-c:\opt\consul\nomad stop broker
-c:\opt\consul\nomad stop authorizer
-c:\opt\consul\nomad stop nginx
-c:\opt\consul\nomad stop file_transfer
+c:\opt\consul\nomad stop -purge receiver
+c:\opt\consul\nomad stop -purge discovery
+c:\opt\consul\nomad stop -purge broker
+c:\opt\consul\nomad stop -purge authorizer
+c:\opt\consul\nomad stop -purge nginx
+c:\opt\consul\nomad stop -purge file_transfer
c:\opt\consul\nomad run nginx_kill.nmd && c:\opt\consul\nomad stop -yes -purge nginx_kill
diff --git a/tests/automatic/common_scripts/stop_services.sh b/tests/automatic/common_scripts/stop_services.sh
index d10019f51..084630d7f 100755
--- a/tests/automatic/common_scripts/stop_services.sh
+++ b/tests/automatic/common_scripts/stop_services.sh
@@ -1,7 +1,7 @@
-nomad stop nginx
+nomad stop -purge nginx
nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill
-nomad stop discovery
-nomad stop broker
-nomad stop receiver
-nomad stop authorizer
-nomad stop file_transfer
+nomad stop -purge discovery
+nomad stop -purge broker
+nomad stop -purge receiver
+nomad stop -purge authorizer
+nomad stop -purge file_transfer
diff --git a/tests/automatic/settings/receiver_fabric.json.tpl.lin.in b/tests/automatic/settings/receiver_fabric.json.tpl.lin.in
index 6cf20b1cd..f7524f9e9 100644
--- a/tests/automatic/settings/receiver_fabric.json.tpl.lin.in
+++ b/tests/automatic/settings/receiver_fabric.json.tpl.lin.in
@@ -10,6 +10,10 @@
"ListenPort": {{ env "NOMAD_PORT_recv_ds" }},
"NetworkMode": ["fabric"]
},
+ "Metrics": {
+ "Expose": true,
+ "ListenPort": {{ env "NOMAD_PORT_recv_metrics" }}
+ },
"DataCache": {
"Use": @RECEIVER_USE_CACHE@,
"SizeGB": 1,
diff --git a/tests/automatic/settings/receiver_tcp.json.tpl.lin.in b/tests/automatic/settings/receiver_tcp.json.tpl.lin.in
index a6f98fe37..498d9fdcc 100644
--- a/tests/automatic/settings/receiver_tcp.json.tpl.lin.in
+++ b/tests/automatic/settings/receiver_tcp.json.tpl.lin.in
@@ -10,6 +10,10 @@
"ListenPort": {{ env "NOMAD_PORT_recv_ds" }},
"NetworkMode": ["tcp"]
},
+ "Metrics": {
+ "Expose": true,
+ "ListenPort": {{ env "NOMAD_PORT_recv_metrics" }}
+ },
"DataCache": {
"Use": @RECEIVER_USE_CACHE@,
"SizeGB": 1,
diff --git a/tests/automatic/settings/receiver_tcp.json.tpl.win.in b/tests/automatic/settings/receiver_tcp.json.tpl.win.in
index c0989d12e..446ebcbf2 100644
--- a/tests/automatic/settings/receiver_tcp.json.tpl.win.in
+++ b/tests/automatic/settings/receiver_tcp.json.tpl.win.in
@@ -13,6 +13,10 @@
"ListenPort": {{ env "NOMAD_PORT_recv_ds" }},
"NetworkMode": ["tcp"]
},
+ "Metrics": {
+ "Expose": true,
+ "ListenPort": {{ env "NOMAD_PORT_recv_metrics" }}
+ },
"DataCache": {
"Use": @RECEIVER_USE_CACHE@,
"SizeGB": 1,
--
GitLab