From a8267e8b973bb2d5bb50b49c63bc950e6eec39b4 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon, 15 Nov 2021 13:30:27 +0100
Subject: [PATCH] update errors and deployment
---
common/cpp/include/asapo/common/error.tpp | 7 +--
common/cpp/include/asapo/common/utils.h | 49 +++++++++++++++++++
.../include/asapo/json_parser/json_parser.h | 1 -
common/cpp/src/database/mongodb_client.cpp | 4 +-
common/cpp/src/logger/logger.cpp | 6 +--
common/cpp/src/logger/spd_logger.cpp | 41 +---------------
common/cpp/src/logger/spd_logger.h | 2 +-
deploy/asapo_services/run_maxwell.sh | 2 +-
.../scripts/asapo-monitoring.nmd.tpl | 4 ++
docs/site/examples/start_asapo_socket.sh | 2 +-
docs/site/examples/start_asapo_tcp.sh | 2 +-
.../request_handler/request_handler_db.cpp | 4 +-
12 files changed, 70 insertions(+), 54 deletions(-)
create mode 100644 common/cpp/include/asapo/common/utils.h
diff --git a/common/cpp/include/asapo/common/error.tpp b/common/cpp/include/asapo/common/error.tpp
index aa14d4015..931abd205 100644
--- a/common/cpp/include/asapo/common/error.tpp
+++ b/common/cpp/include/asapo/common/error.tpp
@@ -1,6 +1,7 @@
-
#include "error.h"
+#include "asapo/common/utils.h"
+
namespace asapo {
template<typename ServiceErrorType>
@@ -91,13 +92,13 @@ template<typename ServiceErrorType>
std::string ServiceError<ServiceErrorType>::ExplainInJSON() const noexcept {
std::string err = WrapInQuotes("error") + ":" + WrapInQuotes(error_name_);
if (!error_message_.empty()) {
- err += "," + WrapInQuotes("message") + ":" + WrapInQuotes(error_message_);
+ err += "," + WrapInQuotes("message") + ":" + WrapInQuotes(EscapeJson(error_message_));
}
if (!details_.empty()) {
err += "," + WrapInQuotes("details") + ":{";
auto i = 0;
for (const auto &kv : details_) {
- err += (i > 0 ? ", " : "") + WrapInQuotes(kv.first) + ":" + WrapInQuotes(kv.second);
+ err += (i > 0 ? ", " : "") + WrapInQuotes(kv.first) + ":" + WrapInQuotes(EscapeJson(kv.second));
i++;
}
err += "}";
diff --git a/common/cpp/include/asapo/common/utils.h b/common/cpp/include/asapo/common/utils.h
new file mode 100644
index 000000000..103fb8f0a
--- /dev/null
+++ b/common/cpp/include/asapo/common/utils.h
@@ -0,0 +1,49 @@
+#ifndef ASAPO_COMMON_CPP_INCLUDE_ASAPO_COMMON_UTILS_H_
+#define ASAPO_COMMON_CPP_INCLUDE_ASAPO_COMMON_UTILS_H_
+
+#include <iomanip>
+#include <sstream>
+
+
+namespace asapo {
+
+inline std::string EscapeJson(const std::string& s) {
+ std::ostringstream o;
+ for (auto c = s.cbegin(); c != s.cend(); c++) {
+ switch (*c) {
+ case '"':
+ o << "\\\"";
+ break;
+ case '\\':
+ o << "\\\\";
+ break;
+ case '\b':
+ o << "\\b";
+ break;
+ case '\f':
+ o << "\\f";
+ break;
+ case '\n':
+ o << "\\n";
+ break;
+ case '\r':
+ o << "\\r";
+ break;
+ case '\t':
+ o << "\\t";
+ break;
+ default:
+ if ('\x00' <= *c && *c <= '\x1f') {
+ o << "\\u"
+ << std::hex << std::setw(4) << std::setfill('0') << (int)*c;
+ } else {
+ o << *c;
+ }
+ }
+ }
+ return o.str();
+}
+
+}
+
+#endif //ASAPO_COMMON_CPP_INCLUDE_ASAPO_COMMON_UTILS_H_
diff --git a/common/cpp/include/asapo/json_parser/json_parser.h b/common/cpp/include/asapo/json_parser/json_parser.h
index b8bd14f69..a2baa383d 100644
--- a/common/cpp/include/asapo/json_parser/json_parser.h
+++ b/common/cpp/include/asapo/json_parser/json_parser.h
@@ -49,7 +49,6 @@ class JsonFileParser : public JsonParser {
JsonFileParser(const std::string& json, const std::unique_ptr<IO>* io = nullptr): JsonParser(json, io) {};
};
-
}
diff --git a/common/cpp/src/database/mongodb_client.cpp b/common/cpp/src/database/mongodb_client.cpp
index eff22561e..bfb0d8ffb 100644
--- a/common/cpp/src/database/mongodb_client.cpp
+++ b/common/cpp/src/database/mongodb_client.cpp
@@ -46,7 +46,7 @@ Error MongoDBClient::Ping() {
bson_destroy(&reply);
bson_destroy(command);
- return !retval ? DBErrorTemplates::kConnectionError.Generate() : nullptr;
+ return !retval ? DBErrorTemplates::kConnectionError.Generate("cannot ping database") : nullptr;
}
MongoDBClient::MongoDBClient() {
@@ -58,7 +58,7 @@ Error MongoDBClient::InitializeClient(const std::string& address) {
client_ = mongoc_client_new(uri_str.c_str());
if (client_ == nullptr) {
- return DBErrorTemplates::kBadAddress.Generate();
+ return DBErrorTemplates::kBadAddress.Generate("cannot initialize database");
}
write_concern_ = mongoc_write_concern_new();
diff --git a/common/cpp/src/logger/logger.cpp b/common/cpp/src/logger/logger.cpp
index e9613d768..e277c2da5 100644
--- a/common/cpp/src/logger/logger.cpp
+++ b/common/cpp/src/logger/logger.cpp
@@ -61,12 +61,12 @@ LogMessageWithFields::LogMessageWithFields(std::string key, double val, int prec
LogMessageWithFields::LogMessageWithFields(std::string val) {
if (!val.empty()) {
- log_string_ = EncloseQuotes("message") + ":" + EncloseQuotes(escape_json(val));
+ log_string_ = EncloseQuotes("message") + ":" + EncloseQuotes(EscapeJson(val));
}
}
LogMessageWithFields::LogMessageWithFields(std::string key, std::string val) {
- log_string_ = EncloseQuotes(std::move(key)) + ":" + EncloseQuotes(escape_json(val));
+ log_string_ = EncloseQuotes(std::move(key)) + ":" + EncloseQuotes(EscapeJson(val));
}
inline std::string LogMessageWithFields::CommaIfNeeded() {
@@ -85,7 +85,7 @@ LogMessageWithFields &LogMessageWithFields::Append(std::string key, double val,
}
LogMessageWithFields &LogMessageWithFields::Append(std::string key, std::string val) {
- log_string_ += CommaIfNeeded() + EncloseQuotes(std::move(key)) + ":" + EncloseQuotes(escape_json(val));
+ log_string_ += CommaIfNeeded() + EncloseQuotes(std::move(key)) + ":" + EncloseQuotes(EscapeJson(val));
return *this;
}
diff --git a/common/cpp/src/logger/spd_logger.cpp b/common/cpp/src/logger/spd_logger.cpp
index 21f9f6ec3..cf66d44ff 100644
--- a/common/cpp/src/logger/spd_logger.cpp
+++ b/common/cpp/src/logger/spd_logger.cpp
@@ -3,8 +3,8 @@
#include "fluentd_sink.h"
#include <sstream>
-#include <iomanip>
+#include "asapo/common/utils.h"
namespace asapo {
@@ -30,46 +30,9 @@ void SpdLogger::SetLogLevel(LogLevel level) {
}
}
-std::string escape_json(const std::string& s) {
- std::ostringstream o;
- for (auto c = s.cbegin(); c != s.cend(); c++) {
- switch (*c) {
- case '"':
- o << "\\\"";
- break;
- case '\\':
- o << "\\\\";
- break;
- case '\b':
- o << "\\b";
- break;
- case '\f':
- o << "\\f";
- break;
- case '\n':
- o << "\\n";
- break;
- case '\r':
- o << "\\r";
- break;
- case '\t':
- o << "\\t";
- break;
- default:
- if ('\x00' <= *c && *c <= '\x1f') {
- o << "\\u"
- << std::hex << std::setw(4) << std::setfill('0') << (int)*c;
- } else {
- o << *c;
- }
- }
- }
- return o.str();
-}
-
std::string EncloseMsg(std::string msg) {
if (msg.find("\"") != 0) {
- return std::string(R"("message":")") + escape_json(msg) + "\"";
+ return std::string(R"("message":")") + EscapeJson(msg) + "\"";
} else {
return msg;
}
diff --git a/common/cpp/src/logger/spd_logger.h b/common/cpp/src/logger/spd_logger.h
index 1c2485f95..4b4ea94a2 100644
--- a/common/cpp/src/logger/spd_logger.h
+++ b/common/cpp/src/logger/spd_logger.h
@@ -38,7 +38,7 @@ class SpdLogger : public AbstractLogger {
};
std::string EncloseMsg(std::string msg);
-std::string escape_json(const std::string& s);
+std::string EscapeJson(const std::string& s);
}
diff --git a/deploy/asapo_services/run_maxwell.sh b/deploy/asapo_services/run_maxwell.sh
index d2a74ed1f..13bd97f0a 100755
--- a/deploy/asapo_services/run_maxwell.sh
+++ b/deploy/asapo_services/run_maxwell.sh
@@ -45,7 +45,7 @@ ASAPO_LIGHTWEIGHT_SERVICE_NODES=`scontrol show hostnames $SLURM_JOB_NODELIST | h
mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED $MONGO_DIR
chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED $MONGO_DIR
cd $SERVICE_DATA_CLUSTER_SHARED
-mkdir esdatadir fluentd grafana influxdb mongodb
+mkdir esdatadir fluentd grafana influxdb mongodb prometheus alertmanager
chmod 777 *
#todo: elastic search check
diff --git a/deploy/asapo_services/scripts/asapo-monitoring.nmd.tpl b/deploy/asapo_services/scripts/asapo-monitoring.nmd.tpl
index 677e5413f..0f1a15493 100644
--- a/deploy/asapo_services/scripts/asapo-monitoring.nmd.tpl
+++ b/deploy/asapo_services/scripts/asapo-monitoring.nmd.tpl
@@ -27,6 +27,8 @@ job "asapo-monitoring" {
driver = "docker"
user = "${asapo_user}"
config {
+ security_opt = ["no-new-privileges"]
+ userns_mode = "host"
image = "prom/alertmanager:${alertmanager_version}"
args = [
"--web.route-prefix=/alertmanager/",
@@ -87,6 +89,8 @@ job "asapo-monitoring" {
driver = "docker"
user = "${asapo_user}"
config {
+ security_opt = ["no-new-privileges"]
+ userns_mode = "host"
image = "prom/prometheus:${prometheus_version}"
args = [
"--web.external-url=/prometheus/",
diff --git a/docs/site/examples/start_asapo_socket.sh b/docs/site/examples/start_asapo_socket.sh
index a50a87ff9..0150a44a8 100644
--- a/docs/site/examples/start_asapo_socket.sh
+++ b/docs/site/examples/start_asapo_socket.sh
@@ -16,7 +16,7 @@ mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHAR
chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED $DATA_GLOBAL_SHARED_ONLINE
cd $SERVICE_DATA_CLUSTER_SHARED
-mkdir -p fluentd grafana influxdb influxdb2 mongodb
+mkdir -p fluentd grafana influxdb influxdb2 mongodb prometheus alertmanager
chmod 777 *
docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \
diff --git a/docs/site/examples/start_asapo_tcp.sh b/docs/site/examples/start_asapo_tcp.sh
index 007595fe5..4debb8e82 100644
--- a/docs/site/examples/start_asapo_tcp.sh
+++ b/docs/site/examples/start_asapo_tcp.sh
@@ -22,7 +22,7 @@ mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHAR
chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED $DATA_GLOBAL_SHARED_ONLINE
cd $SERVICE_DATA_CLUSTER_SHAREDdetector
-mkdir -p fluentd grafana influxdb2 mongodb
+mkdir -p fluentd grafana influxdb influxdb2 mongodb prometheus alertmanager
chmod 777 *
docker run --privileged --userns=host --security-opt no-new-privileges --rm \
diff --git a/receiver/src/request_handler/request_handler_db.cpp b/receiver/src/request_handler/request_handler_db.cpp
index ed201b179..54b9fadc0 100644
--- a/receiver/src/request_handler/request_handler_db.cpp
+++ b/receiver/src/request_handler/request_handler_db.cpp
@@ -81,9 +81,9 @@ Error RequestHandlerDb::DBErrorToReceiverError(Error err) const {
Error return_err;
if (err == DBErrorTemplates::kWrongInput || err == DBErrorTemplates::kNoRecord
|| err == DBErrorTemplates::kJsonParseError) {
- return_err = ReceiverErrorTemplates::kBadRequest.Generate();
+ return_err = ReceiverErrorTemplates::kBadRequest.Generate("error from database");
} else {
- return_err = ReceiverErrorTemplates::kInternalServerError.Generate();
+ return_err = ReceiverErrorTemplates::kInternalServerError.Generate("error from database");
}
return_err->SetCause(std::move(err));
return return_err;
--
GitLab