From 129d86ae16278bbcafd7003193a87556a3cd845d Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Fri, 4 May 2018 09:31:03 +0200
Subject: [PATCH] logging for receiver
---
broker/src/hidra2_broker/logger/logger.go | 26 ++++++-
.../src/hidra2_broker/logger/logrus_logger.go | 29 ++++---
.../src/hidra2_broker/logger/mock_logger.go | 2 +-
broker/src/hidra2_broker/main/broker.go | 8 +-
broker/src/hidra2_broker/server/server.go | 1 +
.../hidra2_broker/server/server_nottested.go | 16 ++--
common/cpp/include/logger/logger.h | 13 +++-
common/cpp/include/unittests/MockLogger.h | 9 +--
common/cpp/src/logger/logger_factory.cpp | 11 +++
common/cpp/src/logger/spd_logger.cpp | 8 +-
common/cpp/src/logger/spd_logger.h | 8 +-
common/cpp/unittests/logger/test_logger.cpp | 40 +++++++---
producer/api/src/producer_impl.cpp | 2 +-
receiver/CMakeLists.txt | 3 +-
receiver/src/connection.cpp | 20 +++--
receiver/src/connection.h | 4 +-
receiver/src/main.cpp | 11 ++-
receiver/src/receiver.cpp | 10 ++-
receiver/src/receiver.h | 6 +-
receiver/src/receiver_config.cpp | 9 ++-
receiver/src/receiver_config.h | 3 +-
receiver/src/receiver_logger.cpp | 11 +++
receiver/src/receiver_logger.h | 14 ++++
receiver/src/request_handler_db_write.cpp | 11 ++-
receiver/src/request_handler_db_write.h | 2 +
receiver/src/request_handler_file_write.cpp | 10 ++-
receiver/src/request_handler_file_write.h | 2 +
receiver/src/statistics_sender_influx_db.cpp | 12 ++-
receiver/src/statistics_sender_influx_db.h | 3 +-
receiver/unittests/mock_receiver_config.cpp | 20 +++++
receiver/unittests/test_config.cpp | 6 +-
receiver/unittests/test_connection.cpp | 75 +++++++++----------
receiver/unittests/test_receiver.cpp | 24 +++++-
.../test_request_handler_db_writer.cpp | 54 ++++++-------
.../test_request_handler_file_write.cpp | 51 +++++++++++--
.../test_statistics_sender_influx_db.cpp | 68 +++++++++++++----
.../full_chain/simple_chain/check_linux.sh | 2 +-
tests/automatic/settings/broker_settings.json | 3 +-
tests/automatic/settings/receiver.json | 3 +-
.../performance_full_chain_simple/broker.json | 3 +-
.../fluentd.conf | 49 ++++++++++++
.../receiver.json | 3 +-
.../performance_full_chain_simple/test.sh | 15 ++--
.../receiver.json | 3 +-
44 files changed, 486 insertions(+), 197 deletions(-)
create mode 100644 receiver/src/receiver_logger.cpp
create mode 100644 receiver/src/receiver_logger.h
create mode 100644 tests/manual/performance_full_chain_simple/fluentd.conf
diff --git a/broker/src/hidra2_broker/logger/logger.go b/broker/src/hidra2_broker/logger/logger.go
index 0fa02db8b..e198cd23d 100644
--- a/broker/src/hidra2_broker/logger/logger.go
+++ b/broker/src/hidra2_broker/logger/logger.go
@@ -1,5 +1,10 @@
package logger
+import (
+ "errors"
+ "strings"
+)
+
type Level uint32
//log levels
@@ -17,7 +22,7 @@ type Logger interface {
Fatal(args ...interface{})
Warning(args ...interface{})
Error(args ...interface{})
- SetLevel(level string)
+ SetLevel(level Level)
}
var my_logger Logger = &logRusLogger{}
@@ -42,6 +47,23 @@ func Fatal(args ...interface{}) {
my_logger.Fatal(args...)
}
-func SetLevel(level string) {
+func SetLevel(level Level) {
my_logger.SetLevel(level)
}
+
+func LevelFromString(str string) (Level, error) {
+ switch strings.ToLower(str) {
+ case "debug":
+ return DebugLevel, nil
+ case "info":
+ return InfoLevel, nil
+ case "warning":
+ return WarnLevel, nil
+ case "error":
+ return ErrorLevel, nil
+ case "fatal", "none":
+ return FatalLevel, nil
+ }
+ return FatalLevel, errors.New("wrong log level")
+
+}
diff --git a/broker/src/hidra2_broker/logger/logrus_logger.go b/broker/src/hidra2_broker/logger/logrus_logger.go
index 19e33eb50..875a05c04 100644
--- a/broker/src/hidra2_broker/logger/logrus_logger.go
+++ b/broker/src/hidra2_broker/logger/logrus_logger.go
@@ -2,7 +2,6 @@ package logger
import (
log "github.com/sirupsen/logrus"
- "strings"
)
type logRusLogger struct {
@@ -56,21 +55,21 @@ func (l *logRusLogger) Fatal(args ...interface{}) {
return
}
-func (l *logRusLogger) SetLevel(level string) {
- logrus_level := log.InfoLevel
- switch strings.ToLower(level) {
- case "debug":
- logrus_level = log.DebugLevel
- case "info":
- logrus_level = log.InfoLevel
- case "warning":
- logrus_level = log.WarnLevel
- case "error":
- logrus_level = log.ErrorLevel
- case "fatal":
- logrus_level = log.FatalLevel
+func (l *logRusLogger) SetLevel(level Level) {
+ logrusLevel := log.InfoLevel
+ switch level {
+ case DebugLevel:
+ logrusLevel = log.DebugLevel
+ case InfoLevel:
+ logrusLevel = log.InfoLevel
+ case WarnLevel:
+ logrusLevel = log.WarnLevel
+ case ErrorLevel:
+ logrusLevel = log.ErrorLevel
+ case FatalLevel:
+ logrusLevel = log.FatalLevel
}
- log.SetLevel(logrus_level)
+ log.SetLevel(logrusLevel)
return
}
diff --git a/broker/src/hidra2_broker/logger/mock_logger.go b/broker/src/hidra2_broker/logger/mock_logger.go
index ad21f5fa8..0e597978b 100644
--- a/broker/src/hidra2_broker/logger/mock_logger.go
+++ b/broker/src/hidra2_broker/logger/mock_logger.go
@@ -45,7 +45,7 @@ func (l *MockLogger) Fatal(args ...interface{}) {
return
}
-func (l *MockLogger) SetLevel(level string) {
+func (l *MockLogger) SetLevel(level Level) {
l.Called(level)
return
}
diff --git a/broker/src/hidra2_broker/main/broker.go b/broker/src/hidra2_broker/main/broker.go
index fc3122a14..8153c2c5c 100644
--- a/broker/src/hidra2_broker/main/broker.go
+++ b/broker/src/hidra2_broker/main/broker.go
@@ -15,24 +15,24 @@ func NewDefaultDatabase() database.Agent {
}
func PrintUsage() {
- log.Fatal("Usage: " + os.Args[0] + " -config <config file> [-logging debug|info|warning|error|fatal]")
+ log.Fatal("Usage: " + os.Args[0] + " -config <config file>")
}
func main() {
var fname = flag.String("config", "", "config file path")
- var log_level = flag.String("logging", "info", "logging level")
flag.Parse()
if *fname == "" {
PrintUsage()
}
- log.SetLevel(*log_level)
- err := server.ReadConfig(*fname)
+ logLevel, err := server.ReadConfig(*fname)
if err != nil {
log.Fatal(err.Error())
}
+ log.SetLevel(logLevel)
+
err = server.InitDB(NewDefaultDatabase())
if err != nil {
log.Fatal(err.Error())
diff --git a/broker/src/hidra2_broker/server/server.go b/broker/src/hidra2_broker/server/server.go
index df1fd327f..fe7dd2266 100644
--- a/broker/src/hidra2_broker/server/server.go
+++ b/broker/src/hidra2_broker/server/server.go
@@ -11,6 +11,7 @@ type serverSettings struct {
MonitorDbAddress string
MonitorDbName string
Port int
+ LogLevel string
}
var settings serverSettings
diff --git a/broker/src/hidra2_broker/server/server_nottested.go b/broker/src/hidra2_broker/server/server_nottested.go
index 9a90f3689..8444fbf28 100644
--- a/broker/src/hidra2_broker/server/server_nottested.go
+++ b/broker/src/hidra2_broker/server/server_nottested.go
@@ -23,26 +23,28 @@ func Start() {
log.Fatal(http.ListenAndServe(":"+strconv.Itoa(settings.Port), http.HandlerFunc(mux.ServeHTTP)))
}
-func ReadConfig(fname string) error {
+func ReadConfig(fname string) (log.Level, error) {
if err := utils.ReadJsonFromFile(fname, &settings); err != nil {
- return err
+ return log.FatalLevel, err
}
if settings.BrokerDbAddress == "" {
- return errors.New("BrokerDbAddress not set")
+ return log.FatalLevel, errors.New("BrokerDbAddress not set")
}
if settings.MonitorDbAddress == "" {
- return errors.New("MonitorDbAddress not set")
+ return log.FatalLevel, errors.New("MonitorDbAddress not set")
}
if settings.Port == 0 {
- return errors.New("Server port not set")
+ return log.FatalLevel, errors.New("Server port not set")
}
if settings.MonitorDbName == "" {
- return errors.New("MonitorDbName not set")
+ return log.FatalLevel, errors.New("MonitorDbName not set")
}
- return nil
+ level, err := log.LevelFromString(settings.LogLevel)
+
+ return level, err
}
diff --git a/common/cpp/include/logger/logger.h b/common/cpp/include/logger/logger.h
index fd146c95b..4cb60f02f 100644
--- a/common/cpp/include/logger/logger.h
+++ b/common/cpp/include/logger/logger.h
@@ -4,6 +4,8 @@
#include <memory>
#include <string>
+#include "common/error.h"
+
namespace hidra2 {
enum class LogLevel {
@@ -17,10 +19,10 @@ enum class LogLevel {
class AbstractLogger {
public:
virtual void SetLogLevel(LogLevel level) = 0;
- virtual void Info(const std::string& text) = 0;
- virtual void Error(const std::string& text) = 0;
- virtual void Debug(const std::string& text) = 0;
- virtual void Warning(const std::string& text) = 0;
+ virtual void Info(const std::string& text) const = 0;
+ virtual void Error(const std::string& text) const = 0;
+ virtual void Debug(const std::string& text) const = 0;
+ virtual void Warning(const std::string& text) const = 0;
virtual void EnableLocalLog(bool enable) = 0;
virtual void EnableRemoteLog(bool enable) = 0;
virtual ~AbstractLogger() = default;
@@ -31,6 +33,9 @@ using Logger = std::unique_ptr<AbstractLogger>;
Logger CreateDefaultLoggerBin(const std::string& name);
Logger CreateDefaultLoggerApi(const std::string& name, const std::string& endpoint_uri);
+LogLevel StringToLogLevel(const std::string& name, Error* err);
+
+
}
#endif //HIDRA2_LOGGER_H
diff --git a/common/cpp/include/unittests/MockLogger.h b/common/cpp/include/unittests/MockLogger.h
index 61aa52bfc..cc7ac8a19 100644
--- a/common/cpp/include/unittests/MockLogger.h
+++ b/common/cpp/include/unittests/MockLogger.h
@@ -10,14 +10,13 @@ namespace hidra2 {
class MockLogger : public AbstractLogger {
public:
- MOCK_METHOD1(Info, void(const std::string&));
- MOCK_METHOD1(Error, void(const std::string& ));
- MOCK_METHOD1(Debug, void(const std::string& ));
- MOCK_METHOD1(Warning, void(const std::string& ));
+ MOCK_CONST_METHOD1(Info, void(const std::string&));
+ MOCK_CONST_METHOD1(Error, void(const std::string& ));
+ MOCK_CONST_METHOD1(Debug, void(const std::string& ));
+ MOCK_CONST_METHOD1(Warning, void(const std::string& ));
MOCK_METHOD1(SetLogLevel, void(LogLevel));
MOCK_METHOD1(EnableLocalLog, void(bool));
MOCK_METHOD1(EnableRemoteLog, void(bool));
-
};
}
diff --git a/common/cpp/src/logger/logger_factory.cpp b/common/cpp/src/logger/logger_factory.cpp
index 9a57f02d5..b42271d3f 100644
--- a/common/cpp/src/logger/logger_factory.cpp
+++ b/common/cpp/src/logger/logger_factory.cpp
@@ -24,5 +24,16 @@ Logger CreateDefaultLoggerApi(const std::string& name, const std::string& endpoi
return CreateLogger(name, false, true, endpoint_uri);
}
+LogLevel StringToLogLevel(const std::string& name, Error* err) {
+ *err = nullptr;
+ if (name == "debug") return LogLevel::Debug;
+ if (name == "info") return LogLevel::Info;
+ if (name == "warning") return LogLevel::Warning;
+ if (name == "none") return LogLevel::None;
+ if (name == "error") return LogLevel::Error;
+
+ *err = TextError("wrong log level: " + name);
+ return LogLevel::None;
+}
};
\ No newline at end of file
diff --git a/common/cpp/src/logger/spd_logger.cpp b/common/cpp/src/logger/spd_logger.cpp
index 5db29c122..5846f5c98 100644
--- a/common/cpp/src/logger/spd_logger.cpp
+++ b/common/cpp/src/logger/spd_logger.cpp
@@ -26,7 +26,7 @@ void SpdLogger::SetLogLevel(LogLevel level) {
}
}
-void SpdLogger::Info(const std::string& text) {
+void SpdLogger::Info(const std::string& text)const {
if (log__) {
log__->info(text);
}
@@ -53,20 +53,20 @@ void SpdLogger::UpdateLoggerSinks() {
SpdLogger::SpdLogger(const std::string& name, const std::string& endpoint_uri): name_{name}, endpoint_uri_{endpoint_uri} {
}
-void SpdLogger::Error(const std::string& text) {
+void SpdLogger::Error(const std::string& text)const {
if (log__) {
log__->error(text);
}
}
-void SpdLogger::Debug(const std::string& text) {
+void SpdLogger::Debug(const std::string& text) const {
if (log__) {
log__->debug(text);
}
}
-void SpdLogger::Warning(const std::string& text) {
+void SpdLogger::Warning(const std::string& text)const {
if (log__) {
log__->warn(text);
}
diff --git a/common/cpp/src/logger/spd_logger.h b/common/cpp/src/logger/spd_logger.h
index 32097315a..ae79abf2a 100644
--- a/common/cpp/src/logger/spd_logger.h
+++ b/common/cpp/src/logger/spd_logger.h
@@ -10,10 +10,10 @@ class SpdLogger : public AbstractLogger {
public:
explicit SpdLogger(const std::string& name, const std::string& endpoint_uri);
void SetLogLevel(LogLevel level) override;
- void Info(const std::string& text) override;
- void Error(const std::string& text) override;
- void Debug(const std::string& text) override;
- void Warning(const std::string& text) override;
+ void Info(const std::string& text) const override;
+ void Error(const std::string& text) const override;
+ void Debug(const std::string& text) const override;
+ void Warning(const std::string& text) const override;
void EnableLocalLog(bool enable) override;
void EnableRemoteLog(bool enable) override;
~SpdLogger() = default;
diff --git a/common/cpp/unittests/logger/test_logger.cpp b/common/cpp/unittests/logger/test_logger.cpp
index 9fd923ce6..71253227b 100644
--- a/common/cpp/unittests/logger/test_logger.cpp
+++ b/common/cpp/unittests/logger/test_logger.cpp
@@ -19,6 +19,30 @@ using hidra2::LogLevel;
namespace {
+void CheckConvert(const std::string& str, LogLevel level) {
+ hidra2::Error err;
+ auto loglev = hidra2::StringToLogLevel(str, &err);
+ ASSERT_THAT(err, Eq(nullptr));
+ ASSERT_THAT(loglev, Eq(level));
+}
+
+TEST(StringToLogLevel, ConvertOK) {
+ CheckConvert("debug", LogLevel::Debug);
+ CheckConvert("info", LogLevel::Info);
+ CheckConvert("warning", LogLevel::Warning);
+ CheckConvert("error", LogLevel::Error);
+ CheckConvert("none", LogLevel::None);
+}
+
+
+TEST(StringToLogLevel, ConvertError) {
+ hidra2::Error err;
+ auto loglev = hidra2::StringToLogLevel("wrong", &err);
+ ASSERT_THAT(err, Ne(nullptr));
+ ASSERT_THAT(loglev, Eq(hidra2::LogLevel::None));
+}
+
+
TEST(DefaultLogger, BinLogger) {
auto logger = hidra2::CreateDefaultLoggerBin("test");
ASSERT_THAT(dynamic_cast<hidra2::SpdLogger*>(logger.get()), Ne(nullptr));
@@ -29,19 +53,18 @@ TEST(DefaultLogger, ApiLogger) {
ASSERT_THAT(dynamic_cast<hidra2::SpdLogger*>(logger.get()), Ne(nullptr));
}
-
class MockSink : public spdlog::sinks::base_sink<std::mutex> {
public:
MockSink(const std::string& endpoint_uri) {};
public:
- MOCK_METHOD1(_sink_it, void (const spdlog::details::log_msg& msg));
- MOCK_METHOD0(_flush, void ());
+ MOCK_METHOD1(_sink_it, void(
+ const spdlog::details::log_msg& msg));
+ MOCK_METHOD0(_flush, void());
};
-
class LoggerTests : public Test {
public:
- std::shared_ptr<MockSink>mock_sink{new MockSink{"test_url"}};
+ std::shared_ptr<MockSink> mock_sink{new MockSink{"test_url"}};
std::unique_ptr<spdlog::logger> log;
hidra2::SpdLogger logger{"test", "test_uri"};
spdlog::details::log_msg msg;
@@ -55,7 +78,6 @@ class LoggerTests : public Test {
}
};
-
MATCHER_P(CompareMsg, msg, "") {
if (arg.level != (*msg).level) return false;
if (arg.raw.str() != (*msg).raw.c_str()) return false;
@@ -63,7 +85,6 @@ MATCHER_P(CompareMsg, msg, "") {
return true;
}
-
TEST_F(LoggerTests, Info) {
msg.level = spdlog::level::info;
logger.SetLogLevel(LogLevel::Info);
@@ -90,7 +111,6 @@ TEST_F(LoggerTests, Error) {
logger.Error(test_string);
}
-
TEST_F(LoggerTests, Warning) {
msg.level = spdlog::level::warn;
@@ -108,7 +128,6 @@ TEST_F(LoggerTests, NoWarningOnErrorLevel) {
logger.Warning(test_string);
}
-
TEST_F(LoggerTests, NoInfoOnWarningLevel) {
msg.level = spdlog::level::info;
logger.SetLogLevel(LogLevel::Warning);
@@ -118,7 +137,6 @@ TEST_F(LoggerTests, NoInfoOnWarningLevel) {
logger.Info(test_string);
}
-
TEST_F(LoggerTests, NoDebugOnNoneLevel) {
msg.level = spdlog::level::debug;
logger.SetLogLevel(LogLevel::None);
@@ -128,5 +146,5 @@ TEST_F(LoggerTests, NoDebugOnNoneLevel) {
logger.Info(test_string);
}
-
}
+
diff --git a/producer/api/src/producer_impl.cpp b/producer/api/src/producer_impl.cpp
index 66f90ab08..b7bd9ecb0 100644
--- a/producer/api/src/producer_impl.cpp
+++ b/producer/api/src/producer_impl.cpp
@@ -11,7 +11,7 @@ const size_t ProducerImpl::kMaxChunkSize = size_t(1024) * size_t(1024) * size_t(
ProducerImpl::ProducerImpl(): io__{GenerateDefaultIO()} {
//todo get fluentd uri from service discovery
- log__ = CreateDefaultLoggerApi("producer_api", "http://localhost:9880/asapo");
+ log__ = CreateDefaultLoggerApi("producer_api", "http://max-wgs.desy.de:9880/asapo");
}
uint64_t ProducerImpl::GetVersion() const {
diff --git a/receiver/CMakeLists.txt b/receiver/CMakeLists.txt
index a5d795a7c..cb9c6267f 100644
--- a/receiver/CMakeLists.txt
+++ b/receiver/CMakeLists.txt
@@ -8,6 +8,7 @@ set(SOURCE_FILES
src/statistics.cpp
src/statistics_sender_influx_db.cpp
src/receiver_config.cpp
+ src/receiver_logger.cpp
src/request_handler_db_write.cpp)
@@ -17,7 +18,7 @@ set(SOURCE_FILES
add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io> $<TARGET_OBJECTS:curl_http_client>
- $<TARGET_OBJECTS:json_parser>)
+ $<TARGET_OBJECTS:json_parser> $<TARGET_OBJECTS:logger>)
set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR} ${CURL_INCLUDE_DIRS})
target_link_libraries(${TARGET_NAME} ${CURL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} database)
diff --git a/receiver/src/connection.cpp b/receiver/src/connection.cpp
index 1f3794491..d74affd2b 100644
--- a/receiver/src/connection.cpp
+++ b/receiver/src/connection.cpp
@@ -4,13 +4,15 @@
#include "receiver_error.h"
#include "io/io_factory.h"
+#include "receiver_logger.h"
+
namespace hidra2 {
size_t Connection::kRequestHandlerMaxBufferSize;
std::atomic<uint32_t> Connection::kNetworkProducerPeerImplGlobalCounter(0);
Connection::Connection(SocketDescriptor socket_fd, const std::string& address): request_factory__{new RequestFactory},
-io__{GenerateDefaultIO()}, statistics__{new Statistics} {
+io__{GenerateDefaultIO()}, statistics__{new Statistics}, log__{GetDefaultReceiverLogger()} {
socket_fd_ = socket_fd;
connection_id_ = kNetworkProducerPeerImplGlobalCounter++;
address_ = address;
@@ -29,7 +31,6 @@ NetworkErrorCode GetNetworkCodeFromError(const Error& err) {
}
}
return NetworkErrorCode::kNetErrorNoError;
-
}
Error Connection::ProcessRequest(const std::unique_ptr<Request>& request) const noexcept {
@@ -38,9 +39,12 @@ Error Connection::ProcessRequest(const std::unique_ptr<Request>& request) const
GenericNetworkResponse generic_response;
generic_response.error_code = GetNetworkCodeFromError(err);
if(err) {
- std::cerr << "[" << GetId() << "] Error while handling request: " << err << std::endl;
+ log__->Error("error while processing request from " + address_ + " - " + err->Explain());
}
io__->Send(socket_fd_, &generic_response, sizeof(GenericNetworkResponse), &err);
+ if(err) {
+ log__->Error("error sending response to " + address_ + " - " + err->Explain());
+ }
return err;
}
@@ -57,20 +61,22 @@ void Connection::Listen() const noexcept {
Error err;
auto request = WaitForNewRequest(&err);
if(err) {
- std::cerr << "[" << GetId() << "] Error while waiting for request: " << err << std::endl;
+ if (err != ErrorTemplates::kEndOfFile) {
+ log__->Error("error while waiting for request from " + address_ + " - " + err->Explain());
+ }
break;
}
if (!request) continue; //no error, but timeout
+ log__->Debug("processing request from " + address_);
err = ProcessRequest(request);
- if(err) {
- std::cerr << "[" << GetId() << "] Error sending response: " << err << std::endl;
+ if (err) {
break;
}
ProcessStatisticsAfterRequest(request);
}
io__->CloseSocket(socket_fd_, nullptr);
statistics__->Send();
- std::cerr << "[" << GetId() << "] Disconnected." << std::endl;
+ log__->Info("disconnected from " + address_);
}
diff --git a/receiver/src/connection.h b/receiver/src/connection.h
index 377019c82..3fbc728d1 100644
--- a/receiver/src/connection.h
+++ b/receiver/src/connection.h
@@ -15,6 +15,7 @@
#include "io/io.h"
#include "request.h"
#include "statistics.h"
+#include "logger/logger.h"
namespace hidra2 {
@@ -37,8 +38,7 @@ class Connection {
std::unique_ptr<RequestFactory> request_factory__;
std::unique_ptr<IO> io__;
mutable std::unique_ptr<Statistics> statistics__;
-
-
+ const AbstractLogger* log__;
private:
std::unique_ptr<Request> WaitForNewRequest(Error* err) const noexcept;
Error ProcessRequest(const std::unique_ptr<Request>& request) const noexcept;
diff --git a/receiver/src/main.cpp b/receiver/src/main.cpp
index dcb19c776..258d81d56 100644
--- a/receiver/src/main.cpp
+++ b/receiver/src/main.cpp
@@ -4,6 +4,7 @@
#include "receiver_config_factory.h"
#include "receiver_config.h"
+#include "receiver_logger.h"
hidra2::Error ReadConfigFile(int argc, char* argv[]) {
if (argc != 2) {
@@ -17,21 +18,25 @@ hidra2::Error ReadConfigFile(int argc, char* argv[]) {
int main (int argc, char* argv[]) {
auto err = ReadConfigFile(argc, argv);
+ const auto& logger = hidra2::GetDefaultReceiverLogger();
+
if (err) {
- std::cerr << "Cannot read config file: " << err << std::endl;
+ logger->Error("cannot read config file: " + err->Explain());
return 1;
}
auto config = hidra2::GetReceiverConfig();
+ logger->SetLogLevel(config->log_level);
+
static const std::string address = "0.0.0.0:" + std::to_string(config->listen_port);
auto* receiver = new hidra2::Receiver();
- std::cout << "Listening on " << address << std::endl;
+ logger->Info("listening on " + address);
receiver->Listen(address, &err);
if(err) {
- std::cerr << "Failed to start receiver: " << err << std::endl;
+ logger->Error("failed to start receiver: " + err->Explain());
return 1;
}
return 0;
diff --git a/receiver/src/receiver.cpp b/receiver/src/receiver.cpp
index c614fd1e8..54f79db87 100644
--- a/receiver/src/receiver.cpp
+++ b/receiver/src/receiver.cpp
@@ -10,7 +10,7 @@ namespace hidra2 {
const int Receiver::kMaxUnacceptedConnectionsBacklog = 5;
-Receiver::Receiver(): io__{GenerateDefaultIO()} {
+Receiver::Receiver(): io__{GenerateDefaultIO()}, log__{GetDefaultReceiverLogger()} {
}
@@ -18,6 +18,9 @@ Error Receiver::PrepareListener(std::string listener_address) {
Error err = nullptr;
listener_fd_ = io__->CreateAndBindIPTCPSocketListener(listener_address, kMaxUnacceptedConnectionsBacklog,
&err);
+ if (err) {
+ log__->Error("prepare listener: " + err->Explain());
+ }
return err;
}
@@ -43,7 +46,7 @@ void Receiver::ProcessConnections(Error* err) {
auto client_info_tuple = io__->InetAcceptConnection(listener_fd_, err);
if(*err) {
//TODO: this can produce a lot of error messages
- std::cerr << "An error occurred while accepting an incoming connection: " << err << std::endl;
+ log__->Error("accepting an incoming connection: " + (*err)->Explain());
return;
}
std::tie(address, connection_socket_fd) = *client_info_tuple;
@@ -51,9 +54,9 @@ void Receiver::ProcessConnections(Error* err) {
}
void Receiver::StartNewConnectionInSeparateThread(int connection_socket_fd, const std::string& address) {
+ log__->Info("new connection from " + address);
auto thread = io__->NewThread([connection_socket_fd, address] {
auto connection = std::unique_ptr<Connection>(new Connection(connection_socket_fd, address));
- std::cout << "[" << connection->GetId() << "] New connection from " << address << std::endl;
connection->Listen();
});
@@ -61,7 +64,6 @@ void Receiver::StartNewConnectionInSeparateThread(int connection_socket_fd, cons
thread->detach();
}
return;
-
}
}
\ No newline at end of file
diff --git a/receiver/src/receiver.h b/receiver/src/receiver.h
index a1da1fb41..55a94678a 100644
--- a/receiver/src/receiver.h
+++ b/receiver/src/receiver.h
@@ -3,9 +3,12 @@
#include <string>
#include <thread>
-#include "connection.h"
#include <list>
+
+#include "connection.h"
+#include "receiver_logger.h"
+
namespace hidra2 {
class Receiver {
@@ -22,6 +25,7 @@ class Receiver {
void Listen(std::string listener_address, Error* err, bool exit_after_first_connection = false);
std::unique_ptr<IO> io__;
+ const AbstractLogger* log__;
};
}
diff --git a/receiver/src/receiver_config.cpp b/receiver/src/receiver_config.cpp
index 984218ee0..fc81efa63 100644
--- a/receiver/src/receiver_config.cpp
+++ b/receiver/src/receiver_config.cpp
@@ -13,6 +13,7 @@ ReceiverConfigFactory::ReceiverConfigFactory() : io__{GenerateDefaultIO()} {
Error ReceiverConfigFactory::SetConfigFromFile(std::string file_name) {
JsonFileParser parser(file_name, &io__);
+ std::string log_level;
Error err;
(err = parser.GetString("MonitorDbAddress", &config.monitor_db_uri)) ||
(err = parser.GetUInt64("ListenPort", &config.listen_port)) ||
@@ -20,9 +21,15 @@ Error ReceiverConfigFactory::SetConfigFromFile(std::string file_name) {
(err = parser.GetBool("WriteToDb", &config.write_to_db)) ||
(err = parser.GetString("BrokerDbAddress", &config.broker_db_uri)) ||
(err = parser.GetString("BrokerDbName", &config.broker_db_name)) ||
-
(err = parser.GetString("MonitorDbName", &config.monitor_db_name));
+ (err = parser.GetString("LogLevel", &log_level));
+ if (err) {
+ return err;
+ }
+
+ config.log_level = StringToLogLevel(log_level, &err);
return err;
+
}
const ReceiverConfig* GetReceiverConfig() {
diff --git a/receiver/src/receiver_config.h b/receiver/src/receiver_config.h
index e348dc704..8a1653607 100644
--- a/receiver/src/receiver_config.h
+++ b/receiver/src/receiver_config.h
@@ -3,6 +3,7 @@
#include "io/io.h"
#include "common/error.h"
+#include "logger/logger.h"
namespace hidra2 {
@@ -14,7 +15,7 @@ struct ReceiverConfig {
uint64_t listen_port = 0;
bool write_to_disk = false;
bool write_to_db = false;
-
+ LogLevel log_level = LogLevel::Info;
};
const ReceiverConfig* GetReceiverConfig();
diff --git a/receiver/src/receiver_logger.cpp b/receiver/src/receiver_logger.cpp
new file mode 100644
index 000000000..b8b6ff788
--- /dev/null
+++ b/receiver/src/receiver_logger.cpp
@@ -0,0 +1,11 @@
+#include "receiver_logger.h"
+
+namespace hidra2 {
+
+
+AbstractLogger* GetDefaultReceiverLogger() {
+ static Logger logger = hidra2::CreateDefaultLoggerBin("receiver");
+ return logger.get();
+}
+
+}
diff --git a/receiver/src/receiver_logger.h b/receiver/src/receiver_logger.h
new file mode 100644
index 000000000..9eb7982bb
--- /dev/null
+++ b/receiver/src/receiver_logger.h
@@ -0,0 +1,14 @@
+#ifndef HIDRA2_RECEIVER_LOGGER_H
+#define HIDRA2_RECEIVER_LOGGER_H
+
+#include "logger/logger.h"
+
+namespace hidra2 {
+
+
+AbstractLogger* GetDefaultReceiverLogger();
+
+}
+
+
+#endif //HIDRA2_RECEIVER_LOGGER_H
diff --git a/receiver/src/request_handler_db_write.cpp b/receiver/src/request_handler_db_write.cpp
index b3b73bfb0..3b85b3770 100644
--- a/receiver/src/request_handler_db_write.cpp
+++ b/receiver/src/request_handler_db_write.cpp
@@ -1,6 +1,7 @@
#include "request_handler_db_write.h"
#include "request.h"
#include "receiver_config.h"
+#include "receiver_logger.h"
namespace hidra2 {
@@ -13,11 +14,15 @@ Error RequestHandlerDbWrite::ProcessRequest(const Request& request) const {
file_info.name = request.GetFileName();
file_info.size = request.GetDataSize();
file_info.id = request.GetDataID();
- return db_client__->Insert(file_info, false);
-
+ auto err = db_client__->Insert(file_info, false);
+ if (!err) {
+ log__->Debug(std::string{"insert record to "} + kDBCollectionName + " in " + GetReceiverConfig()->broker_db_name +
+ " at " + GetReceiverConfig()->broker_db_uri);
+ }
+ return err;
}
-RequestHandlerDbWrite::RequestHandlerDbWrite() {
+RequestHandlerDbWrite::RequestHandlerDbWrite(): log__{GetDefaultReceiverLogger()} {
DatabaseFactory factory;
Error err;
db_client__ = factory.Create(&err);
diff --git a/receiver/src/request_handler_db_write.h b/receiver/src/request_handler_db_write.h
index ddd57350a..5faf24852 100644
--- a/receiver/src/request_handler_db_write.h
+++ b/receiver/src/request_handler_db_write.h
@@ -3,6 +3,7 @@
#include "request_handler.h"
#include "database/database.h"
+#include "logger/logger.h"
#include "io/io.h"
@@ -14,6 +15,7 @@ class RequestHandlerDbWrite final: public RequestHandler {
StatisticEntity GetStatisticEntity() const override;
Error ProcessRequest(const Request& request) const override;
std::unique_ptr<Database> db_client__;
+ const AbstractLogger* log__;
private:
Error ConnectToDbIfNeeded() const;
mutable bool connected_to_db = false;
diff --git a/receiver/src/request_handler_file_write.cpp b/receiver/src/request_handler_file_write.cpp
index 3db786945..83e7a1905 100644
--- a/receiver/src/request_handler_file_write.cpp
+++ b/receiver/src/request_handler_file_write.cpp
@@ -1,6 +1,8 @@
#include "request_handler_file_write.h"
#include "io/io_factory.h"
#include "request.h"
+#include "receiver_logger.h"
+
namespace hidra2 {
Error RequestHandlerFileWrite::ProcessRequest(const Request& request) const {
@@ -13,11 +15,15 @@ Error RequestHandlerFileWrite::ProcessRequest(const Request& request) const {
auto fname = request.GetFileName();
//TODO: folder to write in config file
- return io__->WriteDataToFile("files/" + fname, data, fsize);
+ auto err = io__->WriteDataToFile("files/" + fname, data, fsize);
+ if (!err) {
+ log__->Debug("saved file of size " + std::to_string(fsize) + " to files/" + fname);
+ }
+ return err;
}
-RequestHandlerFileWrite::RequestHandlerFileWrite() : io__{GenerateDefaultIO()} {
+RequestHandlerFileWrite::RequestHandlerFileWrite() : io__{GenerateDefaultIO()} , log__{GetDefaultReceiverLogger()} {
}
diff --git a/receiver/src/request_handler_file_write.h b/receiver/src/request_handler_file_write.h
index af81e2d95..99d25b4ef 100644
--- a/receiver/src/request_handler_file_write.h
+++ b/receiver/src/request_handler_file_write.h
@@ -2,6 +2,7 @@
#define HIDRA2_REQUEST_HANDLER_FILE_WRITE_H
#include "request_handler.h"
+#include "logger/logger.h"
#include "io/io.h"
@@ -15,6 +16,7 @@ class RequestHandlerFileWrite final: public RequestHandler {
StatisticEntity GetStatisticEntity() const override;
Error ProcessRequest(const Request& request) const override;
std::unique_ptr<IO> io__;
+ const AbstractLogger* log__;
};
}
diff --git a/receiver/src/statistics_sender_influx_db.cpp b/receiver/src/statistics_sender_influx_db.cpp
index 8e43792ac..cf60a5f01 100644
--- a/receiver/src/statistics_sender_influx_db.cpp
+++ b/receiver/src/statistics_sender_influx_db.cpp
@@ -4,6 +4,7 @@
#include "statistics.h"
#include "receiver_config.h"
+#include "receiver_logger.h"
namespace hidra2 {
@@ -22,14 +23,19 @@ void StatisticsSenderInfluxDb::SendStatistics(const StatisticsToSend& statistic)
auto responce = httpclient__->Post(GetReceiverConfig()->monitor_db_uri + "/write?db=" +
GetReceiverConfig()->monitor_db_name, StatisticsToString(statistic),
&code, &err);
+ std::string msg = "sending statistics to " + GetReceiverConfig()->monitor_db_name + " at " +
+ GetReceiverConfig()->monitor_db_uri;
if (err) {
- std::cerr << "Error sending statistics: " << err << std::endl;
+ log__->Error(msg + " - " + err->Explain());
return;
}
if (code != HttpCode::OK && code != HttpCode::NoContent) {
- std::cerr << "Error sending statistics: " << responce << std::endl;
+ log__->Error(msg + " - " + responce);
+ return;
}
+
+ log__->Debug(msg);
}
std::string StatisticsSenderInfluxDb::StatisticsToString(const StatisticsToSend& statistic) const noexcept {
@@ -44,7 +50,7 @@ std::string StatisticsSenderInfluxDb::StatisticsToString(const StatisticsToSend&
return str;
}
-StatisticsSenderInfluxDb::StatisticsSenderInfluxDb(): httpclient__{DefaultHttpClient()} {
+StatisticsSenderInfluxDb::StatisticsSenderInfluxDb(): httpclient__{DefaultHttpClient()}, log__{GetDefaultReceiverLogger()} {
};
diff --git a/receiver/src/statistics_sender_influx_db.h b/receiver/src/statistics_sender_influx_db.h
index aeff17333..61cdad751 100644
--- a/receiver/src/statistics_sender_influx_db.h
+++ b/receiver/src/statistics_sender_influx_db.h
@@ -3,7 +3,7 @@
#include "http_client/http_client.h"
#include "statistics_sender.h"
-
+#include "logger/logger.h"
namespace hidra2 {
@@ -12,6 +12,7 @@ class StatisticsSenderInfluxDb : public StatisticsSender {
StatisticsSenderInfluxDb();
virtual void SendStatistics(const StatisticsToSend& statistic) const noexcept override;
std::unique_ptr<HttpClient> httpclient__;
+ const AbstractLogger* log__;
private:
std::string StatisticsToString(const StatisticsToSend& statistic) const noexcept;
diff --git a/receiver/unittests/mock_receiver_config.cpp b/receiver/unittests/mock_receiver_config.cpp
index c1d5108bf..14b665d07 100644
--- a/receiver/unittests/mock_receiver_config.cpp
+++ b/receiver/unittests/mock_receiver_config.cpp
@@ -15,6 +15,25 @@ Error SetReceiverConfig (const ReceiverConfig& config) {
ReceiverConfigFactory config_factory;
config_factory.io__ = std::unique_ptr<IO> {&mock_io};
+ std::string log_level;
+ switch (config.log_level) {
+ case LogLevel::Error:
+ log_level = "error";
+ break;
+ case LogLevel::Warning:
+ log_level = "warning";
+ break;
+ case LogLevel::Info:
+ log_level = "info";
+ break;
+ case LogLevel::Debug:
+ log_level = "debug";
+ break;
+ case LogLevel::None:
+ log_level = "none";
+ break;
+ }
+
auto config_string = std::string("{\"MonitorDbAddress\":") + "\"" + config.monitor_db_uri + "\"";
config_string += "," + std::string("\"MonitorDbName\":") + "\"" + config.monitor_db_name + "\"";
config_string += "," + std::string("\"BrokerDbName\":") + "\"" + config.broker_db_name + "\"";
@@ -22,6 +41,7 @@ Error SetReceiverConfig (const ReceiverConfig& config) {
config_string += "," + std::string("\"ListenPort\":") + std::to_string(config.listen_port);
config_string += "," + std::string("\"WriteToDisk\":") + (config.write_to_disk ? "true" : "false");
config_string += "," + std::string("\"WriteToDb\":") + (config.write_to_db ? "true" : "false");
+ config_string += "," + std::string("\"LogLevel\":") + "\"" + log_level + "\"";
config_string += "}";
diff --git a/receiver/unittests/test_config.cpp b/receiver/unittests/test_config.cpp
index 008b965a5..cfcec334b 100644
--- a/receiver/unittests/test_config.cpp
+++ b/receiver/unittests/test_config.cpp
@@ -56,6 +56,7 @@ TEST_F(ConfigTests, ReadSettings) {
test_config.write_to_db = true;
test_config.broker_db_uri = "localhost:27017";
test_config.broker_db_name = "test";
+ test_config.log_level = hidra2::LogLevel::Error;
auto err = hidra2::SetReceiverConfig(test_config);
@@ -67,8 +68,9 @@ TEST_F(ConfigTests, ReadSettings) {
ASSERT_THAT(config->broker_db_uri, Eq("localhost:27017"));
ASSERT_THAT(config->broker_db_name, Eq("test"));
ASSERT_THAT(config->listen_port, Eq(4200));
- ASSERT_THAT(config->write_to_disk, true);
- ASSERT_THAT(config->write_to_db, true);
+ ASSERT_THAT(config->write_to_disk, Eq(true));
+ ASSERT_THAT(config->write_to_db, Eq(true));
+ ASSERT_THAT(config->log_level, Eq(hidra2::LogLevel::Error));
}
diff --git a/receiver/unittests/test_connection.cpp b/receiver/unittests/test_connection.cpp
index d09e26507..d585fa862 100644
--- a/receiver/unittests/test_connection.cpp
+++ b/receiver/unittests/test_connection.cpp
@@ -1,7 +1,8 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
-#include <unittests/MockIO.h>
+#include "unittests/MockIO.h"
+#include "unittests/MockLogger.h"
#include "../src/connection.h"
#include "../src/receiver_error.h"
#include "../src/request.h"
@@ -21,18 +22,22 @@ using ::testing::NiceMock;
using ::testing::SaveArg;
using ::testing::SaveArgPointee;
using ::testing::InSequence;
+using ::testing::HasSubstr;
using ::testing::SetArgPointee;
-using ::hidra2::Error;
-using ::hidra2::ErrorInterface;
-using ::hidra2::FileDescriptor;
-using ::hidra2::SocketDescriptor;
-using ::hidra2::GenericNetworkRequestHeader;
-using ::hidra2::SendDataResponse;
-using ::hidra2::GenericNetworkRequestHeader;
-using ::hidra2::GenericNetworkResponse;
-using ::hidra2::Opcode;
-using ::hidra2::Connection;
-using ::hidra2::MockIO;
+using ::testing::AllOf;
+
+using hidra2::Error;
+using hidra2::ErrorInterface;
+using hidra2::FileDescriptor;
+using hidra2::SocketDescriptor;
+using hidra2::GenericNetworkRequestHeader;
+using hidra2::SendDataResponse;
+using hidra2::GenericNetworkRequestHeader;
+using hidra2::GenericNetworkResponse;
+using hidra2::Opcode;
+using hidra2::Connection;
+using hidra2::MockIO;
+using hidra2::MockLogger;
using hidra2::Request;
using hidra2::Statistics;
using hidra2::StatisticEntity;
@@ -45,9 +50,9 @@ TEST(Connection, Constructor) {
ASSERT_THAT(dynamic_cast<hidra2::Statistics*>(connection.statistics__.get()), Ne(nullptr));
ASSERT_THAT(dynamic_cast<hidra2::IO*>(connection.io__.get()), Ne(nullptr));
ASSERT_THAT(dynamic_cast<hidra2::RequestFactory*>(connection.request_factory__.get()), Ne(nullptr));
-}
-
+ ASSERT_THAT(dynamic_cast<const hidra2::AbstractLogger*>(connection.log__), Ne(nullptr));
+}
class MockRequest: public Request {
public:
@@ -79,14 +84,19 @@ class MockRequestFactory: public hidra2::RequestFactory {
class ConnectionTests : public Test {
public:
- Connection connection{0, "some_address"};
+ std::string connected_uri{"some_address"};
+ Connection connection{0, connected_uri};
MockIO mock_io;
MockRequestFactory mock_factory;
NiceMock<MockStatistics> mock_statictics;
+ NiceMock<hidra2::MockLogger> mock_logger;
+
void SetUp() override {
connection.io__ = std::unique_ptr<hidra2::IO> {&mock_io};
connection.statistics__ = std::unique_ptr<hidra2::Statistics> {&mock_statictics};
connection.request_factory__ = std::unique_ptr<hidra2::RequestFactory> {&mock_factory};
+ connection.log__ = &mock_logger;
+
ON_CALL(mock_io, ReceiveWithTimeout_t(_, _, _, _, _)).
WillByDefault(DoAll(testing::SetArgPointee<4>(nullptr),
testing::Return(0)));
@@ -114,6 +124,9 @@ TEST_F(ConnectionTests, ErrorWaitForNewRequest) {
Return(0))
);
+ EXPECT_CALL(mock_logger, Error(AllOf(HasSubstr("waiting for request"), HasSubstr(connected_uri))));
+
+
connection.Listen();
}
@@ -138,39 +151,22 @@ TEST_F(ConnectionTests, CallsHandleRequest) {
Return(new hidra2::SimpleError{""})
);
- EXPECT_CALL(mock_io, Send_t(_, _, _, _)).WillOnce(
- DoAll(SetArgPointee<3>(new hidra2::IOError("Test Send Error", hidra2::IOErrorType::kUnknownIOError)),
- Return(0)
- ));
-
-
- connection.Listen();
-}
-
-TEST_F(ConnectionTests, SendsNoErrorToProducer) {
+ EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("processing request"), HasSubstr(connected_uri))));
- GenericNetworkRequestHeader header;
- auto request = new MockRequest{header, 1};
- EXPECT_CALL(mock_io, ReceiveWithTimeout_t(_, _, _, _, _));
+ EXPECT_CALL(mock_logger, Error(AllOf(HasSubstr("processing request"), HasSubstr(connected_uri))));
- EXPECT_CALL(mock_factory, GenerateRequest_t(_, _, _)).WillOnce(
- Return(request)
- );
- EXPECT_CALL(*request, Handle_t()).WillOnce(
- Return(nullptr)
- );
- GenericNetworkResponse response;
- EXPECT_CALL(mock_io, Send_t(_, _, sizeof(GenericNetworkResponse), _)).WillOnce(
+ EXPECT_CALL(mock_io, Send_t(_, _, _, _)).WillOnce(
DoAll(SetArgPointee<3>(new hidra2::IOError("Test Send Error", hidra2::IOErrorType::kUnknownIOError)),
- SaveArg1ToGenericNetworkResponse(&response),
Return(0)
));
- connection.Listen();
+ EXPECT_CALL(mock_logger, Error(AllOf(HasSubstr("sending response"), HasSubstr(connected_uri))));
+
+ EXPECT_CALL(mock_logger, Info(AllOf(HasSubstr("disconnected"), HasSubstr(connected_uri))));
- ASSERT_THAT(response.error_code, Eq(hidra2::NetworkErrorCode::kNetErrorNoError));
+ connection.Listen();
}
TEST_F(ConnectionTests, SendsErrorToProducer) {
@@ -236,7 +232,6 @@ TEST_F(ConnectionTests, FillsStatistics) {
Return(nullptr)
);
-
EXPECT_CALL(mock_io, Send_t(_, _, _, _)).WillOnce(
DoAll(SetArgPointee<3>(nullptr),
Return(0)
diff --git a/receiver/unittests/test_receiver.cpp b/receiver/unittests/test_receiver.cpp
index 0226b182b..707bfaacb 100644
--- a/receiver/unittests/test_receiver.cpp
+++ b/receiver/unittests/test_receiver.cpp
@@ -1,6 +1,8 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
-#include <unittests/MockIO.h>
+
+#include "unittests/MockIO.h"
+#include "unittests/MockLogger.h"
#include "../src/receiver.h"
#include "../src/receiver_error.h"
#include "../src/connection.h"
@@ -12,8 +14,10 @@ using ::testing::SetArgReferee;
using ::testing::SetArgPointee;
using ::testing::Gt;
using ::testing::Eq;
+using ::testing::Ne;
using ::testing::Mock;
using ::testing::InSequence;
+using ::testing::HasSubstr;
using ::hidra2::Error;
using ::hidra2::FileDescriptor;
using ::hidra2::ErrorInterface;
@@ -22,6 +26,13 @@ using ::hidra2::SocketDescriptor;
namespace {
+TEST(Receiver, Constructor) {
+ hidra2::Receiver receiver;
+ ASSERT_THAT(dynamic_cast<const hidra2::AbstractLogger*>(receiver.log__), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<hidra2::IO*>(receiver.io__.get()), Ne(nullptr));
+}
+
+
class StartListenerFixture : public testing::Test {
public:
const hidra2::SocketDescriptor expected_socket_descriptor = 20;
@@ -32,13 +43,14 @@ class StartListenerFixture : public testing::Test {
const FileDescriptor expected_fd = 12643;
Error err;
-
+ ::testing::NiceMock<hidra2::MockLogger> mock_logger;
::testing::NiceMock<hidra2::MockIO> mock_io;
hidra2::Receiver receiver;
void SetUp() override {
err = nullptr;
receiver.io__ = std::unique_ptr<hidra2::IO> {&mock_io};
+ receiver.log__ = &mock_logger;
}
void TearDown() override {
receiver.io__.release();
@@ -54,6 +66,9 @@ TEST_F(StartListenerFixture, CreateAndBindIPTCPSocketListenerError) {
Return(0)
));
+ EXPECT_CALL(mock_logger, Error(HasSubstr("prepare listener")));
+
+
receiver.Listen(expected_address, &err, true);
ASSERT_THAT(err, Eq(hidra2::IOErrorTemplates::kUnknownIOError));
@@ -67,6 +82,8 @@ TEST_F(StartListenerFixture, InetAcceptConnectionError) {
Return(new std::tuple<std::string, SocketDescriptor>(expected_address, expected_socket_descriptor_client))
));
+ EXPECT_CALL(mock_logger, Error(HasSubstr("incoming connection")));
+
receiver.Listen(expected_address, &err, true);
ASSERT_THAT(err, Eq(hidra2::IOErrorTemplates::kUnknownIOError));
@@ -85,6 +102,9 @@ TEST_F(StartListenerFixture, Ok) {
Return(nullptr)
);
+ EXPECT_CALL(mock_logger, Info(HasSubstr("new connection from " + expected_address)));
+
+
receiver.Listen(expected_address, &err, true);
ASSERT_THAT(err, Eq(nullptr));
diff --git a/receiver/unittests/test_request_handler_db_writer.cpp b/receiver/unittests/test_request_handler_db_writer.cpp
index 99075925a..fe1799522 100644
--- a/receiver/unittests/test_request_handler_db_writer.cpp
+++ b/receiver/unittests/test_request_handler_db_writer.cpp
@@ -3,6 +3,7 @@
#include "unittests/MockIO.h"
#include "unittests/MockDatabase.h"
+#include "unittests/MockLogger.h"
#include "../src/receiver_error.h"
#include "../src/request.h"
@@ -28,6 +29,10 @@ using ::testing::Mock;
using ::testing::NiceMock;
using ::testing::InSequence;
using ::testing::SetArgPointee;
+using ::testing::AllOf;
+using ::testing::HasSubstr;
+
+
using ::hidra2::Error;
using ::hidra2::ErrorInterface;
using ::hidra2::FileDescriptor;
@@ -62,11 +67,13 @@ class DbWriterHandlerTests : public Test {
NiceMock<MockIO> mock_io;
std::unique_ptr<NiceMock<MockRequest>> mock_request;
NiceMock<MockDatabase> mock_db;
+ NiceMock<hidra2::MockLogger> mock_logger;
ReceiverConfig config;
void SetUp() override {
GenericNetworkRequestHeader request_header;
request_header.data_id = 2;
handler.db_client__ = std::unique_ptr<hidra2::Database> {&mock_db};
+ handler.log__ = &mock_logger;
mock_request.reset(new NiceMock<MockRequest> {request_header, 1});
}
void TearDown() override {
@@ -74,9 +81,11 @@ class DbWriterHandlerTests : public Test {
}
};
-TEST(DBWritewr, HandlerHasCorrectDbFactory) {
+TEST(DBWritewr, Constructor) {
RequestHandlerDbWrite handler;
ASSERT_THAT(dynamic_cast<hidra2::MongoDBClient*>(handler.db_client__.get()), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<const hidra2::AbstractLogger*>(handler.log__), Ne(nullptr));
+
}
@@ -130,8 +139,11 @@ MATCHER_P(CompareFileInfo, file, "") {
TEST_F(DbWriterHandlerTests, CallsInsert) {
+ config.broker_db_name = "test";
+ config.broker_db_uri = "127.0.0.1:27017";
+ SetReceiverConfig(config);
- EXPECT_CALL(mock_db, Connect_t(_, _, hidra2::kDBCollectionName)).
+ EXPECT_CALL(mock_db, Connect_t(config.broker_db_uri, config.broker_db_name, hidra2::kDBCollectionName)).
WillOnce(testing::Return(nullptr));
std::string expected_file_name = "2.bin";
@@ -158,37 +170,15 @@ TEST_F(DbWriterHandlerTests, CallsInsert) {
EXPECT_CALL(mock_db, Insert_t(CompareFileInfo(file_info), _)).
WillOnce(testing::Return(nullptr));
- handler.ProcessRequest(*mock_request);
-}
+ EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("insert record"),
+ HasSubstr(config.broker_db_uri),
+ HasSubstr(config.broker_db_name),
+ HasSubstr(hidra2::kDBCollectionName)
+ )
+ )
+ );
-
-/*
-TEST_F(DbWriterHandlerTests, CallsWriteFile) {
- std::string expected_file_name = "2.bin";
- uint64_t expected_file_size = 10;
- EXPECT_CALL(*mock_request, GetDataSize())
- .WillOnce(Return(expected_file_size))
- ;
-
- hidra2::FileData data;
- EXPECT_CALL(*mock_request, GetData())
- .WillOnce(ReturnRef(data))
- ;
-
- EXPECT_CALL(*mock_request, GetFileName())
- .WillOnce(Return(expected_file_name))
- ;
-
-
- EXPECT_CALL(mock_io, WriteDataToFile_t("files/" + expected_file_name, _, expected_file_size))
- .WillOnce(
- Return(hidra2::IOErrorTemplates::kUnknownIOError.Generate().release())
- );
-
- auto err = handler.ProcessRequest(*mock_request);
-
- ASSERT_THAT(err, Eq(hidra2::IOErrorTemplates::kUnknownIOError));
+ handler.ProcessRequest(*mock_request);
}
-*/
}
\ No newline at end of file
diff --git a/receiver/unittests/test_request_handler_file_write.cpp b/receiver/unittests/test_request_handler_file_write.cpp
index 5640ce453..41ee22dcf 100644
--- a/receiver/unittests/test_request_handler_file_write.cpp
+++ b/receiver/unittests/test_request_handler_file_write.cpp
@@ -1,12 +1,16 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
-#include <unittests/MockIO.h>
+
+#include "unittests/MockIO.h"
+#include "unittests/MockLogger.h"
+
#include "../src/receiver_error.h"
#include "../src/request.h"
#include "../src/request_handler.h"
#include "../src/request_handler_file_write.h"
#include "common/networking.h"
+
using ::testing::Test;
using ::testing::Return;
using ::testing::ReturnRef;
@@ -20,6 +24,10 @@ using ::testing::Mock;
using ::testing::NiceMock;
using ::testing::InSequence;
using ::testing::SetArgPointee;
+using ::testing::AllOf;
+using ::testing::HasSubstr;
+
+
using ::hidra2::Error;
using ::hidra2::ErrorInterface;
using ::hidra2::FileDescriptor;
@@ -31,6 +39,13 @@ using ::hidra2::GenericNetworkRequestHeader;
namespace {
+TEST(FileWrite, Constructor) {
+ RequestHandlerFileWrite handler;
+ ASSERT_THAT(dynamic_cast<hidra2::IO*>(handler.io__.get()), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<const hidra2::AbstractLogger*>(handler.log__), Ne(nullptr));
+}
+
+
class MockRequest: public Request {
public:
MockRequest(const GenericNetworkRequestHeader& request_header, SocketDescriptor socket_fd):
@@ -46,15 +61,16 @@ class FileWriteHandlerTests : public Test {
RequestHandlerFileWrite handler;
NiceMock<MockIO> mock_io;
std::unique_ptr<MockRequest> mock_request;
+ NiceMock<hidra2::MockLogger> mock_logger;
+ std::string expected_file_name = "2.bin";
+ uint64_t expected_file_size = 10;
+ void MockRequestData();
void SetUp() override {
GenericNetworkRequestHeader request_header;
request_header.data_id = 2;
mock_request.reset(new MockRequest{request_header, 1});
handler.io__ = std::unique_ptr<hidra2::IO> {&mock_io};
- /* ON_CALL(mock_io, Receive_t(socket_fd_, _, data_size_, _)).WillByDefault(
- DoAll(SetArgPointee<3>(nullptr),
- Return(0)
- ));*/
+ handler.log__ = &mock_logger;
}
void TearDown() override {
handler.io__.release();
@@ -88,9 +104,7 @@ TEST_F(FileWriteHandlerTests, ErrorWhenTooBigFileSize) {
ASSERT_THAT(err, Eq(hidra2::ReceiverErrorTemplates::kBadRequest));
}
-TEST_F(FileWriteHandlerTests, CallsWriteFile) {
- std::string expected_file_name = "2.bin";
- uint64_t expected_file_size = 10;
+void FileWriteHandlerTests::MockRequestData() {
EXPECT_CALL(*mock_request, GetDataSize())
.WillOnce(Return(expected_file_size))
;
@@ -103,7 +117,11 @@ TEST_F(FileWriteHandlerTests, CallsWriteFile) {
EXPECT_CALL(*mock_request, GetFileName())
.WillOnce(Return(expected_file_name))
;
+}
+TEST_F(FileWriteHandlerTests, CallsWriteFile) {
+
+ MockRequestData();
EXPECT_CALL(mock_io, WriteDataToFile_t("files/" + expected_file_name, _, expected_file_size))
.WillOnce(
@@ -116,4 +134,21 @@ TEST_F(FileWriteHandlerTests, CallsWriteFile) {
}
+TEST_F(FileWriteHandlerTests, WritesToLog) {
+
+ MockRequestData();
+
+ EXPECT_CALL(mock_io, WriteDataToFile_t(_, _, _))
+ .WillOnce(Return(nullptr));
+
+ EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("saved file"),
+ HasSubstr(expected_file_name),
+ HasSubstr(std::to_string(expected_file_size))
+ )
+ )
+ );
+ handler.ProcessRequest(*mock_request);
+}
+
+
}
\ No newline at end of file
diff --git a/receiver/unittests/test_statistics_sender_influx_db.cpp b/receiver/unittests/test_statistics_sender_influx_db.cpp
index f12c4aff2..904194745 100644
--- a/receiver/unittests/test_statistics_sender_influx_db.cpp
+++ b/receiver/unittests/test_statistics_sender_influx_db.cpp
@@ -1,6 +1,8 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
-#include <unittests/MockIO.h>
+
+#include "unittests/MockIO.h"
+#include "unittests/MockLogger.h"
#include "../src/statistics_sender_influx_db.h"
#include "../src/statistics_sender.h"
@@ -23,6 +25,8 @@ using ::testing::Ne;
using ::testing::Mock;
using ::testing::NiceMock;
using ::testing::SaveArg;
+using ::testing::HasSubstr;
+using ::testing::AllOf;
using ::testing::SaveArgPointee;
using ::testing::InSequence;
using ::testing::SetArgPointee;
@@ -38,6 +42,7 @@ namespace {
TEST(SenderInfluxDb, Constructor) {
StatisticsSenderInfluxDb sender;
ASSERT_THAT(dynamic_cast<hidra2::CurlHttpClient*>(sender.httpclient__.get()), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<const hidra2::AbstractLogger*>(sender.log__), Ne(nullptr));
}
@@ -45,8 +50,24 @@ class SenderInfluxDbTests : public Test {
public:
StatisticsSenderInfluxDb sender;
MockHttpClient mock_http_client;
+ NiceMock<hidra2::MockLogger> mock_logger;
+ StatisticsToSend statistics;
+ ReceiverConfig config;
+
void SetUp() override {
+ statistics.n_requests = 4;
+ statistics.entity_shares[hidra2::StatisticEntity::kDisk] = 0.6;
+ statistics.entity_shares[hidra2::StatisticEntity::kNetwork] = 0.3;
+ statistics.entity_shares[hidra2::StatisticEntity::kDatabase] = 0.1;
+ statistics.elapsed_ms = 100;
+ statistics.data_volume = 1000;
+
+ config.monitor_db_uri = "test_uri";
+ config.monitor_db_name = "test_name";
+ SetReceiverConfig(config);
+
sender.httpclient__.reset(&mock_http_client);
+ sender.log__ = &mock_logger;
}
void TearDown() override {
sender.httpclient__.release();
@@ -55,19 +76,6 @@ class SenderInfluxDbTests : public Test {
TEST_F(SenderInfluxDbTests, SendStatisticsCallsPost) {
- StatisticsToSend statistics;
- statistics.n_requests = 4;
- statistics.entity_shares[hidra2::StatisticEntity::kDisk] = 0.6;
- statistics.entity_shares[hidra2::StatisticEntity::kNetwork] = 0.3;
- statistics.entity_shares[hidra2::StatisticEntity::kDatabase] = 0.1;
- statistics.elapsed_ms = 100;
- statistics.data_volume = 1000;
-
- ReceiverConfig config;
- config.monitor_db_uri = "test_uri";
- config.monitor_db_name = "test_name";
- SetReceiverConfig(config);
-
std::string expect_string = "statistics,receiver=1,connection=1 elapsed_ms=100,data_volume=1000,"
"n_requests=4,db_share=0.1000,network_share=0.3000,disk_share=0.6000";
EXPECT_CALL(mock_http_client, Post_t("test_uri/write?db=test_name", expect_string, _, _)).
@@ -76,6 +84,38 @@ TEST_F(SenderInfluxDbTests, SendStatisticsCallsPost) {
Return("")
));
+ EXPECT_CALL(mock_logger, Error(AllOf(HasSubstr("sending statistics"), HasSubstr(config.monitor_db_uri))));
+
+
+ sender.SendStatistics(statistics);
+}
+
+TEST_F(SenderInfluxDbTests, LogErrorWithWrongResponceSendStatistics) {
+ EXPECT_CALL(mock_http_client, Post_t(_, _, _, _)).
+ WillOnce(
+ DoAll(SetArgPointee<2>(hidra2::HttpCode::BadRequest), SetArgPointee<3>(nullptr), Return("error response")
+ ));
+
+ EXPECT_CALL(mock_logger, Error(AllOf(HasSubstr("sending statistics"), HasSubstr("error response"))));
+
+
+ sender.SendStatistics(statistics);
+}
+
+TEST_F(SenderInfluxDbTests, LogDebugSendStatistics) {
+ EXPECT_CALL(mock_http_client, Post_t(_, _, _, _)).
+ WillOnce(
+ DoAll(SetArgPointee<3>(nullptr), SetArgPointee<2>(hidra2::HttpCode::OK), Return("error response")
+ ));
+
+ EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("sending statistics"),
+ HasSubstr(config.monitor_db_uri),
+ HasSubstr(config.monitor_db_name)
+ )
+ )
+ );
+
+
sender.SendStatistics(statistics);
}
diff --git a/tests/automatic/full_chain/simple_chain/check_linux.sh b/tests/automatic/full_chain/simple_chain/check_linux.sh
index a85cbe6c4..5e4304a60 100644
--- a/tests/automatic/full_chain/simple_chain/check_linux.sh
+++ b/tests/automatic/full_chain/simple_chain/check_linux.sh
@@ -36,6 +36,6 @@ brokerid=`echo $!`
mkdir files
$1 localhost:4200 100 100 &
#producerrid=`echo $!`
-sleep 0.1
+sleep 0.3
$4 ${broker_address} ${broker_database_name} 2 | grep "Processed 100 file(s)"
diff --git a/tests/automatic/settings/broker_settings.json b/tests/automatic/settings/broker_settings.json
index 56bf15f52..a5ccb0752 100644
--- a/tests/automatic/settings/broker_settings.json
+++ b/tests/automatic/settings/broker_settings.json
@@ -2,5 +2,6 @@
"BrokerDbAddress":"127.0.0.1:27017",
"MonitorDbAddress": "localhost:8086",
"MonitorDbName": "db_test",
- "port":5005
+ "port":5005,
+ "LogLevel":"info"
}
\ No newline at end of file
diff --git a/tests/automatic/settings/receiver.json b/tests/automatic/settings/receiver.json
index 5939e7d82..709682ff6 100644
--- a/tests/automatic/settings/receiver.json
+++ b/tests/automatic/settings/receiver.json
@@ -5,5 +5,6 @@
"BrokerDbName": "test_run",
"ListenPort":4200,
"WriteToDisk":true,
- "WriteToDb":true
+ "WriteToDb":true,
+ "LogLevel" : "info"
}
\ No newline at end of file
diff --git a/tests/manual/performance_full_chain_simple/broker.json b/tests/manual/performance_full_chain_simple/broker.json
index c45d16f2f..31aef140d 100644
--- a/tests/manual/performance_full_chain_simple/broker.json
+++ b/tests/manual/performance_full_chain_simple/broker.json
@@ -2,5 +2,6 @@
"BrokerDbAddress":"localhost:27017",
"MonitorDbAddress": "localhost:8086",
"MonitorDbName": "db_test",
- "port":5005
+ "port":5005,
+ "LogLevel":"debug"
}
\ No newline at end of file
diff --git a/tests/manual/performance_full_chain_simple/fluentd.conf b/tests/manual/performance_full_chain_simple/fluentd.conf
new file mode 100644
index 000000000..e24b4f127
--- /dev/null
+++ b/tests/manual/performance_full_chain_simple/fluentd.conf
@@ -0,0 +1,49 @@
+#used to run fluentd on Maxwell
+<source>
+ @type http
+ port 9880
+ bind 0.0.0.0
+ add_remote_addr true
+</source>
+
+<filter asapo>
+@type parser
+key_name message
+reserve_data true
+<parse>
+ @type regexp
+ expression /\[(?<time>[^\]]*)\] \[(?<source>[^ ]*)\] \[(?<level>[^ ]*)\] (?<message>[^\n]*)/
+ time_key time
+ time_format %Y-%m-%d %H:%M:%S.%N
+ </parse>
+</filter>
+
+<source>
+ @type tail
+ path /logs/*.broker
+ pos_file /tmp/asapo.logrus.log.pos1
+ tag asapo.logrus
+ format json
+ time_format %Y-%m-%d %H:%M:%S.%N
+</source>
+
+<source>
+ @type tail
+ path /logs/*.receiver
+ pos_file /tmp/asapo.logrus.log.pos2
+ tag asapo
+ format none
+</source>
+
+<match asapo.**>
+ @type elasticsearch
+ host max-wgs001.desy.de
+ port 9200
+ flush_interval 5s
+ logstash_format true
+ time_key_format %Y-%m-%dT%H:%M:%S.%N
+ time_key time
+ time_key_exclude_timestamp true
+ buffer_type memory
+ flush_interval 1s
+</match>
diff --git a/tests/manual/performance_full_chain_simple/receiver.json b/tests/manual/performance_full_chain_simple/receiver.json
index 5939e7d82..5330d75a7 100644
--- a/tests/manual/performance_full_chain_simple/receiver.json
+++ b/tests/manual/performance_full_chain_simple/receiver.json
@@ -5,5 +5,6 @@
"BrokerDbName": "test_run",
"ListenPort":4200,
"WriteToDisk":true,
- "WriteToDb":true
+ "WriteToDb":true,
+ "LogLevel":"debug"
}
\ No newline at end of file
diff --git a/tests/manual/performance_full_chain_simple/test.sh b/tests/manual/performance_full_chain_simple/test.sh
index f8b533a5b..8846a9972 100755
--- a/tests/manual/performance_full_chain_simple/test.sh
+++ b/tests/manual/performance_full_chain_simple/test.sh
@@ -18,11 +18,14 @@ monitor_node=zitpcx27016
monitor_port=8086
+#logs
+log_dir=~/fullchain_tests/logs
+
# starts receiver on $receiver_node
# runs producer with various file sizes from $producer_node and measures performance
-file_size=100
-file_num=$((100000000 / $file_size))
+file_size=1000
+file_num=$((10000000 / $file_size))
echo filesize: ${file_size}K, filenum: $file_num
# receiver_setup
@@ -30,7 +33,7 @@ receiver_node=max-wgs
receiver_ip=`resolveip -s ${receiver_node}`
receiver_port=4201
receiver_dir=/gpfs/petra3/scratch/yakubov/receiver_tests
-ssh ${receiver_node} mkdir -p ${receiver_dir}
+ssh ${receiver_node} mkdir -p ${receiver_dir}/logs
ssh ${receiver_node} mkdir -p ${receiver_dir}/files
scp ../../../cmake-build-release/receiver/receiver ${receiver_node}:${receiver_dir}
cat receiver.json |
@@ -49,12 +52,12 @@ scp settings_tmp.json ${receiver_node}:${receiver_dir}/settings.json
producer_node=max-display001
#producer_node=max-wgs
producer_dir=~/fullchain_tests
-ssh ${producer_node} mkdir -p ${producer_dir}
scp ../../../cmake-build-release/examples/producer/dummy-data-producer/dummy-data-producer ${producer_node}:${producer_dir}
#broker_setup
broker_node=max-wgs
broker_dir=~/fullchain_tests
+ssh ${broker_node} mkdir -p ${broker_dir}/logs
cat broker.json |
jq "to_entries |
map(if .key == \"MonitorDbAddress\"
@@ -82,11 +85,11 @@ ssh ${monitor_node} influx -execute \"create database db_test\"
ssh ${broker_node} docker run -d -p 27017:27017 --name mongo mongo
#receiver_start
-ssh ${receiver_node} "bash -c 'cd ${receiver_dir}; nohup ./receiver settings.json &> ${receiver_dir}/receiver.log &'"
+ssh ${receiver_node} "bash -c 'cd ${receiver_dir}; nohup ./receiver settings.json &> ${log_dir}/log.receiver &'"
sleep 0.3
#broker_start
-ssh ${broker_node} "bash -c 'cd ${broker_dir}; nohup ./hidra2-broker -config broker.json &> ${broker_dir}/broker.log &'"
+ssh ${broker_node} "bash -c 'cd ${broker_dir}; nohup ./hidra2-broker -config broker.json &> ${log_dir}/log.broker &'"
sleep 0.3
#producer_start
diff --git a/tests/manual/performance_producer_receiver/receiver.json b/tests/manual/performance_producer_receiver/receiver.json
index 2479a88d5..dcb387955 100644
--- a/tests/manual/performance_producer_receiver/receiver.json
+++ b/tests/manual/performance_producer_receiver/receiver.json
@@ -5,5 +5,6 @@
"BrokerDbName": "test_run",
"ListenPort":4200,
"WriteToDisk":true,
- "WriteToDb":true
+ "WriteToDb":true,
+ "LogLevel":"info"
}
--
GitLab