diff --git a/CMakeLists.txt b/CMakeLists.txt
index d4e6c35a31a5615fd037d2fa899c98fdbf564709..e50c24502162da1734ccc05caa5da3eae86716a7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,6 +29,11 @@ set (HIDRA2_CXX_COMMON_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/common/cpp/includ
find_package (Threads)
+set (CMAKE_PREFIX_PATH "${LIBCURL_DIR}")
+find_package (CURL REQUIRED)
+message (STATUS "Found cURL libraries: ${CURL_LIBRARIES}")
+message (STATUS "cURL include: ${CURL_INCLUDE_DIRS}")
+
# format sources
include(astyle)
diff --git a/broker/src/hidra2_broker/database/database_test.go b/broker/src/hidra2_broker/database/database_test.go
index 207b40683c2a6575b546a468a4b698d6f7071d9a..3073ed61b1a621ae47e3a41903e75f2a85836255 100644
--- a/broker/src/hidra2_broker/database/database_test.go
+++ b/broker/src/hidra2_broker/database/database_test.go
@@ -12,7 +12,10 @@ func TestMockDataBase(t *testing.T) {
db.On("Close").Return()
db.On("Copy").Return(nil)
db.On("GetNextRecord", "").Return([]byte(""), nil)
-
db.Connect("")
db.GetNextRecord("")
+ db.Close()
+ db.Copy()
+ var err DBError
+ err.Error()
}
diff --git a/broker/src/hidra2_broker/database/mock_database.go b/broker/src/hidra2_broker/database/mock_database.go
index fbeb4582c4e4304979a1d05f7ee70b7dd2a04ce8..cf25a1eea71a73fa8973027b8a70cfc574643fda 100644
--- a/broker/src/hidra2_broker/database/mock_database.go
+++ b/broker/src/hidra2_broker/database/mock_database.go
@@ -19,7 +19,7 @@ func (db *MockedDatabase) Close() {
db.Called()
}
-func (db *MockedDatabase) Copy() Agent{
+func (db *MockedDatabase) Copy() Agent {
db.Called()
return db
}
diff --git a/broker/src/hidra2_broker/server/get_next.go b/broker/src/hidra2_broker/server/get_next.go
index ae29d355f5d7f1181d867b32cc811df9df7910cd..672a398987e0a85d40f93d98537beb0e68d77263 100644
--- a/broker/src/hidra2_broker/server/get_next.go
+++ b/broker/src/hidra2_broker/server/get_next.go
@@ -15,8 +15,8 @@ func extractRequestParameters(r *http.Request) (string, bool) {
func routeGetNext(w http.ResponseWriter, r *http.Request) {
r.Header.Set("Content-type", "application/json")
- // w.Write([]byte("Hello"))
- // return
+ // w.Write([]byte("Hello"))
+ // return
db_name, ok := extractRequestParameters(r)
if !ok {
w.WriteHeader(http.StatusBadRequest)
@@ -31,6 +31,7 @@ func routeGetNext(w http.ResponseWriter, r *http.Request) {
func getNextRecord(db_name string) (answer []byte, code int) {
db_new := db.Copy()
defer db_new.Close()
+ statistics.IncreaseCounter()
answer, err := db_new.GetNextRecord(db_name)
if err != nil {
err_db, ok := err.(*database.DBError)
diff --git a/broker/src/hidra2_broker/server/get_next_test.go b/broker/src/hidra2_broker/server/get_next_test.go
index 11385f121b5412533e9d30ff6c87357ca211fe07..36f5cad439a721ed9f94d552752f3605b6d876be 100644
--- a/broker/src/hidra2_broker/server/get_next_test.go
+++ b/broker/src/hidra2_broker/server/get_next_test.go
@@ -3,6 +3,7 @@ package server
import (
"errors"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/suite"
"hidra2_broker/database"
"hidra2_broker/utils"
"net/http"
@@ -30,45 +31,57 @@ func TestGetNextWithoutDatabaseName(t *testing.T) {
assert.Equal(t, http.StatusNotFound, w.Code, "no database name")
}
-func ExpectCopyClose(mock_db *database.MockedDatabase){
+func ExpectCopyClose(mock_db *database.MockedDatabase) {
mock_db.On("Copy").Return(mock_db)
mock_db.On("Close").Return()
}
-func TestGetNextWithWrongDatabaseName(t *testing.T) {
- mock_db := new(database.MockedDatabase)
- db = mock_db
- defer func() { db = nil }()
- ExpectCopyClose(mock_db)
- mock_db.On("GetNextRecord", "foo").Return([]byte(""),
+type GetNextTestSuite struct {
+ suite.Suite
+ mock_db *database.MockedDatabase
+}
+
+func (suite *GetNextTestSuite) SetupTest() {
+ statistics.Reset()
+ suite.mock_db = new(database.MockedDatabase)
+ db = suite.mock_db
+ ExpectCopyClose(suite.mock_db)
+}
+
+func (suite *GetNextTestSuite) TearDownTest() {
+ assertExpectations(suite.T(), suite.mock_db)
+ db = nil
+}
+
+func TestGetNextTestSuite(t *testing.T) {
+ suite.Run(t, new(GetNextTestSuite))
+}
+
+func (suite *GetNextTestSuite) TestGetNextWithWrongDatabaseName() {
+ suite.mock_db.On("GetNextRecord", "foo").Return([]byte(""),
&database.DBError{utils.StatusWrongInput, ""})
w := doRequest("/database/foo/next")
- assert.Equal(t, http.StatusBadRequest, w.Code, "wrong database name")
- assertExpectations(t, mock_db)
+ suite.Equal(http.StatusBadRequest, w.Code, "wrong database name")
}
-func TestGetNextWithInternalDBError(t *testing.T) {
- mock_db := new(database.MockedDatabase)
- db = mock_db
- defer func() { db = nil }()
- ExpectCopyClose(mock_db)
- mock_db.On("GetNextRecord", "foo").Return([]byte(""), errors.New(""))
-
+func (suite *GetNextTestSuite) TestGetNextWithInternalDBError() {
+ suite.mock_db.On("GetNextRecord", "foo").Return([]byte(""), errors.New(""))
w := doRequest("/database/foo/next")
- assert.Equal(t, http.StatusInternalServerError, w.Code, "internal error")
- assertExpectations(t, mock_db)
+ suite.Equal(http.StatusInternalServerError, w.Code, "internal error")
}
-func TestGetNextWithGoodDatabaseName(t *testing.T) {
- mock_db := new(database.MockedDatabase)
- db = mock_db
- defer func() { db = nil }()
- ExpectCopyClose(mock_db)
- mock_db.On("GetNextRecord", "dbname").Return([]byte("Hello"), nil)
+func (suite *GetNextTestSuite) TestGetNextWithGoodDatabaseName() {
+ suite.mock_db.On("GetNextRecord", "dbname").Return([]byte("Hello"), nil)
w := doRequest("/database/dbname/next")
- assert.Equal(t, http.StatusOK, w.Code, "GetNext OK")
- assert.Equal(t, "Hello", string(w.Body.Bytes()), "GetNext sends data")
- assertExpectations(t, mock_db)
+ suite.Equal(http.StatusOK, w.Code, "GetNext OK")
+ suite.Equal("Hello", string(w.Body.Bytes()), "GetNext sends data")
+}
+
+func (suite *GetNextTestSuite) TestGetNextAddsCounter() {
+ suite.mock_db.On("GetNextRecord", "dbname").Return([]byte("Hello"), nil)
+
+ doRequest("/database/dbname/next")
+ suite.Equal(1, statistics.GetCounter(), "GetNext increases counter")
}
diff --git a/broker/src/hidra2_broker/server/server.go b/broker/src/hidra2_broker/server/server.go
index 7e85cbca759de0a7b43a76bd1c25af1aa12dd1d6..eadd2ba6b3cc0b202c92f3901ecec985f0d7ceb8 100644
--- a/broker/src/hidra2_broker/server/server.go
+++ b/broker/src/hidra2_broker/server/server.go
@@ -7,15 +7,18 @@ import (
var db database.Agent
type serverSettings struct {
- DbAddress string
- Port int
+ BrokerDbAddress string
+ MonitorDbAddress string
+ MonitorDbName string
+ Port int
}
var settings serverSettings
+var statistics serverStatistics
func InitDB(dbAgent database.Agent) error {
db = dbAgent
- return db.Connect(settings.DbAddress)
+ return db.Connect(settings.BrokerDbAddress)
}
func CleanupDB() {
diff --git a/broker/src/hidra2_broker/server/server_nottested.go b/broker/src/hidra2_broker/server/server_nottested.go
index cf6bd72f72fff462af1860df2d8bcec5587ed83f..35ed53f930526c6be8c30f32204db8a8cc0d23ea 100644
--- a/broker/src/hidra2_broker/server/server_nottested.go
+++ b/broker/src/hidra2_broker/server/server_nottested.go
@@ -3,17 +3,45 @@
package server
import (
+ "errors"
"hidra2_broker/utils"
"log"
"net/http"
"strconv"
)
+func StartStatistics() {
+ statistics.Writer = new(StatisticInfluxDbWriter)
+ statistics.Reset()
+ go statistics.Monitor()
+}
+
func Start() {
+ StartStatistics()
mux := utils.NewRouter(listRoutes)
- log.Fatal(http.ListenAndServe("localhost:"+strconv.Itoa(settings.Port), http.HandlerFunc(mux.ServeHTTP)))
+ log.Fatal(http.ListenAndServe(":"+strconv.Itoa(settings.Port), http.HandlerFunc(mux.ServeHTTP)))
}
func ReadConfig(fname string) error {
- return utils.ReadJsonFromFile(fname, &settings)
+ if err := utils.ReadJsonFromFile(fname, &settings); err != nil {
+ return err
+ }
+
+ if settings.BrokerDbAddress == "" {
+ return errors.New("BrokerDbAddress not set")
+ }
+
+ if settings.MonitorDbAddress == "" {
+ return errors.New("MonitorDbAddress not set")
+ }
+
+ if settings.Port == 0 {
+ return errors.New("Server port not set")
+ }
+
+ if settings.MonitorDbName == "" {
+ return errors.New("MonitorDbName not set")
+ }
+
+ return nil
}
diff --git a/broker/src/hidra2_broker/server/server_test.go b/broker/src/hidra2_broker/server/server_test.go
index 9d6d3d73074f68c138b54fe02b3d3339b9b41363..baba3ed60dbd538d5b1fa85b01c2490fa1d62e13 100644
--- a/broker/src/hidra2_broker/server/server_test.go
+++ b/broker/src/hidra2_broker/server/server_test.go
@@ -34,7 +34,7 @@ func TestInitDBWithWrongAddress(t *testing.T) {
mock_db.ExpectedCalls = nil
- settings.DbAddress = "0.0.0.0:0000"
+ settings.BrokerDbAddress = "0.0.0.0:0000"
for _, test := range initDBTests {
mock_db.On("Connect", "0.0.0.0:0000").Return(test.answer)
diff --git a/broker/src/hidra2_broker/server/statistics.go b/broker/src/hidra2_broker/server/statistics.go
new file mode 100644
index 0000000000000000000000000000000000000000..f0851ae33f58070035d51fc3588f99843718043d
--- /dev/null
+++ b/broker/src/hidra2_broker/server/statistics.go
@@ -0,0 +1,55 @@
+package server
+
+import (
+ "fmt"
+ "log"
+ "time"
+ "sync"
+)
+
+type statisticsWriter interface {
+ Write(*serverStatistics) error
+}
+
+type serverStatistics struct {
+ counter int
+ mux sync.Mutex
+ Writer statisticsWriter
+}
+
+func (st *serverStatistics) IncreaseCounter() {
+ st.mux.Lock()
+ defer st.mux.Unlock()
+ st.counter++
+}
+
+func (st *serverStatistics) GetCounter() int {
+ st.mux.Lock()
+ defer st.mux.Unlock()
+ return st.counter
+}
+
+func (st *serverStatistics) Reset() {
+ st.mux.Lock()
+ defer st.mux.Unlock()
+ st.counter = 0
+}
+
+func (st *serverStatistics) WriteStatistic() (err error) {
+ defer func() {
+ if p := recover(); p != nil {
+ err = fmt.Errorf("WriteStatistic error: %v", p)
+ }
+ }()
+ return st.Writer.Write(st)
+}
+
+func (st *serverStatistics) Monitor() {
+ for {
+ time.Sleep(1000 * time.Millisecond)
+ if err := st.WriteStatistic(); err != nil {
+ log.Println(err.Error())
+ }
+ st.Reset()
+ }
+}
diff --git a/broker/src/hidra2_broker/server/statistics_test.go b/broker/src/hidra2_broker/server/statistics_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..a16a8be4a10fccdfb1b60ce8b81c6b8e1dc97d85
--- /dev/null
+++ b/broker/src/hidra2_broker/server/statistics_test.go
@@ -0,0 +1,41 @@
+package server
+
+import (
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "testing"
+)
+
+type mockWriter struct {
+ mock.Mock
+}
+
+func (writer *mockWriter) Write(statistics *serverStatistics) error {
+ args := writer.Called(statistics)
+ return args.Error(0)
+}
+
+func assertMockWriterExpectations(t *testing.T, mock_writer *mockWriter) {
+ mock_writer.AssertExpectations(t)
+ mock_writer.ExpectedCalls = nil
+}
+
+func TestMonitorOK(t *testing.T) {
+ mock_writer := new(mockWriter)
+ statistics.Writer = mock_writer
+ statistics.Reset()
+ statistics.IncreaseCounter()
+
+ mock_writer.On("Write", &statistics).Return(nil)
+
+ err := statistics.WriteStatistic()
+ assert.Nil(t, err, "Statistics written")
+
+ assertMockWriterExpectations(t, mock_writer)
+}
+
+func TestMonitorCatchesError(t *testing.T) {
+ statistics.Writer = nil
+ err := statistics.WriteStatistic()
+ assert.NotNil(t, err, "Error with nil pointer")
+}
diff --git a/broker/src/hidra2_broker/server/statistics_writers.go b/broker/src/hidra2_broker/server/statistics_writers.go
new file mode 100644
index 0000000000000000000000000000000000000000..db6e46fb293895c77fdfe9599d635f8f4f52ace9
--- /dev/null
+++ b/broker/src/hidra2_broker/server/statistics_writers.go
@@ -0,0 +1,46 @@
+//+build !test
+
+package server
+
+import (
+ "github.com/influxdata/influxdb/client/v2"
+ "log"
+ "time"
+)
+
+type StatisticLogWriter struct {
+}
+
+func (writer *StatisticLogWriter) Write(statistics *serverStatistics) error {
+ log.Println(statistics.GetCounter())
+ return nil
+}
+
+type StatisticInfluxDbWriter struct {
+}
+
+func (writer *StatisticInfluxDbWriter) Write(statistics *serverStatistics) error {
+ c, err := client.NewHTTPClient(client.HTTPConfig{
+ Addr: "http://"+ settings.MonitorDbAddress,
+ })
+ if err != nil {
+ return err
+ }
+ defer c.Close()
+
+ bp, _ := client.NewBatchPoints(client.BatchPointsConfig{
+ Database: settings.MonitorDbName,
+ })
+
+ tags := map[string]string{"Group ID": "0"}
+ fields := map[string]interface{}{
+ "rate": statistics.GetCounter(),
+ }
+ pt, err := client.NewPoint("RequestsRate", tags, fields, time.Now())
+ if err != nil {
+ return err
+ }
+ bp.AddPoint(pt)
+
+ return c.Write(bp)
+}
diff --git a/broker/src/hidra2_broker/utils/helpers.go b/broker/src/hidra2_broker/utils/helpers.go
index 02bb46796c257c6f8f784e25d8a47ee202e84e57..f1d0da4dc985d0231cbe34602f1942ce01fde362 100644
--- a/broker/src/hidra2_broker/utils/helpers.go
+++ b/broker/src/hidra2_broker/utils/helpers.go
@@ -33,5 +33,6 @@ func ReadJsonFromFile(fname string, config interface{}) error {
if err != nil {
return err
}
+
return nil
}
diff --git a/common/cpp/CMakeLists.txt b/common/cpp/CMakeLists.txt
index 2e79b345e35a1a91c74e90320c463f8f66b3800d..981bb1f4484a0033056be6f49ccdde39cf9796e1 100644
--- a/common/cpp/CMakeLists.txt
+++ b/common/cpp/CMakeLists.txt
@@ -6,6 +6,8 @@ add_subdirectory(src/json_parser)
add_subdirectory(src/data_structs)
+add_subdirectory(src/http_client)
+
if(BUILD_MONGODB_CLIENTLIB)
add_subdirectory(src/database)
endif()
diff --git a/worker/api/cpp/src/curl_http_client.h b/common/cpp/include/http_client/curl_http_client.h
similarity index 61%
rename from worker/api/cpp/src/curl_http_client.h
rename to common/cpp/include/http_client/curl_http_client.h
index 135a6cd5c5b38af9e3d54421538a2197ee9b0889..9b65491aac43110c1e4de6bed792c9bf77ba7ab6 100644
--- a/worker/api/cpp/src/curl_http_client.h
+++ b/common/cpp/include/http_client/curl_http_client.h
@@ -13,8 +13,12 @@ class CurlHttpClient final : public HttpClient {
public:
CurlHttpClient();
std::string Get(const std::string& uri, HttpCode* response_code, Error* err) const noexcept override;
+ std::string Post(const std::string& uri, const std::string& data, HttpCode* response_code,
+ Error* err) const noexcept override;
virtual ~CurlHttpClient();
private:
+ std::string Command(bool post, const std::string& uri, const std::string& data, HttpCode* response_code,
+ Error* err) const noexcept;
mutable std::mutex mutex_;
CURL* curl_ = 0;
};
diff --git a/worker/api/cpp/src/http_client.h b/common/cpp/include/http_client/http_client.h
similarity index 92%
rename from worker/api/cpp/src/http_client.h
rename to common/cpp/include/http_client/http_client.h
index 31b838a62924f70fe83aecd5e70b4654421671aa..92a9299617bc6bea7b2979bc5d04865a44fc35bc 100644
--- a/worker/api/cpp/src/http_client.h
+++ b/common/cpp/include/http_client/http_client.h
@@ -1,7 +1,7 @@
#ifndef HIDRA2_HTTP_CLIENT_H
#define HIDRA2_HTTP_CLIENT_H
-#include <hidra2_worker.h>
+#include "common/error.h"
namespace hidra2 {
@@ -10,12 +10,12 @@ enum class HttpCode;
class HttpClient {
public:
virtual std::string Get(const std::string& uri, HttpCode* response_code, Error* err) const noexcept = 0;
+ virtual std::string Post(const std::string& uri, const std::string& data, HttpCode* response_code,
+ Error* err) const noexcept = 0;
virtual ~HttpClient() = default;
};
-Error HttpCodeToWorkerError(const HttpCode& code);
-
enum class HttpCode : int {
Continue = 100,
SwitchingProtocols = 101,
diff --git a/worker/api/cpp/src/http_error.h b/common/cpp/include/http_client/http_error.h
similarity index 100%
rename from worker/api/cpp/src/http_error.h
rename to common/cpp/include/http_client/http_error.h
diff --git a/common/cpp/include/json_parser/json_parser.h b/common/cpp/include/json_parser/json_parser.h
index 7373e91a74bf5043487846b675be1ce29fc62363..231df49fd713664d6fd306195ff78641ed269fea 100644
--- a/common/cpp/include/json_parser/json_parser.h
+++ b/common/cpp/include/json_parser/json_parser.h
@@ -7,6 +7,7 @@
#include <string>
#include "common/error.h"
+#include "io/io.h"
namespace hidra2 {
@@ -14,20 +15,37 @@ class RapidJson;
class JsonParser {
public:
- JsonParser(const std::string& json, bool read_from_file);
- ~JsonParser();
- JsonParser(JsonParser&& other);
Error GetUInt64(const std::string& name, uint64_t* val) const noexcept;
+ Error GetBool(const std::string& name, bool* val) const noexcept;
Error GetString(const std::string& name, std::string* val) const noexcept;
Error GetArrayUInt64(const std::string& name, std::vector<uint64_t>* val) const noexcept;
Error GetArrayString(const std::string& name, std::vector<std::string>* val) const noexcept;
JsonParser Embedded(const std::string& name) const noexcept;
+ ~JsonParser();
+ protected:
+ JsonParser(const std::string& json, const std::unique_ptr<IO>* io); // nullptr as second parameter will use default IO
+ JsonParser(const std::string& json);
+ JsonParser(JsonParser&& other);
private:
+ std::unique_ptr<IO> default_io_;
std::unique_ptr<RapidJson> rapid_json_;
JsonParser(RapidJson* rapid_json_);
};
+
+class JsonStringParser : public JsonParser {
+ public:
+ JsonStringParser(const std::string& json): JsonParser(json) {};
+};
+
+
+class JsonFileParser : public JsonParser {
+ public:
+ JsonFileParser(const std::string& json, const std::unique_ptr<IO>* io = nullptr): JsonParser(json, io) {};
+};
+
+
}
diff --git a/common/cpp/include/unittests/MockHttpClient.h b/common/cpp/include/unittests/MockHttpClient.h
new file mode 100644
index 0000000000000000000000000000000000000000..a497c396e4a03ab2b52f26a39f701dce2c14c9fd
--- /dev/null
+++ b/common/cpp/include/unittests/MockHttpClient.h
@@ -0,0 +1,35 @@
+#ifndef HIDRA2_MOCKHTTPCLIENT_H
+#define HIDRA2_MOCKHTTPCLIENT_H
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
+#include "http_client/http_client.h"
+
+namespace hidra2 {
+
+class MockHttpClient : public HttpClient {
+ public:
+ std::string Get(const std::string& uri, HttpCode* code, Error* err) const noexcept override {
+ ErrorInterface* error = nullptr;
+ auto responce = Get_t(uri, code, &error);
+ err->reset(error);
+ return responce;
+ }
+ std::string Post(const std::string& uri, const std::string& data, HttpCode* code, Error* err) const noexcept override {
+ ErrorInterface* error = nullptr;
+ auto responce = Post_t(uri, data, code, &error);
+ err->reset(error);
+ return responce;
+ }
+ MOCK_CONST_METHOD3(Get_t,
+ std::string(const std::string& uri, HttpCode* code, ErrorInterface** err));
+ MOCK_CONST_METHOD4(Post_t,
+ std::string(const std::string& uri, const std::string& data, HttpCode* code, ErrorInterface** err));
+
+};
+
+
+}
+
+#endif //HIDRA2_MOCKHTTPCLIENT_H
diff --git a/common/cpp/src/data_structs/data_structs.cpp b/common/cpp/src/data_structs/data_structs.cpp
index 331a741a3ef2380e662901b2f95b77a961be8ee6..5887f4e6994e66f0356f2464c99fcd9798a8b235 100644
--- a/common/cpp/src/data_structs/data_structs.cpp
+++ b/common/cpp/src/data_structs/data_structs.cpp
@@ -16,7 +16,7 @@ std::string FileInfo::Json() const {
}
-bool TimeFromJson(const JsonParser& parser, const std::string name, std::chrono::system_clock::time_point* val) {
+bool TimeFromJson(const JsonStringParser& parser, const std::string name, std::chrono::system_clock::time_point* val) {
uint64_t nanoseconds_from_epoch;
if (parser.GetUInt64(name, &nanoseconds_from_epoch)) {
return false;
@@ -34,7 +34,7 @@ bool TimeFromJson(const JsonParser& parser, const std::string name, std::chrono:
bool FileInfo::SetFromJson(const std::string& json_string) {
auto old = *this;
- JsonParser parser(json_string, false);
+ JsonStringParser parser(json_string);
if (parser.GetUInt64("_id", &id) ||
parser.GetUInt64("size", &size) ||
diff --git a/common/cpp/src/http_client/CMakeLists.txt b/common/cpp/src/http_client/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b274c9319e20e484c12d9f22f8668fda3f6a39f9
--- /dev/null
+++ b/common/cpp/src/http_client/CMakeLists.txt
@@ -0,0 +1,16 @@
+set(TARGET_NAME curl_http_client)
+set(SOURCE_FILES
+ curl_http_client.cpp
+ )
+
+
+################################
+# Library
+################################
+add_library(${TARGET_NAME} OBJECT ${SOURCE_FILES})
+IF(WIN32)
+ target_compile_definitions(${TARGET_NAME} PUBLIC -DCURL_STATICLIB)
+ENDIF()
+target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR} ${CURL_INCLUDE_DIRS})
+
+
diff --git a/worker/api/cpp/src/curl_http_client.cpp b/common/cpp/src/http_client/curl_http_client.cpp
similarity index 69%
rename from worker/api/cpp/src/curl_http_client.cpp
rename to common/cpp/src/http_client/curl_http_client.cpp
index 364b87cd1975a90de5ec53cc3eefaeb8e126265c..a6c98fa136c99d1d530c54f2cb7ece2b0e79bce6 100644
--- a/worker/api/cpp/src/curl_http_client.cpp
+++ b/common/cpp/src/http_client/curl_http_client.cpp
@@ -1,4 +1,4 @@
-#include "curl_http_client.h"
+#include "http_client/curl_http_client.h"
#include <cstring>
@@ -25,13 +25,20 @@ size_t curl_write( void* ptr, size_t size, size_t nmemb, void* buffer) {
return size * nmemb;
}
-void SetCurlOptions(CURL* curl, const std::string& uri, char* errbuf, std::string* buffer) {
+void SetCurlOptions(CURL* curl, bool post, const std::string& data, const std::string& uri, char* errbuf,
+ std::string* buffer) {
errbuf[0] = 0;
curl_easy_setopt(curl, CURLOPT_URL, uri.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, buffer);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 0L);
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf);
+ //todo use a config parameter for this
+ curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 5000L);
+
+ if (post) {
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str());
+ }
}
@@ -56,24 +63,38 @@ Error ProcessCurlResponse(CURL* curl, CURLcode res, const char* errbuf,
return nullptr;
} else {
*buffer = GetCurlError(curl, res, errbuf);
- return TextError(std::string(WorkerErrorMessage::kErrorReadingSource) + ": " + *buffer);
+ return TextError("Curl client error: " + *buffer);
}
}
-std::string CurlHttpClient::Get(const std::string& uri, HttpCode* response_code, Error* err) const noexcept {
+std::string CurlHttpClient::Command(bool post, const std::string& uri, const std::string& data, HttpCode* response_code,
+ Error* err) const noexcept {
std::lock_guard<std::mutex> lock{mutex_};
std::string buffer;
char errbuf[CURL_ERROR_SIZE];
- SetCurlOptions(curl_, uri, errbuf, &buffer);
+
+ SetCurlOptions(curl_, post, data, uri, errbuf, &buffer);
auto res = curl_easy_perform(curl_);
*err = ProcessCurlResponse(curl_, res, errbuf, &buffer, response_code);
return buffer;
+
+}
+
+
+std::string CurlHttpClient::Get(const std::string& uri, HttpCode* response_code, Error* err) const noexcept {
+ return Command(false, uri, "", response_code, err);
}
+std::string CurlHttpClient::Post(const std::string& uri, const std::string& data, HttpCode* response_code,
+ Error* err) const noexcept {
+ return Command(true, uri, data, response_code, err);
+}
+
+
CurlHttpClient::CurlHttpClient() {
curl_ = curl_easy_init();
if (!curl_) {
diff --git a/common/cpp/src/json_parser/json_parser.cpp b/common/cpp/src/json_parser/json_parser.cpp
index 1a848557881de69cf1fbcab77fcc2997e6f28d00..03de7dd24d25f90f9ac85585dee3974f5d536fd5 100644
--- a/common/cpp/src/json_parser/json_parser.cpp
+++ b/common/cpp/src/json_parser/json_parser.cpp
@@ -1,14 +1,18 @@
#include "json_parser/json_parser.h"
#include "rapid_json.h"
+#include "io/io_factory.h"
namespace hidra2 {
-
JsonParser::~JsonParser() {
}
-JsonParser::JsonParser(const std::string& json, bool read_from_file) : rapid_json_{new RapidJson(json, read_from_file)} {
+JsonParser::JsonParser(const std::string& json, const std::unique_ptr<IO>* io ) :
+ default_io_{GenerateDefaultIO()}, rapid_json_{new RapidJson(json, io != nullptr ? io : & default_io_)} {
+}
+
+JsonParser::JsonParser(const std::string& json) : rapid_json_{new RapidJson(json, nullptr)} {
}
Error JsonParser::GetArrayUInt64(const std::string& name, std::vector<uint64_t>* val) const noexcept {
@@ -21,6 +25,11 @@ Error JsonParser::GetArrayString(const std::string& name, std::vector<std::strin
}
+Error JsonParser::GetBool(const std::string& name, bool* val) const noexcept {
+ return rapid_json_->GetBool(name, val);
+}
+
+
Error JsonParser::GetUInt64(const std::string& name, uint64_t* val) const noexcept {
return rapid_json_->GetUInt64(name, val);
}
@@ -30,7 +39,7 @@ Error JsonParser::GetString(const std::string& name, std::string* val) const noe
}
JsonParser JsonParser::Embedded(const std::string& name) const noexcept {
- RapidJson* rapid_json = new RapidJson(*rapid_json_.get(), name);
+ RapidJson* rapid_json = new RapidJson(*rapid_json_.get(), name) ;
return JsonParser(rapid_json);
}
diff --git a/common/cpp/src/json_parser/rapid_json.cpp b/common/cpp/src/json_parser/rapid_json.cpp
index 37d1c8771aa04d6f4150df97c229157ce6c10916..e63474f99e6c92dc39e124f6b32b2f593650312d 100644
--- a/common/cpp/src/json_parser/rapid_json.cpp
+++ b/common/cpp/src/json_parser/rapid_json.cpp
@@ -3,12 +3,9 @@
using namespace rapidjson;
-#include "io/io_factory.h"
-
namespace hidra2 {
-RapidJson::RapidJson(const std::string& json, bool read_from_file): io__{GenerateDefaultIO()}, json_{json},
- read_from_file_{read_from_file} {
+RapidJson::RapidJson(const std::string& json, const std::unique_ptr<IO>* io): io__{io}, json_{json} {
}
@@ -21,9 +18,9 @@ Error RapidJson::LazyInitialize()const noexcept {
return nullptr;
auto str = json_;
- if (read_from_file_) {
+ if (io__) {
Error err;
- str = io__->ReadFileToString(json_, &err);
+ str = (*io__)->ReadFileToString(json_, &err);
if (err != nullptr) {
return err;
}
@@ -34,24 +31,28 @@ Error RapidJson::LazyInitialize()const noexcept {
}
object_ = doc_.GetObject();
-
+ object_p_ = &object_;
+ initialized_ = true;
return nullptr;
}
-hidra2::Error CheckValueType(const std::string& name, ValueType type, const Value& val) {
+hidra2::Error CheckValueType(const std::string& name, ValueType type, const Value* val) {
bool res = false;
switch (type) {
case ValueType::kObject:
- res = val.IsObject();
+ res = val->IsObject();
break;
case ValueType::kString:
- res = val.IsString();
+ res = val->IsString();
break;
case ValueType::kUint64:
- res = val.IsInt64();
+ res = val->IsInt64();
+ break;
+ case ValueType::kBool:
+ res = val->IsBool();
break;
case ValueType::kArray:
- res = val.IsArray();
+ res = val->IsArray();
break;
}
if (!res) {
@@ -62,46 +63,56 @@ hidra2::Error CheckValueType(const std::string& name, ValueType type, const Valu
}
-hidra2::Error RapidJson::GetValue(const std::string& name, ValueType type, Value* val)const noexcept {
+hidra2::Error RapidJson::GetValuePointer(const std::string& name, ValueType type, Value** val)const noexcept {
if (Error err = LazyInitialize()) {
return err;
}
- auto iterator = object_.FindMember(name.c_str());
- if (iterator == object_.MemberEnd()) {
+ auto iterator = object_p_->FindMember(name.c_str());
+ if (iterator == object_p_->MemberEnd()) {
return TextError("cannot find: " + name);
}
- *val = iterator->value;
+ *val = &iterator->value;
return CheckValueType(name, type, *val);
}
Error RapidJson::GetUInt64(const std::string& name, uint64_t* val) const noexcept {
- Value json_val;
- if (Error err = GetValue(name, ValueType::kUint64, &json_val)) {
+ Value* json_val;
+ if (Error err = GetValuePointer(name, ValueType::kUint64, &json_val)) {
+ return err;
+ }
+ *val = json_val->GetInt64();
+ return nullptr;
+}
+
+Error RapidJson::GetBool(const std::string& name, bool* val) const noexcept {
+ Value* json_val;
+ if (Error err = GetValuePointer(name, ValueType::kBool, &json_val)) {
return err;
}
- *val = json_val.GetInt64();
+ *val = json_val->GetBool();
return nullptr;
}
Error RapidJson::GetString(const std::string& name, std::string* val) const noexcept {
- Value json_val;
- if (Error err = GetValue(name, ValueType::kString, &json_val)) {
+ Value* json_val;
+ if (Error err = GetValuePointer(name, ValueType::kString, &json_val)) {
return err;
}
- *val = json_val.GetString();
+ *val = json_val->GetString();
return nullptr;
}
Error RapidJson::GetArrayUInt64(const std::string& name, std::vector<uint64_t>* val) const noexcept {
- Value json_val;
- if (Error err = GetValue(name, ValueType::kArray, &json_val)) {
+ Value* json_val;
+ if (Error err = GetValuePointer(name, ValueType::kArray, &json_val)) {
return err;
}
- for (auto& v : json_val.GetArray()) {
+ val->clear();
+ for (auto& v : json_val->GetArray()) {
if (!v.IsInt64()) {
return TextError("wrong type of array element: " + name);
}
@@ -112,12 +123,13 @@ Error RapidJson::GetArrayUInt64(const std::string& name, std::vector<uint64_t>*
}
Error RapidJson::GetArrayString(const std::string& name, std::vector<std::string>* val) const noexcept {
- Value json_val;
- if (Error err = GetValue(name, ValueType::kArray, &json_val)) {
+ Value* json_val;
+ if (Error err = GetValuePointer(name, ValueType::kArray, &json_val)) {
return err;
}
- for (auto& v : json_val.GetArray()) {
+ val->clear();
+ for (auto& v : json_val->GetArray()) {
if (!v.IsString()) {
return TextError("wrong type of array element: " + name);
}
@@ -127,9 +139,8 @@ Error RapidJson::GetArrayString(const std::string& name, std::vector<std::string
}
-
RapidJson::RapidJson(const RapidJson& parent, const std::string& subname) {
- auto err = parent.GetValue(subname, ValueType::kObject, &object_);
+ auto err = parent.GetValuePointer(subname, ValueType::kObject, &object_p_);
if (err) {
embedded_error_ = std::move(err);
return;
diff --git a/common/cpp/src/json_parser/rapid_json.h b/common/cpp/src/json_parser/rapid_json.h
index ef26611e4abcafc3137fd18e7eef18ab07a717ee..f0369ba1430d1c9b8afff0ff8e95df5e39999e88 100644
--- a/common/cpp/src/json_parser/rapid_json.h
+++ b/common/cpp/src/json_parser/rapid_json.h
@@ -12,28 +12,30 @@ enum class ValueType {
kUint64,
kString,
kObject,
- kArray
+ kArray,
+ kBool
};
class RapidJson {
public:
- RapidJson(const std::string& json, bool read_from_file);
+ RapidJson(const std::string& json, const std::unique_ptr<IO>* io);
RapidJson(const RapidJson& parent, const std::string& subname);
Error GetUInt64(const std::string& name, uint64_t* val) const noexcept;
+ Error GetBool(const std::string& name, bool* val) const noexcept;
Error GetString(const std::string& name, std::string* val) const noexcept;
Error GetArrayUInt64(const std::string& name, std::vector<uint64_t>* val) const noexcept;
Error GetArrayString(const std::string& name, std::vector<std::string>* val) const noexcept;
- std::unique_ptr<IO> io__;
private:
+ const std::unique_ptr<IO>* io__;
mutable rapidjson::Document doc_;
mutable rapidjson::Value object_;
+ mutable rapidjson::Value* object_p_;
std::string json_;
- bool read_from_file_;
mutable bool initialized_ = false;
Error LazyInitialize() const noexcept;
Error embedded_error_ = nullptr;
- hidra2::Error GetValue(const std::string& name, ValueType type, rapidjson::Value* val)const noexcept;
+ hidra2::Error GetValuePointer(const std::string& name, ValueType type, rapidjson::Value** val)const noexcept;
};
}
diff --git a/common/cpp/src/system_io/system_io.cpp b/common/cpp/src/system_io/system_io.cpp
index 70c4a93ddfbbe4fd6c6872491370d2539e651ff5..1315a8eb613e98f34a5a7e1ff27acabe33474327 100644
--- a/common/cpp/src/system_io/system_io.cpp
+++ b/common/cpp/src/system_io/system_io.cpp
@@ -23,7 +23,7 @@
namespace hidra2 {
-const int SystemIO::kNetBufferSize = 1024 * 1024 ; //MiByte
+const int SystemIO::kNetBufferSize = 1024 * 1024; //* 1024 ; //MiByte
/*******************************************************************************
* system_io.cpp *
diff --git a/common/cpp/src/system_io/system_io_linux.cpp b/common/cpp/src/system_io/system_io_linux.cpp
index e0faf78b744ddc6becc4e556d2d575ec9e6fdcf6..c84e2f1e93e49dbc45fa632cef0c86216b90cbce 100644
--- a/common/cpp/src/system_io/system_io_linux.cpp
+++ b/common/cpp/src/system_io/system_io_linux.cpp
@@ -176,7 +176,6 @@ void SystemIO::CollectFileInformationRecursively(const std::string& path,
void SystemIO::ApplyNetworkOptions(SocketDescriptor socket_fd, Error* err) const {
//TODO: Need to change network layer code, so everything can be NonBlocking
- // in use and one have to wait for some time until the system cleans up the stuff
int flag;
if (
/*(flags = fcntl(socket_fd, F_GETFL, 0)) == -1
diff --git a/common/cpp/unittests/json_parser/test_json_parser.cpp b/common/cpp/unittests/json_parser/test_json_parser.cpp
index 0767fd02065553e54cafbc86556183eb40702eec..90ef15799972e9e560e7a7022346cf8101f41ba8 100644
--- a/common/cpp/unittests/json_parser/test_json_parser.cpp
+++ b/common/cpp/unittests/json_parser/test_json_parser.cpp
@@ -20,7 +20,8 @@ using ::testing::SetArgPointee;
using ::testing::HasSubstr;
using ::testing::ElementsAre;
-using hidra2::JsonParser;
+using hidra2::JsonFileParser;
+using hidra2::JsonStringParser;
using hidra2::RapidJson;
using hidra2::MockIO;
using hidra2::IO;
@@ -29,30 +30,35 @@ using hidra2::IO;
namespace {
TEST(ParseString, SimpleConvertToJson) {
- std::string json = R"({"_id":2,"foo":"foo","bar":1})";
+ std::string json = R"({"_id":2,"foo":"foo","bar":1,"flag":true})";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
uint64_t id, bar;
std::string foo;
+ bool flag;
auto err1 = parser.GetUInt64("_id", &id);
auto err2 = parser.GetString("foo", &foo);
auto err3 = parser.GetUInt64("bar", &bar);
+ auto err4 = parser.GetBool("flag", &flag);
ASSERT_THAT(err1, Eq(nullptr));
ASSERT_THAT(err2, Eq(nullptr));
ASSERT_THAT(err3, Eq(nullptr));
+ ASSERT_THAT(err4, Eq(nullptr));
+
ASSERT_THAT(id, Eq(2));
ASSERT_THAT(foo, Eq("foo"));
ASSERT_THAT(bar, Eq(1));
+ ASSERT_THAT(flag, true);
}
TEST(ParseString, EmbeddedConvertToJson) {
std::string json = R"({"id":{"test":2}})";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
uint64_t id;
auto err1 = parser.Embedded("id").GetUInt64("test", &id);
@@ -64,7 +70,7 @@ TEST(ParseString, EmbeddedConvertToJson) {
TEST(ParseString, DoubleEmbeddedConvertToJson) {
std::string json = R"({"id":{"test":{"test2":2}}})";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
uint64_t id;
auto err1 = parser.Embedded("id").Embedded("test").GetUInt64("test2", &id);
@@ -76,7 +82,7 @@ TEST(ParseString, DoubleEmbeddedConvertToJson) {
TEST(ParseString, ErrorOnWrongEmbeddedKey) {
std::string json = R"({"id1":{"test":2}})";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
uint64_t id;
auto err = parser.Embedded("id").GetUInt64("test", &id);
@@ -88,7 +94,7 @@ TEST(ParseString, ErrorOnWrongEmbeddedKey) {
TEST(ParseString, ErrorOnWrongEmbeddedSubKey) {
std::string json = R"({"id1":{"test1":2}})";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
uint64_t id;
auto err = parser.Embedded("id").GetUInt64("test", &id);
@@ -100,7 +106,7 @@ TEST(ParseString, ErrorOnWrongEmbeddedSubKey) {
TEST(ParseString, ErrorOnWrongKey) {
std::string json = R"({"_id":"2"})";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
uint64_t id;
auto err = parser.GetUInt64("_id1", &id);
@@ -112,7 +118,7 @@ TEST(ParseString, ErrorOnWrongKey) {
TEST(ParseString, ErrorOnWrongType) {
std::string json = R"({"_id":"2"})";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
uint64_t id;
auto err = parser.GetUInt64("_id", &id);
@@ -125,7 +131,7 @@ TEST(ParseString, ErrorOnWrongType) {
TEST(ParseString, ErrorOnWrongDocument) {
std::string json = R"({"_id":2)";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
uint64_t id;
auto err = parser.GetUInt64("_id", &id);
@@ -139,7 +145,7 @@ TEST(ParseString, ErrorOnWrongDocument) {
TEST(ParseString, IntArrayConvertToJson) {
std::string json = R"({"array":[1,2,3]})";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
std::vector<uint64_t> vec;
auto err = parser.GetArrayUInt64("array", &vec);
@@ -148,10 +154,25 @@ TEST(ParseString, IntArrayConvertToJson) {
ASSERT_THAT(vec, ElementsAre(1, 2, 3));
}
+TEST(ParseString, IntArrayConvertToJsonTwice) {
+ std::string json = R"({"array":[1,2,3]})";
+
+ JsonStringParser parser{json};
+
+ std::vector<uint64_t> vec;
+ auto err = parser.GetArrayUInt64("array", &vec);
+ auto err2 = parser.GetArrayUInt64("array", &vec);
+
+ ASSERT_THAT(err, Eq(nullptr));
+ ASSERT_THAT(err2, Eq(nullptr));
+ ASSERT_THAT(vec, ElementsAre(1, 2, 3));
+}
+
+
TEST(ParseString, IntArrayErrorConvertToJson) {
std::string json = R"({"array":[1,2,"3"]})";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
std::vector<uint64_t> vec;
auto err = parser.GetArrayUInt64("array", &vec);
@@ -161,10 +182,11 @@ TEST(ParseString, IntArrayErrorConvertToJson) {
}
+
TEST(ParseString, StringArrayConvertToJson) {
std::string json = R"({"array":["s1","s2","s3"]})";
- JsonParser parser{json, false};
+ JsonStringParser parser{json};
std::vector<std::string> vec;
auto err = parser.GetArrayString("array", &vec);
@@ -175,13 +197,16 @@ TEST(ParseString, StringArrayConvertToJson) {
class ParseFileTests : public Test {
public:
- RapidJson parser{"filename", true};
NiceMock<MockIO> mock_io;
+ std::unique_ptr<IO> io_ptr = std::unique_ptr<IO> {
+ &mock_io
+ };
+ JsonFileParser parser{"filename", &io_ptr};
+
void SetUp() override {
- parser.io__ = std::unique_ptr<IO> {&mock_io};
}
void TearDown() override {
- parser.io__.release();
+ io_ptr.release();
}
};
@@ -195,6 +220,24 @@ TEST_F(ParseFileTests, CorrectConvertFileToJson) {
auto err = parser.GetUInt64("_id", &id);
ASSERT_THAT(id, Eq(2));
}
+
+TEST_F(ParseFileTests, InitializedOnlyOnce) {
+ std::string json = R"({"_id":2})";
+
+ EXPECT_CALL(mock_io, ReadFileToString_t("filename", _)).
+ WillOnce(DoAll(testing::SetArgPointee<1>(nullptr), testing::Return(json)));
+
+ uint64_t id, id2;
+ auto err1 = parser.GetUInt64("_id", &id);
+ auto err2 = parser.GetUInt64("_id", &id2);
+
+ ASSERT_THAT(err1, Eq(nullptr));
+ ASSERT_THAT(err2, Eq(nullptr));
+ ASSERT_THAT(id, Eq(2));
+ ASSERT_THAT(id2, Eq(2));
+}
+
+
TEST_F(ParseFileTests, CannotReadFile) {
std::string json = R"({"_id":2})";
diff --git a/examples/producer/dummy-data-producer/check_linux.sh b/examples/producer/dummy-data-producer/check_linux.sh
index 04f592decbf9d22d1301157df898e0f9c2b764fa..51a14177248525416ad5e995fcca02f755bddf75 100644
--- a/examples/producer/dummy-data-producer/check_linux.sh
+++ b/examples/producer/dummy-data-producer/check_linux.sh
@@ -6,5 +6,5 @@ database_name=test_run
#just test that it starts, no reciever is running
-$@ 0.0.0.0 1 1 2>&1 | grep "refused"
+$@ 0.0.0.0 1 1 2>&1 | grep "Failed to connect"
diff --git a/examples/producer/dummy-data-producer/dummy_data_producer.cpp b/examples/producer/dummy-data-producer/dummy_data_producer.cpp
index a9476d52422276874a68b692957d4d034c19191b..c7bae8553d6b740df188573d851196d2089ba107 100644
--- a/examples/producer/dummy-data-producer/dummy_data_producer.cpp
+++ b/examples/producer/dummy-data-producer/dummy_data_producer.cpp
@@ -28,7 +28,7 @@ bool SendDummyData(hidra2::Producer* producer, size_t number_of_byte, uint64_t i
auto buffer = std::unique_ptr<uint8_t>(new uint8_t[number_of_byte]);
for(uint64_t i = 0; i < iterations; i++) {
- std::cout << "Send file " << i + 1 << "/" << iterations << std::endl;
+// std::cerr << "Send file " << i + 1 << "/" << iterations << std::endl;
auto err = producer->Send(i, buffer.get(), number_of_byte);
@@ -36,7 +36,7 @@ bool SendDummyData(hidra2::Producer* producer, size_t number_of_byte, uint64_t i
std::cerr << "File was not successfully send: " << err << std::endl;
return false;
} else {
- std::cout << "File was successfully send." << std::endl;
+// std::cerr << "File was successfully send." << std::endl;
}
}
@@ -45,12 +45,12 @@ bool SendDummyData(hidra2::Producer* producer, size_t number_of_byte, uint64_t i
int main (int argc, char* argv[]) {
std::string receiver_address;
- size_t number_of_byte;
+ size_t number_of_kbytes;
uint64_t iterations;
- std::tie(receiver_address, number_of_byte, iterations) = ProcessCommandArguments(argc, argv);
+ std::tie(receiver_address, number_of_kbytes, iterations) = ProcessCommandArguments(argc, argv);
std::cout << "receiver_address: " << receiver_address << std::endl
- << "number_of_byte: " << number_of_byte << std::endl
+ << "Package size: " << number_of_kbytes << "k" << std::endl
<< "iterations: " << iterations << std::endl
<< std::endl;
@@ -62,9 +62,16 @@ int main (int argc, char* argv[]) {
}
std::cout << "Successfully connected" << std::endl;
- if(!SendDummyData(producer.get(), number_of_byte, iterations)) {
+ high_resolution_clock::time_point t1 = high_resolution_clock::now();
+ if(!SendDummyData(producer.get(), number_of_kbytes * 1024, iterations)) {
return EXIT_FAILURE;
}
+ high_resolution_clock::time_point t2 = high_resolution_clock::now();
+ double duration_sec = std::chrono::duration_cast<std::chrono::milliseconds>( t2 - t1 ).count() / 1000.0;
+ double size_gb = double(number_of_kbytes) * iterations / 1024.0 / 1024.0 * 8.0;
+ double rate = iterations / duration_sec;
+ std::cout << "Rate: " << rate << " Hz" << std::endl;
+ std::cout << "Bandwidth " << size_gb / duration_sec << " Gbit/s" << std::endl;
return EXIT_SUCCESS;
}
diff --git a/examples/worker/getnext_broker/CMakeLists.txt b/examples/worker/getnext_broker/CMakeLists.txt
index 58f244cf82c5f36d1100d5ea47622f2671c00175..903ff9db59315b9c4c03dd680adf631f48c8efd8 100644
--- a/examples/worker/getnext_broker/CMakeLists.txt
+++ b/examples/worker/getnext_broker/CMakeLists.txt
@@ -10,7 +10,7 @@ set_target_properties(${TARGET_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY
get_target_property(VAR ${TARGET_NAME} RUNTIME_OUTPUT_DIRECTORY)
-configure_file(${CMAKE_SOURCE_DIR}/tests/settings/broker_settings.json settings.json COPYONLY)
+configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/broker_settings.json settings.json COPYONLY)
add_script_test("${TARGET_NAME}" "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} $<TARGET_PROPERTY:hidra2-broker,EXENAME>")
set (dir examples/worker/${TARGET_NAME})
diff --git a/receiver/CMakeLists.txt b/receiver/CMakeLists.txt
index 3fd260ef2dad27c224f83a2311bca432f79626e3..6f5ee995cbb7c61e4178e5c3687602836b493292 100644
--- a/receiver/CMakeLists.txt
+++ b/receiver/CMakeLists.txt
@@ -5,25 +5,27 @@ set(SOURCE_FILES
src/receiver_error.h
src/request.cpp
src/request_handler_file_write.cpp
- )
+ src/statistics.cpp
+ src/statistics_sender_influx_db.cpp
+ src/receiver_config.cpp src/receiver_config.h)
################################
# Library
################################
-add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io>)
+
+
+add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io> $<TARGET_OBJECTS:curl_http_client>
+ $<TARGET_OBJECTS:json_parser>)
set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
-target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR})
+target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR} ${CURL_INCLUDE_DIRS})
+target_link_libraries(${TARGET_NAME} ${CURL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+
add_executable(${TARGET_NAME}-bin src/main.cpp)
set_target_properties(${TARGET_NAME}-bin PROPERTIES OUTPUT_NAME ${TARGET_NAME})
target_link_libraries(${TARGET_NAME}-bin ${TARGET_NAME})
-#Add all necessary common libraries
-GET_PROPERTY(HIDRA2_COMMON_IO_LIBRARIES GLOBAL PROPERTY HIDRA2_COMMON_IO_LIBRARIES)
-target_link_libraries(${TARGET_NAME}-bin ${HIDRA2_COMMON_IO_LIBRARIES})
-set_target_properties(${TARGET_NAME}-bin PROPERTIES LINKER_LANGUAGE CXX)
-
################################
# Testing
################################
@@ -33,8 +35,12 @@ set_property(TARGET ${TARGET_NAME} PROPERTY ENABLE_EXPORTS true)
set(TEST_SOURCE_FILES
unittests/test_receiver.cpp
unittests/test_connection.cpp
+ unittests/test_statistics.cpp
+ unittests/test_config.cpp
unittests/test_request.cpp
unittests/test_request_handler_file_write.cpp
+ unittests/test_statistics_sender_influx_db.cpp
+ unittests/mock_receiver_config.cpp
)
#
set(TEST_LIBRARIES "${TARGET_NAME};system_io")
diff --git a/receiver/src/connection.cpp b/receiver/src/connection.cpp
index b97358222717c8f4e26dad76f22d4bd56b9a276e..1f3794491fae420280a1c2442c0faec06b500ca0 100644
--- a/receiver/src/connection.cpp
+++ b/receiver/src/connection.cpp
@@ -10,7 +10,7 @@ size_t Connection::kRequestHandlerMaxBufferSize;
std::atomic<uint32_t> Connection::kNetworkProducerPeerImplGlobalCounter(0);
Connection::Connection(SocketDescriptor socket_fd, const std::string& address): request_factory__{new RequestFactory},
-io__{GenerateDefaultIO()} {
+io__{GenerateDefaultIO()}, statistics__{new Statistics} {
socket_fd_ = socket_fd;
connection_id_ = kNetworkProducerPeerImplGlobalCounter++;
address_ = address;
@@ -34,7 +34,7 @@ NetworkErrorCode GetNetworkCodeFromError(const Error& err) {
Error Connection::ProcessRequest(const std::unique_ptr<Request>& request) const noexcept {
Error err;
- err = request->Handle();
+ err = request->Handle(&statistics__);
GenericNetworkResponse generic_response;
generic_response.error_code = GetNetworkCodeFromError(err);
if(err) {
@@ -45,6 +45,13 @@ Error Connection::ProcessRequest(const std::unique_ptr<Request>& request) const
}
+void Connection::ProcessStatisticsAfterRequest(const std::unique_ptr<Request>& request) const noexcept {
+ statistics__->IncreaseRequestCounter();
+ statistics__->IncreaseRequestDataVolume(request->GetDataSize() + sizeof(GenericNetworkRequestHeader) +
+ sizeof(GenericNetworkResponse));
+ statistics__->SendIfNeeded();
+}
+
void Connection::Listen() const noexcept {
while(true) {
Error err;
@@ -59,15 +66,18 @@ void Connection::Listen() const noexcept {
std::cerr << "[" << GetId() << "] Error sending response: " << err << std::endl;
break;
}
+ ProcessStatisticsAfterRequest(request);
}
io__->CloseSocket(socket_fd_, nullptr);
- std::cout << "[" << GetId() << "] Disconnected." << std::endl;
+ statistics__->Send();
+ std::cerr << "[" << GetId() << "] Disconnected." << std::endl;
}
std::unique_ptr<Request> Connection::WaitForNewRequest(Error* err) const noexcept {
//TODO: to be overwritten with MessagePack (or similar)
GenericNetworkRequestHeader generic_request_header;
+ statistics__->StartTimer(StatisticEntity::kNetwork);
io__->ReceiveWithTimeout(socket_fd_, &generic_request_header, sizeof(GenericNetworkRequestHeader), 50, err);
if(*err) {
if(*err == IOErrorTemplates::kTimeout) {
@@ -75,6 +85,7 @@ std::unique_ptr<Request> Connection::WaitForNewRequest(Error* err) const noexcep
}
return nullptr;
}
+ statistics__->StopTimer();
return request_factory__->GenerateRequest(generic_request_header, socket_fd_, err);
}
diff --git a/receiver/src/connection.h b/receiver/src/connection.h
index 9861ca0d00cdce67138b6cf51f38c8881b94b953..377019c82f34bff696c026c249853829116ff35a 100644
--- a/receiver/src/connection.h
+++ b/receiver/src/connection.h
@@ -14,6 +14,7 @@
#include "common/networking.h"
#include "io/io.h"
#include "request.h"
+#include "statistics.h"
namespace hidra2 {
@@ -35,10 +36,13 @@ class Connection {
std::unique_ptr<RequestFactory> request_factory__;
std::unique_ptr<IO> io__;
+ mutable std::unique_ptr<Statistics> statistics__;
+
private:
std::unique_ptr<Request> WaitForNewRequest(Error* err) const noexcept;
Error ProcessRequest(const std::unique_ptr<Request>& request) const noexcept;
+ void ProcessStatisticsAfterRequest(const std::unique_ptr<Request>& request) const noexcept;
};
}
diff --git a/receiver/src/main.cpp b/receiver/src/main.cpp
index 3b871296c3a2838b2fc4561e5e01579380ae74cb..dcb19c776da0237e932246dc89794e5fbbc8a69e 100644
--- a/receiver/src/main.cpp
+++ b/receiver/src/main.cpp
@@ -1,12 +1,32 @@
#include <iostream>
#include "receiver.h"
+#include "receiver_config_factory.h"
+#include "receiver_config.h"
+
+
+hidra2::Error ReadConfigFile(int argc, char* argv[]) {
+ if (argc != 2) {
+ std::cerr << "Usage: " << argv[0] << " <config file>" << std::endl;
+ exit(EXIT_FAILURE);
+ }
+ hidra2::ReceiverConfigFactory factory;
+ return factory.SetConfigFromFile(argv[1]);
+}
+
int main (int argc, char* argv[]) {
- static const std::string address = "0.0.0.0:4200";
- auto* receiver = new hidra2::Receiver();
+ auto err = ReadConfigFile(argc, argv);
+ if (err) {
+ std::cerr << "Cannot read config file: " << err << std::endl;
+ return 1;
+ }
+
+ auto config = hidra2::GetReceiverConfig();
- hidra2::Error err;
+ static const std::string address = "0.0.0.0:" + std::to_string(config->listen_port);
+
+ auto* receiver = new hidra2::Receiver();
std::cout << "Listening on " << address << std::endl;
receiver->Listen(address, &err);
diff --git a/receiver/src/receiver_config.cpp b/receiver/src/receiver_config.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ba4e4b250daea114344e47c6c8c6a300e9f6ab0d
--- /dev/null
+++ b/receiver/src/receiver_config.cpp
@@ -0,0 +1,29 @@
+#include "receiver_config.h"
+#include "receiver_config_factory.h"
+#include "io/io_factory.h"
+#include "json_parser/json_parser.h"
+
+namespace hidra2 {
+
+ReceiverConfig config;
+
+ReceiverConfigFactory::ReceiverConfigFactory() : io__{GenerateDefaultIO()} {
+
+}
+
+Error ReceiverConfigFactory::SetConfigFromFile(std::string file_name) {
+ JsonFileParser parser(file_name, &io__);
+ Error err;
+ (err = parser.GetString("MonitorDbAddress", &config.monitor_db_uri)) ||
+ (err = parser.GetUInt64("ListenPort", &config.listen_port)) ||
+ (err = parser.GetBool("WriteToDisk", &config.write_to_disk)) ||
+ (err = parser.GetString("MonitorDbName", &config.monitor_db_name));
+ return err;
+}
+
+const ReceiverConfig* GetReceiverConfig() {
+ return &config;
+}
+
+
+}
\ No newline at end of file
diff --git a/receiver/src/receiver_config.h b/receiver/src/receiver_config.h
new file mode 100644
index 0000000000000000000000000000000000000000..f0aa92adf8e6317c698f2a5621de6407ad25d4c2
--- /dev/null
+++ b/receiver/src/receiver_config.h
@@ -0,0 +1,21 @@
+#ifndef HIDRA2_RECEIVER_CONFIG_H
+#define HIDRA2_RECEIVER_CONFIG_H
+
+#include "io/io.h"
+#include "common/error.h"
+
+namespace hidra2 {
+
+struct ReceiverConfig {
+ std::string monitor_db_uri;
+ std::string monitor_db_name;
+ uint64_t listen_port = 0;
+ bool write_to_disk = false;
+};
+
+const ReceiverConfig* GetReceiverConfig();
+
+}
+
+
+#endif //HIDRA2_RECEIVER_CONFIG_H
diff --git a/receiver/src/receiver_config_factory.h b/receiver/src/receiver_config_factory.h
new file mode 100644
index 0000000000000000000000000000000000000000..ea4087e85c6d906d75244f6b76ee9dc4cd77c738
--- /dev/null
+++ b/receiver/src/receiver_config_factory.h
@@ -0,0 +1,20 @@
+#ifndef HIDRA2_RECEIVER_CONFIG_FACTORY__H
+#define HIDRA2_RECEIVER_CONFIG_FACTORY__H
+
+#include "io/io.h"
+#include "common/error.h"
+
+namespace hidra2 {
+
+class ReceiverConfigFactory {
+ public:
+ ReceiverConfigFactory();
+ Error SetConfigFromFile(std::string file_name);
+ public:
+ std::unique_ptr<IO> io__;
+};
+
+}
+
+
+#endif //HIDRA2_RECEIVER_CONFIG_FACTORY__H
diff --git a/receiver/src/request.cpp b/receiver/src/request.cpp
index 7f38d168f1af06cc33e3efae9f3e96bd08a2f453..19ac5ca8a7dfd394f652412dcdb56e069ae0b54c 100644
--- a/receiver/src/request.cpp
+++ b/receiver/src/request.cpp
@@ -1,6 +1,7 @@
#include "request.h"
#include "io/io_factory.h"
+#include "receiver_config.h"
namespace hidra2 {
Request::Request(const GenericNetworkRequestHeader& header,
@@ -28,19 +29,23 @@ Error Request::ReceiveData() {
}
-Error Request::Handle() {
+Error Request::Handle(std::unique_ptr<Statistics>* statistics) {
Error err;
if (request_header_.data_size != 0) {
+ (*statistics)->StartTimer(StatisticEntity::kNetwork);
auto err = ReceiveData();
if (err) {
return err;
}
+ (*statistics)->StopTimer();
}
for (auto handler : handlers_) {
+ (*statistics)->StartTimer(handler->GetStatisticEntity());
auto err = handler->ProcessRequest(*this);
if (err) {
return err;
}
+ (*statistics)->StopTimer();
}
return nullptr;
}
@@ -75,7 +80,9 @@ std::unique_ptr<Request> RequestFactory::GenerateRequest(const GenericNetworkReq
switch (request_header.op_code) {
case Opcode::kNetOpcodeSendData: {
auto request = std::unique_ptr<Request> {new Request{request_header, socket_fd}};
- request->AddHandler(&request_handler_filewrite_);
+ if (GetReceiverConfig()->write_to_disk) {
+ request->AddHandler(&request_handler_filewrite_);
+ }
return request;
}
default:
diff --git a/receiver/src/request.h b/receiver/src/request.h
index 3d8ef1025dddc481ee9bd2dca594668f93367eab..b32a9e580401fa666c3fbcc404f03c68f8fb8ae8 100644
--- a/receiver/src/request.h
+++ b/receiver/src/request.h
@@ -6,6 +6,7 @@
#include "io/io.h"
#include "request_handler.h"
#include "request_handler_file_write.h"
+#include "statistics.h"
namespace hidra2 {
@@ -13,7 +14,7 @@ using RequestHandlerList = std::vector<const RequestHandler*>;
class Request {
public:
- virtual Error Handle();
+ virtual Error Handle(std::unique_ptr<Statistics>*);
virtual ~Request() = default;
Request(const GenericNetworkRequestHeader& request_header, SocketDescriptor socket_fd);
void AddHandler(const RequestHandler*);
diff --git a/receiver/src/request_handler.h b/receiver/src/request_handler.h
index 2d208c636cf6eb8d3e8a62c961ecb6c2db054584..179171dfe63fb594437c16b5e2801560ad0aca91 100644
--- a/receiver/src/request_handler.h
+++ b/receiver/src/request_handler.h
@@ -2,6 +2,7 @@
#define HIDRA2_REQUEST_HANDLER_H
#include "receiver_error.h"
+#include "statistics.h"
namespace hidra2 {
@@ -10,6 +11,7 @@ class Request;
class RequestHandler {
public:
virtual Error ProcessRequest(const Request& request) const = 0;
+ virtual StatisticEntity GetStatisticEntity() const = 0;
virtual ~RequestHandler() = default;
private:
};
diff --git a/receiver/src/request_handler_file_write.cpp b/receiver/src/request_handler_file_write.cpp
index 3b1a3c2bc4925502ed2789d3ac3e121f459edebb..3db786945e361be77064ea2cb4929c5611b6d1c4 100644
--- a/receiver/src/request_handler_file_write.cpp
+++ b/receiver/src/request_handler_file_write.cpp
@@ -21,4 +21,9 @@ RequestHandlerFileWrite::RequestHandlerFileWrite() : io__{GenerateDefaultIO()} {
}
+StatisticEntity RequestHandlerFileWrite::GetStatisticEntity() const {
+ return StatisticEntity::kDisk;
+}
+
+
}
diff --git a/receiver/src/request_handler_file_write.h b/receiver/src/request_handler_file_write.h
index 8b9828ccd7310c98ea5d9ceaae4c66a4abb12d3f..af81e2d959b54099dea75f6d494725e0de2db14e 100644
--- a/receiver/src/request_handler_file_write.h
+++ b/receiver/src/request_handler_file_write.h
@@ -12,6 +12,7 @@ const uint64_t kMaxFileSize = uint64_t(1024) * 1024 * 1024 * 2; //2GB
class RequestHandlerFileWrite final: public RequestHandler {
public:
RequestHandlerFileWrite();
+ StatisticEntity GetStatisticEntity() const override;
Error ProcessRequest(const Request& request) const override;
std::unique_ptr<IO> io__;
};
diff --git a/receiver/src/statistics.cpp b/receiver/src/statistics.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fa806eb46bb702d55c95b4ff690411a8c6b248b8
--- /dev/null
+++ b/receiver/src/statistics.cpp
@@ -0,0 +1,80 @@
+#include "statistics.h"
+#include "statistics_sender_influx_db.h"
+#include <algorithm>
+
+using std::chrono::high_resolution_clock;
+
+namespace hidra2 {
+
+void Statistics::SendIfNeeded() noexcept {
+ if (GetTotalElapsedMs() > write_interval_) {
+ Send();
+ }
+}
+
+void Statistics::Send() noexcept {
+ statistics_sender__->SendStatistics(PrepareStatisticsToSend());
+ ResetStatistics();
+}
+
+
+StatisticsToSend Statistics::PrepareStatisticsToSend() const noexcept {
+ StatisticsToSend stat;
+ stat.n_requests = nrequests_;
+ stat.data_volume = volume_counter_;
+ stat.elapsed_ms = std::max(uint64_t{1}, GetTotalElapsedMs());
+ for (auto i = 0; i < kNStatisticEntities; i++) {
+ stat.entity_shares[i] = double(GetElapsedMs(StatisticEntity(i))) / stat.elapsed_ms;
+ }
+ return stat;
+}
+
+uint64_t Statistics::GetTotalElapsedMs() const noexcept {
+ return std::chrono::duration_cast<std::chrono::milliseconds>
+ ( high_resolution_clock::now() - last_timepoint_).count();
+}
+
+uint64_t Statistics::GetElapsedMs(StatisticEntity entity) const noexcept {
+ return std::chrono::duration_cast<std::chrono::milliseconds>(time_counters_[entity]).count();
+}
+
+void Statistics::SetWriteInterval(uint64_t interval_ms) {
+ write_interval_ = interval_ms;
+}
+
+void Statistics::ResetStatistics() noexcept {
+ last_timepoint_ = high_resolution_clock::now();
+ nrequests_ = 0;
+ for (int i = 0; i < kNStatisticEntities; i++) {
+ time_counters_[i] = std::chrono::nanoseconds{0};
+ }
+ volume_counter_ = 0;
+}
+
+void Statistics::IncreaseRequestCounter() noexcept {
+ nrequests_++;
+}
+
+Statistics::Statistics(unsigned int write_frequency) : statistics_sender__{new StatisticsSenderInfluxDb},
+write_interval_{write_frequency} {
+ ResetStatistics();
+}
+
+void Statistics::StartTimer(const StatisticEntity& entity) noexcept {
+ current_statistic_entity_ = entity;
+ current_timer_last_timepoint_ = high_resolution_clock::now();
+}
+
+void Statistics::IncreaseRequestDataVolume(uint64_t transferred_data_volume) noexcept {
+ volume_counter_ += transferred_data_volume;
+}
+
+
+void Statistics::StopTimer() noexcept {
+ auto elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>
+ (high_resolution_clock::now() - current_timer_last_timepoint_);
+ time_counters_[current_statistic_entity_] += elapsed;
+}
+
+
+}
\ No newline at end of file
diff --git a/receiver/src/statistics.h b/receiver/src/statistics.h
new file mode 100644
index 0000000000000000000000000000000000000000..de374ed981a2dc9236cbeec7ae792770b32b6ffb
--- /dev/null
+++ b/receiver/src/statistics.h
@@ -0,0 +1,56 @@
+#ifndef HIDRA2_STATISTICS_H
+#define HIDRA2_STATISTICS_H
+
+#include <chrono>
+#include <memory>
+
+#include "statistics_sender.h"
+
+namespace hidra2 {
+
+static const auto kNStatisticEntities = 3;
+enum StatisticEntity : int {
+ kDatabase = 0,
+ kDisk,
+ kNetwork,
+};
+
+struct StatisticsToSend {
+ double entity_shares[kNStatisticEntities];
+ uint64_t elapsed_ms;
+ uint64_t data_volume;
+ uint64_t n_requests;
+};
+
+class Statistics {
+ public:
+// virtual needed for unittests, could be replaced with #define VIRTUAL ... in case of performance issues
+ virtual void SendIfNeeded() noexcept;
+ virtual void Send() noexcept;
+ explicit Statistics(unsigned int write_interval = kDefaultStatisticWriteIntervalMs);
+ virtual void IncreaseRequestCounter() noexcept;
+ virtual void StartTimer(const StatisticEntity& entity) noexcept;
+ virtual void IncreaseRequestDataVolume(uint64_t transferred_data_volume) noexcept;
+ virtual void StopTimer() noexcept;
+
+ void SetWriteInterval(uint64_t interval_ms);
+ std::unique_ptr<StatisticsSender> statistics_sender__;
+ private:
+ uint64_t GetElapsedMs(StatisticEntity entity) const noexcept;
+ void ResetStatistics() noexcept;
+ uint64_t GetTotalElapsedMs() const noexcept;
+ StatisticsToSend PrepareStatisticsToSend() const noexcept;
+ static const unsigned int kDefaultStatisticWriteIntervalMs = 10000;
+ uint64_t nrequests_;
+ std::chrono::high_resolution_clock::time_point last_timepoint_;
+ std::chrono::high_resolution_clock::time_point current_timer_last_timepoint_;
+ StatisticEntity current_statistic_entity_ = StatisticEntity::kDatabase;
+ std::chrono::nanoseconds time_counters_[kNStatisticEntities];
+ uint64_t volume_counter_;
+ unsigned int write_interval_;
+
+};
+
+}
+
+#endif //HIDRA2_STATISTICS_H
diff --git a/receiver/src/statistics_sender.h b/receiver/src/statistics_sender.h
new file mode 100644
index 0000000000000000000000000000000000000000..2cf12b272a7b70bcfbaebadb493db5a47972cb29
--- /dev/null
+++ b/receiver/src/statistics_sender.h
@@ -0,0 +1,18 @@
+#ifndef HIDRA2_STATISTICS_SENDER_H
+#define HIDRA2_STATISTICS_SENDER_H
+
+#include <cstdint>
+
+namespace hidra2 {
+
+struct StatisticsToSend;
+
+class StatisticsSender {
+ public:
+ virtual void SendStatistics(const StatisticsToSend& statistic) const noexcept = 0;
+ virtual ~StatisticsSender() = default;
+};
+
+}
+
+#endif //HIDRA2_STATISTICS_SENDER_H
diff --git a/receiver/src/statistics_sender_influx_db.cpp b/receiver/src/statistics_sender_influx_db.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..84f8b07d4db80011c8fabf1673ee33d91e855719
--- /dev/null
+++ b/receiver/src/statistics_sender_influx_db.cpp
@@ -0,0 +1,52 @@
+#include "statistics_sender_influx_db.h"
+
+#include <iostream>
+
+#include "statistics.h"
+#include "http_client/curl_http_client.h"
+#include "receiver_config.h"
+
+namespace hidra2 {
+
+template<typename ... Args>
+std::string string_format( const std::string& format, Args ... args ) {
+ size_t size = snprintf( nullptr, 0, format.c_str(), args ... ) + 1;
+ std::unique_ptr<char[]> buf( new char[ size ] );
+ snprintf( buf.get(), size, format.c_str(), args ... );
+ return std::string( buf.get(), buf.get() + size - 1 );
+}
+
+
+void StatisticsSenderInfluxDb::SendStatistics(const StatisticsToSend& statistic) const noexcept {
+ HttpCode code;
+ Error err;
+ auto responce = httpclient__->Post(GetReceiverConfig()->monitor_db_uri + "/write?db=" +
+ GetReceiverConfig()->monitor_db_name, StatisticsToString(statistic),
+ &code, &err);
+ if (err) {
+ std::cerr << "Error sending statistics: " << err << std::endl;
+ return;
+ }
+
+ if (code != HttpCode::OK && code != HttpCode::NoContent) {
+ std::cerr << "Error sending statistics: " << responce << std::endl;
+ }
+}
+
+std::string StatisticsSenderInfluxDb::StatisticsToString(const StatisticsToSend& statistic) const noexcept {
+ std::string str;
+ std::string tags = "receiver=1,connection=1";
+ str = "statistics," + tags + " elapsed_ms=" + string_format("%ld", statistic.elapsed_ms);
+ str += ",data_volume=" + string_format("%ld", statistic.data_volume);
+ str += ",n_requests=" + string_format("%ld", statistic.n_requests);
+ str += ",db_share=" + string_format("%.4f", statistic.entity_shares[StatisticEntity::kDatabase]);
+ str += ",network_share=" + string_format("%.4f", statistic.entity_shares[StatisticEntity::kNetwork]);
+ str += ",disk_share=" + string_format("%.4f", statistic.entity_shares[StatisticEntity::kDisk]);
+ return str;
+}
+
+StatisticsSenderInfluxDb::StatisticsSenderInfluxDb(): httpclient__{new CurlHttpClient} {
+};
+
+
+}
diff --git a/receiver/src/statistics_sender_influx_db.h b/receiver/src/statistics_sender_influx_db.h
new file mode 100644
index 0000000000000000000000000000000000000000..aeff173333ec802cea81ae732e28d3dfa703832c
--- /dev/null
+++ b/receiver/src/statistics_sender_influx_db.h
@@ -0,0 +1,22 @@
+#ifndef HIDRA2_STATISTICS_SENDER_INFLUX_DB_H
+#define HIDRA2_STATISTICS_SENDER_INFLUX_DB_H
+
+#include "http_client/http_client.h"
+#include "statistics_sender.h"
+
+
+namespace hidra2 {
+
+class StatisticsSenderInfluxDb : public StatisticsSender {
+ public:
+ StatisticsSenderInfluxDb();
+ virtual void SendStatistics(const StatisticsToSend& statistic) const noexcept override;
+ std::unique_ptr<HttpClient> httpclient__;
+ private:
+ std::string StatisticsToString(const StatisticsToSend& statistic) const noexcept;
+
+};
+
+}
+
+#endif //HIDRA2_STATISTICS_SENDER_INFLUX_DB_H
diff --git a/receiver/unittests/mock_receiver_config.cpp b/receiver/unittests/mock_receiver_config.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7819fe13a934464ce8ddd2fbbb8f8abd90d7403c
--- /dev/null
+++ b/receiver/unittests/mock_receiver_config.cpp
@@ -0,0 +1,37 @@
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
+#include "mock_receiver_config.h"
+#include "../src/receiver_config_factory.h"
+
+#include <unittests/MockIO.h>
+
+using testing::_;
+
+namespace hidra2 {
+
+Error SetReceiverConfig (const ReceiverConfig& config) {
+ MockIO mock_io;
+ ReceiverConfigFactory config_factory;
+ config_factory.io__ = std::unique_ptr<IO> {&mock_io};
+
+ auto config_string = std::string("{\"MonitorDbAddress\":") + "\"" + config.monitor_db_uri + "\"";
+ config_string += "," + std::string("\"MonitorDbName\":") + "\"" + config.monitor_db_name + "\"";
+ config_string += "," + std::string("\"ListenPort\":") + std::to_string(config.listen_port);
+ config_string += "," + std::string("\"WriteToDisk\":") + (config.write_to_disk ? "true" : "false");
+ config_string += "}";
+
+ EXPECT_CALL(mock_io, ReadFileToString_t("fname", _)).WillOnce(
+ testing::Return(config_string)
+ );
+
+ auto err = config_factory.SetConfigFromFile("fname");
+
+ config_factory.io__.release();
+
+ return err;
+}
+
+}
+
+
diff --git a/receiver/unittests/mock_receiver_config.h b/receiver/unittests/mock_receiver_config.h
new file mode 100644
index 0000000000000000000000000000000000000000..9a724668cf0286d448bb9fb0129bd0ee18e1af12
--- /dev/null
+++ b/receiver/unittests/mock_receiver_config.h
@@ -0,0 +1,14 @@
+#ifndef HIDRA2_MOCK_RECEIVER_CONFIG_H
+#define HIDRA2_MOCK_RECEIVER_CONFIG_H
+
+#include "../src/receiver_config.h"
+
+
+namespace hidra2 {
+
+Error SetReceiverConfig (const ReceiverConfig& config);
+
+}
+
+
+#endif
diff --git a/receiver/unittests/mock_statistics.h b/receiver/unittests/mock_statistics.h
new file mode 100644
index 0000000000000000000000000000000000000000..dc4077a72ff9f683a106df5368c1de03da4c4f9c
--- /dev/null
+++ b/receiver/unittests/mock_statistics.h
@@ -0,0 +1,47 @@
+#ifndef HIDRA2_MOCK_STATISTICS_H
+#define HIDRA2_MOCK_STATISTICS_H
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
+#include "../src/statistics.h"
+
+namespace hidra2 {
+
+class MockStatistics : public hidra2::Statistics {
+ public:
+ void SendIfNeeded() noexcept override {
+ SendIfNeeded_t();
+ }
+
+ void Send() noexcept override {
+ Send_t();
+ }
+
+ void IncreaseRequestCounter() noexcept override {
+ IncreaseRequestCounter_t();
+ }
+ void StartTimer(const hidra2::StatisticEntity& entity) noexcept override {
+ StartTimer_t(entity);
+ }
+ void IncreaseRequestDataVolume(uint64_t transferred_data_volume) noexcept override {
+ IncreaseRequestDataVolume_t(transferred_data_volume);
+ }
+ void StopTimer() noexcept override {
+ StopTimer_t();
+ }
+
+ MOCK_METHOD0(SendIfNeeded_t, void());
+ MOCK_METHOD0(Send_t, void());
+ MOCK_METHOD0(IncreaseRequestCounter_t, void());
+ MOCK_METHOD0(StopTimer_t, void());
+ MOCK_METHOD1(IncreaseRequestDataVolume_t, void (uint64_t
+ transferred_data_volume));
+ MOCK_METHOD1(StartTimer_t, void(
+ const hidra2::StatisticEntity& entity));
+
+};
+
+}
+
+#endif //HIDRA2_MOCK_STATISTICS_H
diff --git a/receiver/unittests/test_config.cpp b/receiver/unittests/test_config.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ea15cede3b0eb2ba3bb0511f4d3381ee575a1454
--- /dev/null
+++ b/receiver/unittests/test_config.cpp
@@ -0,0 +1,70 @@
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+#include <unittests/MockIO.h>
+
+#include "../src/receiver_config.h"
+#include "../src/receiver_config_factory.h"
+#include "mock_receiver_config.h"
+
+using ::testing::Test;
+using ::testing::Return;
+using ::testing::_;
+using ::testing::DoAll;
+using ::testing::SetArgReferee;
+using ::testing::Gt;
+using ::testing::Eq;
+using ::testing::Ne;
+using ::testing::Mock;
+using ::testing::NiceMock;
+using ::testing::SaveArg;
+using ::testing::SaveArgPointee;
+using ::testing::InSequence;
+using ::testing::SetArgPointee;
+using ::hidra2::Error;
+using ::hidra2::ErrorInterface;
+using ::hidra2::FileDescriptor;
+using ::hidra2::SocketDescriptor;
+using ::hidra2::MockIO;
+
+using ::hidra2::ReceiverConfigFactory;
+using hidra2::GetReceiverConfig;
+
+namespace {
+
+
+class ConfigTests : public Test {
+ public:
+ MockIO mock_io;
+ ReceiverConfigFactory config_factory;
+ void SetUp() override {
+ config_factory.io__ = std::unique_ptr<hidra2::IO> {&mock_io};
+ }
+ void TearDown() override {
+ config_factory.io__.release();
+ }
+
+};
+
+
+TEST_F(ConfigTests, ReadSettings) {
+
+ hidra2::ReceiverConfig test_config;
+ test_config.listen_port = 4200;
+ test_config.monitor_db_name = "db_test";
+ test_config.monitor_db_uri = "localhost:8086";
+ test_config.write_to_disk = true;
+
+
+ auto err = hidra2::SetReceiverConfig(test_config);
+
+ auto config = GetReceiverConfig();
+
+ ASSERT_THAT(err, Eq(nullptr));
+ ASSERT_THAT(config->monitor_db_uri, Eq("localhost:8086"));
+ ASSERT_THAT(config->monitor_db_name, Eq("db_test"));
+ ASSERT_THAT(config->listen_port, Eq(4200));
+ ASSERT_THAT(config->write_to_disk, true);
+
+}
+
+}
diff --git a/receiver/unittests/test_connection.cpp b/receiver/unittests/test_connection.cpp
index 846e6df26be5d0283e761d1d1d73cdd54765b18e..d09e26507ecb8e02de64b065bd1673799f5ac736 100644
--- a/receiver/unittests/test_connection.cpp
+++ b/receiver/unittests/test_connection.cpp
@@ -1,9 +1,12 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include <unittests/MockIO.h>
+
#include "../src/connection.h"
#include "../src/receiver_error.h"
#include "../src/request.h"
+#include "../src/statistics.h"
+#include "mock_statistics.h"
using ::testing::Test;
using ::testing::Return;
@@ -12,6 +15,7 @@ using ::testing::DoAll;
using ::testing::SetArgReferee;
using ::testing::Gt;
using ::testing::Eq;
+using ::testing::Ne;
using ::testing::Mock;
using ::testing::NiceMock;
using ::testing::SaveArg;
@@ -30,19 +34,32 @@ using ::hidra2::Opcode;
using ::hidra2::Connection;
using ::hidra2::MockIO;
using hidra2::Request;
+using hidra2::Statistics;
+using hidra2::StatisticEntity;
+using hidra2::MockStatistics;
namespace {
+TEST(Connection, Constructor) {
+ Connection connection{0, "some_address"};
+ ASSERT_THAT(dynamic_cast<hidra2::Statistics*>(connection.statistics__.get()), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<hidra2::IO*>(connection.io__.get()), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<hidra2::RequestFactory*>(connection.request_factory__.get()), Ne(nullptr));
+}
+
+
+
class MockRequest: public Request {
public:
MockRequest(const GenericNetworkRequestHeader& request_header, SocketDescriptor socket_fd):
Request(request_header, socket_fd) {};
- Error Handle() override {
+ Error Handle(std::unique_ptr<Statistics>* statistics) override {
return Error{Handle_t()};
};
MOCK_CONST_METHOD0(Handle_t, ErrorInterface * ());
};
+
class MockRequestFactory: public hidra2::RequestFactory {
public:
std::unique_ptr<Request> GenerateRequest(const GenericNetworkRequestHeader& request_header,
@@ -65,18 +82,22 @@ class ConnectionTests : public Test {
Connection connection{0, "some_address"};
MockIO mock_io;
MockRequestFactory mock_factory;
+ NiceMock<MockStatistics> mock_statictics;
void SetUp() override {
- connection.io__ = std::unique_ptr<hidra2::IO> {&mock_io};;
+ connection.io__ = std::unique_ptr<hidra2::IO> {&mock_io};
+ connection.statistics__ = std::unique_ptr<hidra2::Statistics> {&mock_statictics};
connection.request_factory__ = std::unique_ptr<hidra2::RequestFactory> {&mock_factory};
ON_CALL(mock_io, ReceiveWithTimeout_t(_, _, _, _, _)).
WillByDefault(DoAll(testing::SetArgPointee<4>(nullptr),
testing::Return(0)));
EXPECT_CALL(mock_io, CloseSocket_t(_, _));
+ EXPECT_CALL(mock_statictics, Send_t());
}
void TearDown() override {
connection.io__.release();
connection.request_factory__.release();
+ connection.statistics__.release();
}
};
@@ -180,4 +201,63 @@ TEST_F(ConnectionTests, SendsErrorToProducer) {
}
+void MockExitCycle(const MockIO& mock_io, MockStatistics& mock_statictics) {
+ EXPECT_CALL(mock_statictics, StartTimer_t(StatisticEntity::kNetwork));
+
+ EXPECT_CALL(mock_io, ReceiveWithTimeout_t(_, _, _, _, _))
+ .WillOnce(
+ DoAll(SetArgPointee<4>(new hidra2::IOError("", hidra2::IOErrorType::kUnknownIOError)),
+ Return(0))
+ );
+}
+
+MockRequest* MockWaitRequest(const MockRequestFactory& mock_factory) {
+ GenericNetworkRequestHeader header;
+ header.data_size = 1;
+ auto request = new MockRequest{header, 1};
+ EXPECT_CALL(mock_factory, GenerateRequest_t(_, _, _)).WillOnce(
+ Return(request)
+ );
+ return request;
+}
+
+TEST_F(ConnectionTests, FillsStatistics) {
+ InSequence sequence;
+
+ EXPECT_CALL(mock_statictics, StartTimer_t(StatisticEntity::kNetwork));
+
+ EXPECT_CALL(mock_io, ReceiveWithTimeout_t(_, _, _, _, _));
+
+ EXPECT_CALL(mock_statictics, StopTimer_t());
+
+ auto request = MockWaitRequest(mock_factory);
+
+ EXPECT_CALL(*request, Handle_t()).WillOnce(
+ Return(nullptr)
+ );
+
+
+ EXPECT_CALL(mock_io, Send_t(_, _, _, _)).WillOnce(
+ DoAll(SetArgPointee<3>(nullptr),
+ Return(0)
+ ));
+
+
+ EXPECT_CALL(mock_statictics, IncreaseRequestCounter_t());
+
+ EXPECT_CALL(mock_statictics, IncreaseRequestDataVolume_t(1 + sizeof(hidra2::GenericNetworkRequestHeader) +
+ sizeof(hidra2::GenericNetworkResponse)));
+
+
+ EXPECT_CALL(mock_statictics, SendIfNeeded_t());
+
+ MockExitCycle(mock_io, mock_statictics);
+
+ connection.Listen();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+
+}
+
+
}
diff --git a/receiver/unittests/test_request.cpp b/receiver/unittests/test_request.cpp
index 9b263826fe308534afd922688f6a64cf2311619e..1d1e175279e9c6f3a2588b6a014c48ad22f9969c 100644
--- a/receiver/unittests/test_request.cpp
+++ b/receiver/unittests/test_request.cpp
@@ -7,6 +7,9 @@
#include "../src/request_handler.h"
#include "../src/request_handler_file_write.h"
+#include "mock_statistics.h"
+#include "mock_receiver_config.h"
+
using ::testing::Test;
using ::testing::Return;
using ::testing::_;
@@ -31,6 +34,13 @@ using ::hidra2::Opcode;
using ::hidra2::Connection;
using ::hidra2::MockIO;
using hidra2::Request;
+using hidra2::MockStatistics;
+
+using hidra2::StatisticEntity;
+
+using hidra2::ReceiverConfig;
+using hidra2::SetReceiverConfig;
+
namespace {
@@ -40,6 +50,10 @@ class MockReqestHandler : public hidra2::RequestHandler {
return Error{ProcessRequest_t(request)};
}
+ StatisticEntity GetStatisticEntity() const override {
+ return StatisticEntity::kDisk;
+ }
+
MOCK_CONST_METHOD1(ProcessRequest_t, ErrorInterface * (const Request& request));
};
@@ -49,7 +63,10 @@ class FactoryTests : public Test {
hidra2::RequestFactory factory;
Error err{nullptr};
GenericNetworkRequestHeader generic_request_header;
+ ReceiverConfig config;
void SetUp() override {
+ config.write_to_disk = true;
+ SetReceiverConfig(config);
}
void TearDown() override {
}
@@ -71,6 +88,15 @@ TEST_F(FactoryTests, ReturnsDataRequestOnkNetOpcodeSendDataCode) {
ASSERT_THAT(dynamic_cast<const hidra2::RequestHandlerFileWrite*>(request->GetListHandlers()[0]), Ne(nullptr));
}
+TEST_F(FactoryTests, DoNotAddWriterIfNotWanted) {
+ generic_request_header.op_code = hidra2::Opcode::kNetOpcodeSendData;
+ config.write_to_disk = false;
+ SetReceiverConfig(config);
+
+ auto request = factory.GenerateRequest(generic_request_header, 1, &err);
+ ASSERT_THAT(err, Eq(nullptr));
+ ASSERT_THAT(request->GetListHandlers().size(), Eq(0));
+}
@@ -82,7 +108,10 @@ class RequestTests : public Test {
uint64_t data_id_{15};
std::unique_ptr<Request> request;
NiceMock<MockIO> mock_io;
+ NiceMock<MockStatistics> mock_statistics;
+ std::unique_ptr<hidra2::Statistics> stat;
void SetUp() override {
+ stat = std::unique_ptr<hidra2::Statistics> {&mock_statistics};
generic_request_header.data_size = data_size_;
generic_request_header.data_id = data_id_;
request.reset(new Request{generic_request_header, socket_fd_});
@@ -94,6 +123,7 @@ class RequestTests : public Test {
}
void TearDown() override {
request->io__.release();
+ stat.release();
}
};
@@ -106,7 +136,7 @@ TEST_F(RequestTests, HandleDoesNotReceiveEmptyData) {
EXPECT_CALL(mock_io, Receive_t(_, _, _, _)).Times(0);
- auto err = request->Handle();
+ auto err = request->Handle(&stat);
ASSERT_THAT(err, Eq(nullptr));
}
@@ -117,15 +147,35 @@ TEST_F(RequestTests, HandleReturnsErrorOnDataReceive) {
Return(0)
));
- auto err = request->Handle();
+ auto err = request->Handle(&stat);
ASSERT_THAT(err, Eq(hidra2::IOErrorTemplates::kReadError));
}
+
+TEST_F(RequestTests, HandleMeasuresTimeOnDataReceive) {
+
+ EXPECT_CALL(mock_statistics, StartTimer_t(hidra2::StatisticEntity::kNetwork));
+
+ EXPECT_CALL(mock_io, Receive_t(socket_fd_, _, data_size_, _)).WillOnce(
+ DoAll(SetArgPointee<3>(nullptr),
+ Return(0)
+ ));
+
+ EXPECT_CALL(mock_statistics, StopTimer_t());
+
+ request->Handle(&stat);
+}
+
+
+
+
TEST_F(RequestTests, HandleProcessesRequests) {
MockReqestHandler mock_request_handler;
+ EXPECT_CALL(mock_statistics, StartTimer_t(hidra2::StatisticEntity::kNetwork));
+
EXPECT_CALL(mock_request_handler, ProcessRequest_t(_)).WillOnce(
Return(nullptr)
).WillOnce(
@@ -135,7 +185,12 @@ TEST_F(RequestTests, HandleProcessesRequests) {
request->AddHandler(&mock_request_handler);
request->AddHandler(&mock_request_handler);
- auto err = request->Handle();
+ EXPECT_CALL(mock_statistics, StartTimer_t(hidra2::StatisticEntity::kDisk)).Times(2);
+
+ EXPECT_CALL(mock_statistics, StopTimer_t()).Times(2);
+
+
+ auto err = request->Handle(&stat);
ASSERT_THAT(err, Eq(hidra2::IOErrorTemplates::kUnknownIOError));
}
@@ -148,7 +203,7 @@ TEST_F(RequestTests, DataIsNullAtInit) {
TEST_F(RequestTests, GetDataIsNotNullptr) {
- request->Handle();
+ request->Handle(&stat);
auto& data = request->GetData();
diff --git a/receiver/unittests/test_request_handler_file_write.cpp b/receiver/unittests/test_request_handler_file_write.cpp
index 7c3dc1d09a1f0cc8396cd961e1c5ac7de93a986d..5640ce4539a64d0dc036a2f5731e5c91bb06804e 100644
--- a/receiver/unittests/test_request_handler_file_write.cpp
+++ b/receiver/unittests/test_request_handler_file_write.cpp
@@ -62,6 +62,12 @@ class FileWriteHandlerTests : public Test {
};
+TEST_F(FileWriteHandlerTests, CheckStatisticEntity) {
+ auto entity = handler.GetStatisticEntity();
+ ASSERT_THAT(entity, Eq(hidra2::StatisticEntity::kDisk));
+}
+
+
TEST_F(FileWriteHandlerTests, ErrorWhenZeroFileSize) {
EXPECT_CALL(*mock_request, GetDataSize())
.WillOnce(Return(0))
diff --git a/receiver/unittests/test_statistics.cpp b/receiver/unittests/test_statistics.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..13115b7b7eb2b9d47f1ab9b5524453ebf18f42d3
--- /dev/null
+++ b/receiver/unittests/test_statistics.cpp
@@ -0,0 +1,213 @@
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+#include <thread>
+
+#include "../src/statistics.h"
+#include "../src/statistics_sender.h"
+#include "../src/statistics_sender_influx_db.h"
+
+using ::testing::Test;
+using ::testing::Gt;
+using ::testing::Ge;
+using ::testing::Le;
+using ::testing::Eq;
+using ::testing::Ne;
+using ::testing::Ref;
+using ::testing::_;
+
+using hidra2::Statistics;
+using hidra2::StatisticEntity;
+using hidra2::StatisticsSender;
+using hidra2::StatisticsSenderInfluxDb;
+using hidra2::StatisticsToSend;
+
+
+
+namespace {
+
+
+TEST(StatisticTestsConstructor, Constructor) {
+ Statistics statistics;
+ ASSERT_THAT(dynamic_cast<hidra2::StatisticsSenderInfluxDb*>(statistics.statistics_sender__.get()), Ne(nullptr));
+}
+
+
+class MockStatisticsSender: public StatisticsSender {
+ public:
+ void SendStatistics(const StatisticsToSend& statistics) const noexcept override {
+ SendStatistics_t(statistics);
+ }
+ MOCK_CONST_METHOD1(SendStatistics_t, void (const StatisticsToSend&));
+};
+
+class StatisticTests : public Test {
+ public:
+ Statistics statistics{0};
+ void TestTimer(const StatisticEntity& entity);
+ MockStatisticsSender mock_statistics_sender;
+ void SetUp() override {
+ statistics.statistics_sender__.reset(&mock_statistics_sender);
+ }
+ void TearDown() override {
+ statistics.statistics_sender__.release();
+ }
+ StatisticsToSend ExtractStat();
+};
+
+
+ACTION_P(SaveArg1ToSendStat, value) {
+ auto resp = static_cast<const StatisticsToSend&>(arg0);
+ value->n_requests = resp.n_requests;
+ value->data_volume = resp.data_volume;
+ value->elapsed_ms = resp.elapsed_ms;
+ for (int i = 0; i < hidra2::kNStatisticEntities; i++) {
+ value->entity_shares[i] = resp.entity_shares[i];
+ }
+
+}
+
+
+StatisticsToSend StatisticTests::ExtractStat() {
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ StatisticsToSend stat;
+ stat.elapsed_ms = 0;
+ stat.n_requests = 0;
+ stat.data_volume = 0;
+ for (int i = 0; i < hidra2::kNStatisticEntities; i++) {
+ stat.entity_shares[i] = 0.0;
+ }
+
+ EXPECT_CALL(mock_statistics_sender, SendStatistics_t(_)).
+ WillOnce(SaveArg1ToSendStat(&stat));
+
+ statistics.SendIfNeeded();
+ return stat;
+}
+
+TEST_F(StatisticTests, IncreaseRequestCounter) {
+ statistics.IncreaseRequestCounter();
+
+ auto stat = ExtractStat();
+
+ ASSERT_THAT(stat.n_requests, Eq(1));
+}
+
+TEST_F(StatisticTests, StatisticsResetAfterSend) {
+ statistics.IncreaseRequestCounter();
+
+ ExtractStat();
+ auto stat = ExtractStat();
+
+ ASSERT_THAT(stat.n_requests, Eq(0));
+}
+
+
+TEST_F(StatisticTests, ElapsedTime) {
+
+ auto stat = ExtractStat();
+
+ ASSERT_THAT(stat.elapsed_ms, Ge(1));
+}
+
+
+
+TEST_F(StatisticTests, RequestCounterZeroAtInit) {
+ auto stat = ExtractStat();
+
+ ASSERT_THAT(stat.n_requests, Eq(0));
+}
+
+TEST_F(StatisticTests, GetDataVolume) {
+ statistics.IncreaseRequestDataVolume(100);
+
+ auto stat = ExtractStat();
+
+ ASSERT_THAT(stat.data_volume, Eq(100));
+}
+
+TEST_F(StatisticTests, DataVolumeZeroAtInit) {
+ auto stat = ExtractStat();
+
+ ASSERT_THAT(stat.data_volume, Eq(0));
+}
+
+void StatisticTests::TestTimer(const StatisticEntity& entity) {
+ statistics.StartTimer(entity);
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ statistics.StopTimer();
+
+ auto stat = ExtractStat();
+
+ ASSERT_THAT(stat.entity_shares[entity], Ge(0.4));
+
+}
+
+TEST_F(StatisticTests, TimerForDatabase) {
+ TestTimer(StatisticEntity::kDatabase);
+}
+
+TEST_F(StatisticTests, TimerForNetwork) {
+ TestTimer(StatisticEntity::kNetwork);
+}
+
+TEST_F(StatisticTests, TimerForDisk) {
+ TestTimer(StatisticEntity::kDisk);
+}
+
+TEST_F(StatisticTests, TimerForAll) {
+ statistics.StartTimer(StatisticEntity::kDatabase);
+ std::this_thread::sleep_for(std::chrono::milliseconds(20));
+ statistics.StopTimer();
+ statistics.StartTimer(StatisticEntity::kNetwork);
+ std::this_thread::sleep_for(std::chrono::milliseconds(30));
+ statistics.StopTimer();
+
+ statistics.StartTimer(StatisticEntity::kDisk);
+ std::this_thread::sleep_for(std::chrono::milliseconds(40));
+ statistics.StopTimer();
+
+ auto stat = ExtractStat();
+
+ ASSERT_THAT(stat.entity_shares[StatisticEntity::kDatabase], Ge(0.15));
+ ASSERT_THAT(stat.entity_shares[StatisticEntity::kDatabase], Le(0.25));
+
+ ASSERT_THAT(stat.entity_shares[StatisticEntity::kNetwork], Ge(0.25));
+ ASSERT_THAT(stat.entity_shares[StatisticEntity::kNetwork], Le(0.35));
+
+ ASSERT_THAT(stat.entity_shares[StatisticEntity::kDisk], Ge(0.35));
+ ASSERT_THAT(stat.entity_shares[StatisticEntity::kDisk], Le(0.45));
+}
+
+
+TEST_F(StatisticTests, SendStaticsDoesCallsSender) {
+ statistics.SetWriteInterval(1000);
+
+ EXPECT_CALL(mock_statistics_sender, SendStatistics_t(_)).Times(0);
+
+ statistics.SendIfNeeded();
+}
+
+
+TEST_F(StatisticTests, StatisticsSend) {
+ statistics.IncreaseRequestCounter();
+
+ StatisticsToSend stat;
+ stat.elapsed_ms = 0;
+ stat.n_requests = 0;
+ stat.data_volume = 0;
+ for (int i = 0; i < hidra2::kNStatisticEntities; i++) {
+ stat.entity_shares[i] = 0.0;
+ }
+
+ EXPECT_CALL(mock_statistics_sender, SendStatistics_t(_)).
+ WillOnce(SaveArg1ToSendStat(&stat));
+
+ statistics.Send();
+ std::cout << stat.elapsed_ms << std::endl;
+
+ ASSERT_THAT(stat.elapsed_ms, Ge(1));
+}
+
+
+}
diff --git a/receiver/unittests/test_statistics_sender_influx_db.cpp b/receiver/unittests/test_statistics_sender_influx_db.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0324253dd8c8773d06404abb209ba0f8b58eb403
--- /dev/null
+++ b/receiver/unittests/test_statistics_sender_influx_db.cpp
@@ -0,0 +1,83 @@
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+#include <unittests/MockIO.h>
+
+#include "../src/statistics_sender_influx_db.h"
+#include "../src/statistics_sender.h"
+#include "http_client/curl_http_client.h"
+#include "unittests/MockHttpClient.h"
+#include "../src/statistics.h"
+
+#include "../src/receiver_config.h"
+#include "mock_receiver_config.h"
+
+
+using ::testing::Test;
+using ::testing::Return;
+using ::testing::_;
+using ::testing::DoAll;
+using ::testing::SetArgReferee;
+using ::testing::Gt;
+using ::testing::Eq;
+using ::testing::Ne;
+using ::testing::Mock;
+using ::testing::NiceMock;
+using ::testing::SaveArg;
+using ::testing::SaveArgPointee;
+using ::testing::InSequence;
+using ::testing::SetArgPointee;
+
+using hidra2::StatisticsSenderInfluxDb;
+using hidra2::MockHttpClient;
+using hidra2::StatisticsToSend;
+using hidra2::ReceiverConfig;
+using hidra2::SetReceiverConfig;
+
+namespace {
+
+TEST(SenderInfluxDb, Constructor) {
+ StatisticsSenderInfluxDb sender;
+ ASSERT_THAT(dynamic_cast<hidra2::CurlHttpClient*>(sender.httpclient__.get()), Ne(nullptr));
+}
+
+
+class SenderInfluxDbTests : public Test {
+ public:
+ StatisticsSenderInfluxDb sender;
+ MockHttpClient mock_http_client;
+ void SetUp() override {
+ sender.httpclient__.reset(&mock_http_client);
+ }
+ void TearDown() override {
+ sender.httpclient__.release();
+ }
+};
+
+
+TEST_F(SenderInfluxDbTests, SendStatisticsCallsPost) {
+ StatisticsToSend statistics;
+ statistics.n_requests = 4;
+ statistics.entity_shares[hidra2::StatisticEntity::kDisk] = 0.6;
+ statistics.entity_shares[hidra2::StatisticEntity::kNetwork] = 0.3;
+ statistics.entity_shares[hidra2::StatisticEntity::kDatabase] = 0.1;
+ statistics.elapsed_ms = 100;
+ statistics.data_volume = 1000;
+
+ ReceiverConfig config;
+ config.monitor_db_uri = "test_uri";
+ config.monitor_db_name = "test_name";
+ SetReceiverConfig(config);
+
+ std::string expect_string = "statistics,receiver=1,connection=1 elapsed_ms=100,data_volume=1000,"
+ "n_requests=4,db_share=0.1000,network_share=0.3000,disk_share=0.6000";
+ EXPECT_CALL(mock_http_client, Post_t("test_uri/write?db=test_name", expect_string, _, _)).
+ WillOnce(
+ DoAll(SetArgPointee<3>(new hidra2::IOError("Test Read Error", hidra2::IOErrorType::kReadError)),
+ Return("")
+ ));
+
+ sender.SendStatistics(statistics);
+}
+
+
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index fed871cd51912dfa1d57fd8626413cf56a76cd9b..3e431d34cd6ac7e59e99566ff0151bea49a95dbf 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1,23 +1,4 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 3.7) # needed for fixtures
-find_package(Threads REQUIRED)
+add_subdirectory(automatic)
-add_subdirectory(common/cpp)
-add_subdirectory(system_io)
-
-add_subdirectory(json_parser)
-
-if(BUILD_MONGODB_CLIENTLIB)
- add_subdirectory(mongo_db)
-endif()
-
-
-if(BUILD_BROKER)
- add_subdirectory(broker)
-endif()
-
-
-add_subdirectory(worker)
-
-add_subdirectory(producer_receiver)
diff --git a/tests/automatic/CMakeLists.txt b/tests/automatic/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b0a661944ef8fd479b8d5aa8d4b4c69139995420
--- /dev/null
+++ b/tests/automatic/CMakeLists.txt
@@ -0,0 +1,23 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 3.7) # needed for fixtures
+find_package(Threads)
+
+add_subdirectory(common/cpp)
+add_subdirectory(system_io)
+
+add_subdirectory(json_parser)
+
+if(BUILD_MONGODB_CLIENTLIB)
+ add_subdirectory(mongo_db)
+endif()
+
+
+if(BUILD_BROKER)
+ add_subdirectory(broker)
+endif()
+
+
+add_subdirectory(worker)
+
+add_subdirectory(curl_http_client)
+
+add_subdirectory(producer_receiver)
\ No newline at end of file
diff --git a/tests/broker/CMakeLists.txt b/tests/automatic/broker/CMakeLists.txt
similarity index 52%
rename from tests/broker/CMakeLists.txt
rename to tests/automatic/broker/CMakeLists.txt
index c3741747744a36054595915d91f42ecd5eb96943..6f1685c7d12bf036a9fa6f126b357fd962d2b954 100644
--- a/tests/broker/CMakeLists.txt
+++ b/tests/automatic/broker/CMakeLists.txt
@@ -1,3 +1,6 @@
add_subdirectory(get_next)
add_subdirectory(read_config)
+if (UNIX)
+add_subdirectory(check_monitoring)
+endif()
\ No newline at end of file
diff --git a/tests/automatic/broker/check_monitoring/CMakeLists.txt b/tests/automatic/broker/check_monitoring/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..19c68e39b9dc264e6ef541df207ad418a27af8f7
--- /dev/null
+++ b/tests/automatic/broker/check_monitoring/CMakeLists.txt
@@ -0,0 +1,8 @@
+set(TARGET_NAME hidra2-broker)
+
+################################
+# Testing
+################################
+configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/broker_settings.json settings.json COPYONLY)
+add_script_test("${TARGET_NAME}-monitoring" "$<TARGET_PROPERTY:${TARGET_NAME},EXENAME>" nomem
+ )
diff --git a/tests/automatic/broker/check_monitoring/check_linux.sh b/tests/automatic/broker/check_monitoring/check_linux.sh
new file mode 100644
index 0000000000000000000000000000000000000000..9af76490853e00d86cc70d3118295d9a5faba76f
--- /dev/null
+++ b/tests/automatic/broker/check_monitoring/check_linux.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+database_name=db_test
+
+set -e
+
+trap Cleanup EXIT
+
+Cleanup() {
+ echo cleanup
+ influx -execute "drop database ${database_name}"
+ kill -9 $brokerid
+}
+
+influx -execute "create database ${database_name}"
+
+$@ settings.json &
+
+sleep 0.3
+
+brokerid=`echo $!`
+
+for i in `seq 1 50`;
+do
+ curl --silent 127.0.0.1:5005/database/data/next >/dev/null 2>&1 &
+done
+
+
+sleep 2
+
+influx -execute "select sum(rate) from RequestsRate" -database=${database_name} -format=json | jq .results[0].series[0].values[0][1] | grep 50
diff --git a/tests/broker/get_next/CMakeLists.txt b/tests/automatic/broker/get_next/CMakeLists.txt
similarity index 66%
rename from tests/broker/get_next/CMakeLists.txt
rename to tests/automatic/broker/get_next/CMakeLists.txt
index 4eb27c1142b81b97dcc2c6326c739b2ca1fecaac..8b5f3b6f02b77436f882d1aecebf94026cc8222a 100644
--- a/tests/broker/get_next/CMakeLists.txt
+++ b/tests/automatic/broker/get_next/CMakeLists.txt
@@ -3,6 +3,6 @@ set(TARGET_NAME hidra2-broker)
################################
# Testing
################################
-configure_file(${CMAKE_SOURCE_DIR}/tests/settings/broker_settings.json settings.json COPYONLY)
+configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/broker_settings.json settings.json COPYONLY)
add_script_test("${TARGET_NAME}-getnext" "$<TARGET_PROPERTY:${TARGET_NAME},EXENAME>" nomem
)
diff --git a/tests/broker/get_next/check_linux.sh b/tests/automatic/broker/get_next/check_linux.sh
similarity index 100%
rename from tests/broker/get_next/check_linux.sh
rename to tests/automatic/broker/get_next/check_linux.sh
diff --git a/tests/broker/get_next/check_windows.bat b/tests/automatic/broker/get_next/check_windows.bat
similarity index 100%
rename from tests/broker/get_next/check_windows.bat
rename to tests/automatic/broker/get_next/check_windows.bat
diff --git a/tests/broker/read_config/CMakeLists.txt b/tests/automatic/broker/read_config/CMakeLists.txt
similarity index 71%
rename from tests/broker/read_config/CMakeLists.txt
rename to tests/automatic/broker/read_config/CMakeLists.txt
index a8949ad818264d362b46d90222f3ad5348f623a4..76f6467cfd21c94965a03b39d446a95e53d2ac4c 100644
--- a/tests/broker/read_config/CMakeLists.txt
+++ b/tests/automatic/broker/read_config/CMakeLists.txt
@@ -3,7 +3,7 @@ set(TARGET_NAME hidra2-broker)
################################
# Testing
################################
-configure_file(${CMAKE_SOURCE_DIR}/tests/settings/broker_settings.json settings_good.json COPYONLY)
+configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/broker_settings.json settings_good.json COPYONLY)
configure_file(settings_bad.json settings_bad.json COPYONLY)
add_script_test("${TARGET_NAME}-readconfig" "$<TARGET_PROPERTY:${TARGET_NAME},EXENAME>" nomem
)
diff --git a/tests/broker/read_config/check_linux.sh b/tests/automatic/broker/read_config/check_linux.sh
similarity index 85%
rename from tests/broker/read_config/check_linux.sh
rename to tests/automatic/broker/read_config/check_linux.sh
index e491171a3c8d00802b7e1beb55c77acc3bb3040a..5b9e9c78c8b9c5c814b907ea08dfd4c745043e73 100644
--- a/tests/broker/read_config/check_linux.sh
+++ b/tests/automatic/broker/read_config/check_linux.sh
@@ -9,7 +9,7 @@ brokerid=`echo $!`
kill -9 $brokerid
# check if gives error with bad json file
-$@ settings_bad.json 2>&1 | grep "invalid"
+$@ settings_bad.json 2>&1 | grep "not set"
# check if gives error non-existing file
$@ settings_notexist.json 2>&1 | grep "no such"
diff --git a/tests/broker/read_config/check_windows.bat b/tests/automatic/broker/read_config/check_windows.bat
similarity index 77%
rename from tests/broker/read_config/check_windows.bat
rename to tests/automatic/broker/read_config/check_windows.bat
index 76f042ab8dd9116ef59851b116dc7361313d7c99..e78f9380f8728c8dbf81bede444dc009d428d6e5 100644
--- a/tests/broker/read_config/check_windows.bat
+++ b/tests/automatic/broker/read_config/check_windows.bat
@@ -6,7 +6,7 @@ start /B "" "%full_name%" settings_good.json
ping 1.0.0.0 -n 1 -w 100 > nul
Taskkill /IM "%short_name%" /F
-%full_name% settings_bad.json 2>&1 | findstr invalid || goto :error
+%full_name% settings_bad.json 2>&1 | findstr /c:"not set" || goto :error
%full_name% settings_notexist.json 2>&1| findstr cannot || goto :error
diff --git a/tests/automatic/broker/read_config/settings_bad.json b/tests/automatic/broker/read_config/settings_bad.json
new file mode 100644
index 0000000000000000000000000000000000000000..88157fb1b06da72f529ea9f42a808a58e91533cb
--- /dev/null
+++ b/tests/automatic/broker/read_config/settings_bad.json
@@ -0,0 +1,4 @@
+{
+ "BrokerDbAddres":"127.0.0.1:27017",
+ "port":12
+}
\ No newline at end of file
diff --git a/tests/common/cpp/CMakeLists.txt b/tests/automatic/common/cpp/CMakeLists.txt
similarity index 100%
rename from tests/common/cpp/CMakeLists.txt
rename to tests/automatic/common/cpp/CMakeLists.txt
diff --git a/tests/common/cpp/include/testing.h b/tests/automatic/common/cpp/include/testing.h
similarity index 100%
rename from tests/common/cpp/include/testing.h
rename to tests/automatic/common/cpp/include/testing.h
diff --git a/tests/common/cpp/src/testing.cpp b/tests/automatic/common/cpp/src/testing.cpp
similarity index 100%
rename from tests/common/cpp/src/testing.cpp
rename to tests/automatic/common/cpp/src/testing.cpp
diff --git a/tests/automatic/curl_http_client/CMakeLists.txt b/tests/automatic/curl_http_client/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c704851020a6b314c6172cedde15c2e58cbe489c
--- /dev/null
+++ b/tests/automatic/curl_http_client/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(curl_http_client_command)
diff --git a/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt b/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..503b9e91b51343704ccd80b856da664d5c99aab7
--- /dev/null
+++ b/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt
@@ -0,0 +1,27 @@
+set(TARGET_NAME curl_httpclient_command)
+set(SOURCE_FILES curl_httpclient_command.cpp)
+
+
+################################
+# Executable and link
+################################
+add_executable(${TARGET_NAME} ${SOURCE_FILES})
+target_link_libraries(${TARGET_NAME} test_common hidra2-worker)
+
+#set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
+#if (CMAKE_COMPILER_IS_GNUCXX)
+# set_target_properties(${TARGET_NAME} PROPERTIES LINK_FLAGS_DEBUG "--coverage")
+#endif()
+
+
+################################
+# Testing
+################################
+
+#add_test_setup_cleanup(${TARGET_NAME})
+add_integration_test(${TARGET_NAME} get_google "GET google.com moved 302")
+add_integration_test(${TARGET_NAME} get_badaddress "GET google.com/badaddress found 404")
+add_integration_test(${TARGET_NAME} get_badaddress2 "GET 111 clienterror 404")
+add_integration_test(${TARGET_NAME} post "POST httpbin.org/post testdata 200")
+add_integration_test(${TARGET_NAME} post_badaddress "POST google.com/badaddress found 404")
+add_integration_test(${TARGET_NAME} post_badaddress2 "POST 111 clienterror 404")
\ No newline at end of file
diff --git a/tests/worker/curl_http_client_get/curl_httpclient_get.cpp b/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp
similarity index 66%
rename from tests/worker/curl_http_client_get/curl_httpclient_get.cpp
rename to tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp
index 478839271d364ce2bcfa98cc00de5ac4f602707d..f049bc15573a7e502e53719fa4128944c3848a87 100644
--- a/tests/worker/curl_http_client_get/curl_httpclient_get.cpp
+++ b/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp
@@ -8,20 +8,22 @@ using hidra2::M_AssertEq;
using hidra2::M_AssertContains;
struct Args {
+ std::string command;
std::string uri;
int code;
std::string answer;
};
Args GetArgs(int argc, char* argv[]) {
- if (argc != 4) {
+ if (argc != 5) {
std::cout << "Wrong number of arguments" << std::endl;
exit(EXIT_FAILURE);
}
- std::string uri{argv[1]};
- std::string answer {argv[2]};
- int code = std::stoi(argv[3]);
- return Args{uri, code, answer};
+ std::string command{argv[1]};
+ std::string uri{argv[2]};
+ std::string answer {argv[3]};
+ int code = std::stoi(argv[4]);
+ return Args{command, uri, code, answer};
}
@@ -34,7 +36,12 @@ int main(int argc, char* argv[]) {
auto server_broker = static_cast<hidra2::ServerDataBroker*>(broker.get());
hidra2::HttpCode code;
- auto response = server_broker->httpclient__->Get(args.uri, &code, &err);
+ std::string response;
+ if (args.command == "GET") {
+ response = server_broker->httpclient__->Get(args.uri, &code, &err);
+ } else if (args.command == "POST") {
+ response = server_broker->httpclient__->Post(args.uri, "testdata", &code, &err);
+ }
if (err != nullptr) {
M_AssertEq("clienterror", args.answer);
diff --git a/tests/json_parser/CMakeLists.txt b/tests/automatic/json_parser/CMakeLists.txt
similarity index 100%
rename from tests/json_parser/CMakeLists.txt
rename to tests/automatic/json_parser/CMakeLists.txt
diff --git a/tests/json_parser/parse_config_file/CMakeLists.txt b/tests/automatic/json_parser/parse_config_file/CMakeLists.txt
similarity index 100%
rename from tests/json_parser/parse_config_file/CMakeLists.txt
rename to tests/automatic/json_parser/parse_config_file/CMakeLists.txt
diff --git a/tests/json_parser/parse_config_file/config.json b/tests/automatic/json_parser/parse_config_file/config.json
similarity index 100%
rename from tests/json_parser/parse_config_file/config.json
rename to tests/automatic/json_parser/parse_config_file/config.json
diff --git a/tests/json_parser/parse_config_file/config_bad.json b/tests/automatic/json_parser/parse_config_file/config_bad.json
similarity index 100%
rename from tests/json_parser/parse_config_file/config_bad.json
rename to tests/automatic/json_parser/parse_config_file/config_bad.json
diff --git a/tests/json_parser/parse_config_file/parse_config_file.cpp b/tests/automatic/json_parser/parse_config_file/parse_config_file.cpp
similarity index 97%
rename from tests/json_parser/parse_config_file/parse_config_file.cpp
rename to tests/automatic/json_parser/parse_config_file/parse_config_file.cpp
index c185240553dc07c5786fcb8565ce38b3b0dc57f7..4fb391b756cf2b45059808fc330a9e3100ca496c 100644
--- a/tests/json_parser/parse_config_file/parse_config_file.cpp
+++ b/tests/automatic/json_parser/parse_config_file/parse_config_file.cpp
@@ -39,7 +39,8 @@ void AssertSettings(const Settings& settings) {
}
Settings Parse(const std::string& fname, Error* err) {
- hidra2::JsonParser parser(fname, true);
+
+ hidra2::JsonFileParser parser(fname);
Settings settings;
diff --git a/tests/mongo_db/CMakeLists.txt b/tests/automatic/mongo_db/CMakeLists.txt
similarity index 100%
rename from tests/mongo_db/CMakeLists.txt
rename to tests/automatic/mongo_db/CMakeLists.txt
diff --git a/tests/mongo_db/connect/CMakeLists.txt b/tests/automatic/mongo_db/connect/CMakeLists.txt
similarity index 100%
rename from tests/mongo_db/connect/CMakeLists.txt
rename to tests/automatic/mongo_db/connect/CMakeLists.txt
diff --git a/tests/mongo_db/connect/connect_mongodb.cpp b/tests/automatic/mongo_db/connect/connect_mongodb.cpp
similarity index 100%
rename from tests/mongo_db/connect/connect_mongodb.cpp
rename to tests/automatic/mongo_db/connect/connect_mongodb.cpp
diff --git a/tests/mongo_db/insert/CMakeLists.txt b/tests/automatic/mongo_db/insert/CMakeLists.txt
similarity index 100%
rename from tests/mongo_db/insert/CMakeLists.txt
rename to tests/automatic/mongo_db/insert/CMakeLists.txt
diff --git a/tests/mongo_db/insert/cleanup_linux.sh b/tests/automatic/mongo_db/insert/cleanup_linux.sh
similarity index 100%
rename from tests/mongo_db/insert/cleanup_linux.sh
rename to tests/automatic/mongo_db/insert/cleanup_linux.sh
diff --git a/tests/mongo_db/insert/cleanup_windows.bat b/tests/automatic/mongo_db/insert/cleanup_windows.bat
similarity index 100%
rename from tests/mongo_db/insert/cleanup_windows.bat
rename to tests/automatic/mongo_db/insert/cleanup_windows.bat
diff --git a/tests/mongo_db/insert/insert_mongodb.cpp b/tests/automatic/mongo_db/insert/insert_mongodb.cpp
similarity index 100%
rename from tests/mongo_db/insert/insert_mongodb.cpp
rename to tests/automatic/mongo_db/insert/insert_mongodb.cpp
diff --git a/tests/automatic/producer_receiver/CMakeLists.txt b/tests/automatic/producer_receiver/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ad36394912da0e110b2707f73e0270452030d41f
--- /dev/null
+++ b/tests/automatic/producer_receiver/CMakeLists.txt
@@ -0,0 +1,4 @@
+add_subdirectory(transfer_single_file)
+if (UNIX)
+ add_subdirectory(check_monitoring)
+endif()
\ No newline at end of file
diff --git a/tests/automatic/producer_receiver/check_monitoring/CMakeLists.txt b/tests/automatic/producer_receiver/check_monitoring/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..298b81ed25e3e9ef14c159b37e04c9a8a000d29f
--- /dev/null
+++ b/tests/automatic/producer_receiver/check_monitoring/CMakeLists.txt
@@ -0,0 +1,8 @@
+set(TARGET_NAME receiver)
+
+################################
+# Testing
+################################
+configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/receiver.json receiver.json COPYONLY)
+add_script_test("${TARGET_NAME}-monitoring" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:receiver-bin>" nomem
+ )
diff --git a/tests/automatic/producer_receiver/check_monitoring/check_linux.sh b/tests/automatic/producer_receiver/check_monitoring/check_linux.sh
new file mode 100644
index 0000000000000000000000000000000000000000..3b4b82ec54b48718f5b00412ca0e9a83ddbbdd0e
--- /dev/null
+++ b/tests/automatic/producer_receiver/check_monitoring/check_linux.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+database_name=db_test
+
+set -e
+
+trap Cleanup EXIT
+
+Cleanup() {
+ echo cleanup
+ influx -execute "drop database ${database_name}"
+ kill $receiverid
+ rm -rf files
+
+}
+
+influx -execute "create database ${database_name}"
+
+nohup $2 receiver.json &>/dev/null &
+sleep 0.3
+receiverid=`echo $!`
+
+mkdir files
+
+$1 localhost:4200 100 112
+
+sleep 1
+
+influx -execute "select sum(n_requests) from statistics" -database=${database_name} -format=json | jq .results[0].series[0].values[0][1] | grep 112
diff --git a/tests/producer_receiver/transfer_single_file/CMakeLists.txt b/tests/automatic/producer_receiver/transfer_single_file/CMakeLists.txt
similarity index 69%
rename from tests/producer_receiver/transfer_single_file/CMakeLists.txt
rename to tests/automatic/producer_receiver/transfer_single_file/CMakeLists.txt
index 83a9f59580c5427848720b28aea7353d86335d23..f745213b85ef22c898bce3130b1ae24dcfa33a13 100644
--- a/tests/producer_receiver/transfer_single_file/CMakeLists.txt
+++ b/tests/automatic/producer_receiver/transfer_single_file/CMakeLists.txt
@@ -3,4 +3,5 @@ set(TARGET_NAME transfer-single-file)
################################
# Testing
################################
+configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/receiver.json receiver.json COPYONLY)
add_script_test("${TARGET_NAME}" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:receiver-bin>" nomem)
diff --git a/tests/producer_receiver/transfer_single_file/check_linux.sh b/tests/automatic/producer_receiver/transfer_single_file/check_linux.sh
similarity index 66%
rename from tests/producer_receiver/transfer_single_file/check_linux.sh
rename to tests/automatic/producer_receiver/transfer_single_file/check_linux.sh
index 163d4ab08b8144ea133ca0f6f48899b304234e63..d7c69e6a25b6189b0d84220d635a15b386f29449 100644
--- a/tests/producer_receiver/transfer_single_file/check_linux.sh
+++ b/tests/automatic/producer_receiver/transfer_single_file/check_linux.sh
@@ -6,11 +6,11 @@ trap Cleanup EXIT
Cleanup() {
echo cleanup
- kill $receiverid
rm -rf files
+ kill $receiverid
}
-nohup $2 &>/dev/null &
+nohup $2 receiver.json &>/dev/null &
sleep 0.3
receiverid=`echo $!`
@@ -18,4 +18,4 @@ mkdir files
$1 localhost:4200 100 1
-ls -ln files/0.bin | awk '{ print $5 }'| grep 100
+ls -ln files/0.bin | awk '{ print $5 }'| grep 102400
diff --git a/tests/producer_receiver/transfer_single_file/check_windows.bat b/tests/automatic/producer_receiver/transfer_single_file/check_windows.bat
similarity index 80%
rename from tests/producer_receiver/transfer_single_file/check_windows.bat
rename to tests/automatic/producer_receiver/transfer_single_file/check_windows.bat
index 9413a82a228200e5a50ed87135e10450ab471714..ee15242cd4354e178f16d6e4059b635cf01b3db4 100644
--- a/tests/producer_receiver/transfer_single_file/check_windows.bat
+++ b/tests/automatic/producer_receiver/transfer_single_file/check_windows.bat
@@ -1,7 +1,7 @@
set full_recv_name="%2"
set short_recv_name="%~nx2"
-start /B "" "%full_recv_name%"
+start /B "" "%full_recv_name%" receiver.json
ping 1.0.0.0 -n 1 -w 100 > nul
@@ -13,7 +13,7 @@ ping 1.0.0.0 -n 1 -w 100 > nul
FOR /F "usebackq" %%A IN ('files\0.bin') DO set size=%%~zA
-if %size% NEQ 100 goto :error
+if %size% NEQ 102400 goto :error
goto :clean
diff --git a/tests/automatic/settings/broker_settings.json b/tests/automatic/settings/broker_settings.json
new file mode 100644
index 0000000000000000000000000000000000000000..56bf15f52fa9666103ac542903195def198733d7
--- /dev/null
+++ b/tests/automatic/settings/broker_settings.json
@@ -0,0 +1,6 @@
+{
+ "BrokerDbAddress":"127.0.0.1:27017",
+ "MonitorDbAddress": "localhost:8086",
+ "MonitorDbName": "db_test",
+ "port":5005
+}
\ No newline at end of file
diff --git a/tests/automatic/settings/receiver.json b/tests/automatic/settings/receiver.json
new file mode 100644
index 0000000000000000000000000000000000000000..e23a556d1cd5a8c03ce0c32b37b7b73161bfa89d
--- /dev/null
+++ b/tests/automatic/settings/receiver.json
@@ -0,0 +1,6 @@
+{
+ "MonitorDbAddress":"localhost:8086",
+ "MonitorDbName": "db_test",
+ "ListenPort":4200,
+ "WriteToDisk":true
+}
\ No newline at end of file
diff --git a/tests/system_io/CMakeLists.txt b/tests/automatic/system_io/CMakeLists.txt
similarity index 100%
rename from tests/system_io/CMakeLists.txt
rename to tests/automatic/system_io/CMakeLists.txt
diff --git a/tests/system_io/ip_tcp_network/CMakeLists.txt b/tests/automatic/system_io/ip_tcp_network/CMakeLists.txt
similarity index 100%
rename from tests/system_io/ip_tcp_network/CMakeLists.txt
rename to tests/automatic/system_io/ip_tcp_network/CMakeLists.txt
diff --git a/tests/system_io/ip_tcp_network/ip_tcp_network.cpp b/tests/automatic/system_io/ip_tcp_network/ip_tcp_network.cpp
similarity index 100%
rename from tests/system_io/ip_tcp_network/ip_tcp_network.cpp
rename to tests/automatic/system_io/ip_tcp_network/ip_tcp_network.cpp
diff --git a/tests/system_io/read_file_content/CMakeLists.txt b/tests/automatic/system_io/read_file_content/CMakeLists.txt
similarity index 100%
rename from tests/system_io/read_file_content/CMakeLists.txt
rename to tests/automatic/system_io/read_file_content/CMakeLists.txt
diff --git a/tests/system_io/read_file_content/cleanup_linux.sh b/tests/automatic/system_io/read_file_content/cleanup_linux.sh
similarity index 100%
rename from tests/system_io/read_file_content/cleanup_linux.sh
rename to tests/automatic/system_io/read_file_content/cleanup_linux.sh
diff --git a/tests/system_io/read_file_content/cleanup_windows.bat b/tests/automatic/system_io/read_file_content/cleanup_windows.bat
similarity index 100%
rename from tests/system_io/read_file_content/cleanup_windows.bat
rename to tests/automatic/system_io/read_file_content/cleanup_windows.bat
diff --git a/tests/system_io/read_file_content/read_file_content.cpp b/tests/automatic/system_io/read_file_content/read_file_content.cpp
similarity index 100%
rename from tests/system_io/read_file_content/read_file_content.cpp
rename to tests/automatic/system_io/read_file_content/read_file_content.cpp
diff --git a/tests/system_io/read_file_content/setup_linux.sh b/tests/automatic/system_io/read_file_content/setup_linux.sh
similarity index 100%
rename from tests/system_io/read_file_content/setup_linux.sh
rename to tests/automatic/system_io/read_file_content/setup_linux.sh
diff --git a/tests/system_io/read_file_content/setup_windows.bat b/tests/automatic/system_io/read_file_content/setup_windows.bat
similarity index 100%
rename from tests/system_io/read_file_content/setup_windows.bat
rename to tests/automatic/system_io/read_file_content/setup_windows.bat
diff --git a/tests/system_io/read_folder_content/CMakeLists.txt b/tests/automatic/system_io/read_folder_content/CMakeLists.txt
similarity index 100%
rename from tests/system_io/read_folder_content/CMakeLists.txt
rename to tests/automatic/system_io/read_folder_content/CMakeLists.txt
diff --git a/tests/system_io/read_folder_content/cleanup_linux.sh b/tests/automatic/system_io/read_folder_content/cleanup_linux.sh
similarity index 100%
rename from tests/system_io/read_folder_content/cleanup_linux.sh
rename to tests/automatic/system_io/read_folder_content/cleanup_linux.sh
diff --git a/tests/system_io/read_folder_content/cleanup_windows.bat b/tests/automatic/system_io/read_folder_content/cleanup_windows.bat
similarity index 100%
rename from tests/system_io/read_folder_content/cleanup_windows.bat
rename to tests/automatic/system_io/read_folder_content/cleanup_windows.bat
diff --git a/tests/system_io/read_folder_content/read_folder_content.cpp b/tests/automatic/system_io/read_folder_content/read_folder_content.cpp
similarity index 100%
rename from tests/system_io/read_folder_content/read_folder_content.cpp
rename to tests/automatic/system_io/read_folder_content/read_folder_content.cpp
diff --git a/tests/system_io/read_folder_content/setup_linux.sh b/tests/automatic/system_io/read_folder_content/setup_linux.sh
similarity index 100%
rename from tests/system_io/read_folder_content/setup_linux.sh
rename to tests/automatic/system_io/read_folder_content/setup_linux.sh
diff --git a/tests/system_io/read_folder_content/setup_windows.bat b/tests/automatic/system_io/read_folder_content/setup_windows.bat
similarity index 100%
rename from tests/system_io/read_folder_content/setup_windows.bat
rename to tests/automatic/system_io/read_folder_content/setup_windows.bat
diff --git a/tests/system_io/read_string_from_file/CMakeLists.txt b/tests/automatic/system_io/read_string_from_file/CMakeLists.txt
similarity index 100%
rename from tests/system_io/read_string_from_file/CMakeLists.txt
rename to tests/automatic/system_io/read_string_from_file/CMakeLists.txt
diff --git a/tests/system_io/read_string_from_file/cleanup_linux.sh b/tests/automatic/system_io/read_string_from_file/cleanup_linux.sh
similarity index 100%
rename from tests/system_io/read_string_from_file/cleanup_linux.sh
rename to tests/automatic/system_io/read_string_from_file/cleanup_linux.sh
diff --git a/tests/system_io/read_string_from_file/cleanup_windows.bat b/tests/automatic/system_io/read_string_from_file/cleanup_windows.bat
similarity index 100%
rename from tests/system_io/read_string_from_file/cleanup_windows.bat
rename to tests/automatic/system_io/read_string_from_file/cleanup_windows.bat
diff --git a/tests/system_io/read_string_from_file/read_string_from_file.cpp b/tests/automatic/system_io/read_string_from_file/read_string_from_file.cpp
similarity index 100%
rename from tests/system_io/read_string_from_file/read_string_from_file.cpp
rename to tests/automatic/system_io/read_string_from_file/read_string_from_file.cpp
diff --git a/tests/system_io/read_string_from_file/setup_linux.sh b/tests/automatic/system_io/read_string_from_file/setup_linux.sh
similarity index 100%
rename from tests/system_io/read_string_from_file/setup_linux.sh
rename to tests/automatic/system_io/read_string_from_file/setup_linux.sh
diff --git a/tests/system_io/read_string_from_file/setup_windows.bat b/tests/automatic/system_io/read_string_from_file/setup_windows.bat
similarity index 100%
rename from tests/system_io/read_string_from_file/setup_windows.bat
rename to tests/automatic/system_io/read_string_from_file/setup_windows.bat
diff --git a/tests/system_io/resolve_hostname_to_ip/CMakeLists.txt b/tests/automatic/system_io/resolve_hostname_to_ip/CMakeLists.txt
similarity index 100%
rename from tests/system_io/resolve_hostname_to_ip/CMakeLists.txt
rename to tests/automatic/system_io/resolve_hostname_to_ip/CMakeLists.txt
diff --git a/tests/system_io/resolve_hostname_to_ip/resolve_hostname_to_ip.cpp b/tests/automatic/system_io/resolve_hostname_to_ip/resolve_hostname_to_ip.cpp
similarity index 100%
rename from tests/system_io/resolve_hostname_to_ip/resolve_hostname_to_ip.cpp
rename to tests/automatic/system_io/resolve_hostname_to_ip/resolve_hostname_to_ip.cpp
diff --git a/tests/system_io/write_data_to_file/CMakeLists.txt b/tests/automatic/system_io/write_data_to_file/CMakeLists.txt
similarity index 100%
rename from tests/system_io/write_data_to_file/CMakeLists.txt
rename to tests/automatic/system_io/write_data_to_file/CMakeLists.txt
diff --git a/tests/system_io/write_data_to_file/cleanup_linux.sh b/tests/automatic/system_io/write_data_to_file/cleanup_linux.sh
similarity index 100%
rename from tests/system_io/write_data_to_file/cleanup_linux.sh
rename to tests/automatic/system_io/write_data_to_file/cleanup_linux.sh
diff --git a/tests/system_io/write_data_to_file/cleanup_windows.bat b/tests/automatic/system_io/write_data_to_file/cleanup_windows.bat
similarity index 100%
rename from tests/system_io/write_data_to_file/cleanup_windows.bat
rename to tests/automatic/system_io/write_data_to_file/cleanup_windows.bat
diff --git a/tests/system_io/write_data_to_file/setup_linux.sh b/tests/automatic/system_io/write_data_to_file/setup_linux.sh
similarity index 100%
rename from tests/system_io/write_data_to_file/setup_linux.sh
rename to tests/automatic/system_io/write_data_to_file/setup_linux.sh
diff --git a/tests/system_io/write_data_to_file/setup_windows.bat b/tests/automatic/system_io/write_data_to_file/setup_windows.bat
similarity index 100%
rename from tests/system_io/write_data_to_file/setup_windows.bat
rename to tests/automatic/system_io/write_data_to_file/setup_windows.bat
diff --git a/tests/system_io/write_data_to_file/write_data_to_file.cpp b/tests/automatic/system_io/write_data_to_file/write_data_to_file.cpp
similarity index 100%
rename from tests/system_io/write_data_to_file/write_data_to_file.cpp
rename to tests/automatic/system_io/write_data_to_file/write_data_to_file.cpp
diff --git a/tests/worker/CMakeLists.txt b/tests/automatic/worker/CMakeLists.txt
similarity index 86%
rename from tests/worker/CMakeLists.txt
rename to tests/automatic/worker/CMakeLists.txt
index e5145961a17f9444a92f8f471c1cc5380c7f6363..f8a2dd31597411f2a2297fc6436cce592cfbd8e9 100644
--- a/tests/worker/CMakeLists.txt
+++ b/tests/automatic/worker/CMakeLists.txt
@@ -4,8 +4,6 @@ add_subdirectory(next_multithread_folder)
add_subdirectory(next_multithread_broker)
add_subdirectory(connect_multithread)
-add_subdirectory(curl_http_client_get)
-
if(BUILD_WORKER_TOOLS)
add_subdirectory(folder_to_db)
endif()
diff --git a/tests/worker/connect_multithread/CMakeLists.txt b/tests/automatic/worker/connect_multithread/CMakeLists.txt
similarity index 89%
rename from tests/worker/connect_multithread/CMakeLists.txt
rename to tests/automatic/worker/connect_multithread/CMakeLists.txt
index abe83367dc6b2796d7ca4d734b6ed4d37b8ed167..64224018bc011daf620bca4a96fcc5730004d9b4 100644
--- a/tests/worker/connect_multithread/CMakeLists.txt
+++ b/tests/automatic/worker/connect_multithread/CMakeLists.txt
@@ -7,7 +7,6 @@ set(SOURCE_FILES content_multithread.cpp)
################################
add_executable(${TARGET_NAME} ${SOURCE_FILES})
target_link_libraries(${TARGET_NAME} test_common hidra2-worker ${HIDRA2_COMMON_IO_LIBRARIES})
-set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
if (CMAKE_COMPILER_IS_GNUCXX)
set_target_properties(${TARGET_NAME} PROPERTIES LINK_FLAGS_DEBUG "--coverage")
endif()
diff --git a/tests/worker/connect_multithread/cleanup_linux.sh b/tests/automatic/worker/connect_multithread/cleanup_linux.sh
similarity index 100%
rename from tests/worker/connect_multithread/cleanup_linux.sh
rename to tests/automatic/worker/connect_multithread/cleanup_linux.sh
diff --git a/tests/worker/connect_multithread/cleanup_windows.bat b/tests/automatic/worker/connect_multithread/cleanup_windows.bat
similarity index 100%
rename from tests/worker/connect_multithread/cleanup_windows.bat
rename to tests/automatic/worker/connect_multithread/cleanup_windows.bat
diff --git a/tests/worker/connect_multithread/content_multithread.cpp b/tests/automatic/worker/connect_multithread/content_multithread.cpp
similarity index 100%
rename from tests/worker/connect_multithread/content_multithread.cpp
rename to tests/automatic/worker/connect_multithread/content_multithread.cpp
diff --git a/tests/worker/connect_multithread/setup_linux.sh b/tests/automatic/worker/connect_multithread/setup_linux.sh
similarity index 100%
rename from tests/worker/connect_multithread/setup_linux.sh
rename to tests/automatic/worker/connect_multithread/setup_linux.sh
diff --git a/tests/worker/connect_multithread/setup_windows.bat b/tests/automatic/worker/connect_multithread/setup_windows.bat
similarity index 100%
rename from tests/worker/connect_multithread/setup_windows.bat
rename to tests/automatic/worker/connect_multithread/setup_windows.bat
diff --git a/tests/worker/folder_to_db/CMakeLists.txt b/tests/automatic/worker/folder_to_db/CMakeLists.txt
similarity index 100%
rename from tests/worker/folder_to_db/CMakeLists.txt
rename to tests/automatic/worker/folder_to_db/CMakeLists.txt
diff --git a/tests/worker/folder_to_db/check_linux.sh b/tests/automatic/worker/folder_to_db/check_linux.sh
similarity index 100%
rename from tests/worker/folder_to_db/check_linux.sh
rename to tests/automatic/worker/folder_to_db/check_linux.sh
diff --git a/tests/worker/folder_to_db/check_windows.bat b/tests/automatic/worker/folder_to_db/check_windows.bat
similarity index 100%
rename from tests/worker/folder_to_db/check_windows.bat
rename to tests/automatic/worker/folder_to_db/check_windows.bat
diff --git a/tests/worker/next_multithread_broker/CMakeLists.txt b/tests/automatic/worker/next_multithread_broker/CMakeLists.txt
similarity index 69%
rename from tests/worker/next_multithread_broker/CMakeLists.txt
rename to tests/automatic/worker/next_multithread_broker/CMakeLists.txt
index e97403117f3b1c7e2a9f5708484ec6b71831c069..9c22693a5cbb40f0778ce0ff145ef428fc3e1d8f 100644
--- a/tests/worker/next_multithread_broker/CMakeLists.txt
+++ b/tests/automatic/worker/next_multithread_broker/CMakeLists.txt
@@ -6,12 +6,12 @@ set(SOURCE_FILES next_multithread_broker.cpp)
# Executable and link
################################
add_executable(${TARGET_NAME} ${SOURCE_FILES})
-target_link_libraries(${TARGET_NAME} test_common hidra2-worker ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET_NAME} test_common hidra2-worker)
################################
# Testing
################################
-configure_file(${CMAKE_SOURCE_DIR}/tests/settings/broker_settings.json settings.json COPYONLY)
+configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/broker_settings.json settings.json COPYONLY)
add_script_test("${TARGET_NAME}" "$<TARGET_FILE:${TARGET_NAME}> $<TARGET_PROPERTY:hidra2-broker,EXENAME>"
)
diff --git a/tests/worker/next_multithread_broker/check_linux.sh b/tests/automatic/worker/next_multithread_broker/check_linux.sh
similarity index 100%
rename from tests/worker/next_multithread_broker/check_linux.sh
rename to tests/automatic/worker/next_multithread_broker/check_linux.sh
diff --git a/tests/worker/next_multithread_broker/check_windows.bat b/tests/automatic/worker/next_multithread_broker/check_windows.bat
similarity index 100%
rename from tests/worker/next_multithread_broker/check_windows.bat
rename to tests/automatic/worker/next_multithread_broker/check_windows.bat
diff --git a/tests/worker/next_multithread_broker/next_multithread_broker.cpp b/tests/automatic/worker/next_multithread_broker/next_multithread_broker.cpp
similarity index 100%
rename from tests/worker/next_multithread_broker/next_multithread_broker.cpp
rename to tests/automatic/worker/next_multithread_broker/next_multithread_broker.cpp
diff --git a/tests/worker/next_multithread_folder/CMakeLists.txt b/tests/automatic/worker/next_multithread_folder/CMakeLists.txt
similarity index 91%
rename from tests/worker/next_multithread_folder/CMakeLists.txt
rename to tests/automatic/worker/next_multithread_folder/CMakeLists.txt
index eed3cc9b92840d9f2fe0aebda644df863acb553a..b0920c971f868faacd93ed7fbd60e630651f87c4 100644
--- a/tests/worker/next_multithread_folder/CMakeLists.txt
+++ b/tests/automatic/worker/next_multithread_folder/CMakeLists.txt
@@ -12,7 +12,6 @@ GET_PROPERTY(HIDRA2_COMMON_IO_LIBRARIES GLOBAL PROPERTY HIDRA2_COMMON_IO_LIBRARI
target_link_libraries(${TARGET_NAME} ${HIDRA2_COMMON_IO_LIBRARIES})
target_link_libraries(${TARGET_NAME} test_common hidra2-worker)
-set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
if (CMAKE_COMPILER_IS_GNUCXX)
set_target_properties(${TARGET_NAME} PROPERTIES LINK_FLAGS_DEBUG "--coverage")
endif()
diff --git a/tests/worker/next_multithread_folder/cleanup_linux.sh b/tests/automatic/worker/next_multithread_folder/cleanup_linux.sh
similarity index 100%
rename from tests/worker/next_multithread_folder/cleanup_linux.sh
rename to tests/automatic/worker/next_multithread_folder/cleanup_linux.sh
diff --git a/tests/worker/next_multithread_folder/cleanup_windows.bat b/tests/automatic/worker/next_multithread_folder/cleanup_windows.bat
similarity index 100%
rename from tests/worker/next_multithread_folder/cleanup_windows.bat
rename to tests/automatic/worker/next_multithread_folder/cleanup_windows.bat
diff --git a/tests/worker/next_multithread_folder/next_multithread_folder.cpp b/tests/automatic/worker/next_multithread_folder/next_multithread_folder.cpp
similarity index 100%
rename from tests/worker/next_multithread_folder/next_multithread_folder.cpp
rename to tests/automatic/worker/next_multithread_folder/next_multithread_folder.cpp
diff --git a/tests/worker/next_multithread_folder/setup_linux.sh b/tests/automatic/worker/next_multithread_folder/setup_linux.sh
similarity index 100%
rename from tests/worker/next_multithread_folder/setup_linux.sh
rename to tests/automatic/worker/next_multithread_folder/setup_linux.sh
diff --git a/tests/worker/next_multithread_folder/setup_windows.bat b/tests/automatic/worker/next_multithread_folder/setup_windows.bat
similarity index 100%
rename from tests/worker/next_multithread_folder/setup_windows.bat
rename to tests/automatic/worker/next_multithread_folder/setup_windows.bat
diff --git a/tests/broker/read_config/settings_bad.json b/tests/broker/read_config/settings_bad.json
deleted file mode 100644
index 01598594a92d5cbcd01cacc8e8c27e27cf2c42ea..0000000000000000000000000000000000000000
--- a/tests/broker/read_config/settings_bad.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "dbaddress":"127.0.0.1:27017",
- "port":
-}
\ No newline at end of file
diff --git a/tests/manual/performance_broker/settings.json b/tests/manual/performance_broker/settings.json
new file mode 100644
index 0000000000000000000000000000000000000000..c45d16f2f7b59b7966ad9d2d406ef530da720a2b
--- /dev/null
+++ b/tests/manual/performance_broker/settings.json
@@ -0,0 +1,6 @@
+{
+ "BrokerDbAddress":"localhost:27017",
+ "MonitorDbAddress": "localhost:8086",
+ "MonitorDbName": "db_test",
+ "port":5005
+}
\ No newline at end of file
diff --git a/tests/manual/performance_broker/test.sh b/tests/manual/performance_broker/test.sh
new file mode 100755
index 0000000000000000000000000000000000000000..3083e4e001d2898a99d52ac368b0a66e02ac4cad
--- /dev/null
+++ b/tests/manual/performance_broker/test.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# starts broker, mongodb on $service_node
+# reads fileset into database
+# calls getnext_broker example from $worker_node
+
+nthreads=16
+# a directory with many files in it
+dir=/gpfs/petra3/scratch/yakubov/test
+run_name=test
+service_node=max-wgs
+
+monitor_node=zitpcx27016
+monitor_port=8086
+
+worker_node=max-display
+#worker_node=max-wgs
+
+worker_dir=~/broker_test
+service_dir=~/broker_test
+
+
+cat settings.json |
+ jq "to_entries |
+ map(if .key == \"MonitorDbAddress\"
+ then . + {value:\"${monitor_node}:${monitor_port}\"}
+ else .
+ end
+ ) |
+ from_entries" > settings_tmp.json
+
+ssh ${monitor_node} influx -execute \"create database db_test\"
+
+ssh ${service_node} docker run -d -p 27017:27017 --name mongo mongo
+#ssh ${service_node} docker run -d -p 8086 -p 8086 --name influxdb influxdb
+
+ssh ${service_node} mkdir ${service_dir}
+ssh ${worker_node} mkdir ${worker_dir}
+
+
+scp settings_tmp.json ${service_node}:${service_dir}/settings.json
+rm settings_tmp.json
+scp ../../../cmake-build-release/broker/hidra2-broker ${service_node}:${service_dir}
+ssh ${service_node} "bash -c 'cd ${service_dir}; nohup ./hidra2-broker settings.json &> ${service_dir}/broker.log &'"
+sleep 0.3
+scp ../../../cmake-build-release/worker/tools/folder_to_db/folder2db ${worker_node}:${worker_dir}
+ssh ${worker_node} ${worker_dir}/folder2db -n ${nthreads} ${dir} ${run_name} ${service_node}
+
+sleep 3
+
+scp ../../../cmake-build-release/examples/worker/getnext_broker/getnext_broker ${worker_node}:${worker_dir}
+ssh ${worker_node} ${worker_dir}/getnext_broker ${service_node}:5005 ${run_name} ${nthreads}
+
+
+
+ssh ${service_node} killall hidra2-broker
+ssh ${service_node} docker rm -f mongo
+#ssh ${service_node} docker rm -f influxdb
diff --git a/tests/manual/performance_producer_receiver/receiver.json b/tests/manual/performance_producer_receiver/receiver.json
new file mode 100644
index 0000000000000000000000000000000000000000..e23a556d1cd5a8c03ce0c32b37b7b73161bfa89d
--- /dev/null
+++ b/tests/manual/performance_producer_receiver/receiver.json
@@ -0,0 +1,6 @@
+{
+ "MonitorDbAddress":"localhost:8086",
+ "MonitorDbName": "db_test",
+ "ListenPort":4200,
+ "WriteToDisk":true
+}
\ No newline at end of file
diff --git a/tests/manual/performance_producer_receiver/settings_tmp.json b/tests/manual/performance_producer_receiver/settings_tmp.json
new file mode 100644
index 0000000000000000000000000000000000000000..35b3649bcb182f9b0121d9e85cac59e9c710e679
--- /dev/null
+++ b/tests/manual/performance_producer_receiver/settings_tmp.json
@@ -0,0 +1,6 @@
+{
+ "MonitorDbAddress": "zitpcx27016:8086",
+ "MonitorDbName": "db_test",
+ "ListenPort": 4201,
+ "WriteToDisk": false
+}
diff --git a/tests/manual/performance_producer_receiver/test.sh b/tests/manual/performance_producer_receiver/test.sh
new file mode 100755
index 0000000000000000000000000000000000000000..317f32d77331e639f2d1ed564fd5ba4fc31aa70c
--- /dev/null
+++ b/tests/manual/performance_producer_receiver/test.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+set -e
+
+# starts receiver on $service_node
+# runs producer with various file sizes from $worker_node and measures performance
+
+# a working directory
+service_node=max-wgs
+service_ip=`resolveip -s ${service_node}`
+service_port=4201
+
+monitor_node=zitpcx27016
+monitor_port=8086
+
+worker_node=max-display
+#worker_node=max-wgs
+
+worker_dir=~/producer_tests
+service_dir=/gpfs/petra3/scratch/yakubov/receiver_tests
+
+ssh ${monitor_node} influx -execute \"create database db_test\"
+
+#ssh ${monitor_node} docker run -d -p 8086 -p 8086 --name influxdb influxdb
+
+ssh ${service_node} mkdir -p ${service_dir}
+ssh ${service_node} mkdir -p ${service_dir}/files
+ssh ${worker_node} mkdir -p ${worker_dir}
+
+scp ../../../cmake-build-release/receiver/receiver ${service_node}:${service_dir}
+scp ../../../cmake-build-release/examples/producer/dummy-data-producer/dummy-data-producer ${worker_node}:${worker_dir}
+
+
+function do_work {
+cat receiver.json |
+ jq "to_entries |
+ map(if .key == \"MonitorDbAddress\"
+ then . + {value:\"${monitor_node}:${monitor_port}\"}
+ elif .key == \"ListenPort\"
+ then . + {value:${service_port}}
+ elif .key == \"WriteToDisk\"
+ then . + {value:$1}
+ else .
+ end
+ ) |
+ from_entries" > settings_tmp.json
+scp settings_tmp.json ${service_node}:${service_dir}/settings.json
+ssh ${service_node} "bash -c 'cd ${service_dir}; nohup ./receiver settings.json &> ${service_dir}/receiver.log &'"
+sleep 0.3
+for size in 100 1000 10000
+do
+echo ===================================================================
+ssh ${worker_node} ${worker_dir}/dummy-data-producer ${service_ip}:${service_port} ${size} 1000
+ssh ${service_node} rm -f ${service_dir}/files/*
+done
+ssh ${service_node} killall receiver
+}
+
+echo
+echo "With write to disk:"
+do_work true
+
+echo
+echo "Without write to disk:"
+do_work false
+
+
+#rm settings_tmp.json
+#ssh ${service_node} docker rm -f influxdb
diff --git a/tests/producer_receiver/CMakeLists.txt b/tests/producer_receiver/CMakeLists.txt
deleted file mode 100644
index c4da20d000bc7375e799fcd9b063a640d75c506a..0000000000000000000000000000000000000000
--- a/tests/producer_receiver/CMakeLists.txt
+++ /dev/null
@@ -1 +0,0 @@
-add_subdirectory(transfer_single_file)
diff --git a/tests/settings/broker_settings.json b/tests/settings/broker_settings.json
deleted file mode 100644
index 841f1994677df8ab075b7e71859991527f438b67..0000000000000000000000000000000000000000
--- a/tests/settings/broker_settings.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "dbaddress":"127.0.0.1:27017",
- "port":5005
-}
\ No newline at end of file
diff --git a/tests/worker/curl_http_client_get/CMakeLists.txt b/tests/worker/curl_http_client_get/CMakeLists.txt
deleted file mode 100644
index e233d8a7893f9126dfd81a27a739936d11be1f6b..0000000000000000000000000000000000000000
--- a/tests/worker/curl_http_client_get/CMakeLists.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-set(TARGET_NAME curl_httpclient_get)
-set(SOURCE_FILES curl_httpclient_get.cpp)
-
-
-################################
-# Executable and link
-################################
-add_executable(${TARGET_NAME} ${SOURCE_FILES})
-target_link_libraries(${TARGET_NAME} test_common hidra2-worker)
-
-#set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
-#if (CMAKE_COMPILER_IS_GNUCXX)
-# set_target_properties(${TARGET_NAME} PROPERTIES LINK_FLAGS_DEBUG "--coverage")
-#endif()
-
-
-################################
-# Testing
-################################
-
-#add_test_setup_cleanup(${TARGET_NAME})
-add_integration_test(${TARGET_NAME} get_google "google.com moved 302")
-add_integration_test(${TARGET_NAME} get_badaddress "google.com/badaddress found 404")
-add_integration_test(${TARGET_NAME} get_badaddress2 "111 clienterror 404")
-
diff --git a/worker/api/cpp/CMakeLists.txt b/worker/api/cpp/CMakeLists.txt
index 914d87840dff2058631c83a683401fb10bf9f959..f60a3268647407b76e3c858f0baaa73ec59df718 100644
--- a/worker/api/cpp/CMakeLists.txt
+++ b/worker/api/cpp/CMakeLists.txt
@@ -2,33 +2,23 @@ set(TARGET_NAME hidra2-worker)
set(SOURCE_FILES
src/data_broker.cpp
- src/http_client.cpp
src/folder_data_broker.cpp
src/server_data_broker.cpp
- src/curl_http_client.cpp
- src/http_error.cpp)
+)
################################
# Library
################################
add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io>
- $<TARGET_OBJECTS:json_parser> $<TARGET_OBJECTS:data_structs> )
-
-set (CMAKE_PREFIX_PATH "${LIBCURL_DIR}")
-find_package (CURL REQUIRED)
-message (STATUS "Found cURL libraries: ${CURL_LIBRARIES}")
-message (STATUS "cURL include: ${CURL_INCLUDE_DIRS}")
-
+ $<TARGET_OBJECTS:json_parser> $<TARGET_OBJECTS:data_structs> $<TARGET_OBJECTS:curl_http_client> )
target_include_directories(${TARGET_NAME} PUBLIC include ${HIDRA2_CXX_COMMON_INCLUDE_DIR}
${CURL_INCLUDE_DIRS})
-IF(WIN32)
- target_compile_definitions(${TARGET_NAME} PUBLIC -DCURL_STATICLIB)
-ELSEIF(CMAKE_C_COMPILER_ID STREQUAL "GNU")
+IF(CMAKE_C_COMPILER_ID STREQUAL "GNU")
SET( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
-ENDIF(WIN32)
+ENDIF()
target_link_libraries(${TARGET_NAME} ${CURL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
diff --git a/worker/api/cpp/src/http_client.cpp b/worker/api/cpp/src/http_client.cpp
deleted file mode 100644
index d5c86c6c07343e163714d36a2b22053138bec407..0000000000000000000000000000000000000000
--- a/worker/api/cpp/src/http_client.cpp
+++ /dev/null
@@ -1,29 +0,0 @@
-#include "http_client.h"
-#include "http_error.h"
-namespace hidra2 {
-
-Error HttpCodeToWorkerError(const HttpCode& code) {
- const char* message;
- switch (code) {
- case HttpCode::OK:
- return nullptr;
- case HttpCode::BadRequest:
- message = WorkerErrorMessage::kWrongInput;
- break;
- case HttpCode::InternalServerError:
- message = WorkerErrorMessage::kErrorReadingSource;
- break;
- case HttpCode::NoContent:
- message = WorkerErrorMessage::kNoData;
- return TextErrorWithType(message, ErrorType::kEndOfFile);
- case HttpCode::NotFound:
- message = WorkerErrorMessage::kSourceNotFound;
- break;
- default:
- message = WorkerErrorMessage::kErrorReadingSource;
- break;
- }
- return Error{new HttpError(message, code)};
-}
-
-}
diff --git a/worker/api/cpp/src/http_error.cpp b/worker/api/cpp/src/http_error.cpp
deleted file mode 100644
index 72189fc37a93b34175987bac04642cc283b570c3..0000000000000000000000000000000000000000
--- a/worker/api/cpp/src/http_error.cpp
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "http_error.h"
-
-namespace hidra2 {
-
-
-
-}
\ No newline at end of file
diff --git a/worker/api/cpp/src/server_data_broker.cpp b/worker/api/cpp/src/server_data_broker.cpp
index 0298c82db4d538a0c0202ff7efb82fa10b16752a..0b739e6e7d720e713c91a8623ea9153d08f4678a 100644
--- a/worker/api/cpp/src/server_data_broker.cpp
+++ b/worker/api/cpp/src/server_data_broker.cpp
@@ -1,9 +1,37 @@
#include "server_data_broker.h"
#include "io/io_factory.h"
-#include "curl_http_client.h"
+#include "http_client/curl_http_client.h"
+#include "http_client/http_error.h"
+
namespace hidra2 {
+Error HttpCodeToWorkerError(const HttpCode& code) {
+ const char* message;
+ switch (code) {
+ case HttpCode::OK:
+ return nullptr;
+ case HttpCode::BadRequest:
+ message = WorkerErrorMessage::kWrongInput;
+ break;
+ case HttpCode::InternalServerError:
+ message = WorkerErrorMessage::kErrorReadingSource;
+ break;
+ case HttpCode::NoContent:
+ message = WorkerErrorMessage::kNoData;
+ return TextErrorWithType(message, ErrorType::kEndOfFile);
+ case HttpCode::NotFound:
+ message = WorkerErrorMessage::kSourceNotFound;
+ break;
+ default:
+ message = WorkerErrorMessage::kErrorReadingSource;
+ break;
+ }
+ return Error{new HttpError(message, code)};
+}
+
+
+
ServerDataBroker::ServerDataBroker(const std::string& server_uri,
const std::string& source_name):
io__{GenerateDefaultIO()}, httpclient__{new CurlHttpClient},
diff --git a/worker/api/cpp/src/server_data_broker.h b/worker/api/cpp/src/server_data_broker.h
index 2081261b832c9884e8c762983f58c43adf91579e..5be1b4236e818fa303cde1829811219bc7e099d9 100644
--- a/worker/api/cpp/src/server_data_broker.h
+++ b/worker/api/cpp/src/server_data_broker.h
@@ -3,18 +3,20 @@
#include "worker/data_broker.h"
#include "io/io.h"
-#include "http_client.h"
+#include "http_client/http_client.h"
namespace hidra2 {
+Error HttpCodeToWorkerError(const HttpCode& code);
+
class ServerDataBroker final : public hidra2::DataBroker {
public:
explicit ServerDataBroker(const std::string& server_uri, const std::string& source_name);
Error Connect() override;
Error GetNext(FileInfo* info, FileData* data) override;
- std::unique_ptr<hidra2::IO> io__; // modified in testings to mock system calls,otherwise do not touch
- std::unique_ptr<hidra2::HttpClient> httpclient__;
+ std::unique_ptr<IO> io__; // modified in testings to mock system calls,otherwise do not touch
+ std::unique_ptr<HttpClient> httpclient__;
private:
Error GetFileInfoFromServer(FileInfo* info, const std::string& operation);
std::string server_uri_;
diff --git a/worker/api/cpp/unittests/MockHttpClient.h b/worker/api/cpp/unittests/MockHttpClient.h
deleted file mode 100644
index a386321db331ea9549646faff4fa4302c17e5434..0000000000000000000000000000000000000000
--- a/worker/api/cpp/unittests/MockHttpClient.h
+++ /dev/null
@@ -1,25 +0,0 @@
-#ifndef HIDRA2_MOCKHTTPCLIENT_H
-#define HIDRA2_MOCKHTTPCLIENT_H
-
-#include <gtest/gtest.h>
-#include <gmock/gmock.h>
-
-#include "../src/http_client.h"
-#include "worker/data_broker.h"
-
-
-namespace hidra2 {
-
-class MockHttpClient : public HttpClient {
- public:
- std::string Get(const std::string& uri, HttpCode* code, Error* err) const noexcept override {
- return Get_t(uri, code, err);
- }
- MOCK_CONST_METHOD3(Get_t,
- std::string(const std::string& uri, HttpCode* code, Error* err));
-};
-
-
-}
-
-#endif //HIDRA2_MOCKHTTPCLIENT_H
diff --git a/worker/api/cpp/unittests/test_server_broker.cpp b/worker/api/cpp/unittests/test_server_broker.cpp
index 48296c46fdb53baa0cdca421aa692b6e22c34de8..7987769fdd31e72e715f1959375ec806525e94aa 100644
--- a/worker/api/cpp/unittests/test_server_broker.cpp
+++ b/worker/api/cpp/unittests/test_server_broker.cpp
@@ -5,10 +5,10 @@
#include "io/io.h"
#include "../../../../common/cpp/src/system_io/system_io.h"
#include "../src/server_data_broker.h"
-#include "../src/curl_http_client.h"
+#include "http_client/curl_http_client.h"
#include "unittests/MockIO.h"
-#include "MockHttpClient.h"
-#include "../src/http_error.h"
+#include "unittests/MockHttpClient.h"
+#include "http_client/http_error.h"
using hidra2::DataBrokerFactory;
using hidra2::DataBroker;