diff --git a/CMakeModules/testing_go.cmake b/CMakeModules/testing_go.cmake index 13c133adaf72d3143aa5f7b76a3e80d53dea5cb0..f47e68e66d4c4aae09e89f725f4a7651bfcd4a2a 100644 --- a/CMakeModules/testing_go.cmake +++ b/CMakeModules/testing_go.cmake @@ -27,3 +27,16 @@ function(gotest target test_source_files) endif () endfunction() +function(go_integration_test target test_source_files label) + if (BUILD_TESTS) + add_test(NAME test-${target} COMMAND go test ${test_source_files} -run ${label} + -tags integration_tests + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + set_property( + TEST + test-${target} + PROPERTY + ENVIRONMENT GOPATH=${gopath}) + message(STATUS "Added test 'test-${target}'") + endif () +endfunction() diff --git a/broker/CMakeLists.txt b/broker/CMakeLists.txt index c116442f05ab26dbc79c8dc558094f576dc83a5d..f017083c263adaf11b345feb486045135912b610 100644 --- a/broker/CMakeLists.txt +++ b/broker/CMakeLists.txt @@ -11,4 +11,5 @@ add_custom_target(hidra2-broker ALL install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} DESTINATION bin) -gotest(${TARGET_NAME} "./...") \ No newline at end of file +gotest(${TARGET_NAME} "./...") +go_integration_test(${TARGET_NAME}-connectdb "./..." "MongoDBConnect") diff --git a/broker/bin/main b/broker/bin/main new file mode 100755 index 0000000000000000000000000000000000000000..60fc519df924ef0d6eae045e88ff9b2cffe47632 Binary files /dev/null and b/broker/bin/main differ diff --git a/broker/src/hidra2_broker/database/database.go b/broker/src/hidra2_broker/database/database.go index bf217210619bafc6a66a691b7b5c8c1231f51692..0ae98da857eae82e154d1a61d81207dbe4b496ec 100644 --- a/broker/src/hidra2_broker/database/database.go +++ b/broker/src/hidra2_broker/database/database.go @@ -1,12 +1,7 @@ package database -import "fmt" - type Agent interface { + GetNextRecord(db_name string, collection_name string) (answer []byte, ok bool) Connect(string) error Close() } - -func Test_Hidra2() { - fmt.Println("aaa") -} diff --git a/broker/src/hidra2_broker/database/database_test.go b/broker/src/hidra2_broker/database/database_test.go index 2da556d137b0eea209c53d308df3fdbf41daa950..fbe6ac907cea91e3c1106f57fae5ed55cbc3e731 100644 --- a/broker/src/hidra2_broker/database/database_test.go +++ b/broker/src/hidra2_broker/database/database_test.go @@ -1,22 +1,18 @@ package database import ( - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "testing" ) -func TestCreateRecord(t *testing.T) { - Test_Hidra2() - assert.Equal(t, "111", "111", "record created") -} - // we run this test just to get 100% coverage for mock_database.go func TestMockDataBase(t *testing.T) { var db MockedDatabase db.On("Connect", mock.AnythingOfType("string")).Return(nil) db.On("Close").Return() + db.On("GetNextRecord", "", "").Return([]byte(""), false) db.Connect("") + db.GetNextRecord("", "") db.Close() } diff --git a/broker/src/hidra2_broker/database/mock_database.go b/broker/src/hidra2_broker/database/mock_database.go index 5250c5d853acaa593a28552505b2776bd5d607d1..7d318c942fe850915998c06c03c99fbabd21bf7c 100644 --- a/broker/src/hidra2_broker/database/mock_database.go +++ b/broker/src/hidra2_broker/database/mock_database.go @@ -18,3 +18,8 @@ func (db *MockedDatabase) Connect(address string) error { func (db *MockedDatabase) Close() { db.Called() } + +func (db *MockedDatabase) GetNextRecord(db_name string, collection_name string) (answer []byte, ok bool) { + args := db.Called(db_name, collection_name) + return args.Get(0).([]byte), args.Bool(1) +} diff --git a/broker/src/hidra2_broker/database/mongodb.go b/broker/src/hidra2_broker/database/mongodb.go index 0e67e482d01b78ac471fb611242bbd5fde9e4592..e3eb944be18092d1b480f844e161ba2b1542fbe9 100644 --- a/broker/src/hidra2_broker/database/mongodb.go +++ b/broker/src/hidra2_broker/database/mongodb.go @@ -13,8 +13,9 @@ type Mongodb struct { timeout time.Duration } -func (db *Mongodb) Connect(string) error { +func (db *Mongodb) Connect(address string) error { var err error + db.main_session, err = mgo.DialWithTimeout(address, time.Second) return err } @@ -24,3 +25,7 @@ func (db *Mongodb) Close() { } } + +func (db *Mongodb) GetNextRecord(db_name string, collection_name string) (answer []byte, ok bool) { + return nil, true +} diff --git a/broker/src/hidra2_broker/database/mongodb_test.go b/broker/src/hidra2_broker/database/mongodb_test.go new file mode 100644 index 0000000000000000000000000000000000000000..5b91334cdfef29f4402bbe930de46468f8e5f685 --- /dev/null +++ b/broker/src/hidra2_broker/database/mongodb_test.go @@ -0,0 +1,24 @@ +// +build integration_tests + +package database + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +// these are tjhe integration tests. They assume mongo db is runnig on 127.0.0.1:27027 +// test names shlud contain MongoDB*** so that go test could find them: +// go_integration_test(${TARGET_NAME}-connectdb "./..." "MongoDBConnect") +func TestMongoDBConnectFails(t *testing.T) { + var db Mongodb + err := db.Connect("blabla") + assert.NotNil(t, err) +} + +func TestMongoDBConnectOK(t *testing.T) { + var db Mongodb + err := db.Connect("127.0.0.1:27017") + assert.Nil(t, err) + db.Close() +} diff --git a/broker/src/hidra2_broker/main/broker.go b/broker/src/hidra2_broker/main/broker.go index f7462384fa087eccaefa9a28d0f6400c9eecf41e..5b35622e0d3ba77261ad3d6e5fdc9d0c61bc90da 100644 --- a/broker/src/hidra2_broker/main/broker.go +++ b/broker/src/hidra2_broker/main/broker.go @@ -5,17 +5,15 @@ package main import ( "hidra2_broker/database" "hidra2_broker/server" + "log" ) func NewDefaultDatabase() database.Agent { return new(database.Mongodb) } -// global variable since we only have one instance -var srv server.Server - func main() { - srv.InitDB(NewDefaultDatabase()) - defer srv.CleanupDB() - srv.Start() + log.Fatal(server.InitDB(NewDefaultDatabase())) + defer server.CleanupDB() + server.Start() } diff --git a/broker/src/hidra2_broker/server/get_next.go b/broker/src/hidra2_broker/server/get_next.go index 3af29a08a778d3830885ac36a8c7c6c2958edb57..83849163a16b87d137b44f712a792a7d7685c352 100644 --- a/broker/src/hidra2_broker/server/get_next.go +++ b/broker/src/hidra2_broker/server/get_next.go @@ -4,7 +4,29 @@ import ( "net/http" ) +func extractRequestParameters(r *http.Request) (string, bool) { + db_name := r.URL.Query().Get("database") + return db_name, db_name != "" +} + func routeGetNext(w http.ResponseWriter, r *http.Request) { r.Header.Set("Content-type", "application/json") - w.WriteHeader(http.StatusOK) + + db_name, ok := extractRequestParameters(r) + if !ok { + w.WriteHeader(http.StatusBadRequest) + return + } + + answer, ok := getNextRecord(db_name) + if !ok { + w.WriteHeader(http.StatusBadRequest) + } else { + w.WriteHeader(http.StatusOK) + } + w.Write(answer) +} + +func getNextRecord(db_name string) (answer []byte, ok bool) { + return db.GetNextRecord(db_name, "data") } diff --git a/broker/src/hidra2_broker/server/get_next_test.go b/broker/src/hidra2_broker/server/get_next_test.go index a6d0feff9178e03ccbcfd8f93238d2e72f06ce9f..cc0d8f4c79a5b48287ab8054d2dffbf6d78f9089 100644 --- a/broker/src/hidra2_broker/server/get_next_test.go +++ b/broker/src/hidra2_broker/server/get_next_test.go @@ -2,6 +2,7 @@ package server import ( "github.com/stretchr/testify/assert" + "hidra2_broker/database" "hidra2_broker/utils" "net/http" "net/http/httptest" @@ -15,21 +16,39 @@ type request struct { message string } -var getNextTests = []request{ - {"next", "GET", http.StatusOK, "get next job"}, +func doRequest(path string) *httptest.ResponseRecorder { + mux := utils.NewRouter(listRoutes) + req, _ := http.NewRequest("GET", path, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + return w } -func TestGetNext(t *testing.T) { - mux := utils.NewRouter(listRoutes) +func TestGetNextWithoutDatabaseName(t *testing.T) { + w := doRequest("/next") + assert.Equal(t, http.StatusBadRequest, w.Code, "no database name") +} - for _, test := range getNextTests { +func TestGetNextWithWrongDatabaseName(t *testing.T) { + mock_db := new(database.MockedDatabase) + db = mock_db + defer func() { db = nil }() + mock_db.On("GetNextRecord", "foo", "data").Return([]byte(""), false) + + w := doRequest("/next?database=foo") + assert.Equal(t, http.StatusBadRequest, w.Code, "no database name") + assertExpectations(t, mock_db) +} - req, err := http.NewRequest(test.cmd, "/"+test.path+"/", nil) +func TestGetNextWithGoodDatabaseName(t *testing.T) { + mock_db := new(database.MockedDatabase) + db = mock_db + defer func() { db = nil }() + mock_db.On("GetNextRecord", "database", "data").Return([]byte("Hello"), true) - assert.Nil(t, err, "Should not be error") + w := doRequest("/next?database=database") + assert.Equal(t, http.StatusOK, w.Code, "GetNext OK") + assert.Equal(t, "Hello", string(w.Body.Bytes()), "GetNext sends data") + assertExpectations(t, mock_db) - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - assert.Equal(t, test.answer, w.Code, test.message) - } } diff --git a/broker/src/hidra2_broker/server/listroutes.go b/broker/src/hidra2_broker/server/listroutes.go index b50c1f514bce4a0a9d39117bfed3df71e61d4717..86baeb470e3f028cbd68c5f7719f0bdee20eec51 100644 --- a/broker/src/hidra2_broker/server/listroutes.go +++ b/broker/src/hidra2_broker/server/listroutes.go @@ -8,7 +8,7 @@ var listRoutes = utils.Routes{ utils.Route{ "GetNext", "Get", - "/next/", + "/next", routeGetNext, }, } diff --git a/broker/src/hidra2_broker/server/server.go b/broker/src/hidra2_broker/server/server.go index 1157143124fe6c6952698c32ec341c89750c4043..f369c648044a83808910a949898c8669e7162a90 100644 --- a/broker/src/hidra2_broker/server/server.go +++ b/broker/src/hidra2_broker/server/server.go @@ -4,17 +4,15 @@ import ( "hidra2_broker/database" ) -type Server struct { - db database.Agent -} +var db database.Agent -func (srv *Server) InitDB(db database.Agent) error { - srv.db = db - return srv.db.Connect("127.0.0.1:27017") +func InitDB(dbAgent database.Agent) error { + db = dbAgent + return db.Connect("127.0.0.1:27017") } -func (srv *Server) CleanupDB() { - if srv.db != nil { - srv.db.Close() +func CleanupDB() { + if db != nil { + db.Close() } } diff --git a/broker/src/hidra2_broker/server/server_nottested.go b/broker/src/hidra2_broker/server/server_nottested.go index 4b05ad004d701361ed9e4ff124d838b73247de8a..640e6d9b8727639fef5ff15d6994b7ac42a3b041 100644 --- a/broker/src/hidra2_broker/server/server_nottested.go +++ b/broker/src/hidra2_broker/server/server_nottested.go @@ -8,7 +8,7 @@ import ( "net/http" ) -func (srv *Server) Start() { +func Start() { mux := utils.NewRouter(listRoutes) log.Fatal(http.ListenAndServe("127.0.0.1:5005", http.HandlerFunc(mux.ServeHTTP))) } diff --git a/broker/src/hidra2_broker/server/server_test.go b/broker/src/hidra2_broker/server/server_test.go index e75fc91b5011bfc15aacb4dc100d252168a48e04..52c835ad01c52963ed0cefd91a90d72ac2095a9a 100644 --- a/broker/src/hidra2_broker/server/server_test.go +++ b/broker/src/hidra2_broker/server/server_test.go @@ -8,17 +8,16 @@ import ( "testing" ) -func setup() (*database.MockedDatabase, *Server) { - db := new(database.MockedDatabase) - srv := new(Server) - db.On("Connect", mock.AnythingOfType("string")).Return(nil) - return db, srv +func setup() *database.MockedDatabase { + mock_db := new(database.MockedDatabase) + mock_db.On("Connect", mock.AnythingOfType("string")).Return(nil) + return mock_db } -func assertExpectations(t *testing.T, db *database.MockedDatabase) { - db.AssertExpectations(t) - db.ExpectedCalls = nil +func assertExpectations(t *testing.T, mock_db *database.MockedDatabase) { + mock_db.AssertExpectations(t) + mock_db.ExpectedCalls = nil } var initDBTests = []struct { @@ -31,34 +30,36 @@ var initDBTests = []struct { } func TestInitDBWithWrongAddress(t *testing.T) { - db, srv := setup() - db.ExpectedCalls = nil + mock_db := setup() + + mock_db.ExpectedCalls = nil for _, test := range initDBTests { - db.On("Connect", mock.AnythingOfType("string")).Return(test.answer) + mock_db.On("Connect", mock.AnythingOfType("string")).Return(test.answer) - err := srv.InitDB(db) + err := InitDB(mock_db) assert.Equal(t, test.answer, err, test.message) - assertExpectations(t, db) + assertExpectations(t, mock_db) } + db = nil } func TestCleanupDBWithoutInit(t *testing.T) { - db, srv := setup() + mock_db := setup() - db.AssertNotCalled(t, "Close") + mock_db.AssertNotCalled(t, "Close") - srv.CleanupDB() + CleanupDB() } func TestCleanupDBInit(t *testing.T) { - db, srv := setup() + mock_db := setup() - db.On("Close").Return() + mock_db.On("Close").Return() - srv.InitDB(db) - srv.CleanupDB() + InitDB(mock_db) + CleanupDB() - assertExpectations(t, db) + assertExpectations(t, mock_db) } diff --git a/common/cpp/include/common/file_info.h b/common/cpp/include/common/file_info.h index 2d264128707bfcc3cd36e0a16d473efdd4919f92..59fd134f7a8e86e57409ae11c73cfa657dcfb508 100644 --- a/common/cpp/include/common/file_info.h +++ b/common/cpp/include/common/file_info.h @@ -16,11 +16,12 @@ struct FileInfo { uint64_t size{0}; uint64_t id{0}; std::string Json() const { - auto periods = modify_date.time_since_epoch().count(); + auto nanoseconds_from_epoch = std::chrono::time_point_cast<std::chrono::nanoseconds>(modify_date). + time_since_epoch().count(); std::string s = "{\"_id\":" + std::to_string(id) + "," "\"size\":" + std::to_string(size) + "," "\"base_name\":\"" + base_name + "\"," - "\"lastchange\":" + std::to_string(periods) + "," + "\"lastchange\":" + std::to_string(nanoseconds_from_epoch) + "," "\"relative_path\":\"" + relative_path + "\"}"; return s; } diff --git a/worker/tools/folder_to_db/src/folder_db_importer.cpp b/worker/tools/folder_to_db/src/folder_db_importer.cpp index 26311936ab064d9633d2b3d4c8ff4db25531bad1..f7b5f10274a39dfb1096d49a3f5e14cf237c1e8b 100644 --- a/worker/tools/folder_to_db/src/folder_db_importer.cpp +++ b/worker/tools/folder_to_db/src/folder_db_importer.cpp @@ -100,7 +100,7 @@ std::unique_ptr<hidra2::Database> FolderToDbImporter::CreateDbClient(FolderToDbI return db; } -FolderToDbImportError WaitParallelTasks(std::vector<std::future<FolderToDbImportError>>* res){ +FolderToDbImportError WaitParallelTasks(std::vector<std::future<FolderToDbImportError>>* res) { FolderToDbImportError err{FolderToDbImportError::kOK}; for (auto& fut : *res) { auto task_result = fut.get(); @@ -112,31 +112,32 @@ FolderToDbImportError WaitParallelTasks(std::vector<std::future<FolderToDbImport } -TaskSplitParameters ComputeSplitParameters(const FileInfos& file_list,int ntasks) { +TaskSplitParameters ComputeSplitParameters(const FileInfos& file_list, int ntasks) { TaskSplitParameters parameters; parameters.chunk = file_list.size() / ntasks; parameters.remainder = file_list.size() % ntasks; return parameters; } -void FolderToDbImporter::ProcessNextChunk(const FileInfos& file_list,std::vector<std::future<FolderToDbImportError>> *res, - TaskSplitParameters* p) const{ +void FolderToDbImporter::ProcessNextChunk(const FileInfos& file_list, + std::vector<std::future<FolderToDbImportError>>* res, + TaskSplitParameters* p) const { p->next_chunk_size = p->chunk + (p->remainder ? 1 : 0); if (p->next_chunk_size == 0) return; res->push_back(std::async(std::launch::async, &FolderToDbImporter::PerformParallelTask, this, - file_list, p->begin, p->begin + p->next_chunk_size)); + file_list, p->begin, p->begin + p->next_chunk_size)); p->begin = p->begin + p->next_chunk_size; if (p->remainder) p->remainder -= 1; } FolderToDbImportError FolderToDbImporter::ImportFilelist(const FileInfos& file_list) const { - auto split_parameters = ComputeSplitParameters(file_list,n_tasks_); + auto split_parameters = ComputeSplitParameters(file_list, n_tasks_); std::vector<std::future<FolderToDbImportError>>res; for (auto i = 0; i < n_tasks_; i++) { - ProcessNextChunk(file_list,&res,&split_parameters); + ProcessNextChunk(file_list, &res, &split_parameters); } return WaitParallelTasks(&res); @@ -173,7 +174,8 @@ FolderToDbImportError FolderToDbImporter::Convert(const std::string& uri, const if (err == FolderToDbImportError::kOK && statistics) { statistics->n_files_converted = file_list.size(); statistics->time_read_folder = std::chrono::duration_cast<std::chrono::nanoseconds>( time_end_read_folder - time_begin); - statistics->time_import_files = std::chrono::duration_cast<std::chrono::nanoseconds>( time_end_import - time_end_read_folder); + statistics->time_import_files = std::chrono::duration_cast<std::chrono::nanoseconds> + ( time_end_import - time_end_read_folder); } return err; diff --git a/worker/tools/folder_to_db/src/folder_db_importer.h b/worker/tools/folder_to_db/src/folder_db_importer.h index 5ce846d2351f4a896e312759f0ff3b32360f520c..6e840d9b96705247f380d2b1829ededaa58e0818 100644 --- a/worker/tools/folder_to_db/src/folder_db_importer.h +++ b/worker/tools/folder_to_db/src/folder_db_importer.h @@ -27,11 +27,11 @@ struct FolderImportStatistics { friend std::ostream& operator<<(std::ostream& os, const FolderImportStatistics& stat); }; -struct TaskSplitParameters{ - uint64_t chunk; - uint64_t remainder; - uint64_t begin{0}; - uint64_t next_chunk_size; +struct TaskSplitParameters { + uint64_t chunk; + uint64_t remainder; + uint64_t begin{0}; + uint64_t next_chunk_size; }; class FolderToDbImporter { @@ -64,10 +64,10 @@ class FolderToDbImporter { const FileInfos& file_list, uint64_t begin, uint64_t end) const; std::unique_ptr<Database> CreateDbClient(FolderToDbImportError* err) const; - void ProcessNextChunk(const FileInfos& file_list,std::vector<std::future<FolderToDbImportError>> *res, - TaskSplitParameters* p) const; + void ProcessNextChunk(const FileInfos& file_list, std::vector<std::future<FolderToDbImportError>>* res, + TaskSplitParameters* p) const; - }; +}; }