start adding integration tests

CMakeModules/testing_go.cmake

@@ -27,3 +27,16 @@ function(gotest target test_source_files)
     endif ()
 endfunction()
 
+function(go_integration_test target test_source_files label)
+    if (BUILD_TESTS)
+        add_test(NAME test-${target} COMMAND go test ${test_source_files} -run ${label}
+                -tags integration_tests
+                WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+        set_property(
+                TEST
+                test-${target}
+                PROPERTY
+                ENVIRONMENT GOPATH=${gopath})
+        message(STATUS "Added test 'test-${target}'")
+    endif ()
+endfunction()

broker/CMakeLists.txt

@@ -11,4 +11,5 @@ add_custom_target(hidra2-broker ALL
 
 install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} DESTINATION bin)
 
-gotest(${TARGET_NAME} "./...")
\ No newline at end of file
+gotest(${TARGET_NAME} "./...")
+go_integration_test(${TARGET_NAME}-connectdb "./..." "MongoDBConnect")

broker/src/hidra2_broker/database/database.go

 package database
 
-import "fmt"
-
 type Agent interface {
+	GetNextRecord(db_name string, collection_name string) (answer []byte, ok bool)
 	Connect(string) error
 	Close()
 }
-
-func Test_Hidra2() {
-	fmt.Println("aaa")
-}

broker/src/hidra2_broker/database/database_test.go

 package database
 
 import (
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"testing"
 )
 
-func TestCreateRecord(t *testing.T) {
-	Test_Hidra2()
-	assert.Equal(t, "111", "111", "record created")
-}
-
 // we run this test just to get 100% coverage for mock_database.go
 func TestMockDataBase(t *testing.T) {
 	var db MockedDatabase
 	db.On("Connect", mock.AnythingOfType("string")).Return(nil)
 	db.On("Close").Return()
+	db.On("GetNextRecord", "", "").Return([]byte(""), false)
 
 	db.Connect("")
+	db.GetNextRecord("", "")
 	db.Close()
 }

broker/src/hidra2_broker/database/mock_database.go

@@ -18,3 +18,8 @@ func (db *MockedDatabase) Connect(address string) error {
 func (db *MockedDatabase) Close() {
 	db.Called()
 }
+
+func (db *MockedDatabase) GetNextRecord(db_name string, collection_name string) (answer []byte, ok bool) {
+	args := db.Called(db_name, collection_name)
+	return args.Get(0).([]byte), args.Bool(1)
+}

broker/src/hidra2_broker/database/mongodb.go

@@ -13,8 +13,9 @@ type Mongodb struct {
 	timeout      time.Duration
 }
 
-func (db *Mongodb) Connect(string) error {
+func (db *Mongodb) Connect(address string) error {
 	var err error
+	db.main_session, err = mgo.DialWithTimeout(address, time.Second)
 	return err
 }
 
@@ -24,3 +25,7 @@ func (db *Mongodb) Close() {
 	}
 
 }
+
+func (db *Mongodb) GetNextRecord(db_name string, collection_name string) (answer []byte, ok bool) {
+	return nil, true
+}

broker/src/hidra2_broker/database/mongodb_test.go

+// +build integration_tests
+
+package database
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+// these are tjhe integration tests. They assume mongo db is runnig on 127.0.0.1:27027
+// test names shlud contain MongoDB*** so that go test could find them:
+// go_integration_test(${TARGET_NAME}-connectdb "./..." "MongoDBConnect")
+func TestMongoDBConnectFails(t *testing.T) {
+	var db Mongodb
+	err := db.Connect("blabla")
+	assert.NotNil(t, err)
+}
+
+func TestMongoDBConnectOK(t *testing.T) {
+	var db Mongodb
+	err := db.Connect("127.0.0.1:27017")
+	assert.Nil(t, err)
+	db.Close()
+}

broker/src/hidra2_broker/main/broker.go

@@ -5,17 +5,15 @@ package main
 import (
 	"hidra2_broker/database"
 	"hidra2_broker/server"
+	"log"
 )
 
 func NewDefaultDatabase() database.Agent {
 	return new(database.Mongodb)
 }
 
-// global variable since we only have one instance
-var srv server.Server
-
 func main() {
-	srv.InitDB(NewDefaultDatabase())
-	defer srv.CleanupDB()
-	srv.Start()
+	log.Fatal(server.InitDB(NewDefaultDatabase()))
+	defer server.CleanupDB()
+	server.Start()
 }

broker/src/hidra2_broker/server/get_next.go

@@ -4,7 +4,29 @@ import (
 	"net/http"
 )
 
+func extractRequestParameters(r *http.Request) (string, bool) {
+	db_name := r.URL.Query().Get("database")
+	return db_name, db_name != ""
+}
+
 func routeGetNext(w http.ResponseWriter, r *http.Request) {
 	r.Header.Set("Content-type", "application/json")
-	w.WriteHeader(http.StatusOK)
+
+	db_name, ok := extractRequestParameters(r)
+	if !ok {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	answer, ok := getNextRecord(db_name)
+	if !ok {
+		w.WriteHeader(http.StatusBadRequest)
+	} else {
+		w.WriteHeader(http.StatusOK)
+	}
+	w.Write(answer)
+}
+
+func getNextRecord(db_name string) (answer []byte, ok bool) {
+	return db.GetNextRecord(db_name, "data")
 }

broker/src/hidra2_broker/server/get_next_test.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"github.com/stretchr/testify/assert"
+	"hidra2_broker/database"
 	"hidra2_broker/utils"
 	"net/http"
 	"net/http/httptest"
@@ -15,21 +16,39 @@ type request struct {
 	message string
 }
 
-var getNextTests = []request{
-	{"next", "GET", http.StatusOK, "get next job"},
+func doRequest(path string) *httptest.ResponseRecorder {
+	mux := utils.NewRouter(listRoutes)
+	req, _ := http.NewRequest("GET", path, nil)
+	w := httptest.NewRecorder()
+	mux.ServeHTTP(w, req)
+	return w
 }
 
-func TestGetNext(t *testing.T) {
-	mux := utils.NewRouter(listRoutes)
+func TestGetNextWithoutDatabaseName(t *testing.T) {
+	w := doRequest("/next")
+	assert.Equal(t, http.StatusBadRequest, w.Code, "no database name")
+}
 
-	for _, test := range getNextTests {
+func TestGetNextWithWrongDatabaseName(t *testing.T) {
+	mock_db := new(database.MockedDatabase)
+	db = mock_db
+	defer func() { db = nil }()
+	mock_db.On("GetNextRecord", "foo", "data").Return([]byte(""), false)
+
+	w := doRequest("/next?database=foo")
+	assert.Equal(t, http.StatusBadRequest, w.Code, "no database name")
+	assertExpectations(t, mock_db)
+}
 
-		req, err := http.NewRequest(test.cmd, "/"+test.path+"/", nil)
+func TestGetNextWithGoodDatabaseName(t *testing.T) {
+	mock_db := new(database.MockedDatabase)
+	db = mock_db
+	defer func() { db = nil }()
+	mock_db.On("GetNextRecord", "database", "data").Return([]byte("Hello"), true)
 
-		assert.Nil(t, err, "Should not be error")
+	w := doRequest("/next?database=database")
+	assert.Equal(t, http.StatusOK, w.Code, "GetNext OK")
+	assert.Equal(t, "Hello", string(w.Body.Bytes()), "GetNext sends data")
+	assertExpectations(t, mock_db)
 
-		w := httptest.NewRecorder()
-		mux.ServeHTTP(w, req)
-		assert.Equal(t, test.answer, w.Code, test.message)
-	}
 }

broker/src/hidra2_broker/server/listroutes.go

@@ -8,7 +8,7 @@ var listRoutes = utils.Routes{
 	utils.Route{
 		"GetNext",
 		"Get",
-		"/next/",
+		"/next",
 		routeGetNext,
 	},
 }

broker/src/hidra2_broker/server/server.go

@@ -4,17 +4,15 @@ import (
 	"hidra2_broker/database"
 )
 
-type Server struct {
-	db database.Agent
-}
+var db database.Agent
 
-func (srv *Server) InitDB(db database.Agent) error {
-	srv.db = db
-	return srv.db.Connect("127.0.0.1:27017")
+func InitDB(dbAgent database.Agent) error {
+	db = dbAgent
+	return db.Connect("127.0.0.1:27017")
 }
 
-func (srv *Server) CleanupDB() {
-	if srv.db != nil {
-		srv.db.Close()
+func CleanupDB() {
+	if db != nil {
+		db.Close()
 	}
 }

broker/src/hidra2_broker/server/server_nottested.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 )
 
-func (srv *Server) Start() {
+func Start() {
 	mux := utils.NewRouter(listRoutes)
 	log.Fatal(http.ListenAndServe("127.0.0.1:5005", http.HandlerFunc(mux.ServeHTTP)))
 }

broker/src/hidra2_broker/server/server_test.go

@@ -8,17 +8,16 @@ import (
 	"testing"
 )
 
-func setup() (*database.MockedDatabase, *Server) {
-	db := new(database.MockedDatabase)
-	srv := new(Server)
-	db.On("Connect", mock.AnythingOfType("string")).Return(nil)
-	return db, srv
+func setup() *database.MockedDatabase {
+	mock_db := new(database.MockedDatabase)
+	mock_db.On("Connect", mock.AnythingOfType("string")).Return(nil)
+	return mock_db
 
 }
 
-func assertExpectations(t *testing.T, db *database.MockedDatabase) {
-	db.AssertExpectations(t)
-	db.ExpectedCalls = nil
+func assertExpectations(t *testing.T, mock_db *database.MockedDatabase) {
+	mock_db.AssertExpectations(t)
+	mock_db.ExpectedCalls = nil
 }
 
 var initDBTests = []struct {
@@ -31,34 +30,36 @@ var initDBTests = []struct {
 }
 
 func TestInitDBWithWrongAddress(t *testing.T) {
-	db, srv := setup()
-	db.ExpectedCalls = nil
+	mock_db := setup()
+
+	mock_db.ExpectedCalls = nil
 
 	for _, test := range initDBTests {
-		db.On("Connect", mock.AnythingOfType("string")).Return(test.answer)
+		mock_db.On("Connect", mock.AnythingOfType("string")).Return(test.answer)
 
-		err := srv.InitDB(db)
+		err := InitDB(mock_db)
 
 		assert.Equal(t, test.answer, err, test.message)
-		assertExpectations(t, db)
+		assertExpectations(t, mock_db)
 	}
+	db = nil
 }
 
 func TestCleanupDBWithoutInit(t *testing.T) {
-	db, srv := setup()
+	mock_db := setup()
 
-	db.AssertNotCalled(t, "Close")
+	mock_db.AssertNotCalled(t, "Close")
 
-	srv.CleanupDB()
+	CleanupDB()
 }
 
 func TestCleanupDBInit(t *testing.T) {
-	db, srv := setup()
+	mock_db := setup()
 
-	db.On("Close").Return()
+	mock_db.On("Close").Return()
 
-	srv.InitDB(db)
-	srv.CleanupDB()
+	InitDB(mock_db)
+	CleanupDB()
 
-	assertExpectations(t, db)
+	assertExpectations(t, mock_db)
 }

common/cpp/include/common/file_info.h

@@ -16,11 +16,12 @@ struct FileInfo {
     uint64_t size{0};
     uint64_t id{0};
     std::string Json() const {
-        auto periods = modify_date.time_since_epoch().count();
+        auto nanoseconds_from_epoch = std::chrono::time_point_cast<std::chrono::nanoseconds>(modify_date).
+                       time_since_epoch().count();
         std::string s = "{\"_id\":" + std::to_string(id) + ","
                         "\"size\":" + std::to_string(size) + ","
                         "\"base_name\":\"" + base_name + "\","
-                        "\"lastchange\":" + std::to_string(periods) + ","
+                        "\"lastchange\":" + std::to_string(nanoseconds_from_epoch) + ","
                         "\"relative_path\":\"" + relative_path + "\"}";
         return s;
     }

worker/tools/folder_to_db/src/folder_db_importer.cpp

@@ -100,7 +100,7 @@ std::unique_ptr<hidra2::Database> FolderToDbImporter::CreateDbClient(FolderToDbI
     return db;
 }
 
-FolderToDbImportError WaitParallelTasks(std::vector<std::future<FolderToDbImportError>>* res){
+FolderToDbImportError WaitParallelTasks(std::vector<std::future<FolderToDbImportError>>* res) {
     FolderToDbImportError err{FolderToDbImportError::kOK};
     for (auto& fut : *res) {
         auto task_result = fut.get();
@@ -112,31 +112,32 @@ FolderToDbImportError WaitParallelTasks(std::vector<std::future<FolderToDbImport
 }
 
 
-TaskSplitParameters ComputeSplitParameters(const FileInfos& file_list,int ntasks) {
+TaskSplitParameters ComputeSplitParameters(const FileInfos& file_list, int ntasks) {
     TaskSplitParameters parameters;
     parameters.chunk = file_list.size() / ntasks;
     parameters.remainder = file_list.size() % ntasks;
     return parameters;
 }
 
-void FolderToDbImporter::ProcessNextChunk(const FileInfos& file_list,std::vector<std::future<FolderToDbImportError>> *res,
-                                          TaskSplitParameters* p) const{
+void FolderToDbImporter::ProcessNextChunk(const FileInfos& file_list,
+                                          std::vector<std::future<FolderToDbImportError>>* res,
+                                          TaskSplitParameters* p) const {
     p->next_chunk_size = p->chunk + (p->remainder ? 1 : 0);
     if (p->next_chunk_size == 0) return;
 
     res->push_back(std::async(std::launch::async, &FolderToDbImporter::PerformParallelTask, this,
-                             file_list, p->begin, p->begin + p->next_chunk_size));
+                              file_list, p->begin, p->begin + p->next_chunk_size));
 
     p->begin = p->begin + p->next_chunk_size;
     if (p->remainder) p->remainder -= 1;
 }
 
 FolderToDbImportError FolderToDbImporter::ImportFilelist(const FileInfos& file_list) const {
-    auto split_parameters = ComputeSplitParameters(file_list,n_tasks_);
+    auto split_parameters = ComputeSplitParameters(file_list, n_tasks_);
 
     std::vector<std::future<FolderToDbImportError>>res;
     for (auto i = 0; i < n_tasks_; i++) {
-        ProcessNextChunk(file_list,&res,&split_parameters);
+        ProcessNextChunk(file_list, &res, &split_parameters);
     }
 
     return WaitParallelTasks(&res);
@@ -173,7 +174,8 @@ FolderToDbImportError FolderToDbImporter::Convert(const std::string& uri, const
     if (err == FolderToDbImportError::kOK && statistics) {
         statistics->n_files_converted = file_list.size();
         statistics->time_read_folder = std::chrono::duration_cast<std::chrono::nanoseconds>( time_end_read_folder - time_begin);
-        statistics->time_import_files = std::chrono::duration_cast<std::chrono::nanoseconds>( time_end_import - time_end_read_folder);
+        statistics->time_import_files = std::chrono::duration_cast<std::chrono::nanoseconds>
+                                        ( time_end_import - time_end_read_folder);
     }
 
     return err;

worker/tools/folder_to_db/src/folder_db_importer.h

@@ -27,11 +27,11 @@ struct FolderImportStatistics {
     friend std::ostream& operator<<(std::ostream& os, const FolderImportStatistics& stat);
 };
 
-struct TaskSplitParameters{
-  uint64_t chunk;
-  uint64_t remainder;
-  uint64_t begin{0};
-  uint64_t next_chunk_size;
+struct TaskSplitParameters {
+    uint64_t chunk;
+    uint64_t remainder;
+    uint64_t begin{0};
+    uint64_t next_chunk_size;
 };
 
 class FolderToDbImporter {
@@ -64,10 +64,10 @@ class FolderToDbImporter {
                                               const FileInfos& file_list, uint64_t begin, uint64_t end) const;
 
     std::unique_ptr<Database> CreateDbClient(FolderToDbImportError* err) const;
-    void ProcessNextChunk(const FileInfos& file_list,std::vector<std::future<FolderToDbImportError>> *res,
-                                            TaskSplitParameters* p) const;
+    void ProcessNextChunk(const FileInfos& file_list, std::vector<std::future<FolderToDbImportError>>* res,
+                          TaskSplitParameters* p) const;
 
-  };
+};
 
 }