diff --git a/3d_party/mongo-c-driver/install.sh b/3d_party/mongo-c-driver/install.sh index 6644bf5a99e276b865585e65e8eea5bcf2f56fc5..33d1d5d8a87be513db430128f461da90cb405513 100755 --- a/3d_party/mongo-c-driver/install.sh +++ b/3d_party/mongo-c-driver/install.sh @@ -4,7 +4,7 @@ cd $1 wget https://github.com/mongodb/mongo-c-driver/releases/download/1.9.0/mongo-c-driver-1.9.0.tar.gz tar xzf mongo-c-driver-1.9.0.tar.gz cd mongo-c-driver-1.9.0 -./configure --disable-automatic-init-and-cleanup --enable-static=yes --enable-shared=no --enable-ssl=no --enable-sasl=no +./configure --disable-automatic-init-and-cleanup --enable-static=yes --enable-shared=no --enable-examples=no --enable-ssl=no --enable-sasl=no make #sudo make install diff --git a/CMakeModules/testing_cpp.cmake b/CMakeModules/testing_cpp.cmake index d8d07bbc8ffc0cdd5fbe6171ea459afc554a663f..76f5d247776f259d1fea6cf770a91bb4c11abe40 100644 --- a/CMakeModules/testing_cpp.cmake +++ b/CMakeModules/testing_cpp.cmake @@ -140,8 +140,10 @@ function(add_script_test testname exename) separate_arguments(memargs) add_test(NAME test-${testname} COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/check_linux.sh ${args}) + if (MEMORYCHECK_COMMAND) add_test(NAME memtest-${testname} COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/check_linux.sh ${memargs}) + endif() ENDIF() set_tests_properties(test-${testname} PROPERTIES LABELS "example;all" diff --git a/common/cpp/include/database/database.h b/common/cpp/include/database/database.h index 0c614ad03e1f925c38145c21d005560104c04af4..925cf97241c48c17d35e1e748276776bcf61c996 100644 --- a/common/cpp/include/database/database.h +++ b/common/cpp/include/database/database.h @@ -8,6 +8,7 @@ namespace hidra2 { enum class DBError { + KUnknownError, kConnectionError, kInsertError, kDuplicateID, @@ -30,7 +31,8 @@ class Database { class DatabaseFactory { public: - virtual std::unique_ptr<Database> Create(DBError* err) const noexcept = 0; + virtual std::unique_ptr<Database> Create(DBError* err) const noexcept; + virtual ~DatabaseFactory() = default; }; diff --git a/common/cpp/src/database/CMakeLists.txt b/common/cpp/src/database/CMakeLists.txt index 5479f1f444f0a141a27989d6e6e5c2f0a62bb2af..9ca47b5986b45354df6907b754b4d4713ac42265 100644 --- a/common/cpp/src/database/CMakeLists.txt +++ b/common/cpp/src/database/CMakeLists.txt @@ -1,6 +1,7 @@ -set(TARGET_NAME mongo_db) +set(TARGET_NAME database) set(SOURCE_FILES - mongodb_client.cpp) + mongodb_client.cpp + database.cpp) ################################ # Library diff --git a/common/cpp/src/database/database.cpp b/common/cpp/src/database/database.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5d4dfa61a11aa982cb48ce51fe1582e53533ed71 --- /dev/null +++ b/common/cpp/src/database/database.cpp @@ -0,0 +1,17 @@ +#include "database/database.h" +#include "mongodb_client.h" + +namespace hidra2 { + +std::unique_ptr<Database> DatabaseFactory::Create(DBError* err) const noexcept { + std::unique_ptr<Database> p = nullptr; + try { + p.reset(new MongoDBClient()); + *err = DBError::kNoError; + } catch (...) { + *err = DBError::kMemoryError; + } + return p; +}; + +} \ No newline at end of file diff --git a/common/cpp/src/database/mongodb_client.cpp b/common/cpp/src/database/mongodb_client.cpp index 1c857501a374c0f5ecc11afb1d552f2a281f3077..07e2b9bba6d218b54fee86712413cb4dec44fe70 100644 --- a/common/cpp/src/database/mongodb_client.cpp +++ b/common/cpp/src/database/mongodb_client.cpp @@ -1,22 +1,10 @@ -#include "database/mongodb_client.h" +#include "mongodb_client.h" namespace hidra2 { using std::string; using hidra2::Database; -std::unique_ptr<Database> MongoDatabaseFactory::Create(DBError* err) const noexcept { - std::unique_ptr<Database> p = nullptr; - try { - p.reset(new MongoDBClient()); - *err = DBError::kNoError; - } catch (...) { // we do not test this part - *err = DBError::kMemoryError; - } - return p; -}; - - MongoDbInstance::MongoDbInstance() { mongoc_init (); } diff --git a/common/cpp/include/database/mongodb_client.h b/common/cpp/src/database/mongodb_client.h similarity index 90% rename from common/cpp/include/database/mongodb_client.h rename to common/cpp/src/database/mongodb_client.h index 9966b18569ff6ee49c8fce36238ee083d6b4fcb1..4144e8ec559951f77eb38ddbbbf84f496f365d80 100644 --- a/common/cpp/include/database/mongodb_client.h +++ b/common/cpp/src/database/mongodb_client.h @@ -5,7 +5,7 @@ #include <string> -#include "database.h" +#include "database/database.h" namespace hidra2 { @@ -54,12 +54,6 @@ class MongoDBClient final : public Database { DBError InsertBsonDocument(const bson_p& document, bool ignore_duplicates) const; }; -class MongoDatabaseFactory final : public DatabaseFactory { - public: - std::unique_ptr<hidra2::Database> Create(DBError* err) const noexcept override; -}; - - } #endif //HIDRA2_MONGO_DATABASE_H diff --git a/tests/mongo_db/connect/CMakeLists.txt b/tests/mongo_db/connect/CMakeLists.txt index 780baba6fe7f6c2211519e5d75d9b89da7912e23..6c2dc7c5b35a44e5c8160132b11e5acf19ef9854 100644 --- a/tests/mongo_db/connect/CMakeLists.txt +++ b/tests/mongo_db/connect/CMakeLists.txt @@ -6,7 +6,7 @@ set(SOURCE_FILES connect_mongodb.cpp) # Executable and link ################################ add_executable(${TARGET_NAME} ${SOURCE_FILES}) -target_link_libraries(${TARGET_NAME} test_common mongo_db) +target_link_libraries(${TARGET_NAME} test_common database) target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR}) ################################ diff --git a/tests/mongo_db/connect/connect_mongodb.cpp b/tests/mongo_db/connect/connect_mongodb.cpp index 12247a5241bcd145ad476b497f01fc5aefe6dccb..74fab0b984ff9429bbbf436bebc961a22975ab5c 100644 --- a/tests/mongo_db/connect/connect_mongodb.cpp +++ b/tests/mongo_db/connect/connect_mongodb.cpp @@ -1,7 +1,7 @@ #include <iostream> -#include "database/mongodb_client.h" +#include "../../../common/cpp/src/database/mongodb_client.h" #include "testing.h" diff --git a/tests/mongo_db/insert/CMakeLists.txt b/tests/mongo_db/insert/CMakeLists.txt index bb0ad4a08e67a871e10a8224d83b2a1e1bf04a92..1ff3205b80370c065c4a17a75c1325f41b18400a 100644 --- a/tests/mongo_db/insert/CMakeLists.txt +++ b/tests/mongo_db/insert/CMakeLists.txt @@ -6,7 +6,7 @@ set(SOURCE_FILES insert_mongodb.cpp) # Executable and link ################################ add_executable(${TARGET_NAME} ${SOURCE_FILES}) -target_link_libraries(${TARGET_NAME} test_common mongo_db) +target_link_libraries(${TARGET_NAME} test_common database) target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR}) ################################ diff --git a/tests/mongo_db/insert/insert_mongodb.cpp b/tests/mongo_db/insert/insert_mongodb.cpp index 65c50419d7528954fa1f2d24e336cd574fc0a733..a9194c3c8eb0c1dae6fdb0c7b77283736e69f822 100644 --- a/tests/mongo_db/insert/insert_mongodb.cpp +++ b/tests/mongo_db/insert/insert_mongodb.cpp @@ -1,7 +1,7 @@ #include <iostream> #include <chrono> -#include "database/mongodb_client.h" +#include "../../../common/cpp/src/database/mongodb_client.h" #include "testing.h" diff --git a/worker/tools/folder_to_db/CMakeLists.txt b/worker/tools/folder_to_db/CMakeLists.txt index 37c179e69c9a30717390101ce69bb25e40dfe554..9f66a53cd4253efd07ec6ba2c408ae41c53658f1 100644 --- a/worker/tools/folder_to_db/CMakeLists.txt +++ b/worker/tools/folder_to_db/CMakeLists.txt @@ -7,7 +7,7 @@ set(SOURCE_FILES src/folder_db_importer.cpp) # Library ################################ add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io>) -target_link_libraries(${TARGET_NAME} mongo_db) +target_link_libraries(${TARGET_NAME} database) add_executable(${TARGET_NAME}-bin src/main.cpp) set_target_properties(${TARGET_NAME}-bin diff --git a/worker/tools/folder_to_db/src/folder_db_importer.cpp b/worker/tools/folder_to_db/src/folder_db_importer.cpp index c5e4b3f435dba7ed2b55a3f9285fb4e6ff550317..26311936ab064d9633d2b3d4c8ff4db25531bad1 100644 --- a/worker/tools/folder_to_db/src/folder_db_importer.cpp +++ b/worker/tools/folder_to_db/src/folder_db_importer.cpp @@ -4,7 +4,7 @@ #include <algorithm> #include "system_wrappers/system_io.h" -#include "database/mongodb_client.h" +#include "database/database.h" namespace hidra2 { @@ -52,7 +52,7 @@ FolderToDbImportError MapDBError(DBError db_err) { FolderToDbImporter::FolderToDbImporter() : - io__{new hidra2::SystemIO}, db_factory__{new hidra2::MongoDatabaseFactory} { + io__{new hidra2::SystemIO}, db_factory__{new hidra2::DatabaseFactory} { } FolderToDbImportError FolderToDbImporter::ConnectToDb(const std::unique_ptr<hidra2::Database>& db) const { @@ -67,16 +67,8 @@ FolderToDbImportError FolderToDbImporter::ImportSingleFile(const std::unique_ptr return MapDBError(err); } -FolderToDbImportError FolderToDbImporter::ImportPartOfFilelist(const FileInfos& file_list, uint64_t begin, - uint64_t end) const { - DBError db_err; - std::unique_ptr<hidra2::Database> db = db_factory__->Create(&db_err); - - auto err = ConnectToDb(db); - if (err != FolderToDbImportError::kOK) { - return err; - } - +FolderToDbImportError FolderToDbImporter::ImportFilelistChunk(const std::unique_ptr<hidra2::Database>& db, + const FileInfos& file_list, uint64_t begin, uint64_t end) const { for (auto i = begin; i < end; i++) { auto err = ImportSingleFile(db, file_list[i]); if (err != FolderToDbImportError::kOK) { @@ -86,20 +78,31 @@ FolderToDbImportError FolderToDbImporter::ImportPartOfFilelist(const FileInfos& return FolderToDbImportError::kOK; } -FolderToDbImportError FolderToDbImporter::ImportFilelist(const FileInfos& file_list) const { - auto grain = file_list.size() / n_tasks_; - if (grain == 0) { - grain = file_list.size(); +FolderToDbImportError FolderToDbImporter::PerformParallelTask(const FileInfos& file_list, uint64_t begin, + uint64_t end) const { + FolderToDbImportError err; + auto db = CreateDbClient(&err); + if (err != FolderToDbImportError::kOK) { + return err; } - std::vector<std::future<FolderToDbImportError>>res; - for (auto i = 0; i < file_list.size(); i += grain) { - auto end = std::min(i + grain, file_list.size()); - res.push_back(std::async(std::launch::async, &FolderToDbImporter::ImportPartOfFilelist, this, - file_list, i, end)); + err = ConnectToDb(db); + if (err != FolderToDbImportError::kOK) { + return err; } + + return ImportFilelistChunk(db, file_list, begin, end); +} +std::unique_ptr<hidra2::Database> FolderToDbImporter::CreateDbClient(FolderToDbImportError* err) const { + DBError db_err; + auto db = db_factory__->Create(&db_err); + *err = MapDBError(db_err); + return db; +} + +FolderToDbImportError WaitParallelTasks(std::vector<std::future<FolderToDbImportError>>* res){ FolderToDbImportError err{FolderToDbImportError::kOK}; - for (auto& fut : res) { + for (auto& fut : *res) { auto task_result = fut.get(); if (task_result != FolderToDbImportError::kOK) { err = task_result; @@ -109,6 +112,37 @@ FolderToDbImportError FolderToDbImporter::ImportFilelist(const FileInfos& file_l } +TaskSplitParameters ComputeSplitParameters(const FileInfos& file_list,int ntasks) { + TaskSplitParameters parameters; + parameters.chunk = file_list.size() / ntasks; + parameters.remainder = file_list.size() % ntasks; + return parameters; +} + +void FolderToDbImporter::ProcessNextChunk(const FileInfos& file_list,std::vector<std::future<FolderToDbImportError>> *res, + TaskSplitParameters* p) const{ + p->next_chunk_size = p->chunk + (p->remainder ? 1 : 0); + if (p->next_chunk_size == 0) return; + + res->push_back(std::async(std::launch::async, &FolderToDbImporter::PerformParallelTask, this, + file_list, p->begin, p->begin + p->next_chunk_size)); + + p->begin = p->begin + p->next_chunk_size; + if (p->remainder) p->remainder -= 1; +} + +FolderToDbImportError FolderToDbImporter::ImportFilelist(const FileInfos& file_list) const { + auto split_parameters = ComputeSplitParameters(file_list,n_tasks_); + + std::vector<std::future<FolderToDbImportError>>res; + for (auto i = 0; i < n_tasks_; i++) { + ProcessNextChunk(file_list,&res,&split_parameters); + } + + return WaitParallelTasks(&res); +} + + FileInfos FolderToDbImporter::GetFilesInFolder(const std::string& folder, FolderToDbImportError* err) const { IOError err_io; auto file_list = io__->FilesInFolder(folder, &err_io); @@ -122,7 +156,7 @@ FolderToDbImportError FolderToDbImporter::Convert(const std::string& uri, const db_uri_ = uri; db_collection_name = folder; - high_resolution_clock::time_point t1 = high_resolution_clock::now(); + auto time_begin = high_resolution_clock::now(); FolderToDbImportError err; auto file_list = GetFilesInFolder(folder, &err); @@ -130,30 +164,29 @@ FolderToDbImportError FolderToDbImporter::Convert(const std::string& uri, const return err; } - high_resolution_clock::time_point t2 = high_resolution_clock::now(); + auto time_end_read_folder = high_resolution_clock::now(); err = ImportFilelist(file_list); - high_resolution_clock::time_point t3 = high_resolution_clock::now(); + auto time_end_import = high_resolution_clock::now(); if (err == FolderToDbImportError::kOK && statistics) { statistics->n_files_converted = file_list.size(); - statistics->time_read_folder = std::chrono::duration_cast<std::chrono::nanoseconds>( t2 - t1); - statistics->time_import_files = std::chrono::duration_cast<std::chrono::nanoseconds>( t3 - t2); + statistics->time_read_folder = std::chrono::duration_cast<std::chrono::nanoseconds>( time_end_read_folder - time_begin); + statistics->time_import_files = std::chrono::duration_cast<std::chrono::nanoseconds>( time_end_import - time_end_read_folder); } return err; - - } void FolderToDbImporter::IgnoreDuplicates(bool ignore_duplicates) { ignore_duplicates_ = ignore_duplicates; } -void FolderToDbImporter::RunInParallel(unsigned int ntasks) { +unsigned int FolderToDbImporter::SetNParallelTasks(unsigned int ntasks) { unsigned int nthreads = std::thread::hardware_concurrency(); n_tasks_ = std::max((unsigned int)1, std::min(ntasks, nthreads)); + return n_tasks_; } std::ostream& operator<<(std::ostream& os, const FolderImportStatistics& stat) { diff --git a/worker/tools/folder_to_db/src/folder_db_importer.h b/worker/tools/folder_to_db/src/folder_db_importer.h index 7c2f8df624099ef6d264a905cd2631c5905ee9f2..5ce846d2351f4a896e312759f0ff3b32360f520c 100644 --- a/worker/tools/folder_to_db/src/folder_db_importer.h +++ b/worker/tools/folder_to_db/src/folder_db_importer.h @@ -3,6 +3,7 @@ #include <iostream> #include <chrono> +#include <future> #include "system_wrappers/io.h" @@ -26,6 +27,13 @@ struct FolderImportStatistics { friend std::ostream& operator<<(std::ostream& os, const FolderImportStatistics& stat); }; +struct TaskSplitParameters{ + uint64_t chunk; + uint64_t remainder; + uint64_t begin{0}; + uint64_t next_chunk_size; +}; + class FolderToDbImporter { public: FolderToDbImporter(); @@ -35,12 +43,11 @@ class FolderToDbImporter { FolderToDbImportError Convert(const std::string& uri, const std::string& folder, FolderImportStatistics* statistics = nullptr) const; + unsigned int SetNParallelTasks(unsigned int ntasks); + void IgnoreDuplicates(bool ignore_duplicates = true); std::unique_ptr<hidra2::DatabaseFactory> db_factory__; // modified in testings to mock system calls,otherwise do not touch std::unique_ptr<hidra2::IO> io__; // modified in testings to mock system calls,otherwise do not touch - public: - void RunInParallel(unsigned int ntasks); - void IgnoreDuplicates(bool ignore_duplicates); private: bool ignore_duplicates_{false}; unsigned int n_tasks_{1}; @@ -49,11 +56,18 @@ class FolderToDbImporter { FolderToDbImportError ConnectToDb(const std::unique_ptr<hidra2::Database>& db) const; FileInfos GetFilesInFolder(const std::string& folder, FolderToDbImportError* err) const; FolderToDbImportError ImportFilelist(const FileInfos& file_list) const; - FolderToDbImportError ImportPartOfFilelist(const FileInfos& file_list, uint64_t begin, - uint64_t end) const; + FolderToDbImportError PerformParallelTask(const FileInfos& file_list, uint64_t begin, + uint64_t end) const; FolderToDbImportError ImportSingleFile(const std::unique_ptr<hidra2::Database>& db, const FileInfo& file) const; -}; + FolderToDbImportError ImportFilelistChunk(const std::unique_ptr<hidra2::Database>& db, + const FileInfos& file_list, uint64_t begin, uint64_t end) const; + + std::unique_ptr<Database> CreateDbClient(FolderToDbImportError* err) const; + void ProcessNextChunk(const FileInfos& file_list,std::vector<std::future<FolderToDbImportError>> *res, + TaskSplitParameters* p) const; + + }; } diff --git a/worker/tools/folder_to_db/src/main.cpp b/worker/tools/folder_to_db/src/main.cpp index d20f023a610b1ed4711a76e6fe73397a407411de..5e3993d1b614a8ff8f0fd028c9778962e4a1e3e6 100644 --- a/worker/tools/folder_to_db/src/main.cpp +++ b/worker/tools/folder_to_db/src/main.cpp @@ -64,7 +64,7 @@ int main(int argc, char* argv[]) { auto import_params = ProcessCommandArguments(argc, argv); hidra2::FolderToDbImporter importer; - importer.RunInParallel(import_params.ntasks); + importer.SetNParallelTasks(import_params.ntasks); importer.IgnoreDuplicates(import_params.ignore_duplicates); hidra2::FolderImportStatistics statistics; diff --git a/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp b/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp index abdeb1babed64db26fc74cbad8907184a35fc4c0..a27abc2ffe413682aa9f710e74a3bc8b20f59240 100644 --- a/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp +++ b/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp @@ -1,12 +1,12 @@ #include <gmock/gmock.h> #include "gtest/gtest.h" +#include <thread> #include "system_wrappers/io.h" #include "system_wrappers/system_io.h" #include "database/database.h" -#include "database/mongodb_client.h" #include "common/file_info.h" @@ -48,17 +48,24 @@ TEST(FolderDBConverter, SetCorrectIO) { TEST(FolderDBConverter, SetCorrectDBFactory) { FolderToDbImporter converter{}; - ASSERT_THAT(dynamic_cast<hidra2::MongoDatabaseFactory*>(converter.db_factory__.get()), Ne(nullptr)); + ASSERT_THAT(dynamic_cast<hidra2::DatabaseFactory*>(converter.db_factory__.get()), Ne(nullptr)); } -class MockDatabaseFactory : public DatabaseFactory { - public: - Database* db_; - std::unique_ptr<Database> Create(DBError* err) const noexcept { - return std::unique_ptr<Database> {db_}; - } -}; +TEST(FolderDBConverter, SetNTasksCorrectly) { + + unsigned int max_tasks = std::thread::hardware_concurrency(); + + FolderToDbImporter converter{}; + auto res = converter.SetNParallelTasks(0); + ASSERT_THAT(res, Eq(1)); + res = converter.SetNParallelTasks(2); + ASSERT_THAT(res, Eq(2)); + + res = converter.SetNParallelTasks(1000000); + ASSERT_THAT(res, Eq(max_tasks)); + +} class MockDatabase : public Database { public: @@ -74,6 +81,36 @@ class MockDatabase : public Database { bool check_destructor{false}; }; +class MockDatabaseFactory : public DatabaseFactory { + public: + std::vector<NiceMock<MockDatabase>*> db; + mutable int n{0}; + void CreateDBs(int n) { + for (int i = 0; i < n; i++) { + auto val = new NiceMock<MockDatabase>; + db.push_back(val); + ON_CALL(*val, Connect(_, _, _)) + .WillByDefault(Return(DBError::kNoError)); + } + } + std::unique_ptr<Database> Create(DBError* err) const noexcept override { + *err = DBError::kNoError; + return std::unique_ptr<Database> {db[n++]}; + } + ~MockDatabaseFactory() { + for (int i = n; i < db.size(); i++) { + delete db[i]; + } + } +}; + +class FakeDatabaseFactory : public DatabaseFactory { + std::unique_ptr<Database> Create(DBError* err) const noexcept override { + *err = DBError::kMemoryError; + return {}; + } +}; + FileInfos CreateTestFileInfos() { FileInfos file_infos; FileInfo fi; @@ -91,20 +128,18 @@ class FolderDBConverterTests : public Test { public: FolderToDbImporter converter{}; NiceMock<MockIO> mock_io; - NiceMock<MockDatabase>* mock_db = new NiceMock<MockDatabase>; - MockDatabaseFactory* mock_dbf = new MockDatabaseFactory; + + MockDatabaseFactory* mock_dbf; FileInfos file_infos; std::string folder, uri; void SetUp() override { - mock_dbf->db_ = mock_db; converter.io__ = std::unique_ptr<IO> {&mock_io}; + mock_dbf = new MockDatabaseFactory; + mock_dbf->CreateDBs(3); converter.db_factory__ = std::unique_ptr<DatabaseFactory> {mock_dbf}; file_infos = CreateTestFileInfos(); folder = "folder"; uri = "db_address"; - - ON_CALL(*mock_db, Connect(_, _, _)) - .WillByDefault(Return(DBError::kNoError)); ON_CALL(mock_io, FilesInFolder(_, _)). WillByDefault(DoAll(testing::SetArgPointee<1>(IOError::kNoError), testing::Return(file_infos))); @@ -114,19 +149,33 @@ class FolderDBConverterTests : public Test { } }; + TEST_F(FolderDBConverterTests, ErrorWhenCannotConnect) { - EXPECT_CALL(*mock_db, Connect(uri, kDBName, _)). + EXPECT_CALL(*(mock_dbf->db[0]), Connect(uri, kDBName, _)). + WillOnce(testing::Return(DBError::kConnectionError)); + + auto error = converter.Convert(uri, folder); + ASSERT_THAT(error, Eq(FolderToDbImportError::kDBConnectionError)); +} + +TEST_F(FolderDBConverterTests, ErrorWhenCannotCreateDbParallel) { + int nparallel = 3; + EXPECT_CALL(*(mock_dbf->db[0]), Connect(uri, kDBName, _)). + WillOnce(testing::Return(DBError::kConnectionError)); + EXPECT_CALL(*(mock_dbf->db[1]), Connect(uri, kDBName, _)). + WillOnce(testing::Return(DBError::kConnectionError)); + EXPECT_CALL(*(mock_dbf->db[2]), Connect(uri, kDBName, _)). WillOnce(testing::Return(DBError::kConnectionError)); + converter.SetNParallelTasks(nparallel); auto error = converter.Convert(uri, folder); ASSERT_THAT(error, Eq(FolderToDbImportError::kDBConnectionError)); } TEST_F(FolderDBConverterTests, DBDestructorCalled) { - mock_db->check_destructor = true; - EXPECT_CALL(*mock_db, Die()); - delete mock_db; + mock_dbf->db[0]->check_destructor = true; + EXPECT_CALL(*(mock_dbf->db[0]), Die()); } TEST_F(FolderDBConverterTests, ErrorWhenCannotGetFileList) { @@ -138,23 +187,21 @@ TEST_F(FolderDBConverterTests, ErrorWhenCannotGetFileList) { auto error = converter.Convert(uri, folder); ASSERT_THAT(error, Eq(FolderToDbImportError::kIOError)); - delete mock_db; - } TEST_F(FolderDBConverterTests, PassesIgnoreDuplicates) { - EXPECT_CALL(*mock_db, Insert(_, true)); + EXPECT_CALL(*(mock_dbf->db[0]), Insert(_, true)); converter.IgnoreDuplicates(true); converter.Convert(uri, folder); - converter.IgnoreDuplicates(false); - } + + TEST_F(FolderDBConverterTests, ErrorWhenCannotImportFileListToDb) { - EXPECT_CALL(*mock_db, Insert(_, _)). + EXPECT_CALL(*(mock_dbf->db[0]), Insert(_, _)). WillOnce(testing::Return(DBError::kInsertError)); auto error = converter.Convert(uri, folder); @@ -173,7 +220,7 @@ MATCHER_P(CompareFileInfo, file, "") { TEST_F(FolderDBConverterTests, PassesFileListToInsert) { for (auto& file : file_infos) { - EXPECT_CALL(*mock_db, Insert(CompareFileInfo(file), _)). + EXPECT_CALL(*(mock_dbf->db[0]), Insert(CompareFileInfo(file), _)). WillOnce(testing::Return(DBError::kNoError)); } @@ -182,9 +229,37 @@ TEST_F(FolderDBConverterTests, PassesFileListToInsert) { } +TEST_F(FolderDBConverterTests, PassesFileListToInsertInParallel3by3) { + + EXPECT_CALL(*(mock_dbf->db[0]), Insert(CompareFileInfo(file_infos[0]), _)). + WillOnce(testing::Return(DBError::kNoError)); + EXPECT_CALL(*(mock_dbf->db[1]), Insert(CompareFileInfo(file_infos[1]), _)). + WillOnce(testing::Return(DBError::kNoError)); + EXPECT_CALL(*(mock_dbf->db[2]), Insert(CompareFileInfo(file_infos[2]), _)). + WillOnce(testing::Return(DBError::kNoError)); + + converter.SetNParallelTasks(3); + auto error = converter.Convert(uri, folder); + ASSERT_THAT(error, Eq(FolderToDbImportError::kOK)); +} + +TEST_F(FolderDBConverterTests, PassesFileListToInsertInParallel3by2) { + + EXPECT_CALL(*(mock_dbf->db[0]), Insert(CompareFileInfo(file_infos[0]), _)). + WillOnce(testing::Return(DBError::kNoError)); + EXPECT_CALL(*(mock_dbf->db[0]), Insert(CompareFileInfo(file_infos[1]), _)). + WillOnce(testing::Return(DBError::kNoError)); + EXPECT_CALL(*(mock_dbf->db[1]), Insert(CompareFileInfo(file_infos[2]), _)). + WillOnce(testing::Return(DBError::kNoError)); + + converter.SetNParallelTasks(2); + auto error = converter.Convert(uri, folder); + ASSERT_THAT(error, Eq(FolderToDbImportError::kOK)); +} + TEST_F(FolderDBConverterTests, ComputesStatistics) { - EXPECT_CALL(*mock_db, Insert(_, false)). + EXPECT_CALL(*mock_dbf->db[0], Insert(_, false)). Times(file_infos.size()). WillRepeatedly(testing::Return(DBError::kNoError)); @@ -198,4 +273,15 @@ TEST_F(FolderDBConverterTests, ComputesStatistics) { ASSERT_THAT(statistics.time_import_files.count(), Gt(0)); } + +TEST_F(FolderDBConverterTests, ErrorWhenCannotCreateDB) { + converter.db_factory__ = std::unique_ptr<DatabaseFactory> {new FakeDatabaseFactory}; + + auto err = converter.Convert("", ""); + + ASSERT_THAT(err, Eq(FolderToDbImportError::kMemoryError)); + +} + + }