diff --git a/CMakeLists.txt b/CMakeLists.txt index eeade3b9e256f6dcff785845511b5674bf4caaa1..bb5c947b864274b122c9b257d9905b6b33f22299 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,16 +13,24 @@ option(BUILD_DOCS "Uses doxygen to build the documentaion" OFF) set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMakeModules/) +set (HIDRA2_CXX_COMMON_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/common/cpp/include) + + # format sources include(astyle) include(testing_cpp) -add_subdirectory(producer/api) +if(BUILD_WORKER_TOOLS) + set (BUILD_MONGODB_CLIENTLIB ON) +endif() add_subdirectory(common/cpp) + +add_subdirectory(producer/api) add_subdirectory(worker/api/cpp) if(BUILD_WORKER_TOOLS) + set (BUILD_MONGODB ON) add_subdirectory(worker/tools) endif() diff --git a/CMakeModules/testing_cpp.cmake b/CMakeModules/testing_cpp.cmake index d875ea84e1fcf1bdc931c71431ad3cd6eace6a00..3d9f1fa941d40887ecb30ee28c21ff7c67f7670a 100644 --- a/CMakeModules/testing_cpp.cmake +++ b/CMakeModules/testing_cpp.cmake @@ -109,13 +109,16 @@ function(add_integration_test exename testname commandargs) endif () endfunction() -function(add_example_test testname) +function(add_script_test testname exename) if (BUILD_EXAMPLES) + set(args ${exename}) separate_arguments(args) IF (WIN32) - add_test(NAME test-${testname} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/check_windows.bat) + add_test(NAME test-${testname} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/check_windows.bat + ${args}) ELSE() - add_test(NAME test-${testname} COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/check_linux.sh) + add_test(NAME test-${testname} COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/check_linux.sh + ${args}) ENDIF() set_tests_properties(test-${testname} PROPERTIES LABELS "example;all" diff --git a/common/cpp/CMakeLists.txt b/common/cpp/CMakeLists.txt index 3d3950be158b3509e113124482a763dbb2fc6085..75587b54394fd0397147b955141d022a15b71b18 100644 --- a/common/cpp/CMakeLists.txt +++ b/common/cpp/CMakeLists.txt @@ -1,23 +1,7 @@ -set(TARGET_NAME common) -set(SOURCE_FILES - include/common/os.h - include/common/networking.h - include/system_wrappers/io.h - include/system_wrappers/system_io.h - src/system_io.cpp) -IF(WIN32) - set(SOURCE_FILES ${SOURCE_FILES} src/system_io_windows.cpp) -ELSEIF(UNIX) - set(SOURCE_FILES ${SOURCE_FILES} src/system_io_linux.cpp) -ENDIF(WIN32) +add_subdirectory(src/system_io) +if(BUILD_MONGODB_CLIENTLIB) + add_subdirectory(src/database) +endif() -################################ -# Library -################################ -add_library(${TARGET_NAME} OBJECT ${SOURCE_FILES}) -#add_library(${TARGET_NAME} SHARED ${SOURCE_FILES}) -target_include_directories(${TARGET_NAME} PUBLIC include) -set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX) - -install(DIRECTORY include/ DESTINATION include) +install(DIRECTORY ${HIDRA2_CXX_COMMON_INCLUDE_DIR} DESTINATION include) \ No newline at end of file diff --git a/common/cpp/include/common/file_info.h b/common/cpp/include/common/file_info.h index 16031c85cc1d6da0d1f082aa500e623fc28cdbff..d10773beb6b2c273239b7c81fdf9e5ef644558e1 100644 --- a/common/cpp/include/common/file_info.h +++ b/common/cpp/include/common/file_info.h @@ -4,6 +4,7 @@ #include <cinttypes> #include <chrono> #include <memory> +#include <vector> namespace hidra2 { @@ -15,7 +16,7 @@ struct FileInfo { }; using FileData = std::unique_ptr<uint8_t[]>; - +using FileInfos = std::vector<FileInfo>; } #endif //HIDRA2_FILE_INFO_H diff --git a/common/cpp/include/database/database.h b/common/cpp/include/database/database.h index 40f908fcf3d478fd61dfff89cb69f3cd7e8c7dc1..abe4a2dcc967666c9bab1256ce2da213d468be13 100644 --- a/common/cpp/include/database/database.h +++ b/common/cpp/include/database/database.h @@ -3,11 +3,14 @@ #include <string> +#include "common/file_info.h" + namespace hidra2 { enum class DBError { kNoError, - kConnectionError + kConnectionError, + kImportError }; constexpr char kDBName[] = "data"; @@ -16,6 +19,7 @@ class Database { public: virtual DBError Connect(const std::string& address, const std::string& database, const std::string& collection ) = 0; + virtual DBError Import(const FileInfos& files) const = 0; virtual ~Database() = default; }; diff --git a/common/cpp/include/database/mongo_database.h b/common/cpp/include/database/mongodb_client.h similarity index 76% rename from common/cpp/include/database/mongo_database.h rename to common/cpp/include/database/mongodb_client.h index e9ec8be639230ae7eebeab5e28fc091d4f3e3b64..7ae6dadd1465aa1e4e24d0779b12529a0588d369 100644 --- a/common/cpp/include/database/mongo_database.h +++ b/common/cpp/include/database/mongodb_client.h @@ -8,8 +8,10 @@ namespace hidra2 { class MongoDB final: public Database { public: virtual DBError Connect(const std::string& address, const std::string& database, - const std::string& collection ) override {}; - virtual ~MongoDB() override {}; + const std::string& collection ) override; + virtual DBError Import(const FileInfos& files) const override; + virtual ~MongoDB() override; + }; } diff --git a/common/cpp/include/system_wrappers/io.h b/common/cpp/include/system_wrappers/io.h index f9df7cfae3d251a2028f352e3399de32dea17796..fb180f96177ecc1e88a606cc5c324b2014798b17 100644 --- a/common/cpp/include/system_wrappers/io.h +++ b/common/cpp/include/system_wrappers/io.h @@ -34,7 +34,7 @@ class IO { virtual int64_t write(int __fd, const void* __buf, size_t __n) const noexcept = 0; // this is not standard function - to be implemented differently in windows and linux - virtual std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const = 0; + virtual FileInfos FilesInFolder(const std::string& folder, IOError* err) const = 0; }; } diff --git a/common/cpp/include/system_wrappers/system_io.h b/common/cpp/include/system_wrappers/system_io.h index f5ef95f04c8f4a978885cf6fcfbcb6367b9cc403..7a326abe24075767f4baca3d19193e4d717d0c88 100644 --- a/common/cpp/include/system_wrappers/system_io.h +++ b/common/cpp/include/system_wrappers/system_io.h @@ -12,11 +12,11 @@ class SystemIO final : public IO { int close(int __fd) const noexcept override; int64_t read(int __fd, void* buf, size_t count) const noexcept override; int64_t write(int __fd, const void* __buf, size_t __n) const noexcept override; - std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override; + FileInfos FilesInFolder(const std::string& folder, IOError* err) const override; private: void ReadWholeFile(int fd, uint8_t* array, uint64_t fsize, IOError* err) const noexcept; void CollectFileInformationRecursivly(const std::string& path, - std::vector<FileInfo>* files, IOError* err) const; + FileInfos* files, IOError* err) const; }; } diff --git a/common/cpp/include/unittests/MockIO.h b/common/cpp/include/unittests/MockIO.h index 9c71d323655f2b7232ee317d3a6661df42d7763b..b639f773bf3066fa541b4b48eac626b5cb77e553 100644 --- a/common/cpp/include/unittests/MockIO.h +++ b/common/cpp/include/unittests/MockIO.h @@ -28,7 +28,7 @@ class MockIO : public IO { MOCK_CONST_METHOD3(GetDataFromFile_t, FileData(const std::string& fname, uint64_t fsize, IOError* err)); MOCK_CONST_METHOD2(FilesInFolder, - std::vector<FileInfo>( + FileInfos( const std::string& folder, IOError *err)); diff --git a/common/cpp/src/database/CMakeLists.txt b/common/cpp/src/database/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..592a79029f2f1eb7deade9963290b520bd1a81d1 --- /dev/null +++ b/common/cpp/src/database/CMakeLists.txt @@ -0,0 +1,11 @@ +set(TARGET_NAME mongo_db) +set(SOURCE_FILES + mongodb_client.cpp) + +################################ +# Library +################################ +add_library(${TARGET_NAME} OBJECT ${SOURCE_FILES}) +target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR}) + + diff --git a/common/cpp/src/database/mongodb_client.cpp b/common/cpp/src/database/mongodb_client.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a45169fe070bbd026b1b3b3b16dd7cc73b912146 --- /dev/null +++ b/common/cpp/src/database/mongodb_client.cpp @@ -0,0 +1,21 @@ +#include "database/mongodb_client.h" + +namespace hidra2 { + +DBError MongoDB::Connect(const std::string& address, const std::string& database, + const std::string& collection ) { + return DBError::kNoError; +} + +DBError MongoDB::Import(const FileInfos& files) const { + return DBError::kNoError; +} + + +MongoDB::~MongoDB() { + +} + + + +} diff --git a/common/cpp/src/system_io/CMakeLists.txt b/common/cpp/src/system_io/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..627606d1a12053ea62b8d9dacdc344ff06faa53d --- /dev/null +++ b/common/cpp/src/system_io/CMakeLists.txt @@ -0,0 +1,17 @@ +set(TARGET_NAME system_io) +set(SOURCE_FILES + system_io.cpp) +IF(WIN32) + set(SOURCE_FILES ${SOURCE_FILES} system_io_windows.cpp) +ELSEIF(UNIX) + set(SOURCE_FILES ${SOURCE_FILES} system_io_linux.cpp) +ENDIF(WIN32) + + +################################ +# Library +################################ +add_library(${TARGET_NAME} OBJECT ${SOURCE_FILES}) +target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR}) + + diff --git a/common/cpp/src/system_io.cpp b/common/cpp/src/system_io/system_io.cpp similarity index 88% rename from common/cpp/src/system_io.cpp rename to common/cpp/src/system_io/system_io.cpp index 22780881e50f32042af564ef67f168855c298ad0..f3f79eba1045439ba911cc675be17d750e6bbbd8 100644 --- a/common/cpp/src/system_io.cpp +++ b/common/cpp/src/system_io/system_io.cpp @@ -72,22 +72,22 @@ FileData SystemIO::GetDataFromFile(const std::string& fname, uint64_t fsize, IOE return data; } -void SortFileList(std::vector<FileInfo>* file_list) { +void SortFileList(FileInfos* file_list) { std::sort(file_list->begin(), file_list->end(), [](FileInfo const & a, FileInfo const & b) { return a.modify_date < b.modify_date; }); } -void StripBasePath(const std::string& folder, std::vector<FileInfo>* file_list) { +void StripBasePath(const std::string& folder, FileInfos* file_list) { auto n_erase = folder.size() + 1; for (auto& file : *file_list) { file.relative_path.erase(0, n_erase); } } -std::vector<FileInfo> SystemIO::FilesInFolder(const std::string& folder, IOError* err) const { - std::vector<FileInfo> files{}; +FileInfos SystemIO::FilesInFolder(const std::string& folder, IOError* err) const { + FileInfos files{}; CollectFileInformationRecursivly(folder, &files, err); if (*err != IOError::kNoError) { return {}; diff --git a/common/cpp/src/system_io_linux.cpp b/common/cpp/src/system_io/system_io_linux.cpp similarity index 94% rename from common/cpp/src/system_io_linux.cpp rename to common/cpp/src/system_io/system_io_linux.cpp index 45452107b44b627726eeff817c0482dc036cf533..263ac3b6f0ac9d496f699f6d92ff79fa043f4b0a 100644 --- a/common/cpp/src/system_io_linux.cpp +++ b/common/cpp/src/system_io/system_io_linux.cpp @@ -73,7 +73,7 @@ FileInfo GetFileInfo(const string& path, const string& name, IOError* err) { } void ProcessFileEntity(const struct dirent* entity, const std::string& path, - std::vector<FileInfo>* files, IOError* err) { + FileInfos* files, IOError* err) { *err = IOError::kNoError; if (entity->d_type != DT_REG) { @@ -89,7 +89,7 @@ void ProcessFileEntity(const struct dirent* entity, const std::string& path, } void SystemIO::CollectFileInformationRecursivly(const std::string& path, - std::vector<FileInfo>* files, IOError* err) const { + FileInfos* files, IOError* err) const { auto dir = opendir((path).c_str()); if (dir == nullptr) { *err = IOErrorFromErrno(); diff --git a/common/cpp/src/system_io_windows.cpp b/common/cpp/src/system_io/system_io_windows.cpp similarity index 95% rename from common/cpp/src/system_io_windows.cpp rename to common/cpp/src/system_io/system_io_windows.cpp index ed1d3752e3b0e6b50ca5975d1b1aa1af460d231e..f1ee0a563505bc6db116fd63045322d42adf0856 100644 --- a/common/cpp/src/system_io_windows.cpp +++ b/common/cpp/src/system_io/system_io_windows.cpp @@ -59,7 +59,7 @@ bool IsDirectory(const WIN32_FIND_DATA f) { } void ProcessFileEntity(const WIN32_FIND_DATA f, const std::string& path, - std::vector<FileInfo>* files, IOError* err) { + FileInfos* files, IOError* err) { *err = IOError::kNoError; if (f.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { @@ -79,7 +79,7 @@ void ProcessFileEntity(const WIN32_FIND_DATA f, const std::string& path, void SystemIO::CollectFileInformationRecursivly(const std::string& path, - std::vector<FileInfo>* files, IOError* err) const { + FileInfos* files, IOError* err) const { WIN32_FIND_DATA find_data; HANDLE handle = FindFirstFile((path + "\\*.*").c_str(), &find_data); if (handle == INVALID_HANDLE_VALUE) { diff --git a/common/cpp/unittests/test_common.cpp b/common/cpp/unittests/test_common.cpp deleted file mode 100644 index 1e1b911ef88017e2c98dc5915b52b210f2e18fbf..0000000000000000000000000000000000000000 --- a/common/cpp/unittests/test_common.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include <gtest/gtest.h> - -TEST(EMPTY, REMOVEME) { - EXPECT_EQ(1, 1); -} diff --git a/examples/worker/process_folder/CMakeLists.txt b/examples/worker/process_folder/CMakeLists.txt index 4ec508a102a525b0d1ffdf7eab74fbf9057973b4..f023b46f85b975a24fb970daeb5724eeea23c5c5 100644 --- a/examples/worker/process_folder/CMakeLists.txt +++ b/examples/worker/process_folder/CMakeLists.txt @@ -10,13 +10,12 @@ set_target_properties(${TARGET_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ) get_target_property(VAR ${TARGET_NAME} RUNTIME_OUTPUT_DIRECTORY) -message (STATUS ${VAR}) if (CMAKE_COMPILER_IS_GNUCXX) set_target_properties(${TARGET_NAME} PROPERTIES LINK_FLAGS_DEBUG "--coverage") endif() -add_example_test("${TARGET_NAME}") +add_script_test("${TARGET_NAME}" "") set (dir examples/worker/process_folder) diff --git a/producer/api/CMakeLists.txt b/producer/api/CMakeLists.txt index 4bd5421dbf0434ebd19096fbea9fce167d124e96..db938ae4c1ad7dde2364954b8600eee6d27cc698 100644 --- a/producer/api/CMakeLists.txt +++ b/producer/api/CMakeLists.txt @@ -5,7 +5,7 @@ set(SOURCE_FILES src/producer/producer.cpp include/producer/producer.h) ################################ # Library ################################ -add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:common>) +add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io>) target_include_directories(${TARGET_NAME} PUBLIC include) set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX) diff --git a/tests/system_io/read_file_content/CMakeLists.txt b/tests/system_io/read_file_content/CMakeLists.txt index 55a84f07f8ee50efc8b61c927bfcac6e7e206270..9b1b0b4457692aba08559c3aa381d7a1008f5f4b 100644 --- a/tests/system_io/read_file_content/CMakeLists.txt +++ b/tests/system_io/read_file_content/CMakeLists.txt @@ -5,9 +5,9 @@ set(SOURCE_FILES read_file_content.cpp) ################################ # Executable and link ################################ -add_executable(${TARGET_NAME} ${SOURCE_FILES} $<TARGET_OBJECTS:common> ) +add_executable(${TARGET_NAME} ${SOURCE_FILES} $<TARGET_OBJECTS:system_io> ) target_link_libraries(${TARGET_NAME} test_common) -target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_SOURCE_DIR}/common/cpp/include) +target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR}) set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX) ################################ diff --git a/tests/system_io/read_files_in_folder/CMakeLists.txt b/tests/system_io/read_files_in_folder/CMakeLists.txt index cb22f8bd778be2e6b9df6421cc1b522108a35ff8..b87f07381a9ba8bd905a4632e10ed33fd7e4a0ff 100644 --- a/tests/system_io/read_files_in_folder/CMakeLists.txt +++ b/tests/system_io/read_files_in_folder/CMakeLists.txt @@ -5,9 +5,9 @@ set(SOURCE_FILES read_folder_content.cpp) ################################ # Executable and link ################################ -add_executable(${TARGET_NAME} ${SOURCE_FILES} $<TARGET_OBJECTS:common>) +add_executable(${TARGET_NAME} ${SOURCE_FILES} $<TARGET_OBJECTS:system_io>) target_link_libraries(${TARGET_NAME} test_common) -target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_SOURCE_DIR}/common/cpp/include) +target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR}) set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX) ################################ diff --git a/tests/worker/CMakeLists.txt b/tests/worker/CMakeLists.txt index 474587d3b3d595034555552a51ec52d9745fc260..87667dc2228b52dbddbf13520b29c6e6934c33c8 100644 --- a/tests/worker/CMakeLists.txt +++ b/tests/worker/CMakeLists.txt @@ -2,5 +2,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.7) # needed for fixtures add_subdirectory(next_multithread) add_subdirectory(connect_multithread) +add_subdirectory(folder_to_db) diff --git a/tests/worker/folder_to_db/CMakeLists.txt b/tests/worker/folder_to_db/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..cbd3410dca3eb1c4bfdc150c3ed6d6e05741f0f5 --- /dev/null +++ b/tests/worker/folder_to_db/CMakeLists.txt @@ -0,0 +1,7 @@ +set(TARGET_NAME folder2db) + +################################ +# Testing +################################ +add_script_test("${TARGET_NAME}" "$<TARGET_FILE:${TARGET_NAME}-bin>" + ) diff --git a/tests/worker/folder_to_db/check_linux.sh b/tests/worker/folder_to_db/check_linux.sh new file mode 100644 index 0000000000000000000000000000000000000000..3f3eae30637c1c8bf3667a757979453f8a938140 --- /dev/null +++ b/tests/worker/folder_to_db/check_linux.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +mkdir -p test +touch test/1 + +$1 test 123 + +rm -rf test diff --git a/tests/worker/folder_to_db/check_windows.bat b/tests/worker/folder_to_db/check_windows.bat new file mode 100644 index 0000000000000000000000000000000000000000..667999e8e6223227f9aedfbc032d093d11181ffc --- /dev/null +++ b/tests/worker/folder_to_db/check_windows.bat @@ -0,0 +1,7 @@ +mkdir test +echo "" > test/1 + +%1 test 123 + + +rmdir /S /Q test diff --git a/worker/api/cpp/CMakeLists.txt b/worker/api/cpp/CMakeLists.txt index da6376a3f0c99db6c85b9360c99499b61a69a958..97ab1f315d2cdc732fc269db94d831b8f09820ba 100644 --- a/worker/api/cpp/CMakeLists.txt +++ b/worker/api/cpp/CMakeLists.txt @@ -6,9 +6,9 @@ set(SOURCE_FILES include/worker/data_broker.h src/data_broker.cpp src/folder_dat ################################ # Library ################################ -add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:common>) +add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io>) -target_include_directories(${TARGET_NAME} PUBLIC include ${CMAKE_SOURCE_DIR}/common/cpp/include) +target_include_directories(${TARGET_NAME} PUBLIC include ${HIDRA2_CXX_COMMON_INCLUDE_DIR}) #target_link_libraries(${TARGET_NAME} $<TARGET_OBJECTS:common>) set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX) diff --git a/worker/api/cpp/src/folder_data_broker.h b/worker/api/cpp/src/folder_data_broker.h index f5b622bdaf66663f4ebcc023f51339198861ea81..ad8ca6b0624989f7e50e9e04a5987c1d45aa0a40 100644 --- a/worker/api/cpp/src/folder_data_broker.h +++ b/worker/api/cpp/src/folder_data_broker.h @@ -22,7 +22,7 @@ class FolderDataBroker final : public hidra2::DataBroker { bool is_connected_; int current_file_; std::string base_path_; - std::vector<FileInfo> filelist_; + FileInfos filelist_; WorkerErrorCode CanGetData(FileInfo* info, FileData* data, int nfile) const noexcept; std::mutex mutex_; diff --git a/worker/api/cpp/unittests/test_folder_broker.cpp b/worker/api/cpp/unittests/test_folder_broker.cpp index 324ec46bc1d79523ff1f82735e0c2802f5f8b98a..6cc57cef89248780f9b8253088ad4a949b767721 100644 --- a/worker/api/cpp/unittests/test_folder_broker.cpp +++ b/worker/api/cpp/unittests/test_folder_broker.cpp @@ -12,6 +12,7 @@ using hidra2::FolderDataBroker; using hidra2::WorkerErrorCode; using hidra2::IO; using hidra2::IOError; +using hidra2::FileInfos; using hidra2::FileInfo; using hidra2::FileData; @@ -58,9 +59,9 @@ class FakeIO: public IO { int64_t write(int __fd, const void* __buf, size_t __n) const noexcept override { return 0; }; - std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override { + FileInfos FilesInFolder(const std::string& folder, IOError* err) const override { *err = IOError::kNoError; - std::vector<FileInfo> file_infos; + FileInfos file_infos; FileInfo fi; fi.size = 100; fi.base_name = "1"; @@ -76,7 +77,7 @@ class FakeIO: public IO { class IOFolderNotFound: public FakeIO { public: - std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override { + FileInfos FilesInFolder(const std::string& folder, IOError* err) const override { *err = IOError::kFileNotFound; return {}; } @@ -84,7 +85,7 @@ class IOFolderNotFound: public FakeIO { class IOFodlerUnknownError: public FakeIO { public: - std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override { + FileInfos FilesInFolder(const std::string& folder, IOError* err) const override { *err = IOError::kUnknownError; return {}; } @@ -92,7 +93,7 @@ class IOFodlerUnknownError: public FakeIO { class IOEmptyFodler: public FakeIO { public: - std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override { + FileInfos FilesInFolder(const std::string& folder, IOError* err) const override { *err = IOError::kNoError; return {}; } diff --git a/worker/tools/folder_to_db/CMakeLists.txt b/worker/tools/folder_to_db/CMakeLists.txt index f93fca48dbdf73b0240a1d4c81d009fd4f107430..ebd8940178f88b8afea8387c35edd343983fd99a 100644 --- a/worker/tools/folder_to_db/CMakeLists.txt +++ b/worker/tools/folder_to_db/CMakeLists.txt @@ -1,14 +1,20 @@ set(TARGET_NAME folder2db) -set(SOURCE_FILES src/FolderDBConverter.cpp) +set(SOURCE_FILES src/folder_db_importer.cpp) ################################ # Library ################################ -add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:common>) +add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io> + $<TARGET_OBJECTS:mongo_db>) -target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_SOURCE_DIR}/common/cpp/include) +target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR}) + +add_executable(${TARGET_NAME}-bin src/main.cpp) +set_target_properties(${TARGET_NAME}-bin + PROPERTIES OUTPUT_NAME ${TARGET_NAME}) +target_link_libraries(${TARGET_NAME}-bin ${TARGET_NAME}) ################################ diff --git a/worker/tools/folder_to_db/src/FolderDBConverter.cpp b/worker/tools/folder_to_db/src/FolderDBConverter.cpp deleted file mode 100644 index 286937f5c2234f01742a29e5089dfa15759b23f4..0000000000000000000000000000000000000000 --- a/worker/tools/folder_to_db/src/FolderDBConverter.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include "FolderDBConverter.h" - -#include "system_wrappers/system_io.h" -#include "database/mongo_database.h" - -namespace hidra2 { - -FolderDBConverterError MapIOError(IOError io_err) { - FolderDBConverterError err; - switch (io_err) { - case IOError::kNoError: - err = FolderDBConverterError::kOK; - break; - default: - err = FolderDBConverterError::kIOError; - break; - } - return err; -} - -FolderDBConverterError MapDBError(DBError db_err) { - FolderDBConverterError err; - switch (db_err) { - case DBError::kNoError: - err = FolderDBConverterError::kOK; - break; - case DBError::kConnectionError: - err = FolderDBConverterError::kDBConnectionError; - break; - } - return err; -} - - - -FolderDBConverter::FolderDBConverter() : - io__{new hidra2::SystemIO}, db__{new hidra2::MongoDB} { -} - -FolderDBConverterError FolderDBConverter::Convert(const std::string& uri, const std::string& folder) { - DBError err_db = db__->Connect(uri, kDBName, folder); - if (err_db != DBError::kNoError) { - return MapDBError(err_db); - } - - IOError err_io; - auto file_list = io__->FilesInFolder(folder,&err_io); - if (err_io != IOError::kNoError){ - return MapIOError(err_io); - } - - return FolderDBConverterError::kOK; -} - -} \ No newline at end of file diff --git a/worker/tools/folder_to_db/src/folder_db_importer.cpp b/worker/tools/folder_to_db/src/folder_db_importer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..628ef0407ffc6633bc7b96ecf940d1753ec534f9 --- /dev/null +++ b/worker/tools/folder_to_db/src/folder_db_importer.cpp @@ -0,0 +1,74 @@ +#include "folder_db_importer.h" + +#include "system_wrappers/system_io.h" +#include "database/mongodb_client.h" + +namespace hidra2 { + +FolderToDbImportError MapIOError(IOError io_err) { + FolderToDbImportError err; + switch (io_err) { + case IOError::kNoError: + err = FolderToDbImportError::kOK; + break; + default: + err = FolderToDbImportError::kIOError; + break; + } + return err; +} + +FolderToDbImportError MapDBError(DBError db_err) { + FolderToDbImportError err; + switch (db_err) { + case DBError::kNoError: + err = FolderToDbImportError::kOK; + break; + case DBError::kConnectionError: + err = FolderToDbImportError::kDBConnectionError; + break; + case DBError::kImportError: + err = FolderToDbImportError::kImportError; + break; + } + return err; +} + + +FolderToDbImporter::FolderToDbImporter() : + io__{new hidra2::SystemIO}, db__{new hidra2::MongoDB} { +} + +FolderToDbImportError FolderToDbImporter::ConnectToDb(const std::string& uri, const std::string& folder) const { + DBError err = db__->Connect(uri, kDBName, folder); + return MapDBError(err); +} + +FolderToDbImportError FolderToDbImporter::ImportFilelist(FileInfos file_list) const { + auto err = db__->Import(file_list); + return MapDBError(err); +} + + +FileInfos FolderToDbImporter::GetFilesInFolder(const std::string& folder, FolderToDbImportError* err) const { + IOError err_io; + auto file_list = io__->FilesInFolder(folder, &err_io); + *err = MapIOError(err_io); + return file_list; +} + + +FolderToDbImportError FolderToDbImporter::Convert(const std::string& folder, const std::string& uri) const { + auto err = ConnectToDb(uri, folder); + if (err != FolderToDbImportError::kOK) { + return err; + } + + auto file_list = GetFilesInFolder(folder, &err); + if (err != FolderToDbImportError::kOK) { + return err; + } + + return ImportFilelist(file_list); +} +} \ No newline at end of file diff --git a/worker/tools/folder_to_db/src/FolderDBConverter.h b/worker/tools/folder_to_db/src/folder_db_importer.h similarity index 50% rename from worker/tools/folder_to_db/src/FolderDBConverter.h rename to worker/tools/folder_to_db/src/folder_db_importer.h index d78be1e8793c348fc7c2cc747da92725826f6251..48cf1449c55a5bbe1e5429fc0822480aacbf4b3e 100644 --- a/worker/tools/folder_to_db/src/FolderDBConverter.h +++ b/worker/tools/folder_to_db/src/folder_db_importer.h @@ -6,20 +6,25 @@ namespace hidra2 { -enum class FolderDBConverterError { +enum class FolderToDbImportError { kOK, kDBConnectionError, + kImportError, kIOError }; -class FolderDBConverter { +class FolderToDbImporter { public: - explicit FolderDBConverter(); + explicit FolderToDbImporter(); - FolderDBConverterError Convert(const std::string& uri, const std::string& folder); + FolderToDbImportError Convert(const std::string& folder, const std::string& uri) const; std::unique_ptr<hidra2::IO> io__; // modified in testings to mock system calls,otherwise do not touch std::unique_ptr<hidra2::Database> db__; // modified in testings to mock system calls,otherwise do not touch + private: + FolderToDbImportError ConnectToDb(const std::string& uri, const std::string& folder) const; + FileInfos GetFilesInFolder(const std::string& folder, FolderToDbImportError* err) const; + FolderToDbImportError ImportFilelist(FileInfos file_list) const; }; diff --git a/worker/tools/folder_to_db/src/main.cpp b/worker/tools/folder_to_db/src/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7f19e0f58a90bfef43f7fd0454c70f4291c80d7c --- /dev/null +++ b/worker/tools/folder_to_db/src/main.cpp @@ -0,0 +1,34 @@ +#include <string> +#include <iostream> + +#include "folder_db_importer.h" + + +struct InputParameters { + std::string folder; + std::string uri; +}; + +InputParameters ProcessCommandArguments(int argc, char* argv[]) { + if (argc != 3) { + std::cout << "Usage: " + std::string{argv[0]} +" <path to folder> <database uri>" << std::endl; + exit(EXIT_FAILURE); + } + + return InputParameters{argv[1], argv[2]}; +} + + +int main(int argc, char* argv[]) { + auto command_args = ProcessCommandArguments(argc, argv); + + hidra2::FolderToDbImporter importer; + + auto err = importer.Convert(command_args.folder, command_args.uri); + if (err != hidra2::FolderToDbImportError::kOK) { + return 1; + } + + return 0; +} + diff --git a/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp b/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp index 92fb23afc19ffe668943b1c21de9b4ed29d8d216..0f136e31d98b0e8b71b6ee83de9adce6f51d3f28 100644 --- a/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp +++ b/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp @@ -6,11 +6,11 @@ #include "database/database.h" -#include "database/mongo_database.h" +#include "database/mongodb_client.h" #include "common/file_info.h" -#include "../src/FolderDBConverter.h" +#include "../src/folder_db_importer.h" #include "unittests/MockIO.h" @@ -22,14 +22,16 @@ using ::testing::Test; using ::testing::_; using ::testing::Mock; using ::testing::NiceMock; +using ::testing::Ref; -using hidra2::FolderDBConverter; -using hidra2::FolderDBConverterError; +using hidra2::FolderToDbImporter; +using hidra2::FolderToDbImportError; using hidra2::Database; using hidra2::IO; using hidra2::DBError; using hidra2::IOError; using hidra2::kDBName; +using hidra2::FileInfos; using hidra2::FileInfo; using hidra2::MockIO; @@ -37,18 +39,19 @@ using hidra2::MockIO; namespace { TEST(FolderDBConverter, SetCorrectIO) { - FolderDBConverter converter{}; + FolderToDbImporter converter{}; ASSERT_THAT(dynamic_cast<hidra2::SystemIO*>(converter.io__.get()), Ne(nullptr)); } TEST(FolderDBConverter, SetCorrectDB) { - FolderDBConverter converter{}; + FolderToDbImporter converter{}; ASSERT_THAT(dynamic_cast<hidra2::MongoDB*>(converter.db__.get()), Ne(nullptr)); } class MockDatabase : public Database { public: MOCK_METHOD3(Connect, DBError (const std::string&, const std::string&, const std::string&)); + MOCK_CONST_METHOD1(Import, DBError (const FileInfos&)); // stuff to test db destructor is called and avoid "uninteresting call" messages MOCK_METHOD0(Die, void()); @@ -60,11 +63,9 @@ class MockDatabase : public Database { }; - - class FolderDBConverterTests : public Test { public: - FolderDBConverter converter{}; + FolderToDbImporter converter{}; NiceMock<MockDatabase> mock_db; NiceMock<MockIO> mock_io; @@ -89,8 +90,8 @@ TEST_F(FolderDBConverterTests, ErrorWhenCannotConnect) { EXPECT_CALL(mock_db, Connect(uri, kDBName, "")). WillOnce(testing::Return(DBError::kConnectionError)); - auto error = converter.Convert(uri, ""); - ASSERT_THAT(error, Eq(FolderDBConverterError::kDBConnectionError)); + auto error = converter.Convert("", uri); + ASSERT_THAT(error, Eq(FolderToDbImportError::kDBConnectionError)); } TEST_F(FolderDBConverterTests, DBDestructorCalled) { @@ -103,14 +104,66 @@ TEST_F(FolderDBConverterTests, ErrorWhenCannotGetFileList) { std::string folder{"folder"}; EXPECT_CALL(mock_io, FilesInFolder(folder, _)). - WillOnce(DoAll(testing::SetArgPointee<1>(IOError::kReadError), testing::Return(std::vector<FileInfo>{}))); + WillOnce(DoAll(testing::SetArgPointee<1>(IOError::kReadError), testing::Return(FileInfos {}))); + + + auto error = converter.Convert(folder, ""); + ASSERT_THAT(error, Eq(FolderToDbImportError::kIOError)); + +} + +TEST_F(FolderDBConverterTests, ErrorWhenCannotImportFileListToDb) { + + EXPECT_CALL(mock_io, FilesInFolder(_, _)). + WillOnce(DoAll(testing::SetArgPointee<1>(IOError::kNoError), + testing::Return(FileInfos {}))); + EXPECT_CALL(mock_db, Import(_)). + WillOnce(testing::Return(DBError::kImportError)); - auto error = converter.Convert("", folder); - ASSERT_THAT(error, Eq(FolderDBConverterError::kIOError)); + auto error = converter.Convert("", ""); + ASSERT_THAT(error, Eq(FolderToDbImportError::kImportError)); } +// a matcher to compare file_infos (size and basename only) for testing purposes +// (we do not want to create an == operator for FileInfo) +MATCHER_P(CompareFileInfos, file_infos, "") { + if (arg.size() != file_infos.size()) return false; + for (int i = 0; i < arg.size(); i++) { + if (arg[i].size != file_infos[i].size) return false; + if (arg[i].base_name != file_infos[i].base_name) return false; + } +} +FileInfos CreateTestFileInfos() { + FileInfos file_infos; + FileInfo fi; + fi.size = 100; + fi.base_name = "1"; + file_infos.push_back(fi); + fi.base_name = "2"; + file_infos.push_back(fi); + fi.base_name = "3"; + file_infos.push_back(fi); + return file_infos; +} + + +TEST_F(FolderDBConverterTests, PassesFileListToImport) { + + auto file_infos = CreateTestFileInfos(); + + EXPECT_CALL(mock_io, FilesInFolder(_, _)). + WillOnce(DoAll(testing::SetArgPointee<1>(IOError::kNoError), + testing::Return(file_infos))); + + EXPECT_CALL(mock_db, Import(CompareFileInfos(file_infos))). + WillOnce(testing::Return(DBError::kNoError)); + + auto error = converter.Convert("", ""); + ASSERT_THAT(error, Eq(FolderToDbImportError::kOK)); + +} }