From f4f539dd5c0c339a28db2d2b056b007108236cb4 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon, 8 Jan 2018 14:41:42 +0100
Subject: [PATCH] refactor, added db import tool with tests, started class for
mongodb
---
CMakeLists.txt | 10 ++-
CMakeModules/testing_cpp.cmake | 9 ++-
common/cpp/CMakeLists.txt | 26 ++----
common/cpp/include/common/file_info.h | 3 +-
common/cpp/include/database/database.h | 6 +-
.../{mongo_database.h => mongodb_client.h} | 6 +-
common/cpp/include/system_wrappers/io.h | 2 +-
.../cpp/include/system_wrappers/system_io.h | 4 +-
common/cpp/include/unittests/MockIO.h | 2 +-
common/cpp/src/database/CMakeLists.txt | 11 +++
common/cpp/src/database/mongodb_client.cpp | 21 +++++
common/cpp/src/system_io/CMakeLists.txt | 17 ++++
common/cpp/src/{ => system_io}/system_io.cpp | 8 +-
.../src/{ => system_io}/system_io_linux.cpp | 4 +-
.../src/{ => system_io}/system_io_windows.cpp | 4 +-
common/cpp/unittests/test_common.cpp | 5 --
examples/worker/process_folder/CMakeLists.txt | 3 +-
producer/api/CMakeLists.txt | 2 +-
.../read_file_content/CMakeLists.txt | 4 +-
.../read_files_in_folder/CMakeLists.txt | 4 +-
tests/worker/CMakeLists.txt | 1 +
tests/worker/folder_to_db/CMakeLists.txt | 7 ++
tests/worker/folder_to_db/check_linux.sh | 10 +++
tests/worker/folder_to_db/check_windows.bat | 7 ++
worker/api/cpp/CMakeLists.txt | 4 +-
worker/api/cpp/src/folder_data_broker.h | 2 +-
.../api/cpp/unittests/test_folder_broker.cpp | 11 +--
worker/tools/folder_to_db/CMakeLists.txt | 12 ++-
.../folder_to_db/src/FolderDBConverter.cpp | 55 -------------
.../folder_to_db/src/folder_db_importer.cpp | 74 +++++++++++++++++
...lderDBConverter.h => folder_db_importer.h} | 13 ++-
worker/tools/folder_to_db/src/main.cpp | 34 ++++++++
.../unittests/test_folder_to_db.cpp | 81 +++++++++++++++----
33 files changed, 325 insertions(+), 137 deletions(-)
rename common/cpp/include/database/{mongo_database.h => mongodb_client.h} (76%)
create mode 100644 common/cpp/src/database/CMakeLists.txt
create mode 100644 common/cpp/src/database/mongodb_client.cpp
create mode 100644 common/cpp/src/system_io/CMakeLists.txt
rename common/cpp/src/{ => system_io}/system_io.cpp (88%)
rename common/cpp/src/{ => system_io}/system_io_linux.cpp (94%)
rename common/cpp/src/{ => system_io}/system_io_windows.cpp (95%)
delete mode 100644 common/cpp/unittests/test_common.cpp
create mode 100644 tests/worker/folder_to_db/CMakeLists.txt
create mode 100644 tests/worker/folder_to_db/check_linux.sh
create mode 100644 tests/worker/folder_to_db/check_windows.bat
delete mode 100644 worker/tools/folder_to_db/src/FolderDBConverter.cpp
create mode 100644 worker/tools/folder_to_db/src/folder_db_importer.cpp
rename worker/tools/folder_to_db/src/{FolderDBConverter.h => folder_db_importer.h} (50%)
create mode 100644 worker/tools/folder_to_db/src/main.cpp
diff --git a/CMakeLists.txt b/CMakeLists.txt
index eeade3b9e..bb5c947b8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -13,16 +13,24 @@ option(BUILD_DOCS "Uses doxygen to build the documentaion" OFF)
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMakeModules/)
+set (HIDRA2_CXX_COMMON_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/common/cpp/include)
+
+
# format sources
include(astyle)
include(testing_cpp)
-add_subdirectory(producer/api)
+if(BUILD_WORKER_TOOLS)
+ set (BUILD_MONGODB_CLIENTLIB ON)
+endif()
add_subdirectory(common/cpp)
+
+add_subdirectory(producer/api)
add_subdirectory(worker/api/cpp)
if(BUILD_WORKER_TOOLS)
+ set (BUILD_MONGODB ON)
add_subdirectory(worker/tools)
endif()
diff --git a/CMakeModules/testing_cpp.cmake b/CMakeModules/testing_cpp.cmake
index d875ea84e..3d9f1fa94 100644
--- a/CMakeModules/testing_cpp.cmake
+++ b/CMakeModules/testing_cpp.cmake
@@ -109,13 +109,16 @@ function(add_integration_test exename testname commandargs)
endif ()
endfunction()
-function(add_example_test testname)
+function(add_script_test testname exename)
if (BUILD_EXAMPLES)
+ set(args ${exename})
separate_arguments(args)
IF (WIN32)
- add_test(NAME test-${testname} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/check_windows.bat)
+ add_test(NAME test-${testname} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/check_windows.bat
+ ${args})
ELSE()
- add_test(NAME test-${testname} COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/check_linux.sh)
+ add_test(NAME test-${testname} COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/check_linux.sh
+ ${args})
ENDIF()
set_tests_properties(test-${testname} PROPERTIES
LABELS "example;all"
diff --git a/common/cpp/CMakeLists.txt b/common/cpp/CMakeLists.txt
index 3d3950be1..75587b543 100644
--- a/common/cpp/CMakeLists.txt
+++ b/common/cpp/CMakeLists.txt
@@ -1,23 +1,7 @@
-set(TARGET_NAME common)
-set(SOURCE_FILES
- include/common/os.h
- include/common/networking.h
- include/system_wrappers/io.h
- include/system_wrappers/system_io.h
- src/system_io.cpp)
-IF(WIN32)
- set(SOURCE_FILES ${SOURCE_FILES} src/system_io_windows.cpp)
-ELSEIF(UNIX)
- set(SOURCE_FILES ${SOURCE_FILES} src/system_io_linux.cpp)
-ENDIF(WIN32)
+add_subdirectory(src/system_io)
+if(BUILD_MONGODB_CLIENTLIB)
+ add_subdirectory(src/database)
+endif()
-################################
-# Library
-################################
-add_library(${TARGET_NAME} OBJECT ${SOURCE_FILES})
-#add_library(${TARGET_NAME} SHARED ${SOURCE_FILES})
-target_include_directories(${TARGET_NAME} PUBLIC include)
-set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
-
-install(DIRECTORY include/ DESTINATION include)
+install(DIRECTORY ${HIDRA2_CXX_COMMON_INCLUDE_DIR} DESTINATION include)
\ No newline at end of file
diff --git a/common/cpp/include/common/file_info.h b/common/cpp/include/common/file_info.h
index 16031c85c..d10773beb 100644
--- a/common/cpp/include/common/file_info.h
+++ b/common/cpp/include/common/file_info.h
@@ -4,6 +4,7 @@
#include <cinttypes>
#include <chrono>
#include <memory>
+#include <vector>
namespace hidra2 {
@@ -15,7 +16,7 @@ struct FileInfo {
};
using FileData = std::unique_ptr<uint8_t[]>;
-
+using FileInfos = std::vector<FileInfo>;
}
#endif //HIDRA2_FILE_INFO_H
diff --git a/common/cpp/include/database/database.h b/common/cpp/include/database/database.h
index 40f908fcf..abe4a2dcc 100644
--- a/common/cpp/include/database/database.h
+++ b/common/cpp/include/database/database.h
@@ -3,11 +3,14 @@
#include <string>
+#include "common/file_info.h"
+
namespace hidra2 {
enum class DBError {
kNoError,
- kConnectionError
+ kConnectionError,
+ kImportError
};
constexpr char kDBName[] = "data";
@@ -16,6 +19,7 @@ class Database {
public:
virtual DBError Connect(const std::string& address, const std::string& database,
const std::string& collection ) = 0;
+ virtual DBError Import(const FileInfos& files) const = 0;
virtual ~Database() = default;
};
diff --git a/common/cpp/include/database/mongo_database.h b/common/cpp/include/database/mongodb_client.h
similarity index 76%
rename from common/cpp/include/database/mongo_database.h
rename to common/cpp/include/database/mongodb_client.h
index e9ec8be63..7ae6dadd1 100644
--- a/common/cpp/include/database/mongo_database.h
+++ b/common/cpp/include/database/mongodb_client.h
@@ -8,8 +8,10 @@ namespace hidra2 {
class MongoDB final: public Database {
public:
virtual DBError Connect(const std::string& address, const std::string& database,
- const std::string& collection ) override {};
- virtual ~MongoDB() override {};
+ const std::string& collection ) override;
+ virtual DBError Import(const FileInfos& files) const override;
+ virtual ~MongoDB() override;
+
};
}
diff --git a/common/cpp/include/system_wrappers/io.h b/common/cpp/include/system_wrappers/io.h
index f9df7cfae..fb180f961 100644
--- a/common/cpp/include/system_wrappers/io.h
+++ b/common/cpp/include/system_wrappers/io.h
@@ -34,7 +34,7 @@ class IO {
virtual int64_t write(int __fd, const void* __buf, size_t __n) const noexcept = 0;
// this is not standard function - to be implemented differently in windows and linux
- virtual std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const = 0;
+ virtual FileInfos FilesInFolder(const std::string& folder, IOError* err) const = 0;
};
}
diff --git a/common/cpp/include/system_wrappers/system_io.h b/common/cpp/include/system_wrappers/system_io.h
index f5ef95f04..7a326abe2 100644
--- a/common/cpp/include/system_wrappers/system_io.h
+++ b/common/cpp/include/system_wrappers/system_io.h
@@ -12,11 +12,11 @@ class SystemIO final : public IO {
int close(int __fd) const noexcept override;
int64_t read(int __fd, void* buf, size_t count) const noexcept override;
int64_t write(int __fd, const void* __buf, size_t __n) const noexcept override;
- std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override;
+ FileInfos FilesInFolder(const std::string& folder, IOError* err) const override;
private:
void ReadWholeFile(int fd, uint8_t* array, uint64_t fsize, IOError* err) const noexcept;
void CollectFileInformationRecursivly(const std::string& path,
- std::vector<FileInfo>* files, IOError* err) const;
+ FileInfos* files, IOError* err) const;
};
}
diff --git a/common/cpp/include/unittests/MockIO.h b/common/cpp/include/unittests/MockIO.h
index 9c71d3236..b639f773b 100644
--- a/common/cpp/include/unittests/MockIO.h
+++ b/common/cpp/include/unittests/MockIO.h
@@ -28,7 +28,7 @@ class MockIO : public IO {
MOCK_CONST_METHOD3(GetDataFromFile_t,
FileData(const std::string& fname, uint64_t fsize, IOError* err));
MOCK_CONST_METHOD2(FilesInFolder,
- std::vector<FileInfo>(
+ FileInfos(
const std::string& folder, IOError
*err));
diff --git a/common/cpp/src/database/CMakeLists.txt b/common/cpp/src/database/CMakeLists.txt
new file mode 100644
index 000000000..592a79029
--- /dev/null
+++ b/common/cpp/src/database/CMakeLists.txt
@@ -0,0 +1,11 @@
+set(TARGET_NAME mongo_db)
+set(SOURCE_FILES
+ mongodb_client.cpp)
+
+################################
+# Library
+################################
+add_library(${TARGET_NAME} OBJECT ${SOURCE_FILES})
+target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR})
+
+
diff --git a/common/cpp/src/database/mongodb_client.cpp b/common/cpp/src/database/mongodb_client.cpp
new file mode 100644
index 000000000..a45169fe0
--- /dev/null
+++ b/common/cpp/src/database/mongodb_client.cpp
@@ -0,0 +1,21 @@
+#include "database/mongodb_client.h"
+
+namespace hidra2 {
+
+DBError MongoDB::Connect(const std::string& address, const std::string& database,
+ const std::string& collection ) {
+ return DBError::kNoError;
+}
+
+DBError MongoDB::Import(const FileInfos& files) const {
+ return DBError::kNoError;
+}
+
+
+MongoDB::~MongoDB() {
+
+}
+
+
+
+}
diff --git a/common/cpp/src/system_io/CMakeLists.txt b/common/cpp/src/system_io/CMakeLists.txt
new file mode 100644
index 000000000..627606d1a
--- /dev/null
+++ b/common/cpp/src/system_io/CMakeLists.txt
@@ -0,0 +1,17 @@
+set(TARGET_NAME system_io)
+set(SOURCE_FILES
+ system_io.cpp)
+IF(WIN32)
+ set(SOURCE_FILES ${SOURCE_FILES} system_io_windows.cpp)
+ELSEIF(UNIX)
+ set(SOURCE_FILES ${SOURCE_FILES} system_io_linux.cpp)
+ENDIF(WIN32)
+
+
+################################
+# Library
+################################
+add_library(${TARGET_NAME} OBJECT ${SOURCE_FILES})
+target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR})
+
+
diff --git a/common/cpp/src/system_io.cpp b/common/cpp/src/system_io/system_io.cpp
similarity index 88%
rename from common/cpp/src/system_io.cpp
rename to common/cpp/src/system_io/system_io.cpp
index 22780881e..f3f79eba1 100644
--- a/common/cpp/src/system_io.cpp
+++ b/common/cpp/src/system_io/system_io.cpp
@@ -72,22 +72,22 @@ FileData SystemIO::GetDataFromFile(const std::string& fname, uint64_t fsize, IOE
return data;
}
-void SortFileList(std::vector<FileInfo>* file_list) {
+void SortFileList(FileInfos* file_list) {
std::sort(file_list->begin(), file_list->end(),
[](FileInfo const & a, FileInfo const & b) {
return a.modify_date < b.modify_date;
});
}
-void StripBasePath(const std::string& folder, std::vector<FileInfo>* file_list) {
+void StripBasePath(const std::string& folder, FileInfos* file_list) {
auto n_erase = folder.size() + 1;
for (auto& file : *file_list) {
file.relative_path.erase(0, n_erase);
}
}
-std::vector<FileInfo> SystemIO::FilesInFolder(const std::string& folder, IOError* err) const {
- std::vector<FileInfo> files{};
+FileInfos SystemIO::FilesInFolder(const std::string& folder, IOError* err) const {
+ FileInfos files{};
CollectFileInformationRecursivly(folder, &files, err);
if (*err != IOError::kNoError) {
return {};
diff --git a/common/cpp/src/system_io_linux.cpp b/common/cpp/src/system_io/system_io_linux.cpp
similarity index 94%
rename from common/cpp/src/system_io_linux.cpp
rename to common/cpp/src/system_io/system_io_linux.cpp
index 45452107b..263ac3b6f 100644
--- a/common/cpp/src/system_io_linux.cpp
+++ b/common/cpp/src/system_io/system_io_linux.cpp
@@ -73,7 +73,7 @@ FileInfo GetFileInfo(const string& path, const string& name, IOError* err) {
}
void ProcessFileEntity(const struct dirent* entity, const std::string& path,
- std::vector<FileInfo>* files, IOError* err) {
+ FileInfos* files, IOError* err) {
*err = IOError::kNoError;
if (entity->d_type != DT_REG) {
@@ -89,7 +89,7 @@ void ProcessFileEntity(const struct dirent* entity, const std::string& path,
}
void SystemIO::CollectFileInformationRecursivly(const std::string& path,
- std::vector<FileInfo>* files, IOError* err) const {
+ FileInfos* files, IOError* err) const {
auto dir = opendir((path).c_str());
if (dir == nullptr) {
*err = IOErrorFromErrno();
diff --git a/common/cpp/src/system_io_windows.cpp b/common/cpp/src/system_io/system_io_windows.cpp
similarity index 95%
rename from common/cpp/src/system_io_windows.cpp
rename to common/cpp/src/system_io/system_io_windows.cpp
index ed1d3752e..f1ee0a563 100644
--- a/common/cpp/src/system_io_windows.cpp
+++ b/common/cpp/src/system_io/system_io_windows.cpp
@@ -59,7 +59,7 @@ bool IsDirectory(const WIN32_FIND_DATA f) {
}
void ProcessFileEntity(const WIN32_FIND_DATA f, const std::string& path,
- std::vector<FileInfo>* files, IOError* err) {
+ FileInfos* files, IOError* err) {
*err = IOError::kNoError;
if (f.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
@@ -79,7 +79,7 @@ void ProcessFileEntity(const WIN32_FIND_DATA f, const std::string& path,
void SystemIO::CollectFileInformationRecursivly(const std::string& path,
- std::vector<FileInfo>* files, IOError* err) const {
+ FileInfos* files, IOError* err) const {
WIN32_FIND_DATA find_data;
HANDLE handle = FindFirstFile((path + "\\*.*").c_str(), &find_data);
if (handle == INVALID_HANDLE_VALUE) {
diff --git a/common/cpp/unittests/test_common.cpp b/common/cpp/unittests/test_common.cpp
deleted file mode 100644
index 1e1b911ef..000000000
--- a/common/cpp/unittests/test_common.cpp
+++ /dev/null
@@ -1,5 +0,0 @@
-#include <gtest/gtest.h>
-
-TEST(EMPTY, REMOVEME) {
- EXPECT_EQ(1, 1);
-}
diff --git a/examples/worker/process_folder/CMakeLists.txt b/examples/worker/process_folder/CMakeLists.txt
index 4ec508a10..f023b46f8 100644
--- a/examples/worker/process_folder/CMakeLists.txt
+++ b/examples/worker/process_folder/CMakeLists.txt
@@ -10,13 +10,12 @@ set_target_properties(${TARGET_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY
)
get_target_property(VAR ${TARGET_NAME} RUNTIME_OUTPUT_DIRECTORY)
-message (STATUS ${VAR})
if (CMAKE_COMPILER_IS_GNUCXX)
set_target_properties(${TARGET_NAME} PROPERTIES LINK_FLAGS_DEBUG "--coverage")
endif()
-add_example_test("${TARGET_NAME}")
+add_script_test("${TARGET_NAME}" "")
set (dir examples/worker/process_folder)
diff --git a/producer/api/CMakeLists.txt b/producer/api/CMakeLists.txt
index 4bd5421db..db938ae4c 100644
--- a/producer/api/CMakeLists.txt
+++ b/producer/api/CMakeLists.txt
@@ -5,7 +5,7 @@ set(SOURCE_FILES src/producer/producer.cpp include/producer/producer.h)
################################
# Library
################################
-add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:common>)
+add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io>)
target_include_directories(${TARGET_NAME} PUBLIC include)
set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
diff --git a/tests/system_io/read_file_content/CMakeLists.txt b/tests/system_io/read_file_content/CMakeLists.txt
index 55a84f07f..9b1b0b445 100644
--- a/tests/system_io/read_file_content/CMakeLists.txt
+++ b/tests/system_io/read_file_content/CMakeLists.txt
@@ -5,9 +5,9 @@ set(SOURCE_FILES read_file_content.cpp)
################################
# Executable and link
################################
-add_executable(${TARGET_NAME} ${SOURCE_FILES} $<TARGET_OBJECTS:common> )
+add_executable(${TARGET_NAME} ${SOURCE_FILES} $<TARGET_OBJECTS:system_io> )
target_link_libraries(${TARGET_NAME} test_common)
-target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_SOURCE_DIR}/common/cpp/include)
+target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR})
set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
################################
diff --git a/tests/system_io/read_files_in_folder/CMakeLists.txt b/tests/system_io/read_files_in_folder/CMakeLists.txt
index cb22f8bd7..b87f07381 100644
--- a/tests/system_io/read_files_in_folder/CMakeLists.txt
+++ b/tests/system_io/read_files_in_folder/CMakeLists.txt
@@ -5,9 +5,9 @@ set(SOURCE_FILES read_folder_content.cpp)
################################
# Executable and link
################################
-add_executable(${TARGET_NAME} ${SOURCE_FILES} $<TARGET_OBJECTS:common>)
+add_executable(${TARGET_NAME} ${SOURCE_FILES} $<TARGET_OBJECTS:system_io>)
target_link_libraries(${TARGET_NAME} test_common)
-target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_SOURCE_DIR}/common/cpp/include)
+target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR})
set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
################################
diff --git a/tests/worker/CMakeLists.txt b/tests/worker/CMakeLists.txt
index 474587d3b..87667dc22 100644
--- a/tests/worker/CMakeLists.txt
+++ b/tests/worker/CMakeLists.txt
@@ -2,5 +2,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.7) # needed for fixtures
add_subdirectory(next_multithread)
add_subdirectory(connect_multithread)
+add_subdirectory(folder_to_db)
diff --git a/tests/worker/folder_to_db/CMakeLists.txt b/tests/worker/folder_to_db/CMakeLists.txt
new file mode 100644
index 000000000..cbd3410dc
--- /dev/null
+++ b/tests/worker/folder_to_db/CMakeLists.txt
@@ -0,0 +1,7 @@
+set(TARGET_NAME folder2db)
+
+################################
+# Testing
+################################
+add_script_test("${TARGET_NAME}" "$<TARGET_FILE:${TARGET_NAME}-bin>"
+ )
diff --git a/tests/worker/folder_to_db/check_linux.sh b/tests/worker/folder_to_db/check_linux.sh
new file mode 100644
index 000000000..3f3eae306
--- /dev/null
+++ b/tests/worker/folder_to_db/check_linux.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+mkdir -p test
+touch test/1
+
+$1 test 123
+
+rm -rf test
diff --git a/tests/worker/folder_to_db/check_windows.bat b/tests/worker/folder_to_db/check_windows.bat
new file mode 100644
index 000000000..667999e8e
--- /dev/null
+++ b/tests/worker/folder_to_db/check_windows.bat
@@ -0,0 +1,7 @@
+mkdir test
+echo "" > test/1
+
+%1 test 123
+
+
+rmdir /S /Q test
diff --git a/worker/api/cpp/CMakeLists.txt b/worker/api/cpp/CMakeLists.txt
index da6376a3f..97ab1f315 100644
--- a/worker/api/cpp/CMakeLists.txt
+++ b/worker/api/cpp/CMakeLists.txt
@@ -6,9 +6,9 @@ set(SOURCE_FILES include/worker/data_broker.h src/data_broker.cpp src/folder_dat
################################
# Library
################################
-add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:common>)
+add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io>)
-target_include_directories(${TARGET_NAME} PUBLIC include ${CMAKE_SOURCE_DIR}/common/cpp/include)
+target_include_directories(${TARGET_NAME} PUBLIC include ${HIDRA2_CXX_COMMON_INCLUDE_DIR})
#target_link_libraries(${TARGET_NAME} $<TARGET_OBJECTS:common>)
set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
diff --git a/worker/api/cpp/src/folder_data_broker.h b/worker/api/cpp/src/folder_data_broker.h
index f5b622bda..ad8ca6b06 100644
--- a/worker/api/cpp/src/folder_data_broker.h
+++ b/worker/api/cpp/src/folder_data_broker.h
@@ -22,7 +22,7 @@ class FolderDataBroker final : public hidra2::DataBroker {
bool is_connected_;
int current_file_;
std::string base_path_;
- std::vector<FileInfo> filelist_;
+ FileInfos filelist_;
WorkerErrorCode CanGetData(FileInfo* info, FileData* data, int nfile) const noexcept;
std::mutex mutex_;
diff --git a/worker/api/cpp/unittests/test_folder_broker.cpp b/worker/api/cpp/unittests/test_folder_broker.cpp
index 324ec46bc..6cc57cef8 100644
--- a/worker/api/cpp/unittests/test_folder_broker.cpp
+++ b/worker/api/cpp/unittests/test_folder_broker.cpp
@@ -12,6 +12,7 @@ using hidra2::FolderDataBroker;
using hidra2::WorkerErrorCode;
using hidra2::IO;
using hidra2::IOError;
+using hidra2::FileInfos;
using hidra2::FileInfo;
using hidra2::FileData;
@@ -58,9 +59,9 @@ class FakeIO: public IO {
int64_t write(int __fd, const void* __buf, size_t __n) const noexcept override {
return 0;
};
- std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override {
+ FileInfos FilesInFolder(const std::string& folder, IOError* err) const override {
*err = IOError::kNoError;
- std::vector<FileInfo> file_infos;
+ FileInfos file_infos;
FileInfo fi;
fi.size = 100;
fi.base_name = "1";
@@ -76,7 +77,7 @@ class FakeIO: public IO {
class IOFolderNotFound: public FakeIO {
public:
- std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override {
+ FileInfos FilesInFolder(const std::string& folder, IOError* err) const override {
*err = IOError::kFileNotFound;
return {};
}
@@ -84,7 +85,7 @@ class IOFolderNotFound: public FakeIO {
class IOFodlerUnknownError: public FakeIO {
public:
- std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override {
+ FileInfos FilesInFolder(const std::string& folder, IOError* err) const override {
*err = IOError::kUnknownError;
return {};
}
@@ -92,7 +93,7 @@ class IOFodlerUnknownError: public FakeIO {
class IOEmptyFodler: public FakeIO {
public:
- std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override {
+ FileInfos FilesInFolder(const std::string& folder, IOError* err) const override {
*err = IOError::kNoError;
return {};
}
diff --git a/worker/tools/folder_to_db/CMakeLists.txt b/worker/tools/folder_to_db/CMakeLists.txt
index f93fca48d..ebd894017 100644
--- a/worker/tools/folder_to_db/CMakeLists.txt
+++ b/worker/tools/folder_to_db/CMakeLists.txt
@@ -1,14 +1,20 @@
set(TARGET_NAME folder2db)
-set(SOURCE_FILES src/FolderDBConverter.cpp)
+set(SOURCE_FILES src/folder_db_importer.cpp)
################################
# Library
################################
-add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:common>)
+add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:system_io>
+ $<TARGET_OBJECTS:mongo_db>)
-target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_SOURCE_DIR}/common/cpp/include)
+target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR})
+
+add_executable(${TARGET_NAME}-bin src/main.cpp)
+set_target_properties(${TARGET_NAME}-bin
+ PROPERTIES OUTPUT_NAME ${TARGET_NAME})
+target_link_libraries(${TARGET_NAME}-bin ${TARGET_NAME})
################################
diff --git a/worker/tools/folder_to_db/src/FolderDBConverter.cpp b/worker/tools/folder_to_db/src/FolderDBConverter.cpp
deleted file mode 100644
index 286937f5c..000000000
--- a/worker/tools/folder_to_db/src/FolderDBConverter.cpp
+++ /dev/null
@@ -1,55 +0,0 @@
-#include "FolderDBConverter.h"
-
-#include "system_wrappers/system_io.h"
-#include "database/mongo_database.h"
-
-namespace hidra2 {
-
-FolderDBConverterError MapIOError(IOError io_err) {
- FolderDBConverterError err;
- switch (io_err) {
- case IOError::kNoError:
- err = FolderDBConverterError::kOK;
- break;
- default:
- err = FolderDBConverterError::kIOError;
- break;
- }
- return err;
-}
-
-FolderDBConverterError MapDBError(DBError db_err) {
- FolderDBConverterError err;
- switch (db_err) {
- case DBError::kNoError:
- err = FolderDBConverterError::kOK;
- break;
- case DBError::kConnectionError:
- err = FolderDBConverterError::kDBConnectionError;
- break;
- }
- return err;
-}
-
-
-
-FolderDBConverter::FolderDBConverter() :
- io__{new hidra2::SystemIO}, db__{new hidra2::MongoDB} {
-}
-
-FolderDBConverterError FolderDBConverter::Convert(const std::string& uri, const std::string& folder) {
- DBError err_db = db__->Connect(uri, kDBName, folder);
- if (err_db != DBError::kNoError) {
- return MapDBError(err_db);
- }
-
- IOError err_io;
- auto file_list = io__->FilesInFolder(folder,&err_io);
- if (err_io != IOError::kNoError){
- return MapIOError(err_io);
- }
-
- return FolderDBConverterError::kOK;
-}
-
-}
\ No newline at end of file
diff --git a/worker/tools/folder_to_db/src/folder_db_importer.cpp b/worker/tools/folder_to_db/src/folder_db_importer.cpp
new file mode 100644
index 000000000..628ef0407
--- /dev/null
+++ b/worker/tools/folder_to_db/src/folder_db_importer.cpp
@@ -0,0 +1,74 @@
+#include "folder_db_importer.h"
+
+#include "system_wrappers/system_io.h"
+#include "database/mongodb_client.h"
+
+namespace hidra2 {
+
+FolderToDbImportError MapIOError(IOError io_err) {
+ FolderToDbImportError err;
+ switch (io_err) {
+ case IOError::kNoError:
+ err = FolderToDbImportError::kOK;
+ break;
+ default:
+ err = FolderToDbImportError::kIOError;
+ break;
+ }
+ return err;
+}
+
+FolderToDbImportError MapDBError(DBError db_err) {
+ FolderToDbImportError err;
+ switch (db_err) {
+ case DBError::kNoError:
+ err = FolderToDbImportError::kOK;
+ break;
+ case DBError::kConnectionError:
+ err = FolderToDbImportError::kDBConnectionError;
+ break;
+ case DBError::kImportError:
+ err = FolderToDbImportError::kImportError;
+ break;
+ }
+ return err;
+}
+
+
+FolderToDbImporter::FolderToDbImporter() :
+ io__{new hidra2::SystemIO}, db__{new hidra2::MongoDB} {
+}
+
+FolderToDbImportError FolderToDbImporter::ConnectToDb(const std::string& uri, const std::string& folder) const {
+ DBError err = db__->Connect(uri, kDBName, folder);
+ return MapDBError(err);
+}
+
+FolderToDbImportError FolderToDbImporter::ImportFilelist(FileInfos file_list) const {
+ auto err = db__->Import(file_list);
+ return MapDBError(err);
+}
+
+
+FileInfos FolderToDbImporter::GetFilesInFolder(const std::string& folder, FolderToDbImportError* err) const {
+ IOError err_io;
+ auto file_list = io__->FilesInFolder(folder, &err_io);
+ *err = MapIOError(err_io);
+ return file_list;
+}
+
+
+FolderToDbImportError FolderToDbImporter::Convert(const std::string& folder, const std::string& uri) const {
+ auto err = ConnectToDb(uri, folder);
+ if (err != FolderToDbImportError::kOK) {
+ return err;
+ }
+
+ auto file_list = GetFilesInFolder(folder, &err);
+ if (err != FolderToDbImportError::kOK) {
+ return err;
+ }
+
+ return ImportFilelist(file_list);
+}
+}
\ No newline at end of file
diff --git a/worker/tools/folder_to_db/src/FolderDBConverter.h b/worker/tools/folder_to_db/src/folder_db_importer.h
similarity index 50%
rename from worker/tools/folder_to_db/src/FolderDBConverter.h
rename to worker/tools/folder_to_db/src/folder_db_importer.h
index d78be1e87..48cf1449c 100644
--- a/worker/tools/folder_to_db/src/FolderDBConverter.h
+++ b/worker/tools/folder_to_db/src/folder_db_importer.h
@@ -6,20 +6,25 @@
namespace hidra2 {
-enum class FolderDBConverterError {
+enum class FolderToDbImportError {
kOK,
kDBConnectionError,
+ kImportError,
kIOError
};
-class FolderDBConverter {
+class FolderToDbImporter {
public:
- explicit FolderDBConverter();
+ explicit FolderToDbImporter();
- FolderDBConverterError Convert(const std::string& uri, const std::string& folder);
+ FolderToDbImportError Convert(const std::string& folder, const std::string& uri) const;
std::unique_ptr<hidra2::IO> io__; // modified in testings to mock system calls,otherwise do not touch
std::unique_ptr<hidra2::Database> db__; // modified in testings to mock system calls,otherwise do not touch
+ private:
+ FolderToDbImportError ConnectToDb(const std::string& uri, const std::string& folder) const;
+ FileInfos GetFilesInFolder(const std::string& folder, FolderToDbImportError* err) const;
+ FolderToDbImportError ImportFilelist(FileInfos file_list) const;
};
diff --git a/worker/tools/folder_to_db/src/main.cpp b/worker/tools/folder_to_db/src/main.cpp
new file mode 100644
index 000000000..7f19e0f58
--- /dev/null
+++ b/worker/tools/folder_to_db/src/main.cpp
@@ -0,0 +1,34 @@
+#include <string>
+#include <iostream>
+
+#include "folder_db_importer.h"
+
+
+struct InputParameters {
+ std::string folder;
+ std::string uri;
+};
+
+InputParameters ProcessCommandArguments(int argc, char* argv[]) {
+ if (argc != 3) {
+ std::cout << "Usage: " + std::string{argv[0]} +" <path to folder> <database uri>" << std::endl;
+ exit(EXIT_FAILURE);
+ }
+
+ return InputParameters{argv[1], argv[2]};
+}
+
+
+int main(int argc, char* argv[]) {
+ auto command_args = ProcessCommandArguments(argc, argv);
+
+ hidra2::FolderToDbImporter importer;
+
+ auto err = importer.Convert(command_args.folder, command_args.uri);
+ if (err != hidra2::FolderToDbImportError::kOK) {
+ return 1;
+ }
+
+ return 0;
+}
+
diff --git a/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp b/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp
index 92fb23afc..0f136e31d 100644
--- a/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp
+++ b/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp
@@ -6,11 +6,11 @@
#include "database/database.h"
-#include "database/mongo_database.h"
+#include "database/mongodb_client.h"
#include "common/file_info.h"
-#include "../src/FolderDBConverter.h"
+#include "../src/folder_db_importer.h"
#include "unittests/MockIO.h"
@@ -22,14 +22,16 @@ using ::testing::Test;
using ::testing::_;
using ::testing::Mock;
using ::testing::NiceMock;
+using ::testing::Ref;
-using hidra2::FolderDBConverter;
-using hidra2::FolderDBConverterError;
+using hidra2::FolderToDbImporter;
+using hidra2::FolderToDbImportError;
using hidra2::Database;
using hidra2::IO;
using hidra2::DBError;
using hidra2::IOError;
using hidra2::kDBName;
+using hidra2::FileInfos;
using hidra2::FileInfo;
using hidra2::MockIO;
@@ -37,18 +39,19 @@ using hidra2::MockIO;
namespace {
TEST(FolderDBConverter, SetCorrectIO) {
- FolderDBConverter converter{};
+ FolderToDbImporter converter{};
ASSERT_THAT(dynamic_cast<hidra2::SystemIO*>(converter.io__.get()), Ne(nullptr));
}
TEST(FolderDBConverter, SetCorrectDB) {
- FolderDBConverter converter{};
+ FolderToDbImporter converter{};
ASSERT_THAT(dynamic_cast<hidra2::MongoDB*>(converter.db__.get()), Ne(nullptr));
}
class MockDatabase : public Database {
public:
MOCK_METHOD3(Connect, DBError (const std::string&, const std::string&, const std::string&));
+ MOCK_CONST_METHOD1(Import, DBError (const FileInfos&));
// stuff to test db destructor is called and avoid "uninteresting call" messages
MOCK_METHOD0(Die, void());
@@ -60,11 +63,9 @@ class MockDatabase : public Database {
};
-
-
class FolderDBConverterTests : public Test {
public:
- FolderDBConverter converter{};
+ FolderToDbImporter converter{};
NiceMock<MockDatabase> mock_db;
NiceMock<MockIO> mock_io;
@@ -89,8 +90,8 @@ TEST_F(FolderDBConverterTests, ErrorWhenCannotConnect) {
EXPECT_CALL(mock_db, Connect(uri, kDBName, "")).
WillOnce(testing::Return(DBError::kConnectionError));
- auto error = converter.Convert(uri, "");
- ASSERT_THAT(error, Eq(FolderDBConverterError::kDBConnectionError));
+ auto error = converter.Convert("", uri);
+ ASSERT_THAT(error, Eq(FolderToDbImportError::kDBConnectionError));
}
TEST_F(FolderDBConverterTests, DBDestructorCalled) {
@@ -103,14 +104,66 @@ TEST_F(FolderDBConverterTests, ErrorWhenCannotGetFileList) {
std::string folder{"folder"};
EXPECT_CALL(mock_io, FilesInFolder(folder, _)).
- WillOnce(DoAll(testing::SetArgPointee<1>(IOError::kReadError), testing::Return(std::vector<FileInfo>{})));
+ WillOnce(DoAll(testing::SetArgPointee<1>(IOError::kReadError), testing::Return(FileInfos {})));
+
+
+ auto error = converter.Convert(folder, "");
+ ASSERT_THAT(error, Eq(FolderToDbImportError::kIOError));
+
+}
+
+TEST_F(FolderDBConverterTests, ErrorWhenCannotImportFileListToDb) {
+
+ EXPECT_CALL(mock_io, FilesInFolder(_, _)).
+ WillOnce(DoAll(testing::SetArgPointee<1>(IOError::kNoError),
+ testing::Return(FileInfos {})));
+ EXPECT_CALL(mock_db, Import(_)).
+ WillOnce(testing::Return(DBError::kImportError));
- auto error = converter.Convert("", folder);
- ASSERT_THAT(error, Eq(FolderDBConverterError::kIOError));
+ auto error = converter.Convert("", "");
+ ASSERT_THAT(error, Eq(FolderToDbImportError::kImportError));
}
+// a matcher to compare file_infos (size and basename only) for testing purposes
+// (we do not want to create an == operator for FileInfo)
+MATCHER_P(CompareFileInfos, file_infos, "") {
+ if (arg.size() != file_infos.size()) return false;
+ for (int i = 0; i < arg.size(); i++) {
+ if (arg[i].size != file_infos[i].size) return false;
+ if (arg[i].base_name != file_infos[i].base_name) return false;
+ }
+}
+FileInfos CreateTestFileInfos() {
+ FileInfos file_infos;
+ FileInfo fi;
+ fi.size = 100;
+ fi.base_name = "1";
+ file_infos.push_back(fi);
+ fi.base_name = "2";
+ file_infos.push_back(fi);
+ fi.base_name = "3";
+ file_infos.push_back(fi);
+ return file_infos;
+}
+
+
+TEST_F(FolderDBConverterTests, PassesFileListToImport) {
+
+ auto file_infos = CreateTestFileInfos();
+
+ EXPECT_CALL(mock_io, FilesInFolder(_, _)).
+ WillOnce(DoAll(testing::SetArgPointee<1>(IOError::kNoError),
+ testing::Return(file_infos)));
+
+ EXPECT_CALL(mock_db, Import(CompareFileInfos(file_infos))).
+ WillOnce(testing::Return(DBError::kNoError));
+
+ auto error = converter.Convert("", "");
+ ASSERT_THAT(error, Eq(FolderToDbImportError::kOK));
+
+}
}
--
GitLab