From f515c1f3a3c576dd8a44475b72376f5c48addef6 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Fri, 5 Jan 2018 16:20:55 +0100
Subject: [PATCH] start working on convert tool
---
CMakeLists.txt | 5 +
common/cpp/include/common/file_info.h | 2 +-
common/cpp/include/database/database.h | 24 ++++
common/cpp/include/database/mongo_database.h | 17 +++
common/cpp/include/system_wrappers/io.h | 8 +-
.../cpp/include/system_wrappers/system_io.h | 8 +-
common/cpp/include/unittests/MockIO.h | 55 +++++++++
common/cpp/src/system_io.cpp | 30 ++---
common/cpp/src/system_io_linux.cpp | 16 +--
common/cpp/src/system_io_windows.cpp | 28 ++---
.../read_file_content/read_file_content.cpp | 12 +-
.../read_folder_content.cpp | 10 +-
worker/api/cpp/src/folder_data_broker.cpp | 18 +--
.../api/cpp/unittests/test_folder_broker.cpp | 40 +++---
worker/tools/CMakeLists.txt | 2 +
worker/tools/folder_to_db/CMakeLists.txt | 21 ++++
.../folder_to_db/src/FolderDBConverter.cpp | 55 +++++++++
.../folder_to_db/src/FolderDBConverter.h | 28 +++++
.../unittests/test_folder_to_db.cpp | 116 ++++++++++++++++++
19 files changed, 409 insertions(+), 86 deletions(-)
create mode 100644 common/cpp/include/database/database.h
create mode 100644 common/cpp/include/database/mongo_database.h
create mode 100644 common/cpp/include/unittests/MockIO.h
create mode 100644 worker/tools/CMakeLists.txt
create mode 100644 worker/tools/folder_to_db/CMakeLists.txt
create mode 100644 worker/tools/folder_to_db/src/FolderDBConverter.cpp
create mode 100644 worker/tools/folder_to_db/src/FolderDBConverter.h
create mode 100644 worker/tools/folder_to_db/unittests/test_folder_to_db.cpp
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6ee7afc36..eeade3b9e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -21,6 +21,11 @@ include(testing_cpp)
add_subdirectory(producer/api)
add_subdirectory(common/cpp)
add_subdirectory(worker/api/cpp)
+
+if(BUILD_WORKER_TOOLS)
+ add_subdirectory(worker/tools)
+endif()
+
IF(UNIX)
add_subdirectory(receiver)
ENDIF(UNIX)
diff --git a/common/cpp/include/common/file_info.h b/common/cpp/include/common/file_info.h
index 88bd4f4af..16031c85c 100644
--- a/common/cpp/include/common/file_info.h
+++ b/common/cpp/include/common/file_info.h
@@ -2,8 +2,8 @@
#define HIDRA2_FILE_INFO_H
#include <cinttypes>
-#include <memory>
#include <chrono>
+#include <memory>
namespace hidra2 {
diff --git a/common/cpp/include/database/database.h b/common/cpp/include/database/database.h
new file mode 100644
index 000000000..40f908fcf
--- /dev/null
+++ b/common/cpp/include/database/database.h
@@ -0,0 +1,24 @@
+#ifndef HIDRA2_DATABASE_H
+#define HIDRA2_DATABASE_H
+
+#include <string>
+
+namespace hidra2 {
+
+enum class DBError {
+ kNoError,
+ kConnectionError
+};
+
+constexpr char kDBName[] = "data";
+
+class Database {
+ public:
+ virtual DBError Connect(const std::string& address, const std::string& database,
+ const std::string& collection ) = 0;
+ virtual ~Database() = default;
+};
+
+}
+
+#endif //HIDRA2_DATABASE_H
diff --git a/common/cpp/include/database/mongo_database.h b/common/cpp/include/database/mongo_database.h
new file mode 100644
index 000000000..e9ec8be63
--- /dev/null
+++ b/common/cpp/include/database/mongo_database.h
@@ -0,0 +1,17 @@
+#ifndef HIDRA2_MONGO_DATABASE_H
+#define HIDRA2_MONGO_DATABASE_H
+
+#include "database.h"
+
+namespace hidra2 {
+
+class MongoDB final: public Database {
+ public:
+ virtual DBError Connect(const std::string& address, const std::string& database,
+ const std::string& collection ) override {};
+ virtual ~MongoDB() override {};
+};
+
+}
+
+#endif //HIDRA2_MONGO_DATABASE_H
diff --git a/common/cpp/include/system_wrappers/io.h b/common/cpp/include/system_wrappers/io.h
index 88bb23570..f9df7cfae 100644
--- a/common/cpp/include/system_wrappers/io.h
+++ b/common/cpp/include/system_wrappers/io.h
@@ -11,7 +11,7 @@
namespace hidra2 {
-enum class IOErrors {
+enum class IOError {
kNoError,
kFileNotFound,
kReadError,
@@ -20,13 +20,13 @@ enum class IOErrors {
kMemoryAllocationError
};
-IOErrors IOErrorFromErrno();
+IOError IOErrorFromErrno();
class IO {
public:
- virtual FileData GetDataFromFile(const std::string& fname, uint64_t fsize, IOErrors* err) const noexcept = 0;
+ virtual FileData GetDataFromFile(const std::string& fname, uint64_t fsize, IOError* err) const noexcept = 0;
virtual int open(const char* __file, int __oflag) const noexcept = 0;
virtual int close(int __fd) const noexcept = 0;
@@ -34,7 +34,7 @@ class IO {
virtual int64_t write(int __fd, const void* __buf, size_t __n) const noexcept = 0;
// this is not standard function - to be implemented differently in windows and linux
- virtual std::vector<FileInfo> FilesInFolder(const std::string& folder, IOErrors* err) const = 0;
+ virtual std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const = 0;
};
}
diff --git a/common/cpp/include/system_wrappers/system_io.h b/common/cpp/include/system_wrappers/system_io.h
index 6c2c62646..f5ef95f04 100644
--- a/common/cpp/include/system_wrappers/system_io.h
+++ b/common/cpp/include/system_wrappers/system_io.h
@@ -7,16 +7,16 @@ namespace hidra2 {
class SystemIO final : public IO {
public:
- FileData GetDataFromFile(const std::string& fname, uint64_t fsize, IOErrors* err) const noexcept override;
+ FileData GetDataFromFile(const std::string& fname, uint64_t fsize, IOError* err) const noexcept override;
int open(const char* __file, int __oflag) const noexcept override;
int close(int __fd) const noexcept override;
int64_t read(int __fd, void* buf, size_t count) const noexcept override;
int64_t write(int __fd, const void* __buf, size_t __n) const noexcept override;
- std::vector<FileInfo> FilesInFolder(const std::string& folder, IOErrors* err) const override;
+ std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override;
private:
- void ReadWholeFile(int fd, uint8_t* array, uint64_t fsize, IOErrors* err) const noexcept;
+ void ReadWholeFile(int fd, uint8_t* array, uint64_t fsize, IOError* err) const noexcept;
void CollectFileInformationRecursivly(const std::string& path,
- std::vector<FileInfo>* files, IOErrors* err) const;
+ std::vector<FileInfo>* files, IOError* err) const;
};
}
diff --git a/common/cpp/include/unittests/MockIO.h b/common/cpp/include/unittests/MockIO.h
new file mode 100644
index 000000000..9c71d3236
--- /dev/null
+++ b/common/cpp/include/unittests/MockIO.h
@@ -0,0 +1,55 @@
+#ifndef HIDRA2_COMMON__MOCKIO_H
+#define HIDRA2_COMMON__MOCKIO_H
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
+#include "system_wrappers/io.h"
+
+namespace hidra2 {
+
+class MockIO : public IO {
+ public:
+ FileData GetDataFromFile(const std::string& fname, uint64_t fsize, IOError* err) const noexcept override {
+ return GetDataFromFile_t(fname, fsize, err);
+ }
+ int open(const char* __file, int __oflag) const noexcept override {
+ return open_t(__file, __oflag);
+ }
+ int close(int __fd) const noexcept override {
+ return close_t(__fd);
+ }
+ int64_t read(int __fd, void* buf, size_t count) const noexcept override {
+ return read_t(__fd, buf, count);
+ }
+ int64_t write(int __fd, const void* __buf, size_t __n) const noexcept override {
+ return write_t(__fd, __buf, __n);
+ }
+ MOCK_CONST_METHOD3(GetDataFromFile_t,
+ FileData(const std::string& fname, uint64_t fsize, IOError* err));
+ MOCK_CONST_METHOD2(FilesInFolder,
+ std::vector<FileInfo>(
+ const std::string& folder, IOError
+ *err));
+
+ MOCK_CONST_METHOD3(read_t,
+ int64_t(int
+ __fd, void* buf, size_t
+ count));
+
+ MOCK_CONST_METHOD3(write_t,
+ int64_t(int
+ __fd,
+ const void* __buf, size_t
+ __n));
+
+ MOCK_CONST_METHOD2(open_t,
+ int(const char* __file, int __oflag));
+
+ MOCK_CONST_METHOD1(close_t,
+ int(int __fd));
+};
+
+}
+
+#endif //HIDRA2_COMMON__MOCKIO_H
\ No newline at end of file
diff --git a/common/cpp/src/system_io.cpp b/common/cpp/src/system_io.cpp
index 6ee383e0c..22780881e 100644
--- a/common/cpp/src/system_io.cpp
+++ b/common/cpp/src/system_io.cpp
@@ -9,27 +9,27 @@
namespace hidra2 {
-IOErrors IOErrorFromErrno() {
- IOErrors err;
+IOError IOErrorFromErrno() {
+ IOError err;
switch (errno) {
case 0:
- err = IOErrors::kNoError;
+ err = IOError::kNoError;
break;
case ENOENT:
case ENOTDIR:
- err = IOErrors::kFileNotFound;
+ err = IOError::kFileNotFound;
break;
case EACCES:
- err = IOErrors::kPermissionDenied;
+ err = IOError::kPermissionDenied;
break;
default:
- err = IOErrors::kUnknownError;
+ err = IOError::kUnknownError;
break;
}
return err;
}
-void SystemIO::ReadWholeFile(int fd, uint8_t* array, uint64_t fsize, IOErrors* err) const noexcept {
+void SystemIO::ReadWholeFile(int fd, uint8_t* array, uint64_t fsize, IOError* err) const noexcept {
uint64_t totalbytes = 0;
int64_t readbytes = 0;
do {
@@ -38,34 +38,34 @@ void SystemIO::ReadWholeFile(int fd, uint8_t* array, uint64_t fsize, IOErrors* e
} while (readbytes > 0 && totalbytes < fsize);
if (totalbytes != fsize) {
- *err = IOErrors::kReadError;
+ *err = IOError::kReadError;
}
}
-FileData SystemIO::GetDataFromFile(const std::string& fname, uint64_t fsize, IOErrors* err) const noexcept {
+FileData SystemIO::GetDataFromFile(const std::string& fname, uint64_t fsize, IOError* err) const noexcept {
int fd = open(fname.c_str(), O_RDONLY);
*err = IOErrorFromErrno();
- if (*err != IOErrors::kNoError) {
+ if (*err != IOError::kNoError) {
return nullptr;
}
uint8_t* data_array = nullptr;
try {
data_array = new uint8_t[fsize];
} catch (...) {
- *err = IOErrors::kMemoryAllocationError;
+ *err = IOError::kMemoryAllocationError;
return nullptr;
}
ReadWholeFile(fd, data_array, fsize, err);
FileData data{data_array};
- if (*err != IOErrors::kNoError) {
+ if (*err != IOError::kNoError) {
close(fd);
return nullptr;
}
close(fd);
*err = IOErrorFromErrno();
- if (*err != IOErrors::kNoError) {
+ if (*err != IOError::kNoError) {
return nullptr;
}
@@ -86,10 +86,10 @@ void StripBasePath(const std::string& folder, std::vector<FileInfo>* file_list)
}
}
-std::vector<FileInfo> SystemIO::FilesInFolder(const std::string& folder, IOErrors* err) const {
+std::vector<FileInfo> SystemIO::FilesInFolder(const std::string& folder, IOError* err) const {
std::vector<FileInfo> files{};
CollectFileInformationRecursivly(folder, &files, err);
- if (*err != IOErrors::kNoError) {
+ if (*err != IOError::kNoError) {
return {};
}
StripBasePath(folder, &files);
diff --git a/common/cpp/src/system_io_linux.cpp b/common/cpp/src/system_io_linux.cpp
index 3c1bc09bb..45452107b 100644
--- a/common/cpp/src/system_io_linux.cpp
+++ b/common/cpp/src/system_io_linux.cpp
@@ -46,7 +46,7 @@ void SetFileName(const string& path, const string& name, FileInfo* file_info) {
file_info->base_name = name;
}
-struct stat FileStat(const string& fname, IOErrors* err) {
+struct stat FileStat(const string& fname, IOError* err) {
struct stat t_stat {};
int res = stat(fname.c_str(), &t_stat);
if (res < 0) {
@@ -55,13 +55,13 @@ struct stat FileStat(const string& fname, IOErrors* err) {
return t_stat;
}
-FileInfo GetFileInfo(const string& path, const string& name, IOErrors* err) {
+FileInfo GetFileInfo(const string& path, const string& name, IOError* err) {
FileInfo file_info;
SetFileName(path, name, &file_info);
auto t_stat = FileStat(path + "/" + name, err);
- if (*err != IOErrors::kNoError) {
+ if (*err != IOError::kNoError) {
return FileInfo{};
}
@@ -73,15 +73,15 @@ FileInfo GetFileInfo(const string& path, const string& name, IOErrors* err) {
}
void ProcessFileEntity(const struct dirent* entity, const std::string& path,
- std::vector<FileInfo>* files, IOErrors* err) {
+ std::vector<FileInfo>* files, IOError* err) {
- *err = IOErrors::kNoError;
+ *err = IOError::kNoError;
if (entity->d_type != DT_REG) {
return;
}
FileInfo file_info = GetFileInfo(path, entity->d_name, err);
- if (*err != IOErrors::kNoError) {
+ if (*err != IOError::kNoError) {
return;
}
@@ -89,7 +89,7 @@ void ProcessFileEntity(const struct dirent* entity, const std::string& path,
}
void SystemIO::CollectFileInformationRecursivly(const std::string& path,
- std::vector<FileInfo>* files, IOErrors* err) const {
+ std::vector<FileInfo>* files, IOError* err) const {
auto dir = opendir((path).c_str());
if (dir == nullptr) {
*err = IOErrorFromErrno();
@@ -103,7 +103,7 @@ void SystemIO::CollectFileInformationRecursivly(const std::string& path,
} else {
ProcessFileEntity(current_entity, path, files, err);
}
- if (*err != IOErrors::kNoError) {
+ if (*err != IOError::kNoError) {
closedir(dir);
return;
}
diff --git a/common/cpp/src/system_io_windows.cpp b/common/cpp/src/system_io_windows.cpp
index 5653ce87c..ed1d3752e 100644
--- a/common/cpp/src/system_io_windows.cpp
+++ b/common/cpp/src/system_io_windows.cpp
@@ -12,27 +12,27 @@ using std::chrono::system_clock;
namespace hidra2 {
-IOErrors IOErrorFromGetLastError() {
- IOErrors err;
+IOError IOErrorFromGetLastError() {
+ IOError err;
switch (GetLastError()) {
case ERROR_SUCCESS :
- err = IOErrors::kNoError;
+ err = IOError::kNoError;
break;
case ERROR_PATH_NOT_FOUND:
case ERROR_FILE_NOT_FOUND:
- err = IOErrors::kFileNotFound;
+ err = IOError::kFileNotFound;
break;
case ERROR_ACCESS_DENIED:
- err = IOErrors::kPermissionDenied;
+ err = IOError::kPermissionDenied;
break;
default:
- err = IOErrors::kUnknownError;
+ err = IOError::kUnknownError;
break;
}
return err;
}
-std::chrono::system_clock::time_point FileTime2TimePoint(const FILETIME& ft, IOErrors* err) {
+std::chrono::system_clock::time_point FileTime2TimePoint(const FILETIME& ft, IOError* err) {
SYSTEMTIME st = {0};
if (!FileTimeToSystemTime(&ft, &st)) {
*err = IOErrorFromGetLastError();
@@ -48,7 +48,7 @@ std::chrono::system_clock::time_point FileTime2TimePoint(const FILETIME& ft, IOE
auto tp = std::chrono::system_clock::from_time_t(secs);
tp += ms;
- *err = IOErrors::kNoError;
+ *err = IOError::kNoError;
return tp;
}
@@ -59,16 +59,16 @@ bool IsDirectory(const WIN32_FIND_DATA f) {
}
void ProcessFileEntity(const WIN32_FIND_DATA f, const std::string& path,
- std::vector<FileInfo>* files, IOErrors* err) {
+ std::vector<FileInfo>* files, IOError* err) {
- *err = IOErrors::kNoError;
+ *err = IOError::kNoError;
if (f.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
return;
}
FileInfo file_info;
file_info.modify_date = FileTime2TimePoint(f.ftLastWriteTime, err);
- if (*err != IOErrors::kNoError) {
+ if (*err != IOError::kNoError) {
return;
}
@@ -79,7 +79,7 @@ void ProcessFileEntity(const WIN32_FIND_DATA f, const std::string& path,
void SystemIO::CollectFileInformationRecursivly(const std::string& path,
- std::vector<FileInfo>* files, IOErrors* err) const {
+ std::vector<FileInfo>* files, IOError* err) const {
WIN32_FIND_DATA find_data;
HANDLE handle = FindFirstFile((path + "\\*.*").c_str(), &find_data);
if (handle == INVALID_HANDLE_VALUE) {
@@ -93,14 +93,14 @@ void SystemIO::CollectFileInformationRecursivly(const std::string& path,
} else {
ProcessFileEntity(find_data, path, files, err);
}
- if (*err != IOErrors::kNoError) {
+ if (*err != IOError::kNoError) {
FindClose(handle);
return;
}
} while (FindNextFile(handle, &find_data));
if (FindClose(handle)) {
- *err = IOErrors ::kNoError;
+ *err = IOError ::kNoError;
} else {
*err = IOErrorFromGetLastError();
}
diff --git a/tests/system_io/read_file_content/read_file_content.cpp b/tests/system_io/read_file_content/read_file_content.cpp
index 7853299e7..ab0ecc130 100644
--- a/tests/system_io/read_file_content/read_file_content.cpp
+++ b/tests/system_io/read_file_content/read_file_content.cpp
@@ -4,7 +4,7 @@
#include "testing.h"
using hidra2::SystemIO;
-using hidra2::IOErrors;
+using hidra2::IOError;
using hidra2::M_AssertEq;
int main(int argc, char* argv[]) {
@@ -14,24 +14,24 @@ int main(int argc, char* argv[]) {
}
std::string expect{argv[2]};
- IOErrors err;
+ IOError err;
auto io = std::unique_ptr<SystemIO> {new SystemIO};
auto data = io->GetDataFromFile(argv[1], expect.size(), &err);
std::string result;
switch (err) {
- case IOErrors::kFileNotFound:
+ case IOError::kFileNotFound:
result = "notfound";
break;
- case IOErrors::kNoError:
+ case IOError::kNoError:
for(int i = 0; i < expect.size(); i++)
result += data[i];
break;
- case IOErrors::kPermissionDenied:
+ case IOError::kPermissionDenied:
result = "noaccess";
break;
- case IOErrors::kReadError:
+ case IOError::kReadError:
result = "readerror";
break;
diff --git a/tests/system_io/read_files_in_folder/read_folder_content.cpp b/tests/system_io/read_files_in_folder/read_folder_content.cpp
index 4df949413..926126dc9 100644
--- a/tests/system_io/read_files_in_folder/read_folder_content.cpp
+++ b/tests/system_io/read_files_in_folder/read_folder_content.cpp
@@ -4,7 +4,7 @@
#include "testing.h"
using hidra2::SystemIO;
-using hidra2::IOErrors;
+using hidra2::IOError;
using hidra2::M_AssertEq;
@@ -15,20 +15,20 @@ int main(int argc, char* argv[]) {
}
std::string expect{argv[2]};
- IOErrors err;
+ IOError err;
auto io = std::unique_ptr<SystemIO> {new SystemIO};
auto files = io->FilesInFolder(argv[1], &err);
std::string result;
switch (err) {
- case IOErrors::kFileNotFound:
+ case IOError::kFileNotFound:
result = "notfound";
break;
- case IOErrors::kNoError:
+ case IOError::kNoError:
for(auto file_info : files)
result += file_info.relative_path + file_info.base_name;
break;
- case IOErrors::kPermissionDenied:
+ case IOError::kPermissionDenied:
result = "noaccess";
break;
default:
diff --git a/worker/api/cpp/src/folder_data_broker.cpp b/worker/api/cpp/src/folder_data_broker.cpp
index a6c263c44..bcb6eafd1 100644
--- a/worker/api/cpp/src/folder_data_broker.cpp
+++ b/worker/api/cpp/src/folder_data_broker.cpp
@@ -4,22 +4,22 @@
namespace hidra2 {
-WorkerErrorCode MapIOError(IOErrors io_err) {
+WorkerErrorCode MapIOError(IOError io_err) {
WorkerErrorCode err;
switch (io_err) { // we do not use map due to performance reasons
- case IOErrors::kNoError:
+ case IOError::kNoError:
err = WorkerErrorCode::kOK;
break;
- case IOErrors::kFileNotFound:
+ case IOError::kFileNotFound:
err = WorkerErrorCode::kSourceNotFound;
break;
- case IOErrors::kPermissionDenied:
+ case IOError::kPermissionDenied:
err = WorkerErrorCode::kPermissionDenied;
break;
- case IOErrors::kReadError:
+ case IOError::kReadError:
err = WorkerErrorCode::kErrorReadingSource;
break;
- case IOErrors::kMemoryAllocationError:
+ case IOError::kMemoryAllocationError:
err = WorkerErrorCode::kMemoryError;
break;
default:
@@ -43,10 +43,10 @@ WorkerErrorCode FolderDataBroker::Connect() {
return WorkerErrorCode::kSourceAlreadyConnected;
}
- IOErrors io_err;
+ IOError io_err;
filelist_ = io__->FilesInFolder(base_path_, &io_err);
- if (io_err == IOErrors::kNoError) {
+ if (io_err == IOError::kNoError) {
is_connected_ = true;
}
return MapIOError(io_err);
@@ -88,7 +88,7 @@ WorkerErrorCode FolderDataBroker::GetNext(FileInfo* info, FileData* data) {
return WorkerErrorCode::kOK;
}
- IOErrors ioerr;
+ IOError ioerr;
*data = io__->GetDataFromFile(base_path_ + "/" + file_info.relative_path +
(file_info.relative_path.empty() ? "" : "/") +
file_info.base_name, file_info.size, &ioerr);
diff --git a/worker/api/cpp/unittests/test_folder_broker.cpp b/worker/api/cpp/unittests/test_folder_broker.cpp
index 7da3645ce..324ec46bc 100644
--- a/worker/api/cpp/unittests/test_folder_broker.cpp
+++ b/worker/api/cpp/unittests/test_folder_broker.cpp
@@ -11,7 +11,7 @@ using hidra2::DataBroker;
using hidra2::FolderDataBroker;
using hidra2::WorkerErrorCode;
using hidra2::IO;
-using hidra2::IOErrors;
+using hidra2::IOError;
using hidra2::FileInfo;
using hidra2::FileData;
@@ -36,12 +36,12 @@ TEST(FolderDataBroker, SetCorrectIO) {
class FakeIO: public IO {
public:
- virtual uint8_t* GetDataFromFileProxy(const std::string& fname, uint64_t fsize, IOErrors* err) const {
- *err = IOErrors::kNoError;
+ virtual uint8_t* GetDataFromFileProxy(const std::string& fname, uint64_t fsize, IOError* err) const {
+ *err = IOError::kNoError;
return nullptr;
};
- FileData GetDataFromFile(const std::string& fname, uint64_t fsize, IOErrors* err) const noexcept override {
+ FileData GetDataFromFile(const std::string& fname, uint64_t fsize, IOError* err) const noexcept override {
return FileData(GetDataFromFileProxy(fname, fsize, err));
};
@@ -58,8 +58,8 @@ class FakeIO: public IO {
int64_t write(int __fd, const void* __buf, size_t __n) const noexcept override {
return 0;
};
- std::vector<FileInfo> FilesInFolder(const std::string& folder, IOErrors* err) const override {
- *err = IOErrors::kNoError;
+ std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override {
+ *err = IOError::kNoError;
std::vector<FileInfo> file_infos;
FileInfo fi;
fi.size = 100;
@@ -76,32 +76,32 @@ class FakeIO: public IO {
class IOFolderNotFound: public FakeIO {
public:
- std::vector<FileInfo> FilesInFolder(const std::string& folder, IOErrors* err) const override {
- *err = IOErrors::kFileNotFound;
+ std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override {
+ *err = IOError::kFileNotFound;
return {};
}
};
class IOFodlerUnknownError: public FakeIO {
public:
- std::vector<FileInfo> FilesInFolder(const std::string& folder, IOErrors* err) const override {
- *err = IOErrors::kUnknownError;
+ std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override {
+ *err = IOError::kUnknownError;
return {};
}
};
class IOEmptyFodler: public FakeIO {
public:
- std::vector<FileInfo> FilesInFolder(const std::string& folder, IOErrors* err) const override {
- *err = IOErrors::kNoError;
+ std::vector<FileInfo> FilesInFolder(const std::string& folder, IOError* err) const override {
+ *err = IOError::kNoError;
return {};
}
};
class IOCannotOpenFile: public FakeIO {
public:
- FileData GetDataFromFile(const std::string& fname, uint64_t fsize, IOErrors* err) const noexcept override {
- *err = IOErrors::kPermissionDenied;
+ FileData GetDataFromFile(const std::string& fname, uint64_t fsize, IOError* err) const noexcept override {
+ *err = IOError::kPermissionDenied;
return {};
};
};
@@ -211,7 +211,7 @@ TEST_F(FolderDataBrokerTests, GetNextReturnsErrorWhenFilePermissionsDenied) {
class OpenFileMock : public FakeIO {
public:
- MOCK_CONST_METHOD3(GetDataFromFileProxy, uint8_t* (const std::string&, uint64_t, IOErrors*));
+ MOCK_CONST_METHOD3(GetDataFromFileProxy, uint8_t* (const std::string&, uint64_t, IOError*));
};
@@ -233,7 +233,7 @@ class GetDataFromFileTests : public Test {
TEST_F(GetDataFromFileTests, GetNextCallsGetDataFileWithFileName) {
EXPECT_CALL(mock, GetDataFromFileProxy("/path/to/file/1", _, _)).
- WillOnce(DoAll(testing::SetArgPointee<2>(IOErrors::kNoError), testing::Return(nullptr)));
+ WillOnce(DoAll(testing::SetArgPointee<2>(IOError::kNoError), testing::Return(nullptr)));
data_broker->GetNext(&fi, &data);
}
@@ -242,7 +242,7 @@ TEST_F(GetDataFromFileTests, GetNextCallsGetDataFileWithFileName) {
TEST_F(GetDataFromFileTests, GetNextReturnsDataAndInfo) {
EXPECT_CALL(mock, GetDataFromFileProxy(_, _, _)).
- WillOnce(DoAll(testing::SetArgPointee<2>(IOErrors::kNoError), testing::Return(new uint8_t[1] {'1'})));
+ WillOnce(DoAll(testing::SetArgPointee<2>(IOError::kNoError), testing::Return(new uint8_t[1] {'1'})));
data_broker->GetNext(&fi, &data);
@@ -253,7 +253,7 @@ TEST_F(GetDataFromFileTests, GetNextReturnsDataAndInfo) {
TEST_F(GetDataFromFileTests, GetNextReturnsOnlyData) {
EXPECT_CALL(mock, GetDataFromFileProxy(_, _, _)).
- WillOnce(DoAll(testing::SetArgPointee<2>(IOErrors::kNoError), testing::Return(new uint8_t[1] {'1'})));
+ WillOnce(DoAll(testing::SetArgPointee<2>(IOError::kNoError), testing::Return(new uint8_t[1] {'1'})));
data_broker->GetNext(nullptr, &data);
@@ -263,7 +263,7 @@ TEST_F(GetDataFromFileTests, GetNextReturnsOnlyData) {
TEST_F(GetDataFromFileTests, GetNextReturnsErrorWhenCannotReadData) {
EXPECT_CALL(mock, GetDataFromFileProxy(_, _, _)).
- WillOnce(DoAll(testing::SetArgPointee<2>(IOErrors::kReadError), testing::Return(nullptr)));
+ WillOnce(DoAll(testing::SetArgPointee<2>(IOError::kReadError), testing::Return(nullptr)));
auto err = data_broker->GetNext(&fi, &data);
@@ -272,7 +272,7 @@ TEST_F(GetDataFromFileTests, GetNextReturnsErrorWhenCannotReadData) {
TEST_F(GetDataFromFileTests, GetNextReturnsErrorWhenCannotAllocateData) {
EXPECT_CALL(mock, GetDataFromFileProxy(_, _, _)).
- WillOnce(DoAll(testing::SetArgPointee<2>(IOErrors::kMemoryAllocationError), testing::Return(nullptr)));
+ WillOnce(DoAll(testing::SetArgPointee<2>(IOError::kMemoryAllocationError), testing::Return(nullptr)));
auto err = data_broker->GetNext(&fi, &data);
diff --git a/worker/tools/CMakeLists.txt b/worker/tools/CMakeLists.txt
new file mode 100644
index 000000000..1d90f764c
--- /dev/null
+++ b/worker/tools/CMakeLists.txt
@@ -0,0 +1,2 @@
+add_subdirectory(folder_to_db)
+
diff --git a/worker/tools/folder_to_db/CMakeLists.txt b/worker/tools/folder_to_db/CMakeLists.txt
new file mode 100644
index 000000000..f93fca48d
--- /dev/null
+++ b/worker/tools/folder_to_db/CMakeLists.txt
@@ -0,0 +1,21 @@
+set(TARGET_NAME folder2db)
+
+set(SOURCE_FILES src/FolderDBConverter.cpp)
+
+
+################################
+# Library
+################################
+add_library(${TARGET_NAME} STATIC ${SOURCE_FILES} $<TARGET_OBJECTS:common>)
+
+target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_SOURCE_DIR}/common/cpp/include)
+
+
+################################
+# Testing
+################################
+set(TEST_SOURCE_FILES unittests/test_folder_to_db.cpp)
+set(TEST_LIBRARIES "${TARGET_NAME}")
+
+
+gtest(${TARGET_NAME} "${TEST_SOURCE_FILES}" "${TEST_LIBRARIES}")
diff --git a/worker/tools/folder_to_db/src/FolderDBConverter.cpp b/worker/tools/folder_to_db/src/FolderDBConverter.cpp
new file mode 100644
index 000000000..286937f5c
--- /dev/null
+++ b/worker/tools/folder_to_db/src/FolderDBConverter.cpp
@@ -0,0 +1,55 @@
+#include "FolderDBConverter.h"
+
+#include "system_wrappers/system_io.h"
+#include "database/mongo_database.h"
+
+namespace hidra2 {
+
+FolderDBConverterError MapIOError(IOError io_err) {
+ FolderDBConverterError err;
+ switch (io_err) {
+ case IOError::kNoError:
+ err = FolderDBConverterError::kOK;
+ break;
+ default:
+ err = FolderDBConverterError::kIOError;
+ break;
+ }
+ return err;
+}
+
+FolderDBConverterError MapDBError(DBError db_err) {
+ FolderDBConverterError err;
+ switch (db_err) {
+ case DBError::kNoError:
+ err = FolderDBConverterError::kOK;
+ break;
+ case DBError::kConnectionError:
+ err = FolderDBConverterError::kDBConnectionError;
+ break;
+ }
+ return err;
+}
+
+
+
+FolderDBConverter::FolderDBConverter() :
+ io__{new hidra2::SystemIO}, db__{new hidra2::MongoDB} {
+}
+
+FolderDBConverterError FolderDBConverter::Convert(const std::string& uri, const std::string& folder) {
+ DBError err_db = db__->Connect(uri, kDBName, folder);
+ if (err_db != DBError::kNoError) {
+ return MapDBError(err_db);
+ }
+
+ IOError err_io;
+ auto file_list = io__->FilesInFolder(folder,&err_io);
+ if (err_io != IOError::kNoError){
+ return MapIOError(err_io);
+ }
+
+ return FolderDBConverterError::kOK;
+}
+
+}
\ No newline at end of file
diff --git a/worker/tools/folder_to_db/src/FolderDBConverter.h b/worker/tools/folder_to_db/src/FolderDBConverter.h
new file mode 100644
index 000000000..d78be1e87
--- /dev/null
+++ b/worker/tools/folder_to_db/src/FolderDBConverter.h
@@ -0,0 +1,28 @@
+#ifndef HIDRA2_FOLDERDBCONVERTER_H
+#define HIDRA2_FOLDERDBCONVERTER_H
+
+#include "system_wrappers/io.h"
+#include "database/database.h"
+
+namespace hidra2 {
+
+enum class FolderDBConverterError {
+ kOK,
+ kDBConnectionError,
+ kIOError
+};
+
+class FolderDBConverter {
+ public:
+ explicit FolderDBConverter();
+
+ FolderDBConverterError Convert(const std::string& uri, const std::string& folder);
+
+ std::unique_ptr<hidra2::IO> io__; // modified in testings to mock system calls,otherwise do not touch
+ std::unique_ptr<hidra2::Database> db__; // modified in testings to mock system calls,otherwise do not touch
+
+};
+
+}
+
+#endif //HIDRA2_FOLDERDBCONVERTER_H
diff --git a/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp b/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp
new file mode 100644
index 000000000..92fb23afc
--- /dev/null
+++ b/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp
@@ -0,0 +1,116 @@
+#include <gmock/gmock.h>
+#include "gtest/gtest.h"
+
+#include "system_wrappers/io.h"
+#include "system_wrappers/system_io.h"
+
+
+#include "database/database.h"
+#include "database/mongo_database.h"
+
+#include "common/file_info.h"
+
+#include "../src/FolderDBConverter.h"
+
+#include "unittests/MockIO.h"
+
+
+using ::testing::AtLeast;
+using ::testing::Eq;
+using ::testing::Ne;
+using ::testing::Test;
+using ::testing::_;
+using ::testing::Mock;
+using ::testing::NiceMock;
+
+using hidra2::FolderDBConverter;
+using hidra2::FolderDBConverterError;
+using hidra2::Database;
+using hidra2::IO;
+using hidra2::DBError;
+using hidra2::IOError;
+using hidra2::kDBName;
+using hidra2::FileInfo;
+using hidra2::MockIO;
+
+
+namespace {
+
+TEST(FolderDBConverter, SetCorrectIO) {
+ FolderDBConverter converter{};
+ ASSERT_THAT(dynamic_cast<hidra2::SystemIO*>(converter.io__.get()), Ne(nullptr));
+}
+
+TEST(FolderDBConverter, SetCorrectDB) {
+ FolderDBConverter converter{};
+ ASSERT_THAT(dynamic_cast<hidra2::MongoDB*>(converter.db__.get()), Ne(nullptr));
+}
+
+class MockDatabase : public Database {
+ public:
+ MOCK_METHOD3(Connect, DBError (const std::string&, const std::string&, const std::string&));
+
+ // stuff to test db destructor is called and avoid "uninteresting call" messages
+ MOCK_METHOD0(Die, void());
+ virtual ~MockDatabase() override {
+ if (check_destructor)
+ Die();
+ }
+ bool check_destructor{false};
+
+};
+
+
+
+class FolderDBConverterTests : public Test {
+ public:
+ FolderDBConverter converter{};
+ NiceMock<MockDatabase> mock_db;
+ NiceMock<MockIO> mock_io;
+
+ void SetUp() override {
+ converter.db__ = std::unique_ptr<Database> {&mock_db};
+ converter.io__ = std::unique_ptr<IO> {&mock_io};
+
+ }
+ void TearDown() override {
+ converter.db__.release();
+ converter.io__.release();
+
+ }
+};
+
+
+TEST_F(FolderDBConverterTests, ErrorWhenCannotConnect) {
+
+ std::string uri{"db_address"};
+
+
+ EXPECT_CALL(mock_db, Connect(uri, kDBName, "")).
+ WillOnce(testing::Return(DBError::kConnectionError));
+
+ auto error = converter.Convert(uri, "");
+ ASSERT_THAT(error, Eq(FolderDBConverterError::kDBConnectionError));
+}
+
+TEST_F(FolderDBConverterTests, DBDestructorCalled) {
+ mock_db.check_destructor = true;
+ EXPECT_CALL(mock_db, Die());
+}
+
+TEST_F(FolderDBConverterTests, ErrorWhenCannotGetFileList) {
+
+ std::string folder{"folder"};
+
+ EXPECT_CALL(mock_io, FilesInFolder(folder, _)).
+ WillOnce(DoAll(testing::SetArgPointee<1>(IOError::kReadError), testing::Return(std::vector<FileInfo>{})));
+
+
+ auto error = converter.Convert("", folder);
+ ASSERT_THAT(error, Eq(FolderDBConverterError::kIOError));
+
+}
+
+
+
+}
--
GitLab