From 7e15ee5b71e9eb39ab20e7e51e81baf6e930cd15 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Tue, 9 Jan 2018 17:09:51 +0100
Subject: [PATCH] connect to db done

---
 3d_party/mongo-c-driver/install.sh            |   2 +-
 CMakeModules/testing_cpp.cmake                |   6 +-
 common/cpp/include/database/database.h        |   5 +-
 common/cpp/include/database/mongodb_client.h  |  34 ++++-
 common/cpp/src/database/CMakeLists.txt        |   2 +-
 common/cpp/src/database/mongodb_client.cpp    | 137 ++++++++++--------
 tests/CMakeLists.txt                          |   5 +
 tests/mongo_db/CMakeLists.txt                 |   5 +
 tests/mongo_db/connect/CMakeLists.txt         |  20 +++
 tests/mongo_db/connect/cleanup_linux.sh       |   5 +
 tests/mongo_db/connect/connect_mongodb.cpp    |  63 ++++++++
 tests/mongo_db/connect/setup_linux.sh         |   3 +
 tests/valgrind.suppressions                   |  79 ++++++++++
 tests/worker/CMakeLists.txt                   |   4 +-
 tests/worker/folder_to_db/CMakeLists.txt      |   2 +-
 tests/worker/folder_to_db/check_linux.sh      |  20 ++-
 .../folder_to_db/src/folder_db_importer.cpp   |   2 +-
 .../unittests/test_folder_to_db.cpp           |   2 +-
 18 files changed, 319 insertions(+), 77 deletions(-)
 create mode 100644 tests/mongo_db/CMakeLists.txt
 create mode 100644 tests/mongo_db/connect/CMakeLists.txt
 create mode 100644 tests/mongo_db/connect/cleanup_linux.sh
 create mode 100644 tests/mongo_db/connect/connect_mongodb.cpp
 create mode 100644 tests/mongo_db/connect/setup_linux.sh
 create mode 100644 tests/valgrind.suppressions

diff --git a/3d_party/mongo-c-driver/install.sh b/3d_party/mongo-c-driver/install.sh
index fc9ec2e39..6644bf5a9 100755
--- a/3d_party/mongo-c-driver/install.sh
+++ b/3d_party/mongo-c-driver/install.sh
@@ -4,7 +4,7 @@ cd $1
 wget https://github.com/mongodb/mongo-c-driver/releases/download/1.9.0/mongo-c-driver-1.9.0.tar.gz
 tar xzf mongo-c-driver-1.9.0.tar.gz
 cd mongo-c-driver-1.9.0
-./configure --disable-automatic-init-and-cleanup --enable-static=yes --enable-shared=no
+./configure --disable-automatic-init-and-cleanup --enable-static=yes --enable-shared=no --enable-ssl=no --enable-sasl=no
 make
 #sudo make install
 
diff --git a/CMakeModules/testing_cpp.cmake b/CMakeModules/testing_cpp.cmake
index 3d9f1fa94..0ed139b3f 100644
--- a/CMakeModules/testing_cpp.cmake
+++ b/CMakeModules/testing_cpp.cmake
@@ -6,7 +6,8 @@ if (BUILD_TESTS)
     set(HIDRA2_MINIMUM_COVERAGE 70)
     find_package(Threads)
     find_program(MEMORYCHECK_COMMAND valgrind)
-    set(MEMORYCHECK_COMMAND_OPTIONS "--trace-children=yes --leak-check=full --error-exitcode=1")
+    set(MEMORYCHECK_COMMAND_OPTIONS
+            "--trace-children=yes --leak-check=full --error-exitcode=1 --suppressions=${CMAKE_SOURCE_DIR}/tests/valgrind.suppressions")
     if (NOT "$ENV{gtest_SOURCE_DIR}" STREQUAL "")
         set(gtest_SOURCE_DIR $ENV{gtest_SOURCE_DIR})
     endif ()
@@ -101,7 +102,8 @@ function(add_integration_test exename testname commandargs)
         if (ARGN)
             set(commandargs ${ARGN})
         endif ()
-        if (NOT ${ARGN} STREQUAL nomem)
+        if (NOT "${ARGN}" STREQUAL nomem)
+            message(STATUS "memory check ${exename}")
             add_memory_test(${exename}-${testname} ${exename}
                     "${commandargs}" test-${exename}-fixture
                     "integration")
diff --git a/common/cpp/include/database/database.h b/common/cpp/include/database/database.h
index abe4a2dcc..f1bdb9b6f 100644
--- a/common/cpp/include/database/database.h
+++ b/common/cpp/include/database/database.h
@@ -10,7 +10,10 @@ namespace hidra2 {
 enum class DBError {
     kNoError,
     kConnectionError,
-    kImportError
+    kImportError,
+    kAlreadyConnected,
+    kNotConnected,
+    kBadAddress
 };
 
 constexpr char kDBName[] = "data";
diff --git a/common/cpp/include/database/mongodb_client.h b/common/cpp/include/database/mongodb_client.h
index 26fd80256..61cf5bbfc 100644
--- a/common/cpp/include/database/mongodb_client.h
+++ b/common/cpp/include/database/mongodb_client.h
@@ -1,17 +1,47 @@
 #ifndef HIDRA2_MONGO_DATABASE_H
 #define HIDRA2_MONGO_DATABASE_H
 
+#include "mongoc.h"
+
+#include <string>
+
 #include "database.h"
 
 namespace hidra2 {
 
-class MongoDB final: public Database {
+// An attempt to automize mongoc_init/mongoc_cleanup.
+// One has to be carefull with cleanup order - since it is called after main exits
+// it may conflict with cleanup of libraries that mongodb uses (ssl,sasl which are not used now)
+// see http://mongoc.org/libmongoc/current/init-cleanup.html
+class MongoDbInstance {
+  public:
+    static void Instantiate() {
+        static MongoDbInstance instance;
+    }
+  private:
+    MongoDbInstance();
+    ~MongoDbInstance();
+};
+
+class MongoDBClient final: public Database {
   public:
+    MongoDBClient();
     DBError Connect(const std::string& address, const std::string& database,
                     const std::string& collection ) override;
     DBError Import(const FileInfos& files) const override;
-    ~MongoDB() override;
+    ~MongoDBClient() override;
+  private:
+    mongoc_client_t* client_{nullptr};
+    mongoc_collection_t* collection_{nullptr};
+    bool connected_{false};
+    void CleanUp();
+    std::string DBAddress(const std::string& address) const;
+    DBError InitializeClient(const std::string& address);
+    void InitializeCollection(const std::string& database_name,
+                              const std::string& collection_name);
 
+    DBError Ping();
+    DBError TryConnectDatabase();
 };
 
 }
diff --git a/common/cpp/src/database/CMakeLists.txt b/common/cpp/src/database/CMakeLists.txt
index 405284814..5479f1f44 100644
--- a/common/cpp/src/database/CMakeLists.txt
+++ b/common/cpp/src/database/CMakeLists.txt
@@ -13,6 +13,6 @@ message ("--   mongoc libraries \"${MONGOC_STATIC_LIBRARIES}\"")
 
 add_library(${TARGET_NAME} STATIC ${SOURCE_FILES})
 target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR}
-        PRIVATE "${MONGOC_STATIC_INCLUDE_DIRS}")
+        PUBLIC "${MONGOC_STATIC_INCLUDE_DIRS}")
 target_link_libraries (${TARGET_NAME} PRIVATE "${MONGOC_STATIC_LIBRARIES}")
 target_compile_definitions (${TARGET_NAME} PRIVATE "${MONGOC_STATIC_DEFINITIONS}")
diff --git a/common/cpp/src/database/mongodb_client.cpp b/common/cpp/src/database/mongodb_client.cpp
index 3910786fa..4f014d41d 100644
--- a/common/cpp/src/database/mongodb_client.cpp
+++ b/common/cpp/src/database/mongodb_client.cpp
@@ -1,91 +1,100 @@
-#include "mongoc.h"
-
-
 #include "database/mongodb_client.h"
 
 namespace hidra2 {
 
-DBError MongoDB::Connect(const std::string& address, const std::string& database_name,
-                         const std::string& collection_name ) {
-    const char* uri_str = address.c_str();
-    mongoc_client_t* client;
-    mongoc_database_t* database;
-    mongoc_collection_t* collection;
-    bson_t* command, reply, *insert;
+MongoDbInstance::MongoDbInstance() {
+    mongoc_init ();
+}
+
+MongoDbInstance::~MongoDbInstance() {
+    mongoc_cleanup ();
+}
+
+DBError MongoDBClient::Ping() {
+    bson_t* command, reply;
     bson_error_t error;
-    char* str;
     bool retval;
 
-    /*
-     * Required to initialize libmongoc's internals
-     */
-    mongoc_init ();
-
-    /*
-     * Create a new client instance
-     */
-    client = mongoc_client_new (uri_str);
-
-    /*
-     * Register the application name so we can track it in the profile logs
-     * on the server. This can also be done from the URI (see other examples).
-     */
-    mongoc_client_set_appname (client, "connect-example");
-
-    /*
-     * Get a handle on the database "db_name" and collection "coll_name"
-     */
-    database = mongoc_client_get_database (client, database_name.c_str());
-    collection = mongoc_client_get_collection (client, database_name.c_str(), collection_name.c_str());
-
-    /*
-     * Do work. This example pings the database, prints the result as JSON and
-     * performs an insert
-     */
     command = BCON_NEW ("ping", BCON_INT32 (1));
-
     retval = mongoc_client_command_simple (
-                 client, "admin", command, NULL, &reply, &error);
+                 client_, "admin", command, NULL, &reply, &error);
 
-    if (!retval) {
-        fprintf (stderr, "%s\n", error.message);
-        return DBError::kConnectionError;
-    }
+    bson_destroy (&reply);
+    bson_destroy (command);
 
-    str = bson_as_json (&reply, NULL);
-    printf ("%s\n", str);
+    return !retval ? DBError::kConnectionError : DBError::kNoError;
 
-    insert = BCON_NEW ("hello", BCON_UTF8 ("world"));
+}
+MongoDBClient::MongoDBClient() {
+    MongoDbInstance::Instantiate();
+}
 
-    if (!mongoc_collection_insert_one (collection, insert, NULL, NULL, &error)) {
-        fprintf (stderr, "%s\n", error.message);
+DBError MongoDBClient::InitializeClient(const std::string& address) {
+    auto uri_str = DBAddress(address);
+    client_ = mongoc_client_new (uri_str.c_str());
+    if (client_ == nullptr) {
+        return DBError::kBadAddress;
     }
+    return DBError::kNoError;
 
-    bson_destroy (insert);
-    bson_destroy (&reply);
-    bson_destroy (command);
-    bson_free (str);
-
-    /*
-     * Release our handles and clean up libmongoc
-     */
-    mongoc_collection_destroy (collection);
-    mongoc_database_destroy (database);
-    mongoc_client_destroy (client);
-    mongoc_cleanup ();
+}
 
-    return DBError::kNoError;
+void MongoDBClient::InitializeCollection(const std::string& database_name,
+                                         const std::string& collection_name) {
+    collection_ = mongoc_client_get_collection (client_, database_name.c_str(),
+                                                collection_name.c_str());
 }
 
-DBError MongoDB::Import(const FileInfos& files) const {
-    return DBError::kNoError;
+DBError MongoDBClient::TryConnectDatabase() {
+    auto err = Ping();
+    if (err == DBError::kNoError) {
+        connected_ = true;
+    }
+    return err;
 }
 
+DBError MongoDBClient::Connect(const std::string& address, const std::string& database_name,
+                               const std::string& collection_name) {
+    if (connected_) {
+        return DBError::kAlreadyConnected;
+    }
+
+    auto err = InitializeClient(address);
+    if (err != DBError::kNoError) {
+        return err;
+    }
 
-MongoDB::~MongoDB() {
+    InitializeCollection(database_name, collection_name);
 
+    err = TryConnectDatabase();
+    if (err != DBError::kNoError) {
+        CleanUp();
+    }
+    return err;
 }
 
+std::string MongoDBClient::DBAddress(const std::string& address) const {
+    return "mongodb://" + address + "/?appname=hidra2";
+}
 
+void MongoDBClient::CleanUp() {
+    mongoc_collection_destroy (collection_);
+    mongoc_client_destroy (client_);
+}
+
+DBError MongoDBClient::Import(const FileInfos& files) const {
+    if (!connected_) {
+        return DBError::kNotConnected;
+    }
+
+    return DBError::kNoError;
+}
+
+MongoDBClient::~MongoDBClient() {
+    if (!connected_) {
+        return;
+    }
+    CleanUp();
+}
 
 }
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 59f4f8214..dc25044a5 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -3,6 +3,11 @@ find_package(Threads)
 
 add_subdirectory(common/cpp)
 add_subdirectory(system_io)
+
+if(BUILD_MONGODB_CLIENTLIB)
+    add_subdirectory(mongo_db)
+endif()
+
 add_subdirectory(worker)
 
 
diff --git a/tests/mongo_db/CMakeLists.txt b/tests/mongo_db/CMakeLists.txt
new file mode 100644
index 000000000..4426c8c6d
--- /dev/null
+++ b/tests/mongo_db/CMakeLists.txt
@@ -0,0 +1,5 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 3.7) # needed for fixtures
+
+add_subdirectory(connect)
+
+
diff --git a/tests/mongo_db/connect/CMakeLists.txt b/tests/mongo_db/connect/CMakeLists.txt
new file mode 100644
index 000000000..780baba6f
--- /dev/null
+++ b/tests/mongo_db/connect/CMakeLists.txt
@@ -0,0 +1,20 @@
+set(TARGET_NAME connect_mongodb)
+set(SOURCE_FILES connect_mongodb.cpp)
+
+
+################################
+# Executable and link
+################################
+add_executable(${TARGET_NAME} ${SOURCE_FILES})
+target_link_libraries(${TARGET_NAME} test_common mongo_db)
+target_include_directories(${TARGET_NAME} PUBLIC ${HIDRA2_CXX_COMMON_INCLUDE_DIR})
+
+################################
+# Testing
+################################
+
+add_integration_test(${TARGET_NAME} connectOK "127.0.0.1 data test OK")
+add_integration_test(${TARGET_NAME} connectFAILS "127.0.0.0 data test ConnectionError")
+add_integration_test(${TARGET_NAME} connectBadAddress "#?ß// data test BadAddress")
+
+
diff --git a/tests/mongo_db/connect/cleanup_linux.sh b/tests/mongo_db/connect/cleanup_linux.sh
new file mode 100644
index 000000000..8c8a4ada2
--- /dev/null
+++ b/tests/mongo_db/connect/cleanup_linux.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+database_name=data
+
+echo "db.test.deleteMany({})" | mongo ${database_name}
diff --git a/tests/mongo_db/connect/connect_mongodb.cpp b/tests/mongo_db/connect/connect_mongodb.cpp
new file mode 100644
index 000000000..0cfeccec0
--- /dev/null
+++ b/tests/mongo_db/connect/connect_mongodb.cpp
@@ -0,0 +1,63 @@
+#include <iostream>
+
+
+#include "database/mongodb_client.h"
+#include "testing.h"
+
+
+using hidra2::M_AssertEq;
+using hidra2::DBError;
+
+
+void Assert(DBError error, const std::string& expect) {
+    std::string result;
+    switch (error) {
+    case DBError::kConnectionError:
+        result = "ConnectionError";
+        break;
+    case DBError::kAlreadyConnected:
+        result = "AlreadyConnected";
+        break;
+    case DBError::kBadAddress:
+        result = "BadAddress";
+        break;
+    default:
+        result = "OK";
+        break;
+    }
+
+    M_AssertEq(expect, result);
+}
+
+struct Args {
+    std::string address;
+    std::string database_name;
+    std::string collection_name;
+    std::string expect;
+
+};
+
+Args GetArgs(int argc, char* argv[]) {
+    if (argc != 5) {
+        std::cout << "Wrong number of arguments" << std::endl;
+        exit(EXIT_FAILURE);
+    }
+    return Args{argv[1], argv[2], argv[3], argv[4]};
+}
+
+
+int main(int argc, char* argv[]) {
+    auto args = GetArgs(argc, argv);
+    hidra2::MongoDBClient db;
+
+    auto err = db.Connect(args.address, args.database_name, args.collection_name);
+    Assert(err, args.expect);
+
+    if (err == DBError::kNoError) {
+        err = db.Connect(args.address, args.database_name, args.collection_name);
+        Assert(err, "AlreadyConnected");
+    }
+    return 0;
+}
+
+
diff --git a/tests/mongo_db/connect/setup_linux.sh b/tests/mongo_db/connect/setup_linux.sh
new file mode 100644
index 000000000..48889b6e6
--- /dev/null
+++ b/tests/mongo_db/connect/setup_linux.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+#do nothing
\ No newline at end of file
diff --git a/tests/valgrind.suppressions b/tests/valgrind.suppressions
new file mode 100644
index 000000000..3ecb96d95
--- /dev/null
+++ b/tests/valgrind.suppressions
@@ -0,0 +1,79 @@
+{
+   ignore_libcrypto_conditional_jump_errors_in_leak
+   Memcheck:Leak
+   ...
+   obj:*/libcrypto.so.*
+}
+{
+   ignore_libcrypto_conditional_jump_errors_in_cond
+   Memcheck:Cond
+   ...
+   obj:*/libcrypto.so.*
+}
+{
+   ignore_libssl_conditional_jump_errors_in_cond
+   Memcheck:Cond
+   ...
+   obj:*/libssl.so.*
+}
+{
+   ignore_libcrypto_conditional_jump_errors_in_value8
+   Memcheck:Value8
+   ...
+   obj:*/libcrypto.so.*
+}
+{
+   ignore_scram_nonce_uninitialized_warning_cond
+   Memcheck:Cond
+   fun:mongoc_b64_ntop
+   ...
+   fun:_mongoc_scram_step
+   ...
+}
+{
+   ignore_scram_nonce_uninitialized_warning_value8
+   Memcheck:Value8
+   fun:mongoc_b64_ntop
+   ...
+   fun:_mongoc_scram_step
+   ...
+}
+{
+   ignore_libcrypto_conditional_jump_errors_in_param
+   Memcheck:Param
+   sendmsg(msg.msg_iov[0])
+   ...
+   obj:*/libcrypto.so.*
+}
+{
+   ignore_sasl_load_plugin_reachable_warnings
+   Memcheck:Leak
+   ...
+   fun:_sasl_get_plugin
+   ...
+   fun:_mongoc_do_init
+}
+{
+   ignore_sasl_add_plugin_reachable_warnings
+   Memcheck:Leak
+   ...
+   fun:sasl_client_add_plugin
+   ...
+   fun:_mongoc_do_init
+}
+{
+   temp_ignore_vfprintf
+   Memcheck:Cond
+   fun:vfprintf
+   ...
+   fun:bson_vsnprintf
+   fun:bson_strdupv_printf
+   ...
+}
+{
+   temp_ignore_strlen
+   Memcheck:Cond
+   fun:strlen
+   fun:bson_strdup
+   ...
+}
\ No newline at end of file
diff --git a/tests/worker/CMakeLists.txt b/tests/worker/CMakeLists.txt
index 87667dc22..15851c9db 100644
--- a/tests/worker/CMakeLists.txt
+++ b/tests/worker/CMakeLists.txt
@@ -2,6 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.7) # needed for fixtures
 
 add_subdirectory(next_multithread)
 add_subdirectory(connect_multithread)
-add_subdirectory(folder_to_db)
+if(BUILD_WORKER_TOOLS)
+    add_subdirectory(folder_to_db)
+endif()
 
 
diff --git a/tests/worker/folder_to_db/CMakeLists.txt b/tests/worker/folder_to_db/CMakeLists.txt
index cbd3410dc..40df3a0c5 100644
--- a/tests/worker/folder_to_db/CMakeLists.txt
+++ b/tests/worker/folder_to_db/CMakeLists.txt
@@ -3,5 +3,5 @@ set(TARGET_NAME folder2db)
 ################################
 # Testing
 ################################
-add_script_test("${TARGET_NAME}" "$<TARGET_FILE:${TARGET_NAME}-bin>"
+add_script_test("${TARGET_NAME}-bin" "$<TARGET_FILE:${TARGET_NAME}-bin>"
         )
diff --git a/tests/worker/folder_to_db/check_linux.sh b/tests/worker/folder_to_db/check_linux.sh
index c651c8360..443110382 100644
--- a/tests/worker/folder_to_db/check_linux.sh
+++ b/tests/worker/folder_to_db/check_linux.sh
@@ -1,10 +1,26 @@
 #!/usr/bin/env bash
 
+database_name=data
+
 set -e
 
+trap Cleanup EXIT
+
+Cleanup() {
+	echo cleanup
+	echo "db.test.deleteMany({})" | mongo ${database_name}
+	rm -rf test
+}
+
 mkdir -p test
-touch test/1
+touch test/file2
+sleep 0.1
+touch test/file1
 
 $1 test mongodb://127.0.0.1
 
-rm -rf test
+echo "show collections" | mongo ${database_name} | grep test
+echo "db.test.find({"_id":1})" | mongo ${database_name} | grep file2
+echo "db.test.find("_id":2)" | mongo ${database_name} | grep file1
+
+
diff --git a/worker/tools/folder_to_db/src/folder_db_importer.cpp b/worker/tools/folder_to_db/src/folder_db_importer.cpp
index 628ef0407..e858f88b7 100644
--- a/worker/tools/folder_to_db/src/folder_db_importer.cpp
+++ b/worker/tools/folder_to_db/src/folder_db_importer.cpp
@@ -36,7 +36,7 @@ FolderToDbImportError MapDBError(DBError db_err) {
 
 
 FolderToDbImporter::FolderToDbImporter() :
-    io__{new hidra2::SystemIO}, db__{new hidra2::MongoDB} {
+    io__{new hidra2::SystemIO}, db__{new hidra2::MongoDBClient} {
 }
 
 FolderToDbImportError FolderToDbImporter::ConnectToDb(const std::string& uri, const std::string& folder) const {
diff --git a/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp b/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp
index b77a8ac36..6b6f5c4b2 100644
--- a/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp
+++ b/worker/tools/folder_to_db/unittests/test_folder_to_db.cpp
@@ -45,7 +45,7 @@ TEST(FolderDBConverter, SetCorrectIO) {
 
 TEST(FolderDBConverter, SetCorrectDB) {
     FolderToDbImporter converter{};
-    ASSERT_THAT(dynamic_cast<hidra2::MongoDB*>(converter.db__.get()), Ne(nullptr));
+    ASSERT_THAT(dynamic_cast<hidra2::MongoDBClient*>(converter.db__.get()), Ne(nullptr));
 }
 
 class MockDatabase : public Database {
-- 
GitLab