From 3007aad10f92bf07cd6f10861f4395a8eacf39af Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon, 14 Jun 2021 19:34:04 +0200
Subject: [PATCH] fix tests, refactor
---
common/cpp/include/asapo/database/database.h | 4 +--
.../include/asapo/unittests/MockDatabase.h | 10 +++----
common/cpp/src/database/mongodb_client.cpp | 28 ++++++++++---------
common/cpp/src/database/mongodb_client.h | 4 +--
.../request_handler_db_meta_write.cpp | 2 +-
.../test_request_handler_db_meta_writer.cpp | 12 ++++----
.../insert_retrieve_mongodb.cpp | 10 +++----
.../mongo_db/upsert/upsert_mongodb.cpp | 4 +--
8 files changed, 38 insertions(+), 36 deletions(-)
diff --git a/common/cpp/include/asapo/database/database.h b/common/cpp/include/asapo/database/database.h
index 0d1f5c614..678983382 100644
--- a/common/cpp/include/asapo/database/database.h
+++ b/common/cpp/include/asapo/database/database.h
@@ -15,8 +15,8 @@ class Database {
public:
virtual Error Connect(const std::string& address, const std::string& database) = 0;
virtual Error Insert(const std::string& collection, const MessageMeta& file, bool ignore_duplicates) const = 0;
- virtual Error Insert(const std::string& collection, const std::string& id, const uint8_t* data, uint64_t size,
- MetaIngestMode mode) const = 0;
+ virtual Error InsertMeta(const std::string& collection, const std::string& id, const uint8_t* data, uint64_t size,
+ MetaIngestMode mode) const = 0;
virtual Error InsertAsDatasetMessage(const std::string& collection, const MessageMeta& file,
uint64_t dataset_size,
bool ignore_duplicates) const = 0;
diff --git a/common/cpp/include/asapo/unittests/MockDatabase.h b/common/cpp/include/asapo/unittests/MockDatabase.h
index f5d4d75cc..65631cdd8 100644
--- a/common/cpp/include/asapo/unittests/MockDatabase.h
+++ b/common/cpp/include/asapo/unittests/MockDatabase.h
@@ -30,13 +30,13 @@ class MockDatabase : public Database {
MOCK_CONST_METHOD4(InsertAsDatasetMessage_t,
ErrorInterface * (const std::string&, const MessageMeta&, uint64_t, bool));
- Error Insert(const std::string& collection, const std::string& id, const uint8_t* data, uint64_t size,
- MetaIngestMode mode) const override {
- return Error{Insert_t(collection, id, data, size, mode)};
+ Error InsertMeta(const std::string& collection, const std::string& id, const uint8_t* data, uint64_t size,
+ MetaIngestMode mode) const override {
+ return Error{InsertMeta_t(collection, id, data, size, mode)};
}
- MOCK_CONST_METHOD5(Insert_t, ErrorInterface * (const std::string&, const std::string& id, const uint8_t* data,
- uint64_t size, MetaIngestMode mode));
+ MOCK_CONST_METHOD5(InsertMeta_t, ErrorInterface * (const std::string&, const std::string& id, const uint8_t* data,
+ uint64_t size, MetaIngestMode mode));
Error GetById(const std::string& collection, uint64_t id, MessageMeta* file) const override {
return Error{GetById_t(collection, id, file)};
diff --git a/common/cpp/src/database/mongodb_client.cpp b/common/cpp/src/database/mongodb_client.cpp
index 0f496865b..747c4ed23 100644
--- a/common/cpp/src/database/mongodb_client.cpp
+++ b/common/cpp/src/database/mongodb_client.cpp
@@ -190,16 +190,16 @@ Error MongoDBClient::ReplaceBsonDocument(const std::string& id, const bson_p& do
Error err = nullptr;
bson_iter_t iter;
- if (!mongoc_collection_replace_one(current_collection_, selector, document.get(), opts,&reply, &mongo_err)) {
+ if (!mongoc_collection_replace_one(current_collection_, selector, document.get(), opts, &reply, &mongo_err)) {
err = DBErrorTemplates::kInsertError.Generate(mongo_err.message);
}
- if (err==nullptr) {
+ if (err == nullptr) {
bson_iter_init_find(&iter, &reply, "upsertedCount");
auto n_upsert = bson_iter_int32(&iter);
bson_iter_init_find(&iter, &reply, "modifiedCount");
auto n_mod = bson_iter_int32(&iter);
- if (n_mod + n_upsert!=1) {
+ if (n_mod + n_upsert != 1) {
err = DBErrorTemplates::kInsertError.Generate("metadata does not exist");
}
}
@@ -233,8 +233,9 @@ MongoDBClient::~MongoDBClient() {
CleanUp();
}
-Error MongoDBClient::Insert(const std::string& collection, const std::string& id, const uint8_t* data, uint64_t size,
- MetaIngestMode mode) const {
+Error MongoDBClient::InsertMeta(const std::string& collection, const std::string& id, const uint8_t* data,
+ uint64_t size,
+ MetaIngestMode mode) const {
if (!connected_) {
return DBErrorTemplates::kNotConnected.Generate();
}
@@ -248,18 +249,18 @@ Error MongoDBClient::Insert(const std::string& collection, const std::string& id
if (err) {
return err;
}
-
- if (!BSON_APPEND_UTF8(document.get(), "_id", id.c_str())) {
+ auto id_encoded = EncodeColName(id);
+ if (!BSON_APPEND_UTF8(document.get(), "_id", id_encoded.c_str())) {
err = DBErrorTemplates::kInsertError.Generate("cannot assign document id ");
}
switch (mode.op) {
- case MetaIngestOp::kInsert:
- return InsertBsonDocument(document,false);
- case asapo::MetaIngestOp::kReplace:
- return ReplaceBsonDocument(id, document, mode.upsert);
- case MetaIngestOp::kUpdate:
- break;
+ case MetaIngestOp::kInsert:
+ return InsertBsonDocument(document, false);
+ case asapo::MetaIngestOp::kReplace:
+ return ReplaceBsonDocument(id_encoded, document, mode.upsert);
+ case MetaIngestOp::kUpdate:
+ break;
}
@@ -645,6 +646,7 @@ Error MongoDBClient::DeleteStream(const std::string& stream) const {
DeleteCollections(acks_col);
std::string querystr = ".*_" + EscapeQuery(stream_encoded) + "$";
DeleteDocumentsInCollection("current_location", querystr);
+ DeleteDocumentsInCollection("meta", "^" + EscapeQuery(stream_encoded) + "$");
}
return err;
}
diff --git a/common/cpp/src/database/mongodb_client.h b/common/cpp/src/database/mongodb_client.h
index 2114c4c05..517e75a64 100644
--- a/common/cpp/src/database/mongodb_client.h
+++ b/common/cpp/src/database/mongodb_client.h
@@ -49,8 +49,8 @@ class MongoDBClient final : public Database {
Error Insert(const std::string& collection, const MessageMeta& file, bool ignore_duplicates) const override;
Error InsertAsDatasetMessage(const std::string& collection, const MessageMeta& file, uint64_t dataset_size,
bool ignore_duplicates) const override;
- Error Insert(const std::string& collection, const std::string& id, const uint8_t* data, uint64_t size,
- MetaIngestMode mode) const override;
+ Error InsertMeta(const std::string& collection, const std::string& id, const uint8_t* data, uint64_t size,
+ MetaIngestMode mode) const override;
Error GetById(const std::string& collection, uint64_t id, MessageMeta* file) const override;
Error GetDataSetById(const std::string& collection, uint64_t id_in_set, uint64_t id, MessageMeta* file) const override;
Error GetStreamInfo(const std::string& collection, StreamInfo* info) const override;
diff --git a/receiver/src/request_handler/request_handler_db_meta_write.cpp b/receiver/src/request_handler/request_handler_db_meta_write.cpp
index 58aa2af5c..c1b096af1 100644
--- a/receiver/src/request_handler/request_handler_db_meta_write.cpp
+++ b/receiver/src/request_handler/request_handler_db_meta_write.cpp
@@ -27,7 +27,7 @@ Error RequestHandlerDbMetaWrite::ProcessRequest(Request* request) const {
mode.Decode(request->GetCustomData()[kPosMetaIngestMode]);
}
- auto err = db_client__->Insert(collection_name_prefix_, stream.empty() ? "bt" : "st_" + stream, meta, size, mode);
+ auto err = db_client__->InsertMeta(collection_name_prefix_, stream.empty() ? "bt" : "st_" + stream, meta, size, mode);
if (!err) {
if (stream.empty()) {
log__->Debug(std::string{"insert beamtime meta"} + " to " + collection_name_prefix_ + " in " +
diff --git a/receiver/unittests/request_handler/test_request_handler_db_meta_writer.cpp b/receiver/unittests/request_handler/test_request_handler_db_meta_writer.cpp
index 73a88dd21..56d3ad7a8 100644
--- a/receiver/unittests/request_handler/test_request_handler_db_meta_writer.cpp
+++ b/receiver/unittests/request_handler/test_request_handler_db_meta_writer.cpp
@@ -130,8 +130,8 @@ TEST_F(DbMetaWriterHandlerTests, CallsIngestBeamtimeMetaOldVersion) {
asapo::MetaIngestMode expected_mode{asapo::MetaIngestOp::kReplace, true};
- EXPECT_CALL(mock_db, Insert_t(expected_collection_name, "bt", expected_meta, expected_meta_size,
- M_CheckIngestMode(expected_mode))).
+ EXPECT_CALL(mock_db, InsertMeta_t(expected_collection_name, "bt", expected_meta, expected_meta_size,
+ M_CheckIngestMode(expected_mode))).
WillOnce(testing::Return(nullptr));
EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("insert beamtime meta"),
@@ -152,8 +152,8 @@ TEST_F(DbMetaWriterHandlerTests, CallsIngestBeamtimeMeta) {
asapo::MetaIngestMode expected_mode{asapo::MetaIngestOp::kInsert, true};
- EXPECT_CALL(mock_db, Insert_t(expected_collection_name, "bt", expected_meta, expected_meta_size,
- M_CheckIngestMode(expected_mode))).
+ EXPECT_CALL(mock_db, InsertMeta_t(expected_collection_name, "bt", expected_meta, expected_meta_size,
+ M_CheckIngestMode(expected_mode))).
WillOnce(testing::Return(nullptr));
EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("insert beamtime meta"),
@@ -173,8 +173,8 @@ TEST_F(DbMetaWriterHandlerTests, CallsIngestStreamMeta) {
ExpectRequestParams(asapo::GetReceiverApiVersion(), 13, expected_stream);
asapo::MetaIngestMode expected_mode{asapo::MetaIngestOp::kUpdate, true};
- EXPECT_CALL(mock_db, Insert_t(expected_collection_name, "st_" + expected_stream, expected_meta, expected_meta_size,
- M_CheckIngestMode(expected_mode))).
+ EXPECT_CALL(mock_db, InsertMeta_t(expected_collection_name, "st_" + expected_stream, expected_meta, expected_meta_size,
+ M_CheckIngestMode(expected_mode))).
WillOnce(testing::Return(nullptr));
EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("insert stream meta"),
diff --git a/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp b/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp
index 5e05165dd..9fdcc4cac 100644
--- a/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp
+++ b/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp
@@ -100,16 +100,16 @@ int main(int argc, char* argv[]) {
// metadata
- asapo::MetaIngestMode mode{asapo::MetaIngestOp::kInsert,false};
+ asapo::MetaIngestMode mode{asapo::MetaIngestOp::kInsert, false};
std::string meta = R"({"data":"test"})";
- err = db_new.Insert("meta","bt", reinterpret_cast<const uint8_t *>(meta.c_str()),meta.size(),mode);
+ err = db_new.InsertMeta("meta", stream_name, reinterpret_cast<const uint8_t*>(meta.c_str()), meta.size(), mode);
M_AssertEq(nullptr, err);
- err = db_new.Insert("meta","bt", reinterpret_cast<const uint8_t *>(meta.c_str()),meta.size(),mode);
+ err = db_new.InsertMeta("meta", stream_name, reinterpret_cast<const uint8_t*>(meta.c_str()), meta.size(), mode);
M_AssertTrue(err == asapo::DBErrorTemplates::kDuplicateID);
mode.op = asapo::MetaIngestOp::kReplace;
- err = db_new.Insert("meta","bt", reinterpret_cast<const uint8_t *>(meta.c_str()),meta.size(),mode);
+ err = db_new.InsertMeta("meta", stream_name, reinterpret_cast<const uint8_t*>(meta.c_str()), meta.size(), mode);
M_AssertEq(nullptr, err);
- err = db_new.Insert("meta","bt1", reinterpret_cast<const uint8_t *>(meta.c_str()),meta.size(),mode);
+ err = db_new.InsertMeta("meta", "notexist", reinterpret_cast<const uint8_t*>(meta.c_str()), meta.size(), mode);
M_AssertTrue(err == asapo::DBErrorTemplates::kInsertError);
diff --git a/tests/automatic/mongo_db/upsert/upsert_mongodb.cpp b/tests/automatic/mongo_db/upsert/upsert_mongodb.cpp
index 9b14210d0..ffdbfda51 100644
--- a/tests/automatic/mongo_db/upsert/upsert_mongodb.cpp
+++ b/tests/automatic/mongo_db/upsert/upsert_mongodb.cpp
@@ -46,14 +46,14 @@ int main(int argc, char* argv[]) {
}
auto mode = asapo::MetaIngestMode{asapo::MetaIngestOp::kReplace, true};
- auto err = db.Insert("meta", "0", reinterpret_cast<const uint8_t*>(json.c_str()), json.size(), mode);
+ auto err = db.InsertMeta("meta", "0", reinterpret_cast<const uint8_t*>(json.c_str()), json.size(), mode);
if (err) {
std::cout << err->Explain() << std::endl;
}
Assert(err, args.keyword);
- err = db.Insert("meta", "0", reinterpret_cast<const uint8_t*>(json.c_str()), json.size(), mode);
+ err = db.InsertMeta("meta", "0", reinterpret_cast<const uint8_t*>(json.c_str()), json.size(), mode);
if (err) {
std::cout << err->Explain() << std::endl;
}
--
GitLab