From bb5c8fcc66e6298e255099547032e062691a1533 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Thu, 20 Feb 2020 08:11:36 +0100
Subject: [PATCH] Merge pull request #79 in ASAPO/asapo from
bugfix_ASAPO-106-overwriting to develop
Squashed commit of the following:
commit ce33557832515d541b9d2b505c53225885fd61c7
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Wed Feb 19 13:41:28 2020 +0100
update producer side
commit 0bed6f6ecf3d83ca2f958b6574ad63664f1bfc8c
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Wed Feb 19 11:55:56 2020 +0100
receiver sends warnings for duplicated requests
commit fedf16d3eec9f9a7a1582c8b9f1b8028815a31eb
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Wed Feb 19 11:29:18 2020 +0100
db handler deals with duplicate record
commit d9d242e72b522b15a894c23732f261a584574cb5
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Wed Feb 19 10:31:11 2020 +0100
fix memory error in tests
commit a6c382c2a13dd31520a4bf4561bab1c5ca7e9adf
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Tue Feb 18 17:40:48 2020 +0100
fix mongodb bug
commit 79eaa0107c4550b36c9d3d82259a05c39508b7cc
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Tue Feb 18 16:47:01 2020 +0100
refactor write/receive file handlers, check record in db
commit 20b4dc0533128ec192134d0ffc2899351d933e5b
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon Feb 17 16:39:49 2020 +0100
add overwrite flag to IO routines
commit cf23ea3ca5387ed7c7d52726806bae0e3f6ff1d6
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon Feb 17 14:50:14 2020 +0100
introduce timeout for producer
commit 943cf3c837f151708e538813cf8480bf82e50a53
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Fri Feb 14 13:47:39 2020 +0100
implement GetById for c mongo client
commit 0153e0d1e662584b18c3f55a64093dd343176ac2
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Fri Feb 14 12:08:03 2020 +0100
refactor
commit 631ca34a8be489ba3261a697d5ca889a666d78f2
Author: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Thu Feb 13 18:16:42 2020 +0100
receiver request handler to check db
---
CHANGELOG.md | 8 +-
common/cpp/include/common/data_structs.h | 2 +
common/cpp/include/common/networking.h | 3 +-
common/cpp/include/database/database.h | 3 +
common/cpp/include/database/db_error.h | 8 +-
common/cpp/include/io/io.h | 6 +-
common/cpp/include/preprocessor/definitions.h | 2 +
common/cpp/include/request/request.h | 23 +-
common/cpp/include/request/request_handler.h | 1 +
common/cpp/include/unittests/MockDatabase.h | 11 +
common/cpp/include/unittests/MockIO.h | 20 +-
common/cpp/src/data_structs/data_structs.cpp | 22 ++
common/cpp/src/database/mongodb_client.cpp | 91 +++++-
common/cpp/src/database/mongodb_client.h | 4 +
common/cpp/src/request/request_pool.cpp | 10 +-
common/cpp/src/system_io/system_io.cpp | 19 +-
common/cpp/src/system_io/system_io.h | 8 +-
common/cpp/unittests/request/mocking.h | 4 +-
common/cpp/unittests/request/test_request.cpp | 16 +-
.../unittests/request/test_request_pool.cpp | 41 ++-
consumer/api/cpp/src/server_data_broker.cpp | 26 +-
deploy/build_env/Ubuntu16.04/install_cmake.sh | 3 +
docs/doxygen/main/DoxygenLayout.xml | 4 +-
examples/pipeline/in_to_out/in_to_out.cpp | 4 +-
.../pipeline/in_to_out_python/in_to_out.py | 2 +-
.../dummy_data_producer.cpp | 2 +-
producer/api/cpp/include/producer/producer.h | 1 +
.../api/cpp/include/producer/producer_error.h | 5 +
producer/api/cpp/src/producer.cpp | 4 +-
producer/api/cpp/src/producer_impl.cpp | 9 +-
producer/api/cpp/src/producer_impl.h | 4 +-
producer/api/cpp/src/producer_request.cpp | 3 +-
producer/api/cpp/src/producer_request.h | 3 +-
.../cpp/src/request_handler_filesystem.cpp | 6 +-
.../api/cpp/src/request_handler_filesystem.h | 1 +
producer/api/cpp/src/request_handler_tcp.cpp | 67 +++--
producer/api/cpp/src/request_handler_tcp.h | 7 +-
producer/api/cpp/unittests/test_producer.cpp | 10 +-
.../api/cpp/unittests/test_producer_impl.cpp | 4 +-
.../cpp/unittests/test_producer_request.cpp | 4 +-
.../test_request_handler_filesystem.cpp | 14 +-
.../unittests/test_request_handler_tcp.cpp | 101 +++++--
producer/api/python/asapo_producer.pxd | 3 +-
producer/api/python/asapo_producer.pyx.in | 16 +-
.../src/main_eventmon.cpp | 2 +-
receiver/CMakeLists.txt | 15 +-
receiver/src/file_processor.cpp | 12 +
receiver/src/file_processor.h | 22 ++
...receive.cpp => receive_file_processor.cpp} | 26 +-
receiver/src/receive_file_processor.h | 17 ++
.../receiver_data_server_request.cpp | 2 +-
.../receiver_data_server_request_handler.cpp | 4 +
.../receiver_data_server_request_handler.h | 2 +
receiver/src/receiver_error.h | 13 +-
receiver/src/request.cpp | 26 +-
receiver/src/request.h | 15 +-
receiver/src/request_factory.cpp | 8 +-
receiver/src/request_factory.h | 9 +-
.../src/request_handler_db_check_request.cpp | 72 +++++
.../src/request_handler_db_check_request.h | 24 ++
receiver/src/request_handler_db_write.cpp | 33 ++-
receiver/src/request_handler_db_write.h | 2 +
receiver/src/request_handler_file_process.cpp | 48 +++
receiver/src/request_handler_file_process.h | 25 ++
receiver/src/request_handler_file_receive.h | 22 --
receiver/src/request_handler_file_write.h | 22 --
receiver/src/requests_dispatcher.cpp | 47 ++-
receiver/src/requests_dispatcher.h | 4 +
...ile_write.cpp => write_file_processor.cpp} | 31 +-
receiver/src/write_file_processor.h | 16 +
.../receiver_dataserver_mocking.h | 2 +-
receiver/unittests/receiver_mocking.h | 42 ++-
receiver/unittests/test_connection.cpp | 3 +-
...ve.cpp => test_receive_file_processor.cpp} | 82 +++---
receiver/unittests/test_request.cpp | 30 +-
receiver/unittests/test_request_factory.cpp | 12 +-
.../test_request_handler_authorizer.cpp | 2 +-
.../unittests/test_request_handler_db.cpp | 2 +-
.../test_request_handler_db_check_request.cpp | 273 ++++++++++++++++++
.../test_request_handler_db_meta_writer.cpp | 2 +-
.../test_request_handler_db_writer.cpp | 99 +++++--
.../test_request_handler_file_process.cpp | 136 +++++++++
.../test_request_handler_receive_data.cpp | 6 +-
.../test_request_handler_receive_metadata.cpp | 2 +-
.../unittests/test_requests_dispatcher.cpp | 13 +-
...rite.cpp => test_write_file_processor.cpp} | 94 +++---
.../bugfix_callback.py | 2 +-
.../send_recv_substreams.cpp | 2 +-
.../send_recv_substreams.py | 2 +-
tests/automatic/mongo_db/CMakeLists.txt | 4 +-
.../CMakeLists.txt | 4 +-
.../cleanup_linux.sh | 0
.../cleanup_windows.bat | 0
.../insert_retrieve_mongodb.cpp} | 13 +
.../CMakeLists.txt | 4 +-
.../cleanup_linux.sh | 0
.../cleanup_windows.bat | 0
.../insert_retrieve_dataset_mongodb.cpp} | 14 +-
.../beamtime_metadata/beamtime_metadata.cpp | 2 +-
.../producer/python_api/check_linux.sh | 6 +-
.../producer/python_api/check_windows.bat | 11 +-
.../producer/python_api/producer_api.py | 34 ++-
.../client_serv/ip_tcp_network.cpp | 5 +-
.../write_data_to_file/write_data_to_file.cpp | 6 +-
tests/manual/python_tests/producer/test.py | 2 +-
.../producer_wait_bug_mongo/test.py | 2 +-
.../producer_wait_threads/producer_api.py | 4 +-
.../producer_wait_threads/test.py | 2 +-
108 files changed, 1604 insertions(+), 439 deletions(-)
create mode 100644 receiver/src/file_processor.cpp
create mode 100644 receiver/src/file_processor.h
rename receiver/src/{request_handler_file_receive.cpp => receive_file_processor.cpp} (58%)
create mode 100644 receiver/src/receive_file_processor.h
create mode 100644 receiver/src/request_handler_db_check_request.cpp
create mode 100644 receiver/src/request_handler_db_check_request.h
create mode 100644 receiver/src/request_handler_file_process.cpp
create mode 100644 receiver/src/request_handler_file_process.h
delete mode 100644 receiver/src/request_handler_file_receive.h
delete mode 100644 receiver/src/request_handler_file_write.h
rename receiver/src/{request_handler_file_write.cpp => write_file_processor.cpp} (62%)
create mode 100644 receiver/src/write_file_processor.h
rename receiver/unittests/{test_request_handler_file_receive.cpp => test_receive_file_processor.cpp} (61%)
create mode 100644 receiver/unittests/test_request_handler_db_check_request.cpp
create mode 100644 receiver/unittests/test_request_handler_file_process.cpp
rename receiver/unittests/{test_request_handler_file_write.cpp => test_write_file_processor.cpp} (58%)
rename tests/automatic/mongo_db/{insert => insert_retrieve}/CMakeLists.txt (88%)
rename tests/automatic/mongo_db/{insert => insert_retrieve}/cleanup_linux.sh (100%)
rename tests/automatic/mongo_db/{insert => insert_retrieve}/cleanup_windows.bat (100%)
rename tests/automatic/mongo_db/{insert/insert_mongodb.cpp => insert_retrieve/insert_retrieve_mongodb.cpp} (75%)
rename tests/automatic/mongo_db/{insert_dataset => insert_retrieve_dataset}/CMakeLists.txt (84%)
rename tests/automatic/mongo_db/{insert_dataset => insert_retrieve_dataset}/cleanup_linux.sh (100%)
rename tests/automatic/mongo_db/{insert_dataset => insert_retrieve_dataset}/cleanup_windows.bat (100%)
rename tests/automatic/mongo_db/{insert_dataset/insert_dataset_mongodb.cpp => insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp} (80%)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 513b2886d..4ad0fbbd1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,13 +1,15 @@
+## 20.03 (unreleased)
FEATURES
-* introduced substreams for producer/consumer [[JIRA_102](https://agira.desy.de/browse/HIDRA2-102)]
+* introduced substreams for producer/consumer
+* introduced timeout for producer requests
IMPROVEMENTS
* switch to MongoDB 4.2
* receiver use ASAP3 directory structure to save files to
-* API documentation is available at [asapo-docs.desy.de](asapo-docs.desy.de)
+* API documentation is available at
* switch to using cmake 3.7+
* error messages in Python as Python strings, not byte objects
BUG FIXES
-* consumer operation timout - take duration of the operation into account
\ No newline at end of file
+* consumer operation timout - take duration of the operation into account
+* giving warning/error on attempt to send data/metadata with same id
\ No newline at end of file
diff --git a/common/cpp/include/common/data_structs.h b/common/cpp/include/common/data_structs.h
index 2fb0720d4..24b9649fd 100644
--- a/common/cpp/include/common/data_structs.h
+++ b/common/cpp/include/common/data_structs.h
@@ -41,6 +41,8 @@ using FileInfos = std::vector<FileInfo>;
struct DataSet {
uint64_t id;
FileInfos content;
+ bool SetFromJson(const std::string& json_string);
+
};
using SubDirList = std::vector<std::string>;
diff --git a/common/cpp/include/common/networking.h b/common/cpp/include/common/networking.h
index 529541618..8e6dd3046 100644
--- a/common/cpp/include/common/networking.h
+++ b/common/cpp/include/common/networking.h
@@ -25,11 +25,10 @@ enum Opcode : uint8_t {
enum NetworkErrorCode : uint16_t {
kNetErrorNoError,
+ kNetErrorWarning,
kNetErrorWrongRequest,
kNetErrorNoData,
kNetAuthorizationError,
- kNetErrorFileIdAlreadyInUse,
- kNetErrorAllocateStorageFailed,
kNetErrorInternalServerError = 65535,
};
diff --git a/common/cpp/include/database/database.h b/common/cpp/include/database/database.h
index 8d0bca097..32fa67929 100644
--- a/common/cpp/include/database/database.h
+++ b/common/cpp/include/database/database.h
@@ -21,6 +21,9 @@ class Database {
uint64_t subset_size,
bool ignore_duplicates) const = 0;
+ virtual Error GetById(const std::string& collection, uint64_t id, FileInfo* file) const = 0;
+ virtual Error GetDataSetById(const std::string& collection, uint64_t set_id, uint64_t id, FileInfo* file) const = 0;
+
virtual ~Database() = default;
};
diff --git a/common/cpp/include/database/db_error.h b/common/cpp/include/database/db_error.h
index c36e52458..cb3090045 100644
--- a/common/cpp/include/database/db_error.h
+++ b/common/cpp/include/database/db_error.h
@@ -15,7 +15,8 @@ enum class DBErrorType {
kDuplicateID,
kAlreadyConnected,
kBadAddress,
- kMemoryError
+ kMemoryError,
+ kNoRecord
};
using DBError = ServiceError<DBErrorType, ErrorType::kDBError>;
@@ -23,6 +24,11 @@ using DBErrorTemplate = ServiceErrorTemplate<DBErrorType, ErrorType::kDBError>;
namespace DBErrorTemplates {
+auto const kNoRecord = DBErrorTemplate {
+ "No record", DBErrorType::kNoRecord
+};
+
+
auto const kNotConnected = DBErrorTemplate {
"Not connected", DBErrorType::kNotConnected
};
diff --git a/common/cpp/include/io/io.h b/common/cpp/include/io/io.h
index 92e79f10e..b78016f40 100644
--- a/common/cpp/include/io/io.h
+++ b/common/cpp/include/io/io.h
@@ -100,11 +100,11 @@ class IO {
virtual size_t Write (FileDescriptor fd, const void* buf, size_t length, Error* err) const = 0;
virtual Error RemoveFile(const std::string& fname) const = 0;
virtual Error WriteDataToFile (const std::string& root_folder, const std::string& fname, const FileData& data,
- size_t length, bool create_directories) const = 0;
+ size_t length, bool create_directories, bool allow_ovewrite) const = 0;
virtual Error WriteDataToFile (const std::string& root_folder, const std::string& fname, const uint8_t* data,
- size_t length, bool create_directories) const = 0;
+ size_t length, bool create_directories, bool allow_ovewrite) const = 0;
virtual Error ReceiveDataToFile(SocketDescriptor socket, const std::string& root_folder, const std::string& fname,
- size_t length, bool create_directories) const = 0;
+ size_t length, bool create_directories, bool allow_ovewrite) const = 0;
virtual void CreateNewDirectory (const std::string& directory_name, Error* err) const = 0;
virtual FileData GetDataFromFile (const std::string& fname, uint64_t* fsize, Error* err) const = 0;
virtual SubDirList GetSubDirectories(const std::string& path, Error* err) const = 0;
diff --git a/common/cpp/include/preprocessor/definitions.h b/common/cpp/include/preprocessor/definitions.h
index 385ffd372..3ac042caa 100644
--- a/common/cpp/include/preprocessor/definitions.h
+++ b/common/cpp/include/preprocessor/definitions.h
@@ -3,8 +3,10 @@
#ifdef UNIT_TESTS
#define VIRTUAL virtual
+#define FINAL
#else
#define VIRTUAL
+#define FINAL final
#endif
namespace asapo {
diff --git a/common/cpp/include/request/request.h b/common/cpp/include/request/request.h
index 8bb82cb83..0dfb3e548 100644
--- a/common/cpp/include/request/request.h
+++ b/common/cpp/include/request/request.h
@@ -1,6 +1,8 @@
#ifndef ASAPO_GENERIC_REQUEST_H
#define ASAPO_GENERIC_REQUEST_H
+#include <chrono>
+
#include "common/networking.h"
#include "common/data_structs.h"
#include "io/io.h"
@@ -11,9 +13,28 @@ namespace asapo {
class GenericRequest {
public:
GenericRequest() = delete;
- GenericRequest(GenericRequestHeader h): header{std::move(h)} {};
+ GenericRequest(GenericRequestHeader h, uint64_t timeout_ms): header{std::move(h)}, timeout_ms_{timeout_ms} {};
GenericRequestHeader header;
virtual ~GenericRequest() = default;
+ uint64_t GetRetryCounter() {
+ return retry_counter_;
+ }
+ void IncreaseRetryCounter() {
+ retry_counter_++;
+ }
+ bool TimedOut() {
+ if (timeout_ms_ == 0) {
+ return false;
+ }
+ uint64_t elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() -
+ created_at_).count();
+ return elapsed_ms > timeout_ms_;
+ }
+
+ private:
+ uint64_t retry_counter_ = 0;
+ uint64_t timeout_ms_ = 0;
+ std::chrono::system_clock::time_point created_at_ = std::chrono::system_clock::now();
};
using GenericRequestPtr = std::unique_ptr<GenericRequest>;
diff --git a/common/cpp/include/request/request_handler.h b/common/cpp/include/request/request_handler.h
index 25f63d653..60fbd767b 100644
--- a/common/cpp/include/request/request_handler.h
+++ b/common/cpp/include/request/request_handler.h
@@ -14,6 +14,7 @@ class RequestHandler {
virtual void PrepareProcessingRequestLocked() = 0;
virtual void TearDownProcessingRequestLocked(bool processing_succeeded) = 0;
virtual bool ProcessRequestUnlocked(GenericRequest* request) = 0;
+ virtual void ProcessRequestTimeout(GenericRequest* request) = 0;
virtual bool ReadyProcessRequest() = 0;
virtual ~RequestHandler() = default;
};
diff --git a/common/cpp/include/unittests/MockDatabase.h b/common/cpp/include/unittests/MockDatabase.h
index 85ff272d8..b1ac72561 100644
--- a/common/cpp/include/unittests/MockDatabase.h
+++ b/common/cpp/include/unittests/MockDatabase.h
@@ -38,7 +38,18 @@ class MockDatabase : public Database {
}
MOCK_CONST_METHOD4(Upsert_t, ErrorInterface * (const std::string&, uint64_t id, const uint8_t* data, uint64_t size));
+ Error GetById(const std::string& collection, uint64_t id, FileInfo* file) const override {
+ return Error{GetById_t(collection, id, file)};
+ }
+
+ MOCK_CONST_METHOD3(GetById_t, ErrorInterface * (const std::string&, uint64_t id, FileInfo*));
+
+
+ Error GetDataSetById(const std::string& collection, uint64_t set_id, uint64_t id, FileInfo* file) const override {
+ return Error{GetSetById_t(collection, set_id, id, file)};
+ }
+ MOCK_CONST_METHOD4(GetSetById_t, ErrorInterface * (const std::string&, uint64_t set_id, uint64_t id, FileInfo*));
// stuff to test db destructor is called and avoid "uninteresting call" messages
MOCK_METHOD0(Die, void());
diff --git a/common/cpp/include/unittests/MockIO.h b/common/cpp/include/unittests/MockIO.h
index e24eee23d..c57017921 100644
--- a/common/cpp/include/unittests/MockIO.h
+++ b/common/cpp/include/unittests/MockIO.h
@@ -217,16 +217,16 @@ class MockIO : public IO {
MOCK_CONST_METHOD3(SendFile_t, ErrorInterface * (SocketDescriptor socket_fd, const std::string& fname, size_t length));
Error WriteDataToFile(const std::string& root_folder, const std::string& fname, const FileData& data,
- size_t length, bool create_directories) const override {
- return Error{WriteDataToFile_t(root_folder, fname, data.get(), length, create_directories)};
+ size_t length, bool create_directories, bool allow_ovewrite) const override {
+ return Error{WriteDataToFile_t(root_folder, fname, data.get(), length, create_directories, allow_ovewrite)};
}
MOCK_CONST_METHOD1(RemoveFile_t, ErrorInterface * (const std::string& fname));
Error WriteDataToFile(const std::string& root_folder, const std::string& fname, const uint8_t* data,
- size_t length, bool create_directories) const override {
- return Error{WriteDataToFile_t(root_folder, fname, data, length, create_directories)};
+ size_t length, bool create_directories, bool allow_ovewrite) const override {
+ return Error{WriteDataToFile_t(root_folder, fname, data, length, create_directories, allow_ovewrite)};
}
@@ -235,17 +235,17 @@ class MockIO : public IO {
}
- MOCK_CONST_METHOD5(ReceiveDataToFile_t, ErrorInterface * (SocketDescriptor socket, const std::string& root_folder,
- const std::string& fname, size_t fsize, bool create_directories));
+ MOCK_CONST_METHOD6(ReceiveDataToFile_t, ErrorInterface * (SocketDescriptor socket, const std::string& root_folder,
+ const std::string& fname, size_t fsize, bool create_directories, bool allow_ovewrite));
Error ReceiveDataToFile(SocketDescriptor socket, const std::string& root_folder, const std::string& fname,
- size_t length, bool create_directories) const override {
- return Error{ReceiveDataToFile_t(socket, root_folder, fname, length, create_directories)};
+ size_t length, bool create_directories, bool allow_ovewrite) const override {
+ return Error{ReceiveDataToFile_t(socket, root_folder, fname, length, create_directories, allow_ovewrite)};
}
- MOCK_CONST_METHOD5(WriteDataToFile_t, ErrorInterface * (const std::string& root_folder, const std::string& fname,
- const uint8_t* data, size_t fsize, bool create_directories));
+ MOCK_CONST_METHOD6(WriteDataToFile_t, ErrorInterface * (const std::string& root_folder, const std::string& fname,
+ const uint8_t* data, size_t fsize, bool create_directories, bool allow_ovewrite));
FileInfo GetFileInfo(const std::string& name, Error* err) const override {
diff --git a/common/cpp/src/data_structs/data_structs.cpp b/common/cpp/src/data_structs/data_structs.cpp
index 0c87a7f28..bdda7fa5f 100644
--- a/common/cpp/src/data_structs/data_structs.cpp
+++ b/common/cpp/src/data_structs/data_structs.cpp
@@ -64,7 +64,29 @@ bool TimeFromJson(const JsonStringParser& parser, const std::string name, std::c
return true;
}
+bool DataSet::SetFromJson(const std::string& json_string) {
+ auto old = *this;
+
+ auto parser = JsonStringParser(std::move(json_string));
+ std::vector<std::string> vec_fi_endcoded;
+ Error parse_err;
+ (parse_err = parser.GetArrayRawStrings("images", &vec_fi_endcoded)) ||
+ (parse_err = parser.GetUInt64("_id", &id));
+ if (parse_err) {
+ *this = old;
+ return false;
+ }
+ for (auto fi_encoded : vec_fi_endcoded) {
+ FileInfo fi;
+ if (!fi.SetFromJson(fi_encoded)) {
+ *this = old;
+ return false;
+ }
+ content.emplace_back(fi);
+ }
+ return true;
+}
bool FileInfo::SetFromJson(const std::string& json_string) {
auto old = *this;
diff --git a/common/cpp/src/database/mongodb_client.cpp b/common/cpp/src/database/mongodb_client.cpp
index 9380bdcd0..0330c14df 100644
--- a/common/cpp/src/database/mongodb_client.cpp
+++ b/common/cpp/src/database/mongodb_client.cpp
@@ -1,9 +1,9 @@
#include "mongodb_client.h"
+#include "mongodb_client.h"
#include "database/db_error.h"
namespace asapo {
-using std::string;
using asapo::Database;
MongoDbInstance::MongoDbInstance() {
@@ -33,7 +33,7 @@ MongoDBClient::MongoDBClient() {
MongoDbInstance::Instantiate();
}
-Error MongoDBClient::InitializeClient(const string& address) {
+Error MongoDBClient::InitializeClient(const std::string& address) {
auto uri_str = DBAddress(address);
client_ = mongoc_client_new (uri_str.c_str());
@@ -49,7 +49,7 @@ Error MongoDBClient::InitializeClient(const string& address) {
}
-void MongoDBClient::UpdateCurrentCollectionIfNeeded(const string& collection_name) const {
+void MongoDBClient::UpdateCurrentCollectionIfNeeded(const std::string& collection_name) const {
if (collection_name == current_collection_name_) {
return;
}
@@ -71,7 +71,7 @@ Error MongoDBClient::TryConnectDatabase() {
return err;
}
-Error MongoDBClient::Connect(const string& address, const string& database_name) {
+Error MongoDBClient::Connect(const std::string& address, const std::string& database_name) {
if (connected_) {
return DBErrorTemplates::kAlreadyConnected.Generate();
}
@@ -90,7 +90,7 @@ Error MongoDBClient::Connect(const string& address, const string& database_name)
return err;
}
-string MongoDBClient::DBAddress(const string& address) const {
+std::string MongoDBClient::DBAddress(const std::string& address) const {
return "mongodb://" + address + "/?appname=asapo";
}
@@ -271,4 +271,85 @@ Error MongoDBClient::InsertAsSubset(const std::string& collection, const FileInf
return err;
}
+Error MongoDBClient::GetRecordFromDb(const std::string& collection, uint64_t id, std::string* res) const {
+ if (!connected_) {
+ return DBErrorTemplates::kNotConnected.Generate();
+ }
+
+ UpdateCurrentCollectionIfNeeded(collection);
+
+
+ Error err;
+ bson_error_t mongo_err;
+ bson_t* filter;
+ bson_t* opts;
+ mongoc_cursor_t* cursor;
+ const bson_t* doc;
+ char* str;
+
+ filter = BCON_NEW ("_id", BCON_INT64 (id));
+ opts = BCON_NEW ("limit", BCON_INT64 (1));
+
+ cursor = mongoc_collection_find_with_opts (current_collection_, filter, opts, NULL);
+
+ bool found = false;
+ while (mongoc_cursor_next (cursor, &doc)) {
+ str = bson_as_relaxed_extended_json (doc, NULL);
+ *res = str;
+ found = true;
+ bson_free (str);
+ }
+
+ if (mongoc_cursor_error (cursor, &mongo_err)) {
+ err = DBErrorTemplates::kDBError.Generate(mongo_err.message);
+ } else {
+ if (!found) {
+ err = DBErrorTemplates::kNoRecord.Generate();
+ }
+ }
+
+ mongoc_cursor_destroy (cursor);
+ bson_destroy (filter);
+ bson_destroy (opts);
+
+ return err;
+}
+
+
+Error MongoDBClient::GetById(const std::string& collection, uint64_t id, FileInfo* file) const {
+ std::string record_str;
+ auto err = GetRecordFromDb(collection, id, &record_str);
+ if (err) {
+ return err;
+ }
+
+ if (!file->SetFromJson(record_str)) {
+ DBErrorTemplates::kJsonParseError.Generate(record_str);
+ }
+ return nullptr;
+}
+
+Error MongoDBClient::GetDataSetById(const std::string& collection, uint64_t set_id, uint64_t id, FileInfo* file) const {
+ std::string record_str;
+ auto err = GetRecordFromDb(collection, set_id, &record_str);
+ if (err) {
+ return err;
+ }
+
+ DataSet dataset;
+ if (!dataset.SetFromJson(record_str)) {
+ DBErrorTemplates::kJsonParseError.Generate(record_str);
+ }
+
+ for (const auto& fileinfo : dataset.content) {
+ if (fileinfo.id == id) {
+ *file = fileinfo;
+ return nullptr;
+ }
+ }
+
+ return DBErrorTemplates::kNoRecord.Generate();
+
+}
+
}
diff --git a/common/cpp/src/database/mongodb_client.h b/common/cpp/src/database/mongodb_client.h
index 0544d7912..75303a459 100644
--- a/common/cpp/src/database/mongodb_client.h
+++ b/common/cpp/src/database/mongodb_client.h
@@ -41,6 +41,8 @@ class MongoDBClient final : public Database {
Error InsertAsSubset(const std::string& collection, const FileInfo& file, uint64_t subset_id, uint64_t subset_size,
bool ignore_duplicates) const override;
Error Upsert(const std::string& collection, uint64_t id, const uint8_t* data, uint64_t size) const override;
+ Error GetById(const std::string& collection, uint64_t id, FileInfo* file) const override;
+ Error GetDataSetById(const std::string& collection, uint64_t set_id, uint64_t id, FileInfo* file) const override;
~MongoDBClient() override;
private:
mongoc_client_t* client_{nullptr};
@@ -58,6 +60,8 @@ class MongoDBClient final : public Database {
Error InsertBsonDocument(const bson_p& document, bool ignore_duplicates) const;
Error UpdateBsonDocument(uint64_t id, const bson_p& document, bool upsert) const;
Error AddBsonDocumentToArray(bson_t* query, bson_t* update, bool ignore_duplicates) const;
+ Error GetRecordFromDb(const std::string& collection, uint64_t id, std::string* res) const;
+
};
}
diff --git a/common/cpp/src/request/request_pool.cpp b/common/cpp/src/request/request_pool.cpp
index ccf724e36..de91b3f15 100644
--- a/common/cpp/src/request/request_pool.cpp
+++ b/common/cpp/src/request/request_pool.cpp
@@ -35,13 +35,19 @@ GenericRequestPtr RequestPool::GetRequestFromQueue() {
}
void RequestPool::PutRequestBackToQueue(GenericRequestPtr request) {
+// do not need to lock since we already own it
+ request->IncreaseRetryCounter();
request_queue_.emplace_front(std::move(request));
}
void RequestPool::ProcessRequest(const std::unique_ptr<RequestHandler>& request_handler,
ThreadInformation* thread_info) {
- request_handler->PrepareProcessingRequestLocked();
auto request = GetRequestFromQueue();
+ if (request->TimedOut()) {
+ request_handler->ProcessRequestTimeout(request.get());
+ return;
+ }
+ request_handler->PrepareProcessingRequestLocked();
requests_in_progress_++;
thread_info->lock.unlock();
auto success = request_handler->ProcessRequestUnlocked(request.get());
@@ -50,8 +56,8 @@ void RequestPool::ProcessRequest(const std::unique_ptr<RequestHandler>& request_
request_handler->TearDownProcessingRequestLocked(success);
if (!success) {
PutRequestBackToQueue(std::move(request));
- condition_.notify_all();
thread_info->lock.unlock();
+ condition_.notify_all();
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
thread_info->lock.lock();
}
diff --git a/common/cpp/src/system_io/system_io.cpp b/common/cpp/src/system_io/system_io.cpp
index 6bc1e55dd..e3c8b5dc9 100644
--- a/common/cpp/src/system_io/system_io.cpp
+++ b/common/cpp/src/system_io/system_io.cpp
@@ -147,14 +147,15 @@ void asapo::SystemIO::CreateNewDirectory(const std::string& directory_name, Erro
}
FileDescriptor SystemIO::OpenWithCreateFolders(const std::string& root_folder, const std::string& fname,
- bool create_directories, Error* err) const {
+ bool create_directories, bool allow_ovewrite, Error* err) const {
std::string full_name;
if (!root_folder.empty()) {
full_name = root_folder + kPathSeparator + fname;
} else {
full_name = fname;
}
- auto fd = Open(full_name, IO_OPEN_MODE_CREATE | IO_OPEN_MODE_RW | IO_OPEN_MODE_SET_LENGTH_0, err);
+ auto create_flag = allow_ovewrite ? IO_OPEN_MODE_CREATE : IO_OPEN_MODE_CREATE_AND_FAIL_IF_EXISTS;
+ auto fd = Open(full_name, create_flag | IO_OPEN_MODE_RW | IO_OPEN_MODE_SET_LENGTH_0, err);
if (*err == IOErrorTemplates::kFileNotFound && create_directories) {
size_t pos = fname.rfind(kPathSeparator);
if (pos == std::string::npos) {
@@ -165,7 +166,7 @@ FileDescriptor SystemIO::OpenWithCreateFolders(const std::string& root_folder, c
if (*err) {
return -1;
}
- return OpenWithCreateFolders(root_folder, fname, false, err);
+ return OpenWithCreateFolders(root_folder, fname, false, allow_ovewrite, err);
}
return fd;
@@ -173,9 +174,9 @@ FileDescriptor SystemIO::OpenWithCreateFolders(const std::string& root_folder, c
}
Error SystemIO::WriteDataToFile(const std::string& root_folder, const std::string& fname, const uint8_t* data,
- size_t length, bool create_directories) const {
+ size_t length, bool create_directories, bool allow_ovewrite) const {
Error err;
- auto fd = OpenWithCreateFolders(root_folder, fname, create_directories, &err);
+ auto fd = OpenWithCreateFolders(root_folder, fname, create_directories, allow_ovewrite, &err);
if (err) {
return err;
}
@@ -192,8 +193,8 @@ Error SystemIO::WriteDataToFile(const std::string& root_folder, const std::strin
}
Error SystemIO::WriteDataToFile(const std::string& root_folder, const std::string& fname, const FileData& data,
- size_t length, bool create_directories) const {
- return WriteDataToFile(root_folder, fname, data.get(), length, create_directories);
+ size_t length, bool create_directories, bool allow_ovewrite) const {
+ return WriteDataToFile(root_folder, fname, data.get(), length, create_directories, allow_ovewrite);
}
std::string SystemIO::ReadFileToString(const std::string& fname, Error* err) const {
@@ -663,9 +664,9 @@ Error SystemIO::SendFile(SocketDescriptor socket_fd, const std::string& fname, s
}
Error SystemIO:: ReceiveDataToFile(SocketDescriptor socket, const std::string& root_folder, const std::string& fname,
- size_t length, bool create_directories) const {
+ size_t length, bool create_directories, bool allow_ovewrite) const {
Error err;
- auto fd = OpenWithCreateFolders(root_folder, fname, create_directories, &err);
+ auto fd = OpenWithCreateFolders(root_folder, fname, create_directories, allow_ovewrite, &err);
if (err) {
return err;
}
diff --git a/common/cpp/src/system_io/system_io.h b/common/cpp/src/system_io/system_io.h
index 69443af6b..78621e1fc 100644
--- a/common/cpp/src/system_io/system_io.h
+++ b/common/cpp/src/system_io/system_io.h
@@ -84,7 +84,7 @@ class SystemIO final : public IO {
Error CreateDirectoryWithParents(const std::string& root_path, const std::string& path) const;
uint8_t* AllocateArray(uint64_t fsize, Error* err) const;
FileDescriptor OpenWithCreateFolders(const std::string& root_folder, const std::string& fname,
- bool create_directories, Error* err) const;
+ bool create_directories, bool allow_ovewrite, Error* err) const;
public:
~SystemIO();
/*
@@ -133,12 +133,12 @@ class SystemIO final : public IO {
void CreateNewDirectory(const std::string& directory_name, Error* err) const override;
FileData GetDataFromFile(const std::string& fname, uint64_t* fsize, Error* err) const override;
Error WriteDataToFile (const std::string& root_folder, const std::string& fname, const FileData& data,
- size_t length, bool create_directories) const override;
+ size_t length, bool create_directories, bool allow_ovewrite) const override;
Error ReceiveDataToFile(SocketDescriptor socket, const std::string& root_folder, const std::string& fname,
- size_t length, bool create_directories) const override;
+ size_t length, bool create_directories, bool allow_ovewrite) const override;
Error WriteDataToFile(const std::string& root_folder, const std::string& fname, const uint8_t* data,
- size_t length, bool create_directories) const override;
+ size_t length, bool create_directories, bool allow_ovewrite) const override;
SubDirList GetSubDirectories(const std::string& path, Error* err) const override;
std::string ReadFileToString(const std::string& fname, Error* err) const override;
Error RemoveFile(const std::string& fname) const override;
diff --git a/common/cpp/unittests/request/mocking.h b/common/cpp/unittests/request/mocking.h
index 9e9ef6d5c..d37fdd79d 100644
--- a/common/cpp/unittests/request/mocking.h
+++ b/common/cpp/unittests/request/mocking.h
@@ -16,8 +16,10 @@ class MockRequestHandler : public RequestHandler {
MOCK_METHOD0(ReadyProcessRequest, bool());
MOCK_METHOD1(TearDownProcessingRequestLocked, void(bool processing_succeeded));
MOCK_METHOD1(ProcessRequestUnlocked_t, bool (const GenericRequest* request));
-
+ MOCK_METHOD1(ProcessRequestTimeout, void(GenericRequest* request));
+ uint64_t retry_counter = 0;
bool ProcessRequestUnlocked(GenericRequest* request) override {
+ retry_counter = request->GetRetryCounter();
std::this_thread::sleep_for(std::chrono::milliseconds(50));
return ProcessRequestUnlocked_t(request);
}
diff --git a/common/cpp/unittests/request/test_request.cpp b/common/cpp/unittests/request/test_request.cpp
index 80669cddb..c632486ee 100644
--- a/common/cpp/unittests/request/test_request.cpp
+++ b/common/cpp/unittests/request/test_request.cpp
@@ -37,15 +37,27 @@ using asapo::GenericRequest;
using asapo::GenericRequestHeader;
-TEST(Request, Constructor) {
+TEST(Request, Tests) {
GenericRequestHeader header{asapo::kOpcodeTransferData, 1, 2, 3, "hello"};
- GenericRequest r{header};
+ GenericRequest r{header, 90};
ASSERT_THAT(r.header.data_id, Eq(1));
ASSERT_THAT(r.header.op_code, Eq(asapo::kOpcodeTransferData));
ASSERT_THAT(r.header.data_size, Eq(2));
ASSERT_THAT(r.header.meta_size, Eq(3));
ASSERT_THAT(r.header.message, testing::StrEq("hello"));
+ ASSERT_THAT(r.GetRetryCounter(), Eq(0));
+ ASSERT_THAT(r.TimedOut(), Eq(false));
+ std::this_thread::sleep_for(std::chrono::milliseconds(200));
+ ASSERT_THAT(r.TimedOut(), Eq(true));
+ r.IncreaseRetryCounter();
+ ASSERT_THAT(r.GetRetryCounter(), Eq(1));
+
+ GenericRequest r_notimeout{header, 0};
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+ ASSERT_THAT(r_notimeout.TimedOut(), Eq(false));
+
+
}
}
diff --git a/common/cpp/unittests/request/test_request_pool.cpp b/common/cpp/unittests/request/test_request_pool.cpp
index 91c5cb790..447d4c0e9 100644
--- a/common/cpp/unittests/request/test_request_pool.cpp
+++ b/common/cpp/unittests/request/test_request_pool.cpp
@@ -53,7 +53,7 @@ class MockRequestHandlerFactory : public asapo::RequestHandlerFactory {
class TestRequest : public GenericRequest {
public:
- TestRequest(GenericRequestHeader header): GenericRequest(header) {};
+ TestRequest(GenericRequestHeader header, uint64_t timeout): GenericRequest(header, timeout) {};
};
@@ -64,7 +64,7 @@ class RequestPoolTests : public testing::Test {
MockRequestHandlerFactory request_handler_factory{mock_request_handler};
const uint8_t nthreads = 1;
asapo::RequestPool pool {nthreads, &request_handler_factory, &mock_logger};
- std::unique_ptr<GenericRequest> request{new TestRequest{GenericRequestHeader{}}};
+ std::unique_ptr<GenericRequest> request{new TestRequest{GenericRequestHeader{}, 0}};
void SetUp() override {
}
void TearDown() override {
@@ -98,7 +98,20 @@ TEST_F(RequestPoolTests, NRequestsInPoolInitial) {
ASSERT_THAT(nreq, Eq(0));
}
+TEST_F(RequestPoolTests, TimeOut) {
+ std::unique_ptr<GenericRequest> request{new TestRequest{GenericRequestHeader{}, 10}};
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+ EXPECT_CALL(*mock_request_handler, ReadyProcessRequest()).Times(1).WillRepeatedly(Return(true));
+ EXPECT_CALL(*mock_request_handler, PrepareProcessingRequestLocked()).Times(0);
+ EXPECT_CALL(*mock_request_handler, ProcessRequestUnlocked_t(_)).Times(0);
+ EXPECT_CALL(*mock_request_handler, ProcessRequestTimeout(_)).Times(1);
+
+ auto err = pool.AddRequest(std::move(request));
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ ASSERT_THAT(err, Eq(nullptr));
+}
void ExpectSend(MockRequestHandler* mock_handler, int ntimes = 1) {
EXPECT_CALL(*mock_handler, ReadyProcessRequest()).Times(ntimes).WillRepeatedly(Return(true));
@@ -107,6 +120,26 @@ void ExpectSend(MockRequestHandler* mock_handler, int ntimes = 1) {
EXPECT_CALL(*mock_handler, TearDownProcessingRequestLocked(true)).Times(ntimes);
}
+void ExpectFailProcessRequest(MockRequestHandler* mock_handler) {
+ EXPECT_CALL(*mock_handler, ReadyProcessRequest()).Times(AtLeast(1)).WillRepeatedly(Return(true));
+ EXPECT_CALL(*mock_handler, PrepareProcessingRequestLocked()).Times(AtLeast(1));
+ EXPECT_CALL(*mock_handler, ProcessRequestUnlocked_t(_)).Times(AtLeast(1)).WillRepeatedly(Return(false));
+ EXPECT_CALL(*mock_handler, TearDownProcessingRequestLocked(false)).Times(AtLeast(1));
+}
+
+
+
+TEST_F(RequestPoolTests, AddRequestIncreasesRetryCounter) {
+
+ ExpectFailProcessRequest(mock_request_handler);
+
+ auto err = pool.AddRequest(std::move(request));
+ std::this_thread::sleep_for(std::chrono::milliseconds(1500));
+
+ ASSERT_THAT(err, Eq(nullptr));
+ ASSERT_THAT(mock_request_handler->retry_counter, Gt(0));
+}
+
TEST_F(RequestPoolTests, AddRequestCallsSend) {
@@ -145,7 +178,7 @@ TEST_F(RequestPoolTests, NRequestsInPoolAccountsForRequestsInProgress) {
TEST_F(RequestPoolTests, AddRequestCallsSendTwoRequests) {
- TestRequest* request2 = new TestRequest{GenericRequestHeader{}};
+ TestRequest* request2 = new TestRequest{GenericRequestHeader{}, 0};
ExpectSend(mock_request_handler, 2);
@@ -161,7 +194,7 @@ TEST_F(RequestPoolTests, AddRequestCallsSendTwoRequests) {
TEST_F(RequestPoolTests, AddRequestsOk) {
- TestRequest* request2 = new TestRequest{GenericRequestHeader{}};
+ TestRequest* request2 = new TestRequest{GenericRequestHeader{}, 0};
ExpectSend(mock_request_handler, 2);
diff --git a/consumer/api/cpp/src/server_data_broker.cpp b/consumer/api/cpp/src/server_data_broker.cpp
index 685aafb10..4078698d3 100644
--- a/consumer/api/cpp/src/server_data_broker.cpp
+++ b/consumer/api/cpp/src/server_data_broker.cpp
@@ -395,28 +395,14 @@ std::string ServerDataBroker::GetBeamtimeMeta(Error* err) {
}
DataSet ServerDataBroker::DecodeDatasetFromResponse(std::string response, Error* err) {
- auto parser = JsonStringParser(std::move(response));
-
- std::vector<std::string> vec_fi_endcoded;
- Error parse_err;
- uint64_t id;
- (parse_err = parser.GetArrayRawStrings("images", &vec_fi_endcoded)) ||
- (parse_err = parser.GetUInt64("_id", &id));
- if (parse_err) {
- *err = ConsumerErrorTemplates::kInterruptedTransaction.Generate("malformed response:" + parse_err->Explain());
+ DataSet res;
+ if (!res.SetFromJson(std::move(response))) {
+ *err = ConsumerErrorTemplates::kInterruptedTransaction.Generate("malformed response:" + response);
return {0, FileInfos{}};
+ } else {
+ *err = nullptr;
+ return res;
}
-
- auto res = FileInfos{};
- for (auto fi_encoded : vec_fi_endcoded) {
- FileInfo fi;
- if (!fi.SetFromJson(fi_encoded)) {
- *err = ConsumerErrorTemplates::kInterruptedTransaction.Generate("malformed response:" + fi_encoded);
- return {0, FileInfos{}};
- }
- res.emplace_back(fi);
- }
- return {id, std::move(res)};
}
FileInfos ServerDataBroker::QueryImages(std::string query, std::string substream, Error* err) {
diff --git a/deploy/build_env/Ubuntu16.04/install_cmake.sh b/deploy/build_env/Ubuntu16.04/install_cmake.sh
index 55245ffb3..3e399d4ec 100755
--- a/deploy/build_env/Ubuntu16.04/install_cmake.sh
+++ b/deploy/build_env/Ubuntu16.04/install_cmake.sh
@@ -8,6 +8,9 @@ cd cmake-3.*
make -j$(nproc)
make install
+cd ..
+rm -rf cmake-3*
+
cmake --version
/usr/local/bin/cmake --version
diff --git a/docs/doxygen/main/DoxygenLayout.xml b/docs/doxygen/main/DoxygenLayout.xml
index fc5fb0847..702bee96b 100644
--- a/docs/doxygen/main/DoxygenLayout.xml
+++ b/docs/doxygen/main/DoxygenLayout.xml
@@ -1,4 +1,4 @@
<navindex>
- <tab type="user" url="consumer/index.html" title="Consumer API"/>
- <tab type="user" url="producer/index.html" title="Producer API"/>
+ <tab type="user" url="consumer/index.html" visible="$ALPHABETICAL_INDEX" title="Consumer API"/>
+ <tab type="user" url="producer/index.html" visible="$ALPHABETICAL_INDEX" title="Producer API"/>
</navindex>
diff --git a/examples/pipeline/in_to_out/in_to_out.cpp b/examples/pipeline/in_to_out/in_to_out.cpp
index 11ff051b1..714e704c2 100644
--- a/examples/pipeline/in_to_out/in_to_out.cpp
+++ b/examples/pipeline/in_to_out/in_to_out.cpp
@@ -39,7 +39,7 @@ struct Args {
};
void ProcessAfterSend(asapo::GenericRequestHeader header, asapo::Error err) {
- if (err) {
+ if (err && err!=asapo::ProducerErrorTemplates::kServerWarning) {
std::cerr << "Data was not successfully send: " << err << std::endl;
return;
}
@@ -188,7 +188,7 @@ std::unique_ptr<asapo::Producer> CreateProducer(const Args& args) {
asapo::Error err;
auto producer = asapo::Producer::Create(args.server, args.nthreads,
asapo::RequestHandlerType::kTcp,
- asapo::SourceCredentials{args.beamtime_id, args.stream_out, args.token }, &err);
+ asapo::SourceCredentials{args.beamtime_id, args.stream_out, args.token }, 60, &err);
if(err) {
std::cerr << "Cannot start producer. ProducerError: " << err << std::endl;
exit(EXIT_FAILURE);
diff --git a/examples/pipeline/in_to_out_python/in_to_out.py b/examples/pipeline/in_to_out_python/in_to_out.py
index b8a51975c..4c67b5947 100644
--- a/examples/pipeline/in_to_out_python/in_to_out.py
+++ b/examples/pipeline/in_to_out_python/in_to_out.py
@@ -30,7 +30,7 @@ transfer_data=int(transfer_data)>0
broker = asapo_consumer.create_server_broker(source,path, beamtime,stream_in,token,timeout_s*1000)
-producer = asapo_producer.create_producer(source,beamtime, stream_out, token, nthreads)
+producer = asapo_producer.create_producer(source,beamtime, stream_out, token, nthreads, 600)
group_id = broker.generate_group_id()
diff --git a/examples/producer/dummy-data-producer/dummy_data_producer.cpp b/examples/producer/dummy-data-producer/dummy_data_producer.cpp
index e2b034385..9a7be1a9c 100644
--- a/examples/producer/dummy-data-producer/dummy_data_producer.cpp
+++ b/examples/producer/dummy-data-producer/dummy_data_producer.cpp
@@ -180,7 +180,7 @@ std::unique_ptr<asapo::Producer> CreateProducer(const Args& args) {
asapo::Error err;
auto producer = asapo::Producer::Create(args.receiver_address, args.nthreads,
args.mode % 10 == 0 ? asapo::RequestHandlerType::kTcp : asapo::RequestHandlerType::kFilesystem,
- asapo::SourceCredentials{args.beamtime_id, args.stream, args.token }, &err);
+ asapo::SourceCredentials{args.beamtime_id, args.stream, args.token }, 3600, &err);
if(err) {
std::cerr << "Cannot start producer. ProducerError: " << err << std::endl;
exit(EXIT_FAILURE);
diff --git a/producer/api/cpp/include/producer/producer.h b/producer/api/cpp/include/producer/producer.h
index 3dc1def79..f4e4b43c5 100644
--- a/producer/api/cpp/include/producer/producer.h
+++ b/producer/api/cpp/include/producer/producer.h
@@ -20,6 +20,7 @@ class Producer {
*/
static std::unique_ptr<Producer> Create(const std::string& endpoint, uint8_t n_processing_threads,
asapo::RequestHandlerType type, SourceCredentials source_cred,
+ uint64_t timeout_sec,
Error* err);
virtual ~Producer() = default;
diff --git a/producer/api/cpp/include/producer/producer_error.h b/producer/api/cpp/include/producer/producer_error.h
index 0dc331258..077c4108b 100644
--- a/producer/api/cpp/include/producer/producer_error.h
+++ b/producer/api/cpp/include/producer/producer_error.h
@@ -10,6 +10,7 @@ enum class ProducerErrorType {
kRequestPoolIsFull,
kLocalIOError,
kWrongInput,
+ kServerWarning,
kTimeout
};
@@ -17,6 +18,10 @@ using ProducerErrorTemplate = ServiceErrorTemplate<ProducerErrorType, ErrorType:
namespace ProducerErrorTemplates {
+auto const kServerWarning = ProducerErrorTemplate {
+ "server warning", ProducerErrorType::kServerWarning
+};
+
auto const kLocalIOError = ProducerErrorTemplate {
"local i/o error", ProducerErrorType::kLocalIOError
};
diff --git a/producer/api/cpp/src/producer.cpp b/producer/api/cpp/src/producer.cpp
index 84191fcf8..d56b8cab8 100644
--- a/producer/api/cpp/src/producer.cpp
+++ b/producer/api/cpp/src/producer.cpp
@@ -3,7 +3,7 @@
#include "producer/producer_error.h"
std::unique_ptr<asapo::Producer> asapo::Producer::Create(const std::string& endpoint, uint8_t n_processing_threads,
- asapo::RequestHandlerType type, SourceCredentials source_cred, Error* err) {
+ asapo::RequestHandlerType type, SourceCredentials source_cred, uint64_t timeout_sec, Error* err) {
if (n_processing_threads > kMaxProcessingThreads || n_processing_threads == 0) {
*err = ProducerErrorTemplates::kWrongInput.Generate("Set number of processing threads > 0 and <= " + std::to_string(
@@ -13,7 +13,7 @@ std::unique_ptr<asapo::Producer> asapo::Producer::Create(const std::string& endp
std::unique_ptr<asapo::Producer> producer;
try {
- producer.reset(new ProducerImpl(endpoint, n_processing_threads, type));
+ producer.reset(new ProducerImpl(endpoint, n_processing_threads, timeout_sec, type));
} catch (const std::exception& ex) {
*err = TextError(ex.what());
return nullptr;
diff --git a/producer/api/cpp/src/producer_impl.cpp b/producer/api/cpp/src/producer_impl.cpp
index 12c3b0335..a9a1e40e8 100644
--- a/producer/api/cpp/src/producer_impl.cpp
+++ b/producer/api/cpp/src/producer_impl.cpp
@@ -18,8 +18,9 @@ const std::string ProducerImpl::kFinishSubStreamKeyword = "asapo_finish_substrea
const std::string ProducerImpl::kNoNextSubStreamKeyword = "asapo_no_next";
-ProducerImpl::ProducerImpl(std::string endpoint, uint8_t n_processing_threads, asapo::RequestHandlerType type):
- log__{GetDefaultProducerLogger()} {
+ProducerImpl::ProducerImpl(std::string endpoint, uint8_t n_processing_threads, uint64_t timeout_sec,
+ asapo::RequestHandlerType type):
+ log__{GetDefaultProducerLogger()}, timeout_sec_{timeout_sec} {
switch (type) {
case RequestHandlerType::kTcp:
discovery_service_.reset(new ReceiverDiscoveryService{endpoint, ProducerImpl::kDiscoveryServiceUpdateFrequencyMs});
@@ -91,7 +92,7 @@ Error ProducerImpl::Send(const EventHeader& event_header,
auto request_header = GenerateNextSendRequest(event_header, std::move(substream), ingest_mode);
return request_pool__->AddRequest(std::unique_ptr<ProducerRequest> {new ProducerRequest{source_cred_string_, std::move(request_header),
- std::move(data), std::move(event_header.user_metadata), std::move(full_path), callback, manage_data_memory}
+ std::move(data), std::move(event_header.user_metadata), std::move(full_path), callback, manage_data_memory, timeout_sec_ * 1000}
});
}
@@ -187,7 +188,7 @@ Error ProducerImpl::SendMetaData(const std::string& metadata, RequestCallback ca
FileData data{new uint8_t[metadata.size()]};
strncpy((char*)data.get(), metadata.c_str(), metadata.size());
return request_pool__->AddRequest(std::unique_ptr<ProducerRequest> {new ProducerRequest{source_cred_string_, std::move(request_header),
- std::move(data), "", "", callback, true}
+ std::move(data), "", "", callback, true, timeout_sec_}
});
}
diff --git a/producer/api/cpp/src/producer_impl.h b/producer/api/cpp/src/producer_impl.h
index c54644bdb..371277f97 100644
--- a/producer/api/cpp/src/producer_impl.h
+++ b/producer/api/cpp/src/producer_impl.h
@@ -22,7 +22,8 @@ class ProducerImpl : public Producer {
static const std::string kFinishSubStreamKeyword;
static const std::string kNoNextSubStreamKeyword;
- explicit ProducerImpl(std::string endpoint, uint8_t n_processing_threads, asapo::RequestHandlerType type);
+ explicit ProducerImpl(std::string endpoint, uint8_t n_processing_threads, uint64_t timeout_sec,
+ asapo::RequestHandlerType type);
ProducerImpl(const ProducerImpl&) = delete;
ProducerImpl& operator=(const ProducerImpl&) = delete;
@@ -62,6 +63,7 @@ class ProducerImpl : public Producer {
GenericRequestHeader GenerateNextSendRequest(const EventHeader& event_header, std::string substream,
uint64_t ingest_mode);
std::string source_cred_string_;
+ uint64_t timeout_sec_;
};
}
diff --git a/producer/api/cpp/src/producer_request.cpp b/producer/api/cpp/src/producer_request.cpp
index 53f589e0f..afe6d271e 100644
--- a/producer/api/cpp/src/producer_request.cpp
+++ b/producer/api/cpp/src/producer_request.cpp
@@ -16,7 +16,8 @@ ProducerRequest::ProducerRequest(std::string source_credentials,
std::string metadata,
std::string original_filepath,
RequestCallback callback,
- bool manage_data_memory) : GenericRequest(std::move(h)),
+ bool manage_data_memory,
+ uint64_t timeout_ms) : GenericRequest(std::move(h), timeout_ms),
source_credentials{std::move(source_credentials)},
metadata{std::move(metadata)},
data{std::move(data)},
diff --git a/producer/api/cpp/src/producer_request.h b/producer/api/cpp/src/producer_request.h
index 4dbf6e8d0..1dd9a53f8 100644
--- a/producer/api/cpp/src/producer_request.h
+++ b/producer/api/cpp/src/producer_request.h
@@ -16,7 +16,8 @@ class ProducerRequest : public GenericRequest {
std::string metadata,
std::string original_filepath,
RequestCallback callback,
- bool manage_data_memory);
+ bool manage_data_memory,
+ uint64_t timeout_ms);
std::string source_credentials;
std::string metadata;
FileData data;
diff --git a/producer/api/cpp/src/request_handler_filesystem.cpp b/producer/api/cpp/src/request_handler_filesystem.cpp
index 7c1ff56d4..fd75e003f 100644
--- a/producer/api/cpp/src/request_handler_filesystem.cpp
+++ b/producer/api/cpp/src/request_handler_filesystem.cpp
@@ -28,12 +28,16 @@ bool RequestHandlerFilesystem::ProcessRequestUnlocked(GenericRequest* request) {
}
err = io__->WriteDataToFile(destination_folder_, request->header.message, (uint8_t*)producer_request->data.get(),
- (size_t)request->header.data_size, true);
+ (size_t)request->header.data_size, true, true);
if (producer_request->callback) {
producer_request->callback(request->header, std::move(err));
}
return true;
}
+void RequestHandlerFilesystem::ProcessRequestTimeout(GenericRequest* request) {
+ log__->Error("request timeout, id:" + std::to_string(request->header.data_id) + " to " + request->header.substream +
+ " substream");
+}
}
diff --git a/producer/api/cpp/src/request_handler_filesystem.h b/producer/api/cpp/src/request_handler_filesystem.h
index affdb3fab..72fb21061 100644
--- a/producer/api/cpp/src/request_handler_filesystem.h
+++ b/producer/api/cpp/src/request_handler_filesystem.h
@@ -23,6 +23,7 @@ class RequestHandlerFilesystem: public RequestHandler {
};
void PrepareProcessingRequestLocked() override {};
void TearDownProcessingRequestLocked(bool processing_succeeded) override {};
+ void ProcessRequestTimeout(GenericRequest* request) override;
virtual ~RequestHandlerFilesystem() = default;
std::unique_ptr<IO> io__;
diff --git a/producer/api/cpp/src/request_handler_tcp.cpp b/producer/api/cpp/src/request_handler_tcp.cpp
index 9aab14b4e..a1f25ffb5 100644
--- a/producer/api/cpp/src/request_handler_tcp.cpp
+++ b/producer/api/cpp/src/request_handler_tcp.cpp
@@ -83,9 +83,6 @@ Error RequestHandlerTcp::ReceiveResponse(const GenericRequestHeader& request_hea
return err;
}
switch (sendDataResponse.error_code) {
- case kNetErrorFileIdAlreadyInUse :
- return ProducerErrorTemplates::kWrongInput.Generate("file id already in use: " + std::to_string(
- request_header.data_id));
case kNetAuthorizationError : {
auto res_err = ProducerErrorTemplates::kWrongInput.Generate();
res_err->Append(sendDataResponse.message);
@@ -96,6 +93,11 @@ Error RequestHandlerTcp::ReceiveResponse(const GenericRequestHeader& request_hea
res_err->Append(sendDataResponse.message);
return res_err;
}
+ case kNetErrorWarning: {
+ auto res_err = ProducerErrorTemplates::kServerWarning.Generate();
+ res_err->Append(sendDataResponse.message);
+ return res_err;
+ }
case kNetErrorNoError :
return nullptr;
default:
@@ -112,14 +114,15 @@ Error RequestHandlerTcp::TrySendToReceiver(const ProducerRequest* request) {
}
err = ReceiveResponse(request->header);
- if (err) {
- return err;
+ if (err == nullptr || err == ProducerErrorTemplates::kServerWarning) {
+ log__->Debug("successfully sent data, opcode: " + std::to_string(request->header.op_code) +
+ ", id: " + std::to_string(request->header.data_id) + " to " + connected_receiver_uri_);
+ if (err == ProducerErrorTemplates::kServerWarning ) {
+ log__->Warning("warning from server for id " + std::to_string(request->header.data_id) + ": " + err->Explain());
+ }
}
- log__->Debug("successfully sent data, opcode: " + std::to_string(request->header.op_code) +
- ", id: " + std::to_string(request->header.data_id) + " to " + connected_receiver_uri_);
-
- return nullptr;
+ return err;
}
@@ -177,10 +180,32 @@ void RequestHandlerTcp::Disconnect() {
bool RequestHandlerTcp::ServerError(const Error& err) {
return err != nullptr && (err != ProducerErrorTemplates::kWrongInput &&
- err != ProducerErrorTemplates::kLocalIOError
+ err != ProducerErrorTemplates::kLocalIOError &&
+ err != ProducerErrorTemplates::kServerWarning
);
}
+bool RequestHandlerTcp::ProcessErrorFromReceiver(const Error& error,
+ const ProducerRequest* request,
+ const std::string& receiver_uri) {
+ bool is_server_error = ServerError(error);
+
+ if (error && error != ProducerErrorTemplates::kServerWarning) {
+ Disconnect();
+ std::string log_str = "cannot send data, opcode: " + std::to_string(request->header.op_code) +
+ ", id: " + std::to_string(request->header.data_id) + " to " + receiver_uri + ": " +
+ error->Explain();
+ if (is_server_error) {
+ log__->Warning(log_str + ", will try again");
+ } else {
+ log__->Error(log_str + ", request removed from queue");
+ }
+ }
+
+ return is_server_error;
+}
+
+
bool RequestHandlerTcp::SendDataToOneOfTheReceivers(ProducerRequest* request) {
for (auto receiver_uri : receivers_list_) {
if (Disconnected()) {
@@ -189,13 +214,8 @@ bool RequestHandlerTcp::SendDataToOneOfTheReceivers(ProducerRequest* request) {
}
auto err = TrySendToReceiver(request);
- if (err) {
- Disconnect();
- log__->Warning("cannot send data, opcode: " + std::to_string(request->header.op_code) +
- ", id: " + std::to_string(request->header.data_id) + " to " + receiver_uri + ": " +
- err->Explain());
- }
- if (ServerError(err)) {
+ auto retry = ProcessErrorFromReceiver(err, request, receiver_uri);
+ if (retry) {
continue;
}
@@ -250,4 +270,17 @@ void RequestHandlerTcp::TearDownProcessingRequestLocked(bool processing_succeede
}
}
+void RequestHandlerTcp::ProcessRequestTimeout(GenericRequest* request) {
+ auto producer_request = static_cast<ProducerRequest*>(request);
+
+ log__->Error("request timeout, id:" + std::to_string(request->header.data_id) + " to " + request->header.substream +
+ " substream");
+
+ auto err = ProducerErrorTemplates::kTimeout.Generate();
+ if (producer_request->callback) {
+ producer_request->callback(request->header, std::move(err));
+ }
+
+}
+
}
diff --git a/producer/api/cpp/src/request_handler_tcp.h b/producer/api/cpp/src/request_handler_tcp.h
index 311579b79..0cba33e5d 100644
--- a/producer/api/cpp/src/request_handler_tcp.h
+++ b/producer/api/cpp/src/request_handler_tcp.h
@@ -23,6 +23,7 @@ class RequestHandlerTcp: public RequestHandler {
bool ReadyProcessRequest() override;
void PrepareProcessingRequestLocked() override;
void TearDownProcessingRequestLocked(bool processing_succeeded) override;
+ void ProcessRequestTimeout(GenericRequest* request) override;
virtual ~RequestHandlerTcp() = default;
std::unique_ptr<IO> io__;
@@ -44,10 +45,12 @@ class RequestHandlerTcp: public RequestHandler {
bool Disconnected();
void Disconnect();
bool ServerError(const Error& err);
- ReceiversList receivers_list_;
- system_clock::time_point last_receivers_uri_update_;
bool Connected();
bool CanCreateNewConnections();
+ bool ProcessErrorFromReceiver(const Error& error, const ProducerRequest* request, const std::string& receiver_uri);
+ ReceiversList receivers_list_;
+ system_clock::time_point last_receivers_uri_update_;
+
uint64_t thread_id_;
uint64_t* ncurrent_connections_;
std::string connected_receiver_uri_;
diff --git a/producer/api/cpp/unittests/test_producer.cpp b/producer/api/cpp/unittests/test_producer.cpp
index 4788882c0..807bf22be 100644
--- a/producer/api/cpp/unittests/test_producer.cpp
+++ b/producer/api/cpp/unittests/test_producer.cpp
@@ -15,7 +15,7 @@ namespace {
TEST(CreateProducer, TcpProducer) {
asapo::Error err;
std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("endpoint", 4, asapo::RequestHandlerType::kTcp,
- SourceCredentials{"bt", "", ""}, &err);
+ SourceCredentials{"bt", "", ""}, 3600, &err);
ASSERT_THAT(dynamic_cast<asapo::ProducerImpl*>(producer.get()), Ne(nullptr));
ASSERT_THAT(err, Eq(nullptr));
}
@@ -24,7 +24,7 @@ TEST(CreateProducer, ErrorBeamtime) {
asapo::Error err;
std::string expected_beamtimeid(asapo::kMaxMessageSize * 10, 'a');
std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("endpoint", 4, asapo::RequestHandlerType::kTcp,
- SourceCredentials{expected_beamtimeid, "", ""}, &err);
+ SourceCredentials{expected_beamtimeid, "", ""}, 3600, &err);
ASSERT_THAT(producer, Eq(nullptr));
ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput));
}
@@ -44,7 +44,7 @@ TEST(CreateProducer, FileSystemProducer) {
TEST(CreateProducer, TooManyThreads) {
asapo::Error err;
std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("", asapo::kMaxProcessingThreads + 1,
- asapo::RequestHandlerType::kTcp, SourceCredentials{"bt", "", ""}, &err);
+ asapo::RequestHandlerType::kTcp, SourceCredentials{"bt", "", ""}, 3600, &err);
ASSERT_THAT(producer, Eq(nullptr));
ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput));
}
@@ -53,7 +53,7 @@ TEST(CreateProducer, TooManyThreads) {
TEST(CreateProducer, ZeroThreads) {
asapo::Error err;
std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("", 0,
- asapo::RequestHandlerType::kTcp, SourceCredentials{"bt", "", ""}, &err);
+ asapo::RequestHandlerType::kTcp, SourceCredentials{"bt", "", ""}, 3600, &err);
ASSERT_THAT(producer, Eq(nullptr));
ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput));
}
@@ -62,7 +62,7 @@ TEST(CreateProducer, ZeroThreads) {
TEST(Producer, SimpleWorkflowWihoutConnection) {
asapo::Error err;
std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("hello", 5, asapo::RequestHandlerType::kTcp,
- SourceCredentials{"bt", "", ""},
+ SourceCredentials{"bt", "", ""}, 3600,
&err);
asapo::EventHeader event_header{1, 1, "test"};
diff --git a/producer/api/cpp/unittests/test_producer_impl.cpp b/producer/api/cpp/unittests/test_producer_impl.cpp
index 16caa956b..aadeeca48 100644
--- a/producer/api/cpp/unittests/test_producer_impl.cpp
+++ b/producer/api/cpp/unittests/test_producer_impl.cpp
@@ -50,7 +50,7 @@ MATCHER_P10(M_CheckSendDataRequest, op_code, source_credentials, metadata, file_
}
TEST(ProducerImpl, Constructor) {
- asapo::ProducerImpl producer{"", 4, asapo::RequestHandlerType::kTcp};
+ asapo::ProducerImpl producer{"", 4, 3600, asapo::RequestHandlerType::kTcp};
ASSERT_THAT(dynamic_cast<asapo::AbstractLogger*>(producer.log__), Ne(nullptr));
ASSERT_THAT(dynamic_cast<asapo::RequestPool*>(producer.request_pool__.get()), Ne(nullptr));
}
@@ -61,7 +61,7 @@ class ProducerImplTests : public testing::Test {
asapo::ProducerRequestHandlerFactory factory{&service};
testing::NiceMock<asapo::MockLogger> mock_logger;
testing::NiceMock<MockRequestPull> mock_pull{&factory, &mock_logger};
- asapo::ProducerImpl producer{"", 1, asapo::RequestHandlerType::kTcp};
+ asapo::ProducerImpl producer{"", 1, 3600, asapo::RequestHandlerType::kTcp};
uint64_t expected_size = 100;
uint64_t expected_id = 10;
uint64_t expected_subset_id = 100;
diff --git a/producer/api/cpp/unittests/test_producer_request.cpp b/producer/api/cpp/unittests/test_producer_request.cpp
index 2c9323002..eaf1de845 100644
--- a/producer/api/cpp/unittests/test_producer_request.cpp
+++ b/producer/api/cpp/unittests/test_producer_request.cpp
@@ -44,7 +44,7 @@ TEST(ProducerRequest, Constructor) {
asapo::GenericRequestHeader header{expected_op_code, expected_file_id, expected_file_size,
expected_meta_size, expected_file_name};
- asapo::ProducerRequest request{expected_source_credentials, std::move(header), nullptr, expected_meta, "", nullptr, true};
+ asapo::ProducerRequest request{expected_source_credentials, std::move(header), nullptr, expected_meta, "", nullptr, true, 0};
ASSERT_THAT(request.source_credentials, Eq(expected_source_credentials));
ASSERT_THAT(request.metadata, Eq(expected_meta));
@@ -62,7 +62,7 @@ TEST(ProducerRequest, Destructor) {
char data_[100];
asapo::FileData data{(uint8_t*)data_};
asapo::GenericRequestHeader header{asapo::kOpcodeTransferData, 1, 1, 1, ""};
- asapo::ProducerRequest* request = new asapo::ProducerRequest{"", std::move(header), std::move(data), "", "", nullptr, false};
+ asapo::ProducerRequest* request = new asapo::ProducerRequest{"", std::move(header), std::move(data), "", "", nullptr, false, 0};
delete request;
diff --git a/producer/api/cpp/unittests/test_request_handler_filesystem.cpp b/producer/api/cpp/unittests/test_request_handler_filesystem.cpp
index e36fbbc8e..4e443b971 100644
--- a/producer/api/cpp/unittests/test_request_handler_filesystem.cpp
+++ b/producer/api/cpp/unittests/test_request_handler_filesystem.cpp
@@ -61,10 +61,10 @@ class RequestHandlerFilesystemTests : public testing::Test {
called = true;
callback_err = std::move(err);
callback_header = header;
- }, true};
+ }, true, 0};
- asapo::ProducerRequest request_nocallback{"", header, nullptr, "", "", nullptr, true};
- asapo::ProducerRequest request_filesend{"", header, nullptr, "", expected_origin_fullpath, nullptr, true};
+ asapo::ProducerRequest request_nocallback{"", header, nullptr, "", "", nullptr, true, 0};
+ asapo::ProducerRequest request_filesend{"", header, nullptr, "", expected_origin_fullpath, nullptr, true, 0};
testing::NiceMock<asapo::MockLogger> mock_logger;
@@ -97,7 +97,7 @@ MATCHER_P2(M_CheckSendDataRequest, file_id, file_size,
TEST_F(RequestHandlerFilesystemTests, CallBackErrorIfCannotSaveFile) {
EXPECT_CALL(mock_io, WriteDataToFile_t(expected_destination, expected_file_name, nullptr, (size_t)expected_file_size,
- true))
+ true, true))
.WillOnce(
Return(
asapo::IOErrorTemplates::kUnknownIOError.Generate().release())
@@ -113,7 +113,7 @@ TEST_F(RequestHandlerFilesystemTests, CallBackErrorIfCannotSaveFile) {
TEST_F(RequestHandlerFilesystemTests, WorksWithemptyCallback) {
EXPECT_CALL(mock_io, WriteDataToFile_t(expected_destination, expected_file_name, nullptr, (size_t) expected_file_size,
- true))
+ true, true))
.WillOnce(
Return(nullptr)
);
@@ -149,7 +149,7 @@ TEST_F(RequestHandlerFilesystemTests, FileRequestOK) {
));
EXPECT_CALL(mock_io, WriteDataToFile_t(expected_destination, expected_file_name, nullptr, (size_t)expected_file_size,
- true))
+ true, true))
.WillOnce(
Return(nullptr)
);
@@ -162,7 +162,7 @@ TEST_F(RequestHandlerFilesystemTests, FileRequestOK) {
TEST_F(RequestHandlerFilesystemTests, TransferOK) {
EXPECT_CALL(mock_io, WriteDataToFile_t(expected_destination, expected_file_name, nullptr, (size_t) expected_file_size,
- true))
+ true, true))
.WillOnce(
Return(
nullptr)
diff --git a/producer/api/cpp/unittests/test_request_handler_tcp.cpp b/producer/api/cpp/unittests/test_request_handler_tcp.cpp
index d086a932f..ab5cebbc6 100644
--- a/producer/api/cpp/unittests/test_request_handler_tcp.cpp
+++ b/producer/api/cpp/unittests/test_request_handler_tcp.cpp
@@ -56,7 +56,7 @@ class RequestHandlerTcpTests : public testing::Test {
uint64_t expected_file_size = 1337;
uint64_t expected_meta_size = 4;
std::string expected_metadata = "meta";
-
+ std::string expected_warning = "warning";
char expected_file_name[asapo::kMaxMessageSize] = "test_name";
char expected_beamtime_id[asapo::kMaxMessageSize] = "test_beamtime_id";
char expected_substream[asapo::kMaxMessageSize] = "test_substream";
@@ -76,7 +76,7 @@ class RequestHandlerTcpTests : public testing::Test {
callback_called = true;
callback_err = std::move(err);
callback_header = header;
- }, true};
+ }, true, 0};
std::string expected_origin_fullpath = std::string("origin/") + expected_file_name;
asapo::ProducerRequest request_filesend{expected_beamtime_id, header_fromfile, nullptr, expected_metadata,
@@ -84,10 +84,10 @@ class RequestHandlerTcpTests : public testing::Test {
callback_called = true;
callback_err = std::move(err);
callback_header = header;
- }, true};
+ }, true, 0};
- asapo::ProducerRequest request_nocallback{expected_beamtime_id, header, nullptr, expected_metadata, "", nullptr, true};
+ asapo::ProducerRequest request_nocallback{expected_beamtime_id, header, nullptr, expected_metadata, "", nullptr, true, 0};
testing::NiceMock<asapo::MockLogger> mock_logger;
uint64_t n_connections{0};
asapo::RequestHandlerTcp request_handler{&mock_discovery_service, expected_thread_id, &n_connections};
@@ -116,10 +116,11 @@ class RequestHandlerTcpTests : public testing::Test {
void ExpectGetFileSize(bool ok);
void ExpectOKSendData(bool only_once = false);
void ExpectOKSendFile(bool only_once = false);
- void ExpectFailSendFile(const asapo::ProducerErrorTemplate& err_template, bool only_once = false);
+ void ExpectFailSendFile(const asapo::ProducerErrorTemplate& err_template, bool client_error = false);
void ExpectOKSendMetaData(bool only_once = false);
void ExpectFailReceive(bool only_once = false);
- void ExpectOKReceive(bool only_once = true);
+ void ExpectOKReceive(bool only_once = true, asapo::NetworkErrorCode code = asapo::kNetErrorNoError,
+ std::string message = "");
void DoSingleSend(bool connect = true, bool success = true);
void AssertImmediatelyCallBack(asapo::NetworkErrorCode error_code, const asapo::ProducerErrorTemplate& err_template);
void SetUp() override {
@@ -137,10 +138,10 @@ class RequestHandlerTcpTests : public testing::Test {
}
};
-ACTION_P(A_WriteSendDataResponse, error_code) {
+ACTION_P2(A_WriteSendDataResponse, error_code, message) {
((asapo::SendDataResponse*)arg1)->op_code = asapo::kOpcodeTransferData;
((asapo::SendDataResponse*)arg1)->error_code = error_code;
- strcpy(((asapo::SendDataResponse*)arg1)->message, expected_auth_message.c_str());
+ strcpy(((asapo::SendDataResponse*)arg1)->message, message.c_str());
}
MATCHER_P5(M_CheckSendDataRequest, op_code, file_id, file_size, message, substream,
@@ -191,7 +192,7 @@ void RequestHandlerTcpTests::ExpectFailAuthorize(bool only_once) {
.WillOnce(
DoAll(
testing::SetArgPointee<3>(nullptr),
- A_WriteSendDataResponse(asapo::kNetAuthorizationError),
+ A_WriteSendDataResponse(asapo::kNetAuthorizationError, expected_auth_message),
testing::ReturnArg<2>()
));
EXPECT_CALL(mock_io, CloseSocket_t(expected_sd, _));
@@ -230,7 +231,7 @@ void RequestHandlerTcpTests::ExpectOKAuthorize(bool only_once) {
.WillOnce(
DoAll(
testing::SetArgPointee<3>(nullptr),
- A_WriteSendDataResponse(asapo::kNetErrorNoError),
+ A_WriteSendDataResponse(asapo::kNetErrorNoError, expected_auth_message),
testing::ReturnArg<2>()
));
EXPECT_CALL(mock_logger, Info(AllOf(
@@ -275,7 +276,7 @@ void RequestHandlerTcpTests::ExpectFailSendHeader(bool only_once) {
EXPECT_CALL(mock_logger, Warning(HasSubstr("put back")));
}
-void RequestHandlerTcpTests::ExpectFailSendFile(const asapo::ProducerErrorTemplate& err_template, bool only_once) {
+void RequestHandlerTcpTests::ExpectFailSendFile(const asapo::ProducerErrorTemplate& err_template, bool client_error) {
int i = 0;
for (auto expected_sd : expected_sds) {
EXPECT_CALL(mock_io, SendFile_t(expected_sd, expected_origin_fullpath, (size_t) expected_file_size))
@@ -288,13 +289,19 @@ void RequestHandlerTcpTests::ExpectFailSendFile(const asapo::ProducerErrorTempla
HasSubstr(receivers_list[i])
)
));
-
- EXPECT_CALL(mock_logger, Warning(AllOf(
- HasSubstr("cannot send"),
- HasSubstr(receivers_list[i]) )
- ));
+ if (client_error) {
+ EXPECT_CALL(mock_logger, Error(AllOf(
+ HasSubstr("cannot send"),
+ HasSubstr(receivers_list[i]) )
+ ));
+ } else {
+ EXPECT_CALL(mock_logger, Warning(AllOf(
+ HasSubstr("cannot send"),
+ HasSubstr(receivers_list[i]) )
+ ));
+ }
EXPECT_CALL(mock_io, CloseSocket_t(expected_sd, _));
- if (only_once) break;
+ if (client_error) break;
i++;
}
if (err_template != asapo::ProducerErrorTemplates::kLocalIOError.Generate()) {
@@ -448,7 +455,7 @@ void RequestHandlerTcpTests::ExpectOKConnect(bool only_once) {
}
-void RequestHandlerTcpTests::ExpectOKReceive(bool only_once) {
+void RequestHandlerTcpTests::ExpectOKReceive(bool only_once, asapo::NetworkErrorCode code, std::string message) {
int i = 0;
for (auto expected_sd : expected_sds) {
EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendDataResponse), _))
@@ -456,7 +463,7 @@ void RequestHandlerTcpTests::ExpectOKReceive(bool only_once) {
.WillOnce(
DoAll(
testing::SetArgPointee<3>(nullptr),
- A_WriteSendDataResponse(asapo::kNetErrorNoError),
+ A_WriteSendDataResponse(code, message),
testing::ReturnArg<2>()
));
EXPECT_CALL(mock_logger, Debug(AllOf(
@@ -673,7 +680,7 @@ void RequestHandlerTcpTests::AssertImmediatelyCallBack(asapo::NetworkErrorCode e
.WillOnce(
DoAll(
testing::SetArgPointee<3>(nullptr),
- A_WriteSendDataResponse(error_code),
+ A_WriteSendDataResponse(error_code, expected_auth_message),
testing::ReturnArg<2>()
));
EXPECT_CALL(mock_logger, Debug(AllOf(
@@ -682,11 +689,11 @@ void RequestHandlerTcpTests::AssertImmediatelyCallBack(asapo::NetworkErrorCode e
)
));
- EXPECT_CALL(mock_logger, Warning(AllOf(
- HasSubstr("cannot send"),
- HasSubstr(receivers_list[0])
- )
- ));
+ EXPECT_CALL(mock_logger, Error(AllOf(
+ HasSubstr("cannot send"),
+ HasSubstr(receivers_list[0])
+ )
+ ));
request_handler.PrepareProcessingRequestLocked();
auto success = request_handler.ProcessRequestUnlocked(&request);
@@ -712,14 +719,12 @@ TEST_F(RequestHandlerTcpTests, ImmediatelyCallBackErrorIfAuthorizationFailure) {
AssertImmediatelyCallBack(asapo::kNetAuthorizationError, asapo::ProducerErrorTemplates::kWrongInput);
}
-TEST_F(RequestHandlerTcpTests, ImmediatelyCallBackErrorIfFileAlreadyInUse) {
- AssertImmediatelyCallBack(asapo::kNetErrorFileIdAlreadyInUse, asapo::ProducerErrorTemplates::kWrongInput);
-}
TEST_F(RequestHandlerTcpTests, ImmediatelyCallBackErrorIfWrongMetadata) {
AssertImmediatelyCallBack(asapo::kNetErrorWrongRequest, asapo::ProducerErrorTemplates::kWrongInput);
}
+
TEST_F(RequestHandlerTcpTests, SendEmptyCallBack) {
ExpectOKConnect(true);
ExpectOKAuthorize(true);
@@ -870,5 +875,45 @@ TEST_F(RequestHandlerTcpTests, SendMetaOnlyForFileReadOK) {
}
+TEST_F(RequestHandlerTcpTests, TimeoutCallsCallback) {
+ EXPECT_CALL(mock_logger, Error(AllOf(
+ HasSubstr("timeout"),
+ HasSubstr("substream"))
+ ));
+
+ request_handler.ProcessRequestTimeout(&request);
+
+ ASSERT_THAT(callback_err, Eq(asapo::ProducerErrorTemplates::kTimeout));
+ ASSERT_THAT(callback_called, Eq(true));
+}
+
+
+
+TEST_F(RequestHandlerTcpTests, SendWithWarning) {
+ ExpectOKConnect(true);
+ ExpectOKAuthorize(true);
+ ExpectOKSendAll(true);
+ ExpectOKReceive(true, asapo::kNetErrorWarning, expected_warning);
+
+ EXPECT_CALL(mock_logger, Warning(AllOf(
+ HasSubstr("server"),
+ HasSubstr(expected_warning))
+ ));
+
+
+ request_handler.PrepareProcessingRequestLocked();
+ auto success = request_handler.ProcessRequestUnlocked(&request);
+
+ ASSERT_THAT(success, Eq(true));
+ ASSERT_THAT(callback_err, Eq(asapo::ProducerErrorTemplates::kServerWarning));
+ ASSERT_THAT(callback_err->Explain(), HasSubstr(expected_warning));
+ ASSERT_THAT(callback_called, Eq(true));
+ ASSERT_THAT(callback_header.data_size, Eq(header.data_size));
+ ASSERT_THAT(callback_header.op_code, Eq(header.op_code));
+ ASSERT_THAT(callback_header.data_id, Eq(header.data_id));
+ ASSERT_THAT(std::string{callback_header.message}, Eq(std::string{header.message}));
+}
+
+
}
diff --git a/producer/api/python/asapo_producer.pxd b/producer/api/python/asapo_producer.pxd
index 4ce8fda3b..22fdedcba 100644
--- a/producer/api/python/asapo_producer.pxd
+++ b/producer/api/python/asapo_producer.pxd
@@ -21,6 +21,7 @@ cdef extern from "asapo_producer.h" namespace "asapo":
ErrorTemplateInterface kTimeout "asapo::ProducerErrorTemplates::kTimeout"
ErrorTemplateInterface kWrongInput "asapo::ProducerErrorTemplates::kWrongInput"
ErrorTemplateInterface kLocalIOError "asapo::ProducerErrorTemplates::kLocalIOError"
+ ErrorTemplateInterface kServerWarning "asapo::ProducerErrorTemplates::kServerWarning"
cdef extern from "asapo_producer.h" namespace "asapo":
cppclass FileData:
@@ -88,7 +89,7 @@ cdef extern from "asapo_wrappers.h" namespace "asapo":
cdef extern from "asapo_producer.h" namespace "asapo" nogil:
cppclass Producer:
@staticmethod
- unique_ptr[Producer] Create(string endpoint,uint8_t nthreads,RequestHandlerType type, SourceCredentials source,Error* error)
+ unique_ptr[Producer] Create(string endpoint,uint8_t nthreads,RequestHandlerType type, SourceCredentials source,uint64_t timeout_sec, Error* error)
Error SendFile(const EventHeader& event_header, string substream, string full_path, uint64_t ingest_mode,RequestCallback callback)
Error SendData__(const EventHeader& event_header, string substream, void* data, uint64_t ingest_mode,RequestCallback callback)
void StopThreads__()
diff --git a/producer/api/python/asapo_producer.pyx.in b/producer/api/python/asapo_producer.pyx.in
index 48c5fa77a..e606d7d89 100644
--- a/producer/api/python/asapo_producer.pyx.in
+++ b/producer/api/python/asapo_producer.pyx.in
@@ -47,6 +47,10 @@ class AsapoLocalIOError(AsapoProducerError):
class AsapoTimeOutError(AsapoProducerError):
pass
+class AsapoServerWarning(AsapoProducerError):
+ pass
+
+
cdef python_exception_from_error(Error& err):
error_string = _str(err.get().Explain())
if err == kTimeout:
@@ -55,6 +59,8 @@ cdef python_exception_from_error(Error& err):
return AsapoWrongInputError(error_string)
elif err == kLocalIOError:
return AsapoLocalIOError(error_string)
+ elif err == kServerWarning:
+ return AsapoServerWarning(error_string)
else:
return AsapoProducerError(error_string)
@@ -272,19 +278,19 @@ cdef class PyProducer:
if self.c_producer.get() is not NULL:
self.c_producer.get().StopThreads__()
@staticmethod
- def __create_producer(endpoint,beamtime_id,stream,token,nthreads):
+ def __create_producer(endpoint,beamtime_id,stream,token,nthreads,timeout_sec):
pyProd = PyProducer()
cdef Error err
cdef SourceCredentials source
source.beamtime_id = beamtime_id
source.user_token = token
source.stream = stream
- pyProd.c_producer = Producer.Create(endpoint,nthreads,RequestHandlerType_Tcp,source,&err)
+ pyProd.c_producer = Producer.Create(endpoint,nthreads,RequestHandlerType_Tcp,source,timeout_sec,&err)
if err:
throw_exception(err)
return pyProd
-def create_producer(endpoint,beamtime_id,stream,token,nthreads):
+def create_producer(endpoint,beamtime_id,stream,token,nthreads,timeout_sec):
"""
:param endpoint: server endpoint (url:port)
:type endpoint: string
@@ -296,11 +302,13 @@ def create_producer(endpoint,beamtime_id,stream,token,nthreads):
:type token: string
:param nthreads: ingest mode flag
:type nthreads: int
+ :param timeout_sec: send requests timeout
+ :type timeout_sec: int
:raises:
AsapoWrongInputError: wrong input (number of threads, ,,,)
AsapoProducerError: actually should not happen
"""
- return PyProducer.__create_producer(_bytes(endpoint),_bytes(beamtime_id),_bytes(stream),_bytes(token),nthreads)
+ return PyProducer.__create_producer(_bytes(endpoint),_bytes(beamtime_id),_bytes(stream),_bytes(token),nthreads,timeout_sec)
__version__ = "@ASAPO_VERSION_PYTHON@"
diff --git a/producer/event_monitor_producer/src/main_eventmon.cpp b/producer/event_monitor_producer/src/main_eventmon.cpp
index e277e7b11..cd5899c3c 100644
--- a/producer/event_monitor_producer/src/main_eventmon.cpp
+++ b/producer/event_monitor_producer/src/main_eventmon.cpp
@@ -39,7 +39,7 @@ std::unique_ptr<Producer> CreateProducer() {
Error err;
auto producer = Producer::Create(config->asapo_endpoint, (uint8_t) config->nthreads,
- config->mode, asapo::SourceCredentials{config->beamtime_id, config->stream, ""}, &err);
+ config->mode, asapo::SourceCredentials{config->beamtime_id, config->stream, ""}, 3600, &err);
if(err) {
std::cerr << "cannot create producer: " << err << std::endl;
exit(EXIT_FAILURE);
diff --git a/receiver/CMakeLists.txt b/receiver/CMakeLists.txt
index c991bcfa8..182559e7c 100644
--- a/receiver/CMakeLists.txt
+++ b/receiver/CMakeLists.txt
@@ -3,7 +3,7 @@ set(SOURCE_FILES
src/receiver.cpp
src/connection.cpp
src/request.cpp
- src/request_handler_file_write.cpp
+ src/request_handler_file_process.cpp
src/statistics.cpp
src/statistics_sender_influx_db.cpp
src/receiver_config.cpp
@@ -23,9 +23,12 @@ set(SOURCE_FILES
src/receiver_statistics.cpp
src/request_handler_db_meta_write.cpp
src/request_handler_receive_metadata.cpp
- src/request_handler_file_receive.cpp
+ src/request_handler_db_check_request.cpp
src/request_factory.cpp
- src/request_handler_db.cpp)
+ src/write_file_processor.cpp
+ src/request_handler_db.cpp
+ src/file_processor.cpp
+ src/receive_file_processor.cpp)
################################
@@ -66,9 +69,9 @@ set(TEST_SOURCE_FILES
unittests/test_config.cpp
unittests/test_request.cpp
unittests/test_request_factory.cpp
- unittests/test_request_handler_file_write.cpp
- unittests/test_request_handler_file_receive.cpp
+ unittests/test_request_handler_file_process.cpp
unittests/test_request_handler_db_writer.cpp
+ unittests/test_request_handler_db_check_request.cpp
unittests/test_request_handler_db_meta_writer.cpp
unittests/test_request_handler_db.cpp
unittests/test_request_handler_authorizer.cpp
@@ -79,6 +82,8 @@ set(TEST_SOURCE_FILES
unittests/mock_receiver_config.cpp
unittests/test_requests_dispatcher.cpp
unittests/test_datacache.cpp
+ unittests/test_write_file_processor.cpp
+ unittests/test_receive_file_processor.cpp
)
#
set(TEST_LIBRARIES "${TARGET_NAME};system_io")
diff --git a/receiver/src/file_processor.cpp b/receiver/src/file_processor.cpp
new file mode 100644
index 000000000..e8425048e
--- /dev/null
+++ b/receiver/src/file_processor.cpp
@@ -0,0 +1,12 @@
+#include "file_processor.h"
+
+#include "io/io_factory.h"
+#include "receiver_logger.h"
+
+namespace asapo {
+
+FileProcessor::FileProcessor(): io__{GenerateDefaultIO()}, log__{GetDefaultReceiverLogger()} {
+
+}
+
+}
\ No newline at end of file
diff --git a/receiver/src/file_processor.h b/receiver/src/file_processor.h
new file mode 100644
index 000000000..b57ca733c
--- /dev/null
+++ b/receiver/src/file_processor.h
@@ -0,0 +1,22 @@
+#ifndef ASAPO_FILE_PROCESSOR_H
+#define ASAPO_FILE_PROCESSOR_H
+
+#include "io/io.h"
+#include "logger/logger.h"
+
+namespace asapo {
+
+class Request;
+
+class FileProcessor {
+ public:
+ FileProcessor();
+ virtual ~FileProcessor() = default;
+ virtual Error ProcessFile(const Request* request, bool overwrite) const = 0;
+ std::unique_ptr<IO> io__;
+ const AbstractLogger* log__;
+};
+
+}
+
+#endif //ASAPO_FILE_PROCESSOR_H
diff --git a/receiver/src/request_handler_file_receive.cpp b/receiver/src/receive_file_processor.cpp
similarity index 58%
rename from receiver/src/request_handler_file_receive.cpp
rename to receiver/src/receive_file_processor.cpp
index 500f6a9cd..512ca1438 100644
--- a/receiver/src/request_handler_file_receive.cpp
+++ b/receiver/src/receive_file_processor.cpp
@@ -1,31 +1,27 @@
-#include "request_handler_file_receive.h"
+#include "receive_file_processor.h"
+
#include "io/io_factory.h"
+#include "receiver_error.h"
+#include "preprocessor/definitions.h"
#include "request.h"
-#include "receiver_logger.h"
#include "receiver_config.h"
-#include "preprocessor/definitions.h"
namespace asapo {
-Error RequestHandlerFileReceive::ProcessRequest(Request* request) const {
+ReceiveFileProcessor::ReceiveFileProcessor() : FileProcessor() {
+
+}
+
+Error ReceiveFileProcessor::ProcessFile(const Request* request, bool overwrite) const {
auto fsize = request->GetDataSize();
auto socket = request->GetSocket();
auto fname = request->GetFileName();
auto root_folder = request->GetFullPath(GetReceiverConfig()->root_folder);
- auto err = io__->ReceiveDataToFile(socket, root_folder, fname, (size_t) fsize, true);
+ auto err = io__->ReceiveDataToFile(socket, root_folder, fname, (size_t) fsize, true, overwrite);
if (!err) {
log__->Debug("received file of size " + std::to_string(fsize) + " to " + root_folder + kPathSeparator + fname);
}
return err;
}
-RequestHandlerFileReceive::RequestHandlerFileReceive() : io__{GenerateDefaultIO()} , log__{GetDefaultReceiverLogger()} {
-
-}
-
-StatisticEntity RequestHandlerFileReceive::GetStatisticEntity() const {
- return StatisticEntity::kDisk;
-}
-
-
-}
+}
\ No newline at end of file
diff --git a/receiver/src/receive_file_processor.h b/receiver/src/receive_file_processor.h
new file mode 100644
index 000000000..9941e1795
--- /dev/null
+++ b/receiver/src/receive_file_processor.h
@@ -0,0 +1,17 @@
+#ifndef ASAPO_RECEIVE_FILE_PROCESSOR_H
+#define ASAPO_RECEIVE_FILE_PROCESSOR_H
+
+#include "file_processor.h"
+
+namespace asapo {
+
+class ReceiveFileProcessor final : public FileProcessor {
+ public:
+ ReceiveFileProcessor();
+ Error ProcessFile(const Request* request, bool overwrite) const override;
+};
+
+
+}
+
+#endif //ASAPO_RECEIVE_FILE_PROCESSOR_H
diff --git a/receiver/src/receiver_data_server/receiver_data_server_request.cpp b/receiver/src/receiver_data_server/receiver_data_server_request.cpp
index f0e5a5dea..6c30cff9e 100644
--- a/receiver/src/receiver_data_server/receiver_data_server_request.cpp
+++ b/receiver/src/receiver_data_server/receiver_data_server_request.cpp
@@ -4,7 +4,7 @@
namespace asapo {
ReceiverDataServerRequest::ReceiverDataServerRequest(GenericRequestHeader header, uint64_t source_id) :
- GenericRequest(std::move(header)),
+ GenericRequest(std::move(header), 0),
source_id{source_id} {
}
diff --git a/receiver/src/receiver_data_server/receiver_data_server_request_handler.cpp b/receiver/src/receiver_data_server/receiver_data_server_request_handler.cpp
index 1254981a8..f3d1672a8 100644
--- a/receiver/src/receiver_data_server/receiver_data_server_request_handler.cpp
+++ b/receiver/src/receiver_data_server/receiver_data_server_request_handler.cpp
@@ -86,4 +86,8 @@ Error ReceiverDataServerRequestHandler::SendResponce(const ReceiverDataServerReq
return server_->SendData(request->source_id, &responce, sizeof(GenericNetworkResponse));
}
+void ReceiverDataServerRequestHandler::ProcessRequestTimeout(GenericRequest* request) {
+// do nothing
+}
+
}
\ No newline at end of file
diff --git a/receiver/src/receiver_data_server/receiver_data_server_request_handler.h b/receiver/src/receiver_data_server/receiver_data_server_request_handler.h
index e5aabd418..0ca1a46b5 100644
--- a/receiver/src/receiver_data_server/receiver_data_server_request_handler.h
+++ b/receiver/src/receiver_data_server/receiver_data_server_request_handler.h
@@ -17,6 +17,8 @@ class ReceiverDataServerRequestHandler: public RequestHandler {
bool ReadyProcessRequest() override;
void PrepareProcessingRequestLocked() override;
void TearDownProcessingRequestLocked(bool processing_succeeded) override;
+ void ProcessRequestTimeout(GenericRequest* request) override;
+
const AbstractLogger* log__;
Statistics* statistics__;
private:
diff --git a/receiver/src/receiver_error.h b/receiver/src/receiver_error.h
index 0704a69bc..a25c9e78f 100644
--- a/receiver/src/receiver_error.h
+++ b/receiver/src/receiver_error.h
@@ -8,24 +8,25 @@ namespace asapo {
enum class ReceiverErrorType {
kInvalidOpCode,
kBadRequest,
- kReject,
kAuthorizationFailure,
kInternalServerError,
+ kWarningDuplicatedRequest
};
using ReceiverErrorTemplate = ServiceErrorTemplate<ReceiverErrorType, ErrorType::kReceiverError>;
namespace ReceiverErrorTemplates {
-auto const kInvalidOpCode = ReceiverErrorTemplate{
- "Invalid Opcode", ReceiverErrorType::kInvalidOpCode
-};
-auto const kReject = ReceiverErrorTemplate{
- "request rejected", ReceiverErrorType::kReject
+auto const kWarningDuplicatedRequest = ReceiverErrorTemplate{
+ "Duplicated request, possible due to retry", ReceiverErrorType::kWarningDuplicatedRequest
};
+auto const kInvalidOpCode = ReceiverErrorTemplate{
+ "Invalid Opcode", ReceiverErrorType::kInvalidOpCode
+};
+
auto const kInternalServerError = ReceiverErrorTemplate{
"server error", ReceiverErrorType::kInternalServerError
};
diff --git a/receiver/src/request.cpp b/receiver/src/request.cpp
index 2a5cec65f..c34a9f421 100644
--- a/receiver/src/request.cpp
+++ b/receiver/src/request.cpp
@@ -5,9 +5,11 @@
namespace asapo {
Request::Request(const GenericRequestHeader& header,
- SocketDescriptor socket_fd, std::string origin_uri, DataCache* cache) : io__{GenerateDefaultIO()},
+ SocketDescriptor socket_fd, std::string origin_uri, DataCache* cache,
+ const RequestHandlerDbCheckRequest* db_check_handler) : io__{GenerateDefaultIO()},
cache__{cache}, request_header_(header),
- socket_fd_{socket_fd}, origin_uri_{std::move(origin_uri)} {
+ socket_fd_{socket_fd}, origin_uri_{std::move(origin_uri)},
+ check_duplicate_request_handler_{db_check_handler} {
}
Error Request::PrepareDataBufferAndLockIfNeeded() {
@@ -177,4 +179,24 @@ std::string Request::GetFullPath(std::string root_folder) const {
+ GetBeamtimeId();
}
+bool Request::WasAlreadyProcessed() const {
+ return already_processed_;
+}
+
+void Request::SetAlreadyProcessedFlag() {
+ already_processed_ = true;
+}
+
+void Request::SetWarningMessage(std::string message) {
+ warning_message_ = std::move(message);
+}
+
+const std::string& Request::GetWarningMessage() const {
+ return warning_message_;
+}
+
+Error Request::CheckForDuplicates() {
+ return check_duplicate_request_handler_->ProcessRequest(this);
+}
+
}
\ No newline at end of file
diff --git a/receiver/src/request.h b/receiver/src/request.h
index 7af37f61b..11eb6ca2a 100644
--- a/receiver/src/request.h
+++ b/receiver/src/request.h
@@ -7,13 +7,13 @@
#include "common/networking.h"
#include "io/io.h"
#include "request_handler.h"
-#include "request_handler_file_write.h"
+#include "request_handler_file_process.h"
#include "request_handler_db_write.h"
#include "request_handler_authorize.h"
#include "request_handler_db_meta_write.h"
#include "request_handler_receive_data.h"
#include "request_handler_receive_metadata.h"
-#include "request_handler_file_receive.h"
+#include "request_handler_db_check_request.h"
#include "receiver_statistics.h"
#include "data_cache.h"
@@ -27,8 +27,9 @@ class Request {
public:
VIRTUAL Error Handle(ReceiverStatistics*);
~Request() = default;
+ Request() = delete;
Request(const GenericRequestHeader& request_header, SocketDescriptor socket_fd, std::string origin_uri,
- DataCache* cache);
+ DataCache* cache, const RequestHandlerDbCheckRequest* db_check_handler);
VIRTUAL void AddHandler(const ReceiverRequestHandler*);
VIRTUAL const RequestHandlerList& GetListHandlers() const;
VIRTUAL uint64_t GetDataSize() const;
@@ -66,6 +67,11 @@ class Request {
std::unique_ptr<IO> io__;
DataCache* cache__ = nullptr;
VIRTUAL uint64_t GetSlotId() const;
+ VIRTUAL bool WasAlreadyProcessed() const;
+ VIRTUAL void SetAlreadyProcessedFlag();
+ VIRTUAL void SetWarningMessage(std::string message);
+ VIRTUAL const std::string& GetWarningMessage() const;
+ VIRTUAL Error CheckForDuplicates();
private:
const GenericRequestHeader request_header_;
const SocketDescriptor socket_fd_;
@@ -80,6 +86,9 @@ class Request {
std::string beamtime_year_;
std::string metadata_;
CacheMeta* slot_meta_ = nullptr;
+ bool already_processed_ = false;
+ std::string warning_message_;
+ const RequestHandlerDbCheckRequest* check_duplicate_request_handler_;
};
diff --git a/receiver/src/request_factory.cpp b/receiver/src/request_factory.cpp
index 27f31cb64..a01bc4803 100644
--- a/receiver/src/request_factory.cpp
+++ b/receiver/src/request_factory.cpp
@@ -38,7 +38,7 @@ void RequestFactory::AddReceiveViaBufferHandlers(std::unique_ptr<Request>& reque
Error RequestFactory::AddReceiveDirectToFileHandler(std::unique_ptr<Request>& request,
const GenericRequestHeader& request_header) const {
if (!GetReceiverConfig()->write_to_disk) {
- return ReceiverErrorTemplates::kReject.Generate("reciever does not support writing to disk");
+ return ReceiverErrorTemplates::kInternalServerError.Generate("reciever does not support writing to disk");
}
if (! (request_header.custom_data[kPosIngestMode] & kStoreInFilesystem)) {
return ReceiverErrorTemplates::kBadRequest.Generate("ingest mode should include kStoreInFilesystem for large files ");
@@ -69,7 +69,7 @@ Error RequestFactory::AddHandlersToRequest(std::unique_ptr<Request>& request,
request->AddHandler(&request_handler_receivedata_);
request->AddHandler(&request_handler_db_meta_write_);
} else {
- return ReceiverErrorTemplates::kReject.Generate("reciever does not support writing to database");
+ return ReceiverErrorTemplates::kInternalServerError.Generate("reciever does not support writing to database");
}
break;
}
@@ -88,7 +88,9 @@ Error RequestFactory::AddHandlersToRequest(std::unique_ptr<Request>& request,
std::unique_ptr<Request> RequestFactory::GenerateRequest(const GenericRequestHeader&
request_header, SocketDescriptor socket_fd, std::string origin_uri,
Error* err) const noexcept {
- auto request = std::unique_ptr<Request> {new Request{request_header, socket_fd, std::move(origin_uri), cache_.get()}};
+ auto request = std::unique_ptr<Request> {new Request{request_header, socket_fd, std::move(origin_uri), cache_.get(),
+ &request_handler_db_check_}
+ };
*err = AddHandlersToRequest(request, request_header);
if (*err) {
return nullptr;
diff --git a/receiver/src/request_factory.h b/receiver/src/request_factory.h
index 90ce81c28..364225421 100644
--- a/receiver/src/request_factory.h
+++ b/receiver/src/request_factory.h
@@ -2,6 +2,8 @@
#define ASAPO_REQUEST_FACTORY_H
#include "request.h"
+#include "write_file_processor.h"
+#include "receive_file_processor.h"
namespace asapo {
@@ -13,13 +15,16 @@ class RequestFactory {
private:
Error AddHandlersToRequest(std::unique_ptr<Request>& request, const GenericRequestHeader& request_header) const;
Error AddReceiveWriteHandlers(std::unique_ptr<Request>& request, const GenericRequestHeader& request_header) const;
- RequestHandlerFileWrite request_handler_filewrite_;
+ WriteFileProcessor write_file_processor_;
+ ReceiveFileProcessor receive_file_processor_;
+ RequestHandlerFileProcess request_handler_filewrite_{&write_file_processor_};
+ RequestHandlerFileProcess request_handler_filereceive_{&receive_file_processor_};
RequestHandlerReceiveData request_handler_receivedata_;
RequestHandlerReceiveMetaData request_handler_receive_metadata_;
RequestHandlerDbWrite request_handler_dbwrite_{kDBDataCollectionNamePrefix};
RequestHandlerDbMetaWrite request_handler_db_meta_write_{kDBMetaCollectionName};
RequestHandlerAuthorize request_handler_authorize_;
- RequestHandlerFileReceive request_handler_filereceive_;
+ RequestHandlerDbCheckRequest request_handler_db_check_{kDBDataCollectionNamePrefix};;
SharedCache cache_;
bool ReceiveDirectToFile(const GenericRequestHeader& request_header) const;
Error AddReceiveDirectToFileHandler(std::unique_ptr<Request>& request,
diff --git a/receiver/src/request_handler_db_check_request.cpp b/receiver/src/request_handler_db_check_request.cpp
new file mode 100644
index 000000000..03d5393a4
--- /dev/null
+++ b/receiver/src/request_handler_db_check_request.cpp
@@ -0,0 +1,72 @@
+#include "request_handler_db_check_request.h"
+
+#include "database/database.h"
+#include "database/db_error.h"
+#include "logger/logger.h"
+#include "request_handler_db.h"
+#include "receiver_config.h"
+#include "io/io.h"
+#include "request.h"
+
+namespace asapo {
+
+RequestHandlerDbCheckRequest::RequestHandlerDbCheckRequest(std::string collection_name_prefix) : RequestHandlerDb(
+ std::move(
+ collection_name_prefix)) {
+
+}
+
+Error RequestHandlerDbCheckRequest::GetRecordFromDb(const Request* request, FileInfo* record ) const {
+ auto op_code = request->GetOpCode();
+ auto id = request->GetDataID();
+ auto col_name = collection_name_prefix_ + "_" + request->GetSubstream();
+ Error err;
+ if (op_code == Opcode::kOpcodeTransferData) {
+ err = db_client__->GetById(col_name, id, record);
+ if (!err) {
+ log__->Debug(std::string{"get record id "} + std::to_string(id) + " from " + col_name + " in " +
+ db_name_ + " at " + GetReceiverConfig()->database_uri);
+ }
+ return err;
+ } else {
+ auto subset_id = request->GetCustomData()[1];
+ err = db_client__->GetDataSetById(col_name, subset_id, id, record);
+ if (!err) {
+ log__->Debug(std::string{"get subset record id "} + std::to_string(subset_id) + " from " + col_name + " in " +
+ db_name_ + " at " + GetReceiverConfig()->database_uri);
+ }
+ return err;
+ }
+}
+
+
+bool RequestHandlerDbCheckRequest::SameRequestInRecord(const Request* request, const FileInfo& record) const {
+ std::string meta = request->GetMetaData();
+ if (meta.size() == 0) { // so it is stored in database
+ meta = "{}";
+ }
+ return request->GetDataSize() == record.size
+ && request->GetFileName() == record.name
+ && meta == record.metadata;
+}
+
+Error RequestHandlerDbCheckRequest::ProcessRequest(Request* request) const {
+ if (auto err = RequestHandlerDb::ProcessRequest(request) ) {
+ return err;
+ }
+
+ FileInfo record;
+ auto err = GetRecordFromDb(request, &record);
+ if (err) {
+ return err == DBErrorTemplates::kNoRecord ? nullptr : std::move(err);
+ }
+
+ if (SameRequestInRecord(request, record)) {
+ return ReceiverErrorTemplates::kWarningDuplicatedRequest.Generate();
+ } else {
+ return ReceiverErrorTemplates::kBadRequest.Generate("already have record with same id");
+ }
+}
+
+
+}
\ No newline at end of file
diff --git a/receiver/src/request_handler_db_check_request.h b/receiver/src/request_handler_db_check_request.h
new file mode 100644
index 000000000..c5efd1100
--- /dev/null
+++ b/receiver/src/request_handler_db_check_request.h
@@ -0,0 +1,24 @@
+#ifndef ASAPO_REQUEST_HANDLER_DB_CHECK_REQUEST_H
+#define ASAPO_REQUEST_HANDLER_DB_CHECK_REQUEST_H
+
+#include "request_handler.h"
+#include "database/database.h"
+#include "request_handler_db.h"
+#include "io/io.h"
+#include "preprocessor/definitions.h"
+
+namespace asapo {
+
+class RequestHandlerDbCheckRequest FINAL : public RequestHandlerDb {
+ public:
+ RequestHandlerDbCheckRequest(std::string collection_name_prefix);
+ Error ProcessRequest(Request* request) const override;
+ private:
+ Error GetRecordFromDb(const Request* request, FileInfo* record) const;
+ bool SameRequestInRecord(const Request* request, const FileInfo& record) const;
+
+};
+
+}
+
+#endif //ASAPO_REQUEST_HANDLER_DB_CHECK_REQUEST_H
diff --git a/receiver/src/request_handler_db_write.cpp b/receiver/src/request_handler_db_write.cpp
index e2d3c656c..6a0e31bcb 100644
--- a/receiver/src/request_handler_db_write.cpp
+++ b/receiver/src/request_handler_db_write.cpp
@@ -3,6 +3,7 @@
#include "receiver_config.h"
#include "receiver_logger.h"
#include "io/io_factory.h"
+#include "database/db_error.h"
namespace asapo {
@@ -17,13 +18,35 @@ std::string string_format( const std::string& format, Args ... args ) {
Error RequestHandlerDbWrite::ProcessRequest(Request* request) const {
- if (Error err = RequestHandlerDb::ProcessRequest(request) ) {
+ if (request->WasAlreadyProcessed()) {
+ return nullptr;
+ }
+
+ if (auto err = RequestHandlerDb::ProcessRequest(request) ) {
+ return err;
+ }
+
+ auto err = InsertRecordToDb(request);
+ if (err == DBErrorTemplates::kDuplicateID) {
+ return ProcessDuplicateRecordSituation(request);
+ } else {
return err;
}
+}
+
+Error RequestHandlerDbWrite::ProcessDuplicateRecordSituation(Request* request) const {
+ auto check_err = request->CheckForDuplicates();
+ if (check_err == ReceiverErrorTemplates::kWarningDuplicatedRequest) {
+ std::string warn_str = "ignoring duplicate record for id " + std::to_string(request->GetDataID());
+ request->SetWarningMessage(warn_str);
+ log__->Warning(warn_str);
+ return nullptr;
+ }
- return InsertRecordToDb(request);
+ return check_err;
}
+
Error RequestHandlerDbWrite::InsertRecordToDb(const Request* request) const {
auto file_info = PrepareFileInfo(request);
@@ -31,7 +54,7 @@ Error RequestHandlerDbWrite::InsertRecordToDb(const Request* request) const {
auto col_name = collection_name_prefix_ + "_" + request->GetSubstream();
Error err;
if (op_code == Opcode::kOpcodeTransferData) {
- err = db_client__->Insert(col_name, file_info, true);
+ err = db_client__->Insert(col_name, file_info, false);
if (!err) {
log__->Debug(std::string{"insert record id "} + std::to_string(file_info.id) + " to " + col_name + " in " +
db_name_ +
@@ -40,7 +63,7 @@ Error RequestHandlerDbWrite::InsertRecordToDb(const Request* request) const {
} else {
auto subset_id = request->GetCustomData()[1];
auto subset_size = request->GetCustomData()[2];
- err = db_client__->InsertAsSubset(col_name, file_info, subset_id, subset_size, true);
+ err = db_client__->InsertAsSubset(col_name, file_info, subset_id, subset_size, false);
if (!err) {
log__->Debug(std::string{"insert record as subset id "} + std::to_string(subset_id) + ", id: " +
std::to_string(file_info.id) + " to " + col_name + " in " +
@@ -62,9 +85,11 @@ FileInfo RequestHandlerDbWrite::PrepareFileInfo(const Request* request) const {
file_info.metadata = request->GetMetaData();
return file_info;
}
+
RequestHandlerDbWrite::RequestHandlerDbWrite(std::string collection_name_prefix) : RequestHandlerDb(std::move(
collection_name_prefix)) {
}
+
}
diff --git a/receiver/src/request_handler_db_write.h b/receiver/src/request_handler_db_write.h
index 56a9c00a8..4603c028f 100644
--- a/receiver/src/request_handler_db_write.h
+++ b/receiver/src/request_handler_db_write.h
@@ -16,6 +16,8 @@ class RequestHandlerDbWrite final: public RequestHandlerDb {
private:
FileInfo PrepareFileInfo(const Request* request) const;
Error InsertRecordToDb(const Request* request) const;
+ Error ProcessDuplicateRecordSituation(Request* request) const;
+
};
}
diff --git a/receiver/src/request_handler_file_process.cpp b/receiver/src/request_handler_file_process.cpp
new file mode 100644
index 000000000..fde5496a2
--- /dev/null
+++ b/receiver/src/request_handler_file_process.cpp
@@ -0,0 +1,48 @@
+#include "request_handler_file_process.h"
+#include "io/io_factory.h"
+#include "request.h"
+#include "receiver_logger.h"
+#include "receiver_config.h"
+#include "preprocessor/definitions.h"
+
+namespace asapo {
+
+Error RequestHandlerFileProcess::ProcessRequest(Request* request) const {
+
+ auto err = file_processor_->ProcessFile(request, false);
+ if (err == IOErrorTemplates::kFileAlreadyExists) {
+ return ProcessFileExistSituation(request);
+ }
+
+ return err;
+}
+
+Error RequestHandlerFileProcess::ProcessFileExistSituation(Request* request) const {
+ auto err_duplicate = request->CheckForDuplicates();
+ if (err_duplicate == nullptr) {
+ request->SetWarningMessage("file has been overwritten");
+ log__->Warning("overwriting file " + request->GetFullPath(GetReceiverConfig()->root_folder));
+ return file_processor_->ProcessFile(request, true);
+ }
+
+ if (err_duplicate == ReceiverErrorTemplates::kWarningDuplicatedRequest) {
+ request->SetAlreadyProcessedFlag();
+ request->SetWarningMessage("duplicated request, ignored");
+ log__->Warning("duplicated request, id: " + std::to_string(request->GetDataID()));
+ return nullptr;
+ }
+
+ return err_duplicate;
+}
+
+
+RequestHandlerFileProcess::RequestHandlerFileProcess(const FileProcessor* file_processor) : io__{GenerateDefaultIO()},
+ log__{GetDefaultReceiverLogger()}, file_processor_{file_processor} {
+
+}
+
+StatisticEntity RequestHandlerFileProcess::GetStatisticEntity() const {
+ return StatisticEntity::kDisk;
+}
+
+}
diff --git a/receiver/src/request_handler_file_process.h b/receiver/src/request_handler_file_process.h
new file mode 100644
index 000000000..d4db2b5f5
--- /dev/null
+++ b/receiver/src/request_handler_file_process.h
@@ -0,0 +1,25 @@
+#ifndef ASAPO_REQUEST_HANDLER_FILE_PROCESS_H
+#define ASAPO_REQUEST_HANDLER_FILE_PROCESS_H
+
+#include "request_handler.h"
+#include "logger/logger.h"
+#include "file_processor.h"
+#include "io/io.h"
+
+namespace asapo {
+
+class RequestHandlerFileProcess final : public ReceiverRequestHandler {
+ public:
+ RequestHandlerFileProcess() = delete;
+ RequestHandlerFileProcess(const FileProcessor* file_processor);
+ StatisticEntity GetStatisticEntity() const override;
+ Error ProcessRequest(Request* request) const override;
+ std::unique_ptr<IO> io__;
+ const AbstractLogger* log__;
+ private:
+ Error ProcessFileExistSituation(Request* request) const;
+ const FileProcessor* file_processor_;
+
+};
+}
+#endif //ASAPO_REQUEST_HANDLER_FILE_PROCESS_H
diff --git a/receiver/src/request_handler_file_receive.h b/receiver/src/request_handler_file_receive.h
deleted file mode 100644
index 4e9cdb4b5..000000000
--- a/receiver/src/request_handler_file_receive.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef ASAPO_REQUEST_HANDLER_FILE_RECEIVE_H
-#define ASAPO_REQUEST_HANDLER_FILE_RECEIVE_H
-
-#include "request_handler.h"
-#include "logger/logger.h"
-
-#include "io/io.h"
-
-namespace asapo {
-
-class RequestHandlerFileReceive final: public ReceiverRequestHandler {
- public:
- RequestHandlerFileReceive();
- StatisticEntity GetStatisticEntity() const override;
- Error ProcessRequest(Request* request) const override;
- std::unique_ptr<IO> io__;
- const AbstractLogger* log__;
-};
-
-}
-
-#endif //ASAPO_REQUEST_HANDLER_FILE_RECEIVE_H
diff --git a/receiver/src/request_handler_file_write.h b/receiver/src/request_handler_file_write.h
deleted file mode 100644
index 8e77f8086..000000000
--- a/receiver/src/request_handler_file_write.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef ASAPO_REQUEST_HANDLER_FILE_WRITE_H
-#define ASAPO_REQUEST_HANDLER_FILE_WRITE_H
-
-#include "request_handler.h"
-#include "logger/logger.h"
-
-#include "io/io.h"
-
-namespace asapo {
-
-class RequestHandlerFileWrite final: public ReceiverRequestHandler {
- public:
- RequestHandlerFileWrite();
- StatisticEntity GetStatisticEntity() const override;
- Error ProcessRequest(Request* request) const override;
- std::unique_ptr<IO> io__;
- const AbstractLogger* log__;
-};
-
-}
-
-#endif //ASAPO_REQUEST_HANDLER_FILE_WRITE_H
diff --git a/receiver/src/requests_dispatcher.cpp b/receiver/src/requests_dispatcher.cpp
index 4ac17c30d..93c1cade3 100644
--- a/receiver/src/requests_dispatcher.cpp
+++ b/receiver/src/requests_dispatcher.cpp
@@ -16,9 +16,7 @@ producer_uri_{std::move(address)} {
NetworkErrorCode GetNetworkCodeFromError(const Error& err) {
if (err) {
- if (err == IOErrorTemplates::kFileAlreadyExists) {
- return NetworkErrorCode::kNetErrorFileIdAlreadyInUse;
- } else if (err == ReceiverErrorTemplates::kAuthorizationFailure) {
+ if (err == ReceiverErrorTemplates::kAuthorizationFailure) {
return NetworkErrorCode::kNetAuthorizationError;
} else if (err == DBErrorTemplates::kJsonParseError || err == ReceiverErrorTemplates::kBadRequest) {
return NetworkErrorCode::kNetErrorWrongRequest;
@@ -29,25 +27,47 @@ NetworkErrorCode GetNetworkCodeFromError(const Error& err) {
return NetworkErrorCode::kNetErrorNoError;
}
-Error RequestsDispatcher::ProcessRequest(const std::unique_ptr<Request>& request) const noexcept {
+GenericNetworkResponse RequestsDispatcher::CreateResponseToRequest(const std::unique_ptr<Request>& request,
+ const Error& handle_error) const {
+ GenericNetworkResponse generic_response;
+ generic_response.op_code = request->GetOpCode();
+ generic_response.error_code = GetNetworkCodeFromError(handle_error);
+ strcpy(generic_response.message, "");
+ if (handle_error) {
+ strncpy(generic_response.message, handle_error->Explain().c_str(), kMaxMessageSize);
+ }
+ if (request->GetWarningMessage().size() > 0) {
+ generic_response.error_code = kNetErrorWarning;
+ strncpy(generic_response.message, request->GetWarningMessage().c_str(), kMaxMessageSize);
+ }
+ return generic_response;
+}
+
+Error RequestsDispatcher::HandleRequest(const std::unique_ptr<Request>& request) const {
log__->Debug("processing request id " + std::to_string(request->GetDataID()) + ", opcode " +
std::to_string(request->GetOpCode()) + " from " + producer_uri_ );
Error handle_err;
handle_err = request->Handle(statistics__);
- GenericNetworkResponse generic_response;
- generic_response.op_code = request->GetOpCode();
- generic_response.error_code = GetNetworkCodeFromError(handle_err);
- strcpy(generic_response.message, "");
if (handle_err) {
log__->Error("error processing request from " + producer_uri_ + " - " + handle_err->Explain());
- strncpy(generic_response.message, handle_err->Explain().c_str(), kMaxMessageSize);
}
+ return handle_err;
+}
+
+Error RequestsDispatcher::SendResponse(const std::unique_ptr<Request>& request, const Error& handle_error) const {
log__->Debug("sending response to " + producer_uri_ );
Error io_err;
+ GenericNetworkResponse generic_response = CreateResponseToRequest(request, handle_error);
io__->Send(socket_fd_, &generic_response, sizeof(GenericNetworkResponse), &io_err);
if (io_err) {
log__->Error("error sending response to " + producer_uri_ + " - " + io_err->Explain());
}
+ return io_err;
+}
+
+Error RequestsDispatcher::ProcessRequest(const std::unique_ptr<Request>& request) const noexcept {
+ auto handle_err = HandleRequest(request);
+ auto io_err = SendResponse(request, handle_err);
return handle_err == nullptr ? std::move(io_err) : std::move(handle_err);
}
@@ -61,20 +81,15 @@ std::unique_ptr<Request> RequestsDispatcher::GetNextRequest(Error* err) const no
if (*err == ErrorTemplates::kEndOfFile) {
log__->Debug("error getting next request from " + producer_uri_ + " - " + "peer has performed an orderly shutdown");
} else {
- log__->Error("error getting next request from " + producer_uri_ + " - " + (*err)->
- Explain()
- );
+ log__->Error("error getting next request from " + producer_uri_ + " - " + (*err)->Explain());
}
return nullptr;
}
statistics__-> StopTimer();
auto request = request_factory__->GenerateRequest(generic_request_header, socket_fd_, producer_uri_, err);
if (*err) {
- log__->Error("error processing request from " + producer_uri_ + " - " + (*err)->
- Explain()
- );
+ log__->Error("error processing request from " + producer_uri_ + " - " + (*err)->Explain());
}
-
return request;
}
diff --git a/receiver/src/requests_dispatcher.h b/receiver/src/requests_dispatcher.h
index 9389a3d97..047c1b5ef 100644
--- a/receiver/src/requests_dispatcher.h
+++ b/receiver/src/requests_dispatcher.h
@@ -24,6 +24,10 @@ class RequestsDispatcher {
private:
SocketDescriptor socket_fd_;
std::string producer_uri_;
+ GenericNetworkResponse CreateResponseToRequest(const std::unique_ptr<Request>& request,
+ const Error& handle_error) const;
+ Error HandleRequest(const std::unique_ptr<Request>& request) const;
+ Error SendResponse(const std::unique_ptr<Request>& request, const Error& handle_error) const;
};
}
diff --git a/receiver/src/request_handler_file_write.cpp b/receiver/src/write_file_processor.cpp
similarity index 62%
rename from receiver/src/request_handler_file_write.cpp
rename to receiver/src/write_file_processor.cpp
index f30341889..0e92bfe00 100644
--- a/receiver/src/request_handler_file_write.cpp
+++ b/receiver/src/write_file_processor.cpp
@@ -1,37 +1,34 @@
-#include "request_handler_file_write.h"
+#include "write_file_processor.h"
+
#include "io/io_factory.h"
+#include "receiver_error.h"
+#include "preprocessor/definitions.h"
#include "request.h"
-#include "receiver_logger.h"
#include "receiver_config.h"
-#include "preprocessor/definitions.h"
namespace asapo {
-Error RequestHandlerFileWrite::ProcessRequest(Request* request) const {
+WriteFileProcessor::WriteFileProcessor() : FileProcessor() {
+
+}
+
+
+Error WriteFileProcessor::ProcessFile(const Request* request, bool overwrite) const {
auto fsize = request->GetDataSize();
if (fsize <= 0) {
return ReceiverErrorTemplates::kBadRequest.Generate("wrong file size");
}
auto data = request->GetData();
-
auto fname = request->GetFileName();
auto root_folder = request->GetFullPath(GetReceiverConfig()->root_folder);
- auto err = io__->WriteDataToFile(root_folder, fname, (uint8_t*)data, (size_t) fsize, true);
+
+ auto err = io__->WriteDataToFile(root_folder, fname, (uint8_t*)data, (size_t) fsize, true, overwrite);
if (!err) {
log__->Debug("saved file of size " + std::to_string(fsize) + " to " + root_folder + kPathSeparator + fname);
}
- return err;
-
-}
-RequestHandlerFileWrite::RequestHandlerFileWrite() : io__{GenerateDefaultIO()} , log__{GetDefaultReceiverLogger()} {
-
-}
-
-StatisticEntity RequestHandlerFileWrite::GetStatisticEntity() const {
- return StatisticEntity::kDisk;
+ return err;
}
-
-}
+}
\ No newline at end of file
diff --git a/receiver/src/write_file_processor.h b/receiver/src/write_file_processor.h
new file mode 100644
index 000000000..e946491fc
--- /dev/null
+++ b/receiver/src/write_file_processor.h
@@ -0,0 +1,16 @@
+#ifndef ASAPO_WRITE_FILE_PROCESSOR_H
+#define ASAPO_WRITE_FILE_PROCESSOR_H
+
+#include "file_processor.h"
+
+namespace asapo {
+
+class WriteFileProcessor final : public FileProcessor {
+ public:
+ WriteFileProcessor();
+ Error ProcessFile(const Request* request, bool overwrite) const override;
+};
+
+}
+
+#endif //ASAPO_WRITE_FILE_PROCESSOR_H
diff --git a/receiver/unittests/receiver_data_server/receiver_dataserver_mocking.h b/receiver/unittests/receiver_data_server/receiver_dataserver_mocking.h
index 5c21f1e23..6eacdb6c7 100644
--- a/receiver/unittests/receiver_data_server/receiver_dataserver_mocking.h
+++ b/receiver/unittests/receiver_data_server/receiver_dataserver_mocking.h
@@ -47,7 +47,7 @@ class MockPool : public RequestPool {
Error AddRequests(GenericRequests requests) noexcept override {
std::vector<GenericRequest> reqs;
for (const auto& preq : requests) {
- reqs.push_back(GenericRequest{preq->header});
+ reqs.push_back(GenericRequest{preq->header, 0});
}
return Error(AddRequests_t(std::move(reqs)));
diff --git a/receiver/unittests/receiver_mocking.h b/receiver/unittests/receiver_mocking.h
index a0ae63135..cce334542 100644
--- a/receiver/unittests/receiver_mocking.h
+++ b/receiver/unittests/receiver_mocking.h
@@ -7,6 +7,7 @@
#include "../src/receiver_statistics.h"
#include "../src/request.h"
#include "../src/data_cache.h"
+#include "../src/file_processor.h"
namespace asapo {
@@ -40,10 +41,28 @@ class MockStatistics : public asapo::ReceiverStatistics {
};
+class MockHandlerDbCheckRequest : public asapo::RequestHandlerDbCheckRequest {
+ public:
+ MockHandlerDbCheckRequest(std::string collection_name_prefix): RequestHandlerDbCheckRequest(collection_name_prefix) {};
+
+ Error ProcessRequest(Request* request) const override {
+ return Error{ProcessRequest_t(*request)};
+ }
+
+ StatisticEntity GetStatisticEntity() const override {
+ return StatisticEntity::kDatabase;
+ }
+
+ MOCK_CONST_METHOD1(ProcessRequest_t, ErrorInterface * (const Request& request));
+
+};
+
+
class MockRequest: public Request {
public:
- MockRequest(const GenericRequestHeader& request_header, SocketDescriptor socket_fd, std::string origin_uri):
- Request(request_header, socket_fd, std::move(origin_uri), nullptr) {};
+ MockRequest(const GenericRequestHeader& request_header, SocketDescriptor socket_fd, std::string origin_uri,
+ const RequestHandlerDbCheckRequest* db_check_handler ):
+ Request(request_header, socket_fd, std::move(origin_uri), nullptr, db_check_handler) {};
MOCK_CONST_METHOD0(GetFileName, std::string());
MOCK_CONST_METHOD0(GetSubstream, std::string());
@@ -74,7 +93,17 @@ class MockRequest: public Request {
MOCK_METHOD1(SetBeamtimeYear, void (std::string));
MOCK_CONST_METHOD1(GetFullPath, std::string (std::string));
+ MOCK_CONST_METHOD0(WasAlreadyProcessed, bool());
+ MOCK_METHOD0(SetAlreadyProcessedFlag, void());
+ MOCK_METHOD1(SetWarningMessage, void(std::string));
+ MOCK_CONST_METHOD0(GetWarningMessage, const std::string & ());
+
+ Error CheckForDuplicates() override {
+ return Error{CheckForDuplicates_t()};
+ }
+
+ MOCK_METHOD0(CheckForDuplicates_t, ErrorInterface * ());
};
@@ -99,6 +128,15 @@ class MockStatisticsSender: public StatisticsSender {
};
+class MockFileProcessor: public FileProcessor {
+ public:
+ Error ProcessFile(const Request* request, bool overwrite) const override {
+ return Error{ProcessFile_t(request, overwrite)};
+
+ }
+ MOCK_CONST_METHOD2(ProcessFile_t, ErrorInterface * (const Request*, bool));
+};
+
}
#endif //ASAPO_RECEIVER_MOCKING_H
diff --git a/receiver/unittests/test_connection.cpp b/receiver/unittests/test_connection.cpp
index edacbcd6e..844ccc76c 100644
--- a/receiver/unittests/test_connection.cpp
+++ b/receiver/unittests/test_connection.cpp
@@ -120,7 +120,8 @@ class ConnectionTests : public Test {
));
return nullptr;
} else {
- auto request = new Request(GenericRequestHeader{asapo::kOpcodeUnknownOp, 0, 1, 0, ""}, 0, connected_uri, nullptr);
+ auto request = new Request(GenericRequestHeader{asapo::kOpcodeUnknownOp, 0, 1, 0, ""}, 0, connected_uri, nullptr,
+ nullptr);
EXPECT_CALL(mock_dispatcher, GetNextRequest_t(_))
.WillOnce(DoAll(
SetArgPointee<0>(nullptr),
diff --git a/receiver/unittests/test_request_handler_file_receive.cpp b/receiver/unittests/test_receive_file_processor.cpp
similarity index 61%
rename from receiver/unittests/test_request_handler_file_receive.cpp
rename to receiver/unittests/test_receive_file_processor.cpp
index 15d6931bf..6a729431b 100644
--- a/receiver/unittests/test_request_handler_file_receive.cpp
+++ b/receiver/unittests/test_receive_file_processor.cpp
@@ -4,13 +4,10 @@
#include "unittests/MockIO.h"
#include "unittests/MockLogger.h"
-#include "../src/receiver_error.h"
-#include "../src/request.h"
-#include "../src/request_handler.h"
-#include "../src/request_handler_file_receive.h"
+#include "../src/receive_file_processor.h"
#include "common/networking.h"
-#include "mock_receiver_config.h"
#include "preprocessor/definitions.h"
+#include "mock_receiver_config.h"
#include "receiver_mocking.h"
@@ -37,21 +34,22 @@ using ::asapo::FileDescriptor;
using ::asapo::SocketDescriptor;
using ::asapo::MockIO;
using asapo::Request;
-using asapo::RequestHandlerFileReceive;
+using asapo::ReceiveFileProcessor;
using ::asapo::GenericRequestHeader;
using asapo::MockRequest;
namespace {
-TEST(FileReceive, Constructor) {
- RequestHandlerFileReceive handler;
- ASSERT_THAT(dynamic_cast<asapo::IO*>(handler.io__.get()), Ne(nullptr));
- ASSERT_THAT(dynamic_cast<const asapo::AbstractLogger*>(handler.log__), Ne(nullptr));
+TEST(ReceiveFileProcessor, Constructor) {
+ ReceiveFileProcessor processor;
+ ASSERT_THAT(dynamic_cast<asapo::IO*>(processor.io__.get()), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<const asapo::AbstractLogger*>(processor.log__), Ne(nullptr));
+
}
-class FileReceiveHandlerTests : public Test {
+class ReceiveFileProcessorTests : public Test {
public:
- RequestHandlerFileReceive handler;
+ ReceiveFileProcessor processor;
NiceMock<MockIO> mock_io;
std::unique_ptr<MockRequest> mock_request;
NiceMock<asapo::MockLogger> mock_logger;
@@ -59,54 +57,59 @@ class FileReceiveHandlerTests : public Test {
std::string expected_file_name = "2";
std::string expected_beamtime_id = "beamtime_id";
std::string expected_beamline = "beamline";
- std::string expected_root_folder = "root_folder";
std::string expected_facility = "facility";
std::string expected_year = "2020";
+ uint64_t expected_file_size = 10;
+ bool expected_overwrite = false;
+ std::string expected_root_folder = "root_folder";
std::string expected_full_path = expected_root_folder + asapo::kPathSeparator + expected_facility +
asapo::kPathSeparator + "gpfs" +
asapo::kPathSeparator + expected_beamline +
asapo::kPathSeparator + expected_year +
asapo::kPathSeparator + "data" +
asapo::kPathSeparator + expected_beamtime_id;
-
- uint64_t expected_file_size = 10;
+ void ExpectFileWrite(const asapo::SimpleErrorTemplate* error_template);
void MockRequestData();
void SetUp() override {
GenericRequestHeader request_header;
request_header.data_id = 2;
- mock_request.reset(new MockRequest{request_header, expected_socket_id, ""});
- handler.io__ = std::unique_ptr<asapo::IO> {&mock_io};
- handler.log__ = &mock_logger;
+ asapo::ReceiverConfig test_config;
+ test_config.root_folder = expected_root_folder;
+ asapo::SetReceiverConfig(test_config, "none");
+ processor.log__ = &mock_logger;
+ mock_request.reset(new MockRequest{request_header, 1, "", nullptr});
+ processor.io__ = std::unique_ptr<asapo::IO> {&mock_io};
}
void TearDown() override {
- handler.io__.release();
+ processor.io__.release();
}
};
-TEST_F(FileReceiveHandlerTests, CheckStatisticEntity) {
- auto entity = handler.GetStatisticEntity();
- ASSERT_THAT(entity, Eq(asapo::StatisticEntity::kDisk));
-}
-
-void FileReceiveHandlerTests::MockRequestData() {
- EXPECT_CALL(*mock_request, GetDataSize())
- .WillOnce(Return(expected_file_size))
- ;
+void ReceiveFileProcessorTests::MockRequestData() {
EXPECT_CALL(*mock_request, GetSocket())
.WillOnce(Return(expected_socket_id))
;
- EXPECT_CALL(*mock_request, GetFullPath(expected_root_folder))
- .WillOnce(Return(expected_full_path));
+ EXPECT_CALL(*mock_request, GetDataSize()).Times(1)
+ .WillRepeatedly(Return(expected_file_size));
- EXPECT_CALL(*mock_request, GetFileName())
- .WillOnce(Return(expected_file_name))
- ;
+ EXPECT_CALL(*mock_request, GetFullPath(expected_root_folder)).Times(1)
+ .WillRepeatedly(Return(expected_full_path));
+
+ EXPECT_CALL(*mock_request, GetFileName()).Times(1)
+ .WillRepeatedly(Return(expected_file_name));
}
-TEST_F(FileReceiveHandlerTests, CallsReceiveFile) {
+void ReceiveFileProcessorTests::ExpectFileWrite(const asapo::SimpleErrorTemplate* error_template) {
+ EXPECT_CALL(mock_io, WriteDataToFile_t(expected_full_path, expected_file_name, _, expected_file_size, true,
+ expected_overwrite))
+ .WillOnce(
+ Return(error_template == nullptr ? nullptr : error_template->Generate().release()));
+}
+
+TEST_F(ReceiveFileProcessorTests, CallsReceiveFile) {
asapo::ReceiverConfig test_config;
test_config.root_folder = expected_root_folder;
@@ -115,22 +118,22 @@ TEST_F(FileReceiveHandlerTests, CallsReceiveFile) {
MockRequestData();
EXPECT_CALL(mock_io, ReceiveDataToFile_t(expected_socket_id, expected_full_path, expected_file_name, expected_file_size,
- true))
+ true, expected_overwrite))
.WillOnce(
Return(asapo::IOErrorTemplates::kUnknownIOError.Generate().release())
);
- auto err = handler.ProcessRequest(mock_request.get());
+ auto err = processor.ProcessFile(mock_request.get(), expected_overwrite);
ASSERT_THAT(err, Eq(asapo::IOErrorTemplates::kUnknownIOError));
}
-TEST_F(FileReceiveHandlerTests, WritesToLog) {
+TEST_F(ReceiveFileProcessorTests, WritesToLog) {
MockRequestData();
- EXPECT_CALL(mock_io, ReceiveDataToFile_t(_, _, _, _, _))
+ EXPECT_CALL(mock_io, ReceiveDataToFile_t(_, _, _, _, _, _))
.WillOnce(Return(nullptr));
EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("received file"),
@@ -140,8 +143,9 @@ TEST_F(FileReceiveHandlerTests, WritesToLog) {
)
)
);
- handler.ProcessRequest(mock_request.get());
+ processor.ProcessFile(mock_request.get(), expected_overwrite);
}
+
}
\ No newline at end of file
diff --git a/receiver/unittests/test_request.cpp b/receiver/unittests/test_request.cpp
index 392f83a13..2a94be104 100644
--- a/receiver/unittests/test_request.cpp
+++ b/receiver/unittests/test_request.cpp
@@ -5,7 +5,7 @@
#include "../src/receiver_error.h"
#include "../src/request.h"
#include "../src/request_handler.h"
-#include "../src/request_handler_file_write.h"
+#include "../src/request_handler_file_process.h"
#include "../src/request_handler_db_write.h"
#include "database/database.h"
@@ -60,6 +60,16 @@ class MockReqestHandler : public asapo::ReceiverRequestHandler {
};
+
+
+TEST(RequestTest, Constructor) {
+ std::unique_ptr<Request> request;
+ GenericRequestHeader generic_request_header;
+ request.reset(new Request{generic_request_header, 1, "", nullptr, nullptr});
+ ASSERT_THAT(request->WasAlreadyProcessed(), false);
+}
+
+
class RequestTests : public Test {
public:
GenericRequestHeader generic_request_header;
@@ -86,7 +96,7 @@ class RequestTests : public Test {
generic_request_header.op_code = expected_op_code;
generic_request_header.custom_data[asapo::kPosIngestMode] = asapo::kDefaultIngestMode;
strcpy(generic_request_header.message, expected_request_message);
- request.reset(new Request{generic_request_header, expected_socket_id, expected_origin_uri, nullptr});
+ request.reset(new Request{generic_request_header, expected_socket_id, expected_origin_uri, nullptr, nullptr});
request->io__ = std::unique_ptr<asapo::IO> {&mock_io};
ON_CALL(mock_io, Receive_t(expected_socket_id, _, data_size_, _)).WillByDefault(
DoAll(SetArgPointee<3>(nullptr),
@@ -166,7 +176,7 @@ void RequestTests::ExpectFileName(std::string sended, std::string received) {
strcpy(generic_request_header.message, sended.c_str());
request->io__.release();
- request.reset(new Request{generic_request_header, expected_socket_id, expected_origin_uri, nullptr});
+ request.reset(new Request{generic_request_header, expected_socket_id, expected_origin_uri, nullptr, nullptr});
request->io__ = std::unique_ptr<asapo::IO> {&mock_io};;
auto fname = request->GetFileName();
@@ -180,7 +190,7 @@ TEST_F(RequestTests, GetSubstream) {
strcpy(generic_request_header.substream, expected_substream.c_str());
request->io__.release();
- request.reset(new Request{generic_request_header, expected_socket_id, expected_origin_uri, nullptr});
+ request.reset(new Request{generic_request_header, expected_socket_id, expected_origin_uri, nullptr, nullptr});
request->io__ = std::unique_ptr<asapo::IO> {&mock_io};;
auto substream = request->GetSubstream();
@@ -261,6 +271,18 @@ TEST_F(RequestTests, RequestTests_GetFullPath) {
ASSERT_THAT(request->GetFullPath("test_folder"), expected_path);
}
+TEST_F(RequestTests, SetGetWarningMessage) {
+ request->SetWarningMessage("warn");
+
+ ASSERT_THAT(request->GetWarningMessage(), "warn");
+}
+
+
+TEST_F(RequestTests, SetGetOverwriteAllowed) {
+ request->SetAlreadyProcessedFlag();
+
+ ASSERT_THAT(request->WasAlreadyProcessed(), true);
+}
}
diff --git a/receiver/unittests/test_request_factory.cpp b/receiver/unittests/test_request_factory.cpp
index efd613811..223871137 100644
--- a/receiver/unittests/test_request_factory.cpp
+++ b/receiver/unittests/test_request_factory.cpp
@@ -10,7 +10,7 @@
#include "../src/request.h"
#include "../src/request_factory.h"
#include "../src/request_handler.h"
-#include "../src/request_handler_file_write.h"
+#include "../src/request_handler_file_process.h"
#include "../src/request_handler_db_write.h"
#include "../src/request_handler_authorize.h"
#include "../src/request_handler_receive_data.h"
@@ -89,7 +89,7 @@ TEST_F(FactoryTests, ReturnsDataRequestOnkNetOpcodeSendDataCode) {
ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerAuthorize*>(request->GetListHandlers()[0]), Ne(nullptr));
ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerReceiveMetaData*>(request->GetListHandlers()[1]), Ne(nullptr));
ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerReceiveData*>(request->GetListHandlers()[2]), Ne(nullptr));
- ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerFileWrite*>(request->GetListHandlers()[3]), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerFileProcess*>(request->GetListHandlers()[3]), Ne(nullptr));
ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerDbWrite*>(request->GetListHandlers().back()), Ne(nullptr));
}
}
@@ -108,7 +108,7 @@ TEST_F(FactoryTests, ReturnsDataRequestOnkNetOpcodeSendDataCodeLargeFile) {
ASSERT_THAT(request->GetListHandlers().size(), Eq(4));
ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerAuthorize*>(request->GetListHandlers()[0]), Ne(nullptr));
ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerReceiveMetaData*>(request->GetListHandlers()[1]), Ne(nullptr));
- ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerFileReceive*>(request->GetListHandlers()[2]), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerFileProcess*>(request->GetListHandlers()[2]), Ne(nullptr));
ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerDbWrite*>(request->GetListHandlers().back()), Ne(nullptr));
}
}
@@ -156,7 +156,7 @@ TEST_F(FactoryTests, DoNotAddDbWriterIfNotWanted) {
ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerAuthorize*>(request->GetListHandlers()[0]), Ne(nullptr));
ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerReceiveMetaData*>(request->GetListHandlers()[1]), Ne(nullptr));
ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerReceiveData*>(request->GetListHandlers()[2]), Ne(nullptr));
- ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerFileWrite*>(request->GetListHandlers()[3]), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<const asapo::RequestHandlerFileProcess*>(request->GetListHandlers()[3]), Ne(nullptr));
}
TEST_F(FactoryTests, CachePassedToRequest) {
@@ -190,7 +190,7 @@ TEST_F(FactoryTests, DonNotGenerateMetadataRequestIfNoDbConfigured) {
generic_request_header.op_code = asapo::Opcode::kOpcodeTransferMetaData;
auto request = factory.GenerateRequest(generic_request_header, 1, origin_uri, &err);
- ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kReject));
+ ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kInternalServerError));
}
@@ -205,7 +205,7 @@ TEST_F(FactoryTests, DonNotGenerateRequestIfWriteToDiskNotActive) {
auto request = factory.GenerateRequest(generic_request_header, 1, origin_uri, &err);
- ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kReject));
+ ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kInternalServerError));
}
TEST_F(FactoryTests, DonNotGenerateRequestIfIngestModeIsWrong) {
diff --git a/receiver/unittests/test_request_handler_authorizer.cpp b/receiver/unittests/test_request_handler_authorizer.cpp
index b80e4fe99..b6e7ff1b3 100644
--- a/receiver/unittests/test_request_handler_authorizer.cpp
+++ b/receiver/unittests/test_request_handler_authorizer.cpp
@@ -78,7 +78,7 @@ class AuthorizerHandlerTests : public Test {
void MockRequestData();
void SetUp() override {
GenericRequestHeader request_header;
- mock_request.reset(new MockRequest{request_header, 1, expected_producer_uri});
+ mock_request.reset(new MockRequest{request_header, 1, expected_producer_uri, nullptr});
handler.http_client__ = std::unique_ptr<asapo::HttpClient> {&mock_http_client};
handler.log__ = &mock_logger;
config.authorization_server = expected_authorization_server;
diff --git a/receiver/unittests/test_request_handler_db.cpp b/receiver/unittests/test_request_handler_db.cpp
index a2804fae7..1f4dc691c 100644
--- a/receiver/unittests/test_request_handler_db.cpp
+++ b/receiver/unittests/test_request_handler_db.cpp
@@ -79,7 +79,7 @@ class DbHandlerTests : public Test {
handler.db_client__ = std::unique_ptr<asapo::Database> {&mock_db};
handler.log__ = &mock_logger;
handler.http_client__ = std::unique_ptr<asapo::HttpClient> {&mock_http_client};
- mock_request.reset(new NiceMock<MockRequest> {request_header, 1, ""});
+ mock_request.reset(new NiceMock<MockRequest> {request_header, 1, "", nullptr});
ON_CALL(*mock_request, GetBeamtimeId()).WillByDefault(ReturnRef(expected_beamtime_id));
ON_CALL(*mock_request, GetStream()).WillByDefault(ReturnRef(expected_stream));
diff --git a/receiver/unittests/test_request_handler_db_check_request.cpp b/receiver/unittests/test_request_handler_db_check_request.cpp
new file mode 100644
index 000000000..e40cbfbb4
--- /dev/null
+++ b/receiver/unittests/test_request_handler_db_check_request.cpp
@@ -0,0 +1,273 @@
+#include <functional>
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+#include <database/db_error.h>
+
+#include "unittests/MockIO.h"
+#include "unittests/MockDatabase.h"
+#include "unittests/MockLogger.h"
+
+#include "../src/receiver_error.h"
+#include "../src/request.h"
+#include "../src/request_factory.h"
+#include "../src/request_handler.h"
+#include "../src/request_handler_db_check_request.h"
+#include "common/networking.h"
+#include "../../common/cpp/src/database/mongodb_client.h"
+#include "mock_receiver_config.h"
+#include "common/data_structs.h"
+
+#include "receiver_mocking.h"
+
+using asapo::MockRequest;
+using asapo::FileInfo;
+using ::testing::Test;
+using ::testing::Return;
+using ::testing::ReturnRef;
+using ::testing::_;
+using ::testing::DoAll;
+using ::testing::SetArgReferee;
+using ::testing::Gt;
+using ::testing::Eq;
+using ::testing::Ne;
+using ::testing::Mock;
+using ::testing::NiceMock;
+using ::testing::InSequence;
+using ::testing::SetArgPointee;
+using ::testing::AllOf;
+using ::testing::HasSubstr;
+
+
+using ::asapo::Error;
+using ::asapo::ErrorInterface;
+using ::asapo::FileDescriptor;
+using ::asapo::SocketDescriptor;
+using ::asapo::MockIO;
+using asapo::Request;
+using asapo::RequestHandlerDbCheckRequest;
+using ::asapo::GenericRequestHeader;
+
+using asapo::MockDatabase;
+using asapo::RequestFactory;
+using asapo::SetReceiverConfig;
+using asapo::ReceiverConfig;
+
+using MockFunctions = std::vector<std::function<void(asapo::ErrorInterface*, bool )>>;
+
+namespace {
+
+TEST(DbCheckRequestHandler, Constructor) {
+ RequestHandlerDbCheckRequest handler{""};
+ ASSERT_THAT(dynamic_cast<asapo::HttpClient*>(handler.http_client__.get()), Ne(nullptr));
+}
+
+
+class DbCheckRequestHandlerTests : public Test {
+ public:
+ std::string expected_substream = "substream";
+ std::string expected_collection_name = std::string(asapo::kDBDataCollectionNamePrefix) + "_" + expected_substream;
+ RequestHandlerDbCheckRequest handler{asapo::kDBDataCollectionNamePrefix};
+ std::unique_ptr<NiceMock<MockRequest>> mock_request;
+ NiceMock<MockDatabase> mock_db;
+ NiceMock<asapo::MockLogger> mock_logger;
+ ReceiverConfig config;
+ std::string expected_beamtime_id = "beamtime_id";
+ std::string expected_default_stream = "detector";
+ std::string expected_stream = "stream";
+ std::string expected_host_ip = "127.0.0.1";
+ uint64_t expected_port = 1234;
+ uint64_t expected_buf_id = 18446744073709551615ull;
+ std::string expected_file_name = "2";
+ std::string expected_metadata = "meta";
+ uint64_t expected_file_size = 10;
+ uint64_t expected_id = 15;
+ uint64_t expected_subset_id = 16;
+ uint64_t expected_subset_size = 2;
+ uint64_t expected_custom_data[asapo::kNCustomParams] {0, expected_subset_id, expected_subset_size};
+ FileInfo expected_file_info;
+ MockFunctions mock_functions;
+ int n_run = 0;
+ void SetUp() override {
+ GenericRequestHeader request_header;
+ request_header.data_id = 2;
+ request_header.op_code = asapo::Opcode::kOpcodeTransferData;
+ handler.db_client__ = std::unique_ptr<asapo::Database> {&mock_db};
+ handler.log__ = &mock_logger;
+ mock_request.reset(new NiceMock<MockRequest> {request_header, 1, "", nullptr});
+ config.database_uri = "127.0.0.1:27017";
+ config.advertise_ip = expected_host_ip;
+ config.dataserver.listen_port = expected_port;
+ SetReceiverConfig(config, "none");
+ expected_file_info = PrepareFileInfo();
+ mock_functions.push_back([this](asapo::ErrorInterface * error, bool expect_compare) {
+ MockGetByID(error, expect_compare);
+ n_run++;
+ });
+ mock_functions.push_back([this](asapo::ErrorInterface * error, bool expect_compare) {
+ MockGetSetByID(error, expect_compare);
+ n_run++;
+ });
+
+ ON_CALL(*mock_request, GetBeamtimeId()).WillByDefault(ReturnRef(expected_beamtime_id));
+ }
+ void ExpectRequestParams(asapo::Opcode op_code, const std::string& stream, bool expect_compare = true);
+
+ FileInfo PrepareFileInfo();
+ void MockGetByID(asapo::ErrorInterface* error, bool expect_compare);
+ void MockGetSetByID(asapo::ErrorInterface* error, bool expect_compare);
+ void TearDown() override {
+ handler.db_client__.release();
+ }
+
+
+};
+
+MATCHER_P(CompareFileInfo, file, "") {
+ if (arg.size != file.size) return false;
+ if (arg.source != file.source) return false;
+ if (arg.buf_id != file.buf_id) return false;
+ if (arg.name != file.name) return false;
+ if (arg.id != file.id) return false;
+ if (arg.metadata != file.metadata) return false;
+
+ return true;
+}
+
+
+void DbCheckRequestHandlerTests::ExpectRequestParams(asapo::Opcode op_code, const std::string& stream,
+ bool expect_compare) {
+
+ std::string db_name = expected_beamtime_id;
+ db_name += "_" + stream;
+
+ if (n_run == 0) {
+ EXPECT_CALL(mock_db, Connect_t(config.database_uri, db_name)).
+ WillOnce(testing::Return(nullptr));
+ EXPECT_CALL(*mock_request, GetBeamtimeId())
+ .WillOnce(ReturnRef(expected_beamtime_id))
+ ;
+
+ EXPECT_CALL(*mock_request, GetStream())
+ .WillOnce(ReturnRef(stream))
+ ;
+ }
+
+
+ if (expect_compare) {
+ EXPECT_CALL(*mock_request, GetDataSize())
+ .WillOnce(Return(expected_file_size))
+ ;
+
+ EXPECT_CALL(*mock_request, GetFileName())
+ .WillOnce(Return(expected_file_name))
+ ;
+
+ EXPECT_CALL(*mock_request, GetMetaData())
+ .WillOnce(ReturnRef(expected_metadata))
+ ;
+ }
+
+
+ EXPECT_CALL(*mock_request, GetSubstream())
+ .WillOnce(Return(expected_substream))
+ ;
+
+
+
+ EXPECT_CALL(*mock_request, GetDataID())
+ .WillOnce(Return(expected_id))
+ ;
+
+ EXPECT_CALL(*mock_request, GetOpCode())
+ .WillOnce(Return(op_code))
+ ;
+
+ if (op_code == asapo::Opcode::kOpcodeTransferSubsetData) {
+ EXPECT_CALL(*mock_request, GetCustomData_t())
+ .WillOnce(Return(expected_custom_data))
+ ;
+ }
+}
+
+FileInfo DbCheckRequestHandlerTests::PrepareFileInfo() {
+ FileInfo file_info;
+ file_info.size = expected_file_size;
+ file_info.name = expected_file_name;
+ file_info.id = expected_id;
+ file_info.buf_id = expected_buf_id;
+ file_info.source = expected_host_ip + ":" + std::to_string(expected_port);
+ file_info.metadata = expected_metadata;
+ return file_info;
+}
+
+void DbCheckRequestHandlerTests::MockGetByID(asapo::ErrorInterface* error, bool expect_compare ) {
+ ExpectRequestParams(asapo::Opcode::kOpcodeTransferData, expected_stream, expect_compare);
+ EXPECT_CALL(mock_db, GetById_t(expected_collection_name, expected_id, _)).
+ WillOnce(DoAll(
+ SetArgPointee<2>(expected_file_info),
+ testing::Return(error)
+ ));
+}
+
+void DbCheckRequestHandlerTests::MockGetSetByID(asapo::ErrorInterface* error, bool expect_compare ) {
+ ExpectRequestParams(asapo::Opcode::kOpcodeTransferSubsetData, expected_stream, expect_compare);
+ EXPECT_CALL(mock_db, GetSetById_t(expected_collection_name, expected_subset_id, expected_id, _)).
+ WillOnce(DoAll(
+ SetArgPointee<3>(expected_file_info),
+ testing::Return(error)
+ ));
+}
+
+
+TEST_F(DbCheckRequestHandlerTests, ErrorIfRecordsDoNotMatch) {
+ expected_file_info.metadata = expected_metadata + "_";
+
+ for (auto mock : mock_functions) {
+ mock(nullptr, true);
+ auto err = handler.ProcessRequest(mock_request.get());
+ ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kBadRequest));
+ Mock::VerifyAndClearExpectations(mock_request.get());
+ }
+
+}
+
+TEST_F(DbCheckRequestHandlerTests, DuplicateErrorIfRecordsMatch) {
+ for (auto mock : mock_functions) {
+ mock(nullptr, true);
+ auto err = handler.ProcessRequest(mock_request.get());
+ ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kWarningDuplicatedRequest));
+ Mock::VerifyAndClearExpectations(mock_request.get());
+ }
+}
+
+TEST_F(DbCheckRequestHandlerTests, DuplicateErrorIfRecordsMatchWithEmptyMetadata) {
+ expected_file_info.metadata = "{}";
+ expected_metadata = "";
+ for (auto mock : mock_functions) {
+ mock(nullptr, true);
+ auto err = handler.ProcessRequest(mock_request.get());
+ ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kWarningDuplicatedRequest));
+ Mock::VerifyAndClearExpectations(mock_request.get());
+ }
+}
+
+TEST_F(DbCheckRequestHandlerTests, OkIfNotFound) {
+ for (auto mock : mock_functions) {
+ mock(asapo::DBErrorTemplates::kNoRecord.Generate().release(), false);
+ auto err = handler.ProcessRequest(mock_request.get());
+ ASSERT_THAT(err, Eq(nullptr));
+ Mock::VerifyAndClearExpectations(mock_request.get());
+ }
+}
+
+TEST_F(DbCheckRequestHandlerTests, ErrorIfDbError) {
+ for (auto mock : mock_functions) {
+ mock(asapo::DBErrorTemplates::kConnectionError.Generate().release(), false);
+ auto err = handler.ProcessRequest(mock_request.get());
+ ASSERT_THAT(err, Eq(asapo::DBErrorTemplates::kConnectionError));
+ Mock::VerifyAndClearExpectations(mock_request.get());
+ }
+}
+
+}
diff --git a/receiver/unittests/test_request_handler_db_meta_writer.cpp b/receiver/unittests/test_request_handler_db_meta_writer.cpp
index 822bbc42a..af3070b79 100644
--- a/receiver/unittests/test_request_handler_db_meta_writer.cpp
+++ b/receiver/unittests/test_request_handler_db_meta_writer.cpp
@@ -73,7 +73,7 @@ class DbMetaWriterHandlerTests : public Test {
request_header.data_id = 0;
handler.db_client__ = std::unique_ptr<asapo::Database> {&mock_db};
handler.log__ = &mock_logger;
- mock_request.reset(new NiceMock<MockRequest> {request_header, 1, ""});
+ mock_request.reset(new NiceMock<MockRequest> {request_header, 1, "", nullptr});
ON_CALL(*mock_request, GetBeamtimeId()).WillByDefault(ReturnRef(expected_beamtime_id));
}
void TearDown() override {
diff --git a/receiver/unittests/test_request_handler_db_writer.cpp b/receiver/unittests/test_request_handler_db_writer.cpp
index 8c6454049..4f82656c7 100644
--- a/receiver/unittests/test_request_handler_db_writer.cpp
+++ b/receiver/unittests/test_request_handler_db_writer.cpp
@@ -1,5 +1,6 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
+#include <database/db_error.h>
#include "unittests/MockIO.h"
#include "unittests/MockDatabase.h"
@@ -34,6 +35,7 @@ using ::testing::InSequence;
using ::testing::SetArgPointee;
using ::testing::AllOf;
using ::testing::HasSubstr;
+using ::testing::AtLeast;
using ::asapo::Error;
@@ -82,13 +84,15 @@ class DbWriterHandlerTests : public Test {
uint64_t expected_subset_id = 15;
uint64_t expected_subset_size = 2;
uint64_t expected_custom_data[asapo::kNCustomParams] {0, expected_subset_id, expected_subset_size};
+ asapo::MockHandlerDbCheckRequest mock_db_check_handler{asapo::kDBDataCollectionNamePrefix};
+
void SetUp() override {
GenericRequestHeader request_header;
request_header.data_id = 2;
request_header.op_code = asapo::Opcode::kOpcodeTransferData;
handler.db_client__ = std::unique_ptr<asapo::Database> {&mock_db};
handler.log__ = &mock_logger;
- mock_request.reset(new NiceMock<MockRequest> {request_header, 1, ""});
+ mock_request.reset(new NiceMock<MockRequest> {request_header, 1, "", &mock_db_check_handler});
config.database_uri = "127.0.0.1:27017";
config.advertise_ip = expected_host_ip;
config.dataserver.listen_port = expected_port;
@@ -97,7 +101,8 @@ class DbWriterHandlerTests : public Test {
ON_CALL(*mock_request, GetBeamtimeId()).WillByDefault(ReturnRef(expected_beamtime_id));
}
void ExpectRequestParams(asapo::Opcode op_code, const std::string& stream);
-
+ void ExpectLogger();
+ void ExpectDuplicatedID();
FileInfo PrepareFileInfo();
void TearDown() override {
handler.db_client__.release();
@@ -119,6 +124,11 @@ MATCHER_P(CompareFileInfo, file, "") {
void DbWriterHandlerTests::ExpectRequestParams(asapo::Opcode op_code, const std::string& stream) {
+
+ EXPECT_CALL(*mock_request, WasAlreadyProcessed())
+ .WillOnce(Return(false))
+ ;
+
EXPECT_CALL(*mock_request, GetBeamtimeId())
.WillOnce(ReturnRef(expected_beamtime_id))
;
@@ -155,8 +165,8 @@ void DbWriterHandlerTests::ExpectRequestParams(asapo::Opcode op_code, const std:
.WillOnce(ReturnRef(expected_metadata))
;
- EXPECT_CALL(*mock_request, GetDataID())
- .WillOnce(Return(expected_id))
+ EXPECT_CALL(*mock_request, GetDataID()).Times(AtLeast(1)).WillRepeatedly
+ (Return(expected_id))
;
EXPECT_CALL(*mock_request, GetOpCode())
@@ -183,15 +193,7 @@ FileInfo DbWriterHandlerTests::PrepareFileInfo() {
file_info.metadata = expected_metadata;
return file_info;
}
-
-TEST_F(DbWriterHandlerTests, CallsInsert) {
-
- ExpectRequestParams(asapo::Opcode::kOpcodeTransferData, expected_stream);
- auto file_info = PrepareFileInfo();
-
- EXPECT_CALL(mock_db, Insert_t(expected_collection_name, CompareFileInfo(file_info), _)).
- WillOnce(testing::Return(nullptr));
-
+void DbWriterHandlerTests::ExpectLogger() {
EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("insert record"),
HasSubstr(config.database_uri),
HasSubstr(expected_beamtime_id),
@@ -201,6 +203,17 @@ TEST_F(DbWriterHandlerTests, CallsInsert) {
)
);
+}
+
+TEST_F(DbWriterHandlerTests, CallsInsert) {
+
+ ExpectRequestParams(asapo::Opcode::kOpcodeTransferData, expected_stream);
+ auto file_info = PrepareFileInfo();
+
+ EXPECT_CALL(mock_db, Insert_t(expected_collection_name, CompareFileInfo(file_info), false)).
+ WillOnce(testing::Return(nullptr));
+ ExpectLogger();
+
handler.ProcessRequest(mock_request.get());
}
@@ -211,19 +224,61 @@ TEST_F(DbWriterHandlerTests, CallsInsertSubset) {
EXPECT_CALL(mock_db, InsertAsSubset_t(expected_collection_name, CompareFileInfo(file_info), expected_subset_id,
- expected_subset_size, _)).
+ expected_subset_size, false)).
WillOnce(testing::Return(nullptr));
-
- EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("insert record"),
- HasSubstr(config.database_uri),
- HasSubstr(expected_beamtime_id),
- HasSubstr(expected_collection_name)
- )
- )
- );
+ ExpectLogger();
handler.ProcessRequest(mock_request.get());
}
+void DbWriterHandlerTests::ExpectDuplicatedID() {
+ ExpectRequestParams(asapo::Opcode::kOpcodeTransferData, expected_stream);
+ auto file_info = PrepareFileInfo();
+
+ EXPECT_CALL(mock_db, Insert_t(expected_collection_name, CompareFileInfo(file_info), false)).
+ WillOnce(testing::Return(asapo::DBErrorTemplates::kDuplicateID.Generate().release()));
+}
+
+TEST_F(DbWriterHandlerTests, SkipIfWasAlreadyProcessed) {
+ EXPECT_CALL(*mock_request, WasAlreadyProcessed())
+ .WillOnce(Return(true));
+
+ EXPECT_CALL(*mock_request, GetBeamtimeId()).Times(0);
+
+ auto err = handler.ProcessRequest(mock_request.get());
+ ASSERT_THAT(err, Eq(nullptr));
+
+}
+
+TEST_F(DbWriterHandlerTests, DuplicatedRequest_SameRecord) {
+ ExpectDuplicatedID();
+
+ EXPECT_CALL(*mock_request, SetWarningMessage(HasSubstr("duplicate record")));
+ EXPECT_CALL(*mock_request, CheckForDuplicates_t())
+ .WillOnce(
+ Return(asapo::ReceiverErrorTemplates::kWarningDuplicatedRequest.Generate().release())
+ );
+
+ EXPECT_CALL(mock_logger, Warning(HasSubstr("ignoring")));
+
+ auto err = handler.ProcessRequest(mock_request.get());
+ ASSERT_THAT(err, Eq(nullptr));
+
+}
+
+TEST_F(DbWriterHandlerTests, DuplicatedRequest_DifferentRecord) {
+ ExpectDuplicatedID();
+
+ EXPECT_CALL(*mock_request, CheckForDuplicates_t())
+ .WillOnce(
+ Return(asapo::ReceiverErrorTemplates::kBadRequest.Generate().release())
+ );
+
+ auto err = handler.ProcessRequest(mock_request.get());
+ ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kBadRequest));
+
+}
+
+
}
\ No newline at end of file
diff --git a/receiver/unittests/test_request_handler_file_process.cpp b/receiver/unittests/test_request_handler_file_process.cpp
new file mode 100644
index 000000000..0978d571e
--- /dev/null
+++ b/receiver/unittests/test_request_handler_file_process.cpp
@@ -0,0 +1,136 @@
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
+#include "unittests/MockIO.h"
+#include "unittests/MockLogger.h"
+
+#include "../src/receiver_error.h"
+#include "../src/request.h"
+#include "../src/request_handler.h"
+#include "../src/request_handler_file_process.h"
+#include "common/networking.h"
+#include "mock_receiver_config.h"
+#include "preprocessor/definitions.h"
+
+#include "receiver_mocking.h"
+
+using ::testing::Test;
+using ::testing::Return;
+using ::testing::ReturnRef;
+using ::testing::_;
+using ::testing::DoAll;
+using ::testing::SetArgReferee;
+using ::testing::Gt;
+using ::testing::Eq;
+using ::testing::Ne;
+using ::testing::Mock;
+using ::testing::NiceMock;
+using ::testing::InSequence;
+using ::testing::SetArgPointee;
+using ::testing::AllOf;
+using ::testing::HasSubstr;
+
+
+using ::asapo::Error;
+using ::asapo::ErrorInterface;
+using ::asapo::FileDescriptor;
+using ::asapo::SocketDescriptor;
+using ::asapo::MockIO;
+using asapo::Request;
+using asapo::RequestHandlerFileProcess;
+using ::asapo::GenericRequestHeader;
+using asapo::MockRequest;
+
+namespace {
+
+TEST(FileWrite, Constructor) {
+ RequestHandlerFileProcess handler(nullptr);
+ ASSERT_THAT(dynamic_cast<asapo::IO*>(handler.io__.get()), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<const asapo::AbstractLogger*>(handler.log__), Ne(nullptr));
+}
+
+class FileWriteHandlerTests : public Test {
+ public:
+ asapo::MockFileProcessor mock_file_processor;
+ RequestHandlerFileProcess handler{&mock_file_processor};
+ NiceMock<MockIO> mock_io;
+ std::unique_ptr<MockRequest> mock_request;
+ NiceMock<asapo::MockLogger> mock_logger;
+ void ExpecFileProcess(const asapo::SimpleErrorTemplate* error_template, bool overwrite);
+ void SetUp() override {
+ GenericRequestHeader request_header;
+ mock_request.reset(new MockRequest{request_header, 1, "", nullptr});
+ handler.io__ = std::unique_ptr<asapo::IO> {&mock_io};
+ handler.log__ = &mock_logger;
+ }
+ void TearDown() override {
+ handler.io__.release();
+ }
+
+};
+
+TEST_F(FileWriteHandlerTests, CheckStatisticEntity) {
+ auto entity = handler.GetStatisticEntity();
+ ASSERT_THAT(entity, Eq(asapo::StatisticEntity::kDisk));
+}
+
+void FileWriteHandlerTests::ExpecFileProcess(const asapo::SimpleErrorTemplate* error_template, bool overwrite) {
+ EXPECT_CALL(mock_file_processor, ProcessFile_t(mock_request.get(), overwrite))
+ .WillOnce(
+ Return(error_template == nullptr ? nullptr : error_template->Generate().release()));
+}
+
+TEST_F(FileWriteHandlerTests, FileAlreadyExists_NoRecordInDb) {
+ EXPECT_CALL(*mock_request, SetWarningMessage(HasSubstr("overwritten")));
+ EXPECT_CALL(*mock_request, CheckForDuplicates_t())
+ .WillOnce(
+ Return(nullptr)
+ );
+
+ EXPECT_CALL(mock_logger, Warning(HasSubstr("overwriting")));
+
+ ExpecFileProcess(&asapo::IOErrorTemplates::kFileAlreadyExists, false);
+ ExpecFileProcess(nullptr, true);
+
+ auto err = handler.ProcessRequest(mock_request.get());
+
+ ASSERT_THAT(err, Eq(nullptr));
+}
+
+TEST_F(FileWriteHandlerTests, FileAlreadyExists_DuplicatedRecordInDb) {
+
+ EXPECT_CALL(*mock_request, SetWarningMessage(HasSubstr("ignore")));
+ EXPECT_CALL(*mock_request, SetAlreadyProcessedFlag());
+ EXPECT_CALL(mock_logger, Warning(HasSubstr("duplicated")));
+ EXPECT_CALL(*mock_request, GetDataID()).WillOnce(Return(1));
+
+ ExpecFileProcess(&asapo::IOErrorTemplates::kFileAlreadyExists, false);
+
+ EXPECT_CALL(*mock_request, CheckForDuplicates_t())
+ .WillOnce(
+ Return(asapo::ReceiverErrorTemplates::kWarningDuplicatedRequest.Generate().release())
+ );
+
+ auto err = handler.ProcessRequest(mock_request.get());
+
+ ASSERT_THAT(err, Eq(nullptr));
+}
+
+TEST_F(FileWriteHandlerTests, FileAlreadyExists_DifferentRecordInDb) {
+
+ ExpecFileProcess(&asapo::IOErrorTemplates::kFileAlreadyExists, false);
+
+ EXPECT_CALL(*mock_request, CheckForDuplicates_t())
+ .WillOnce(
+ Return(asapo::ReceiverErrorTemplates::kBadRequest.Generate().release())
+ );
+
+
+ auto err = handler.ProcessRequest(mock_request.get());
+
+ ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kBadRequest));
+}
+
+
+
+}
\ No newline at end of file
diff --git a/receiver/unittests/test_request_handler_receive_data.cpp b/receiver/unittests/test_request_handler_receive_data.cpp
index 1b852b766..9991f5c53 100644
--- a/receiver/unittests/test_request_handler_receive_data.cpp
+++ b/receiver/unittests/test_request_handler_receive_data.cpp
@@ -75,7 +75,7 @@ class ReceiveDataHandlerTests : public Test {
generic_request_header.op_code = expected_op_code;
generic_request_header.custom_data[asapo::kPosIngestMode] = asapo::kDefaultIngestMode;
strcpy(generic_request_header.message, expected_request_message);
- request.reset(new Request{generic_request_header, socket_fd_, expected_origin_uri, nullptr});
+ request.reset(new Request{generic_request_header, socket_fd_, expected_origin_uri, nullptr, nullptr});
handler.io__ = std::unique_ptr<asapo::IO> {&mock_io};
handler.log__ = &mock_logger;
}
@@ -114,7 +114,7 @@ TEST_F(ReceiveDataHandlerTests, CheckStatisticEntity) {
TEST_F(ReceiveDataHandlerTests, HandleDoesNotReceiveEmptyData) {
generic_request_header.data_size = 0;
- request.reset(new Request{generic_request_header, socket_fd_, "", nullptr});
+ request.reset(new Request{generic_request_header, socket_fd_, "", nullptr, nullptr});
EXPECT_CALL(mock_io, Receive_t(_, _, _, _)).Times(0);
@@ -127,7 +127,7 @@ TEST_F(ReceiveDataHandlerTests, HandleDoesNotReceiveEmptyData) {
TEST_F(ReceiveDataHandlerTests, HandleDoesNotReceiveDataWhenMetadataOnlyWasSent) {
generic_request_header.data_size = 10;
generic_request_header.custom_data[asapo::kPosIngestMode] = asapo::kTransferMetaDataOnly;
- request.reset(new Request{generic_request_header, socket_fd_, "", nullptr});
+ request.reset(new Request{generic_request_header, socket_fd_, "", nullptr, nullptr});
auto err = handler.ProcessRequest(request.get());
diff --git a/receiver/unittests/test_request_handler_receive_metadata.cpp b/receiver/unittests/test_request_handler_receive_metadata.cpp
index 92d526504..029d586c6 100644
--- a/receiver/unittests/test_request_handler_receive_metadata.cpp
+++ b/receiver/unittests/test_request_handler_receive_metadata.cpp
@@ -73,7 +73,7 @@ class ReceiveMetaDataHandlerTests : public Test {
generic_request_header.meta_size = expected_metadata_size;
generic_request_header.op_code = expected_op_code;
generic_request_header.custom_data[asapo::kPosIngestMode] = asapo::kDefaultIngestMode;
- request.reset(new Request{generic_request_header, socket_fd_, expected_origin_uri, nullptr});
+ request.reset(new Request{generic_request_header, socket_fd_, expected_origin_uri, nullptr, nullptr});
handler.io__ = std::unique_ptr<asapo::IO> {&mock_io};
handler.log__ = &mock_logger;
}
diff --git a/receiver/unittests/test_requests_dispatcher.cpp b/receiver/unittests/test_requests_dispatcher.cpp
index 78b495f67..12bcc973a 100644
--- a/receiver/unittests/test_requests_dispatcher.cpp
+++ b/receiver/unittests/test_requests_dispatcher.cpp
@@ -65,7 +65,7 @@ TEST(RequestDispatcher, Constructor) {
class MockRequest: public Request {
public:
MockRequest(const GenericRequestHeader& request_header, SocketDescriptor socket_fd):
- Request(request_header, socket_fd, "", nullptr) {};
+ Request(request_header, socket_fd, "", nullptr, nullptr) {};
Error Handle(ReceiverStatistics* statistics) override {
return Error{Handle_t()};
};
@@ -259,15 +259,16 @@ TEST_F(RequestsDispatcherTests, OkProcessRequestSendOK) {
}
-TEST_F(RequestsDispatcherTests, ProcessRequestReturnsAlreadyExist) {
- MockHandleRequest(true, asapo::IOErrorTemplates::kFileAlreadyExists.Generate());
+TEST_F(RequestsDispatcherTests, ProcessRequestReturnsOkWithWarning) {
+ MockHandleRequest(false);
MockSendResponse(&response, false);
+ request->SetWarningMessage("duplicate");
auto err = dispatcher->ProcessRequest(request);
- ASSERT_THAT(err, Eq(asapo::IOErrorTemplates::kFileAlreadyExists));
- ASSERT_THAT(response.error_code, Eq(asapo::kNetErrorFileIdAlreadyInUse));
- ASSERT_THAT(std::string(response.message), HasSubstr(std::string("kFileAlreadyExists")));
+ ASSERT_THAT(err, Eq(nullptr));
+ ASSERT_THAT(response.error_code, Eq(asapo::kNetErrorWarning));
+ ASSERT_THAT(std::string(response.message), HasSubstr(std::string("duplicate")));
}
TEST_F(RequestsDispatcherTests, ProcessRequestReturnsAuthorizationFailure) {
diff --git a/receiver/unittests/test_request_handler_file_write.cpp b/receiver/unittests/test_write_file_processor.cpp
similarity index 58%
rename from receiver/unittests/test_request_handler_file_write.cpp
rename to receiver/unittests/test_write_file_processor.cpp
index 7a290e771..592de637d 100644
--- a/receiver/unittests/test_request_handler_file_write.cpp
+++ b/receiver/unittests/test_write_file_processor.cpp
@@ -4,13 +4,10 @@
#include "unittests/MockIO.h"
#include "unittests/MockLogger.h"
-#include "../src/receiver_error.h"
-#include "../src/request.h"
-#include "../src/request_handler.h"
-#include "../src/request_handler_file_write.h"
+#include "../src/write_file_processor.h"
#include "common/networking.h"
-#include "mock_receiver_config.h"
#include "preprocessor/definitions.h"
+#include "mock_receiver_config.h"
#include "receiver_mocking.h"
@@ -37,21 +34,22 @@ using ::asapo::FileDescriptor;
using ::asapo::SocketDescriptor;
using ::asapo::MockIO;
using asapo::Request;
-using asapo::RequestHandlerFileWrite;
+using asapo::WriteFileProcessor;
using ::asapo::GenericRequestHeader;
using asapo::MockRequest;
namespace {
-TEST(FileWrite, Constructor) {
- RequestHandlerFileWrite handler;
- ASSERT_THAT(dynamic_cast<asapo::IO*>(handler.io__.get()), Ne(nullptr));
- ASSERT_THAT(dynamic_cast<const asapo::AbstractLogger*>(handler.log__), Ne(nullptr));
+TEST(WriteFileProcessor, Constructor) {
+ WriteFileProcessor processor;
+ ASSERT_THAT(dynamic_cast<asapo::IO*>(processor.io__.get()), Ne(nullptr));
+ ASSERT_THAT(dynamic_cast<const asapo::AbstractLogger*>(processor.log__), Ne(nullptr));
+
}
-class FileWriteHandlerTests : public Test {
+class WriteFileProcessorTests : public Test {
public:
- RequestHandlerFileWrite handler;
+ WriteFileProcessor processor;
NiceMock<MockIO> mock_io;
std::unique_ptr<MockRequest> mock_request;
NiceMock<asapo::MockLogger> mock_logger;
@@ -61,6 +59,7 @@ class FileWriteHandlerTests : public Test {
std::string expected_facility = "facility";
std::string expected_year = "2020";
uint64_t expected_file_size = 10;
+ bool expected_overwrite = false;
std::string expected_root_folder = "root_folder";
std::string expected_full_path = expected_root_folder + asapo::kPathSeparator + expected_facility +
asapo::kPathSeparator + "gpfs" +
@@ -68,72 +67,69 @@ class FileWriteHandlerTests : public Test {
asapo::kPathSeparator + expected_year +
asapo::kPathSeparator + "data" +
asapo::kPathSeparator + expected_beamtime_id;
- void MockRequestData();
+ void ExpectFileWrite(const asapo::SimpleErrorTemplate* error_template);
+ void MockRequestData(int times = 1);
void SetUp() override {
GenericRequestHeader request_header;
request_header.data_id = 2;
- mock_request.reset(new MockRequest{request_header, 1, ""});
- handler.io__ = std::unique_ptr<asapo::IO> {&mock_io};
- handler.log__ = &mock_logger;
+ asapo::ReceiverConfig test_config;
+ test_config.root_folder = expected_root_folder;
+ asapo::SetReceiverConfig(test_config, "none");
+ processor.log__ = &mock_logger;
+ mock_request.reset(new MockRequest{request_header, 1, "", nullptr});
+ processor.io__ = std::unique_ptr<asapo::IO> {&mock_io};
}
void TearDown() override {
- handler.io__.release();
+ processor.io__.release();
}
};
-TEST_F(FileWriteHandlerTests, CheckStatisticEntity) {
- auto entity = handler.GetStatisticEntity();
- ASSERT_THAT(entity, Eq(asapo::StatisticEntity::kDisk));
-}
-
-TEST_F(FileWriteHandlerTests, ErrorWhenZeroFileSize) {
+TEST_F(WriteFileProcessorTests, ErrorWhenZeroFileSize) {
EXPECT_CALL(*mock_request, GetDataSize())
.WillOnce(Return(0));
- auto err = handler.ProcessRequest(mock_request.get());
+ auto err = processor.ProcessFile(mock_request.get(), false);
ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kBadRequest));
}
-void FileWriteHandlerTests::MockRequestData() {
- EXPECT_CALL(*mock_request, GetDataSize())
- .WillOnce(Return(expected_file_size));
+void WriteFileProcessorTests::MockRequestData(int times) {
+ EXPECT_CALL(*mock_request, GetDataSize()).Times(times)
+ .WillRepeatedly(Return(expected_file_size));
- EXPECT_CALL(*mock_request, GetData())
- .WillOnce(Return(nullptr));
+ EXPECT_CALL(*mock_request, GetData()).Times(times)
+ .WillRepeatedly(Return(nullptr));
- EXPECT_CALL(*mock_request, GetFullPath(expected_root_folder))
- .WillOnce(Return(expected_full_path));
+ EXPECT_CALL(*mock_request, GetFullPath(expected_root_folder)).Times(times)
+ .WillRepeatedly(Return(expected_full_path));
- EXPECT_CALL(*mock_request, GetFileName())
- .WillOnce(Return(expected_file_name));
+ EXPECT_CALL(*mock_request, GetFileName()).Times(times)
+ .WillRepeatedly(Return(expected_file_name));
}
-TEST_F(FileWriteHandlerTests, CallsWriteFile) {
- asapo::ReceiverConfig test_config;
- test_config.root_folder = expected_root_folder;
-
- asapo::SetReceiverConfig(test_config, "none");
+void WriteFileProcessorTests::ExpectFileWrite(const asapo::SimpleErrorTemplate* error_template) {
+ EXPECT_CALL(mock_io, WriteDataToFile_t(expected_full_path, expected_file_name, _, expected_file_size, true,
+ expected_overwrite))
+ .WillOnce(
+ Return(error_template == nullptr ? nullptr : error_template->Generate().release()));
+}
+TEST_F(WriteFileProcessorTests, CallsWriteFile) {
MockRequestData();
- EXPECT_CALL(mock_io, WriteDataToFile_t(expected_full_path, expected_file_name, _, expected_file_size, true))
- .WillOnce(
- Return(asapo::IOErrorTemplates::kUnknownIOError.Generate().release())
- );
+ ExpectFileWrite(&asapo::IOErrorTemplates::kUnknownIOError);
- auto err = handler.ProcessRequest(mock_request.get());
+ auto err = processor.ProcessFile(mock_request.get(), expected_overwrite);
ASSERT_THAT(err, Eq(asapo::IOErrorTemplates::kUnknownIOError));
}
-TEST_F(FileWriteHandlerTests, WritesToLog) {
+TEST_F(WriteFileProcessorTests, WritesToLog) {
MockRequestData();
- EXPECT_CALL(mock_io, WriteDataToFile_t(_, _, _, _, _))
- .WillOnce(Return(nullptr));
+ ExpectFileWrite(nullptr);
EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("saved file"),
HasSubstr(expected_file_name),
@@ -144,7 +140,11 @@ TEST_F(FileWriteHandlerTests, WritesToLog) {
)
)
);
- handler.ProcessRequest(mock_request.get());
+ auto err = processor.ProcessFile(mock_request.get(), expected_overwrite);
+ ASSERT_THAT(err, Eq(nullptr));
}
+
+
+
}
\ No newline at end of file
diff --git a/tests/automatic/bug_fixes/error-sending-data-using-callback-method/bugfix_callback.py b/tests/automatic/bug_fixes/error-sending-data-using-callback-method/bugfix_callback.py
index 5b8d930cb..8f8864647 100644
--- a/tests/automatic/bug_fixes/error-sending-data-using-callback-method/bugfix_callback.py
+++ b/tests/automatic/bug_fixes/error-sending-data-using-callback-method/bugfix_callback.py
@@ -25,7 +25,7 @@ class AsapoSender:
def _callback(self, header, err):
print ("hello self callback")
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads)
+producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads, 600)
producer.set_log_level("debug")
sender = AsapoSender(producer)
diff --git a/tests/automatic/full_chain/send_recv_substreams/send_recv_substreams.cpp b/tests/automatic/full_chain/send_recv_substreams/send_recv_substreams.cpp
index fb00a9997..d4eb41a50 100644
--- a/tests/automatic/full_chain/send_recv_substreams/send_recv_substreams.cpp
+++ b/tests/automatic/full_chain/send_recv_substreams/send_recv_substreams.cpp
@@ -56,7 +56,7 @@ ProducerPtr CreateProducer(const Args& args) {
asapo::Error err;
auto producer = asapo::Producer::Create(args.server, 1,
asapo::RequestHandlerType::kTcp,
- asapo::SourceCredentials{args.beamtime_id, "", args.token }, &err);
+ asapo::SourceCredentials{args.beamtime_id, "", args.token }, 60, &err);
if(err) {
std::cerr << "Cannot start producer. ProducerError: " << err << std::endl;
exit(EXIT_FAILURE);
diff --git a/tests/automatic/full_chain/send_recv_substreams_python/send_recv_substreams.py b/tests/automatic/full_chain/send_recv_substreams_python/send_recv_substreams.py
index a8081ed28..f7c959370 100644
--- a/tests/automatic/full_chain/send_recv_substreams_python/send_recv_substreams.py
+++ b/tests/automatic/full_chain/send_recv_substreams_python/send_recv_substreams.py
@@ -28,7 +28,7 @@ def callback(header,err):
source, beamtime, token = sys.argv[1:]
broker = asapo_consumer.create_server_broker(source,".", beamtime,"",token,timeout)
-producer = asapo_producer.create_producer(source,beamtime, "", token, 1)
+producer = asapo_producer.create_producer(source,beamtime, "", token, 1, 600)
producer.set_log_level("debug")
group_id = broker.generate_group_id()
diff --git a/tests/automatic/mongo_db/CMakeLists.txt b/tests/automatic/mongo_db/CMakeLists.txt
index 1fcf5373d..e02447414 100644
--- a/tests/automatic/mongo_db/CMakeLists.txt
+++ b/tests/automatic/mongo_db/CMakeLists.txt
@@ -1,7 +1,7 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.7) # needed for fixtures
add_subdirectory(connect)
-add_subdirectory(insert)
-add_subdirectory(insert_dataset)
+add_subdirectory(insert_retrieve)
+add_subdirectory(insert_retrieve_dataset)
add_subdirectory(upsert)
diff --git a/tests/automatic/mongo_db/insert/CMakeLists.txt b/tests/automatic/mongo_db/insert_retrieve/CMakeLists.txt
similarity index 88%
rename from tests/automatic/mongo_db/insert/CMakeLists.txt
rename to tests/automatic/mongo_db/insert_retrieve/CMakeLists.txt
index ed386c91e..d1eef84c9 100644
--- a/tests/automatic/mongo_db/insert/CMakeLists.txt
+++ b/tests/automatic/mongo_db/insert_retrieve/CMakeLists.txt
@@ -1,5 +1,5 @@
-set(TARGET_NAME insert_mongodb)
-set(SOURCE_FILES insert_mongodb.cpp)
+set(TARGET_NAME insert_retrieve_mongodb)
+set(SOURCE_FILES insert_retrieve_mongodb.cpp)
################################
diff --git a/tests/automatic/mongo_db/insert/cleanup_linux.sh b/tests/automatic/mongo_db/insert_retrieve/cleanup_linux.sh
similarity index 100%
rename from tests/automatic/mongo_db/insert/cleanup_linux.sh
rename to tests/automatic/mongo_db/insert_retrieve/cleanup_linux.sh
diff --git a/tests/automatic/mongo_db/insert/cleanup_windows.bat b/tests/automatic/mongo_db/insert_retrieve/cleanup_windows.bat
similarity index 100%
rename from tests/automatic/mongo_db/insert/cleanup_windows.bat
rename to tests/automatic/mongo_db/insert_retrieve/cleanup_windows.bat
diff --git a/tests/automatic/mongo_db/insert/insert_mongodb.cpp b/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp
similarity index 75%
rename from tests/automatic/mongo_db/insert/insert_mongodb.cpp
rename to tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp
index 87a525270..abeb042eb 100644
--- a/tests/automatic/mongo_db/insert/insert_mongodb.cpp
+++ b/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp
@@ -6,9 +6,11 @@
using asapo::M_AssertContains;
+using asapo::M_AssertTrue;
using asapo::Error;
+
void Assert(const Error& error, const std::string& expect) {
std::string result;
if (error == nullptr) {
@@ -58,5 +60,16 @@ int main(int argc, char* argv[]) {
Assert(err, args.keyword);
+ if (args.keyword == "OK") { // check retrieve
+ asapo::FileInfo fi_db;
+ asapo::MongoDBClient db_new;
+ db_new.Connect("127.0.0.1", "data");
+ err = db_new.GetById("test", fi.id, &fi_db);
+ M_AssertTrue(fi_db == fi, "get record from db");
+ Assert(err, "OK");
+ err = db_new.GetById("test", 0, &fi_db);
+ Assert(err, "No record");
+ }
+
return 0;
}
diff --git a/tests/automatic/mongo_db/insert_dataset/CMakeLists.txt b/tests/automatic/mongo_db/insert_retrieve_dataset/CMakeLists.txt
similarity index 84%
rename from tests/automatic/mongo_db/insert_dataset/CMakeLists.txt
rename to tests/automatic/mongo_db/insert_retrieve_dataset/CMakeLists.txt
index a7c1d9648..d4c09f197 100644
--- a/tests/automatic/mongo_db/insert_dataset/CMakeLists.txt
+++ b/tests/automatic/mongo_db/insert_retrieve_dataset/CMakeLists.txt
@@ -1,5 +1,5 @@
-set(TARGET_NAME insert_dataset_mongodb)
-set(SOURCE_FILES insert_dataset_mongodb.cpp)
+set(TARGET_NAME insert_retrieve_dataset_mongodb)
+set(SOURCE_FILES insert_retrieve_dataset_mongodb.cpp)
################################
diff --git a/tests/automatic/mongo_db/insert_dataset/cleanup_linux.sh b/tests/automatic/mongo_db/insert_retrieve_dataset/cleanup_linux.sh
similarity index 100%
rename from tests/automatic/mongo_db/insert_dataset/cleanup_linux.sh
rename to tests/automatic/mongo_db/insert_retrieve_dataset/cleanup_linux.sh
diff --git a/tests/automatic/mongo_db/insert_dataset/cleanup_windows.bat b/tests/automatic/mongo_db/insert_retrieve_dataset/cleanup_windows.bat
similarity index 100%
rename from tests/automatic/mongo_db/insert_dataset/cleanup_windows.bat
rename to tests/automatic/mongo_db/insert_retrieve_dataset/cleanup_windows.bat
diff --git a/tests/automatic/mongo_db/insert_dataset/insert_dataset_mongodb.cpp b/tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp
similarity index 80%
rename from tests/automatic/mongo_db/insert_dataset/insert_dataset_mongodb.cpp
rename to tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp
index eea1a1a84..944bfed46 100644
--- a/tests/automatic/mongo_db/insert_dataset/insert_dataset_mongodb.cpp
+++ b/tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp
@@ -6,6 +6,8 @@
using asapo::M_AssertContains;
+using asapo::M_AssertTrue;
+
using asapo::Error;
@@ -44,7 +46,7 @@ int main(int argc, char* argv[]) {
fi.modify_date = std::chrono::system_clock::now();
fi.buf_id = 18446744073709551615ull;
fi.source = "host:1234";
- fi.id = 1;
+ fi.id = 10;
uint64_t subset_size = 2;
@@ -65,5 +67,15 @@ int main(int argc, char* argv[]) {
Assert(err, args.keyword);
+ if (args.keyword == "OK") { // check retrieve
+ asapo::FileInfo fi_db;
+ err = db.GetDataSetById("test", subset_id, fi.id, &fi_db);
+ M_AssertTrue(fi_db == fi, "get record from db");
+ Assert(err, "OK");
+ err = db.GetDataSetById("test", 0, 0, &fi_db);
+ Assert(err, "No record");
+ }
+
+
return 0;
}
diff --git a/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp b/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp
index 044ca7c93..2b72594b7 100644
--- a/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp
+++ b/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp
@@ -69,7 +69,7 @@ std::unique_ptr<asapo::Producer> CreateProducer(const Args& args) {
auto producer = asapo::Producer::Create(args.receiver_address, 1,
args.mode == 0 ? asapo::RequestHandlerType::kTcp
: asapo::RequestHandlerType::kFilesystem,
- asapo::SourceCredentials{args.beamtime_id, "", ""}, &err);
+ asapo::SourceCredentials{args.beamtime_id, "", ""}, 60, &err);
if (err) {
std::cerr << "Cannot start producer. ProducerError: " << err << std::endl;
exit(EXIT_FAILURE);
diff --git a/tests/automatic/producer/python_api/check_linux.sh b/tests/automatic/producer/python_api/check_linux.sh
index 3a332e03d..6cd5fa908 100644
--- a/tests/automatic/producer/python_api/check_linux.sh
+++ b/tests/automatic/producer/python_api/check_linux.sh
@@ -41,5 +41,7 @@ sleep 1
$1 $3 $stream $beamtime_id "127.0.0.1:8400" > out || cat out
cat out
-cat out | grep "successfuly sent" | wc -l | grep 8
-cat out | grep "local i/o error"
\ No newline at end of file
+cat out | grep "successfuly sent" | wc -l | grep 10
+cat out | grep "local i/o error"
+cat out | grep "already have record with same id" | wc -l | grep 4
+cat out | grep "duplicate" | wc -l | grep 4
diff --git a/tests/automatic/producer/python_api/check_windows.bat b/tests/automatic/producer/python_api/check_windows.bat
index 01c838078..6d8d3ddb2 100644
--- a/tests/automatic/producer/python_api/check_windows.bat
+++ b/tests/automatic/producer/python_api/check_windows.bat
@@ -4,7 +4,7 @@ SET beamline=test
SET stream=python
SET receiver_root_folder=c:\tmp\asapo\receiver\files
SET receiver_folder="%receiver_root_folder%\test_facility\gpfs\%beamline%\2019\data\%beamtime_id%"
-SET dbname = %beamtime_id%_%stream%
+SET dbname=%beamtime_id%_%stream%
echo db.%dbname%.insert({dummy:1})" | %mongo_exe% %dbname%
@@ -27,7 +27,14 @@ set PYTHONPATH=%2
type out
set NUM=0
for /F %%N in ('find /C "successfuly sent" ^< "out"') do set NUM=%%N
-echo %NUM% | findstr 8 || goto error
+echo %NUM% | findstr 10 || goto error
+
+for /F %%N in ('find /C "already have record with same id" ^< "out"') do set NUM=%%N
+echo %NUM% | findstr 4 || goto error
+
+
+for /F %%N in ('find /C "duplicate" ^< "out"') do set NUM=%%N
+echo %NUM% | findstr 4 || goto error
goto :clean
diff --git a/tests/automatic/producer/python_api/producer_api.py b/tests/automatic/producer/python_api/producer_api.py
index e6905705c..084af5568 100644
--- a/tests/automatic/producer/python_api/producer_api.py
+++ b/tests/automatic/producer/python_api/producer_api.py
@@ -16,21 +16,23 @@ nthreads = 8
def callback(header,err):
lock.acquire() # to print
- if err is not None:
+ if isinstance(err,asapo_producer.AsapoServerWarning):
+ print("successfuly sent, but with warning from server: ",header,err)
+ elif err is not None:
print("could not sent: ",header,err)
else:
print ("successfuly sent: ",header)
lock.release()
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads)
+producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads,60)
-producer.set_log_level("info")
+producer.set_log_level("debug")
#send single file
producer.send_file(1, local_path = "./file1", exposed_path = stream+"/"+"file1", user_meta = '{"test_key":"test_val"}', callback = callback)
#send single file without callback
-producer.send_file(1, local_path = "./file1", exposed_path = stream+"/"+"file1", user_meta = '{"test_key":"test_val"}',callback=None)
+producer.send_file(10, local_path = "./file1", exposed_path = stream+"/"+"file10", user_meta = '{"test_key":"test_val"}',callback=None)
#send subsets
producer.send_file(2, local_path = "./file1", exposed_path = stream+"/"+"file2",subset=(2,2),user_meta = '{"test_key":"test_val"}', callback = callback)
@@ -62,8 +64,8 @@ producer.send_data(8, stream+"/"+"file8",x,
ingest_mode = asapo_producer.DEFAULT_INGEST_MODE, callback = callback)
try:
- x = x.T
- producer.send_data(8, stream+"/"+"file8",x,
+ x = x.T
+ producer.send_data(8, stream+"/"+"file8",x,
ingest_mode = asapo_producer.DEFAULT_INGEST_MODE, callback = callback)
except asapo_producer.AsapoWrongInputError as e:
print(e)
@@ -72,16 +74,29 @@ else:
sys.exit(1)
+#send single file once again
+producer.send_file(1, local_path = "./file1", exposed_path = stream+"/"+"file1", user_meta = '{"test_key":"test_val"}', callback = callback)
+#send metadata only once again
+producer.send_data(6, stream+"/"+"file7",None,
+ ingest_mode = asapo_producer.INGEST_MODE_TRANSFER_METADATA_ONLY, callback = callback)
+
+#send same id different data
+producer.send_file(1, local_path = "./file1", exposed_path = stream+"/"+"file1", user_meta = '{"test_key1":"test_val"}', callback = callback)#send same id different data
+producer.send_data(6, stream+"/"+"file8",None,
+ ingest_mode = asapo_producer.INGEST_MODE_TRANSFER_METADATA_ONLY, callback = callback)
+
+
+
producer.wait_requests_finished(50000)
n = producer.get_requests_queue_size()
if n!=0:
- print("number of remaining requests should be zero, got ",n)
- sys.exit(1)
+ print("number of remaining requests should be zero, got ",n)
+ sys.exit(1)
# create with error
try:
- producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, 0)
+ producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, 0,0)
except asapo_producer.AsapoWrongInputError as e:
print(e)
else:
@@ -91,4 +106,3 @@ else:
-
diff --git a/tests/automatic/system_io/ip_tcp_network/client_serv/ip_tcp_network.cpp b/tests/automatic/system_io/ip_tcp_network/client_serv/ip_tcp_network.cpp
index 08149f98a..31517dec8 100644
--- a/tests/automatic/system_io/ip_tcp_network/client_serv/ip_tcp_network.cpp
+++ b/tests/automatic/system_io/ip_tcp_network/client_serv/ip_tcp_network.cpp
@@ -17,8 +17,7 @@ using asapo::M_AssertEq;
using namespace std::chrono;
-static const std::unique_ptr<asapo::IO>
-io(asapo::GenerateDefaultIO());
+static const std::unique_ptr<asapo::IO> io(asapo::GenerateDefaultIO());
static const std::string kListenAddress = "127.0.0.1:60123";
static std::promise<void> kThreadStarted;
static const int kNumberOfChecks = 2;
@@ -79,7 +78,7 @@ std::unique_ptr<std::thread> CreateEchoServerThread() {
io->Send(client_fd, buffer.get(), received, &err);
ExitIfErrIsNotOk(&err, 107);
- io->ReceiveDataToFile(client_fd, "", "received", need_to_receive_size, false);
+ io->ReceiveDataToFile(client_fd, "", "received", need_to_receive_size, false, true);
buffer = io->GetDataFromFile("received", &need_to_receive_size, &err);
io->Send(client_fd, buffer.get(), received, &err);
ExitIfErrIsNotOk(&err, 108);
diff --git a/tests/automatic/system_io/write_data_to_file/write_data_to_file.cpp b/tests/automatic/system_io/write_data_to_file/write_data_to_file.cpp
index edc3e5179..8231f5f4b 100644
--- a/tests/automatic/system_io/write_data_to_file/write_data_to_file.cpp
+++ b/tests/automatic/system_io/write_data_to_file/write_data_to_file.cpp
@@ -55,10 +55,14 @@ int main(int argc, char* argv[]) {
auto array = new uint8_t[params.length] {'1', '2', '3'};
FileData data{array};
- auto err = io->WriteDataToFile("", params.fname, data, params.length, true);
+ auto err = io->WriteDataToFile("", params.fname, data, params.length, true, true);
if (params.result == "ok") {
AssertGoodResult(io, err, data, params);
+ // check allow_overwrite works
+ auto err = io->WriteDataToFile("", params.fname, data, params.length, true, false);
+ params.message = asapo::IOErrorTemplates::kFileAlreadyExists.Generate()->Explain();
+ AssertBadResult(err, params);
} else {
AssertBadResult(err, params);
}
diff --git a/tests/manual/python_tests/producer/test.py b/tests/manual/python_tests/producer/test.py
index 83ea7b425..b5354b29d 100644
--- a/tests/manual/python_tests/producer/test.py
+++ b/tests/manual/python_tests/producer/test.py
@@ -27,7 +27,7 @@ def assert_err(err):
print(err)
sys.exit(1)
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads)
+producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads ,0)
producer.set_log_level("info")
diff --git a/tests/manual/python_tests/producer_wait_bug_mongo/test.py b/tests/manual/python_tests/producer_wait_bug_mongo/test.py
index 2725862d9..99d063b46 100644
--- a/tests/manual/python_tests/producer_wait_bug_mongo/test.py
+++ b/tests/manual/python_tests/producer_wait_bug_mongo/test.py
@@ -27,7 +27,7 @@ def assert_err(err):
print(err)
sys.exit(1)
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads)
+producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads, 600)
producer.set_log_level("debug")
diff --git a/tests/manual/python_tests/producer_wait_threads/producer_api.py b/tests/manual/python_tests/producer_wait_threads/producer_api.py
index d4de005f3..e15ce89b8 100644
--- a/tests/manual/python_tests/producer_wait_threads/producer_api.py
+++ b/tests/manual/python_tests/producer_wait_threads/producer_api.py
@@ -22,7 +22,7 @@ def callback(header,err):
print ("successfuly sent: ",header)
lock.release()
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads)
+producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads, 600)
producer.set_log_level("info")
@@ -63,7 +63,7 @@ if n!=0:
# create with error
try:
- producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, 0)
+ producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, 0, 600)
except Exception as Asapo:
print(e)
else:
diff --git a/tests/manual/python_tests/producer_wait_threads/test.py b/tests/manual/python_tests/producer_wait_threads/test.py
index 3a0206d73..70c04381e 100644
--- a/tests/manual/python_tests/producer_wait_threads/test.py
+++ b/tests/manual/python_tests/producer_wait_threads/test.py
@@ -22,7 +22,7 @@ def callback(header,err):
print ("successfuly sent: ",header)
lock.release()
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads)
+producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads, 600)
producer.set_log_level("info")
--
GitLab