diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b7122210c8609028f9b47802223379855e81761..8e1ec90109995c26b2bb9a4e8b1070a88c10b066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,11 +14,14 @@ FEATURES BREAKING CHANGES * Consumer API - get_next_dataset, get_last_dataset, get_dataset_by_id return dictionary with 'id','expected_size','content' fields, not tuple (id,content) as before * Consumer API - remove group_id argument from get_last/get_by_id/get_last_dataset/get_dataset_by_id functions -* Producer API - changed meaning of subsets (subset_id replaced with id_in_subset and this means now id of the image within a subset (e.g. module number for multi-module detector)), file_id is now a global id of a multi-set data (i.g. multi-image id) +* Producer API - changed meaning of subsets (subset_id replaced with id_in_subset and this means now id of the image within a subset (e.g. module number for multi-module detector)), message_id is now a global id of a multi-set data (i.g. multi-image id) #### renaming - general * stream -> data_source, substream -> stream -* use millisecond everywhere for timeout/delay +* use millisecond everywhere for timeout/delay +* use term `message` for blob of information we send around, rename related structs, parameters, ... #### renaming - Producer API +* SendData/send_data -> Send/send +* SendFile/send_file -> SendFromFile/send_from_file #### renaming - Consumer API * broker -> consumer diff --git a/common/cpp/include/asapo/common/data_structs.h b/common/cpp/include/asapo/common/data_structs.h index 598ac41cc494dc930294fa8e0eef03fac93063df..457e4756fe3f6acc77b674f5706d0b9bbc4b1087 100644 --- a/common/cpp/include/asapo/common/data_structs.h +++ b/common/cpp/include/asapo/common/data_structs.h @@ -1,5 +1,5 @@ -#ifndef ASAPO_FILE_INFO_H -#define ASAPO_FILE_INFO_H +#ifndef ASAPO_message_meta_H +#define ASAPO_message_meta_H #include <cinttypes> #include <chrono> @@ -22,7 +22,7 @@ uint64_t NanosecsEpochFromISODate(std::string date_time); bool TimeFromJson(const JsonStringParser& parser, const std::string& name, std::chrono::system_clock::time_point* val); -class FileInfo { +class MessageMeta { public: std::string name; std::chrono::system_clock::time_point timestamp; @@ -48,16 +48,16 @@ struct StreamInfo { using StreamInfos = std::vector<StreamInfo>; -inline bool operator==(const FileInfo& lhs, const FileInfo& rhs) { +inline bool operator==(const MessageMeta& lhs, const MessageMeta& rhs) { return (lhs.name == rhs.name && lhs.id == rhs.id && lhs.timestamp == rhs.timestamp && lhs.size == rhs.size); } -using FileData = std::unique_ptr<uint8_t[]>; +using MessageData = std::unique_ptr<uint8_t[]>; -using FileInfos = std::vector<FileInfo>; +using MessageMetas = std::vector<MessageMeta>; using IdList = std::vector<uint64_t>; @@ -65,7 +65,7 @@ using IdList = std::vector<uint64_t>; struct DataSet { uint64_t id; uint64_t expected_size; - FileInfos content; + MessageMetas content; bool SetFromJson(const std::string& json_string); }; @@ -113,4 +113,4 @@ const std::string kDefaultStream = "default"; } -#endif //ASAPO_FILE_INFO_H +#endif //ASAPO_message_meta_H diff --git a/common/cpp/include/asapo/common/networking.h b/common/cpp/include/asapo/common/networking.h index 4420df93250ea9a5302acad31a8d5daab74e7709..7a479c2a7f262acc5a3dc9a9a17f36ee0d2f6283 100644 --- a/common/cpp/include/asapo/common/networking.h +++ b/common/cpp/include/asapo/common/networking.h @@ -93,7 +93,7 @@ struct GenericNetworkResponse { }; -struct SendDataResponse : GenericNetworkResponse { +struct SendResponse : GenericNetworkResponse { }; } diff --git a/common/cpp/include/asapo/database/database.h b/common/cpp/include/asapo/database/database.h index d08b0f9bd566285ae767591c4f28590ab274396e..289a4c8c09dac53b369524d4af1a3994aed9c9f7 100644 --- a/common/cpp/include/asapo/database/database.h +++ b/common/cpp/include/asapo/database/database.h @@ -15,14 +15,14 @@ constexpr char kDBMetaCollectionName[] = "meta"; class Database { public: virtual Error Connect(const std::string& address, const std::string& database) = 0; - virtual Error Insert(const std::string& collection, const FileInfo& file, bool ignore_duplicates) const = 0; + virtual Error Insert(const std::string& collection, const MessageMeta& file, bool ignore_duplicates) const = 0; virtual Error Upsert(const std::string& collection, uint64_t id, const uint8_t* data, uint64_t size) const = 0; - virtual Error InsertAsSubset(const std::string& collection, const FileInfo& file, uint64_t subset_id, + virtual Error InsertAsSubset(const std::string& collection, const MessageMeta& file, uint64_t subset_id, uint64_t subset_size, bool ignore_duplicates) const = 0; - virtual Error GetById(const std::string& collection, uint64_t id, FileInfo* file) const = 0; - virtual Error GetDataSetById(const std::string& collection, uint64_t set_id, uint64_t id, FileInfo* file) const = 0; + virtual Error GetById(const std::string& collection, uint64_t id, MessageMeta* file) const = 0; + virtual Error GetDataSetById(const std::string& collection, uint64_t set_id, uint64_t id, MessageMeta* file) const = 0; virtual Error GetStreamInfo(const std::string& collection, StreamInfo* info) const = 0; virtual Error GetLastStream(StreamInfo* info) const = 0; virtual ~Database() = default; diff --git a/common/cpp/include/asapo/http_client/http_client.h b/common/cpp/include/asapo/http_client/http_client.h index 1d4a8b0eccbf89a88cadeea24f98ab6b8c802b59..3a41ea96b28013c655b5d51ce4abcee6c80977a4 100644 --- a/common/cpp/include/asapo/http_client/http_client.h +++ b/common/cpp/include/asapo/http_client/http_client.h @@ -15,7 +15,7 @@ class HttpClient { HttpCode* response_code, Error* err) const noexcept = 0; virtual Error Post(const std::string& uri, const std::string& cookie, - const std::string& input_data, FileData* ouput_data, + const std::string& input_data, MessageData* ouput_data, uint64_t output_data_size, HttpCode* response_code) const noexcept = 0; virtual Error Post(const std::string& uri, const std::string& cookie, diff --git a/common/cpp/include/asapo/io/io.h b/common/cpp/include/asapo/io/io.h index 1eb81c619ab78bc6dc88a3ec727bf01c27692a7e..3965506ca4d64ef7ba6b763b648fc9d69e4d0fee 100644 --- a/common/cpp/include/asapo/io/io.h +++ b/common/cpp/include/asapo/io/io.h @@ -109,19 +109,19 @@ class IO { virtual size_t Read (FileDescriptor fd, void* buf, size_t length, Error* err) const = 0; virtual size_t Write (FileDescriptor fd, const void* buf, size_t length, Error* err) const = 0; virtual Error RemoveFile(const std::string& fname) const = 0; - virtual Error WriteDataToFile (const std::string& root_folder, const std::string& fname, const FileData& data, + virtual Error WriteDataToFile (const std::string& root_folder, const std::string& fname, const MessageData& data, size_t length, bool create_directories, bool allow_ovewrite) const = 0; virtual Error WriteDataToFile (const std::string& root_folder, const std::string& fname, const uint8_t* data, size_t length, bool create_directories, bool allow_ovewrite) const = 0; virtual Error ReceiveDataToFile(SocketDescriptor socket, const std::string& root_folder, const std::string& fname, size_t length, bool create_directories, bool allow_ovewrite) const = 0; virtual void CreateNewDirectory (const std::string& directory_name, Error* err) const = 0; - virtual FileData GetDataFromFile (const std::string& fname, uint64_t* fsize, Error* err) const = 0; + virtual MessageData GetDataFromFile (const std::string& fname, uint64_t* fsize, Error* err) const = 0; virtual SubDirList GetSubDirectories(const std::string& path, Error* err) const = 0; - virtual std::vector<FileInfo> FilesInFolder (const std::string& folder, Error* err) const = 0; + virtual std::vector<MessageMeta> FilesInFolder (const std::string& folder, Error* err) const = 0; virtual std::string ReadFileToString (const std::string& fname, Error* err) const = 0; virtual Error GetLastError() const = 0; - virtual FileInfo GetFileInfo(const std::string& name, Error* err) const = 0; + virtual MessageMeta GetMessageMeta(const std::string& name, Error* err) const = 0; virtual ~IO() = default; }; diff --git a/common/cpp/include/asapo/unittests/MockDatabase.h b/common/cpp/include/asapo/unittests/MockDatabase.h index e07a4c51a71a481b904e1bf6687ccdfa4f4b604f..f3c47933d49cf6172bba3824318ec4661298d9d2 100644 --- a/common/cpp/include/asapo/unittests/MockDatabase.h +++ b/common/cpp/include/asapo/unittests/MockDatabase.h @@ -15,21 +15,21 @@ class MockDatabase : public Database { return Error{Connect_t(address, database)}; } - Error Insert(const std::string& collection, const FileInfo& file, bool ignore_duplicates) const override { + Error Insert(const std::string& collection, const MessageMeta& file, bool ignore_duplicates) const override { return Error{Insert_t(collection, file, ignore_duplicates)}; } - Error InsertAsSubset(const std::string& collection, const FileInfo& file, uint64_t subset_id, + Error InsertAsSubset(const std::string& collection, const MessageMeta& file, uint64_t subset_id, uint64_t subset_size, bool ignore_duplicates) const override { return Error{InsertAsSubset_t(collection, file, subset_id, subset_size, ignore_duplicates)}; } MOCK_METHOD2(Connect_t, ErrorInterface * (const std::string&, const std::string&)); - MOCK_CONST_METHOD3(Insert_t, ErrorInterface * (const std::string&, const FileInfo&, bool)); + MOCK_CONST_METHOD3(Insert_t, ErrorInterface * (const std::string&, const MessageMeta&, bool)); - MOCK_CONST_METHOD5(InsertAsSubset_t, ErrorInterface * (const std::string&, const FileInfo&, uint64_t, uint64_t, bool)); + MOCK_CONST_METHOD5(InsertAsSubset_t, ErrorInterface * (const std::string&, const MessageMeta&, uint64_t, uint64_t, bool)); Error Upsert(const std::string& collection, uint64_t id, const uint8_t* data, uint64_t size) const override { @@ -38,18 +38,18 @@ class MockDatabase : public Database { } MOCK_CONST_METHOD4(Upsert_t, ErrorInterface * (const std::string&, uint64_t id, const uint8_t* data, uint64_t size)); - Error GetById(const std::string& collection, uint64_t id, FileInfo* file) const override { + Error GetById(const std::string& collection, uint64_t id, MessageMeta* file) const override { return Error{GetById_t(collection, id, file)}; } - MOCK_CONST_METHOD3(GetById_t, ErrorInterface * (const std::string&, uint64_t id, FileInfo*)); + MOCK_CONST_METHOD3(GetById_t, ErrorInterface * (const std::string&, uint64_t id, MessageMeta*)); - Error GetDataSetById(const std::string& collection, uint64_t set_id, uint64_t id, FileInfo* file) const override { + Error GetDataSetById(const std::string& collection, uint64_t set_id, uint64_t id, MessageMeta* file) const override { return Error{GetSetById_t(collection, set_id, id, file)}; } - MOCK_CONST_METHOD4(GetSetById_t, ErrorInterface * (const std::string&, uint64_t set_id, uint64_t id, FileInfo*)); + MOCK_CONST_METHOD4(GetSetById_t, ErrorInterface * (const std::string&, uint64_t set_id, uint64_t id, MessageMeta*)); Error GetStreamInfo(const std::string& collection, StreamInfo* info) const override { diff --git a/common/cpp/include/asapo/unittests/MockHttpClient.h b/common/cpp/include/asapo/unittests/MockHttpClient.h index 34db9077e54d31189ef683c72205685067fc9966..41a5b5d232e2146c7851928ba503792cb959e8f1 100644 --- a/common/cpp/include/asapo/unittests/MockHttpClient.h +++ b/common/cpp/include/asapo/unittests/MockHttpClient.h @@ -24,7 +24,7 @@ class MockHttpClient : public HttpClient { return response; } - Error Post(const std::string& uri, const std::string& cookie, const std::string& input_data, FileData* ouput_data, + Error Post(const std::string& uri, const std::string& cookie, const std::string& input_data, MessageData* ouput_data, uint64_t output_data_size, HttpCode* response_code) const noexcept override { return Error{PostReturnArray_t(uri, cookie, input_data, ouput_data, output_data_size, response_code)}; @@ -44,7 +44,7 @@ class MockHttpClient : public HttpClient { ErrorInterface** err)); MOCK_CONST_METHOD6(PostReturnArray_t, ErrorInterface * (const std::string& uri, const std::string& cookie, const std::string& input_data, - FileData* ouput_data, uint64_t output_data_size, HttpCode* code)); + MessageData* ouput_data, uint64_t output_data_size, HttpCode* code)); }; diff --git a/common/cpp/include/asapo/unittests/MockIO.h b/common/cpp/include/asapo/unittests/MockIO.h index 94027b2a8ea56d92aac02b889b3b3230d5734cfb..d6c10de678c3cfa3ad8f4d9c0c1220c96b6e7514 100644 --- a/common/cpp/include/asapo/unittests/MockIO.h +++ b/common/cpp/include/asapo/unittests/MockIO.h @@ -207,11 +207,11 @@ class MockIO : public IO { } MOCK_CONST_METHOD2(CreateNewDirectory_t, void(const std::string& directory_name, ErrorInterface** err)); - FileData GetDataFromFile(const std::string& fname, uint64_t* fsize, Error* err) const override { + MessageData GetDataFromFile(const std::string& fname, uint64_t* fsize, Error* err) const override { ErrorInterface* error = nullptr; auto data = GetDataFromFile_t(fname, fsize, &error); err->reset(error); - return FileData(data); + return MessageData(data); } MOCK_CONST_METHOD3(GetDataFromFile_t, uint8_t* (const std::string& fname, uint64_t* fsize, ErrorInterface** err)); @@ -228,7 +228,7 @@ class MockIO : public IO { } MOCK_CONST_METHOD3(SendFile_t, ErrorInterface * (SocketDescriptor socket_fd, const std::string& fname, size_t length)); - Error WriteDataToFile(const std::string& root_folder, const std::string& fname, const FileData& data, + Error WriteDataToFile(const std::string& root_folder, const std::string& fname, const MessageData& data, size_t length, bool create_directories, bool allow_ovewrite) const override { return Error{WriteDataToFile_t(root_folder, fname, data.get(), length, create_directories, allow_ovewrite)}; @@ -260,23 +260,23 @@ class MockIO : public IO { const uint8_t* data, size_t fsize, bool create_directories, bool allow_ovewrite)); - FileInfo GetFileInfo(const std::string& name, Error* err) const override { + MessageMeta GetMessageMeta(const std::string& name, Error* err) const override { ErrorInterface* error = nullptr; - auto data = GetFileInfo_t(name, &error); + auto data = GetMessageMeta_t(name, &error); err->reset(error); return data; } - MOCK_CONST_METHOD2(GetFileInfo_t, FileInfo (const std::string& name, ErrorInterface** err)); + MOCK_CONST_METHOD2(GetMessageMeta_t, MessageMeta (const std::string& name, ErrorInterface** err)); - std::vector<FileInfo> FilesInFolder(const std::string& folder, Error* err) const override { + std::vector<MessageMeta> FilesInFolder(const std::string& folder, Error* err) const override { ErrorInterface* error = nullptr; auto data = FilesInFolder_t(folder, &error); err->reset(error); return data; } - MOCK_CONST_METHOD2(FilesInFolder_t, std::vector<FileInfo>(const std::string& folder, ErrorInterface** err)); + MOCK_CONST_METHOD2(FilesInFolder_t, std::vector<MessageMeta>(const std::string& folder, ErrorInterface** err)); SubDirList GetSubDirectories(const std::string& path, Error* err) const override { diff --git a/common/cpp/src/data_structs/data_structs.cpp b/common/cpp/src/data_structs/data_structs.cpp index f92a8210c5f79c245ba1c01689697fce5622ded6..36023b5d5e92487866a9cf581465ba13942680ce 100644 --- a/common/cpp/src/data_structs/data_structs.cpp +++ b/common/cpp/src/data_structs/data_structs.cpp @@ -41,7 +41,7 @@ Error GetSourceTypeFromString(std::string stype, SourceType* type) { } } -std::string FileInfo::Json() const { +std::string MessageMeta::Json() const { auto nanoseconds_from_epoch = NanosecsEpochFromTimePoint(timestamp); std::string x = name; //todo: change this - use / when sending file from windows @@ -99,7 +99,7 @@ bool DataSet::SetFromJson(const std::string &json_string) { return false; } for (auto fi_encoded : vec_fi_endcoded) { - FileInfo fi; + MessageMeta fi; if (!fi.SetFromJson(fi_encoded)) { *this = old; return false; @@ -109,7 +109,7 @@ bool DataSet::SetFromJson(const std::string &json_string) { return true; } -bool FileInfo::SetFromJson(const std::string &json_string) { +bool MessageMeta::SetFromJson(const std::string &json_string) { auto old = *this; JsonStringParser parser(json_string); @@ -130,7 +130,7 @@ bool FileInfo::SetFromJson(const std::string &json_string) { return true; } -std::string FileInfo::FullName(const std::string &base_path) const { +std::string MessageMeta::FullName(const std::string &base_path) const { std::string full_name; full_name = base_path.empty() ? "" : base_path + kPathSeparator; return full_name + name; diff --git a/common/cpp/src/database/mongodb_client.cpp b/common/cpp/src/database/mongodb_client.cpp index e562670c279684a687265c8db82441399962e943..783b38acbc7604cff8984b0028884980dc0b592d 100644 --- a/common/cpp/src/database/mongodb_client.cpp +++ b/common/cpp/src/database/mongodb_client.cpp @@ -110,7 +110,7 @@ void MongoDBClient::CleanUp() { } } -bson_p PrepareBsonDocument(const FileInfo &file, Error* err) { +bson_p PrepareBsonDocument(const MessageMeta &file, Error* err) { bson_error_t mongo_err; auto s = file.Json(); auto json = reinterpret_cast<const uint8_t*>(s.c_str()); @@ -172,7 +172,7 @@ Error MongoDBClient::UpdateBsonDocument(uint64_t id, const bson_p &document, boo return err; } -Error MongoDBClient::Insert(const std::string &collection, const FileInfo &file, bool ignore_duplicates) const { +Error MongoDBClient::Insert(const std::string &collection, const MessageMeta &file, bool ignore_duplicates) const { if (!connected_) { return DBErrorTemplates::kNotConnected.Generate(); } @@ -237,7 +237,7 @@ Error MongoDBClient::AddBsonDocumentToArray(bson_t* query, bson_t* update, bool return err; } -Error MongoDBClient::InsertAsSubset(const std::string &collection, const FileInfo &file, +Error MongoDBClient::InsertAsSubset(const std::string &collection, const MessageMeta &file, uint64_t subset_id, uint64_t subset_size, bool ignore_duplicates) const { @@ -322,7 +322,7 @@ Error MongoDBClient::GetRecordFromDb(const std::string &collection, uint64_t id, return err; } -Error MongoDBClient::GetById(const std::string &collection, uint64_t id, FileInfo* file) const { +Error MongoDBClient::GetById(const std::string &collection, uint64_t id, MessageMeta* file) const { std::string record_str; auto err = GetRecordFromDb(collection, id, GetRecordMode::kById, &record_str); if (err) { @@ -335,7 +335,7 @@ Error MongoDBClient::GetById(const std::string &collection, uint64_t id, FileInf return nullptr; } -Error MongoDBClient::GetDataSetById(const std::string &collection, uint64_t id_in_set, uint64_t id, FileInfo* file) const { +Error MongoDBClient::GetDataSetById(const std::string &collection, uint64_t id_in_set, uint64_t id, MessageMeta* file) const { std::string record_str; auto err = GetRecordFromDb(collection, id, GetRecordMode::kById, &record_str); if (err) { diff --git a/common/cpp/src/database/mongodb_client.h b/common/cpp/src/database/mongodb_client.h index b74691a3205e6c93ec24e061ae2fb3c6074c5a3e..ad78011173e875f44b3efcd17b5a61f1a4f1f15b 100644 --- a/common/cpp/src/database/mongodb_client.h +++ b/common/cpp/src/database/mongodb_client.h @@ -43,12 +43,12 @@ class MongoDBClient final : public Database { public: MongoDBClient(); Error Connect(const std::string& address, const std::string& database) override; - Error Insert(const std::string& collection, const FileInfo& file, bool ignore_duplicates) const override; - Error InsertAsSubset(const std::string& collection, const FileInfo& file, uint64_t subset_id, uint64_t subset_size, + Error Insert(const std::string& collection, const MessageMeta& file, bool ignore_duplicates) const override; + Error InsertAsSubset(const std::string& collection, const MessageMeta& file, uint64_t subset_id, uint64_t subset_size, bool ignore_duplicates) const override; Error Upsert(const std::string& collection, uint64_t id, const uint8_t* data, uint64_t size) const override; - Error GetById(const std::string& collection, uint64_t id, FileInfo* file) const override; - Error GetDataSetById(const std::string& collection, uint64_t id_in_set, uint64_t id, FileInfo* file) const override; + Error GetById(const std::string& collection, uint64_t id, MessageMeta* file) const override; + Error GetDataSetById(const std::string& collection, uint64_t id_in_set, uint64_t id, MessageMeta* file) const override; Error GetStreamInfo(const std::string& collection, StreamInfo* info) const override; Error GetLastStream(StreamInfo* info) const override; ~MongoDBClient() override; diff --git a/common/cpp/src/http_client/curl_http_client.cpp b/common/cpp/src/http_client/curl_http_client.cpp index 7524e4fc1bf58f959c2e132370c194cb34f60523..0bdefda2189197185f2c5e7797dc1b0f1a491ffa 100644 --- a/common/cpp/src/http_client/curl_http_client.cpp +++ b/common/cpp/src/http_client/curl_http_client.cpp @@ -108,10 +108,10 @@ Error CurlHttpClient::Command(bool post, CurlDataContainer* data_container, cons return ProcessCurlResponse(curl_, res, errbuf, response_code); } -FileData AllocateMemory(uint64_t size, Error* err) { - FileData data; +MessageData AllocateMemory(uint64_t size, Error* err) { + MessageData data; try { - data = FileData{new uint8_t[(size_t)size + 1 ]}; + data = MessageData{new uint8_t[(size_t)size + 1 ]}; } catch (...) { *err = ErrorTemplates::kMemoryAllocationError.Generate(); return nullptr; @@ -123,7 +123,7 @@ FileData AllocateMemory(uint64_t size, Error* err) { Error CurlHttpClient::Post(const std::string& uri, const std::string& cookie, const std::string& input_data, - FileData* output_data, + MessageData* output_data, uint64_t output_data_size, HttpCode* response_code) const noexcept { Error err; diff --git a/common/cpp/src/http_client/curl_http_client.h b/common/cpp/src/http_client/curl_http_client.h index 07412c143a0d6805c26ff6b68046f78a2b92f975..cfc2e7626974422a72d87b371660cd7052cc34ea 100644 --- a/common/cpp/src/http_client/curl_http_client.h +++ b/common/cpp/src/http_client/curl_http_client.h @@ -19,7 +19,7 @@ enum class CurlDataMode { struct CurlDataContainer { std::string string_buffer; - FileData* p_array; + MessageData* p_array; uint64_t array_size; uint64_t bytes_received = 0; CurlDataMode mode; @@ -34,7 +34,7 @@ class CurlHttpClient final : public HttpClient { std::string Get(const std::string& uri, HttpCode* response_code, Error* err) const noexcept override; std::string Post(const std::string& uri, const std::string& cookie, const std::string& data, HttpCode* response_code, Error* err) const noexcept override; - Error Post(const std::string& uri, const std::string& cookie, const std::string& input_data, FileData* output_data, + Error Post(const std::string& uri, const std::string& cookie, const std::string& input_data, MessageData* output_data, uint64_t output_data_size, HttpCode* response_code) const noexcept override; Error Post(const std::string& uri, const std::string& cookie, diff --git a/common/cpp/src/system_io/system_io.cpp b/common/cpp/src/system_io/system_io.cpp index 0c3098be19a2c2f95a89d1a7eaffe7369a5eb503..b15011f379445951910b8fcfa9dfb02ad12a18cd 100644 --- a/common/cpp/src/system_io/system_io.cpp +++ b/common/cpp/src/system_io/system_io.cpp @@ -38,21 +38,21 @@ const size_t SystemIO::kReadWriteBufSize = size_t(1024) * 1024 * 50; //50MiByte // PRIVATE FUNCTIONS - START -void SortFileList(std::vector<FileInfo>* file_list) { +void SortFileList(std::vector<MessageMeta>* file_list) { std::sort(file_list->begin(), file_list->end(), - [](FileInfo const & a, FileInfo const & b) { + [](MessageMeta const & a, MessageMeta const & b) { return a.timestamp < b.timestamp; }); } -void StripBasePath(const std::string& folder, std::vector<FileInfo>* file_list) { +void StripBasePath(const std::string& folder, std::vector<MessageMeta>* file_list) { auto n_erase = folder.size() + 1; for (auto& file : *file_list) { file.name.erase(0, n_erase); } } -void AssignIDs(FileInfos* file_list) { +void AssignIDs(MessageMetas* file_list) { int64_t id = 0; for (auto& file : *file_list) { file.id = ++id; @@ -86,10 +86,10 @@ uint8_t* SystemIO::AllocateArray(uint64_t fsize, Error* err) const { // PRIVATE FUNCTIONS - END -FileData SystemIO::GetDataFromFile(const std::string& fname, uint64_t* fsize, Error* err) const { +MessageData SystemIO::GetDataFromFile(const std::string& fname, uint64_t* fsize, Error* err) const { if (*fsize == 0 && !fname.empty()) { - auto info = GetFileInfo(fname, err); + auto info = GetMessageMeta(fname, err); if (*err != nullptr) { return nullptr; } @@ -115,12 +115,12 @@ FileData SystemIO::GetDataFromFile(const std::string& fname, uint64_t* fsize, Er } Close(fd, err); - return FileData{data_array}; + return MessageData{data_array}; } -FileInfos SystemIO::FilesInFolder(const std::string& folder, Error* err) const { - FileInfos files{}; - CollectFileInformationRecursively(folder, &files, err); +MessageMetas SystemIO::FilesInFolder(const std::string& folder, Error* err) const { + MessageMetas files{}; + CollectMessageMetarmationRecursively(folder, &files, err); if (*err != nullptr) { return {}; } @@ -193,7 +193,7 @@ Error SystemIO::WriteDataToFile(const std::string& root_folder, const std::strin } -Error SystemIO::WriteDataToFile(const std::string& root_folder, const std::string& fname, const FileData& data, +Error SystemIO::WriteDataToFile(const std::string& root_folder, const std::string& fname, const MessageData& data, size_t length, bool create_directories, bool allow_ovewrite) const { return WriteDataToFile(root_folder, fname, data.get(), length, create_directories, allow_ovewrite); } diff --git a/common/cpp/src/system_io/system_io.h b/common/cpp/src/system_io/system_io.h index f964e92e257d20a20bfd485a4f4d3bb35a8b5500..b42cd2b1799f8e8f54c9b5f7eab6ad89c435d178 100644 --- a/common/cpp/src/system_io/system_io.h +++ b/common/cpp/src/system_io/system_io.h @@ -46,7 +46,7 @@ class SystemIO final : public IO { void ApplyNetworkOptions(SocketDescriptor socket_fd, Error* err) const; - //void CollectFileInformationRecursively(const std::string& path, std::vector<FileInfo>* files, IOErrors* err) const; + //void CollectMessageMetarmationRecursively(const std::string& path, std::vector<MessageMeta>* files, IOErrors* err) const; int FileOpenModeToPosixFileOpenMode(int open_flags) const; short AddressFamilyToPosixFamily (AddressFamilies address_family) const; @@ -78,7 +78,7 @@ class SystemIO final : public IO { static ssize_t _recv(SocketDescriptor socket_fd, void* buffer, size_t length); static ssize_t _read(FileDescriptor fd, void* buffer, size_t length); static ssize_t _write(FileDescriptor fd, const void* buffer, size_t count); - void CollectFileInformationRecursively(const std::string& path, std::vector<FileInfo>* files, + void CollectMessageMetarmationRecursively(const std::string& path, std::vector<MessageMeta>* files, Error* err) const; void GetSubDirectoriesRecursively(const std::string& path, SubDirList* subdirs, Error* err) const; Error CreateDirectoryWithParents(const std::string& root_path, const std::string& path) const; @@ -100,7 +100,7 @@ class SystemIO final : public IO { // this is not standard function - to be implemented differently in windows and linux - std::vector<FileInfo> FilesInFolder(const std::string& folder, Error* err) const override; + std::vector<MessageMeta> FilesInFolder(const std::string& folder, Error* err) const override; /* * Network @@ -139,8 +139,8 @@ class SystemIO final : public IO { size_t Read(FileDescriptor fd, void* buf, size_t length, Error* err) const override; size_t Write(FileDescriptor fd, const void* buf, size_t length, Error* err) const override; void CreateNewDirectory(const std::string& directory_name, Error* err) const override; - FileData GetDataFromFile(const std::string& fname, uint64_t* fsize, Error* err) const override; - Error WriteDataToFile (const std::string& root_folder, const std::string& fname, const FileData& data, + MessageData GetDataFromFile(const std::string& fname, uint64_t* fsize, Error* err) const override; + Error WriteDataToFile (const std::string& root_folder, const std::string& fname, const MessageData& data, size_t length, bool create_directories, bool allow_ovewrite) const override; Error ReceiveDataToFile(SocketDescriptor socket, const std::string& root_folder, const std::string& fname, size_t length, bool create_directories, bool allow_ovewrite) const override; @@ -152,7 +152,7 @@ class SystemIO final : public IO { Error RemoveFile(const std::string& fname) const override; Error GetLastError() const override; std::string AddressFromSocket(SocketDescriptor socket) const noexcept override; - FileInfo GetFileInfo(const std::string& name, Error* err) const override; + MessageMeta GetMessageMeta(const std::string& name, Error* err) const override; }; diff --git a/common/cpp/src/system_io/system_io_linux_mac.cpp b/common/cpp/src/system_io/system_io_linux_mac.cpp index 0277406c6e5f723cbd54bf0339918420a1723d10..fd0b7f35ef97f60a680571d13765175946f85e4a 100644 --- a/common/cpp/src/system_io/system_io_linux_mac.cpp +++ b/common/cpp/src/system_io/system_io_linux_mac.cpp @@ -82,7 +82,7 @@ bool IsDirectory(const struct dirent* entity) { strstr(entity->d_name, ".") == nullptr; } -void SetModifyDate(const struct stat& t_stat, FileInfo* file_info) { +void SetModifyDate(const struct stat& t_stat, MessageMeta* message_meta) { #ifdef __APPLE__ #define st_mtim st_mtimespec #endif @@ -92,16 +92,16 @@ void SetModifyDate(const struct stat& t_stat, FileInfo* file_info) { #undef st_mtim #endif - file_info->timestamp = system_clock::time_point + message_meta->timestamp = system_clock::time_point {std::chrono::duration_cast<system_clock::duration>(d)}; } -void SetFileSize(const struct stat& t_stat, FileInfo* file_info) { - file_info->size = t_stat.st_size; +void SetFileSize(const struct stat& t_stat, MessageMeta* message_meta) { + message_meta->size = t_stat.st_size; } -void SetFileName(const string& name, FileInfo* file_info) { - file_info->name = name; +void SetFileName(const string& name, MessageMeta* message_meta) { + message_meta->name = name; } struct stat FileStat(const string& fname, Error* err) { @@ -114,41 +114,41 @@ struct stat FileStat(const string& fname, Error* err) { return t_stat; } -FileInfo GetFileInfo(const string& name, Error* err) { - FileInfo file_info; +MessageMeta GetMessageMeta(const string& name, Error* err) { + MessageMeta message_meta; - SetFileName(name, &file_info); + SetFileName(name, &message_meta); auto t_stat = FileStat(name, err); if (*err != nullptr) { (*err)->Append(name); - return FileInfo{}; + return MessageMeta{}; } - SetFileSize(t_stat, &file_info); + SetFileSize(t_stat, &message_meta); - SetModifyDate(t_stat, &file_info); + SetModifyDate(t_stat, &message_meta); - return file_info; + return message_meta; } -FileInfo SystemIO::GetFileInfo(const string& name, Error* err) const { - return ::asapo::GetFileInfo(name, err); +MessageMeta SystemIO::GetMessageMeta(const string& name, Error* err) const { + return ::asapo::GetMessageMeta(name, err); } void ProcessFileEntity(const struct dirent* entity, const std::string& path, - FileInfos* files, Error* err) { + MessageMetas* files, Error* err) { *err = nullptr; if (entity->d_type != DT_REG) { return; } - FileInfo file_info = GetFileInfo(path + "/" + entity->d_name, err); + MessageMeta message_meta = GetMessageMeta(path + "/" + entity->d_name, err); if (*err != nullptr) { return; } - files->push_back(file_info); + files->push_back(message_meta); } void SystemIO::GetSubDirectoriesRecursively(const std::string& path, SubDirList* subdirs, Error* err) const { @@ -176,8 +176,8 @@ void SystemIO::GetSubDirectoriesRecursively(const std::string& path, SubDirList* closedir(dir); } -void SystemIO::CollectFileInformationRecursively(const std::string& path, - FileInfos* files, Error* err) const { +void SystemIO::CollectMessageMetarmationRecursively(const std::string& path, + MessageMetas* files, Error* err) const { errno = 0; auto dir = opendir((path).c_str()); if (dir == nullptr) { @@ -188,7 +188,7 @@ void SystemIO::CollectFileInformationRecursively(const std::string& path, while (struct dirent* current_entity = readdir(dir)) { if (IsDirectory(current_entity)) { - CollectFileInformationRecursively(path + "/" + current_entity->d_name, + CollectMessageMetarmationRecursively(path + "/" + current_entity->d_name, files, err); } else { ProcessFileEntity(current_entity, path, files, err); diff --git a/common/cpp/src/system_io/system_io_windows.cpp b/common/cpp/src/system_io/system_io_windows.cpp index cb9c6d43a20fc772070c2c683f24309b3ea4f90d..9755e2e91123f293f682db88c2ee618ccc1b3517 100644 --- a/common/cpp/src/system_io/system_io_windows.cpp +++ b/common/cpp/src/system_io/system_io_windows.cpp @@ -126,10 +126,10 @@ bool IsDirectory(const WIN32_FIND_DATA f) { strstr(f.cFileName, ".") == nullptr; } -FileInfo GetFileInfo_win(const WIN32_FIND_DATA& f, const string& name, Error* err) { - FileInfo file_info; +MessageMeta GetMessageMeta_win(const WIN32_FIND_DATA& f, const string& name, Error* err) { + MessageMeta message_meta; - file_info.timestamp = FileTime2TimePoint(f.ftLastWriteTime, err); + message_meta.timestamp = FileTime2TimePoint(f.ftLastWriteTime, err); if (*err) { return {}; } @@ -138,14 +138,14 @@ FileInfo GetFileInfo_win(const WIN32_FIND_DATA& f, const string& name, Error* er fsize.LowPart = f.nFileSizeLow; fsize.HighPart = f.nFileSizeHigh; - file_info.size = fsize.QuadPart; + message_meta.size = fsize.QuadPart; - file_info.name = name + "\\" + f.cFileName; + message_meta.name = name + "\\" + f.cFileName; - return file_info; + return message_meta; } -FileInfo SystemIO::GetFileInfo(const std::string& name, Error* err) const { +MessageMeta SystemIO::GetMessageMeta(const std::string& name, Error* err) const { WIN32_FIND_DATA f; auto hFind = FindFirstFile(name.c_str(), &f); @@ -155,23 +155,23 @@ FileInfo SystemIO::GetFileInfo(const std::string& name, Error* err) const { return {}; } FindClose(hFind); - return GetFileInfo_win(f, name, err); + return GetMessageMeta_win(f, name, err); } void ProcessFileEntity(const WIN32_FIND_DATA& f, const std::string& path, - FileInfos* files, Error* err) { + MessageMetas* files, Error* err) { *err = nullptr; if (f.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { return; } - auto file_info = GetFileInfo_win(f, path, err); + auto message_meta = GetMessageMeta_win(f, path, err); if (*err) { return; } - files->push_back(file_info); + files->push_back(message_meta); } void SystemIO::GetSubDirectoriesRecursively(const std::string& path, SubDirList* subdirs, Error* err) const { @@ -202,8 +202,8 @@ void SystemIO::GetSubDirectoriesRecursively(const std::string& path, SubDirList* } } -void SystemIO::CollectFileInformationRecursively(const std::string& path, - FileInfos* files, Error* err) const { +void SystemIO::CollectMessageMetarmationRecursively(const std::string& path, + MessageMetas* files, Error* err) const { WIN32_FIND_DATA find_data; HANDLE handle = FindFirstFile((path + "\\*.*").c_str(), &find_data); if (handle == INVALID_HANDLE_VALUE) { @@ -214,7 +214,7 @@ void SystemIO::CollectFileInformationRecursively(const std::string& path, do { if (IsDirectory(find_data)) { - CollectFileInformationRecursively(path + "\\" + find_data.cFileName, files, err); + CollectMessageMetarmationRecursively(path + "\\" + find_data.cFileName, files, err); } else { ProcessFileEntity(find_data, path, files, err); } diff --git a/common/cpp/unittests/data_structs/test_data_structs.cpp b/common/cpp/unittests/data_structs/test_data_structs.cpp index 0a877a55359d796fc44a45b8a0d4b1dff4dfdcc1..49ba236dfb244eaf37a28d58cdc70b31ede1f0e8 100644 --- a/common/cpp/unittests/data_structs/test_data_structs.cpp +++ b/common/cpp/unittests/data_structs/test_data_structs.cpp @@ -5,7 +5,7 @@ #include <chrono> -using asapo::FileInfo; +using asapo::MessageMeta; using asapo::StreamInfo; using asapo::SourceType; using asapo::SourceCredentials; @@ -25,29 +25,29 @@ namespace { uint64_t big_uint = 18446744073709551615ull; -FileInfo PrepareFileInfo() { - FileInfo finfo; - finfo.size = 100; - finfo.id = 1; - finfo.name = std::string("folder") + asapo::kPathSeparator + "test"; - finfo.source = "host:1234"; - finfo.buf_id = big_uint; - finfo.timestamp = std::chrono::time_point<std::chrono::system_clock>(std::chrono::milliseconds(1)); - finfo.metadata = "{\"bla\":10}"; - return finfo; +MessageMeta PrepareMessageMeta() { + MessageMeta message_meta; + message_meta.size = 100; + message_meta.id = 1; + message_meta.name = std::string("folder") + asapo::kPathSeparator + "test"; + message_meta.source = "host:1234"; + message_meta.buf_id = big_uint; + message_meta.timestamp = std::chrono::time_point<std::chrono::system_clock>(std::chrono::milliseconds(1)); + message_meta.metadata = "{\"bla\":10}"; + return message_meta; } -TEST(FileInFo, Defaults) { - FileInfo finfo; +TEST(MessageMetaTests, Defaults) { + MessageMeta message_meta; - ASSERT_THAT(finfo.buf_id, Eq(0)); - ASSERT_THAT(finfo.id, Eq(0)); + ASSERT_THAT(message_meta.buf_id, Eq(0)); + ASSERT_THAT(message_meta.id, Eq(0)); } -TEST(FileInFo, CorrectConvertToJson) { - auto finfo = PrepareFileInfo(); - std::string json = finfo.Json(); +TEST(MessageMetaTests, CorrectConvertToJson) { + auto message_meta = PrepareMessageMeta(); + std::string json = message_meta.Json(); if (asapo::kPathSeparator == '/') { ASSERT_THAT(json, Eq( R"({"_id":1,"size":100,"name":"folder/test","timestamp":1000000,"source":"host:1234","buf_id":-1,"meta":{"bla":10}})")); @@ -57,10 +57,10 @@ TEST(FileInFo, CorrectConvertToJson) { } } -TEST(FileInFo, CorrectConvertFromJsonReturnsError) { - auto finfo = PrepareFileInfo(); +TEST(MessageMetaTests, CorrectConvertFromJsonReturnsError) { + auto message_meta = PrepareMessageMeta(); - FileInfo result; + MessageMeta result; result.id = 10; std::string json = R"({"_id":2,"foo":"foo","bar":1})"; @@ -72,10 +72,10 @@ TEST(FileInFo, CorrectConvertFromJsonReturnsError) { } -TEST(FileInFo, CorrectConvertFromJsonReturnsErrorForMetadata) { - auto finfo = PrepareFileInfo(); +TEST(MessageMetaTests, CorrectConvertFromJsonReturnsErrorForMetadata) { + auto message_meta = PrepareMessageMeta(); - FileInfo result; + MessageMeta result; std::string json = R"({"_id":2,"foo":"foo","bar":1,{"meta":err}})"; @@ -87,32 +87,32 @@ TEST(FileInFo, CorrectConvertFromJsonReturnsErrorForMetadata) { -TEST(FileInFo, CorrectConvertFromJson) { - auto finfo = PrepareFileInfo(); - std::string json = finfo.Json(); +TEST(MessageMetaTests, CorrectConvertFromJson) { + auto message_meta = PrepareMessageMeta(); + std::string json = message_meta.Json(); - FileInfo result; + MessageMeta result; auto ok = result.SetFromJson(json); ASSERT_THAT(ok, Eq(true)); - ASSERT_THAT(result.id, Eq(finfo.id)); - ASSERT_THAT(result.name, Eq(finfo.name)); - ASSERT_THAT(result.size, Eq(finfo.size)); - ASSERT_THAT(result.timestamp, Eq(finfo.timestamp)); - ASSERT_THAT(result.buf_id, Eq(finfo.buf_id)); - ASSERT_THAT(result.source, Eq(finfo.source)); - ASSERT_THAT(result.metadata, Eq(finfo.metadata)); + ASSERT_THAT(result.id, Eq(message_meta.id)); + ASSERT_THAT(result.name, Eq(message_meta.name)); + ASSERT_THAT(result.size, Eq(message_meta.size)); + ASSERT_THAT(result.timestamp, Eq(message_meta.timestamp)); + ASSERT_THAT(result.buf_id, Eq(message_meta.buf_id)); + ASSERT_THAT(result.source, Eq(message_meta.source)); + ASSERT_THAT(result.metadata, Eq(message_meta.metadata)); } -TEST(FileInFo, CorrectConvertFromJsonEmptyMeta) { - auto finfo = PrepareFileInfo(); - finfo.metadata = ""; - std::string json = finfo.Json(); +TEST(MessageMetaTests, CorrectConvertFromJsonEmptyMeta) { + auto message_meta = PrepareMessageMeta(); + message_meta.metadata = ""; + std::string json = message_meta.Json(); - FileInfo result; + MessageMeta result; auto ok = result.SetFromJson(json); ASSERT_THAT(ok, Eq(true)); @@ -121,7 +121,7 @@ TEST(FileInFo, CorrectConvertFromJsonEmptyMeta) { } -TEST(FileInFo, EpochNanosecsFromNow) { +TEST(MessageMetaTests, EpochNanosecsFromNow) { auto ns = asapo::EpochNanosecsFromNow(); ASSERT_THAT(ns, ::testing::Gt(0)); } @@ -143,7 +143,7 @@ StreamInfo PrepareStreamInfo() { } -TEST(FileInFo, TimeFromNanosec) { +TEST(MessageMetaTests, TimeFromNanosec) { auto tp = asapo::TimePointfromNanosec(1000); auto res = asapo::NanosecsEpochFromTimePoint(tp); ASSERT_THAT(res, Eq(1000)); @@ -249,7 +249,7 @@ auto tests = std::vector<TestEpochFromISODate> { TestEpochFromISODate{"1970-12-01T00:00:00.", 0}, }; -TEST(FileInFo, NanosecsEpochFromISODate) { +TEST(MessageMetaTests, NanosecsEpochFromISODate) { for (auto test : tests) { auto res = asapo::NanosecsEpochFromISODate(test.iso); ASSERT_THAT(res, Eq(test.ns)); @@ -262,7 +262,7 @@ auto tests2 = std::vector<TestEpochFromISODate> { TestEpochFromISODate{"2019-07-25T15:38:11.100010002", 1564069091100010002}, }; -TEST(FileInFo, ISODateFromNanosecsEpoch) { +TEST(MessageMetaTests, ISODateFromNanosecsEpoch) { for (auto test : tests2) { auto res = asapo::IsoDateFromEpochNanosecs(test.ns); ASSERT_THAT(res, Eq(test.iso)); diff --git a/consumer/api/cpp/include/asapo/consumer/consumer.h b/consumer/api/cpp/include/asapo/consumer/consumer.h index 12a07f4c5e7ee098b1f9c6802e52b91244a3c100..bd4c495bf0ad85560e8ba7dd0915421ea83dcb02 100644 --- a/consumer/api/cpp/include/asapo/consumer/consumer.h +++ b/consumer/api/cpp/include/asapo/consumer/consumer.h @@ -106,8 +106,8 @@ class Consumer { \param data - where to store image data. Can be set to nullptr only image metadata is needed. \return Error if both pointers are nullptr or data cannot be read, nullptr otherwise. */ - virtual Error GetNext(FileInfo* info, std::string group_id, FileData* data) = 0; - virtual Error GetNext(FileInfo* info, std::string group_id, std::string stream, FileData* data) = 0; + virtual Error GetNext(MessageMeta* info, std::string group_id, MessageData* data) = 0; + virtual Error GetNext(MessageMeta* info, std::string group_id, std::string stream, MessageData* data) = 0; //! Retrieves image using fileinfo. /*! @@ -115,7 +115,7 @@ class Consumer { \param data - where to store image data. Can be set to nullptr only image metadata is needed. \return Error if data is nullptr or data cannot be read, nullptr otherwise. */ - virtual Error RetrieveData(FileInfo* info, FileData* data) = 0; + virtual Error RetrieveData(MessageMeta* info, MessageData* data) = 0; //! Receive next available completed dataset. @@ -157,8 +157,8 @@ class Consumer { \param data - where to store image data. Can be set to nullptr only image metadata is needed. \return Error if both pointers are nullptr or data cannot be read, nullptr otherwise. */ - virtual Error GetById(uint64_t id, FileInfo* info, FileData* data) = 0; - virtual Error GetById(uint64_t id, FileInfo* info, std::string stream, FileData* data) = 0; + virtual Error GetById(uint64_t id, MessageMeta* info, MessageData* data) = 0; + virtual Error GetById(uint64_t id, MessageMeta* info, std::string stream, MessageData* data) = 0; //! Receive id of last acknowledged data tuple /*! @@ -176,8 +176,8 @@ class Consumer { \param data - where to store image data. Can be set to nullptr only image metadata is needed. \return Error if both pointers are nullptr or data cannot be read, nullptr otherwise. */ - virtual Error GetLast(FileInfo* info, FileData* data) = 0; - virtual Error GetLast(FileInfo* info, std::string stream, FileData* data) = 0; + virtual Error GetLast(MessageMeta* info, MessageData* data) = 0; + virtual Error GetLast(MessageMeta* info, std::string stream, MessageData* data) = 0; //! Get all images matching the query. /*! @@ -185,8 +185,8 @@ class Consumer { \param err - will be set in case of error, nullptr otherwise \return vector of image metadata matchiing to specified query. Empty if nothing found or error */ - virtual FileInfos QueryImages(std::string query, Error* err) = 0; - virtual FileInfos QueryImages(std::string query, std::string stream, Error* err) = 0; + virtual MessageMetas QueryImages(std::string query, Error* err) = 0; + virtual MessageMetas QueryImages(std::string query, std::string stream, Error* err) = 0; //! Configure resending nonacknowledged data /*! diff --git a/consumer/api/cpp/src/consumer_impl.cpp b/consumer/api/cpp/src/consumer_impl.cpp index 575460028eb76cb27a3056abc0af5e4f8b9b5b49..7afbaeb8e8538e7834bc881b4209426c6d0f49f9 100644 --- a/consumer/api/cpp/src/consumer_impl.cpp +++ b/consumer/api/cpp/src/consumer_impl.cpp @@ -294,11 +294,11 @@ Error ConsumerImpl::GetRecordFromServer(std::string* response, std::string group return nullptr; } -Error ConsumerImpl::GetNext(FileInfo* info, std::string group_id, FileData* data) { +Error ConsumerImpl::GetNext(MessageMeta* info, std::string group_id, MessageData* data) { return GetNext(info, std::move(group_id), kDefaultStream, data); } -Error ConsumerImpl::GetNext(FileInfo* info, std::string group_id, std::string stream, FileData* data) { +Error ConsumerImpl::GetNext(MessageMeta* info, std::string group_id, std::string stream, MessageData* data) { return GetImageFromServer(GetImageServerOperation::GetNext, 0, std::move(group_id), @@ -307,11 +307,11 @@ Error ConsumerImpl::GetNext(FileInfo* info, std::string group_id, std::string st data); } -Error ConsumerImpl::GetLast(FileInfo* info, FileData* data) { +Error ConsumerImpl::GetLast(MessageMeta* info, MessageData* data) { return GetLast(info, kDefaultStream, data); } -Error ConsumerImpl::GetLast(FileInfo* info, std::string stream, FileData* data) { +Error ConsumerImpl::GetLast(MessageMeta* info, std::string stream, MessageData* data) { return GetImageFromServer(GetImageServerOperation::GetLast, 0, "0", @@ -330,8 +330,8 @@ std::string ConsumerImpl::OpToUriCmd(GetImageServerOperation op) { Error ConsumerImpl::GetImageFromServer(GetImageServerOperation op, uint64_t id, std::string group_id, std::string stream, - FileInfo* info, - FileData* data) { + MessageMeta* info, + MessageData* data) { if (info == nullptr) { return ConsumerErrorTemplates::kWrongInput.Generate(); } @@ -353,7 +353,7 @@ Error ConsumerImpl::GetImageFromServer(GetImageServerOperation op, uint64_t id, return GetDataIfNeeded(info, data); } -Error ConsumerImpl::GetDataFromFile(FileInfo* info, FileData* data) { +Error ConsumerImpl::GetDataFromFile(MessageMeta* info, MessageData* data) { Error error; *data = io__->GetDataFromFile(info->FullName(source_path_), &info->size, &error); if (error) { @@ -362,7 +362,7 @@ Error ConsumerImpl::GetDataFromFile(FileInfo* info, FileData* data) { return nullptr; } -Error ConsumerImpl::RetrieveData(FileInfo* info, FileData* data) { +Error ConsumerImpl::RetrieveData(MessageMeta* info, MessageData* data) { if (data == nullptr || info == nullptr) { return ConsumerErrorTemplates::kWrongInput.Generate("pointers are empty"); } @@ -382,7 +382,7 @@ Error ConsumerImpl::RetrieveData(FileInfo* info, FileData* data) { return GetDataFromFileTransferService(info, data, false); } -Error ConsumerImpl::GetDataIfNeeded(FileInfo* info, FileData* data) { +Error ConsumerImpl::GetDataIfNeeded(MessageMeta* info, MessageData* data) { if (data == nullptr) { return nullptr; } @@ -391,11 +391,11 @@ Error ConsumerImpl::GetDataIfNeeded(FileInfo* info, FileData* data) { } -bool ConsumerImpl::DataCanBeInBuffer(const FileInfo* info) { +bool ConsumerImpl::DataCanBeInBuffer(const MessageMeta* info) { return info->buf_id > 0; } -Error ConsumerImpl::CreateNetClientAndTryToGetFile(const FileInfo* info, FileData* data) { +Error ConsumerImpl::CreateNetClientAndTryToGetFile(const MessageMeta* info, MessageData* data) { const std::lock_guard<std::mutex> lock(net_client_mutex__); if (net_client__) { return nullptr; @@ -430,7 +430,7 @@ Error ConsumerImpl::CreateNetClientAndTryToGetFile(const FileInfo* info, FileDat return net_client__->GetData(info, data); } -Error ConsumerImpl::TryGetDataFromBuffer(const FileInfo* info, FileData* data) { +Error ConsumerImpl::TryGetDataFromBuffer(const MessageMeta* info, MessageData* data) { if (!net_client__) { return CreateNetClientAndTryToGetFile(info, data); } @@ -472,7 +472,7 @@ Error ConsumerImpl::ServiceRequestWithTimeout(const std::string &service_name, return err; } -Error ConsumerImpl::FtsSizeRequestWithTimeout(FileInfo* info) { +Error ConsumerImpl::FtsSizeRequestWithTimeout(MessageMeta* info) { RequestInfo ri = CreateFileTransferRequest(info); ri.extra_params = "&sizeonly=true"; ri.output_mode = OutputDataMode::string; @@ -487,7 +487,7 @@ Error ConsumerImpl::FtsSizeRequestWithTimeout(FileInfo* info) { return err; } -Error ConsumerImpl::FtsRequestWithTimeout(FileInfo* info, FileData* data) { +Error ConsumerImpl::FtsRequestWithTimeout(MessageMeta* info, MessageData* data) { RequestInfo ri = CreateFileTransferRequest(info); RequestOutput response; response.data_output_size = info->size; @@ -499,7 +499,7 @@ Error ConsumerImpl::FtsRequestWithTimeout(FileInfo* info, FileData* data) { return nullptr; } -RequestInfo ConsumerImpl::CreateFileTransferRequest(const FileInfo* info) const { +RequestInfo ConsumerImpl::CreateFileTransferRequest(const MessageMeta* info) const { RequestInfo ri; ri.api = "/transfer"; ri.post = true; @@ -559,7 +559,7 @@ uint64_t ConsumerImpl::GetCurrentSize(std::string stream, Error* err) { uint64_t ConsumerImpl::GetCurrentSize(Error* err) { return GetCurrentSize(kDefaultStream, err); } -Error ConsumerImpl::GetById(uint64_t id, FileInfo* info, FileData* data) { +Error ConsumerImpl::GetById(uint64_t id, MessageMeta* info, MessageData* data) { if (id == 0) { return ConsumerErrorTemplates::kWrongInput.Generate("id should be positive"); } @@ -567,7 +567,7 @@ Error ConsumerImpl::GetById(uint64_t id, FileInfo* info, FileData* data) { return GetById(id, info, kDefaultStream, data); } -Error ConsumerImpl::GetById(uint64_t id, FileInfo* info, std::string stream, FileData* data) { +Error ConsumerImpl::GetById(uint64_t id, MessageMeta* info, std::string stream, MessageData* data) { return GetImageFromServer(GetImageServerOperation::GetID, id, "0", stream, info, data); } @@ -600,13 +600,13 @@ DataSet DecodeDatasetFromResponse(std::string response, Error* err) { DataSet res; if (!res.SetFromJson(std::move(response))) { *err = ConsumerErrorTemplates::kInterruptedTransaction.Generate("malformed response:" + response); - return {0,0,FileInfos{}}; + return {0,0,MessageMetas{}}; } else { return res; } } -FileInfos ConsumerImpl::QueryImages(std::string query, std::string stream, Error* err) { +MessageMetas ConsumerImpl::QueryImages(std::string query, std::string stream, Error* err) { RequestInfo ri; ri.api = "/database/" + source_credentials_.beamtime_id + "/" + source_credentials_.data_source + "/" + std::move(stream) + "/0/queryimages"; @@ -615,14 +615,14 @@ FileInfos ConsumerImpl::QueryImages(std::string query, std::string stream, Error auto response = BrokerRequestWithTimeout(ri, err); if (*err) { - return FileInfos{}; + return MessageMetas{}; } auto dataset = DecodeDatasetFromResponse("{\"_id\":0,\"size\":0, \"images\":" + response + "}", err); return dataset.content; } -FileInfos ConsumerImpl::QueryImages(std::string query, Error* err) { +MessageMetas ConsumerImpl::QueryImages(std::string query, Error* err) { return QueryImages(std::move(query), kDefaultStream, err); } @@ -647,7 +647,7 @@ DataSet ConsumerImpl::GetDatasetFromServer(GetImageServerOperation op, std::string group_id, std::string stream, uint64_t min_size, Error* err) { - FileInfos infos; + MessageMetas infos; std::string response; if (op == GetImageServerOperation::GetID) { *err = GetRecordFromServerById(id, &response, std::move(group_id), std::move(stream), true, min_size); @@ -655,7 +655,7 @@ DataSet ConsumerImpl::GetDatasetFromServer(GetImageServerOperation op, *err = GetRecordFromServer(&response, std::move(group_id), std::move(stream), op, true, min_size); } if (*err != nullptr && *err!=ConsumerErrorTemplates::kPartialData) { - return {0, 0,FileInfos{}}; + return {0, 0,MessageMetas{}}; } return DecodeDatasetFromResponse(response, err); } @@ -734,7 +734,7 @@ RequestInfo ConsumerImpl::CreateFolderTokenRequest() const { return ri; } -Error ConsumerImpl::GetDataFromFileTransferService(FileInfo* info, FileData* data, +Error ConsumerImpl::GetDataFromFileTransferService(MessageMeta* info, MessageData* data, bool retry_with_new_token) { auto err = UpdateFolderTokenIfNeeded(retry_with_new_token); if (err) { diff --git a/consumer/api/cpp/src/consumer_impl.h b/consumer/api/cpp/src/consumer_impl.h index 59266f2ce36cbb2754055ecf3e2b9c227106f572..d5c1defba27affde1b67b21fb6234fd8cb2ced67 100644 --- a/consumer/api/cpp/src/consumer_impl.h +++ b/consumer/api/cpp/src/consumer_impl.h @@ -35,7 +35,7 @@ struct RequestInfo { struct RequestOutput { std::string string_output; - FileData data_output; + MessageData data_output; uint64_t data_output_size; const char* to_string() const { if (!data_output) { @@ -76,11 +76,11 @@ class ConsumerImpl final : public asapo::Consumer { Error SetLastReadMarker(uint64_t value, std::string group_id) override; Error SetLastReadMarker(uint64_t value, std::string group_id, std::string stream) override; - Error GetNext(FileInfo* info, std::string group_id, FileData* data) override; - Error GetNext(FileInfo* info, std::string group_id, std::string stream, FileData* data) override; + Error GetNext(MessageMeta* info, std::string group_id, MessageData* data) override; + Error GetNext(MessageMeta* info, std::string group_id, std::string stream, MessageData* data) override; - Error GetLast(FileInfo* info, FileData* data) override; - Error GetLast(FileInfo* info, std::string stream, FileData* data) override; + Error GetLast(MessageMeta* info, MessageData* data) override; + Error GetLast(MessageMeta* info, std::string stream, MessageData* data) override; std::string GenerateNewGroupId(Error* err) override; std::string GetBeamtimeMeta(Error* err) override; @@ -88,8 +88,8 @@ class ConsumerImpl final : public asapo::Consumer { uint64_t GetCurrentSize(Error* err) override; uint64_t GetCurrentSize(std::string stream, Error* err) override; - Error GetById(uint64_t id, FileInfo* info, FileData* data) override; - Error GetById(uint64_t id, FileInfo* info, std::string stream, FileData* data) override; + Error GetById(uint64_t id, MessageMeta* info, MessageData* data) override; + Error GetById(uint64_t id, MessageMeta* info, std::string stream, MessageData* data) override; void SetTimeout(uint64_t timeout_ms) override; @@ -97,8 +97,8 @@ class ConsumerImpl final : public asapo::Consumer { NetworkConnectionType CurrentConnectionType() const override; - FileInfos QueryImages(std::string query, Error* err) override; - FileInfos QueryImages(std::string query, std::string stream, Error* err) override; + MessageMetas QueryImages(std::string query, Error* err) override; + MessageMetas QueryImages(std::string query, std::string stream, Error* err) override; DataSet GetNextDataset(std::string group_id, uint64_t min_size, Error* err) override; DataSet GetNextDataset(std::string group_id, std::string stream, uint64_t min_size, Error* err) override; @@ -109,7 +109,7 @@ class ConsumerImpl final : public asapo::Consumer { DataSet GetDatasetById(uint64_t id, uint64_t min_size, Error* err) override; DataSet GetDatasetById(uint64_t id, std::string stream, uint64_t min_size, Error* err) override; - Error RetrieveData(FileInfo* info, FileData* data) override; + Error RetrieveData(MessageMeta* info, MessageData* data) override; StreamInfos GetStreamList(std::string from, Error* err) override; void SetResendNacs(bool resend, uint64_t delay_ms, uint64_t resend_attempts) override; @@ -122,8 +122,8 @@ class ConsumerImpl final : public asapo::Consumer { std::unique_ptr<NetClient> net_client__; std::mutex net_client_mutex__; // Required for the lazy initialization of net_client private: - Error GetDataFromFileTransferService(FileInfo* info, FileData* data, bool retry_with_new_token); - Error GetDataFromFile(FileInfo* info, FileData* data); + Error GetDataFromFileTransferService(MessageMeta* info, MessageData* data, bool retry_with_new_token); + Error GetDataFromFile(MessageMeta* info, MessageData* data); static const std::string kBrokerServiceName; static const std::string kFileTransferServiceName; std::string RequestWithToken(std::string uri); @@ -131,23 +131,23 @@ class ConsumerImpl final : public asapo::Consumer { bool dataset = false, uint64_t min_size = 0); Error GetRecordFromServerById(uint64_t id, std::string* info, std::string group_id, std::string stream, bool dataset = false, uint64_t min_size = 0); - Error GetDataIfNeeded(FileInfo* info, FileData* data); + Error GetDataIfNeeded(MessageMeta* info, MessageData* data); Error DiscoverService(const std::string& service_name, std::string* uri_to_set); bool SwitchToGetByIdIfNoData(Error* err, const std::string& response, std::string* group_id,std::string* redirect_uri); bool SwitchToGetByIdIfPartialData(Error* err, const std::string& response, std::string* group_id,std::string* redirect_uri); Error ProcessRequest(RequestOutput* response, const RequestInfo& request, std::string* service_uri); Error GetImageFromServer(GetImageServerOperation op, uint64_t id, std::string group_id, std::string stream, - FileInfo* info, FileData* data); + MessageMeta* info, MessageData* data); DataSet GetDatasetFromServer(GetImageServerOperation op, uint64_t id, std::string group_id, std::string stream, uint64_t min_size, Error* err); - bool DataCanBeInBuffer(const FileInfo* info); - Error TryGetDataFromBuffer(const FileInfo* info, FileData* data); - Error CreateNetClientAndTryToGetFile(const FileInfo* info, FileData* data); + bool DataCanBeInBuffer(const MessageMeta* info); + Error TryGetDataFromBuffer(const MessageMeta* info, MessageData* data); + Error CreateNetClientAndTryToGetFile(const MessageMeta* info, MessageData* data); Error ServiceRequestWithTimeout(const std::string& service_name, std::string* service_uri, RequestInfo request, RequestOutput* response); std::string BrokerRequestWithTimeout(RequestInfo request, Error* err); - Error FtsRequestWithTimeout(FileInfo* info, FileData* data); - Error FtsSizeRequestWithTimeout(FileInfo* info); + Error FtsRequestWithTimeout(MessageMeta* info, MessageData* data); + Error FtsSizeRequestWithTimeout(MessageMeta* info); Error ProcessPostRequest(const RequestInfo& request, RequestOutput* response, HttpCode* code); Error ProcessGetRequest(const RequestInfo& request, RequestOutput* response, HttpCode* code); @@ -165,7 +165,7 @@ class ConsumerImpl final : public asapo::Consumer { NetworkConnectionType current_connection_type_ = NetworkConnectionType::kUndefined; std::string folder_token_; RequestInfo CreateFolderTokenRequest() const; - RequestInfo CreateFileTransferRequest(const FileInfo* info) const; + RequestInfo CreateFileTransferRequest(const MessageMeta* info) const; uint64_t resend_timout_ = 0; bool resend_ = false; uint64_t delay_ms_; diff --git a/consumer/api/cpp/src/fabric_consumer_client.cpp b/consumer/api/cpp/src/fabric_consumer_client.cpp index 9027a3295254a067e83d7cfcf565be277c2d61ea..8c80895cb4a1048fb10cab4128677e58f7dbe6b8 100644 --- a/consumer/api/cpp/src/fabric_consumer_client.cpp +++ b/consumer/api/cpp/src/fabric_consumer_client.cpp @@ -10,7 +10,7 @@ FabricConsumerClient::FabricConsumerClient(): factory__(fabric::GenerateDefaultF } -Error FabricConsumerClient::GetData(const FileInfo* info, FileData* data) { +Error FabricConsumerClient::GetData(const MessageMeta* info, MessageData* data) { Error err; if (!client__) { client__ = factory__->CreateClient(&err); @@ -24,7 +24,7 @@ Error FabricConsumerClient::GetData(const FileInfo* info, FileData* data) { return err; } - FileData tempData{new uint8_t[info->size]}; + MessageData tempData{new uint8_t[info->size]}; /* MemoryRegion will be released when out of scope */ auto mr = client__->ShareMemoryRegion(tempData.get(), info->size, &err); @@ -50,7 +50,7 @@ Error FabricConsumerClient::GetData(const FileInfo* info, FileData* data) { return nullptr; } -fabric::FabricAddress FabricConsumerClient::GetAddressOrConnect(const FileInfo* info, Error* error) { +fabric::FabricAddress FabricConsumerClient::GetAddressOrConnect(const MessageMeta* info, Error* error) { std::lock_guard<std::mutex> lock(mutex_); auto tableEntry = known_addresses_.find(info->source); diff --git a/consumer/api/cpp/src/fabric_consumer_client.h b/consumer/api/cpp/src/fabric_consumer_client.h index 6816625b083eeb339a2cd891a05fcd0b89fbec73..a9f47ac11bfffa50b7109eeeaabb392dfc65952b 100644 --- a/consumer/api/cpp/src/fabric_consumer_client.h +++ b/consumer/api/cpp/src/fabric_consumer_client.h @@ -25,9 +25,9 @@ class FabricConsumerClient : public NetClient { std::atomic<fabric::FabricMessageId> global_message_id_{0}; public: - Error GetData(const FileInfo* info, FileData* data) override; + Error GetData(const MessageMeta* info, MessageData* data) override; private: - fabric::FabricAddress GetAddressOrConnect(const FileInfo* info, Error* error); + fabric::FabricAddress GetAddressOrConnect(const MessageMeta* info, Error* error); void PerformNetworkTransfer(fabric::FabricAddress address, const GenericRequestHeader* request_header, GenericNetworkResponse* response, Error* err); diff --git a/consumer/api/cpp/src/net_client.h b/consumer/api/cpp/src/net_client.h index 800c2ea5c840775b67ed220c4a7f22fe803f70ba..831b9b45677af84e74e9d312769ab82cf1ad3e5a 100644 --- a/consumer/api/cpp/src/net_client.h +++ b/consumer/api/cpp/src/net_client.h @@ -8,7 +8,7 @@ namespace asapo { class NetClient { public: - virtual Error GetData(const FileInfo* info, FileData* data) = 0; + virtual Error GetData(const MessageMeta* info, MessageData* data) = 0; virtual ~NetClient() = default; }; diff --git a/consumer/api/cpp/src/tcp_consumer_client.cpp b/consumer/api/cpp/src/tcp_consumer_client.cpp index 5ee4ffed110fcd169d63d9ff143710fd0c571e75..a2bcc349cb90152409dfc3a68f808cfeed68a5dd 100644 --- a/consumer/api/cpp/src/tcp_consumer_client.cpp +++ b/consumer/api/cpp/src/tcp_consumer_client.cpp @@ -10,7 +10,7 @@ TcpConsumerClient::TcpConsumerClient() : io__{GenerateDefaultIO()}, connection_p } -Error TcpConsumerClient::SendGetDataRequest(SocketDescriptor sd, const FileInfo* info) const noexcept { +Error TcpConsumerClient::SendGetDataRequest(SocketDescriptor sd, const MessageMeta* info) const noexcept { Error err; GenericRequestHeader request_header{kOpcodeGetBufferData, info->buf_id, info->size}; io__->Send(sd, &request_header, sizeof(request_header), &err); @@ -21,7 +21,7 @@ Error TcpConsumerClient::SendGetDataRequest(SocketDescriptor sd, const FileInfo* return err; } -Error TcpConsumerClient::ReconnectAndResendGetDataRequest(SocketDescriptor* sd, const FileInfo* info) const noexcept { +Error TcpConsumerClient::ReconnectAndResendGetDataRequest(SocketDescriptor* sd, const MessageMeta* info) const noexcept { Error err; *sd = connection_pool__->Reconnect(*sd, &err); if (err) { @@ -55,7 +55,7 @@ Error TcpConsumerClient::ReceiveResponce(SocketDescriptor sd) const noexcept { return nullptr; } -Error TcpConsumerClient::QueryCacheHasData(SocketDescriptor* sd, const FileInfo* info, bool try_reconnect) const noexcept { +Error TcpConsumerClient::QueryCacheHasData(SocketDescriptor* sd, const MessageMeta* info, bool try_reconnect) const noexcept { Error err; err = SendGetDataRequest(*sd, info); if (err && try_reconnect) { @@ -68,7 +68,7 @@ Error TcpConsumerClient::QueryCacheHasData(SocketDescriptor* sd, const FileInfo* return ReceiveResponce(*sd); } -Error TcpConsumerClient::ReceiveData(SocketDescriptor sd, const FileInfo* info, FileData* data) const noexcept { +Error TcpConsumerClient::ReceiveData(SocketDescriptor sd, const MessageMeta* info, MessageData* data) const noexcept { Error err; uint8_t* data_array = nullptr; try { @@ -80,7 +80,7 @@ Error TcpConsumerClient::ReceiveData(SocketDescriptor sd, const FileInfo* info, io__->Receive(sd, data_array, (size_t)info->size, &err); connection_pool__->ReleaseConnection(sd); if (!err) { - *data = FileData{data_array}; + *data = MessageData{data_array}; } else { io__->CloseSocket(sd, nullptr); delete[] data_array; @@ -88,7 +88,7 @@ Error TcpConsumerClient::ReceiveData(SocketDescriptor sd, const FileInfo* info, return err; } -Error TcpConsumerClient::GetData(const FileInfo* info, FileData* data) { +Error TcpConsumerClient::GetData(const MessageMeta* info, MessageData* data) { Error err; bool reused; auto sd = connection_pool__->GetFreeConnection(info->source, &reused, &err); diff --git a/consumer/api/cpp/src/tcp_consumer_client.h b/consumer/api/cpp/src/tcp_consumer_client.h index cc96afc7b1a78f176065776318c4431f0d2809a7..b7961ee44ab88af22bc45ab9a87a55a5e1ab8b9e 100644 --- a/consumer/api/cpp/src/tcp_consumer_client.h +++ b/consumer/api/cpp/src/tcp_consumer_client.h @@ -10,15 +10,15 @@ namespace asapo { class TcpConsumerClient : public NetClient { public: explicit TcpConsumerClient(); - Error GetData(const FileInfo* info, FileData* data) override; + Error GetData(const MessageMeta* info, MessageData* data) override; std::unique_ptr<IO> io__; std::unique_ptr<TcpConnectionPool> connection_pool__; private: - Error SendGetDataRequest(SocketDescriptor sd, const FileInfo* info) const noexcept; - Error ReconnectAndResendGetDataRequest(SocketDescriptor* sd, const FileInfo* info) const noexcept; + Error SendGetDataRequest(SocketDescriptor sd, const MessageMeta* info) const noexcept; + Error ReconnectAndResendGetDataRequest(SocketDescriptor* sd, const MessageMeta* info) const noexcept; Error ReceiveResponce(SocketDescriptor sd) const noexcept; - Error QueryCacheHasData(SocketDescriptor* sd, const FileInfo* info, bool try_reconnect) const noexcept; - Error ReceiveData(SocketDescriptor sd, const FileInfo* info, FileData* data) const noexcept; + Error QueryCacheHasData(SocketDescriptor* sd, const MessageMeta* info, bool try_reconnect) const noexcept; + Error ReceiveData(SocketDescriptor sd, const MessageMeta* info, MessageData* data) const noexcept; }; } diff --git a/consumer/api/cpp/unittests/mocking.h b/consumer/api/cpp/unittests/mocking.h index 06a1326a43594a9124b36a1043bf6a55cb7c3321..5ded4240f8ff721aae94d699c00d9d1dddd49479 100644 --- a/consumer/api/cpp/unittests/mocking.h +++ b/consumer/api/cpp/unittests/mocking.h @@ -12,11 +12,11 @@ namespace asapo { class MockNetClient : public asapo::NetClient { public: - Error GetData(const FileInfo* info, FileData* data) override { + Error GetData(const MessageMeta* info, MessageData* data) override { return Error(GetData_t(info, data)); } - MOCK_CONST_METHOD2(GetData_t, ErrorInterface * (const FileInfo* info, FileData* data)); + MOCK_CONST_METHOD2(GetData_t, ErrorInterface * (const MessageMeta* info, MessageData* data)); }; diff --git a/consumer/api/cpp/unittests/test_consumer_impl.cpp b/consumer/api/cpp/unittests/test_consumer_impl.cpp index 9fb1918c89673d5d18c7750a700fc553f0d19de6..f2defc92df19a6f82d2c4c6ecf1e10e7c640d802 100644 --- a/consumer/api/cpp/unittests/test_consumer_impl.cpp +++ b/consumer/api/cpp/unittests/test_consumer_impl.cpp @@ -19,8 +19,8 @@ using asapo::ConsumerFactory; using asapo::Consumer; using asapo::ConsumerImpl; using asapo::IO; -using asapo::FileInfo; -using asapo::FileData; +using asapo::MessageMeta; +using asapo::MessageData; using asapo::MockIO; using asapo::MockHttpClient; using asapo::MockNetClient; @@ -63,7 +63,7 @@ class ConsumerImplTests : public Test { NiceMock<MockIO> mock_io; NiceMock<MockHttpClient> mock_http_client; NiceMock<MockNetClient> mock_netclient; - FileInfo info; + MessageMeta info; std::string expected_server_uri = "test:8400"; std::string expected_broker_uri = "asapo-broker:5005"; std::string expected_fts_uri = "asapo-file-transfer:5008"; @@ -142,7 +142,7 @@ class ConsumerImplTests : public Test { Return(result))); } - void MockBeforeFTS(FileData* data); + void MockBeforeFTS(MessageData* data); void MockGetFTSUri() { MockGetServiceUri("asapo-file-transfer", expected_fts_uri); @@ -166,8 +166,8 @@ class ConsumerImplTests : public Test { EXPECT_CALL(mock_io, GetDataFromFile_t(expected_full_path, testing::Pointee(100), _)).Times(times). WillRepeatedly(DoAll(SetArgPointee<2>(new asapo::SimpleError{"s"}), testing::Return(nullptr))); } - FileInfo CreateFI(uint64_t buf_id = expected_buf_id) { - FileInfo fi; + MessageMeta CreateFI(uint64_t buf_id = expected_buf_id) { + MessageMeta fi; fi.size = expected_image_size; fi.id = 1; fi.buf_id = buf_id; @@ -475,7 +475,7 @@ TEST_F(ConsumerImplTests, GetNextImageReturnsImmediatelyOnFinshedStream) { ASSERT_THAT(err, Eq(asapo::ConsumerErrorTemplates::kStreamFinished)); } -TEST_F(ConsumerImplTests, GetImageReturnsFileInfo) { +TEST_F(ConsumerImplTests, GetImageReturnsMessageMeta) { MockGetBrokerUri(); auto to_send = CreateFI(); @@ -517,7 +517,7 @@ TEST_F(ConsumerImplTests, GetImageTriesToGetDataFromMemoryCache) { auto to_send = CreateFI(); auto json = to_send.Json(); MockGet(json); - FileData data; + MessageData data; EXPECT_CALL(mock_netclient, GetData_t(&info, &data)).WillOnce(Return(nullptr)); MockReadDataFromFile(0); @@ -534,7 +534,7 @@ TEST_F(ConsumerImplTests, GetImageCallsReadFromFileIfCannotReadFromCache) { auto json = to_send.Json(); MockGet(json); - FileData data; + MessageData data; EXPECT_CALL(mock_netclient, GetData_t(&info, &data)).WillOnce(Return(asapo::IOErrorTemplates::kUnknownIOError.Generate().release())); @@ -550,7 +550,7 @@ TEST_F(ConsumerImplTests, GetImageCallsReadFromFileIfZeroBufId) { auto json = to_send.Json(); MockGet(json); - FileData data; + MessageData data; EXPECT_CALL(mock_netclient, GetData_t(_, _)).Times(0); @@ -928,7 +928,7 @@ TEST_F(ConsumerImplTests, GetNextDatasetUsesCorrectUri) { consumer->GetNextDataset(expected_group_id, 0, &err); } -TEST_F(ConsumerImplTests, GetDataSetReturnsFileInfos) { +TEST_F(ConsumerImplTests, GetDataSetReturnsMessageMetas) { asapo::Error err; MockGetBrokerUri(); @@ -960,7 +960,7 @@ TEST_F(ConsumerImplTests, GetDataSetReturnsFileInfos) { ASSERT_THAT(dataset.content[2].id, Eq(to_send3.id)); } -TEST_F(ConsumerImplTests, GetDataSetReturnsPartialFileInfos) { +TEST_F(ConsumerImplTests, GetDataSetReturnsPartialMessageMetas) { asapo::Error err; MockGetBrokerUri(); @@ -995,7 +995,7 @@ TEST_F(ConsumerImplTests, GetDataSetReturnsPartialFileInfos) { ASSERT_THAT(dataset.content[1].id, Eq(to_send2.id)); } -TEST_F(ConsumerImplTests, GetDataSetByIdReturnsPartialFileInfos) { +TEST_F(ConsumerImplTests, GetDataSetByIdReturnsPartialMessageMetas) { asapo::Error err; MockGetBrokerUri(); @@ -1119,7 +1119,7 @@ TEST_F(ConsumerImplTests, GetStreamListUsesCorrectUriWithoutFrom) { auto streams = consumer->GetStreamList("", &err); } -void ConsumerImplTests::MockBeforeFTS(FileData* data) { +void ConsumerImplTests::MockBeforeFTS(MessageData* data) { auto to_send = CreateFI(); auto json = to_send.Json(); MockGet(json); @@ -1144,7 +1144,7 @@ void ConsumerImplTests::ExpectFolderToken() { ACTION_P(AssignArg3, assign) { if (assign) { - asapo::FileData data = asapo::FileData{new uint8_t[1]}; + asapo::MessageData data = asapo::MessageData{new uint8_t[1]}; data[0] = expected_value; *arg3 = std::move(data); } @@ -1180,7 +1180,7 @@ void ConsumerImplTests::ExpectRepeatedFileTransfer() { } void ConsumerImplTests::AssertSingleFileTransfer() { - asapo::FileData data = asapo::FileData{new uint8_t[1]}; + asapo::MessageData data = asapo::MessageData{new uint8_t[1]}; MockGetBrokerUri(); MockBeforeFTS(&data); ExpectFolderToken(); @@ -1220,7 +1220,7 @@ TEST_F(ConsumerImplTests, FileTransferReadsFileSize) { Return(nullptr) )); - FileData data; + MessageData data; info.size = 0; info.buf_id = 0; auto err = fts_consumer->RetrieveData(&info, &data); @@ -1229,7 +1229,7 @@ TEST_F(ConsumerImplTests, FileTransferReadsFileSize) { TEST_F(ConsumerImplTests, GetImageReusesTokenAndUri) { AssertSingleFileTransfer(); - asapo::FileData data = asapo::FileData{new uint8_t[1]}; + asapo::MessageData data = asapo::MessageData{new uint8_t[1]}; MockBeforeFTS(&data); ExpectFileTransfer(nullptr); @@ -1239,7 +1239,7 @@ TEST_F(ConsumerImplTests, GetImageReusesTokenAndUri) { TEST_F(ConsumerImplTests, GetImageTriesToGetTokenAgainIfTransferFailed) { AssertSingleFileTransfer(); - asapo::FileData data; + asapo::MessageData data; MockBeforeFTS(&data); ExpectRepeatedFileTransfer(); ExpectFolderToken(); diff --git a/consumer/api/cpp/unittests/test_fabric_consumer_client.cpp b/consumer/api/cpp/unittests/test_fabric_consumer_client.cpp index 4db2da2ec917ab05268fc25b692fc507baefc638..9e32cd3842cc11d96003b342ade885a0d8d7e8a7 100644 --- a/consumer/api/cpp/unittests/test_fabric_consumer_client.cpp +++ b/consumer/api/cpp/unittests/test_fabric_consumer_client.cpp @@ -25,7 +25,7 @@ TEST(FabricConsumerClient, Constructor) { ASSERT_THAT(dynamic_cast<fabric::FabricClient*>(client.client__.get()), Eq(nullptr)); } -MATCHER_P6(M_CheckSendDataRequest, op_code, buf_id, data_size, mr_addr, mr_length, mr_key, +MATCHER_P6(M_CheckSendRequest, op_code, buf_id, data_size, mr_addr, mr_length, mr_key, "Checks if a valid GenericRequestHeader was Send") { auto data = (GenericRequestHeader*) arg; auto mr = (fabric::MemoryRegionDetails*) &data->message; @@ -37,9 +37,9 @@ MATCHER_P6(M_CheckSendDataRequest, op_code, buf_id, data_size, mr_addr, mr_lengt && mr->key == uint64_t(mr_key); } -ACTION_P(A_WriteSendDataResponse, error_code) { - ((asapo::SendDataResponse*)arg2)->op_code = asapo::kOpcodeGetBufferData; - ((asapo::SendDataResponse*)arg2)->error_code = error_code; +ACTION_P(A_WriteSendResponse, error_code) { + ((asapo::SendResponse*)arg2)->op_code = asapo::kOpcodeGetBufferData; + ((asapo::SendResponse*)arg2)->error_code = error_code; } class FabricConsumerClientTests : public Test { @@ -98,7 +98,7 @@ void FabricConsumerClientTests::ExpectTransfer(void** outputData, fabric::Fabric Expectation sendCall = EXPECT_CALL(mock_fabric_client, Send_t(serverAddr, messageId, - M_CheckSendDataRequest(kOpcodeGetBufferData, 78954, 4123, 0x124, 4123, 20), + M_CheckSendRequest(kOpcodeGetBufferData, 78954, 4123, 0x124, 4123, 20), sizeof(GenericRequestHeader), _)).After(getDetailsCall) .WillOnce(SetArgPointee<4>(sendOk ? nullptr : fabric::FabricErrorTemplates::kInternalError.Generate().release())); @@ -108,7 +108,7 @@ void FabricConsumerClientTests::ExpectTransfer(void** outputData, fabric::Fabric .After(sendCall) .WillOnce(DoAll( SetArgPointee<4>(recvOk ? nullptr : fabric::FabricErrorTemplates::kInternalError.Generate().release()), - A_WriteSendDataResponse(serverResponse) + A_WriteSendResponse(serverResponse) )); EXPECT_CALL(*mr, Destructor()).After(recvCall); } else { @@ -120,10 +120,10 @@ void FabricConsumerClientTests::ExpectTransfer(void** outputData, fabric::Fabric TEST_F(FabricConsumerClientTests, GetData_Error_Init) { ExpectInit(false); - FileData expectedFileData; - FileInfo expectedInfo{}; + MessageData expectedMessageData; + MessageMeta expectedInfo{}; expectedInfo.source = "host:1234"; - Error err = client.GetData(&expectedInfo, &expectedFileData); + Error err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Eq(fabric::FabricErrorTemplates::kInternalError)); } @@ -132,15 +132,15 @@ TEST_F(FabricConsumerClientTests, GetData_Error_AddConnection) { ExpectInit(true); ExpectAddedConnection("host:1234", false, -1); - FileData expectedFileData; - FileInfo expectedInfo{}; + MessageData expectedMessageData; + MessageMeta expectedInfo{}; expectedInfo.source = "host:1234"; - Error err = client.GetData(&expectedInfo, &expectedFileData); + Error err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Eq(fabric::FabricErrorTemplates::kInternalError)); // Make sure that the connection was not saved ExpectAddedConnection("host:1234", false, -1); - err = client.GetData(&expectedInfo, &expectedFileData); + err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Eq(fabric::FabricErrorTemplates::kInternalError)); } @@ -149,8 +149,8 @@ TEST_F(FabricConsumerClientTests, GetData_ShareMemoryRegion_Error) { ExpectInit(true); ExpectAddedConnection("host:1234", true, 0); - FileData expectedFileData; - FileInfo expectedInfo{}; + MessageData expectedMessageData; + MessageMeta expectedInfo{}; expectedInfo.source = "host:1234"; expectedInfo.size = 4123; @@ -160,7 +160,7 @@ TEST_F(FabricConsumerClientTests, GetData_ShareMemoryRegion_Error) { Return(nullptr) )); - Error err = client.GetData(&expectedInfo, &expectedFileData); + Error err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Eq(fabric::FabricErrorTemplates::kInternalError)); } @@ -169,8 +169,8 @@ TEST_F(FabricConsumerClientTests, GetData_SendFailed) { ExpectInit(true); ExpectAddedConnection("host:1234", true, 0); - FileData expectedFileData; - FileInfo expectedInfo{}; + MessageData expectedMessageData; + MessageMeta expectedInfo{}; expectedInfo.source = "host:1234"; expectedInfo.size = 4123; expectedInfo.buf_id = 78954; @@ -178,18 +178,18 @@ TEST_F(FabricConsumerClientTests, GetData_SendFailed) { void* outData = nullptr; ExpectTransfer(&outData, 0, 0, false, false, kNetErrorNoError); - Error err = client.GetData(&expectedInfo, &expectedFileData); + Error err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Ne(nullptr)); - ASSERT_THAT(expectedFileData.get(), Eq(nullptr)); + ASSERT_THAT(expectedMessageData.get(), Eq(nullptr)); } TEST_F(FabricConsumerClientTests, GetData_RecvFailed) { ExpectInit(true); ExpectAddedConnection("host:1234", true, 0); - FileData expectedFileData; - FileInfo expectedInfo{}; + MessageData expectedMessageData; + MessageMeta expectedInfo{}; expectedInfo.source = "host:1234"; expectedInfo.size = 4123; expectedInfo.buf_id = 78954; @@ -197,18 +197,18 @@ TEST_F(FabricConsumerClientTests, GetData_RecvFailed) { void* outData = nullptr; ExpectTransfer(&outData, 0, 0, true, false, kNetErrorNoError); - Error err = client.GetData(&expectedInfo, &expectedFileData); + Error err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Ne(nullptr)); - ASSERT_THAT(expectedFileData.get(), Eq(nullptr)); + ASSERT_THAT(expectedMessageData.get(), Eq(nullptr)); } TEST_F(FabricConsumerClientTests, GetData_ServerError) { ExpectInit(true); ExpectAddedConnection("host:1234", true, 0); - FileData expectedFileData; - FileInfo expectedInfo{}; + MessageData expectedMessageData; + MessageMeta expectedInfo{}; expectedInfo.source = "host:1234"; expectedInfo.size = 4123; expectedInfo.buf_id = 78954; @@ -216,18 +216,18 @@ TEST_F(FabricConsumerClientTests, GetData_ServerError) { void* outData = nullptr; ExpectTransfer(&outData, 0, 0, true, true, kNetErrorInternalServerError); - Error err = client.GetData(&expectedInfo, &expectedFileData); + Error err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Ne(nullptr)); - ASSERT_THAT(expectedFileData.get(), Eq(nullptr)); + ASSERT_THAT(expectedMessageData.get(), Eq(nullptr)); } TEST_F(FabricConsumerClientTests, GetData_Ok) { ExpectInit(true); ExpectAddedConnection("host:1234", true, 0); - FileData expectedFileData; - FileInfo expectedInfo{}; + MessageData expectedMessageData; + MessageMeta expectedInfo{}; expectedInfo.source = "host:1234"; expectedInfo.size = 4123; expectedInfo.buf_id = 78954; @@ -235,18 +235,18 @@ TEST_F(FabricConsumerClientTests, GetData_Ok) { void* outData = nullptr; ExpectTransfer(&outData, 0, 0, true, true, kNetErrorNoError); - Error err = client.GetData(&expectedInfo, &expectedFileData); + Error err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Eq(nullptr)); - ASSERT_THAT(expectedFileData.get(), Eq(outData)); + ASSERT_THAT(expectedMessageData.get(), Eq(outData)); } TEST_F(FabricConsumerClientTests, GetData_Ok_UsedCahedConnection) { ExpectInit(true); ExpectAddedConnection("host:1234", true, 0); - FileData expectedFileData; - FileInfo expectedInfo{}; + MessageData expectedMessageData; + MessageMeta expectedInfo{}; expectedInfo.source = "host:1234"; expectedInfo.size = 4123; expectedInfo.buf_id = 78954; @@ -254,26 +254,26 @@ TEST_F(FabricConsumerClientTests, GetData_Ok_UsedCahedConnection) { void* outData = nullptr; ExpectTransfer(&outData, 0, 0, true, true, kNetErrorNoError); - Error err = client.GetData(&expectedInfo, &expectedFileData); + Error err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Eq(nullptr)); - ASSERT_THAT(expectedFileData.get(), Eq(outData)); + ASSERT_THAT(expectedMessageData.get(), Eq(outData)); outData = nullptr; ExpectTransfer(&outData, 0, 1, true, true, kNetErrorNoError); - err = client.GetData(&expectedInfo, &expectedFileData); + err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Eq(nullptr)); - ASSERT_THAT(expectedFileData.get(), Eq(outData)); + ASSERT_THAT(expectedMessageData.get(), Eq(outData)); } TEST_F(FabricConsumerClientTests, GetData_Ok_SecondConnection) { ExpectInit(true); ExpectAddedConnection("host:1234", true, 0); - FileData expectedFileData; - FileInfo expectedInfo{}; + MessageData expectedMessageData; + MessageMeta expectedInfo{}; expectedInfo.source = "host:1234"; expectedInfo.size = 4123; expectedInfo.buf_id = 78954; @@ -281,10 +281,10 @@ TEST_F(FabricConsumerClientTests, GetData_Ok_SecondConnection) { void* outData = nullptr; ExpectTransfer(&outData, 0, 0, true, true, kNetErrorNoError); - Error err = client.GetData(&expectedInfo, &expectedFileData); + Error err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Eq(nullptr)); - ASSERT_THAT(expectedFileData.get(), Eq(outData)); + ASSERT_THAT(expectedMessageData.get(), Eq(outData)); ExpectAddedConnection("host:1235", true, 54); expectedInfo.source = "host:1235"; @@ -292,8 +292,8 @@ TEST_F(FabricConsumerClientTests, GetData_Ok_SecondConnection) { outData = nullptr; ExpectTransfer(&outData, 54, 1, true, true, kNetErrorNoError); - err = client.GetData(&expectedInfo, &expectedFileData); + err = client.GetData(&expectedInfo, &expectedMessageData); ASSERT_THAT(err, Eq(nullptr)); - ASSERT_THAT(expectedFileData.get(), Eq(outData)); + ASSERT_THAT(expectedMessageData.get(), Eq(outData)); } diff --git a/consumer/api/cpp/unittests/test_tcp_connection_pool.cpp b/consumer/api/cpp/unittests/test_tcp_connection_pool.cpp index fcb66a241e93023af7c3419958d5a7f4d8870632..74f3990dff74f074075a62a5a870d4a0bd309bf9 100644 --- a/consumer/api/cpp/unittests/test_tcp_connection_pool.cpp +++ b/consumer/api/cpp/unittests/test_tcp_connection_pool.cpp @@ -9,8 +9,8 @@ using asapo::IO; -using asapo::FileInfo; -using asapo::FileData; +using asapo::MessageMeta; +using asapo::MessageData; using asapo::MockIO; using asapo::SimpleError; using asapo::TcpConnectionPool; @@ -42,7 +42,7 @@ TEST(TcpConnectioPool, Constructor) { class TcpConnectioPoolTests : public Test { public: NiceMock<MockIO> mock_io; - FileInfo info; + MessageMeta info; std::string expected_source = "test:8400"; TcpConnectionPool pool; SocketDescriptor expected_sd = 123; diff --git a/consumer/api/cpp/unittests/test_tcp_consumer_client.cpp b/consumer/api/cpp/unittests/test_tcp_consumer_client.cpp index 08fee5bc4dc299d0b6144eb0bee2eca18cf42a2b..b1df9c9db1e6bd84ae1c21aba9968e0bc7c82a36 100644 --- a/consumer/api/cpp/unittests/test_tcp_consumer_client.cpp +++ b/consumer/api/cpp/unittests/test_tcp_consumer_client.cpp @@ -9,8 +9,8 @@ #include "asapo/common/networking.h" using asapo::IO; -using asapo::FileInfo; -using asapo::FileData; +using asapo::MessageMeta; +using asapo::MessageData; using asapo::MockIO; using asapo::SimpleError; using asapo::TcpConsumerClient; @@ -39,7 +39,7 @@ TEST(TcpClient, Constructor) { ASSERT_THAT(dynamic_cast<asapo::TcpConnectionPool*>(client->connection_pool__.get()), Ne(nullptr)); } -MATCHER_P4(M_CheckSendDataRequest, op_code, buf_id, data_size, message, +MATCHER_P4(M_CheckSendRequest, op_code, buf_id, data_size, message, "Checks if a valid GenericRequestHeader was Send") { return ((asapo::GenericRequestHeader*) arg)->op_code == op_code && ((asapo::GenericRequestHeader*) arg)->data_id == uint64_t(buf_id) @@ -47,9 +47,9 @@ MATCHER_P4(M_CheckSendDataRequest, op_code, buf_id, data_size, message, && strcmp(((asapo::GenericRequestHeader*) arg)->message, message) == 0; } -ACTION_P(A_WriteSendDataResponse, error_code) { - ((asapo::SendDataResponse*)arg1)->op_code = asapo::kOpcodeGetBufferData; - ((asapo::SendDataResponse*)arg1)->error_code = error_code; +ACTION_P(A_WriteSendResponse, error_code) { + ((asapo::SendResponse*)arg1)->op_code = asapo::kOpcodeGetBufferData; + ((asapo::SendResponse*)arg1)->error_code = error_code; } @@ -58,11 +58,11 @@ class TcpClientTests : public Test { std::unique_ptr<TcpConsumerClient> client = std::unique_ptr<TcpConsumerClient> {new TcpConsumerClient()}; NiceMock<MockIO> mock_io; NiceMock<MockTCPConnectionPool> mock_connection_pool; - FileInfo info; + MessageMeta info; std::string expected_uri = "test:8400"; uint64_t expected_buf_id = 123; uint64_t expected_size = 1233; - FileData data; + MessageData data; asapo::SocketDescriptor expected_sd = 1; void SetUp() override { info.source = expected_uri; @@ -93,8 +93,8 @@ class TcpClientTests : public Test { ); } - void ExpectSendDataRequest(asapo::SocketDescriptor sd, bool ok = true) { - EXPECT_CALL(mock_io, Send_t(sd, M_CheckSendDataRequest(asapo::kOpcodeGetBufferData, expected_buf_id, + void ExpectSendRequest(asapo::SocketDescriptor sd, bool ok = true) { + EXPECT_CALL(mock_io, Send_t(sd, M_CheckSendRequest(asapo::kOpcodeGetBufferData, expected_buf_id, expected_size, ""), sizeof(asapo::GenericRequestHeader), _)) .WillOnce( @@ -113,11 +113,11 @@ class TcpClientTests : public Test { void ExpectGetResponce(asapo::SocketDescriptor sd, bool ok, asapo::NetworkErrorCode responce_code) { - EXPECT_CALL(mock_io, Receive_t(sd, _, sizeof(asapo::SendDataResponse), _)) + EXPECT_CALL(mock_io, Receive_t(sd, _, sizeof(asapo::SendResponse), _)) .WillOnce( DoAll( testing::SetArgPointee<3>(ok ? nullptr : asapo::IOErrorTemplates::kConnectionRefused.Generate().release()), - A_WriteSendDataResponse(responce_code), + A_WriteSendResponse(responce_code), testing::ReturnArg<2>() )); if (!ok) { @@ -152,7 +152,7 @@ TEST_F(TcpClientTests, ErrorGetNewConnection) { TEST_F(TcpClientTests, SendHeaderForNewConnectionReturnsError) { ExpectNewConnection(false, true); - ExpectSendDataRequest(expected_sd, false); + ExpectSendRequest(expected_sd, false); auto err = client->GetData(&info, &data); @@ -161,7 +161,7 @@ TEST_F(TcpClientTests, SendHeaderForNewConnectionReturnsError) { TEST_F(TcpClientTests, OnErrorSendHeaderTriesToReconnectAndFails) { ExpectNewConnection(true, true); - ExpectSendDataRequest(expected_sd, false); + ExpectSendRequest(expected_sd, false); ExpectReconnect(false); auto err = client->GetData(&info, &data); @@ -171,9 +171,9 @@ TEST_F(TcpClientTests, OnErrorSendHeaderTriesToReconnectAndFails) { TEST_F(TcpClientTests, OnErrorSendHeaderTriesToReconnectAndSendsAnotherRequest) { ExpectNewConnection(true, true); - ExpectSendDataRequest(expected_sd, false); + ExpectSendRequest(expected_sd, false); ExpectReconnect(true); - ExpectSendDataRequest(expected_sd + 1, false); + ExpectSendRequest(expected_sd + 1, false); auto err = client->GetData(&info, &data); @@ -182,7 +182,7 @@ TEST_F(TcpClientTests, OnErrorSendHeaderTriesToReconnectAndSendsAnotherRequest) TEST_F(TcpClientTests, GetResponceReturnsError) { ExpectNewConnection(false, true); - ExpectSendDataRequest(expected_sd, true); + ExpectSendRequest(expected_sd, true); ExpectGetResponce(expected_sd, false, asapo::kNetErrorNoError); auto err = client->GetData(&info, &data); @@ -192,7 +192,7 @@ TEST_F(TcpClientTests, GetResponceReturnsError) { TEST_F(TcpClientTests, GetResponceReturnsNoData) { ExpectNewConnection(false, true); - ExpectSendDataRequest(expected_sd, true); + ExpectSendRequest(expected_sd, true); ExpectGetResponce(expected_sd, true, asapo::kNetErrorNoData); EXPECT_CALL(mock_connection_pool, ReleaseConnection(expected_sd)); @@ -203,7 +203,7 @@ TEST_F(TcpClientTests, GetResponceReturnsNoData) { TEST_F(TcpClientTests, GetResponceReturnsWrongRequest) { ExpectNewConnection(false, true); - ExpectSendDataRequest(expected_sd, true); + ExpectSendRequest(expected_sd, true); ExpectGetResponce(expected_sd, true, asapo::kNetErrorWrongRequest); EXPECT_CALL(mock_io, CloseSocket_t(expected_sd, _)); @@ -214,7 +214,7 @@ TEST_F(TcpClientTests, GetResponceReturnsWrongRequest) { TEST_F(TcpClientTests, ErrorGettingData) { ExpectNewConnection(false, true); - ExpectSendDataRequest(expected_sd, true); + ExpectSendRequest(expected_sd, true); ExpectGetResponce(expected_sd, true, asapo::kNetErrorNoError); ExpectGetData(expected_sd, false); @@ -225,7 +225,7 @@ TEST_F(TcpClientTests, ErrorGettingData) { TEST_F(TcpClientTests, OkGettingData) { ExpectNewConnection(false, true); - ExpectSendDataRequest(expected_sd, true); + ExpectSendRequest(expected_sd, true); ExpectGetResponce(expected_sd, true, asapo::kNetErrorNoError); ExpectGetData(expected_sd, true); @@ -236,9 +236,9 @@ TEST_F(TcpClientTests, OkGettingData) { TEST_F(TcpClientTests, OkGettingDataWithReconnect) { ExpectNewConnection(true, true); - ExpectSendDataRequest(expected_sd, false); + ExpectSendRequest(expected_sd, false); ExpectReconnect(true); - ExpectSendDataRequest(expected_sd + 1, true); + ExpectSendRequest(expected_sd + 1, true); ExpectGetResponce(expected_sd + 1, true, asapo::kNetErrorNoError); ExpectGetData(expected_sd + 1, true); diff --git a/consumer/api/python/asapo_consumer.pxd b/consumer/api/python/asapo_consumer.pxd index ea868f8cb387d6caf7f6c841914b31b022bfe753..b7a7b450073e02da3e2fb835c9e40560b912aa22 100644 --- a/consumer/api/python/asapo_consumer.pxd +++ b/consumer/api/python/asapo_consumer.pxd @@ -23,24 +23,24 @@ cdef extern from "asapo_wrappers.h" namespace "asapo": cdef string GetErrorString(Error* err) cdef extern from "asapo/asapo_consumer.h" namespace "asapo": - cppclass FileData: + cppclass MessageData: uint8_t[] release() pass cdef extern from "asapo/asapo_consumer.h" namespace "asapo": - cppclass FileInfo: + cppclass MessageMeta: string Json() bool SetFromJson(string json_str) cppclass IdList: vector[uint64_t].iterator begin() vector[uint64_t].iterator end() - cppclass FileInfos: - vector[FileInfo].iterator begin() - vector[FileInfo].iterator end() + cppclass MessageMetas: + vector[MessageMeta].iterator begin() + vector[MessageMeta].iterator end() struct DataSet: uint64_t id uint64_t expected_size - FileInfos content + MessageMetas content struct SourceCredentials: string beamtime_id string data_source @@ -62,9 +62,9 @@ cdef extern from "asapo/asapo_consumer.h" namespace "asapo" nogil: void SetTimeout(uint64_t timeout_ms) void ForceNoRdma() NetworkConnectionType CurrentConnectionType() - Error GetNext(FileInfo* info, string group_id,string stream, FileData* data) - Error GetLast(FileInfo* info, string stream, FileData* data) - Error GetById(uint64_t id, FileInfo* info, string stream, FileData* data) + Error GetNext(MessageMeta* info, string group_id,string stream, MessageData* data) + Error GetLast(MessageMeta* info, string stream, MessageData* data) + Error GetById(uint64_t id, MessageMeta* info, string stream, MessageData* data) uint64_t GetCurrentSize(string stream, Error* err) Error SetLastReadMarker(uint64_t value, string group_id, string stream) Error ResetLastReadMarker(string group_id, string stream) @@ -74,11 +74,11 @@ cdef extern from "asapo/asapo_consumer.h" namespace "asapo" nogil: IdList GetUnacknowledgedTupleIds(string group_id, string stream, uint64_t from_id, uint64_t to_id, Error* error) string GenerateNewGroupId(Error* err) string GetBeamtimeMeta(Error* err) - FileInfos QueryImages(string query, string stream, Error* err) + MessageMetas QueryImages(string query, string stream, Error* err) DataSet GetNextDataset(string group_id, string stream, uint64_t min_size, Error* err) DataSet GetLastDataset(string stream, uint64_t min_size, Error* err) DataSet GetDatasetById(uint64_t id, string stream, uint64_t min_size, Error* err) - Error RetrieveData(FileInfo* info, FileData* data) + Error RetrieveData(MessageMeta* info, MessageData* data) vector[StreamInfo] GetStreamList(string from_stream, Error* err) void SetResendNacs(bool resend, uint64_t delay_ms, uint64_t resend_attempts) void InterruptCurrentOperation() diff --git a/consumer/api/python/asapo_consumer.pyx.in b/consumer/api/python/asapo_consumer.pyx.in index 3ff5c94c119150f778d7f086d80a57bd62d2c631..a664a22aeaf8756034d00a95fca7b8602c4a2965 100644 --- a/consumer/api/python/asapo_consumer.pyx.in +++ b/consumer/api/python/asapo_consumer.pyx.in @@ -105,11 +105,11 @@ cdef throw_exception(Error& err, res = None): cdef class PyConsumer: cdef unique_ptr[Consumer] c_consumer def _op(self, op, group_id, stream, meta_only, uint64_t id): - cdef FileInfo info + cdef MessageMeta info cdef string b_group_id = _bytes(group_id) cdef string b_stream = _bytes(stream) - cdef FileData data - cdef FileData* p_data = <FileData*>NULL if meta_only else &data + cdef MessageData data + cdef MessageData* p_data = <MessageData*>NULL if meta_only else &data cdef Error err cdef np.npy_intp dims[1] if op == "next": @@ -140,11 +140,11 @@ cdef class PyConsumer: return self._op("id","",stream,meta_only,id) def retrieve_data(self,meta): json_str = json.dumps(meta) - cdef FileInfo info + cdef MessageMeta info if not info.SetFromJson(_bytes(json_str)): raise AsapoWrongInputError("wrong metadata") cdef Error err - cdef FileData data + cdef MessageData data with nogil: err = self.c_consumer.get().RetrieveData(&info, &data) if err: @@ -270,19 +270,19 @@ cdef class PyConsumer: cdef string b_query = _bytes(query) cdef string b_stream = _bytes(stream) cdef Error err - cdef FileInfos file_infos + cdef MessageMetas message_metas with nogil: - file_infos = self.c_consumer.get().QueryImages(b_query,b_stream,&err) + message_metas = self.c_consumer.get().QueryImages(b_query,b_stream,&err) if err: throw_exception(err) json_list = [] - for fi in file_infos: + for fi in message_metas: json_list.append(json.loads(_str(fi.Json()))) return json_list def _op_dataset(self, op, group_id, stream, uint64_t min_size, uint64_t id): cdef string b_group_id = _bytes(group_id) cdef string b_stream = _bytes(stream) - cdef FileInfos file_infos + cdef MessageMetas message_metas cdef DataSet dataset cdef Error err if op == "next": diff --git a/consumer/tools/folder_to_db/src/folder_db_importer.cpp b/consumer/tools/folder_to_db/src/folder_db_importer.cpp index 4d3520186eacdaf78ba760d97727ba1284eff129..833ade112f4e6aa09a6b19f4a732a7b6ac889230 100644 --- a/consumer/tools/folder_to_db/src/folder_db_importer.cpp +++ b/consumer/tools/folder_to_db/src/folder_db_importer.cpp @@ -20,12 +20,12 @@ Error FolderToDbImporter::ConnectToDb(const std::unique_ptr<asapo::Database>& db } Error FolderToDbImporter::ImportSingleFile(const std::unique_ptr<asapo::Database>& db, - const FileInfo& file) const { + const MessageMeta& file) const { return db->Insert(std::string(kDBDataCollectionNamePrefix) + "_default", file, ignore_duplicates_); } Error FolderToDbImporter::ImportFilelistChunk(const std::unique_ptr<asapo::Database>& db, - const FileInfos& file_list, uint64_t begin, uint64_t end) const { + const MessageMetas& file_list, uint64_t begin, uint64_t end) const { for (auto i = begin; i < end; i++) { auto err = ImportSingleFile(db, file_list[(size_t)i]); if (err != nullptr) { @@ -35,7 +35,7 @@ Error FolderToDbImporter::ImportFilelistChunk(const std::unique_ptr<asapo::Datab return nullptr; } -Error FolderToDbImporter::PerformParallelTask(const FileInfos& file_list, uint64_t begin, +Error FolderToDbImporter::PerformParallelTask(const MessageMetas& file_list, uint64_t begin, uint64_t end) const { Error err; auto db = CreateDbClient(&err); @@ -66,14 +66,14 @@ Error WaitParallelTasks(std::vector<std::future<Error>>* res) { } -TaskSplitParameters ComputeSplitParameters(const FileInfos& file_list, int ntasks) { +TaskSplitParameters ComputeSplitParameters(const MessageMetas& file_list, int ntasks) { TaskSplitParameters parameters; parameters.chunk = file_list.size() / ntasks; parameters.remainder = file_list.size() % ntasks; return parameters; } -void FolderToDbImporter::ProcessNextChunk(const FileInfos& file_list, +void FolderToDbImporter::ProcessNextChunk(const MessageMetas& file_list, std::vector<std::future<Error>>* res, TaskSplitParameters* p) const { p->next_chunk_size = p->chunk + (p->remainder ? 1 : 0); @@ -88,7 +88,7 @@ void FolderToDbImporter::ProcessNextChunk(const FileInfos& file_list, if (p->remainder) p->remainder -= 1; } -Error FolderToDbImporter::ImportFilelist(const FileInfos& file_list) const { +Error FolderToDbImporter::ImportFilelist(const MessageMetas& file_list) const { auto split_parameters = ComputeSplitParameters(file_list, n_tasks_); std::vector<std::future<Error>>res; @@ -100,7 +100,7 @@ Error FolderToDbImporter::ImportFilelist(const FileInfos& file_list) const { } -FileInfos FolderToDbImporter::GetFilesInFolder(const std::string& folder, Error* err) const { +MessageMetas FolderToDbImporter::GetFilesInFolder(const std::string& folder, Error* err) const { auto file_list = io__->FilesInFolder(folder, err); return file_list; } diff --git a/consumer/tools/folder_to_db/src/folder_db_importer.h b/consumer/tools/folder_to_db/src/folder_db_importer.h index 046cf4f10343e61fa8cee3dd199ec8ceb14411f1..9087b4a763914223125d25d3c91465a9c0434fa1 100644 --- a/consumer/tools/folder_to_db/src/folder_db_importer.h +++ b/consumer/tools/folder_to_db/src/folder_db_importer.h @@ -57,17 +57,17 @@ class FolderToDbImporter { mutable std::string db_uri_ ; mutable std::string db_name_; Error ConnectToDb(const std::unique_ptr<asapo::Database>& db) const; - FileInfos GetFilesInFolder(const std::string& folder, Error* err) const; - Error ImportFilelist(const FileInfos& file_list) const; - Error PerformParallelTask(const FileInfos& file_list, uint64_t begin, + MessageMetas GetFilesInFolder(const std::string& folder, Error* err) const; + Error ImportFilelist(const MessageMetas& file_list) const; + Error PerformParallelTask(const MessageMetas& file_list, uint64_t begin, uint64_t end) const; Error ImportSingleFile(const std::unique_ptr<asapo::Database>& db, - const FileInfo& file) const; + const MessageMeta& file) const; Error ImportFilelistChunk(const std::unique_ptr<asapo::Database>& db, - const FileInfos& file_list, uint64_t begin, uint64_t end) const; + const MessageMetas& file_list, uint64_t begin, uint64_t end) const; std::unique_ptr<Database> CreateDbClient(Error* err) const; - void ProcessNextChunk(const FileInfos& file_list, std::vector<std::future<Error>>* res, + void ProcessNextChunk(const MessageMetas& file_list, std::vector<std::future<Error>>* res, TaskSplitParameters* p) const; }; diff --git a/consumer/tools/folder_to_db/unittests/test_folder_to_db.cpp b/consumer/tools/folder_to_db/unittests/test_folder_to_db.cpp index 1347106a2666660f0829df8d07a3fbf6c1d5f26a..f14a72dd418cfac576bf864ed3b6434ce72a4442 100644 --- a/consumer/tools/folder_to_db/unittests/test_folder_to_db.cpp +++ b/consumer/tools/folder_to_db/unittests/test_folder_to_db.cpp @@ -95,17 +95,17 @@ class FakeDatabaseFactory : public DatabaseFactory { } }; -FileInfos CreateTestFileInfos() { - FileInfos file_infos; - FileInfo fi; +MessageMetas CreateTestMessageMetas() { + MessageMetas message_metas; + MessageMeta fi; fi.size = 100; fi.name = "1"; - file_infos.push_back(fi); + message_metas.push_back(fi); fi.name = "2"; - file_infos.push_back(fi); + message_metas.push_back(fi); fi.name = "3"; - file_infos.push_back(fi); - return file_infos; + message_metas.push_back(fi); + return message_metas; } class FolderDBConverterTests : public Test { @@ -114,20 +114,20 @@ class FolderDBConverterTests : public Test { NiceMock<MockIO> mock_io; std::string expected_collection_name = std::string(kDBDataCollectionNamePrefix) + "_default"; MockDatabaseFactory* mock_dbf; - FileInfos file_infos; + MessageMetas message_metas; std::string folder, uri, db_name; void SetUp() override { converter.io__ = std::unique_ptr<IO> {&mock_io}; mock_dbf = new MockDatabaseFactory; mock_dbf->CreateDBs(3); converter.db_factory__ = std::unique_ptr<DatabaseFactory> {mock_dbf}; - file_infos = CreateTestFileInfos(); + message_metas = CreateTestMessageMetas(); folder = "folder"; db_name = "db_name"; uri = "db_address"; ON_CALL(mock_io, FilesInFolder_t(_, _)). WillByDefault(DoAll(testing::SetArgPointee<1>(nullptr), - testing::Return(file_infos))); + testing::Return(message_metas))); } void TearDown() override { converter.io__.release(); @@ -167,7 +167,7 @@ TEST_F(FolderDBConverterTests, ErrorWhenCannotGetFileList) { EXPECT_CALL(mock_io, FilesInFolder_t(folder, _)). WillOnce(DoAll(testing::SetArgPointee<1>(new asapo::SimpleError("err")), - testing::Return(FileInfos {}))); + testing::Return(MessageMetas {}))); auto error = converter.Convert(uri, folder, db_name); ASSERT_THAT(error, Ne(nullptr)); @@ -193,9 +193,9 @@ TEST_F(FolderDBConverterTests, ErrorWhenCannotImportFileListToDb) { ASSERT_THAT(error, Ne(nullptr)); } -// a matcher to compare file_infos (size and basename only) for testing purposes -// (we do not want to create an == operator for FileInfo) -MATCHER_P(CompareFileInfo, file, "") { +// a matcher to compare message_metas (size and basename only) for testing purposes +// (we do not want to create an == operator for MessageMeta) +MATCHER_P(CompareMessageMeta, file, "") { if (arg.size != file.size) return false; if (arg.name != file.name) return false; return true; @@ -204,8 +204,8 @@ MATCHER_P(CompareFileInfo, file, "") { TEST_F(FolderDBConverterTests, PassesFileListToInsert) { - for (auto& file : file_infos) { - EXPECT_CALL(*(mock_dbf->db[0]), Insert_t(expected_collection_name, CompareFileInfo(file), _)). + for (auto& file : message_metas) { + EXPECT_CALL(*(mock_dbf->db[0]), Insert_t(expected_collection_name, CompareMessageMeta(file), _)). WillOnce(testing::Return(nullptr)); } @@ -216,11 +216,11 @@ TEST_F(FolderDBConverterTests, PassesFileListToInsert) { TEST_F(FolderDBConverterTests, PassesFileListToInsertInParallel3by3) { - EXPECT_CALL(*(mock_dbf->db[0]), Insert_t(expected_collection_name, CompareFileInfo(file_infos[0]), _)). + EXPECT_CALL(*(mock_dbf->db[0]), Insert_t(expected_collection_name, CompareMessageMeta(message_metas[0]), _)). WillOnce(testing::Return(nullptr)); - EXPECT_CALL(*(mock_dbf->db[1]), Insert_t(expected_collection_name, CompareFileInfo(file_infos[1]), _)). + EXPECT_CALL(*(mock_dbf->db[1]), Insert_t(expected_collection_name, CompareMessageMeta(message_metas[1]), _)). WillOnce(testing::Return(nullptr)); - EXPECT_CALL(*(mock_dbf->db[2]), Insert_t(expected_collection_name, CompareFileInfo(file_infos[2]), _)). + EXPECT_CALL(*(mock_dbf->db[2]), Insert_t(expected_collection_name, CompareMessageMeta(message_metas[2]), _)). WillOnce(testing::Return(nullptr)); converter.SetNParallelTasks(3, false); @@ -230,11 +230,11 @@ TEST_F(FolderDBConverterTests, PassesFileListToInsertInParallel3by3) { TEST_F(FolderDBConverterTests, PassesFileListToInsertInParallel3by2) { - EXPECT_CALL(*(mock_dbf->db[0]), Insert_t(expected_collection_name, CompareFileInfo(file_infos[0]), _)). + EXPECT_CALL(*(mock_dbf->db[0]), Insert_t(expected_collection_name, CompareMessageMeta(message_metas[0]), _)). WillOnce(testing::Return(nullptr)); - EXPECT_CALL(*(mock_dbf->db[0]), Insert_t(expected_collection_name, CompareFileInfo(file_infos[1]), _)). + EXPECT_CALL(*(mock_dbf->db[0]), Insert_t(expected_collection_name, CompareMessageMeta(message_metas[1]), _)). WillOnce(testing::Return(nullptr)); - EXPECT_CALL(*(mock_dbf->db[1]), Insert_t(expected_collection_name, CompareFileInfo(file_infos[2]), _)). + EXPECT_CALL(*(mock_dbf->db[1]), Insert_t(expected_collection_name, CompareMessageMeta(message_metas[2]), _)). WillOnce(testing::Return(nullptr)); converter.SetNParallelTasks(2, false); @@ -245,7 +245,7 @@ TEST_F(FolderDBConverterTests, PassesFileListToInsertInParallel3by2) { TEST_F(FolderDBConverterTests, ComputesStatistics) { EXPECT_CALL(*mock_dbf->db[0], Insert_t(_, _, false)). - Times(file_infos.size()). + Times(message_metas.size()). WillRepeatedly(testing::Return(nullptr)); asapo::FolderImportStatistics statistics; @@ -256,7 +256,7 @@ TEST_F(FolderDBConverterTests, ComputesStatistics) { auto error = converter.Convert(uri, folder, db_name, &statistics); ASSERT_THAT(error, Eq(nullptr)); - ASSERT_THAT(statistics.n_files_converted, Eq(file_infos.size())); + ASSERT_THAT(statistics.n_files_converted, Eq(message_metas.size())); ASSERT_THAT(statistics.time_read_folder.count(), Ge(0)); ASSERT_THAT(statistics.time_import_files.count(), Ge(0)); } diff --git a/examples/consumer/getnext/getnext.cpp b/examples/consumer/getnext/getnext.cpp index e1d2d6ed0e55f25c7bcea9921a44aa036b659a22..4d40d8bd59cd2b79db3bdc0b93256635209019d5 100644 --- a/examples/consumer/getnext/getnext.cpp +++ b/examples/consumer/getnext/getnext.cpp @@ -95,7 +95,7 @@ StartThreads(const Args& params, std::vector<int>* nfiles, std::vector<int>* err std::vector<int>* nfiles_total, std::vector<asapo::NetworkConnectionType>* connection_type, LatchedTimer* timer) { auto exec_next = [¶ms, nfiles, errors, nbuf, nfiles_total, connection_type, timer](int i) { - asapo::FileInfo fi; + asapo::MessageMeta fi; Error err; auto consumer = asapo::ConsumerFactory::CreateConsumer(params.server, params.file_path, @@ -111,7 +111,7 @@ StartThreads(const Args& params, std::vector<int>* nfiles, std::vector<int>* err //consumer->ForceNoRdma(); consumer->SetTimeout((uint64_t) params.timeout_ms); - asapo::FileData data; + asapo::MessageData data; lock.lock(); if (group_id.empty()) { @@ -185,7 +185,7 @@ StartThreads(const Args& params, std::vector<int>* nfiles, std::vector<int>* err int ReadAllData(const Args& params, uint64_t* duration_ms, uint64_t* duration_without_first_ms, int* nerrors, int* nbuf, int* nfiles_total, asapo::NetworkConnectionType* connection_type) { - asapo::FileInfo fi; + asapo::MessageMeta fi; std::chrono::system_clock::time_point t1 = std::chrono::system_clock::now(); std::vector<int> nfiles(params.nthreads, 0); diff --git a/examples/consumer/simple-consumer/consume.cpp b/examples/consumer/simple-consumer/consume.cpp index a0a3b6d392e42594d32f1456d3e9a14fd4382094..501fd2af5113052bf978372ff50183bc4b1fe4f7 100644 --- a/examples/consumer/simple-consumer/consume.cpp +++ b/examples/consumer/simple-consumer/consume.cpp @@ -21,8 +21,8 @@ int main(int argc, char* argv[]) { auto group_id = consumer->GenerateNewGroupId(&err); exit_if_error("Cannot create group id", err); - asapo::FileInfo fi; - asapo::FileData data; + asapo::MessageMeta fi; + asapo::MessageData data; err = consumer->GetLast(&fi, group_id, &data); exit_if_error("Cannot get next record", err); diff --git a/examples/pipeline/in_to_out/in_to_out.cpp b/examples/pipeline/in_to_out/in_to_out.cpp index 06907afb8a4ec9e372b15a1614252ef24e4ff687..241773a5ccbde1308faa5be1fdaa26a1e713992f 100644 --- a/examples/pipeline/in_to_out/in_to_out.cpp +++ b/examples/pipeline/in_to_out/in_to_out.cpp @@ -98,16 +98,17 @@ void GetBeamtimeMeta(const ConsumerPtr& consumer) { } } -void SendDataDownstreamThePipeline(const Args& args, const asapo::FileInfo& fi, asapo::FileData data, +void SendDownstreamThePipeline(const Args& args, const asapo::MessageMeta& fi, asapo::MessageData data, const ProducerPtr& producer) { - asapo::EventHeader header{fi.id, fi.size, fi.name, fi.metadata}; + asapo::MessageHeader header{fi.id, fi.size, fi.name, fi.metadata}; Error err_send; if (args.transfer_data) { header.file_name += "_" + args.stream_out; - err_send = producer->SendData(header, std::move(data), asapo::kDefaultIngestMode, ProcessAfterSend); + err_send = producer->Send(header, std::move(data), asapo::kDefaultIngestMode, ProcessAfterSend); } else { header.file_name = args.file_path + asapo::kPathSeparator + header.file_name; - err_send = producer->SendData(header, nullptr, asapo::IngestModeFlags::kTransferMetaDataOnly, ProcessAfterSend); + err_send = + producer->Send(header, nullptr, asapo::IngestModeFlags::kTransferMetaDataOnly, ProcessAfterSend); std::cout << err_send << std::endl; } @@ -124,15 +125,15 @@ void SendDataDownstreamThePipeline(const Args& args, const asapo::FileInfo& fi, } Error ProcessNextEvent(const Args& args, const ConsumerPtr& consumer, const ProducerPtr& producer) { - asapo::FileData data; - asapo::FileInfo fi; + asapo::MessageData data; + asapo::MessageMeta fi; auto err = consumer->GetNext(&fi, group_id, args.transfer_data ? &data : nullptr); if (err) { return err; } - SendDataDownstreamThePipeline(args, fi, std::move(data), producer); + SendDownstreamThePipeline(args, fi, std::move(data), producer); return nullptr; } @@ -141,7 +142,7 @@ std::vector<std::thread> StartConsumerThreads(const Args& args, const ProducerPt std::vector<int>* nfiles, std::vector<int>* errors) { auto exec_next = [&args, nfiles, errors, &producer ](int i) { - asapo::FileInfo fi; + asapo::MessageMeta fi; Error err; auto consumer = CreateConsumerAndGroup(args, &err); if (err) { @@ -169,7 +170,7 @@ std::vector<std::thread> StartConsumerThreads(const Args& args, const ProducerPt } int ProcessAllData(const Args& args, const ProducerPtr& producer, uint64_t* duration_ms, int* nerrors) { - asapo::FileInfo fi; + asapo::MessageMeta fi; system_clock::time_point t1 = system_clock::now(); std::vector<int> nfiles(args.nthreads, 0); diff --git a/examples/producer/dummy-data-producer/dummy_data_producer.cpp b/examples/producer/dummy-data-producer/dummy_data_producer.cpp index 3e67b04ce4513720409949d1e6bde569a3213523..6516f2b345595d1205464e1646a6942909849ce8 100644 --- a/examples/producer/dummy-data-producer/dummy_data_producer.cpp +++ b/examples/producer/dummy-data-producer/dummy_data_producer.cpp @@ -124,8 +124,8 @@ void ProcessAfterMetaDataSend(asapo::RequestCallbackPayload payload, asapo::Erro return; } -asapo::FileData CreateMemoryBuffer(size_t size) { - return asapo::FileData(new uint8_t[size]); +asapo::MessageData CreateMemoryBuffer(size_t size) { + return asapo::MessageData(new uint8_t[size]); } @@ -134,7 +134,7 @@ bool SendDummyData(asapo::Producer* producer, size_t number_of_byte, uint64_t it asapo::Error err; if (iterations == 0) { - err = producer->SendMetaData("{\"dummy_meta\":\"test\"}", &ProcessAfterMetaDataSend); + err = producer->SendMetadata("{\"dummy_meta\":\"test\"}", &ProcessAfterMetaDataSend); if (err) { std::cerr << "Cannot send metadata: " << err << std::endl; return false; @@ -146,16 +146,16 @@ bool SendDummyData(asapo::Producer* producer, size_t number_of_byte, uint64_t it for (uint64_t i = 0; i < iterations; i++) { auto buffer = CreateMemoryBuffer(number_of_byte); - asapo::EventHeader event_header{i + 1, number_of_byte, std::to_string(i + 1)}; + asapo::MessageHeader message_header{i + 1, number_of_byte, std::to_string(i + 1)}; std::string meta = "{\"user_meta\":\"test" + std::to_string(i + 1) + "\"}"; if (!data_source.empty()) { - event_header.file_name = data_source + "/" + event_header.file_name; + message_header.file_name = data_source + "/" + message_header.file_name; } - event_header.file_name = image_folder+event_header.file_name; - event_header.user_metadata = std::move(meta); + message_header.file_name = image_folder+message_header.file_name; + message_header.user_metadata = std::move(meta); if (images_in_set == 1) { - auto err = producer->SendData(event_header, std::move(buffer), write_files ? asapo::kDefaultIngestMode : - asapo::kTransferData, &ProcessAfterSend); + auto err = producer->Send(message_header, std::move(buffer), write_files ? asapo::kDefaultIngestMode : + asapo::kTransferData, &ProcessAfterSend); if (err) { std::cerr << "Cannot send file: " << err << std::endl; return false; @@ -163,17 +163,18 @@ bool SendDummyData(asapo::Producer* producer, size_t number_of_byte, uint64_t it } else { for (uint64_t id = 0; id < images_in_set; id++) { auto buffer = CreateMemoryBuffer(number_of_byte); - event_header.id_in_subset = id + 1; - event_header.subset_size = images_in_set; - event_header.file_id = i + 1; - event_header.file_name = std::to_string(i + 1) + "_" + std::to_string(id + 1); + message_header.id_in_subset = id + 1; + message_header.subset_size = images_in_set; + message_header.message_id = i + 1; + message_header.file_name = std::to_string(i + 1) + "_" + std::to_string(id + 1); if (!data_source.empty()) { - event_header.file_name = data_source + "/" + event_header.file_name; + message_header.file_name = data_source + "/" + message_header.file_name; } - event_header.file_name = image_folder + event_header.file_name; - event_header.user_metadata = meta; - auto err = producer->SendData(event_header, std::move(buffer), write_files ? asapo::kDefaultIngestMode : - asapo::kTransferData, &ProcessAfterSend); + message_header.file_name = image_folder + message_header.file_name; + message_header.user_metadata = meta; + auto err = + producer->Send(message_header, std::move(buffer), write_files ? asapo::kDefaultIngestMode : + asapo::kTransferData, &ProcessAfterSend); if (err) { std::cerr << "Cannot send file: " << err << std::endl; return false; diff --git a/examples/producer/simple-producer/produce.cpp b/examples/producer/simple-producer/produce.cpp index 5ddfe5d41bd3ad88db69722541ef8c7e19b213fb..57c8b0fd68435876c35521b06efbb2980b6b194e 100644 --- a/examples/producer/simple-producer/produce.cpp +++ b/examples/producer/simple-producer/produce.cpp @@ -31,11 +31,11 @@ int main(int argc, char* argv[]) { std::string to_send = "hello"; auto send_size = to_send.size() + 1; - auto buffer = asapo::FileData(new uint8_t[send_size]); + auto buffer = asapo::MessageData(new uint8_t[send_size]); memcpy(buffer.get(), to_send.c_str(), send_size); - asapo::EventHeader event_header{1, send_size, "processed"+asapo::kPathseparator +"test_file"}; - err = producer->SendData(event_header, std::move(buffer), asapo::kDefaultIngestMode, &ProcessAfterSend); + asapo::EventHeader message_header{1, send_size, "processed"+asapo::kPathseparator +"test_file"}; + err = producer->Send(message_header, std::move(buffer), asapo::kDefaultIngestMode, &ProcessAfterSend); exit_if_error("Cannot send file", err); err = producer->WaitRequestsFinished(1000); diff --git a/producer/api/cpp/include/asapo/producer/common.h b/producer/api/cpp/include/asapo/producer/common.h index b1612c8cfe7ae742cc6f53b6cf5aca4f3286e95e..07dd2a110b9733876c2533225aa362ed565e6d49 100644 --- a/producer/api/cpp/include/asapo/producer/common.h +++ b/producer/api/cpp/include/asapo/producer/common.h @@ -14,7 +14,7 @@ const uint8_t kMaxProcessingThreads = 32; struct RequestCallbackPayload { GenericRequestHeader original_header; - FileData data; + MessageData data; std::string response; }; @@ -26,19 +26,19 @@ enum class RequestHandlerType { }; -struct EventHeader { - EventHeader() {}; - EventHeader(uint64_t file_id_i, uint64_t file_size_i, std::string file_name_i, - std::string user_metadata_i = "", - uint64_t id_in_subset_i = 0, - uint64_t subset_size_i = 0 ): - file_id{file_id_i}, file_size{file_size_i}, +struct MessageHeader { + MessageHeader() {}; + MessageHeader(uint64_t message_id_i, uint64_t data_size_i, std::string file_name_i, + std::string user_metadata_i = "", + uint64_t id_in_subset_i = 0, + uint64_t subset_size_i = 0 ): + message_id{message_id_i}, data_size{data_size_i}, file_name{std::move(file_name_i)}, user_metadata{std::move(user_metadata_i)}, id_in_subset{id_in_subset_i}, subset_size{subset_size_i} {}; - uint64_t file_id = 0; - uint64_t file_size = 0; + uint64_t message_id = 0; + uint64_t data_size = 0; std::string file_name; std::string user_metadata; uint64_t id_in_subset = 0; diff --git a/producer/api/cpp/include/asapo/producer/producer.h b/producer/api/cpp/include/asapo/producer/producer.h index 334b4ae72e55d8293184c75464e290ad48a3d404..848c94d2ccb578bd55ae4f0b40134b87126fe1cb 100644 --- a/producer/api/cpp/include/asapo/producer/producer.h +++ b/producer/api/cpp/include/asapo/producer/producer.h @@ -40,58 +40,58 @@ class Producer { virtual StreamInfo GetLastStream(uint64_t timeout_ms, Error* err) const = 0; - //! Sends data to the receiver + //! Sends message to the receiver /*! - \param event_header - A stucture with the meta information (file name, size, a string with user metadata (JSON format)). - \param data - A pointer to the data to send + \param message_header - A stucture with the meta information (file name, size, a string with user metadata (JSON format)). + \param data - A smart pointer to the message data to send \return Error - Will be nullptr on success */ - virtual Error SendData(const EventHeader& event_header, FileData data, uint64_t ingest_mode, - RequestCallback callback) = 0; + virtual Error Send(const MessageHeader& message_header, MessageData data, uint64_t ingest_mode, + RequestCallback callback) = 0; - //! Sends data to the receiver - same as SendData - memory should not be freed until send is finished + //! Sends message to the receiver - same as Send - memory should not be freed until send is finished //! used e.g. for Python bindings - virtual Error SendData__(const EventHeader& event_header, void* data, uint64_t ingest_mode, - RequestCallback callback) = 0; + virtual Error Send__(const MessageHeader& message_header, void* data, uint64_t ingest_mode, + RequestCallback callback) = 0; - //! Sends data to the receiver + //! Sends message to the receiver /*! - \param event_header - A stucture with the meta information (file name, size, a string with user metadata (JSON format)). - \param data - A pointer to the data to send + \param message_header - A stucture with the meta information (file name, size, a string with user metadata (JSON format)). + \param data - A smart pointer to the message data to send, can be nullptr \return Error - Will be nullptr on success */ - virtual Error SendData(const EventHeader& event_header, std::string stream, FileData data, uint64_t ingest_mode, - RequestCallback callback) = 0; + virtual Error Send(const MessageHeader& message_header, std::string stream, MessageData data, uint64_t ingest_mode, + RequestCallback callback) = 0; - //! Sends data to the receiver - same as SendData - memory should not be freed until send is finished + //! Sends data to the receiver - same as Send - memory should not be freed until send is finished //! used e.g. for Python bindings - virtual Error SendData__(const EventHeader& event_header, std::string stream, void* data, uint64_t ingest_mode, - RequestCallback callback) = 0; + virtual Error Send__(const MessageHeader& message_header, std::string stream, void* data, uint64_t ingest_mode, + RequestCallback callback) = 0; //! Stop processing threads //! used e.g. for Python bindings virtual void StopThreads__() = 0; - //! Sends files to the default stream + //! Sends message from a file to the default stream /*! - \param event_header - A stucture with the meta information (file name, size is ignored). + \param message_header - A stucture with the meta information (file name, size is ignored). \param full_path - A full path of the file to send \return Error - Will be nullptr on success */ - virtual Error SendFile(const EventHeader& event_header, std::string full_path, uint64_t ingest_mode, - RequestCallback callback) = 0; + virtual Error SendFromFile(const MessageHeader& message_header, std::string full_path, uint64_t ingest_mode, + RequestCallback callback) = 0; - //! Sends files to the stream + //! Sends message from a file to a stream /*! - \param event_header - A stucture with the meta information (file name, size is ignored). + \param message_header - A stucture with the meta information (file name, size is ignored). \param full_path - A full path of the file to send \return Error - Will be nullptr on success */ - virtual Error SendFile(const EventHeader& event_header, std::string stream, std::string full_path, - uint64_t ingest_mode, - RequestCallback callback) = 0; + virtual Error SendFromFile(const MessageHeader& message_header, std::string stream, std::string full_path, + uint64_t ingest_mode, + RequestCallback callback) = 0; //! Marks stream finished /*! @@ -110,7 +110,7 @@ class Producer { \param callback - callback function \return Error - will be nullptr on success */ - virtual Error SendMetaData(const std::string& metadata, RequestCallback callback) = 0; + virtual Error SendMetadata(const std::string& metadata, RequestCallback callback) = 0; //! Set internal log level virtual void SetLogLevel(LogLevel level) = 0; diff --git a/producer/api/cpp/src/producer_impl.cpp b/producer/api/cpp/src/producer_impl.cpp index 24bdea3c47e31c90c5154ccb178d4da9a5bea75e..72f2ee4a37ed8aeb6dc9a272fd72ade73a3ab989 100644 --- a/producer/api/cpp/src/producer_impl.cpp +++ b/producer/api/cpp/src/producer_impl.cpp @@ -34,14 +34,14 @@ ProducerImpl::ProducerImpl(std::string endpoint, uint8_t n_processing_threads, u request_pool__.reset(new RequestPool{n_processing_threads, request_handler_factory_.get(), log__}); } -GenericRequestHeader ProducerImpl::GenerateNextSendRequest(const EventHeader& event_header, std::string stream, - uint64_t ingest_mode) { - GenericRequestHeader request{kOpcodeTransferData, event_header.file_id, event_header.file_size, - event_header.user_metadata.size(), event_header.file_name, stream}; - if (event_header.id_in_subset != 0) { +GenericRequestHeader ProducerImpl::GenerateNextSendRequest(const MessageHeader& message_header, std::string stream, + uint64_t ingest_mode) { + GenericRequestHeader request{kOpcodeTransferData, message_header.message_id, message_header.data_size, + message_header.user_metadata.size(), message_header.file_name, stream}; + if (message_header.id_in_subset != 0) { request.op_code = kOpcodeTransferSubsetData; - request.custom_data[kPosDataSetId] = event_header.id_in_subset; - request.custom_data[kPosDataSetSize] = event_header.subset_size; + request.custom_data[kPosDataSetId] = message_header.id_in_subset; + request.custom_data[kPosDataSetSize] = message_header.subset_size; } request.custom_data[kPosIngestMode] = ingest_mode; return request; @@ -71,34 +71,34 @@ Error CheckIngestMode(uint64_t ingest_mode) { return nullptr; } -Error CheckProducerRequest(const EventHeader& event_header, uint64_t ingest_mode) { - if (event_header.file_name.size() > kMaxMessageSize) { +Error CheckProducerRequest(const MessageHeader& message_header, uint64_t ingest_mode) { + if (message_header.file_name.size() > kMaxMessageSize) { return ProducerErrorTemplates::kWrongInput.Generate("too long filename"); } - if (event_header.file_name.empty() ) { + if (message_header.file_name.empty() ) { return ProducerErrorTemplates::kWrongInput.Generate("empty filename"); } - if (event_header.id_in_subset > 0 && event_header.subset_size == 0) { + if (message_header.id_in_subset > 0 && message_header.subset_size == 0) { return ProducerErrorTemplates::kWrongInput.Generate("subset dimensions"); } - if (event_header.file_id == 0) { + if (message_header.message_id == 0) { return ProducerErrorTemplates::kWrongInput.Generate("data tuple id should be positive"); } return CheckIngestMode(ingest_mode); } -Error ProducerImpl::Send(const EventHeader& event_header, +Error ProducerImpl::Send(const MessageHeader& message_header, std::string stream, - FileData data, + MessageData data, std::string full_path, uint64_t ingest_mode, RequestCallback callback, bool manage_data_memory) { - auto err = CheckProducerRequest(event_header, ingest_mode); + auto err = CheckProducerRequest(message_header, ingest_mode); if (err) { if (!manage_data_memory) { data.release(); @@ -107,10 +107,10 @@ Error ProducerImpl::Send(const EventHeader& event_header, return err; } - auto request_header = GenerateNextSendRequest(event_header, std::move(stream), ingest_mode); + auto request_header = GenerateNextSendRequest(message_header, std::move(stream), ingest_mode); return request_pool__->AddRequest(std::unique_ptr<ProducerRequest> {new ProducerRequest{source_cred_string_, std::move(request_header), - std::move(data), std::move(event_header.user_metadata), std::move(full_path), callback, manage_data_memory, timeout_ms_} + std::move(data), std::move(message_header.user_metadata), std::move(full_path), callback, manage_data_memory, timeout_ms_} }); } @@ -119,51 +119,51 @@ bool WandTransferData(uint64_t ingest_mode) { return ingest_mode & IngestModeFlags::kTransferData; } -Error CheckData(uint64_t ingest_mode, const EventHeader& event_header, const FileData* data) { +Error CheckData(uint64_t ingest_mode, const MessageHeader& message_header, const MessageData* data) { if (WandTransferData(ingest_mode)) { if (*data == nullptr) { return ProducerErrorTemplates::kWrongInput.Generate("need data for this ingest mode"); } - if (event_header.file_size == 0) { + if (message_header.data_size == 0) { return ProducerErrorTemplates::kWrongInput.Generate("zero data size"); } } return nullptr; } -Error ProducerImpl::SendData(const EventHeader& event_header, FileData data, - uint64_t ingest_mode, RequestCallback callback) { - return SendData(event_header, kDefaultStream, std::move(data), ingest_mode, callback); +Error ProducerImpl::Send(const MessageHeader& message_header, MessageData data, + uint64_t ingest_mode, RequestCallback callback) { + return Send(message_header, kDefaultStream, std::move(data), ingest_mode, callback); } -Error ProducerImpl::SendData(const EventHeader& event_header, - std::string stream, - FileData data, - uint64_t ingest_mode, - RequestCallback callback) { - if (auto err = CheckData(ingest_mode, event_header, &data)) { +Error ProducerImpl::Send(const MessageHeader& message_header, + std::string stream, + MessageData data, + uint64_t ingest_mode, + RequestCallback callback) { + if (auto err = CheckData(ingest_mode, message_header, &data)) { return err; } - return Send(event_header, std::move(stream), std::move(data), "", ingest_mode, callback, true); + return Send(message_header, std::move(stream), std::move(data), "", ingest_mode, callback, true); } Error ProducerImpl::SendStreamFinishedFlag(std::string stream, uint64_t last_id, std::string next_stream, RequestCallback callback) { - EventHeader event_header; - event_header.file_name = kFinishStreamKeyword; - event_header.file_size = 0; - event_header.file_id = last_id + 1; + MessageHeader message_header; + message_header.file_name = kFinishStreamKeyword; + message_header.data_size = 0; + message_header.message_id = last_id + 1; if (next_stream.empty()) { next_stream = kNoNextStreamKeyword; } - event_header.user_metadata = std::string("{\"next_stream\":") + "\"" + next_stream + "\"}"; - return Send(event_header, std::move(stream), nullptr, "", IngestModeFlags::kTransferMetaDataOnly, callback, true); + message_header.user_metadata = std::string("{\"next_stream\":") + "\"" + next_stream + "\"}"; + return Send(message_header, std::move(stream), nullptr, "", IngestModeFlags::kTransferMetaDataOnly, callback, true); } -Error ProducerImpl::SendFile(const EventHeader& event_header, std::string full_path, uint64_t ingest_mode, - RequestCallback callback) { - return SendFile(event_header, kDefaultStream, std::move(full_path), ingest_mode, callback); +Error ProducerImpl::SendFromFile(const MessageHeader& message_header, std::string full_path, uint64_t ingest_mode, + RequestCallback callback) { + return SendFromFile(message_header, kDefaultStream, std::move(full_path), ingest_mode, callback); } @@ -215,36 +215,36 @@ Error ProducerImpl::SetCredentials(SourceCredentials source_cred) { return nullptr; } -Error ProducerImpl::SendMetaData(const std::string& metadata, RequestCallback callback) { +Error ProducerImpl::SendMetadata(const std::string& metadata, RequestCallback callback) { GenericRequestHeader request_header{kOpcodeTransferMetaData, 0, metadata.size(), 0, "beamtime_global.meta"}; request_header.custom_data[kPosIngestMode] = asapo::IngestModeFlags::kTransferData | asapo::IngestModeFlags::kStoreInDatabase; - FileData data{new uint8_t[metadata.size()]}; + MessageData data{new uint8_t[metadata.size()]}; strncpy((char*)data.get(), metadata.c_str(), metadata.size()); return request_pool__->AddRequest(std::unique_ptr<ProducerRequest> {new ProducerRequest{source_cred_string_, std::move(request_header), std::move(data), "", "", callback, true, timeout_ms_} }); } -Error ProducerImpl::SendData__(const EventHeader& event_header, - std::string stream, - void* data, - uint64_t ingest_mode, - RequestCallback callback) { - FileData data_wrapped = FileData{(uint8_t*)data}; +Error ProducerImpl::Send__(const MessageHeader& message_header, + std::string stream, + void* data, + uint64_t ingest_mode, + RequestCallback callback) { + MessageData data_wrapped = MessageData{(uint8_t*)data}; - if (auto err = CheckData(ingest_mode, event_header, &data_wrapped)) { + if (auto err = CheckData(ingest_mode, message_header, &data_wrapped)) { data_wrapped.release(); return err; } - return Send(std::move(event_header), std::move(stream), std::move(data_wrapped), "", ingest_mode, callback, false); + return Send(std::move(message_header), std::move(stream), std::move(data_wrapped), "", ingest_mode, callback, false); } -Error ProducerImpl::SendData__(const EventHeader& event_header, - void* data, - uint64_t ingest_mode, - RequestCallback callback) { - return SendData__(event_header, kDefaultStream, data, ingest_mode, callback); +Error ProducerImpl::Send__(const MessageHeader& message_header, + void* data, + uint64_t ingest_mode, + RequestCallback callback) { + return Send__(message_header, kDefaultStream, data, ingest_mode, callback); } uint64_t ProducerImpl::GetRequestsQueueSize() { @@ -262,16 +262,16 @@ Error ProducerImpl::WaitRequestsFinished(uint64_t timeout_ms) { void ProducerImpl::StopThreads__() { request_pool__->StopThreads(); } -Error ProducerImpl::SendFile(const EventHeader& event_header, - std::string stream, - std::string full_path, - uint64_t ingest_mode, - RequestCallback callback) { +Error ProducerImpl::SendFromFile(const MessageHeader& message_header, + std::string stream, + std::string full_path, + uint64_t ingest_mode, + RequestCallback callback) { if (full_path.empty()) { return ProducerErrorTemplates::kWrongInput.Generate("empty filename"); } - return Send(event_header, std::move(stream), nullptr, std::move(full_path), ingest_mode, callback, true); + return Send(message_header, std::move(stream), nullptr, std::move(full_path), ingest_mode, callback, true); } diff --git a/producer/api/cpp/src/producer_impl.h b/producer/api/cpp/src/producer_impl.h index a2380040f1a054f0d497f3981b8e77054aab5144..18e952f700458c7f0fc75b1cd5065f622b0c70b7 100644 --- a/producer/api/cpp/src/producer_impl.h +++ b/producer/api/cpp/src/producer_impl.h @@ -39,21 +39,21 @@ class ProducerImpl : public Producer { void SetLogLevel(LogLevel level) override; void EnableLocalLog(bool enable) override; void EnableRemoteLog(bool enable) override; - Error SendData(const EventHeader &event_header, - FileData data, - uint64_t ingest_mode, - RequestCallback callback) override; - Error SendData__(const EventHeader &event_header, void* data, uint64_t ingest_mode, - RequestCallback callback) override; - Error SendData(const EventHeader &event_header, std::string stream, FileData data, uint64_t ingest_mode, - RequestCallback callback) override; - Error SendData__(const EventHeader &event_header, std::string stream, void* data, uint64_t ingest_mode, - RequestCallback callback) override; + Error Send(const MessageHeader &message_header, + MessageData data, + uint64_t ingest_mode, + RequestCallback callback) override; + Error Send__(const MessageHeader &message_header, void* data, uint64_t ingest_mode, + RequestCallback callback) override; + Error Send(const MessageHeader &message_header, std::string stream, MessageData data, uint64_t ingest_mode, + RequestCallback callback) override; + Error Send__(const MessageHeader &message_header, std::string stream, void* data, uint64_t ingest_mode, + RequestCallback callback) override; void StopThreads__() override; - Error SendFile(const EventHeader &event_header, std::string full_path, uint64_t ingest_mode, - RequestCallback callback) override; - Error SendFile(const EventHeader &event_header, std::string stream, std::string full_path, uint64_t ingest_mode, - RequestCallback callback) override; + Error SendFromFile(const MessageHeader &message_header, std::string full_path, uint64_t ingest_mode, + RequestCallback callback) override; + Error SendFromFile(const MessageHeader &message_header, std::string stream, std::string full_path, uint64_t ingest_mode, + RequestCallback callback) override; Error SendStreamFinishedFlag(std::string stream, uint64_t last_id, std::string next_stream, RequestCallback callback) override; @@ -63,17 +63,17 @@ class ProducerImpl : public Producer { Error SetCredentials(SourceCredentials source_cred) override; - Error SendMetaData(const std::string &metadata, RequestCallback callback) override; + Error SendMetadata(const std::string &metadata, RequestCallback callback) override; uint64_t GetRequestsQueueSize() override; Error WaitRequestsFinished(uint64_t timeout_ms) override; uint64_t GetRequestsQueueVolumeMb() override; void SetRequestsQueueLimits(uint64_t size, uint64_t volume) override; private: StreamInfo StreamRequest(StreamRequestOp op, std::string stream, uint64_t timeout_ms, Error* err) const; - Error Send(const EventHeader &event_header, std::string stream, FileData data, std::string full_path, + Error Send(const MessageHeader &message_header, std::string stream, MessageData data, std::string full_path, uint64_t ingest_mode, RequestCallback callback, bool manage_data_memory); - GenericRequestHeader GenerateNextSendRequest(const EventHeader &event_header, std::string stream, + GenericRequestHeader GenerateNextSendRequest(const MessageHeader &message_header, std::string stream, uint64_t ingest_mode); std::string source_cred_string_; uint64_t timeout_ms_; diff --git a/producer/api/cpp/src/producer_request.cpp b/producer/api/cpp/src/producer_request.cpp index 5011a4b8ceb7be8d04b5e64e8f22eae2c9b14bd6..7cc63b4f1aeaac497e939fd83187abe17af20b10 100644 --- a/producer/api/cpp/src/producer_request.cpp +++ b/producer/api/cpp/src/producer_request.cpp @@ -4,7 +4,7 @@ namespace asapo { bool ProducerRequest::DataFromFile() const { - if (data != nullptr || original_filepath.empty() || !NeedSendData()) { + if (data != nullptr || original_filepath.empty() || !NeedSend()) { return false; } return true; @@ -12,7 +12,7 @@ bool ProducerRequest::DataFromFile() const { ProducerRequest::ProducerRequest(std::string source_credentials, GenericRequestHeader h, - FileData data, + MessageData data, std::string metadata, std::string original_filepath, RequestCallback callback, @@ -26,7 +26,7 @@ ProducerRequest::ProducerRequest(std::string source_credentials, manage_data_memory{manage_data_memory} { } -bool ProducerRequest::NeedSendData() const { +bool ProducerRequest::NeedSend() const { if (header.op_code == kOpcodeTransferData || header.op_code == kOpcodeTransferSubsetData) { return header.custom_data[kPosIngestMode] & IngestModeFlags::kTransferData; } @@ -49,11 +49,11 @@ Error ProducerRequest::UpdateDataSizeFromFileIfNeeded(const IO* io) { } Error err; - auto finfo = io->GetFileInfo(original_filepath, &err); + auto message_meta = io->GetMessageMeta(original_filepath, &err); if (err) { return ProducerErrorTemplates::kLocalIOError.Generate(err->Explain()); } - header.data_size = finfo.size; + header.data_size = message_meta.size; return nullptr; } diff --git a/producer/api/cpp/src/producer_request.h b/producer/api/cpp/src/producer_request.h index 47a435efb583f49157dd1558f126a3003d8e8623..4d0b73fc1eec36fcb270221f62a570d36f31230d 100644 --- a/producer/api/cpp/src/producer_request.h +++ b/producer/api/cpp/src/producer_request.h @@ -12,7 +12,7 @@ namespace asapo { class ProducerRequest : public GenericRequest { public: ~ProducerRequest(); - ProducerRequest(std::string source_credentials, GenericRequestHeader header, FileData data, + ProducerRequest(std::string source_credentials, GenericRequestHeader header, MessageData data, std::string metadata, std::string original_filepath, RequestCallback callback, @@ -20,12 +20,12 @@ class ProducerRequest : public GenericRequest { uint64_t timeout_ms); std::string source_credentials; std::string metadata; - FileData data; + MessageData data; std::string original_filepath; RequestCallback callback; bool manage_data_memory; bool DataFromFile() const; - bool NeedSendData() const; + bool NeedSend() const; bool NeedSendMetaData() const; Error UpdateDataSizeFromFileIfNeeded(const IO* io); diff --git a/producer/api/cpp/src/request_handler_tcp.cpp b/producer/api/cpp/src/request_handler_tcp.cpp index e257c32c0f8b2d4f888b33105485dd8d40f3ed4d..157c1212e6b24089d7ed8ace19d7de3f2fc26da5 100644 --- a/producer/api/cpp/src/request_handler_tcp.cpp +++ b/producer/api/cpp/src/request_handler_tcp.cpp @@ -61,7 +61,7 @@ Error RequestHandlerTcp::SendRequestContent(const ProducerRequest* request) { } } - if (request->NeedSendData()) { + if (request->NeedSend()) { if (request->DataFromFile()) { io_error = io__->SendFile(sd_, request->original_filepath, (size_t)request->header.data_size); } else { @@ -77,7 +77,7 @@ Error RequestHandlerTcp::SendRequestContent(const ProducerRequest* request) { Error RequestHandlerTcp::ReceiveResponse(const GenericRequestHeader& request_header, std::string* response) { Error err; - SendDataResponse sendDataResponse; + SendResponse sendDataResponse; io__->Receive(sd_, &sendDataResponse, sizeof(sendDataResponse), &err); if(err != nullptr) { return err; @@ -222,7 +222,7 @@ void RequestHandlerTcp::ProcessRequestCallback(Error err, ProducerRequest* reque } -bool RequestHandlerTcp::SendDataToOneOfTheReceivers(ProducerRequest* request, bool* retry) { +bool RequestHandlerTcp::SendToOneOfTheReceivers(ProducerRequest* request, bool* retry) { for (auto receiver_uri : receivers_list_) { if (Disconnected()) { auto err = ConnectToReceiver(request->source_credentials, receiver_uri); @@ -269,7 +269,7 @@ bool RequestHandlerTcp::ProcessRequestUnlocked(GenericRequest* request, bool* re if (NeedRebalance()) { CloseConnectionToPeformRebalance(); } - return SendDataToOneOfTheReceivers(producer_request, retry); + return SendToOneOfTheReceivers(producer_request, retry); } bool RequestHandlerTcp::Connected() { diff --git a/producer/api/cpp/src/request_handler_tcp.h b/producer/api/cpp/src/request_handler_tcp.h index 0de896610fb9324b29cf873cb5442d841a5b1447..a891a81d5e139da8c60f00d98997cb8c70e2ea1e 100644 --- a/producer/api/cpp/src/request_handler_tcp.h +++ b/producer/api/cpp/src/request_handler_tcp.h @@ -32,7 +32,7 @@ class RequestHandlerTcp: public RequestHandler { private: Error Authorize(const std::string& source_credentials); Error ConnectToReceiver(const std::string& source_credentials, const std::string& receiver_address); - bool SendDataToOneOfTheReceivers(ProducerRequest* request, bool* retry); + bool SendToOneOfTheReceivers(ProducerRequest* request, bool* retry); Error SendRequestContent(const ProducerRequest* request); Error ReceiveResponse(const GenericRequestHeader& request_header, std::string* response); Error TrySendToReceiver(const ProducerRequest* request, std::string* response); diff --git a/producer/api/cpp/unittests/test_producer.cpp b/producer/api/cpp/unittests/test_producer.cpp index 4fc2319b1cc3de2dc56f639f36898d40aa3a199b..5fd627a86faaa65d8df486c5f274598887047b37 100644 --- a/producer/api/cpp/unittests/test_producer.cpp +++ b/producer/api/cpp/unittests/test_producer.cpp @@ -62,8 +62,8 @@ TEST(Producer, SimpleWorkflowWihoutConnection) { SourceCredentials{asapo::SourceType::kRaw,"bt", "", "", ""}, 3600000, &err); - asapo::EventHeader event_header{1, 1, "test"}; - auto err_send = producer->SendData(event_header, nullptr, asapo::kTransferMetaDataOnly, nullptr); + asapo::MessageHeader message_header{1, 1, "test"}; + auto err_send = producer->Send(message_header, nullptr, asapo::kTransferMetaDataOnly, nullptr); std::this_thread::sleep_for(std::chrono::milliseconds(100)); ASSERT_THAT(producer, Ne(nullptr)); diff --git a/producer/api/cpp/unittests/test_producer_impl.cpp b/producer/api/cpp/unittests/test_producer_impl.cpp index 81c86cc3ca61a7f636d10578e8c300e9d76fd69c..09f6545f3b6a98fbeae53c108b9ebf3559872765 100644 --- a/producer/api/cpp/unittests/test_producer_impl.cpp +++ b/producer/api/cpp/unittests/test_producer_impl.cpp @@ -30,7 +30,7 @@ using ::testing::HasSubstr; using asapo::RequestPool; using asapo::ProducerRequest; -MATCHER_P10(M_CheckSendDataRequest, op_code, source_credentials, metadata, file_id, file_size, message, stream, +MATCHER_P10(M_CheckSendRequest, op_code, source_credentials, metadata, file_id, file_size, message, stream, ingest_mode, subset_id, subset_size, @@ -99,74 +99,74 @@ class ProducerImplTests : public testing::Test { TEST_F(ProducerImplTests, SendReturnsError) { EXPECT_CALL(mock_pull, AddRequest_t(_, false)).WillOnce(Return( asapo::ProducerErrorTemplates::kRequestPoolIsFull.Generate().release())); - asapo::EventHeader event_header{1, 1, "test"}; - auto err = producer.SendData(event_header, nullptr, expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{1, 1, "test"}; + auto err = producer.Send(message_header, nullptr, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kRequestPoolIsFull)); } TEST_F(ProducerImplTests, ErrorIfFileNameTooLong) { std::string long_string(asapo::kMaxMessageSize + 100, 'a'); - asapo::EventHeader event_header{1, 1, long_string}; - auto err = producer.SendData(event_header, nullptr, expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{1, 1, long_string}; + auto err = producer.Send(message_header, nullptr, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput)); } TEST_F(ProducerImplTests, ErrorIfFileEmpty) { std::string long_string(asapo::kMaxMessageSize + 100, 'a'); - asapo::EventHeader event_header{1, 1, ""}; - auto err = producer.SendData(event_header, nullptr, expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{1, 1, ""}; + auto err = producer.Send(message_header, nullptr, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput)); } TEST_F(ProducerImplTests, ErrorIfSubsetSizeNotDefined) { EXPECT_CALL(mock_logger, Error(testing::HasSubstr("subset dimensions"))); - asapo::EventHeader event_header{1, 1000, "test", "", 1}; - auto err = producer.SendData(event_header, nullptr, expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{1, 1000, "test", "", 1}; + auto err = producer.Send(message_header, nullptr, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput)); } TEST_F(ProducerImplTests, ErrorIfZeroDataSize) { - asapo::FileData data = asapo::FileData{new uint8_t[100]}; - asapo::EventHeader event_header{1, 0, expected_fullpath}; - auto err = producer.SendData(event_header, std::move(data), asapo::kDefaultIngestMode, nullptr); + asapo::MessageData data = asapo::MessageData{new uint8_t[100]}; + asapo::MessageHeader message_header{1, 0, expected_fullpath}; + auto err = producer.Send(message_header, std::move(data), asapo::kDefaultIngestMode, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput)); } TEST_F(ProducerImplTests, ErrorIfNoData) { - asapo::EventHeader event_header{1, 100, expected_fullpath}; - auto err = producer.SendData(event_header, nullptr, asapo::kDefaultIngestMode, nullptr); + asapo::MessageHeader message_header{1, 100, expected_fullpath}; + auto err = producer.Send(message_header, nullptr, asapo::kDefaultIngestMode, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput)); } TEST_F(ProducerImplTests, ErrorIfNoDataSend_) { - asapo::EventHeader event_header{1, 100, expected_fullpath}; - auto err = producer.SendData__(event_header, nullptr, asapo::kDefaultIngestMode, nullptr); + asapo::MessageHeader message_header{1, 100, expected_fullpath}; + auto err = producer.Send__(message_header, nullptr, asapo::kDefaultIngestMode, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput)); } TEST_F(ProducerImplTests, ErrorIfSendingDataWithZeroId) { - asapo::EventHeader event_header{0, 100, expected_fullpath}; - auto err = producer.SendData(event_header, nullptr, asapo::kTransferMetaDataOnly, nullptr); + asapo::MessageHeader message_header{0, 100, expected_fullpath}; + auto err = producer.Send(message_header, nullptr, asapo::kTransferMetaDataOnly, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput)); } TEST_F(ProducerImplTests, OkIfNoDataWithTransferMetadataOnlyMode) { - asapo::EventHeader event_header{1, 100, expected_fullpath}; - auto err = producer.SendData(event_header, nullptr, asapo::kTransferMetaDataOnly, nullptr); + asapo::MessageHeader message_header{1, 100, expected_fullpath}; + auto err = producer.Send(message_header, nullptr, asapo::kTransferMetaDataOnly, nullptr); ASSERT_THAT(err, Eq(nullptr)); } TEST_F(ProducerImplTests, OkIfZeroSizeWithTransferMetadataOnlyMode) { - asapo::FileData data = asapo::FileData{new uint8_t[100]}; - asapo::EventHeader event_header{1, 0, expected_fullpath}; - auto err = producer.SendData(event_header, std::move(data), asapo::kTransferMetaDataOnly, nullptr); + asapo::MessageData data = asapo::MessageData{new uint8_t[100]}; + asapo::MessageHeader message_header{1, 0, expected_fullpath}; + auto err = producer.Send(message_header, std::move(data), asapo::kTransferMetaDataOnly, nullptr); ASSERT_THAT(err, Eq(nullptr)); } TEST_F(ProducerImplTests, UsesDefaultStream) { producer.SetCredentials(expected_default_credentials); - EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendDataRequest(asapo::kOpcodeTransferData, + EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendRequest(asapo::kOpcodeTransferData, expected_default_credentials_str, expected_metadata, expected_id, @@ -177,16 +177,16 @@ TEST_F(ProducerImplTests, UsesDefaultStream) { 0, 0), false)).WillOnce(Return(nullptr)); - asapo::EventHeader event_header{expected_id, expected_size, expected_name, expected_metadata}; - auto err = producer.SendData(event_header, nullptr, expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{expected_id, expected_size, expected_name, expected_metadata}; + auto err = producer.Send(message_header, nullptr, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(nullptr)); } -TEST_F(ProducerImplTests, OKSendingSendDataRequest) { +TEST_F(ProducerImplTests, OKSendingSendRequest) { producer.SetCredentials(expected_credentials); - EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendDataRequest(asapo::kOpcodeTransferData, + EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendRequest(asapo::kOpcodeTransferData, expected_credentials_str, expected_metadata, expected_id, @@ -199,16 +199,16 @@ TEST_F(ProducerImplTests, OKSendingSendDataRequest) { ), false)).WillOnce(Return( nullptr)); - asapo::EventHeader event_header{expected_id, expected_size, expected_name, expected_metadata}; - auto err = producer.SendData(event_header, nullptr, expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{expected_id, expected_size, expected_name, expected_metadata}; + auto err = producer.Send(message_header, nullptr, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(nullptr)); } -TEST_F(ProducerImplTests, OKSendingSendDataRequestWithStream) { +TEST_F(ProducerImplTests, OKSendingSendRequestWithStream) { producer.SetCredentials(expected_credentials); - EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendDataRequest(asapo::kOpcodeTransferData, + EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendRequest(asapo::kOpcodeTransferData, expected_credentials_str, expected_metadata, expected_id, @@ -221,8 +221,8 @@ TEST_F(ProducerImplTests, OKSendingSendDataRequestWithStream) { ), false)).WillOnce(Return( nullptr)); - asapo::EventHeader event_header{expected_id, expected_size, expected_name, expected_metadata}; - auto err = producer.SendData(event_header, expected_stream, nullptr, expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{expected_id, expected_size, expected_name, expected_metadata}; + auto err = producer.Send(message_header, expected_stream, nullptr, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(nullptr)); } @@ -232,7 +232,7 @@ TEST_F(ProducerImplTests, OKSendingStreamFinish) { std::string next_stream_meta = std::string("{\"next_stream\":") + "\"" + expected_next_stream + "\"}"; - EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendDataRequest(asapo::kOpcodeTransferData, + EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendRequest(asapo::kOpcodeTransferData, expected_credentials_str, next_stream_meta.c_str(), expected_id + 1, @@ -257,7 +257,7 @@ TEST_F(ProducerImplTests, OKSendingStreamFinishWithNoNextStream) { next_stream_meta = std::string("{\"next_stream\":") + "\"" + asapo::ProducerImpl::kNoNextStreamKeyword + "\"}"; - EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendDataRequest(asapo::kOpcodeTransferData, + EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendRequest(asapo::kOpcodeTransferData, expected_credentials_str, next_stream_meta.c_str(), expected_id + 1, @@ -277,7 +277,7 @@ TEST_F(ProducerImplTests, OKSendingStreamFinishWithNoNextStream) { TEST_F(ProducerImplTests, OKSendingSendSubsetDataRequest) { producer.SetCredentials(expected_credentials); - EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendDataRequest(asapo::kOpcodeTransferSubsetData, + EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendRequest(asapo::kOpcodeTransferSubsetData, expected_credentials_str, expected_metadata, expected_id, @@ -290,9 +290,9 @@ TEST_F(ProducerImplTests, OKSendingSendSubsetDataRequest) { Return( nullptr)); - asapo::EventHeader event_header + asapo::MessageHeader message_header {expected_id, expected_size, expected_name, expected_metadata, expected_subset_id, expected_subset_size}; - auto err = producer.SendData(event_header, nullptr, expected_ingest_mode, nullptr); + auto err = producer.Send(message_header, nullptr, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(nullptr)); } @@ -304,7 +304,7 @@ TEST_F(ProducerImplTests, OKAddingSendMetaDataRequest) { expected_ingest_mode = asapo::IngestModeFlags::kTransferData | asapo::IngestModeFlags::kStoreInDatabase ; producer.SetCredentials(expected_credentials); - EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendDataRequest(asapo::kOpcodeTransferMetaData, + EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendRequest(asapo::kOpcodeTransferMetaData, expected_credentials_str, "", expected_id, @@ -316,7 +316,7 @@ TEST_F(ProducerImplTests, OKAddingSendMetaDataRequest) { 10), false)).WillOnce(Return( nullptr)); - auto err = producer.SendMetaData(expected_metadata, nullptr); + auto err = producer.SendMetadata(expected_metadata, nullptr); ASSERT_THAT(err, Eq(nullptr)); } @@ -326,8 +326,8 @@ TEST_F(ProducerImplTests, ErrorSendingEmptyFileName) { EXPECT_CALL(mock_pull, AddRequest_t(_, _)).Times(0); - asapo::EventHeader event_header{expected_id, 0, expected_name}; - auto err = producer.SendFile(event_header, "", expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{expected_id, 0, expected_name}; + auto err = producer.SendFromFile(message_header, "", expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput)); @@ -338,8 +338,8 @@ TEST_F(ProducerImplTests, ErrorSendingEmptyRelativeFileName) { EXPECT_CALL(mock_pull, AddRequest_t(_, _)).Times(0); - asapo::EventHeader event_header{expected_id, 0, ""}; - auto err = producer.SendFile(event_header, expected_fullpath, expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{expected_id, 0, ""}; + auto err = producer.SendFromFile(message_header, expected_fullpath, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput)); @@ -348,7 +348,7 @@ TEST_F(ProducerImplTests, ErrorSendingEmptyRelativeFileName) { TEST_F(ProducerImplTests, OKSendingSendFileRequest) { producer.SetCredentials(expected_credentials); - EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendDataRequest(asapo::kOpcodeTransferData, + EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendRequest(asapo::kOpcodeTransferData, expected_credentials_str, "", expected_id, @@ -360,8 +360,8 @@ TEST_F(ProducerImplTests, OKSendingSendFileRequest) { 0), false)).WillOnce(Return( nullptr)); - asapo::EventHeader event_header{expected_id, 0, expected_name}; - auto err = producer.SendFile(event_header, expected_fullpath, expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{expected_id, 0, expected_name}; + auto err = producer.SendFromFile(message_header, expected_fullpath, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(nullptr)); } @@ -369,7 +369,7 @@ TEST_F(ProducerImplTests, OKSendingSendFileRequest) { TEST_F(ProducerImplTests, OKSendingSendFileRequestWithStream) { producer.SetCredentials(expected_credentials); - EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendDataRequest(asapo::kOpcodeTransferData, + EXPECT_CALL(mock_pull, AddRequest_t(M_CheckSendRequest(asapo::kOpcodeTransferData, expected_credentials_str, "", expected_id, @@ -381,8 +381,9 @@ TEST_F(ProducerImplTests, OKSendingSendFileRequestWithStream) { 0), false)).WillOnce(Return( nullptr)); - asapo::EventHeader event_header{expected_id, 0, expected_name}; - auto err = producer.SendFile(event_header, expected_stream, expected_fullpath, expected_ingest_mode, nullptr); + asapo::MessageHeader message_header{expected_id, 0, expected_name}; + auto err = + producer.SendFromFile(message_header, expected_stream, expected_fullpath, expected_ingest_mode, nullptr); ASSERT_THAT(err, Eq(nullptr)); } @@ -408,7 +409,7 @@ TEST_F(ProducerImplTests, ErrorSettingSecondTime) { TEST_F(ProducerImplTests, ErrorSendingWrongIngestMode) { producer.SetCredentials(expected_credentials); - asapo::EventHeader event_header{expected_id, 0, expected_name}; + asapo::MessageHeader message_header{expected_id, 0, expected_name}; uint64_t ingest_modes[] = {0, asapo::IngestModeFlags::kTransferMetaDataOnly | asapo::IngestModeFlags::kTransferData, asapo::IngestModeFlags::kTransferData, asapo::IngestModeFlags::kStoreInDatabase, @@ -420,7 +421,7 @@ TEST_F(ProducerImplTests, ErrorSendingWrongIngestMode) { EXPECT_CALL(mock_pull, AddRequest_t(_, _)).Times(0); for (auto ingest_mode : ingest_modes) { - auto err = producer.SendFile(event_header, expected_fullpath, ingest_mode, nullptr); + auto err = producer.SendFromFile(message_header, expected_fullpath, ingest_mode, nullptr); ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput)); } diff --git a/producer/api/cpp/unittests/test_producer_request.cpp b/producer/api/cpp/unittests/test_producer_request.cpp index 6bb12223cac5d7351bc90a56ce1f6d17a2dccf55..eb087cfa3b3a4dec67b92d1626c7b0bb58d47ce8 100644 --- a/producer/api/cpp/unittests/test_producer_request.cpp +++ b/producer/api/cpp/unittests/test_producer_request.cpp @@ -60,7 +60,7 @@ TEST(ProducerRequest, Constructor) { TEST(ProducerRequest, Destructor) { // fails with data corruption if done wrong char data_[100]; - asapo::FileData data{(uint8_t*)data_}; + asapo::MessageData data{(uint8_t*)data_}; asapo::GenericRequestHeader header{asapo::kOpcodeTransferData, 1, 1, 1, ""}; asapo::ProducerRequest* request = new asapo::ProducerRequest{"", std::move(header), std::move(data), "", "", nullptr, false, 0}; diff --git a/producer/api/cpp/unittests/test_request_handler_filesystem.cpp b/producer/api/cpp/unittests/test_request_handler_filesystem.cpp index 1c95e5126ae092d52576934e8f0dda59a5b545e8..9c666a0561cac488cd3d0826e8b773bc52df4994 100644 --- a/producer/api/cpp/unittests/test_request_handler_filesystem.cpp +++ b/producer/api/cpp/unittests/test_request_handler_filesystem.cpp @@ -86,12 +86,12 @@ class RequestHandlerFilesystemTests : public testing::Test { } }; -ACTION_P(A_WriteSendDataResponse, error_code) { - ((asapo::SendDataResponse*)arg1)->op_code = asapo::kOpcodeTransferData; - ((asapo::SendDataResponse*)arg1)->error_code = error_code; +ACTION_P(A_WriteSendResponse, error_code) { + ((asapo::SendResponse*)arg1)->op_code = asapo::kOpcodeTransferData; + ((asapo::SendResponse*)arg1)->error_code = error_code; } -MATCHER_P2(M_CheckSendDataRequest, file_id, file_size, +MATCHER_P2(M_CheckSendRequest, file_id, file_size, "Checks if a valid GenericRequestHeader was Send") { return ((asapo::GenericRequestHeader*)arg)->op_code == asapo::kOpcodeTransferData && ((asapo::GenericRequestHeader*)arg)->data_id == file_id diff --git a/producer/api/cpp/unittests/test_request_handler_tcp.cpp b/producer/api/cpp/unittests/test_request_handler_tcp.cpp index 780bd64a8e7e0ea7184f0a5bae8e7e97561d0866..d9f4a155e786944579d54840048d4f3d5d2136a6 100644 --- a/producer/api/cpp/unittests/test_request_handler_tcp.cpp +++ b/producer/api/cpp/unittests/test_request_handler_tcp.cpp @@ -73,14 +73,14 @@ class RequestHandlerTcpTests : public testing::Test { asapo::GenericRequestHeader callback_header; std::string callback_response; uint8_t expected_callback_data = 2; - asapo::FileData expected_data{[this]() { + asapo::MessageData expected_data{[this]() { auto a = new uint8_t[expected_file_size]; for (auto i = 0; i < expected_file_size; i++) { a[i] = expected_callback_data; } return a; }()}; - asapo::FileData callback_data; + asapo::MessageData callback_data; asapo::ProducerRequest request{expected_beamtime_id, header, std::move(expected_data), expected_metadata, "", [this](asapo::RequestCallbackPayload payload, asapo::Error err) { @@ -124,16 +124,16 @@ class RequestHandlerTcpTests : public testing::Test { void ExpectOKAuthorize(bool only_once = false); void ExpectFailSendHeader(bool only_once = false); void ExpectFailSend(uint64_t expected_size, bool only_once); - void ExpectFailSendData(bool only_once = false); + void ExpectFailSend(bool only_once = false); void ExpectFailSendMetaData(bool only_once = false); void ExpectOKConnect(bool only_once = false); void ExpectOKSendHeader(bool only_once = false, asapo::Opcode code = expected_op_code); void ExpectOKSend(uint64_t expected_size, bool only_once); void ExpectOKSendAll(bool only_once); void ExpectGetFileSize(bool ok); - void ExpectOKSendData(bool only_once = false); - void ExpectOKSendFile(bool only_once = false); - void ExpectFailSendFile(const asapo::ProducerErrorTemplate &err_template, bool client_error = false); + void ExpectOKSend(bool only_once = false); + void ExpectOKSendFromFile(bool only_once = false); + void ExpectFailSendFromFile(const asapo::ProducerErrorTemplate &err_template, bool client_error = false); void ExpectOKSendMetaData(bool only_once = false); void ExpectFailReceive(bool only_once = false); void ExpectOKReceive(bool only_once = true, asapo::NetworkErrorCode code = asapo::kNetErrorNoError, @@ -155,13 +155,13 @@ class RequestHandlerTcpTests : public testing::Test { } }; -ACTION_P2(A_WriteSendDataResponse, error_code, message) { - ((asapo::SendDataResponse*) arg1)->op_code = asapo::kOpcodeTransferData; - ((asapo::SendDataResponse*) arg1)->error_code = error_code; - strcpy(((asapo::SendDataResponse*) arg1)->message, message.c_str()); +ACTION_P2(A_WriteSendResponse, error_code, message) { + ((asapo::SendResponse*) arg1)->op_code = asapo::kOpcodeTransferData; + ((asapo::SendResponse*) arg1)->error_code = error_code; + strcpy(((asapo::SendResponse*) arg1)->message, message.c_str()); } -MATCHER_P5(M_CheckSendDataRequest, op_code, file_id, file_size, message, stream, +MATCHER_P5(M_CheckSendRequest, op_code, file_id, file_size, message, stream, "Checks if a valid GenericRequestHeader was Send") { return ((asapo::GenericRequestHeader*) arg)->op_code == op_code && ((asapo::GenericRequestHeader*) arg)->data_id == uint64_t(file_id) @@ -195,7 +195,7 @@ void RequestHandlerTcpTests::ExpectFailAuthorize(bool only_once) { int i = 0; for (auto expected_sd : expected_sds) { EXPECT_CALL(mock_io, - Send_t(expected_sd, M_CheckSendDataRequest(asapo::kOpcodeAuthorize, 0, 0, expected_beamtime_id, + Send_t(expected_sd, M_CheckSendRequest(asapo::kOpcodeAuthorize, 0, 0, expected_beamtime_id, ""), sizeof(asapo::GenericRequestHeader), _)) .WillOnce( @@ -204,12 +204,12 @@ void RequestHandlerTcpTests::ExpectFailAuthorize(bool only_once) { Return(sizeof(asapo::GenericRequestHeader)) )); - EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendDataResponse), _)) + EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendResponse), _)) .InSequence(seq_receive[i]) .WillOnce( DoAll( testing::SetArgPointee<3>(nullptr), - A_WriteSendDataResponse(asapo::kNetAuthorizationError, expected_auth_message), + A_WriteSendResponse(asapo::kNetAuthorizationError, expected_auth_message), testing::ReturnArg<2>() )); EXPECT_CALL(mock_io, CloseSocket_t(expected_sd, _)); @@ -236,7 +236,7 @@ void RequestHandlerTcpTests::ExpectOKAuthorize(bool only_once) { int i = 0; for (auto expected_sd : expected_sds) { EXPECT_CALL(mock_io, - Send_t(expected_sd, M_CheckSendDataRequest(asapo::kOpcodeAuthorize, 0, 0, expected_beamtime_id, + Send_t(expected_sd, M_CheckSendRequest(asapo::kOpcodeAuthorize, 0, 0, expected_beamtime_id, ""), sizeof(asapo::GenericRequestHeader), _)) .WillOnce( @@ -245,12 +245,12 @@ void RequestHandlerTcpTests::ExpectOKAuthorize(bool only_once) { Return(sizeof(asapo::GenericRequestHeader)) )); - EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendDataResponse), _)) + EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendResponse), _)) .InSequence(seq_receive[i]) .WillOnce( DoAll( testing::SetArgPointee<3>(nullptr), - A_WriteSendDataResponse(asapo::kNetErrorNoError, expected_auth_message), + A_WriteSendResponse(asapo::kNetErrorNoError, expected_auth_message), testing::ReturnArg<2>() )); if (only_once) { @@ -269,7 +269,7 @@ void RequestHandlerTcpTests::ExpectOKAuthorize(bool only_once) { void RequestHandlerTcpTests::ExpectFailSendHeader(bool only_once) { int i = 0; for (auto expected_sd : expected_sds) { - EXPECT_CALL(mock_io, Send_t(expected_sd, M_CheckSendDataRequest(expected_op_code, + EXPECT_CALL(mock_io, Send_t(expected_sd, M_CheckSendRequest(expected_op_code, expected_file_id, expected_file_size, expected_file_name, @@ -302,7 +302,7 @@ void RequestHandlerTcpTests::ExpectFailSendHeader(bool only_once) { } } -void RequestHandlerTcpTests::ExpectFailSendFile(const asapo::ProducerErrorTemplate &err_template, bool client_error) { +void RequestHandlerTcpTests::ExpectFailSendFromFile(const asapo::ProducerErrorTemplate &err_template, bool client_error) { int i = 0; for (auto expected_sd : expected_sds) { EXPECT_CALL(mock_io, SendFile_t(expected_sd, expected_origin_fullpath, (size_t) expected_file_size)) @@ -366,7 +366,7 @@ void RequestHandlerTcpTests::ExpectFailSend(uint64_t expected_size, bool only_on if (only_once) EXPECT_CALL(mock_logger, Warning(HasSubstr("put back"))); } -void RequestHandlerTcpTests::ExpectFailSendData(bool only_once) { +void RequestHandlerTcpTests::ExpectFailSend(bool only_once) { ExpectFailSend(expected_file_size, only_once); } @@ -377,7 +377,7 @@ void RequestHandlerTcpTests::ExpectFailSendMetaData(bool only_once) { void RequestHandlerTcpTests::ExpectFailReceive(bool only_once) { int i = 0; for (auto expected_sd : expected_sds) { - EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendDataResponse), _)) + EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendResponse), _)) .InSequence(seq_receive[i]) .WillOnce( DoAll( @@ -406,7 +406,7 @@ void RequestHandlerTcpTests::ExpectFailReceive(bool only_once) { void RequestHandlerTcpTests::ExpectOKSendAll(bool only_once) { ExpectOKSendHeader(only_once); ExpectOKSendMetaData(only_once); - ExpectOKSendData(only_once); + ExpectOKSend(only_once); } void RequestHandlerTcpTests::ExpectOKSend(uint64_t expected_size, bool only_once) { @@ -426,11 +426,11 @@ void RequestHandlerTcpTests::ExpectOKSendMetaData(bool only_once) { ExpectOKSend(expected_meta_size, only_once); } -void RequestHandlerTcpTests::ExpectOKSendData(bool only_once) { +void RequestHandlerTcpTests::ExpectOKSend(bool only_once) { ExpectOKSend(expected_file_size, only_once); } -void RequestHandlerTcpTests::ExpectOKSendFile(bool only_once) { +void RequestHandlerTcpTests::ExpectOKSendFromFile(bool only_once) { for (auto expected_sd : expected_sds) { EXPECT_CALL(mock_io, SendFile_t(expected_sd, expected_origin_fullpath, (size_t) expected_file_size)) .Times(1) @@ -441,7 +441,7 @@ void RequestHandlerTcpTests::ExpectOKSendFile(bool only_once) { void RequestHandlerTcpTests::ExpectOKSendHeader(bool only_once, asapo::Opcode opcode) { for (auto expected_sd : expected_sds) { - EXPECT_CALL(mock_io, Send_t(expected_sd, M_CheckSendDataRequest(opcode, + EXPECT_CALL(mock_io, Send_t(expected_sd, M_CheckSendRequest(opcode, expected_file_id, expected_file_size, expected_file_name, @@ -481,12 +481,12 @@ void RequestHandlerTcpTests::ExpectOKConnect(bool only_once) { void RequestHandlerTcpTests::ExpectOKReceive(bool only_once, asapo::NetworkErrorCode code, std::string message) { int i = 0; for (auto expected_sd : expected_sds) { - EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendDataResponse), _)) + EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendResponse), _)) .InSequence(seq_receive[i]) .WillOnce( DoAll( testing::SetArgPointee<3>(nullptr), - A_WriteSendDataResponse(code, message), + A_WriteSendResponse(code, message), testing::ReturnArg<2>() )); if (only_once) { @@ -656,12 +656,12 @@ TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendHeader) { } -TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendData) { +TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSend) { ExpectOKConnect(); ExpectOKAuthorize(); ExpectOKSendHeader(); ExpectOKSendMetaData(); - ExpectFailSendData(); + ExpectFailSend(); request_handler.PrepareProcessingRequestLocked(); auto success = request_handler.ProcessRequestUnlocked(&request, &retry); @@ -708,12 +708,12 @@ void RequestHandlerTcpTests::AssertImmediatelyCallBack(asapo::NetworkErrorCode e ExpectOKAuthorize(true); ExpectOKSendAll(true); - EXPECT_CALL(mock_io, Receive_t(expected_sds[0], _, sizeof(asapo::SendDataResponse), _)) + EXPECT_CALL(mock_io, Receive_t(expected_sds[0], _, sizeof(asapo::SendResponse), _)) .InSequence(seq_receive[0]) .WillOnce( DoAll( testing::SetArgPointee<3>(nullptr), - A_WriteSendDataResponse(error_code, expected_auth_message), + A_WriteSendResponse(error_code, expected_auth_message), testing::ReturnArg<2>() )); EXPECT_CALL(mock_logger, Debug(AllOf( @@ -738,12 +738,12 @@ void RequestHandlerTcpTests::AssertImmediatelyCallBack(asapo::NetworkErrorCode e } void RequestHandlerTcpTests::ExpectGetFileSize(bool ok) { - asapo::FileInfo fi; + asapo::MessageMeta fi; if (ok) { fi.size = expected_file_size; } - EXPECT_CALL(mock_io, GetFileInfo_t(expected_origin_fullpath, _)).WillOnce( + EXPECT_CALL(mock_io, GetMessageMeta_t(expected_origin_fullpath, _)).WillOnce( DoAll( testing::SetArgPointee<1>(ok ? nullptr : asapo::IOErrorTemplates::kFileNotFound.Generate().release()), testing::Return(fi) @@ -772,13 +772,13 @@ TEST_F(RequestHandlerTcpTests, SendEmptyCallBack) { ASSERT_THAT(retry, Eq(false)); } -TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendFileWithReadError) { +TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendFromFileWithReadError) { ExpectGetFileSize(true); ExpectOKConnect(true); ExpectOKAuthorize(true); ExpectOKSendHeader(true); ExpectOKSendMetaData(true); - ExpectFailSendFile(asapo::ProducerErrorTemplates::kLocalIOError, true); + ExpectFailSendFromFile(asapo::ProducerErrorTemplates::kLocalIOError, true); request_handler.PrepareProcessingRequestLocked(); auto success = request_handler.ProcessRequestUnlocked(&request_filesend, &retry); @@ -790,13 +790,13 @@ TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendFileWithReadError) { } -TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendFileWithServerError) { +TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendFromFileWithServerError) { ExpectGetFileSize(true); ExpectOKConnect(); ExpectOKAuthorize(); ExpectOKSendHeader(); ExpectOKSendMetaData(); - ExpectFailSendFile(asapo::ProducerErrorTemplates::kInternalServerError); + ExpectFailSendFromFile(asapo::ProducerErrorTemplates::kInternalServerError); request_handler.PrepareProcessingRequestLocked(); auto success = request_handler.ProcessRequestUnlocked(&request_filesend, &retry); @@ -839,7 +839,7 @@ TEST_F(RequestHandlerTcpTests, FileRequestOK) { ExpectOKAuthorize(true); ExpectOKSendHeader(true); ExpectOKSendMetaData(true); - ExpectOKSendFile(true); + ExpectOKSendFromFile(true); ExpectOKReceive(true, asapo::kNetErrorNoError, expected_response); request_handler.PrepareProcessingRequestLocked(); @@ -883,7 +883,7 @@ TEST_F(RequestHandlerTcpTests, SendMetadataIgnoresIngestMode) { ExpectOKConnect(true); ExpectOKAuthorize(true); ExpectOKSendHeader(true, asapo::kOpcodeTransferMetaData); - ExpectOKSendData(true); + ExpectOKSend(true); ExpectOKSendMetaData(true); ExpectOKReceive(); @@ -928,7 +928,7 @@ TEST_F(RequestHandlerTcpTests, SendMetaOnlyForFileReadOK) { request_handler.PrepareProcessingRequestLocked(); EXPECT_CALL(mock_io, SendFile_t(_, _, _)).Times(0); - EXPECT_CALL(mock_io, GetFileInfo_t(_, _)).Times(0); + EXPECT_CALL(mock_io, GetMessageMeta_t(_, _)).Times(0); auto ingest_mode = asapo::IngestModeFlags::kTransferMetaDataOnly; request_filesend.header.custom_data[asapo::kPosIngestMode] = ingest_mode; diff --git a/producer/api/python/asapo_producer.pxd b/producer/api/python/asapo_producer.pxd index 892766c4079c2db86d59fba92f40f57a536ee8de..c09a2d7df70effa0aa69f6817839d3fc5f2693b7 100644 --- a/producer/api/python/asapo_producer.pxd +++ b/producer/api/python/asapo_producer.pxd @@ -25,7 +25,7 @@ cdef extern from "asapo/asapo_producer.h" namespace "asapo": cdef extern from "asapo/asapo_producer.h" namespace "asapo": - cppclass FileData: + cppclass MessageData: uint8_t[] release() uint8_t[] get() cppclass StreamInfo: @@ -60,18 +60,9 @@ cdef extern from "asapo/asapo_producer.h" namespace "asapo": SourceType type cdef extern from "asapo/asapo_producer.h" namespace "asapo": - struct EventHeader: - uint64_t file_id - uint64_t file_size - string file_name - string user_metadata - uint64_t id_in_subset - uint64_t subset_size - -cdef extern from "asapo/asapo_producer.h" namespace "asapo": - struct EventHeader: - uint64_t file_id - uint64_t file_size + struct MessageHeader: + uint64_t message_id + uint64_t data_size string file_name string user_metadata uint64_t id_in_subset @@ -82,7 +73,7 @@ cdef extern from "asapo/asapo_producer.h" namespace "asapo": string Json() struct RequestCallbackPayload: GenericRequestHeader original_header - FileData data + MessageData data string response cdef extern from "asapo/asapo_producer.h" namespace "asapo": @@ -104,8 +95,8 @@ cdef extern from "asapo/asapo_producer.h" namespace "asapo" nogil: cppclass Producer: @staticmethod unique_ptr[Producer] Create(string endpoint,uint8_t nthreads,RequestHandlerType type, SourceCredentials source,uint64_t timeout_ms, Error* error) - Error SendFile(const EventHeader& event_header, string stream, string full_path, uint64_t ingest_mode,RequestCallback callback) - Error SendData__(const EventHeader& event_header, string stream, void* data, uint64_t ingest_mode,RequestCallback callback) + Error SendFromFile(const MessageHeader& message_header, string stream, string full_path, uint64_t ingest_mode,RequestCallback callback) + Error Send__(const MessageHeader& message_header, string stream, void* data, uint64_t ingest_mode,RequestCallback callback) void StopThreads__() void SetLogLevel(LogLevel level) uint64_t GetRequestsQueueSize() diff --git a/producer/api/python/asapo_producer.pyx.in b/producer/api/python/asapo_producer.pyx.in index a0814c3aaafc171d1e375e4d955e6adf62188819..d6322ae309a2b45a35bebfb998223cf926b29651 100644 --- a/producer/api/python/asapo_producer.pyx.in +++ b/producer/api/python/asapo_producer.pyx.in @@ -108,12 +108,12 @@ cdef class PyProducer: self.c_producer.get().SetLogLevel(log_level) def __send_np_array(self, id, exposed_path,data, user_meta=None,subset=None,stream="default",ingest_mode = DEFAULT_INGEST_MODE,callback=None): - cdef EventHeader event_header = self.create_event_header(id,exposed_path,user_meta,subset,ingest_mode) + cdef MessageHeader message_header = self.create_message_header(id,exposed_path,user_meta,subset,ingest_mode) if data is None: - event_header.file_size = 0 + message_header.data_size = 0 else: - event_header.file_size = data.nbytes - err = self.c_producer.get().SendData__(event_header, _bytes(stream), data_pointer_nparray(data),ingest_mode, + message_header.data_size = data.nbytes + err = self.c_producer.get().Send__(message_header, _bytes(stream), data_pointer_nparray(data),ingest_mode, unwrap_callback_with_memory(<RequestCallbackCythonMemory>self.c_callback_ndarr, <void*>self,<void*>callback, <void*>data)) if err: @@ -126,23 +126,23 @@ cdef class PyProducer: if callback != None: Py_XINCREF(<PyObject*>callback) return - cdef EventHeader create_event_header(self,uint64_t id, exposed_path,user_meta,subset,ingest_mode): - cdef EventHeader event_header - event_header.file_id = id - event_header.file_name = _bytes(exposed_path) - event_header.user_metadata = _bytes(user_meta) if user_meta!=None else "" + cdef MessageHeader create_message_header(self,uint64_t id, exposed_path,user_meta,subset,ingest_mode): + cdef MessageHeader message_header + message_header.message_id = id + message_header.file_name = _bytes(exposed_path) + message_header.user_metadata = _bytes(user_meta) if user_meta!=None else "" if subset == None: - event_header.id_in_subset = 0 - event_header.subset_size = 0 + message_header.id_in_subset = 0 + message_header.subset_size = 0 else: - event_header.id_in_subset = subset[0] - event_header.subset_size = subset[1] - return event_header + message_header.id_in_subset = subset[0] + message_header.subset_size = subset[1] + return message_header def __send_bytes(self, id, exposed_path,data, user_meta=None,subset=None, stream="default", ingest_mode = DEFAULT_INGEST_MODE,callback=None): - cdef EventHeader event_header = self.create_event_header(id,exposed_path,user_meta,subset,ingest_mode) - event_header.file_size = len(data) - err = self.c_producer.get().SendData__(event_header,_bytes(stream), data_pointer_bytes(data), ingest_mode, + cdef MessageHeader message_header = self.create_message_header(id,exposed_path,user_meta,subset,ingest_mode) + message_header.data_size = len(data) + err = self.c_producer.get().Send__(message_header,_bytes(stream), data_pointer_bytes(data), ingest_mode, unwrap_callback_with_memory(<RequestCallbackCythonMemory>self.c_callback_bytesaddr, <void*>self,<void*>callback, <void*>data)) if err: @@ -258,9 +258,9 @@ cdef class PyProducer: AsapoProducerError: actually should not happen """ - cdef EventHeader event_header = self.create_event_header(id,exposed_path,user_meta,subset,ingest_mode) - event_header.file_size = 0 - err = self.c_producer.get().SendFile(event_header, _bytes(stream), _bytes(local_path), ingest_mode, + cdef MessageHeader message_header = self.create_message_header(id,exposed_path,user_meta,subset,ingest_mode) + message_header.data_size = 0 + err = self.c_producer.get().SendFromFile(message_header, _bytes(stream), _bytes(local_path), ingest_mode, unwrap_callback(<RequestCallbackCython>self.c_callback, <void*>self,<void*>callback if callback != None else NULL)) if err: throw_exception(err) diff --git a/producer/event_monitor_producer/src/event_detector.h b/producer/event_monitor_producer/src/event_detector.h index 610b8aaa478b5968a11aec96b999c2d0beee12c2..b38c3294a452af2f55e46ee15131185b9d6ee863 100644 --- a/producer/event_monitor_producer/src/event_detector.h +++ b/producer/event_monitor_producer/src/event_detector.h @@ -8,7 +8,7 @@ namespace asapo { class AbstractEventDetector { public: - virtual Error GetNextEvent(EventHeader* event_header) = 0; + virtual Error GetNextEvent(MessageHeader* message_header) = 0; virtual Error StartMonitoring() = 0; virtual ~AbstractEventDetector() = default; }; diff --git a/producer/event_monitor_producer/src/folder_event_detector.cpp b/producer/event_monitor_producer/src/folder_event_detector.cpp index 6ed45b9923396e5f10d69fdaddfcd14707a63d0d..e108144ba696211143e033c46e1d98d5a897bddf 100644 --- a/producer/event_monitor_producer/src/folder_event_detector.cpp +++ b/producer/event_monitor_producer/src/folder_event_detector.cpp @@ -49,7 +49,7 @@ Error FolderEventDetector::UpdateEventsBuffer() { for (auto& file : files) { if (!IgnoreFile(file) && FileInWhiteList(file) ) { - events_buffer_.emplace_back(EventHeader{0, 0, file}); + events_buffer_.emplace_back(MessageHeader{0, 0, file}); } } @@ -57,7 +57,7 @@ Error FolderEventDetector::UpdateEventsBuffer() { } -Error FolderEventDetector::GetNextEvent(EventHeader* event_header) { +Error FolderEventDetector::GetNextEvent(MessageHeader* message_header) { if (!monitoring_started_) { auto err = TextError("monitoring is not started yet"); return err; @@ -69,7 +69,7 @@ Error FolderEventDetector::GetNextEvent(EventHeader* event_header) { } } - return GetHeaderFromBuffer(event_header); + return GetHeaderFromBuffer(message_header); } bool FolderEventDetector::BufferIsEmpty() const { @@ -90,11 +90,11 @@ Error FolderEventDetector::StartMonitoring() { return nullptr; } -Error FolderEventDetector::GetHeaderFromBuffer(EventHeader* event_header) { +Error FolderEventDetector::GetHeaderFromBuffer(MessageHeader* message_header) { if (events_buffer_.size() == 0) { return EventMonitorErrorTemplates::kNoNewEvent.Generate(); } - *event_header = std::move(events_buffer_.front()); + *message_header = std::move(events_buffer_.front()); events_buffer_.pop_front(); return nullptr; } diff --git a/producer/event_monitor_producer/src/folder_event_detector.h b/producer/event_monitor_producer/src/folder_event_detector.h index 75874ee414a53546c11ec4fa143450d8bc0632ef..f2381a8198beedc18f9c0736f0c2f5d61cd3779a 100644 --- a/producer/event_monitor_producer/src/folder_event_detector.h +++ b/producer/event_monitor_producer/src/folder_event_detector.h @@ -14,16 +14,16 @@ namespace asapo { class FolderEventDetector : public AbstractEventDetector { public: - Error GetNextEvent(EventHeader* event_header) override; + Error GetNextEvent(MessageHeader* message_header) override; Error StartMonitoring() override; FolderEventDetector(const EventMonConfig* config); std::unique_ptr<SystemFolderWatch> system_folder_watch__; private: const EventMonConfig* config_; bool monitoring_started_ = false; - std::deque<EventHeader> events_buffer_; + std::deque<MessageHeader> events_buffer_; Error UpdateEventsBuffer(); - Error GetHeaderFromBuffer(EventHeader* event_header); + Error GetHeaderFromBuffer(MessageHeader* message_header); bool IgnoreFile(const std::string& event); bool FileInWhiteList(const std::string& file); bool BufferIsEmpty() const; diff --git a/producer/event_monitor_producer/src/main_eventmon.cpp b/producer/event_monitor_producer/src/main_eventmon.cpp index 4436f6dca6c41187773ec53f70004f6b723532f9..091c8b7df85da3e1d01240d124a634c012fdab63 100644 --- a/producer/event_monitor_producer/src/main_eventmon.cpp +++ b/producer/event_monitor_producer/src/main_eventmon.cpp @@ -76,14 +76,14 @@ void SignalHandler(int signal) { } -void HandleSubsets(asapo::EventHeader* header) { +void HandleSubsets(asapo::MessageHeader* header) { switch (GetEventMonConfig()->subset_mode) { case asapo::SubSetMode::kNone: return; case asapo::SubSetMode::kBatch: header->subset_size = GetEventMonConfig()->subset_batch_size; - header->id_in_subset = (header->file_id - 1) % header->subset_size + 1; - header->file_id = (header->file_id - 1) / header->subset_size + 1; + header->id_in_subset = (header->message_id - 1) % header->subset_size + 1; + header->message_id = (header->message_id - 1) / header->subset_size + 1; break; case asapo::SubSetMode::kMultiSource: header->subset_size = GetEventMonConfig()->subset_multisource_nsources; @@ -124,8 +124,8 @@ int main (int argc, char* argv[]) { int i = 0; while (true) { - asapo::EventHeader event_header; - auto err = event_detector->GetNextEvent(&event_header); + asapo::MessageHeader message_header; + auto err = event_detector->GetNextEvent(&message_header); if (stop_signal) { break; // we check it here because signal can interrupt system call (ready by inotify and result in incomplete event data) } @@ -135,10 +135,10 @@ int main (int argc, char* argv[]) { } continue; } - event_header.file_id = ++i; - HandleSubsets(&event_header); - producer->SendFile(event_header, GetEventMonConfig()->root_monitored_folder + asapo::kPathSeparator + - event_header.file_name, asapo::kDefaultIngestMode, ProcessAfterSend); + message_header.message_id = ++i; + HandleSubsets(&message_header); + producer->SendFromFile(message_header, GetEventMonConfig()->root_monitored_folder + asapo::kPathSeparator + + message_header.file_name, asapo::kDefaultIngestMode, ProcessAfterSend); } logger->Info("Producer exit. Processed " + std::to_string(i) + " files"); diff --git a/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp b/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp index 7221bbf1322ea6faccd7181a7257cc11cf21fc79..1ce260a12174a0944ba5bae9fa6e535a5854684f 100644 --- a/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp +++ b/producer/event_monitor_producer/unittests/test_folder_event_detector.cpp @@ -65,7 +65,7 @@ class FolderEventDetectorTests : public testing::Test { } void MockStartMonitoring(); void MockGetEvents(); - asapo::EventHeader InitiateAndReadSingleEvent(); + asapo::MessageHeader InitiateAndReadSingleEvent(); }; void FolderEventDetectorTests::MockStartMonitoring() { @@ -120,13 +120,13 @@ TEST_F(FolderEventDetectorTests, GetNextReturnsErrorIfMonitoringNotStarted) { TEST_F(FolderEventDetectorTests, GetNextCallsSystemGetNextFirstTimeNoEvents) { MockStartMonitoring(); - asapo::EventHeader event_header; + asapo::MessageHeader message_header; EXPECT_CALL(mock_system_folder_watch, GetFileEventList_t(_)); detector.StartMonitoring(); - auto err = detector.GetNextEvent(&event_header); + auto err = detector.GetNextEvent(&message_header); ASSERT_THAT(err, Eq(asapo::EventMonitorErrorTemplates::kNoNewEvent)); } @@ -140,8 +140,8 @@ TEST_F(FolderEventDetectorTests, GetNextEventError) { detector.StartMonitoring(); - asapo::EventHeader event_header; - auto err = detector.GetNextEvent(&event_header); + asapo::MessageHeader message_header; + auto err = detector.GetNextEvent(&message_header); ASSERT_THAT(err, Eq(asapo::EventMonitorErrorTemplates::kSystemError)); } @@ -153,14 +153,14 @@ void FolderEventDetectorTests::MockGetEvents() { )); } -asapo::EventHeader FolderEventDetectorTests::InitiateAndReadSingleEvent() { +asapo::MessageHeader FolderEventDetectorTests::InitiateAndReadSingleEvent() { MockStartMonitoring(); MockGetEvents(); detector.StartMonitoring(); - asapo::EventHeader event_header; - detector.GetNextEvent(&event_header); + asapo::MessageHeader message_header; + detector.GetNextEvent(&message_header); Mock::VerifyAndClearExpectations(&mock_system_folder_watch); - return event_header; + return message_header; }; @@ -170,11 +170,11 @@ TEST_F(FolderEventDetectorTests, GetNextEventOK) { detector.StartMonitoring(); - asapo::EventHeader event_header; - auto err = detector.GetNextEvent(&event_header); + asapo::MessageHeader message_header; + auto err = detector.GetNextEvent(&message_header); ASSERT_THAT(err, Eq(nullptr)); - ASSERT_THAT(event_header.file_name, Eq("test1.dat")); + ASSERT_THAT(message_header.file_name, Eq("test1.dat")); } @@ -185,11 +185,11 @@ TEST_F(FolderEventDetectorTests, GetNextEventDoesDoSystemCallIfListNotEmpty) { EXPECT_CALL(mock_system_folder_watch, GetFileEventList_t(_)).Times(0); - asapo::EventHeader event_header; - auto err = detector.GetNextEvent(&event_header); + asapo::MessageHeader message_header; + auto err = detector.GetNextEvent(&message_header); ASSERT_THAT(err, Eq(nullptr)); - ASSERT_THAT(event_header.file_name, Eq("test2.dat")); + ASSERT_THAT(message_header.file_name, Eq("test2.dat")); } @@ -198,36 +198,36 @@ TEST_F(FolderEventDetectorTests, GetNextEventDoesSystemCallIfListEmpty) { EXPECT_CALL(mock_system_folder_watch, GetFileEventList_t(_)).Times(1); // read events 2 to 4 - asapo::EventHeader event_header; - err = detector.GetNextEvent(&event_header); + asapo::MessageHeader message_header; + err = detector.GetNextEvent(&message_header); ASSERT_THAT(err, Eq(nullptr)); - err = detector.GetNextEvent(&event_header); + err = detector.GetNextEvent(&message_header); ASSERT_THAT(err, Eq(nullptr)); - err = detector.GetNextEvent(&event_header); + err = detector.GetNextEvent(&message_header); ASSERT_THAT(err, Eq(nullptr)); // read events - should initiate system call since the buffer is empty now - err = detector.GetNextEvent(&event_header); + err = detector.GetNextEvent(&message_header); ASSERT_THAT(err, Eq(asapo::EventMonitorErrorTemplates::kNoNewEvent)); } TEST_F(FolderEventDetectorTests, GetNextIgnoresTmpFiles) { test_config.ignored_extensions = {"tmp"}; InitiateAndReadSingleEvent(); - asapo::EventHeader event_header; - err = detector.GetNextEvent(&event_header); + asapo::MessageHeader message_header; + err = detector.GetNextEvent(&message_header); ASSERT_THAT(err, Eq(nullptr)); - ASSERT_THAT(event_header.file_name, Eq("test2.dat")); + ASSERT_THAT(message_header.file_name, Eq("test2.dat")); // try read event 3 test3.tmp sould be ignored - err = detector.GetNextEvent(&event_header); + err = detector.GetNextEvent(&message_header); ASSERT_THAT(err, Eq(asapo::EventMonitorErrorTemplates::kNoNewEvent)); } TEST_F(FolderEventDetectorTests, GetNextRespectsWhiteList) { test_config.whitelisted_extensions = {"tmp"}; - auto event_header = InitiateAndReadSingleEvent(); + auto message_header = InitiateAndReadSingleEvent(); ASSERT_THAT(err, Eq(nullptr)); - ASSERT_THAT(event_header.file_name, Eq("test3.tmp")); + ASSERT_THAT(message_header.file_name, Eq("test3.tmp")); } } diff --git a/producer/event_monitor_producer/unittests/test_single_folder_watch_windows.cpp b/producer/event_monitor_producer/unittests/test_single_folder_watch_windows.cpp index 2e83f0f8f4e28bbd826ec41d6f887877ca6e94b6..c6aa2473071b9ad66a38f429b7bc4923bd836005 100644 --- a/producer/event_monitor_producer/unittests/test_single_folder_watch_windows.cpp +++ b/producer/event_monitor_producer/unittests/test_single_folder_watch_windows.cpp @@ -31,8 +31,8 @@ using ::asapo::Error; using ::asapo::ErrorInterface; using asapo::FilesToSend; using asapo::SingleFolderWatch; -using asapo::FileInfos; -using asapo::FileInfo; +using asapo::MessageMetas; +using asapo::MessageMeta; namespace { diff --git a/producer/event_monitor_producer/unittests/test_system_folder_watch_linux.cpp b/producer/event_monitor_producer/unittests/test_system_folder_watch_linux.cpp index 7e1d3fd8eb688fcb8257c28db34f85b852909667..68e852f63c370295ab982613fb933b05ff59c47c 100644 --- a/producer/event_monitor_producer/unittests/test_system_folder_watch_linux.cpp +++ b/producer/event_monitor_producer/unittests/test_system_folder_watch_linux.cpp @@ -27,8 +27,8 @@ using ::asapo::Error; using ::asapo::ErrorInterface; using asapo::FilesToSend; using asapo::SystemFolderWatch; -using asapo::FileInfos; -using asapo::FileInfo; +using asapo::MessageMetas; +using asapo::MessageMeta; namespace { @@ -39,15 +39,15 @@ TEST(SystemFolderWatch, Constructor) { ASSERT_THAT(dynamic_cast<asapo::Inotify*>(watch.inotify__.get()), Ne(nullptr)); } -FileInfos CreateTestFileInfos() { - FileInfos file_infos; - FileInfo fi; +MessageMetas CreateTestMessageMetas() { + MessageMetas message_metas; + MessageMeta fi; fi.size = 100; fi.name = "file1"; - file_infos.push_back(fi); + message_metas.push_back(fi); fi.name = "subfolder/file2"; - file_infos.push_back(fi); - return file_infos; + message_metas.push_back(fi); + return message_metas; } @@ -64,7 +64,7 @@ class SystemFolderWatchTests : public testing::Test { std::vector<std::string> expected_watches{"/tmp/test1", "/tmp/test2", "/tmp/test1/sub11", "/tmp/test2/sub21", "/tmp/test2/sub22", "/tmp/test2/sub21/sub211"}; std::string expected_filename1{"file1"}; std::string expected_filename2{"file2"}; - FileInfos expected_fileinfos = CreateTestFileInfos(); + MessageMetas expected_fileinfos = CreateTestMessageMetas(); int expected_wd = 10; std::vector<int>expected_fds = {1, 2, 3, 4, 5, 6}; void MockStartMonitoring(); @@ -291,7 +291,7 @@ void SystemFolderWatchTests::ExpectCreateFolder(std::string folder, bool with_fi } else { ON_CALL(mock_io, FilesInFolder_t(newfolder, _)). WillByDefault(DoAll(testing::SetArgPointee<1>(nullptr), - testing::Return(FileInfos{}))); + testing::Return(MessageMetas{}))); } } diff --git a/producer/event_monitor_producer/unittests/test_system_folder_watch_windows.cpp b/producer/event_monitor_producer/unittests/test_system_folder_watch_windows.cpp index 3329022a2dcd720a06ef8f4d0d05717f5e023d2d..24faffcc9c07c77bee6f8e8aec77aedd8b742865 100644 --- a/producer/event_monitor_producer/unittests/test_system_folder_watch_windows.cpp +++ b/producer/event_monitor_producer/unittests/test_system_folder_watch_windows.cpp @@ -26,8 +26,8 @@ using ::asapo::Error; using ::asapo::ErrorInterface; using asapo::FilesToSend; using asapo::SystemFolderWatch; -using asapo::FileInfos; -using asapo::FileInfo; +using asapo::MessageMetas; +using asapo::MessageMeta; namespace { @@ -37,15 +37,15 @@ TEST(SystemFolderWatch, Constructor) { ASSERT_THAT(dynamic_cast<asapo::IO*>(watch.io__.get()), Ne(nullptr)); } -FileInfos CreateTestFileInfos() { - FileInfos file_infos; - FileInfo fi; +MessageMetas CreateTestMessageMetas() { + MessageMetas message_metas; + MessageMeta fi; fi.size = 100; fi.name = "file1"; - file_infos.push_back(fi); + message_metas.push_back(fi); fi.name = "subfolder\\file2"; - file_infos.push_back(fi); - return file_infos; + message_metas.push_back(fi); + return message_metas; } diff --git a/receiver/src/request.h b/receiver/src/request.h index 5d462d7f1ab620a5b2c913f88f9b0d9564d72513..f24ad10fcc1927e69e80a2edf91a64efbc73cc26 100644 --- a/receiver/src/request.h +++ b/receiver/src/request.h @@ -83,7 +83,7 @@ class Request { private: const GenericRequestHeader request_header_; const SocketDescriptor socket_fd_; - FileData data_buffer_; + MessageData data_buffer_; void* data_ptr; RequestHandlerList handlers_; std::string origin_uri_; diff --git a/receiver/src/request_handler/request_handler_db_check_request.cpp b/receiver/src/request_handler/request_handler_db_check_request.cpp index 3ce2b9dacff95971fc136f405011f173dc798eaf..60fbe61a1ad7d6b3e5ea20c3e83c330d3eb091ec 100644 --- a/receiver/src/request_handler/request_handler_db_check_request.cpp +++ b/receiver/src/request_handler/request_handler_db_check_request.cpp @@ -16,7 +16,7 @@ RequestHandlerDbCheckRequest::RequestHandlerDbCheckRequest(std::string collectio } -Error RequestHandlerDbCheckRequest::GetRecordFromDb(const Request* request, FileInfo* record ) const { +Error RequestHandlerDbCheckRequest::GetRecordFromDb(const Request* request, MessageMeta* record ) const { auto op_code = request->GetOpCode(); auto id = request->GetDataID(); auto col_name = collection_name_prefix_ + "_" + request->GetStream(); @@ -40,7 +40,7 @@ Error RequestHandlerDbCheckRequest::GetRecordFromDb(const Request* request, File } -bool RequestHandlerDbCheckRequest::SameRequestInRecord(const Request* request, const FileInfo& record) const { +bool RequestHandlerDbCheckRequest::SameRequestInRecord(const Request* request, const MessageMeta& record) const { std::string meta = request->GetMetaData(); if (meta.size() == 0) { // so it is stored in database meta = "{}"; @@ -55,7 +55,7 @@ Error RequestHandlerDbCheckRequest::ProcessRequest(Request* request) const { return err; } - FileInfo record; + MessageMeta record; auto err = GetRecordFromDb(request, &record); if (err) { return err == DBErrorTemplates::kNoRecord ? nullptr : std::move(err); diff --git a/receiver/src/request_handler/request_handler_db_check_request.h b/receiver/src/request_handler/request_handler_db_check_request.h index 9b50673acc3b142f7b25964444869d04b648c4fe..d28dc7dffc3f98b749ca6448a20a54862e7bce62 100644 --- a/receiver/src/request_handler/request_handler_db_check_request.h +++ b/receiver/src/request_handler/request_handler_db_check_request.h @@ -14,8 +14,8 @@ class RequestHandlerDbCheckRequest FINAL : public RequestHandlerDb { RequestHandlerDbCheckRequest(std::string collection_name_prefix); Error ProcessRequest(Request* request) const override; private: - Error GetRecordFromDb(const Request* request, FileInfo* record) const; - bool SameRequestInRecord(const Request* request, const FileInfo& record) const; + Error GetRecordFromDb(const Request* request, MessageMeta* record) const; + bool SameRequestInRecord(const Request* request, const MessageMeta& record) const; }; diff --git a/receiver/src/request_handler/request_handler_db_write.cpp b/receiver/src/request_handler/request_handler_db_write.cpp index 3dece0e77768e3d4537e3b046852029bee38d1ab..2d61d1122cf98b57a23ee13a8c10eaae4ffd79ce 100644 --- a/receiver/src/request_handler/request_handler_db_write.cpp +++ b/receiver/src/request_handler/request_handler_db_write.cpp @@ -52,25 +52,25 @@ Error RequestHandlerDbWrite::ProcessDuplicateRecordSituation(Request* request) c Error RequestHandlerDbWrite::InsertRecordToDb(const Request* request) const { - auto file_info = PrepareFileInfo(request); + auto message_meta = PrepareMessageMeta(request); auto op_code = request->GetOpCode(); auto col_name = collection_name_prefix_ + "_" + request->GetStream(); Error err; if (op_code == Opcode::kOpcodeTransferData) { - err = db_client__->Insert(col_name, file_info, false); + err = db_client__->Insert(col_name, message_meta, false); if (!err) { - log__->Debug(std::string{"insert record id "} + std::to_string(file_info.id) + " to " + col_name + " in " + + log__->Debug(std::string{"insert record id "} + std::to_string(message_meta.id) + " to " + col_name + " in " + db_name_ + " at " + GetReceiverConfig()->database_uri); } } else { - auto subset_id = file_info.id; - file_info.id = request->GetCustomData()[1]; + auto subset_id = message_meta.id; + message_meta.id = request->GetCustomData()[1]; auto subset_size = request->GetCustomData()[2]; - err = db_client__->InsertAsSubset(col_name, file_info, subset_id, subset_size, false); + err = db_client__->InsertAsSubset(col_name, message_meta, subset_id, subset_size, false); if (!err) { - log__->Debug(std::string{"insert record as subset id "} + std::to_string(file_info.id) + ", id in subset: " + + log__->Debug(std::string{"insert record as subset id "} + std::to_string(message_meta.id) + ", id in subset: " + std::to_string(subset_id) + " to " + col_name + " in " + db_name_ + " at " + GetReceiverConfig()->database_uri); @@ -79,16 +79,16 @@ Error RequestHandlerDbWrite::InsertRecordToDb(const Request* request) const { return err; } -FileInfo RequestHandlerDbWrite::PrepareFileInfo(const Request* request) const { - FileInfo file_info; - file_info.name = request->GetFileName(); - file_info.size = request->GetDataSize(); - file_info.id = request->GetDataID(); - file_info.buf_id = request->GetSlotId(); - file_info.source = GetReceiverConfig()->dataserver.advertise_uri; - file_info.metadata = request->GetMetaData(); - file_info.timestamp = std::chrono::system_clock::now(); - return file_info; +MessageMeta RequestHandlerDbWrite::PrepareMessageMeta(const Request* request) const { + MessageMeta message_meta; + message_meta.name = request->GetFileName(); + message_meta.size = request->GetDataSize(); + message_meta.id = request->GetDataID(); + message_meta.buf_id = request->GetSlotId(); + message_meta.source = GetReceiverConfig()->dataserver.advertise_uri; + message_meta.metadata = request->GetMetaData(); + message_meta.timestamp = std::chrono::system_clock::now(); + return message_meta; } RequestHandlerDbWrite::RequestHandlerDbWrite(std::string collection_name_prefix) : RequestHandlerDb(std::move( diff --git a/receiver/src/request_handler/request_handler_db_write.h b/receiver/src/request_handler/request_handler_db_write.h index 089f1b59c2c2aa51ac6a073f48c54f0d26055afa..bb3e78a99599d14ba3c3319b6614e63520d50939 100644 --- a/receiver/src/request_handler/request_handler_db_write.h +++ b/receiver/src/request_handler/request_handler_db_write.h @@ -14,7 +14,7 @@ class RequestHandlerDbWrite final: public RequestHandlerDb { Error ProcessRequest(Request* request) const override; RequestHandlerDbWrite(std::string collection_name_prefix); private: - FileInfo PrepareFileInfo(const Request* request) const; + MessageMeta PrepareMessageMeta(const Request* request) const; Error InsertRecordToDb(const Request* request) const; Error ProcessDuplicateRecordSituation(Request* request) const; diff --git a/receiver/unittests/receiver_data_server/net_server/test_rds_tcp_server.cpp b/receiver/unittests/receiver_data_server/net_server/test_rds_tcp_server.cpp index 25f31dfb592b5ee411f8f667072de842fa574845..1106da959b420f2614429f23f2e925c235d0e28d 100644 --- a/receiver/unittests/receiver_data_server/net_server/test_rds_tcp_server.cpp +++ b/receiver/unittests/receiver_data_server/net_server/test_rds_tcp_server.cpp @@ -279,7 +279,7 @@ TEST_F(RdsTCPServerTests, SendResponseAndSlotData_SendResponseError) { ASSERT_THAT(err, Ne(nullptr)); } -TEST_F(RdsTCPServerTests, SendResponseAndSlotData_SendDataError) { +TEST_F(RdsTCPServerTests, SendResponseAndSlotData_SendError) { asapo::GenericNetworkResponse tmp {}; asapo::ReceiverDataServerRequest expectedRequest {{}, 30}; diff --git a/receiver/unittests/request_handler/test_request_factory.cpp b/receiver/unittests/request_handler/test_request_factory.cpp index 77742d925b9be1f2c9a16c37b502e4dd43283267..bb924ba635e4939ad43a7cad0ad7f5d6a51e3c02 100644 --- a/receiver/unittests/request_handler/test_request_factory.cpp +++ b/receiver/unittests/request_handler/test_request_factory.cpp @@ -79,7 +79,7 @@ TEST_F(FactoryTests, ErrorOnWrongCode) { ASSERT_THAT(err, Ne(nullptr)); } -TEST_F(FactoryTests, ReturnsDataRequestOnkNetOpcodeSendDataCode) { +TEST_F(FactoryTests, ReturnsDataRequestOnkNetOpcodeSendCode) { for (auto code : std::vector<asapo::Opcode> {asapo::Opcode::kOpcodeTransferData, asapo::Opcode::kOpcodeTransferSubsetData}) { generic_request_header.op_code = code; auto request = factory.GenerateRequest(generic_request_header, 1, origin_uri, &err); @@ -95,7 +95,7 @@ TEST_F(FactoryTests, ReturnsDataRequestOnkNetOpcodeSendDataCode) { } } -TEST_F(FactoryTests, ReturnsDataRequestOnkNetOpcodeSendDataCodeLargeFile) { +TEST_F(FactoryTests, ReturnsDataRequestOnkNetOpcodeSendCodeLargeFile) { for (auto code : std::vector<asapo::Opcode> {asapo::Opcode::kOpcodeTransferData, asapo::Opcode::kOpcodeTransferSubsetData}) { generic_request_header.op_code = code; config.receive_to_disk_threshold_mb = 0; diff --git a/receiver/unittests/request_handler/test_request_handler_db.cpp b/receiver/unittests/request_handler/test_request_handler_db.cpp index 0cda482d766393ddc76ef4532943cd2781991d57..8f3bf1549c27064bc53ed1fe7320b9ef45e5d2a7 100644 --- a/receiver/unittests/request_handler/test_request_handler_db.cpp +++ b/receiver/unittests/request_handler/test_request_handler_db.cpp @@ -19,7 +19,7 @@ #include "../receiver_mocking.h" using asapo::MockRequest; -using asapo::FileInfo; +using asapo::MessageMeta; using ::testing::Test; using ::testing::Return; using ::testing::ReturnRef; diff --git a/receiver/unittests/request_handler/test_request_handler_db_check_request.cpp b/receiver/unittests/request_handler/test_request_handler_db_check_request.cpp index 1a5d57574f97b60bf5b3539d88413a4ef960a14e..31d93693bbad712281eb8f2d2340802a4bc8478d 100644 --- a/receiver/unittests/request_handler/test_request_handler_db_check_request.cpp +++ b/receiver/unittests/request_handler/test_request_handler_db_check_request.cpp @@ -21,7 +21,7 @@ #include "../receiver_mocking.h" using asapo::MockRequest; -using asapo::FileInfo; +using asapo::MessageMeta; using ::testing::Test; using ::testing::Return; using ::testing::ReturnRef; @@ -85,7 +85,7 @@ class DbCheckRequestHandlerTests : public Test { uint64_t expected_subset_id = 16; uint64_t expected_subset_size = 2; uint64_t expected_custom_data[asapo::kNCustomParams] {0, expected_subset_id, expected_subset_size}; - FileInfo expected_file_info; + MessageMeta expected_message_meta; MockFunctions mock_functions; int n_run = 0; void SetUp() override { @@ -99,7 +99,7 @@ class DbCheckRequestHandlerTests : public Test { config.dataserver.advertise_uri = expected_host_uri; config.dataserver.listen_port = expected_port; SetReceiverConfig(config, "none"); - expected_file_info = PrepareFileInfo(); + expected_message_meta = PrepareMessageMeta(); mock_functions.push_back([this](asapo::ErrorInterface * error, bool expect_compare) { MockGetByID(error, expect_compare); n_run++; @@ -113,7 +113,7 @@ class DbCheckRequestHandlerTests : public Test { } void ExpectRequestParams(asapo::Opcode op_code, const std::string& data_source, bool expect_compare = true); - FileInfo PrepareFileInfo(); + MessageMeta PrepareMessageMeta(); void MockGetByID(asapo::ErrorInterface* error, bool expect_compare); void MockGetSetByID(asapo::ErrorInterface* error, bool expect_compare); void TearDown() override { @@ -123,7 +123,7 @@ class DbCheckRequestHandlerTests : public Test { }; -MATCHER_P(CompareFileInfo, file, "") { +MATCHER_P(CompareMessageMeta, file, "") { if (arg.size != file.size) return false; if (arg.source != file.source) return false; if (arg.buf_id != file.buf_id) return false; @@ -186,22 +186,22 @@ void DbCheckRequestHandlerTests::ExpectRequestParams(asapo::Opcode op_code, cons } } -FileInfo DbCheckRequestHandlerTests::PrepareFileInfo() { - FileInfo file_info; - file_info.size = expected_file_size; - file_info.name = expected_file_name; - file_info.id = expected_id; - file_info.buf_id = expected_buf_id; - file_info.source = expected_host_uri; - file_info.metadata = expected_metadata; - return file_info; +MessageMeta DbCheckRequestHandlerTests::PrepareMessageMeta() { + MessageMeta message_meta; + message_meta.size = expected_file_size; + message_meta.name = expected_file_name; + message_meta.id = expected_id; + message_meta.buf_id = expected_buf_id; + message_meta.source = expected_host_uri; + message_meta.metadata = expected_metadata; + return message_meta; } void DbCheckRequestHandlerTests::MockGetByID(asapo::ErrorInterface* error, bool expect_compare ) { ExpectRequestParams(asapo::Opcode::kOpcodeTransferData, expected_data_source, expect_compare); EXPECT_CALL(mock_db, GetById_t(expected_collection_name, expected_id, _)). WillOnce(DoAll( - SetArgPointee<2>(expected_file_info), + SetArgPointee<2>(expected_message_meta), testing::Return(error) )); } @@ -210,14 +210,14 @@ void DbCheckRequestHandlerTests::MockGetSetByID(asapo::ErrorInterface* error, bo ExpectRequestParams(asapo::Opcode::kOpcodeTransferSubsetData, expected_data_source, expect_compare); EXPECT_CALL(mock_db, GetSetById_t(expected_collection_name, expected_subset_id, expected_id, _)). WillOnce(DoAll( - SetArgPointee<3>(expected_file_info), + SetArgPointee<3>(expected_message_meta), testing::Return(error) )); } TEST_F(DbCheckRequestHandlerTests, ErrorIfRecordsDoNotMatch) { - expected_file_info.metadata = expected_metadata + "_"; + expected_message_meta.metadata = expected_metadata + "_"; for (auto mock : mock_functions) { mock(nullptr, true); @@ -238,7 +238,7 @@ TEST_F(DbCheckRequestHandlerTests, DuplicateErrorIfRecordsMatch) { } TEST_F(DbCheckRequestHandlerTests, DuplicateErrorIfRecordsMatchWithEmptyMetadata) { - expected_file_info.metadata = "{}"; + expected_message_meta.metadata = "{}"; expected_metadata = ""; for (auto mock : mock_functions) { mock(nullptr, true); diff --git a/receiver/unittests/request_handler/test_request_handler_db_last_stream.cpp b/receiver/unittests/request_handler/test_request_handler_db_last_stream.cpp index da6740f13b0c4ff9f47b3ec3e70d65273108670a..2e6762cb541f123ce9b529e28b23bbf648792934 100644 --- a/receiver/unittests/request_handler/test_request_handler_db_last_stream.cpp +++ b/receiver/unittests/request_handler/test_request_handler_db_last_stream.cpp @@ -18,7 +18,7 @@ #include "../receiver_mocking.h" using asapo::MockRequest; -using asapo::FileInfo; +using asapo::MessageMeta; using ::testing::Test; using ::testing::Return; using ::testing::ReturnRef; diff --git a/receiver/unittests/request_handler/test_request_handler_db_meta_writer.cpp b/receiver/unittests/request_handler/test_request_handler_db_meta_writer.cpp index 7cec10a2cef7c4edb6a13e6840cd67fab9675a1b..ef677673aa55abcf99bb13dee9b078c796652c4b 100644 --- a/receiver/unittests/request_handler/test_request_handler_db_meta_writer.cpp +++ b/receiver/unittests/request_handler/test_request_handler_db_meta_writer.cpp @@ -18,7 +18,7 @@ #include "../receiver_mocking.h" using asapo::MockRequest; -using asapo::FileInfo; +using asapo::MessageMeta; using ::testing::Test; using ::testing::Return; using ::testing::ReturnRef; diff --git a/receiver/unittests/request_handler/test_request_handler_db_stream_info.cpp b/receiver/unittests/request_handler/test_request_handler_db_stream_info.cpp index 2596660e025102afcbb0eb769ab35c37012ccf3a..1d1d96d3d2cf47a84928f603fd5660af329b4804 100644 --- a/receiver/unittests/request_handler/test_request_handler_db_stream_info.cpp +++ b/receiver/unittests/request_handler/test_request_handler_db_stream_info.cpp @@ -18,7 +18,7 @@ #include "../receiver_mocking.h" using asapo::MockRequest; -using asapo::FileInfo; +using asapo::MessageMeta; using ::testing::Test; using ::testing::Return; using ::testing::ReturnRef; diff --git a/receiver/unittests/request_handler/test_request_handler_db_writer.cpp b/receiver/unittests/request_handler/test_request_handler_db_writer.cpp index f3b26a8c47adc8a5791def34adf505bd635f4540..b4962bedc273ebade9d5d48b256d7f27e1e203e4 100644 --- a/receiver/unittests/request_handler/test_request_handler_db_writer.cpp +++ b/receiver/unittests/request_handler/test_request_handler_db_writer.cpp @@ -21,7 +21,7 @@ #include "../receiver_mocking.h" using asapo::MockRequest; -using asapo::FileInfo; +using asapo::MessageMeta; using ::testing::Test; using ::testing::Return; using ::testing::ReturnRef; @@ -105,7 +105,7 @@ class DbWriterHandlerTests : public Test { void ExpectRequestParams(asapo::Opcode op_code, const std::string& data_source); void ExpectLogger(); void ExpectDuplicatedID(); - FileInfo PrepareFileInfo(); + MessageMeta PrepareMessageMeta(); void TearDown() override { handler.db_client__.release(); } @@ -113,7 +113,7 @@ class DbWriterHandlerTests : public Test { }; -MATCHER_P(CompareFileInfo, file, "") { +MATCHER_P(CompareMessageMeta, file, "") { if (arg.size != file.size) return false; if (arg.source != file.source) return false; if (arg.buf_id != file.buf_id) return false; @@ -189,15 +189,15 @@ void DbWriterHandlerTests::ExpectRequestParams(asapo::Opcode op_code, const std: -FileInfo DbWriterHandlerTests::PrepareFileInfo() { - FileInfo file_info; - file_info.size = expected_file_size; - file_info.name = expected_file_name; - file_info.id = expected_id; - file_info.buf_id = expected_buf_id; - file_info.source = expected_host_ip + ":" + std::to_string(expected_port); - file_info.metadata = expected_metadata; - return file_info; +MessageMeta DbWriterHandlerTests::PrepareMessageMeta() { + MessageMeta message_meta; + message_meta.size = expected_file_size; + message_meta.name = expected_file_name; + message_meta.id = expected_id; + message_meta.buf_id = expected_buf_id; + message_meta.source = expected_host_ip + ":" + std::to_string(expected_port); + message_meta.metadata = expected_metadata; + return message_meta; } void DbWriterHandlerTests::ExpectLogger() { EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("insert record"), @@ -214,9 +214,9 @@ void DbWriterHandlerTests::ExpectLogger() { TEST_F(DbWriterHandlerTests, CallsInsert) { ExpectRequestParams(asapo::Opcode::kOpcodeTransferData, expected_data_source); - auto file_info = PrepareFileInfo(); + auto message_meta = PrepareMessageMeta(); - EXPECT_CALL(mock_db, Insert_t(expected_collection_name, CompareFileInfo(file_info), false)). + EXPECT_CALL(mock_db, Insert_t(expected_collection_name, CompareMessageMeta(message_meta), false)). WillOnce(testing::Return(nullptr)); ExpectLogger(); @@ -226,10 +226,10 @@ TEST_F(DbWriterHandlerTests, CallsInsert) { TEST_F(DbWriterHandlerTests, CallsInsertSubset) { ExpectRequestParams(asapo::Opcode::kOpcodeTransferSubsetData, expected_data_source); - auto file_info = PrepareFileInfo(); + auto message_meta = PrepareMessageMeta(); - EXPECT_CALL(mock_db, InsertAsSubset_t(expected_collection_name, CompareFileInfo(file_info), expected_subset_id, + EXPECT_CALL(mock_db, InsertAsSubset_t(expected_collection_name, CompareMessageMeta(message_meta), expected_subset_id, expected_subset_size, false)). WillOnce(testing::Return( nullptr)); ExpectLogger(); @@ -240,9 +240,9 @@ TEST_F(DbWriterHandlerTests, CallsInsertSubset) { void DbWriterHandlerTests::ExpectDuplicatedID() { ExpectRequestParams(asapo::Opcode::kOpcodeTransferData, expected_data_source); - auto file_info = PrepareFileInfo(); + auto message_meta = PrepareMessageMeta(); - EXPECT_CALL(mock_db, Insert_t(expected_collection_name, CompareFileInfo(file_info), false)). + EXPECT_CALL(mock_db, Insert_t(expected_collection_name, CompareMessageMeta(message_meta), false)). WillOnce(testing::Return(asapo::DBErrorTemplates::kDuplicateID.Generate().release())); } diff --git a/receiver/unittests/request_handler/test_request_handler_receive_data.cpp b/receiver/unittests/request_handler/test_request_handler_receive_data.cpp index 5f4eca41f980cff3838720118e5517e8de5a8eb5..bc9a9116a8b62dc0d31825e3a31b299be211fbbd 100644 --- a/receiver/unittests/request_handler/test_request_handler_receive_data.cpp +++ b/receiver/unittests/request_handler/test_request_handler_receive_data.cpp @@ -29,7 +29,7 @@ using ::asapo::ErrorInterface; using ::asapo::FileDescriptor; using ::asapo::SocketDescriptor; using ::asapo::GenericRequestHeader; -using ::asapo::SendDataResponse; +using ::asapo::SendResponse; using ::asapo::GenericRequestHeader; using ::asapo::GenericNetworkResponse; using ::asapo::Opcode; diff --git a/receiver/unittests/request_handler/test_request_handler_receive_metadata.cpp b/receiver/unittests/request_handler/test_request_handler_receive_metadata.cpp index 0c52fa4252bea349b7ab178946199fcc2e0b9d7f..fbef728d127d162d4aebf03bb2255b31d322cdff 100644 --- a/receiver/unittests/request_handler/test_request_handler_receive_metadata.cpp +++ b/receiver/unittests/request_handler/test_request_handler_receive_metadata.cpp @@ -29,7 +29,7 @@ using ::asapo::ErrorInterface; using ::asapo::FileDescriptor; using ::asapo::SocketDescriptor; using ::asapo::GenericRequestHeader; -using ::asapo::SendDataResponse; +using ::asapo::SendResponse; using ::asapo::GenericRequestHeader; using ::asapo::GenericNetworkResponse; using ::asapo::Opcode; diff --git a/receiver/unittests/request_handler/test_requests_dispatcher.cpp b/receiver/unittests/request_handler/test_requests_dispatcher.cpp index 0a5227d121911ff6cced611d9f6432cfafff9418..e5d6daaf0354bcd3a9a40f0cd5de60674bf06152 100644 --- a/receiver/unittests/request_handler/test_requests_dispatcher.cpp +++ b/receiver/unittests/request_handler/test_requests_dispatcher.cpp @@ -36,7 +36,7 @@ using asapo::Error; using asapo::ErrorInterface; using asapo::SocketDescriptor; using asapo::GenericRequestHeader; -using asapo::SendDataResponse; +using asapo::SendResponse; using asapo::GenericRequestHeader; using asapo::GenericNetworkResponse; using asapo::Opcode; diff --git a/receiver/unittests/test_connection.cpp b/receiver/unittests/test_connection.cpp index 1366ce7a617b0934e6b7289e8da81184acbbdd7e..fca987832b01204c8708a8bfe85840e9476d1c68 100644 --- a/receiver/unittests/test_connection.cpp +++ b/receiver/unittests/test_connection.cpp @@ -39,7 +39,7 @@ using asapo::ErrorInterface; using asapo::FileDescriptor; using asapo::SocketDescriptor; using asapo::GenericRequestHeader; -using asapo::SendDataResponse; +using asapo::SendResponse; using asapo::GenericRequestHeader; using asapo::GenericNetworkResponse; using asapo::Opcode; diff --git a/receiver/unittests/test_request.cpp b/receiver/unittests/test_request.cpp index 07722c9406556bbf62ab21dcd9ccb167c7be2184..cea29e89dc78ee98dc004c9452cb33ac1ca40e96 100644 --- a/receiver/unittests/test_request.cpp +++ b/receiver/unittests/test_request.cpp @@ -29,7 +29,7 @@ using ::asapo::ErrorInterface; using ::asapo::FileDescriptor; using ::asapo::SocketDescriptor; using ::asapo::GenericRequestHeader; -using ::asapo::SendDataResponse; +using ::asapo::SendResponse; using ::asapo::GenericRequestHeader; using ::asapo::GenericNetworkResponse; using ::asapo::Opcode; diff --git a/tests/automatic/consumer/consumer_api/consumer_api.cpp b/tests/automatic/consumer/consumer_api/consumer_api.cpp index d5994af5ef735eb5c961e4a8338d1686e3da8395..101279ed0475d6ed435770352cf79b0f3de391d2 100644 --- a/tests/automatic/consumer/consumer_api/consumer_api.cpp +++ b/tests/automatic/consumer/consumer_api/consumer_api.cpp @@ -28,7 +28,7 @@ Args GetArgs(int argc, char* argv[]) { void TestSingle(const std::unique_ptr<asapo::Consumer>& consumer, const std::string& group_id) { - asapo::FileInfo fi; + asapo::MessageMeta fi; asapo::Error err; err = consumer->GetNext(&fi, group_id, nullptr); @@ -39,7 +39,7 @@ void TestSingle(const std::unique_ptr<asapo::Consumer>& consumer, const std::str M_AssertTrue(fi.name == "1", "GetNext filename"); M_AssertTrue(fi.metadata == "{\"test\":10}", "GetNext metadata"); - asapo::FileData data; + asapo::MessageData data; err = consumer->RetrieveData(&fi, &data); M_AssertTrue(err == nullptr, "RetrieveData no error"); M_AssertEq("hello1", std::string(data.get(), data.get() + fi.size)); @@ -203,7 +203,7 @@ void TestSingle(const std::unique_ptr<asapo::Consumer>& consumer, const std::str void TestDataset(const std::unique_ptr<asapo::Consumer>& consumer, const std::string& group_id) { - asapo::FileInfo fi; + asapo::MessageMeta fi; asapo::Error err; auto dataset = consumer->GetNextDataset(group_id, 0, &err); @@ -216,7 +216,7 @@ void TestDataset(const std::unique_ptr<asapo::Consumer>& consumer, const std::st M_AssertTrue(dataset.content[2].name == "1_3", "GetNextDataSet filename"); M_AssertTrue(dataset.content[0].metadata == "{\"test\":10}", "GetNext metadata"); - asapo::FileData data; + asapo::MessageData data; err = consumer->RetrieveData(&dataset.content[0], &data); M_AssertTrue(err == nullptr, "RetrieveData no error"); M_AssertEq("hello1", std::string(data.get(), data.get() + dataset.content[0].size)); diff --git a/tests/automatic/consumer/next_multithread_broker/next_multithread_broker.cpp b/tests/automatic/consumer/next_multithread_broker/next_multithread_broker.cpp index f4e67411d6a2d94c11fadaed43f979dcc243a2fc..bc89b9660cdc1aa5ee0558171a8df62f3115295d 100644 --- a/tests/automatic/consumer/next_multithread_broker/next_multithread_broker.cpp +++ b/tests/automatic/consumer/next_multithread_broker/next_multithread_broker.cpp @@ -5,15 +5,15 @@ #include "asapo/consumer/consumer.h" #include "testing.h" -void Assert(std::vector<asapo::FileInfos> file_infos, int nthreads, int nfiles) { +void Assert(std::vector<asapo::MessageMetas> message_metas, int nthreads, int nfiles) { std::vector<std::string> expect, result; for (int i = 1; i <= nfiles; i++) { expect.push_back(std::to_string(i)); } int nfiles_read = 0; for (int i = 0; i < nthreads; i++) { - nfiles_read += file_infos[i].size(); - for (const auto& fi : file_infos[i]) { + nfiles_read += message_metas[i].size(); + for (const auto& fi : message_metas[i]) { result.push_back(fi.name); } } @@ -66,11 +66,11 @@ void TestAll(const Args& args) { auto group_id = consumer->GenerateNewGroupId(&err); consumer->SetTimeout(10000); - std::vector<asapo::FileInfos>file_infos(args.nthreads); + std::vector<asapo::MessageMetas>message_metas(args.nthreads); auto exec_next = [&](int i) { - asapo::FileInfo fi; + asapo::MessageMeta fi; while ((err = consumer->GetNext(&fi, group_id, nullptr)) == nullptr) { - file_infos[i].emplace_back(fi); + message_metas[i].emplace_back(fi); } printf("%s\n", err->Explain().c_str()); }; @@ -86,7 +86,7 @@ void TestAll(const Args& args) { } } - Assert(file_infos, args.nthreads, args.nfiles); + Assert(message_metas, args.nthreads, args.nfiles); } int main(int argc, char* argv[]) { diff --git a/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp b/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp index 4e0fdc1450754595d221add00e7e6f287871f092..33e1b6a6665255ebc048374b96c67505155b77ff 100644 --- a/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp +++ b/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp @@ -65,7 +65,7 @@ int main(int argc, char* argv[]) { M_AssertEq("hello", content); M_AssertTrue(code == asapo::HttpCode::OK); // with array - asapo::FileData data; + asapo::MessageData data; err = consumer_impl->httpclient__->Post(args.uri_fts + "/transfer", cookie, transfer, &data, 5, &code); M_AssertEq( "hello", reinterpret_cast<char const*>(data.get())); M_AssertTrue(code == asapo::HttpCode::OK); diff --git a/tests/automatic/full_chain/send_recv_streams/send_recv_streams.cpp b/tests/automatic/full_chain/send_recv_streams/send_recv_streams.cpp index 4ebb3a8847cdc1962dfd083ea257679df24cd823..c80d5ecfef5ee794743acc513a1843be9017fddd 100644 --- a/tests/automatic/full_chain/send_recv_streams/send_recv_streams.cpp +++ b/tests/automatic/full_chain/send_recv_streams/send_recv_streams.cpp @@ -90,8 +90,8 @@ int main(int argc, char* argv[]) { uint64_t n = 1; for (uint64_t i = 0; i < n; i++) { - asapo::EventHeader event_header{i + 1, 0, std::to_string(i + 1)}; - producer->SendData(event_header, "stream1", nullptr, asapo::kTransferMetaDataOnly, ProcessAfterSend); + asapo::MessageHeader message_header{i + 1, 0, std::to_string(i + 1)}; + producer->Send(message_header, "stream1", nullptr, asapo::kTransferMetaDataOnly, ProcessAfterSend); } producer->SendStreamFinishedFlag("stream1", n, "stream2", ProcessAfterSend); producer->WaitRequestsFinished(10000); @@ -103,7 +103,7 @@ int main(int argc, char* argv[]) { exit(EXIT_FAILURE); } - asapo::FileInfo fi; + asapo::MessageMeta fi; for (uint64_t i = 0; i < n; i++) { consumer->GetNext(&fi, group_id, "stream1", nullptr); } diff --git a/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp b/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp index 318dfb64e7e9289fe2be8a27a3934fe3c903369b..c360f0339ac45070731b91e92a5c139ec903af9c 100644 --- a/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp +++ b/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp @@ -35,7 +35,7 @@ int main(int argc, char* argv[]) { auto args = GetArgs(argc, argv); asapo::MongoDBClient db; - asapo::FileInfo fi; + asapo::MessageMeta fi; fi.size = 100; fi.name = "relpath/1"; fi.id = args.file_id; @@ -66,7 +66,7 @@ int main(int argc, char* argv[]) { Assert(err, args.keyword); if (args.keyword == "OK") { // check retrieve - asapo::FileInfo fi_db; + asapo::MessageMeta fi_db; asapo::MongoDBClient db_new; db_new.Connect("127.0.0.1", "data"); err = db_new.GetById("data_test", fi.id, &fi_db); diff --git a/tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp b/tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp index 8e04f6e4790e39145b3217d6438b0db8a4104606..b10952d39d6c9a18d173f957bc8bc38185fe950c 100644 --- a/tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp +++ b/tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp @@ -34,7 +34,7 @@ int main(int argc, char* argv[]) { auto args = GetArgs(argc, argv); asapo::MongoDBClient db; - asapo::FileInfo fi; + asapo::MessageMeta fi; fi.size = 100; fi.name = "relpath/1"; uint64_t subset_id = args.file_id; @@ -63,7 +63,7 @@ int main(int argc, char* argv[]) { Assert(err, args.keyword); if (args.keyword == "OK") { // check retrieve - asapo::FileInfo fi_db; + asapo::MessageMeta fi_db; err = db.GetDataSetById("test", fi.id,subset_id, &fi_db); M_AssertTrue(fi_db == fi, "get record from db"); M_AssertEq(nullptr, err); diff --git a/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp b/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp index 15ac3fbeb3d0dc2302abc1b61c33f8609978296e..5d91a1fd83076ae400f20452e146cbd31629e058 100644 --- a/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp +++ b/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp @@ -55,7 +55,7 @@ void ProcessAfterSend(asapo::RequestCallbackPayload payload, asapo::Error err) { bool SendMetaData(asapo::Producer* producer) { - auto err = producer->SendMetaData("hello", &ProcessAfterSend); + auto err = producer->SendMetadata("hello", &ProcessAfterSend); if (err) { std::cerr << "Cannot send metadata: " << err << std::endl; return false; diff --git a/tests/automatic/system_io/read_file_content/read_file_content.cpp b/tests/automatic/system_io/read_file_content/read_file_content.cpp index fa4aa0a34182ad0e6b4e18fcc0caf96686e8ab38..743fb25e2f302451dfdcc6387a4ab3510feb1b1f 100644 --- a/tests/automatic/system_io/read_file_content/read_file_content.cpp +++ b/tests/automatic/system_io/read_file_content/read_file_content.cpp @@ -14,7 +14,7 @@ int main(int argc, char* argv[]) { asapo::Error err; auto io = std::unique_ptr<asapo::IO> {asapo::GenerateDefaultIO()}; - asapo::FileData data; + asapo::MessageData data; uint64_t size = 0; if (expect == "unknown_size") { data = io->GetDataFromFile(argv[1], &size, &err); diff --git a/tests/automatic/system_io/read_folder_content/read_folder_content.cpp b/tests/automatic/system_io/read_folder_content/read_folder_content.cpp index 4a8bddd5b7d0cdd1c4e593c807ad1d2f8f30ba80..a820b7a499d5dcef478b3d85362e1745c24d68ea 100644 --- a/tests/automatic/system_io/read_folder_content/read_folder_content.cpp +++ b/tests/automatic/system_io/read_folder_content/read_folder_content.cpp @@ -20,12 +20,12 @@ int main(int argc, char* argv[]) { std::string result{}; if (err == nullptr) { int64_t id = 0; - for(auto file_info : files) { - M_AssertEq(file_info.id, ++id); - if (file_info.name == "1") { - M_AssertEq(4, file_info.size); + for(auto message_meta : files) { + M_AssertEq(message_meta.id, ++id); + if (message_meta.name == "1") { + M_AssertEq(4, message_meta.size); } - result += file_info.name; + result += message_meta.name; } } else { result = err->Explain(); diff --git a/tests/automatic/system_io/write_data_to_file/write_data_to_file.cpp b/tests/automatic/system_io/write_data_to_file/write_data_to_file.cpp index f34c385b9978b5a8e66738f97a5ea7043038c604..5cbe7939899823846e3459bea39279d74357ff82 100644 --- a/tests/automatic/system_io/write_data_to_file/write_data_to_file.cpp +++ b/tests/automatic/system_io/write_data_to_file/write_data_to_file.cpp @@ -5,7 +5,7 @@ using asapo::IO; using asapo::Error; -using asapo::FileData; +using asapo::MessageData; struct Args { @@ -27,7 +27,7 @@ Args GetParams(int argc, char* argv[]) { return Args{fname, result, message, 3}; } -void AssertGoodResult(const std::unique_ptr<IO>& io, const Error& err, const FileData& data, +void AssertGoodResult(const std::unique_ptr<IO>& io, const Error& err, const MessageData& data, const Args& params) { if (err) { std::cerr << err << std::endl; @@ -53,7 +53,7 @@ int main(int argc, char* argv[]) { auto io = std::unique_ptr<asapo::IO> {asapo::GenerateDefaultIO()}; auto array = new uint8_t[params.length] {'1', '2', '3'}; - FileData data{array}; + MessageData data{array}; auto err = io->WriteDataToFile("", params.fname, data, params.length, true, true); diff --git a/tests/manual/asapo_fabric/fabric_client.cpp b/tests/manual/asapo_fabric/fabric_client.cpp index 47edfc2a22df0111c52d145fa13e0d1e6ebc4bbd..462a84f7c030005b7f761f86d75e4fbe8863f79f 100644 --- a/tests/manual/asapo_fabric/fabric_client.cpp +++ b/tests/manual/asapo_fabric/fabric_client.cpp @@ -39,7 +39,7 @@ int main(int argc, char* argv[]) { } size_t dataBufferSize = 1024 * kByte; - FileData dataBuffer = FileData{new uint8_t[dataBufferSize]}; + MessageData dataBuffer = MessageData{new uint8_t[dataBufferSize]}; std::cout << "Expected file size: " << dataBufferSize << " byte" << std::endl; auto serverAddress = client->AddServerAddress(serverAddressString, &error); diff --git a/tests/manual/asapo_fabric/fabric_server.cpp b/tests/manual/asapo_fabric/fabric_server.cpp index fa6f7865d40db1c021759aea7c79f88b863d4cda..981840e6ec785bb7b7dfc2dc48fc693a55fc1bab 100644 --- a/tests/manual/asapo_fabric/fabric_server.cpp +++ b/tests/manual/asapo_fabric/fabric_server.cpp @@ -9,7 +9,7 @@ using namespace asapo::fabric; volatile bool running = false; -void ServerThread(FabricServer* server, size_t bufferSize, FileData* buffer) { +void ServerThread(FabricServer* server, size_t bufferSize, MessageData* buffer) { Error error; while(running && !error) { FabricAddress clientAddress; @@ -69,7 +69,7 @@ int main(int argc, char* argv[]) { std::cout << "Server is listening on " << server->GetAddress() << std::endl; size_t dataBufferSize = 1024 * kByte; - FileData dataBuffer = FileData{new uint8_t[dataBufferSize]}; + MessageData dataBuffer = MessageData{new uint8_t[dataBufferSize]}; strcpy((char*)dataBuffer.get(), "I (the server) wrote into your buffer."); std::cout << "Expected file size: " << dataBufferSize << " byte" << std::endl; diff --git a/tests/manual/performance_broker_receiver/getlast_broker.cpp b/tests/manual/performance_broker_receiver/getlast_broker.cpp index 9ec15280db66a9fd9fbe7c95166c2290a4e7e0da..a3ee8ba05a72f6e9dbfc64c2f2bed0934b825577 100644 --- a/tests/manual/performance_broker_receiver/getlast_broker.cpp +++ b/tests/manual/performance_broker_receiver/getlast_broker.cpp @@ -58,14 +58,14 @@ std::vector<std::thread> StartThreads(const Args& params, std::vector<int>* nbuf, std::vector<int>* nfiles_total) { auto exec_next = [¶ms, nfiles, errors, nbuf, nfiles_total](int i) { - asapo::FileInfo fi; + asapo::MessageMeta fi; Error err; auto consumer = asapo::ConsumerFactory::CreateConsumer(params.server, params.file_path, true, asapo::SourceCredentials{asapo::SourceType::kProcessed, params.beamtime_id, "", "", params.token}, &err); consumer->SetTimeout((uint64_t) params.timeout_ms); - asapo::FileData data; + asapo::MessageData data; lock.lock(); @@ -119,7 +119,7 @@ std::vector<std::thread> StartThreads(const Args& params, int ReadAllData(const Args& params, uint64_t* duration_ms, int* nerrors, int* nbuf, int* nfiles_total, asapo::NetworkConnectionType* connection_type) { - asapo::FileInfo fi; + asapo::MessageMeta fi; system_clock::time_point t1 = system_clock::now(); std::vector<int> nfiles(params.nthreads, 0); diff --git a/tests/manual/producer_cpp/producer.cpp b/tests/manual/producer_cpp/producer.cpp index 25c7b06e499c1f916258e1b05f739d95eaf6f4a0..795d25e21cbc2909b2b34b065257c78760f66e38 100644 --- a/tests/manual/producer_cpp/producer.cpp +++ b/tests/manual/producer_cpp/producer.cpp @@ -72,19 +72,19 @@ int main(int argc, char* argv[]) { + "_part" + format_string(part) // file part id (chunk id) + "_m" + format_string(submodule, std::string("%02d")); auto send_size = to_send.size() + 1; - auto buffer = asapo::FileData(new uint8_t[send_size]); + auto buffer = asapo::MessageData(new uint8_t[send_size]); memcpy(buffer.get(), to_send.c_str(), send_size); std::string stream = std::to_string(start_number); // std::cout<<"submodule:"<<submodule // <<"- stream:"<<stream // <<"- filename:"<<to_send<<std::endl; - asapo::EventHeader event_header{submodule, send_size, to_send,"", part,modules}; - // err = producer->SendData(event_header,stream, std::move(buffer), + asapo::MessageHeader message_header{submodule, send_size, to_send, "", part, modules}; + // err = producer->Send(message_header,stream, std::move(buffer), // asapo::kTransferMetaDataOnly, &ProcessAfterSend); - err = producer->SendData(event_header,stream, std::move(buffer), - asapo::kDefaultIngestMode, &ProcessAfterSend); + err = producer->Send(message_header, stream, std::move(buffer), + asapo::kDefaultIngestMode, &ProcessAfterSend); exit_if_error("Cannot send file", err); err = producer->WaitRequestsFinished(1000);