modified macros, started segfault test

CHANGELOG.md

+## 21.12.1 (in progress)
+
+IMPROVEMENTS
+* renamed and hid C++ macros from client code
+
+BUG FIXES
+* Producer API: fixed bug segfault in Python code when sending data object which memory is from some other object
+
+
 ## 21.12.0
 
 FEATURES

common/cpp/include/asapo/preprocessor/definitions.h

@@ -2,24 +2,13 @@
 #define ASAPO_DEFINITIONS_H
 
 #ifdef UNIT_TESTS
-#define VIRTUAL virtual
-#define FINAL
+#define ASAPO_VIRTUAL virtual
+#define ASAPO_FINAL
 #else
-#define VIRTUAL
-#define FINAL final
+#define ASAPO_VIRTUAL
+#define ASAPO_FINAL final
 #endif
 
-#if defined(__GNUC__) || defined(__clang__)
-#define DEPRECATED(msg) __attribute__((deprecated(msg)))
-#elif defined(_MSC_VER)
-#define DEPRECATED(msg) __declspec(deprecated(msg))
-#else
-#pragma message("WARNING: You need to implement DEPRECATED for this compiler")
-#define DEPRECATED(msg)
-#endif
-
-
-
 namespace  asapo {
 const char kPathSeparator =
 #ifdef WIN32

common/cpp/include/asapo/preprocessor/deprecated.h

+#ifndef ASAPO_DEPRECATED_H
+#define ASAPO_DEPRECATED_H
+
+#if defined(__GNUC__) || defined(__clang__)
+#define ASAPO_DEPRECATED(msg) __attribute__((deprecated(msg)))
+#elif defined(_MSC_VER)
+#define ASAPO_DEPRECATED(msg) __declspec(deprecated(msg))
+#else
+#pragma message("WARNING: You need to implement DEPRECATED for this compiler")
+#define ASAPO_DEPRECATED(msg)
+#endif
+
+#endif //ASAPO_DEPRECATED_H

common/cpp/include/asapo/request/request_pool.h

@@ -27,13 +27,13 @@ class RequestPool {
     };
   public:
     explicit RequestPool(uint8_t n_threads, RequestHandlerFactory* request_handler_factory, const AbstractLogger* log);
-    VIRTUAL Error AddRequest(GenericRequestPtr request, bool top_priority = false);
-    VIRTUAL void SetLimits(RequestPoolLimits limits);
-    VIRTUAL Error AddRequests(GenericRequests requests);
-    VIRTUAL ~RequestPool();
-    VIRTUAL uint64_t NRequestsInPool();
-    VIRTUAL uint64_t UsedMemoryInPool();
-    VIRTUAL Error WaitRequestsFinished(uint64_t timeout_ms);
+    ASAPO_VIRTUAL Error AddRequest(GenericRequestPtr request, bool top_priority = false);
+    ASAPO_VIRTUAL void SetLimits(RequestPoolLimits limits);
+    ASAPO_VIRTUAL Error AddRequests(GenericRequests requests);
+    ASAPO_VIRTUAL ~RequestPool();
+    ASAPO_VIRTUAL uint64_t NRequestsInPool();
+    ASAPO_VIRTUAL uint64_t UsedMemoryInPool();
+    ASAPO_VIRTUAL Error WaitRequestsFinished(uint64_t timeout_ms);
     void StopThreads();
   private:
     const AbstractLogger* log__;

consumer/api/cpp/src/tcp_connection_pool.h

@@ -16,10 +16,10 @@ struct TcpConnectionInfo {
 
 class TcpConnectionPool {
   public:
-    VIRTUAL SocketDescriptor GetFreeConnection(const std::string& source, bool* reused, Error* err);
-    VIRTUAL SocketDescriptor Reconnect(SocketDescriptor sd, Error* err);
-    VIRTUAL  void ReleaseConnection(SocketDescriptor sd);
-    VIRTUAL ~TcpConnectionPool() = default;
+    ASAPO_VIRTUAL SocketDescriptor GetFreeConnection(const std::string& source, bool* reused, Error* err);
+    ASAPO_VIRTUAL SocketDescriptor Reconnect(SocketDescriptor sd, Error* err);
+    ASAPO_VIRTUAL  void ReleaseConnection(SocketDescriptor sd);
+    ASAPO_VIRTUAL ~TcpConnectionPool() = default;
     TcpConnectionPool();
     std::unique_ptr<IO> io__;
   private:

producer/api/cpp/include/asapo/producer/producer.h

@@ -7,7 +7,7 @@
 #include "asapo/logger/logger.h"
 #include "common.h"
 #include "asapo/common/data_structs.h"
-#include "asapo/preprocessor/definitions.h"
+#include "asapo/preprocessor/deprecated.h"
 
 namespace asapo {
 
@@ -127,7 +127,7 @@ class Producer {
       \param callback - callback function
       \return Error - will be nullptr on success
     */
-    virtual Error DEPRECATED("obsolates 01.07.2022, use SendBeamtimeMetadata instead") SendMetadata(
+    virtual Error ASAPO_DEPRECATED("obsolates 01.07.2022, use SendBeamtimeMetadata instead") SendMetadata(
         const std::string& metadata,
         RequestCallback callback)  = 0;
 

producer/api/cpp/src/receiver_discovery_service.h

@@ -19,11 +19,11 @@ using ReceiversList = std::vector<std::string>;
 class ReceiverDiscoveryService {
   public:
     explicit ReceiverDiscoveryService(std::string endpoint, uint64_t update_frequency_ms);
-    VIRTUAL void StartCollectingData();
-    VIRTUAL ~ReceiverDiscoveryService();
-    VIRTUAL uint64_t MaxConnections();
-    VIRTUAL ReceiversList RotatedUriList(uint64_t nthread);
-    VIRTUAL uint64_t UpdateFrequency();
+    ASAPO_VIRTUAL void StartCollectingData();
+    ASAPO_VIRTUAL ~ReceiverDiscoveryService();
+    ASAPO_VIRTUAL uint64_t MaxConnections();
+    ASAPO_VIRTUAL ReceiversList RotatedUriList(uint64_t nthread);
+    ASAPO_VIRTUAL uint64_t UpdateFrequency();
   public:
     std::unique_ptr<HttpClient> httpclient__;
     const AbstractLogger* log__;

producer/event_monitor_producer/src/system_folder_watch_macos_dummy.h

@@ -12,11 +12,11 @@ namespace asapo {
 
 class SystemFolderWatch {
   public:
-    VIRTUAL ~SystemFolderWatch() = default;
-    VIRTUAL Error StartFolderMonitor(const std::string&, const std::vector<std::string>&) {
+    ASAPO_VIRTUAL ~SystemFolderWatch() = default;
+    ASAPO_VIRTUAL Error StartFolderMonitor(const std::string&, const std::vector<std::string>&) {
         return nullptr;
     };
-    VIRTUAL FilesToSend GetFileList(Error*) {
+    ASAPO_VIRTUAL FilesToSend GetFileList(Error*) {
         return {};
     };
 };

receiver/src/data_cache.h

@@ -21,10 +21,10 @@ struct CacheMeta {
 class DataCache {
   public:
     explicit DataCache(uint64_t cache_size_gb, float keepunlocked_ratio);
-    VIRTUAL void* GetFreeSlotAndLock(uint64_t size, CacheMeta** meta);
-    VIRTUAL void* GetSlotToReadAndLock(uint64_t id, uint64_t data_size, CacheMeta** meta);
-    VIRTUAL bool UnlockSlot(CacheMeta* meta);
-    VIRTUAL ~DataCache() = default;
+    ASAPO_VIRTUAL void* GetFreeSlotAndLock(uint64_t size, CacheMeta** meta);
+    ASAPO_VIRTUAL void* GetSlotToReadAndLock(uint64_t id, uint64_t data_size, CacheMeta** meta);
+    ASAPO_VIRTUAL bool UnlockSlot(CacheMeta* meta);
+    ASAPO_VIRTUAL ~DataCache() = default;
   private:
     uint64_t cache_size_;
     float keepunlocked_ratio_;

receiver/src/receiver_data_server/request_handler/receiver_data_server_request_handler_factory.h

@@ -14,7 +14,7 @@ namespace asapo {
 class ReceiverDataServerRequestHandlerFactory : public RequestHandlerFactory {
   public:
     ReceiverDataServerRequestHandlerFactory(RdsNetServer* server, DataCache* data_cache, Statistics* statistics);
-    VIRTUAL std::unique_ptr<RequestHandler> NewRequestHandler(uint64_t thread_id, uint64_t* shared_counter) override;
+    ASAPO_VIRTUAL std::unique_ptr<RequestHandler> NewRequestHandler(uint64_t thread_id, uint64_t* shared_counter) override;
   private:
     RdsNetServer* server_;
     DataCache* data_cache_;

receiver/src/request.h

@@ -26,56 +26,56 @@ class RequestHandlerDbCheckRequest;
 
 class Request {
   public:
-    VIRTUAL Error Handle(ReceiverStatistics*);
-    VIRTUAL ~Request() = default;
+    ASAPO_VIRTUAL Error Handle(ReceiverStatistics*);
+    ASAPO_VIRTUAL ~Request() = default;
     Request() = delete;
     Request(const GenericRequestHeader& request_header, SocketDescriptor socket_fd, std::string origin_uri,
             DataCache* cache, const RequestHandlerDbCheckRequest* db_check_handler);
-    VIRTUAL void AddHandler(const ReceiverRequestHandler*);
-    VIRTUAL const RequestHandlerList& GetListHandlers() const;
-    VIRTUAL uint64_t GetDataSize() const;
-    VIRTUAL uint64_t GetMetaDataSize() const;
-    VIRTUAL uint64_t GetDataID() const;
-    VIRTUAL std::string GetFileName() const;
-    VIRTUAL std::string GetStream() const;
-    VIRTUAL std::string GetApiVersion() const;
-    VIRTUAL void* GetData() const;
-    VIRTUAL Opcode GetOpCode() const;
-    VIRTUAL const char* GetMessage() const;
-
-    VIRTUAL const std::string& GetOriginUri() const;
-    VIRTUAL const std::string& GetOriginHost() const;
-    VIRTUAL const std::string& GetMetaData() const;
-    VIRTUAL const std::string& GetBeamtimeId() const;
-    VIRTUAL void SetBeamtimeId(std::string beamtime_id);
-    VIRTUAL void SetBeamline(std::string beamline);
-
-    VIRTUAL void SetSourceType(SourceType);
-    VIRTUAL SourceType GetSourceType() const;
-
-    VIRTUAL const std::string& GetDataSource() const;
-    VIRTUAL void SetDataSource(std::string data_source);
-    VIRTUAL void SetMetadata(std::string metadata);
-
-    VIRTUAL void SetOnlinePath(std::string facility);
-    VIRTUAL void SetOfflinePath(std::string path);
-    VIRTUAL const std::string& GetOnlinePath() const;
-    VIRTUAL const std::string& GetOfflinePath() const;
-
-    VIRTUAL const std::string& GetBeamline() const;
-    VIRTUAL const CustomRequestData& GetCustomData() const;
-    VIRTUAL Error PrepareDataBufferAndLockIfNeeded();
-    VIRTUAL void UnlockDataBufferIfNeeded();
-    VIRTUAL  SocketDescriptor GetSocket() const ;
+    ASAPO_VIRTUAL void AddHandler(const ReceiverRequestHandler*);
+    ASAPO_VIRTUAL const RequestHandlerList& GetListHandlers() const;
+    ASAPO_VIRTUAL uint64_t GetDataSize() const;
+    ASAPO_VIRTUAL uint64_t GetMetaDataSize() const;
+    ASAPO_VIRTUAL uint64_t GetDataID() const;
+    ASAPO_VIRTUAL std::string GetFileName() const;
+    ASAPO_VIRTUAL std::string GetStream() const;
+    ASAPO_VIRTUAL std::string GetApiVersion() const;
+    ASAPO_VIRTUAL void* GetData() const;
+    ASAPO_VIRTUAL Opcode GetOpCode() const;
+    ASAPO_VIRTUAL const char* GetMessage() const;
+
+    ASAPO_VIRTUAL const std::string& GetOriginUri() const;
+    ASAPO_VIRTUAL const std::string& GetOriginHost() const;
+    ASAPO_VIRTUAL const std::string& GetMetaData() const;
+    ASAPO_VIRTUAL const std::string& GetBeamtimeId() const;
+    ASAPO_VIRTUAL void SetBeamtimeId(std::string beamtime_id);
+    ASAPO_VIRTUAL void SetBeamline(std::string beamline);
+
+    ASAPO_VIRTUAL void SetSourceType(SourceType);
+    ASAPO_VIRTUAL SourceType GetSourceType() const;
+
+    ASAPO_VIRTUAL const std::string& GetDataSource() const;
+    ASAPO_VIRTUAL void SetDataSource(std::string data_source);
+    ASAPO_VIRTUAL void SetMetadata(std::string metadata);
+
+    ASAPO_VIRTUAL void SetOnlinePath(std::string facility);
+    ASAPO_VIRTUAL void SetOfflinePath(std::string path);
+    ASAPO_VIRTUAL const std::string& GetOnlinePath() const;
+    ASAPO_VIRTUAL const std::string& GetOfflinePath() const;
+
+    ASAPO_VIRTUAL const std::string& GetBeamline() const;
+    ASAPO_VIRTUAL const CustomRequestData& GetCustomData() const;
+    ASAPO_VIRTUAL Error PrepareDataBufferAndLockIfNeeded();
+    ASAPO_VIRTUAL void UnlockDataBufferIfNeeded();
+    ASAPO_VIRTUAL  SocketDescriptor GetSocket() const ;
     std::unique_ptr<IO> io__;
     DataCache* cache__ = nullptr;
-    VIRTUAL uint64_t GetSlotId() const;
-    VIRTUAL bool WasAlreadyProcessed() const;
-    VIRTUAL void SetAlreadyProcessedFlag();
-    VIRTUAL void SetResponseMessage(std::string message, ResponseMessageType type);
-    VIRTUAL ResponseMessageType GetResponseMessageType() const;
-    VIRTUAL const std::string& GetResponseMessage() const;
-    VIRTUAL Error CheckForDuplicates();
+    ASAPO_VIRTUAL uint64_t GetSlotId() const;
+    ASAPO_VIRTUAL bool WasAlreadyProcessed() const;
+    ASAPO_VIRTUAL void SetAlreadyProcessedFlag();
+    ASAPO_VIRTUAL void SetResponseMessage(std::string message, ResponseMessageType type);
+    ASAPO_VIRTUAL ResponseMessageType GetResponseMessageType() const;
+    ASAPO_VIRTUAL const std::string& GetResponseMessage() const;
+    ASAPO_VIRTUAL Error CheckForDuplicates();
   private:
     const GenericRequestHeader request_header_;
     const SocketDescriptor socket_fd_;

receiver/src/request_handler/authorization_client.h

@@ -14,10 +14,10 @@ class AbstractLogger;
 class AuthorizationClient {
   public:
     AuthorizationClient();
-    VIRTUAL Error Authorize(const Request* request, AuthorizationData* data) const;
+    ASAPO_VIRTUAL Error Authorize(const Request* request, AuthorizationData* data) const;
     const AbstractLogger* log__;
     std::unique_ptr<HttpClient> http_client__;
-    VIRTUAL ~AuthorizationClient() = default;
+    ASAPO_VIRTUAL ~AuthorizationClient() = default;
   private:
     Error DoServerRequest(const std::string& request_string, std::string* response, HttpCode* code) const;
 

receiver/src/request_handler/request_handler_db_check_request.h

@@ -9,7 +9,7 @@
 
 namespace asapo {
 
-class RequestHandlerDbCheckRequest FINAL : public RequestHandlerDb {
+class RequestHandlerDbCheckRequest ASAPO_FINAL : public RequestHandlerDb {
   public:
     RequestHandlerDbCheckRequest(std::string collection_name_prefix);
     Error ProcessRequest(Request* request) const override;

receiver/src/request_handler/requests_dispatcher.h

@@ -15,9 +15,9 @@ namespace asapo {
 class RequestsDispatcher {
   public:
     RequestsDispatcher(SocketDescriptor socket_fd, std::string address, ReceiverStatistics* statistics, SharedCache cache);
-    VIRTUAL Error ProcessRequest(const std::unique_ptr<Request>& request) const noexcept;
-    VIRTUAL std::unique_ptr<Request> GetNextRequest(Error* err) const noexcept;
-    VIRTUAL ~RequestsDispatcher() = default;
+    ASAPO_VIRTUAL Error ProcessRequest(const std::unique_ptr<Request>& request) const noexcept;
+    ASAPO_VIRTUAL std::unique_ptr<Request> GetNextRequest(Error* err) const noexcept;
+    ASAPO_VIRTUAL ~RequestsDispatcher() = default;
     ReceiverStatistics* statistics__;
     std::unique_ptr<IO> io__;
     const AbstractLogger* log__;

receiver/src/statistics/receiver_statistics.h

@@ -17,8 +17,8 @@ static const std::vector<std::string> kStatisticEntityNames = {"db_share", "disk
 class ReceiverStatistics : public Statistics {
   public:
     ReceiverStatistics(unsigned int write_interval = kDefaultStatisticWriteIntervalMs);
-    VIRTUAL void StartTimer(const StatisticEntity& entity) noexcept;
-    VIRTUAL void StopTimer() noexcept;
+    ASAPO_VIRTUAL void StartTimer(const StatisticEntity& entity) noexcept;
+    ASAPO_VIRTUAL void StopTimer() noexcept;
   private:
     StatisticsToSend PrepareStatisticsToSend() const noexcept override;
     void ResetStatistics() noexcept override;

receiver/src/statistics/statistics.h

@@ -27,10 +27,10 @@ struct StatisticsToSend {
 class Statistics {
   public:
     explicit Statistics(unsigned int write_interval = kDefaultStatisticWriteIntervalMs);
-    VIRTUAL void SendIfNeeded(bool send_always = false) noexcept;
-    VIRTUAL void IncreaseRequestCounter() noexcept;
-    VIRTUAL void IncreaseRequestDataVolume(uint64_t transferred_data_volume) noexcept;
-    VIRTUAL void AddTag(const std::string& name, const std::string& value) noexcept;
+    ASAPO_VIRTUAL void SendIfNeeded(bool send_always = false) noexcept;
+    ASAPO_VIRTUAL void IncreaseRequestCounter() noexcept;
+    ASAPO_VIRTUAL void IncreaseRequestDataVolume(uint64_t transferred_data_volume) noexcept;
+    ASAPO_VIRTUAL void AddTag(const std::string& name, const std::string& value) noexcept;
     void SetWriteInterval(uint64_t interval_ms);
     std::vector<std::unique_ptr<StatisticsSender>> statistics_sender_list__;
     virtual ~Statistics() = default;

tests/automatic/bug_fixes/CMakeLists.txt

@@ -5,6 +5,7 @@ if (UNIX)
         add_subdirectory(python_deadlock)
         add_subdirectory(streamlist_python_multithread)
         add_subdirectory(error-sending-data-using-callback-method)
+        add_subdirectory(python_segfault_dataview)
     endif()
 endif()
 

tests/automatic/bug_fixes/python_segfault_dataview/CMakeLists.txt

+set(TARGET_NAME python_deadlock_producer)
+
+
+find_package (Python3 REQUIRED)
+
+if (UNIX)
+    get_target_property(PYTHON_LIBS_PRODUCER python-lib-producer BINARY_DIR)
+else()
+    get_target_property(PYTHON_LIBS_PRODUCER asapo_producer BINARY_DIR)
+endif()
+
+file(TO_NATIVE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/producer_segfault.py TEST_SCRIPT )
+
+
+add_script_test("${TARGET_NAME}" "${TEST_SCRIPT} ${PYTHON_LIBS_PRODUCER} ${Python3_EXECUTABLE}" nomem)
+
+

tests/automatic/bug_fixes/python_segfault_dataview/check_linux.sh

+#!/usr/bin/env bash
+set -e
+
+export PYTHONPATH=$2:${PYTHONPATH}
+export Python3_EXECUTABLE=$3
+
+$Python3_EXECUTABLE $1 $endpoint $beamtime_id $token
+
+

tests/automatic/bug_fixes/python_segfault_dataview/jira_issue.txt

+https://agira.desy.de/browse/ASAPO-159
\ No newline at end of file