From 07e26c2f244a1c7f56789739407ab15be0845bd6 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon, 12 Oct 2020 12:26:15 +0200
Subject: [PATCH] add timestamp at inserting record to database

---
 common/cpp/include/common/data_structs.h      |  4 ++--
 common/cpp/src/data_structs/data_structs.cpp  | 20 ++++++++++------
 common/cpp/src/system_io/system_io.cpp        |  2 +-
 .../cpp/src/system_io/system_io_linux_mac.cpp |  2 +-
 .../data_structs/test_data_structs.cpp        | 23 ++++++++++---------
 .../api/cpp/unittests/test_server_broker.cpp  |  4 ++--
 .../consumer/getnext_broker/check_linux.sh    |  2 +-
 .../consumer/getnext_broker/check_windows.bat |  2 +-
 .../getnext_broker_python/check_linux.sh      |  2 +-
 .../getnext_broker_python/check_windows.bat   |  2 +-
 examples/pipeline/in_to_out/check_linux.sh    |  2 +-
 examples/pipeline/in_to_out/check_windows.bat |  2 +-
 .../pipeline/in_to_out_python/check_linux.sh  |  2 +-
 .../in_to_out_python/check_windows.bat        |  2 +-
 .../request_handler_db_write.cpp              |  5 ++++
 .../test_request_handler_db_writer.cpp        |  6 +++++
 .../consumer_python_memleak/check_linux.sh    |  2 +-
 .../consumer/consumer_api/check_linux.sh      |  8 +++----
 .../consumer/consumer_api/check_windows.bat   |  8 +++----
 .../consumer_api_python/check_linux.sh        | 10 ++++----
 .../consumer_api_python/check_windows.bat     | 10 ++++----
 .../next_multithread_broker/check_linux.sh    |  2 +-
 .../next_multithread_broker/check_windows.bat |  2 +-
 .../insert_retrieve_mongodb.cpp               |  2 +-
 .../insert_retrieve_dataset_mongodb.cpp       |  2 +-
 25 files changed, 73 insertions(+), 55 deletions(-)

diff --git a/common/cpp/include/common/data_structs.h b/common/cpp/include/common/data_structs.h
index 5c3408dfb..4b399cc5e 100644
--- a/common/cpp/include/common/data_structs.h
+++ b/common/cpp/include/common/data_structs.h
@@ -18,7 +18,7 @@ uint64_t  EpochNanosecsFromNow();
 class FileInfo {
   public:
     std::string name;
-    std::chrono::system_clock::time_point modify_date;
+    std::chrono::system_clock::time_point timestamp;
     uint64_t size{0};
     uint64_t id{0};
     std::string source;
@@ -38,7 +38,7 @@ struct StreamInfo {
 inline bool operator==(const FileInfo& lhs, const FileInfo& rhs) {
     return  (lhs.name == rhs.name &&
              lhs.id == rhs.id &&
-             lhs.modify_date == rhs.modify_date &&
+             lhs.timestamp == rhs.timestamp &&
              lhs.size == rhs.size);
 }
 
diff --git a/common/cpp/src/data_structs/data_structs.cpp b/common/cpp/src/data_structs/data_structs.cpp
index 214a313e2..8262d48a8 100644
--- a/common/cpp/src/data_structs/data_structs.cpp
+++ b/common/cpp/src/data_structs/data_structs.cpp
@@ -44,7 +44,7 @@ Error GetSourceTypeFromString(std::string stype,SourceType *type) {
 }
 
 std::string FileInfo::Json() const {
-    auto nanoseconds_from_epoch = std::chrono::time_point_cast<std::chrono::nanoseconds>(modify_date).
+    auto nanoseconds_from_epoch = std::chrono::time_point_cast<std::chrono::nanoseconds>(timestamp).
                                   time_since_epoch().count();
 
     std::string x = name;
@@ -64,7 +64,7 @@ std::string FileInfo::Json() const {
     std::string s = "{\"_id\":" + std::to_string(id) + ","
                     "\"size\":" + std::to_string(size) + ","
                     "\"name\":\"" + x + "\","
-                    "\"lastchange\":" + std::to_string(nanoseconds_from_epoch) + ","
+                    "\"timestamp\":" + std::to_string(nanoseconds_from_epoch) + ","
                     "\"source\":\"" + source + "\","
                     "\"buf_id\":" + std::to_string(buf_id_int) + ","
                     "\"meta\":" + (metadata.size() == 0 ? std::string("{}") : metadata)
@@ -120,7 +120,7 @@ bool FileInfo::SetFromJson(const std::string& json_string) {
             parser.GetString("source", &source) ||
             parser.GetUInt64("buf_id", &buf_id) ||
             parser.Embedded("meta").GetRawString(&metadata) ||
-            !TimeFromJson(parser, "lastchange", &modify_date)) {
+            !TimeFromJson(parser, "timestamp", &timestamp)) {
         *this = old;
         return false;
     }
@@ -145,7 +145,6 @@ std::string IsoDateFromEpochNanosecs(uint64_t time_from_epoch_nanosec) {
     std::stringstream ssTp;
     auto zz = time_from_epoch_nanosec % 1000000000;
 
-    std::string s;
     char buff[100];
 
     sprintf(buff, "%.4d-%.2d-%.2dT%.2d:%.2d:%.2d", timetm.tm_year + 1900, timetm.tm_mon + 1, timetm.tm_mday,
@@ -153,13 +152,20 @@ std::string IsoDateFromEpochNanosecs(uint64_t time_from_epoch_nanosec) {
     if (zz > 0) {
         sprintf(buff + 19, ".%.9ld", zz);
     }
-
-    return buff;
+    std::string res{buff};
+    return res+"Z";
 }
 
 uint64_t NanosecsEpochFromISODate(std::string date_time) {
     double frac = 0;
-    int pos = date_time.find_first_of('.');
+    size_t pos = date_time.find_last_of("Z");
+    if (pos != std::string::npos) {
+        date_time = date_time.substr(0,pos);
+    } else {
+        return 0;
+    }
+
+    pos = date_time.find_first_of('.');
     if (pos != std::string::npos) {
         std::string frac_str = date_time.substr(pos);
         if (sscanf(frac_str.c_str(), "%lf", &frac) != 1) {
diff --git a/common/cpp/src/system_io/system_io.cpp b/common/cpp/src/system_io/system_io.cpp
index 51ac70d75..8ec63c46b 100644
--- a/common/cpp/src/system_io/system_io.cpp
+++ b/common/cpp/src/system_io/system_io.cpp
@@ -41,7 +41,7 @@ const size_t SystemIO::kReadWriteBufSize = size_t(1024) * 1024 * 50; //50MiByte
 void SortFileList(std::vector<FileInfo>* file_list) {
     std::sort(file_list->begin(), file_list->end(),
     [](FileInfo const & a, FileInfo const & b) {
-        return a.modify_date < b.modify_date;
+        return a.timestamp < b.timestamp;
     });
 }
 
diff --git a/common/cpp/src/system_io/system_io_linux_mac.cpp b/common/cpp/src/system_io/system_io_linux_mac.cpp
index 43137367a..0277406c6 100644
--- a/common/cpp/src/system_io/system_io_linux_mac.cpp
+++ b/common/cpp/src/system_io/system_io_linux_mac.cpp
@@ -92,7 +92,7 @@ void SetModifyDate(const struct stat& t_stat, FileInfo* file_info) {
 #undef st_mtim
 #endif
 
-    file_info->modify_date = system_clock::time_point
+    file_info->timestamp = system_clock::time_point
     {std::chrono::duration_cast<system_clock::duration>(d)};
 }
 
diff --git a/common/cpp/unittests/data_structs/test_data_structs.cpp b/common/cpp/unittests/data_structs/test_data_structs.cpp
index ffd8d7f86..3318773e1 100644
--- a/common/cpp/unittests/data_structs/test_data_structs.cpp
+++ b/common/cpp/unittests/data_structs/test_data_structs.cpp
@@ -32,7 +32,7 @@ FileInfo PrepareFileInfo() {
     finfo.name = std::string("folder") + asapo::kPathSeparator + "test";
     finfo.source = "host:1234";
     finfo.buf_id = big_uint;
-    finfo.modify_date = std::chrono::time_point<std::chrono::system_clock>(std::chrono::milliseconds(1));
+    finfo.timestamp = std::chrono::time_point<std::chrono::system_clock>(std::chrono::milliseconds(1));
     finfo.metadata =  "{\"bla\":10}";
     return finfo;
 }
@@ -50,10 +50,10 @@ TEST(FileInFo, CorrectConvertToJson) {
     std::string json = finfo.Json();
     if (asapo::kPathSeparator == '/') {
         ASSERT_THAT(json, Eq(
-                        R"({"_id":1,"size":100,"name":"folder/test","lastchange":1000000,"source":"host:1234","buf_id":-1,"meta":{"bla":10}})"));
+                        R"({"_id":1,"size":100,"name":"folder/test","timestamp":1000000,"source":"host:1234","buf_id":-1,"meta":{"bla":10}})"));
     } else {
         ASSERT_THAT(json, Eq(
-                        R"({"_id":1,"size":100,"name":"folder\\test","lastchange":1000000,"source":"host:1234","buf_id":-1,"meta":{"bla":10}})"));
+                        R"({"_id":1,"size":100,"name":"folder\\test","timestamp":1000000,"source":"host:1234","buf_id":-1,"meta":{"bla":10}})"));
     }
 }
 
@@ -99,7 +99,7 @@ TEST(FileInFo, CorrectConvertFromJson) {
     ASSERT_THAT(result.id, Eq(finfo.id));
     ASSERT_THAT(result.name, Eq(finfo.name));
     ASSERT_THAT(result.size, Eq(finfo.size));
-    ASSERT_THAT(result.modify_date, Eq(finfo.modify_date));
+    ASSERT_THAT(result.timestamp, Eq(finfo.timestamp));
     ASSERT_THAT(result.buf_id, Eq(finfo.buf_id));
     ASSERT_THAT(result.source, Eq(finfo.source));
     ASSERT_THAT(result.metadata, Eq(finfo.metadata));
@@ -133,13 +133,14 @@ struct TestEpochFromISODate {
 };
 
 auto tests = std::vector<TestEpochFromISODate> {
-    TestEpochFromISODate{"1970-01-01T00:00:00.0", 1}, // 0 reserved for errors
-    TestEpochFromISODate{"1970-01-01", 1},
-    TestEpochFromISODate{"1970-01-01T00:00:00.000000002", 2},
-    TestEpochFromISODate{"2019-07-25T15:38:11.100010002", 1564069091100010002},
+    TestEpochFromISODate{"1970-01-01T00:00:00.0Z", 1}, // 0 reserved for errors
+    TestEpochFromISODate{"1970-01-01Z", 1},
+    TestEpochFromISODate{"1970-01-01T00:00:00.000000002Z", 2},
+    TestEpochFromISODate{"2019-07-25T15:38:11.100010002Z", 1564069091100010002},
 //errors
     TestEpochFromISODate{"1970-13-01T00:00:00.000000002", 0},
     TestEpochFromISODate{"1970-12-01T00:00:00.", 0},
+    TestEpochFromISODate{"1970-01-01T00:00:00.000000002", 0},
 };
 
 TEST(FileInFo, NanosecsEpochFromISODate) {
@@ -150,9 +151,9 @@ TEST(FileInFo, NanosecsEpochFromISODate) {
 }
 
 auto tests2 = std::vector<TestEpochFromISODate> {
-    TestEpochFromISODate{"1970-01-01T00:00:00", 0},
-    TestEpochFromISODate{"1970-01-01T00:00:00.000000002", 2},
-    TestEpochFromISODate{"2019-07-25T15:38:11.100010002", 1564069091100010002},
+    TestEpochFromISODate{"1970-01-01T00:00:00Z", 0},
+    TestEpochFromISODate{"1970-01-01T00:00:00.000000002Z", 2},
+    TestEpochFromISODate{"2019-07-25T15:38:11.100010002Z", 1564069091100010002},
 };
 
 TEST(FileInFo, ISODateFromNanosecsEpoch) {
diff --git a/consumer/api/cpp/unittests/test_server_broker.cpp b/consumer/api/cpp/unittests/test_server_broker.cpp
index 17c6a624a..1cbb4e70d 100644
--- a/consumer/api/cpp/unittests/test_server_broker.cpp
+++ b/consumer/api/cpp/unittests/test_server_broker.cpp
@@ -162,7 +162,7 @@ class ServerDataBrokerTests : public Test {
         fi.id = 1;
         fi.buf_id = buf_id;
         fi.name = expected_filename;
-        fi.modify_date = std::chrono::system_clock::now();
+        fi.timestamp = std::chrono::system_clock::now();
         return fi;
     }
 };
@@ -481,7 +481,7 @@ TEST_F(ServerDataBrokerTests, GetImageReturnsFileInfo) {
     ASSERT_THAT(info.name, Eq(to_send.name));
     ASSERT_THAT(info.size, Eq(to_send.size));
     ASSERT_THAT(info.id, Eq(to_send.id));
-    ASSERT_THAT(info.modify_date, Eq(to_send.modify_date));
+    ASSERT_THAT(info.timestamp, Eq(to_send.timestamp));
 }
 
 TEST_F(ServerDataBrokerTests, GetImageReturnsParseError) {
diff --git a/examples/consumer/getnext_broker/check_linux.sh b/examples/consumer/getnext_broker/check_linux.sh
index 01b701c9b..5e604fedd 100644
--- a/examples/consumer/getnext_broker/check_linux.sh
+++ b/examples/consumer/getnext_broker/check_linux.sh
@@ -25,7 +25,7 @@ nomad run broker.nmd
 
 for i in `seq 1 3`;
 do
-	echo 'db.data_default.insert({"_id":'$i',"size":100,"name":"'$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
+	echo 'db.data_default.insert({"_id":'$i',"size":100,"name":"'$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
 done
 
 sleep 1
diff --git a/examples/consumer/getnext_broker/check_windows.bat b/examples/consumer/getnext_broker/check_windows.bat
index 6945e1dba..8254057ac 100644
--- a/examples/consumer/getnext_broker/check_windows.bat
+++ b/examples/consumer/getnext_broker/check_windows.bat
@@ -9,7 +9,7 @@ set token_test_run=K38Mqc90iRv8fC7prcFHd994mF_wfUiJnWBfIjIzieo=
 
 call start_services.bat
 
-for /l %%x in (1, 1, 3) do echo db.data_default.insert({"_id":%%x,"size":100,"name":"%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 3) do echo db.data_default.insert({"_id":%%x,"size":100,"name":"%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
 
 
 "%1" 127.0.0.1:8400 %source_path% %beamtime_id% 1 %token_test_run% 12000 1 | findstr /c:"Processed 3 file" || goto :error
diff --git a/examples/consumer/getnext_broker_python/check_linux.sh b/examples/consumer/getnext_broker_python/check_linux.sh
index b1a508025..2ea403236 100644
--- a/examples/consumer/getnext_broker_python/check_linux.sh
+++ b/examples/consumer/getnext_broker_python/check_linux.sh
@@ -25,7 +25,7 @@ nomad run broker.nmd
 
 for i in `seq 1 3`;
 do
-	echo 'db.data_default.insert({"_id":'$i',"size":100,"name":"'$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
+	echo 'db.data_default.insert({"_id":'$i',"size":100,"name":"'$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
 done
 
 echo 'db.meta.insert({"_id":0,"meta_test":"test"})' | mongo ${database_name}
diff --git a/examples/consumer/getnext_broker_python/check_windows.bat b/examples/consumer/getnext_broker_python/check_windows.bat
index 7562b1734..ec9120da3 100644
--- a/examples/consumer/getnext_broker_python/check_windows.bat
+++ b/examples/consumer/getnext_broker_python/check_windows.bat
@@ -9,7 +9,7 @@ set group_id=bif31l2uiddd4r0q6b40
 
 call start_services.bat
 
-for /l %%x in (1, 1, 3) do echo db.data_default.insert({"_id":%%x,"size":100,"name":"%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 3) do echo db.data_default.insert({"_id":%%x,"size":100,"name":"%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
 
 
 echo db.meta.insert({"_id":0,"meta_test":"test"}) | %mongo_exe% %database_name%  || goto :error
diff --git a/examples/pipeline/in_to_out/check_linux.sh b/examples/pipeline/in_to_out/check_linux.sh
index 8214bba5b..61d6cea65 100644
--- a/examples/pipeline/in_to_out/check_linux.sh
+++ b/examples/pipeline/in_to_out/check_linux.sh
@@ -55,7 +55,7 @@ echo hello3 > processed/file3
 
 for i in `seq 1 3`;
 do
-	echo 'db.data_default.insert({"_id":'$i',"size":6,"name":"'processed/file$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${indatabase_name}
+	echo 'db.data_default.insert({"_id":'$i',"size":6,"name":"'processed/file$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${indatabase_name}
 done
 
 sleep 1
diff --git a/examples/pipeline/in_to_out/check_windows.bat b/examples/pipeline/in_to_out/check_windows.bat
index 9d45718a5..e72cf9456 100644
--- a/examples/pipeline/in_to_out/check_windows.bat
+++ b/examples/pipeline/in_to_out/check_windows.bat
@@ -20,7 +20,7 @@ SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe"
 
 call start_services.bat
 
-for /l %%x in (1, 1, 3) do echo db.data_default.insert({"_id":%%x,"size":6,"name":"processed\\file%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %indatabase_name%  || goto :error
+for /l %%x in (1, 1, 3) do echo db.data_default.insert({"_id":%%x,"size":6,"name":"processed\\file%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %indatabase_name%  || goto :error
 
 mkdir %receiver_folder%
 
diff --git a/examples/pipeline/in_to_out_python/check_linux.sh b/examples/pipeline/in_to_out_python/check_linux.sh
index eea966070..2c794a0a3 100644
--- a/examples/pipeline/in_to_out_python/check_linux.sh
+++ b/examples/pipeline/in_to_out_python/check_linux.sh
@@ -57,7 +57,7 @@ echo hello3 > processed/file3
 
 for i in `seq 1 3`;
 do
-	echo 'db.data_default.insert({"_id":'$i',"size":6,"name":"'processed/file$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${indatabase_name}
+	echo 'db.data_default.insert({"_id":'$i',"size":6,"name":"'processed/file$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${indatabase_name}
 done
 
 sleep 1
diff --git a/examples/pipeline/in_to_out_python/check_windows.bat b/examples/pipeline/in_to_out_python/check_windows.bat
index c22726a79..14c125bbc 100644
--- a/examples/pipeline/in_to_out_python/check_windows.bat
+++ b/examples/pipeline/in_to_out_python/check_windows.bat
@@ -22,7 +22,7 @@ SET nthreads=4
 
 call start_services.bat
 
-for /l %%x in (1, 1, 3) do echo db.data_default.insert({"_id":%%x,"size":6,"name":"processed\\file%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %indatabase_name%  || goto :error
+for /l %%x in (1, 1, 3) do echo db.data_default.insert({"_id":%%x,"size":6,"name":"processed\\file%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %indatabase_name%  || goto :error
 
 mkdir %receiver_folder%
 mkdir processed
diff --git a/receiver/src/request_handler/request_handler_db_write.cpp b/receiver/src/request_handler/request_handler_db_write.cpp
index b37232ed5..fdeefa68c 100644
--- a/receiver/src/request_handler/request_handler_db_write.cpp
+++ b/receiver/src/request_handler/request_handler_db_write.cpp
@@ -1,4 +1,8 @@
 #include "request_handler_db_write.h"
+
+#include <chrono>
+
+
 #include "../request.h"
 #include "../receiver_config.h"
 #include "../receiver_logger.h"
@@ -82,6 +86,7 @@ FileInfo RequestHandlerDbWrite::PrepareFileInfo(const Request* request) const {
     file_info.buf_id = request->GetSlotId();
     file_info.source = GetReceiverConfig()->dataserver.advertise_uri;
     file_info.metadata = request->GetMetaData();
+    file_info.timestamp = std::chrono::system_clock::now();
     return file_info;
 }
 
diff --git a/receiver/unittests/request_handler/test_request_handler_db_writer.cpp b/receiver/unittests/request_handler/test_request_handler_db_writer.cpp
index a2536c22c..66d7e9afc 100644
--- a/receiver/unittests/request_handler/test_request_handler_db_writer.cpp
+++ b/receiver/unittests/request_handler/test_request_handler_db_writer.cpp
@@ -1,7 +1,9 @@
+#include <chrono>
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
 #include <database/db_error.h>
 
+
 #include "unittests/MockIO.h"
 #include "unittests/MockDatabase.h"
 #include "unittests/MockLogger.h"
@@ -119,6 +121,10 @@ MATCHER_P(CompareFileInfo, file, "") {
     if (arg.id != file.id) return false;
     if (arg.metadata != file.metadata) return false;
 
+    if (arg.timestamp<std::chrono::system_clock::now()-std::chrono::seconds (5)) {
+        return false;
+    }
+
     return true;
 }
 
diff --git a/tests/automatic/bug_fixes/consumer_python_memleak/check_linux.sh b/tests/automatic/bug_fixes/consumer_python_memleak/check_linux.sh
index 766c57341..0da5161d7 100644
--- a/tests/automatic/bug_fixes/consumer_python_memleak/check_linux.sh
+++ b/tests/automatic/bug_fixes/consumer_python_memleak/check_linux.sh
@@ -28,7 +28,7 @@ nomad run broker.nmd
 
 sleep 1
 
-echo 'db.data_default.insert({"_id":'1',"size":'$size',"name":"'$fname'","lastchange":1,"source":"none","buf_id":0,"meta":{}})' | mongo ${beamtime_id}_stream
+echo 'db.data_default.insert({"_id":'1',"size":'$size',"name":"'$fname'","timestamp":1,"source":"none","buf_id":0,"meta":{}})' | mongo ${beamtime_id}_stream
 dd if=/dev/zero of=$fname bs=$size count=1
 
 export PYTHONPATH=$1:${PYTHONPATH}
diff --git a/tests/automatic/consumer/consumer_api/check_linux.sh b/tests/automatic/consumer/consumer_api/check_linux.sh
index be6442d3a..8d58e5cda 100644
--- a/tests/automatic/consumer/consumer_api/check_linux.sh
+++ b/tests/automatic/consumer/consumer_api/check_linux.sh
@@ -28,17 +28,17 @@ sleep 1
 
 for i in `seq 1 10`;
 do
-	echo 'db.data_default.insert({"_id":'$i',"size":6,"name":"'$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
+	echo 'db.data_default.insert({"_id":'$i',"size":6,"name":"'$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
 done
 
 for i in `seq 1 5`;
 do
-	echo 'db.data_stream1.insert({"_id":'$i',"size":6,"name":"'1$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
+	echo 'db.data_stream1.insert({"_id":'$i',"size":6,"name":"'1$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
 done
 
 for i in `seq 1 5`;
 do
-	echo 'db.data_stream2.insert({"_id":'$i',"size":6,"name":"'2$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
+	echo 'db.data_stream2.insert({"_id":'$i',"size":6,"name":"'2$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
 done
 
 
@@ -57,7 +57,7 @@ do
 	images=''
 	for j in `seq 1 3`;
 	do
-		images="$images,{"_id":$j,"size":6,"name":'${i}_${j}',"lastchange":1,"source":'none',"buf_id":0,"meta":{"test":10}}"
+		images="$images,{"_id":$j,"size":6,"name":'${i}_${j}',"timestamp":1,"source":'none',"buf_id":0,"meta":{"test":10}}"
 	done
 	images=${images#?}
 	echo 'db.data_default.insert({"_id":'$i',"size":3,"images":['$images']})' | mongo ${database_name}
diff --git a/tests/automatic/consumer/consumer_api/check_windows.bat b/tests/automatic/consumer/consumer_api/check_windows.bat
index 6619a5c96..227c53f19 100644
--- a/tests/automatic/consumer/consumer_api/check_windows.bat
+++ b/tests/automatic/consumer/consumer_api/check_windows.bat
@@ -8,11 +8,11 @@ set token_test_run=K38Mqc90iRv8fC7prcFHd994mF_wfUiJnWBfIjIzieo=
 
 call start_services.bat
 
-for /l %%x in (1, 1, 10) do echo db.data_default.insert({"_id":%%x,"size":6,"name":"%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 10) do echo db.data_default.insert({"_id":%%x,"size":6,"name":"%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
 
-for /l %%x in (1, 1, 5) do echo db.data_stream1.insert({"_id":%%x,"size":6,"name":"1%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 5) do echo db.data_stream1.insert({"_id":%%x,"size":6,"name":"1%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
 
-for /l %%x in (1, 1, 5) do echo db.data_stream2.insert({"_id":%%x,"size":6,"name":"2%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 5) do echo db.data_stream2.insert({"_id":%%x,"size":6,"name":"2%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
 echo hello1 > 1
 
 
@@ -20,7 +20,7 @@ echo hello1 > 1
 
 echo db.dropDatabase() | %mongo_exe% %database_name%
 
-for /l %%x in (1, 1, 10) do echo db.data_default.insert({"_id":%%x,"size":3,"images":[{"_id":1, "size":6,"name":"%%x_1","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}},{"_id":2, "size":6,"name":"%%x_2","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}},{"_id":3, "size":6,"name":"%%x_3","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}]}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 10) do echo db.data_default.insert({"_id":%%x,"size":3,"images":[{"_id":1, "size":6,"name":"%%x_1","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}},{"_id":2, "size":6,"name":"%%x_2","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}},{"_id":3, "size":6,"name":"%%x_3","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}]}) | %mongo_exe% %database_name%  || goto :error
 
 echo hello1 > 1_1
 
diff --git a/tests/automatic/consumer/consumer_api_python/check_linux.sh b/tests/automatic/consumer/consumer_api_python/check_linux.sh
index 262d2ae2f..f359bd4e2 100644
--- a/tests/automatic/consumer/consumer_api_python/check_linux.sh
+++ b/tests/automatic/consumer/consumer_api_python/check_linux.sh
@@ -35,19 +35,19 @@ echo -n hello1 > $source_path/1_1
 
 for i in `seq 1 5`;
 do
-	echo 'db.data_default.insert({"_id":'$i',"size":6,"name":"'$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
+	echo 'db.data_default.insert({"_id":'$i',"size":6,"name":"'$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
 done
 
-echo 'db.data_streamfts.insert({"_id":'1',"size":0,"name":"'1'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
+echo 'db.data_streamfts.insert({"_id":'1',"size":0,"name":"'1'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
 
 for i in `seq 1 5`;
 do
-	echo 'db.data_stream1.insert({"_id":'$i',"size":6,"name":"'1$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
+	echo 'db.data_stream1.insert({"_id":'$i',"size":6,"name":"'1$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
 done
 
 for i in `seq 1 5`;
 do
-	echo 'db.data_stream2.insert({"_id":'$i',"size":6,"name":"'2$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
+	echo 'db.data_stream2.insert({"_id":'$i',"size":6,"name":"'2$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
 done
 
 sleep 1
@@ -66,7 +66,7 @@ do
 	images=''
 	for j in `seq 1 3`;
 	do
-		images="$images,{"_id":$j,"size":6,"name":'${i}_${j}',"lastchange":1,"source":'none',"buf_id":0,"meta":{"test":10}}"
+		images="$images,{"_id":$j,"size":6,"name":'${i}_${j}',"timestamp":1,"source":'none',"buf_id":0,"meta":{"test":10}}"
 	done
 	images=${images#?}
 	echo 'db.data_default.insert({"_id":'$i',"size":3,"images":['$images']})' | mongo ${database_name} >/dev/null
diff --git a/tests/automatic/consumer/consumer_api_python/check_windows.bat b/tests/automatic/consumer/consumer_api_python/check_windows.bat
index ce5e35ce6..a611558d0 100644
--- a/tests/automatic/consumer/consumer_api_python/check_windows.bat
+++ b/tests/automatic/consumer/consumer_api_python/check_windows.bat
@@ -12,13 +12,13 @@ set token_test_run=K38Mqc90iRv8fC7prcFHd994mF_wfUiJnWBfIjIzieo=
 
 call start_services.bat
 
-for /l %%x in (1, 1, 5) do echo db.data_default.insert({"_id":%%x,"size":6,"name":"%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 5) do echo db.data_default.insert({"_id":%%x,"size":6,"name":"%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
 
-echo db.data_streamfts.insert({"_id":1,"size":0,"name":"1","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
+echo db.data_streamfts.insert({"_id":1,"size":0,"name":"1","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
 
-for /l %%x in (1, 1, 5) do echo db.data_stream1.insert({"_id":%%x,"size":6,"name":"1%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 5) do echo db.data_stream1.insert({"_id":%%x,"size":6,"name":"1%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
 
-for /l %%x in (1, 1, 5) do echo db.data_stream2.insert({"_id":%%x,"size":6,"name":"2%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 5) do echo db.data_stream2.insert({"_id":%%x,"size":6,"name":"2%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
 
 
 mkdir %source_path%
@@ -33,7 +33,7 @@ python %3/consumer_api.py 127.0.0.1:8400 %source_path% %beamtime_id%  %token_tes
 
 echo db.dropDatabase() | %mongo_exe% %database_name%
 
-for /l %%x in (1, 1, 10) do echo db.data_default.insert({"_id":%%x,"size":3,"images":[{"_id":1, "size":6,"name":"%%x_1","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}},{"_id":2, "size":6,"name":"%%x_2","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}},{"_id":3, "size":6,"name":"%%x_3","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}]}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 10) do echo db.data_default.insert({"_id":%%x,"size":3,"images":[{"_id":1, "size":6,"name":"%%x_1","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}},{"_id":2, "size":6,"name":"%%x_2","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}},{"_id":3, "size":6,"name":"%%x_3","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}]}) | %mongo_exe% %database_name%  || goto :error
 
 python %3/consumer_api.py 127.0.0.1:8400 %source_path% %beamtime_id%  %token_test_run% datasets || goto :error
 
diff --git a/tests/automatic/consumer/next_multithread_broker/check_linux.sh b/tests/automatic/consumer/next_multithread_broker/check_linux.sh
index b4a6c53c9..69e790337 100644
--- a/tests/automatic/consumer/next_multithread_broker/check_linux.sh
+++ b/tests/automatic/consumer/next_multithread_broker/check_linux.sh
@@ -25,7 +25,7 @@ sleep 1
 
 for i in `seq 1 10`;
 do
-	echo 'db.data_default.insert({"_id":'$i',"size":100,"name":"'$i'","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
+	echo 'db.data_default.insert({"_id":'$i',"size":100,"name":"'$i'","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}})' | mongo ${database_name}
 done
 
 $@ 127.0.0.1:8400 test_run 4 10 $token_test_run
diff --git a/tests/automatic/consumer/next_multithread_broker/check_windows.bat b/tests/automatic/consumer/next_multithread_broker/check_windows.bat
index 4c4bb2e68..1ff5ba7ba 100644
--- a/tests/automatic/consumer/next_multithread_broker/check_windows.bat
+++ b/tests/automatic/consumer/next_multithread_broker/check_windows.bat
@@ -4,7 +4,7 @@ set token_test_run=K38Mqc90iRv8fC7prcFHd994mF_wfUiJnWBfIjIzieo=
 
 call start_services.bat
 
-for /l %%x in (1, 1, 10) do echo db.data_default.insert({"_id":%%x,"size":100,"name":"%%x","lastchange":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
+for /l %%x in (1, 1, 10) do echo db.data_default.insert({"_id":%%x,"size":100,"name":"%%x","timestamp":1,"source":"none","buf_id":0,"meta":{"test":10}}) | %mongo_exe% %database_name%  || goto :error
 
 
 %1 127.0.0.1:8400 test_run 4 10 %token_test_run% || goto :error
diff --git a/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp b/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp
index e245e6c40..f36e4de35 100644
--- a/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp
+++ b/tests/automatic/mongo_db/insert_retrieve/insert_retrieve_mongodb.cpp
@@ -38,7 +38,7 @@ int main(int argc, char* argv[]) {
     fi.size = 100;
     fi.name = "relpath/1";
     fi.id = args.file_id;
-    fi.modify_date = std::chrono::system_clock::now();
+    fi.timestamp = std::chrono::system_clock::now();
     fi.buf_id = 18446744073709551615ull;
     fi.source = "host:1234";
 
diff --git a/tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp b/tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp
index 6cbaba641..d54e63487 100644
--- a/tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp
+++ b/tests/automatic/mongo_db/insert_retrieve_dataset/insert_retrieve_dataset_mongodb.cpp
@@ -38,7 +38,7 @@ int main(int argc, char* argv[]) {
     fi.size = 100;
     fi.name = "relpath/1";
     uint64_t subset_id = args.file_id;
-    fi.modify_date = std::chrono::system_clock::now();
+    fi.timestamp = std::chrono::system_clock::now();
     fi.buf_id = 18446744073709551615ull;
     fi.source = "host:1234";
     fi.id = 10;
-- 
GitLab