From 483b40f965a80116acc55f5e4b078723b818d7dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Wed, 17 Feb 2021 17:41:45 +0100 Subject: [PATCH 1/6] more debugging in test --- tests/auto.gpfs/mmwatchMove.bash | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/auto.gpfs/mmwatchMove.bash b/tests/auto.gpfs/mmwatchMove.bash index 5ac2588..49c764b 100755 --- a/tests/auto.gpfs/mmwatchMove.bash +++ b/tests/auto.gpfs/mmwatchMove.bash @@ -5,7 +5,8 @@ workdir=$(mktemp -d --suffix .$(basename $0)) pidfile=${workdir}/ewmscp.pid ewmscplog=${workdir}/ewmscp.log ewmscperr=${workdir}/ewmscp.err - +cd workdir +ulimit -c unlimited ewmscp --nThreads 2 \ --requestProvider KafkaStream \ --prohibitive-attrs user.truncated \ @@ -28,6 +29,9 @@ grep -v -e 'json decode no dir mapping found for' \ ls -l ${cmpsrc} ${cmpdst} test -e ${cmpdst}/file && fail=1 test -e ${cmpdst}/moved || fail=1 - -test -d ${workdir} && rm -rv ${workdir} +if [ $fail -eq 0 ]; then + test -d ${workdir} && rm -rv ${workdir} +else + ls -l $workdir +fi exit $fail -- GitLab From cbe80477d33c96db56646ce2d4af1b9e25d5d094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Wed, 17 Feb 2021 18:00:11 +0100 Subject: [PATCH 2/6] more debugging follow json --- src/followJsonRequestProvider.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/followJsonRequestProvider.cpp b/src/followJsonRequestProvider.cpp index e907cba..886d4a8 100644 --- a/src/followJsonRequestProvider.cpp +++ b/src/followJsonRequestProvider.cpp @@ -142,17 +142,23 @@ void followJsonRequestProvider::processJson(const Json::Value& json) { } +std::ostream& operator<< (std::ostream& stream, + const copyRequest::clock_type::time_point& t); + + void followJsonRequestProvider::cleanupRenameEvents(copyRequest::clock_type::duration minAge) { while (auto event = renameHandler.getNextStaleSingle(minAge)) { if (event->moveType == renameHalfEvent::moveTypeEnum::moveFrom) { errMsg::emit(errMsg::level::debug, errMsg::location(), - event->dstPath, "found stale moveFrom, cookie", event->cookie); + event->dstPath, "found stale moveFrom, cookie", event->cookie, + "from", event->timestamp); if (mayDelete) { handleOther(event->srcPath, event->dstPath, event->mapEntry, event->timestamp, true); } } else { errMsg::emit(errMsg::level::debug, errMsg::location(), - event->dstPath, "found stale moveTo, cookie", event->cookie); + event->dstPath, "found stale moveTo, cookie", event->cookie, + "from", event->timestamp); handleOther(event->srcPath, event->dstPath, event->mapEntry, event->timestamp, false); } renameHandler.forgetRenameEvent(event); -- GitLab From 5bd38aee2cb81bbe6af6b2390131476eaefc48b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Wed, 17 Feb 2021 18:07:38 +0100 Subject: [PATCH 3/6] more debugging follow json --- src/followJsonRequestProvider.cpp | 4 ++-- tests/auto.gpfs/mmwatchMove.bash | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/followJsonRequestProvider.cpp b/src/followJsonRequestProvider.cpp index 886d4a8..bdf2995 100644 --- a/src/followJsonRequestProvider.cpp +++ b/src/followJsonRequestProvider.cpp @@ -151,14 +151,14 @@ void followJsonRequestProvider::cleanupRenameEvents(copyRequest::clock_type::dur if (event->moveType == renameHalfEvent::moveTypeEnum::moveFrom) { errMsg::emit(errMsg::level::debug, errMsg::location(), event->dstPath, "found stale moveFrom, cookie", event->cookie, - "from", event->timestamp); + "from", std::fixed, event->timestamp); if (mayDelete) { handleOther(event->srcPath, event->dstPath, event->mapEntry, event->timestamp, true); } } else { errMsg::emit(errMsg::level::debug, errMsg::location(), event->dstPath, "found stale moveTo, cookie", event->cookie, - "from", event->timestamp); + "from", std::fixed, event->timestamp); handleOther(event->srcPath, event->dstPath, event->mapEntry, event->timestamp, false); } renameHandler.forgetRenameEvent(event); diff --git a/tests/auto.gpfs/mmwatchMove.bash b/tests/auto.gpfs/mmwatchMove.bash index 49c764b..897b92b 100755 --- a/tests/auto.gpfs/mmwatchMove.bash +++ b/tests/auto.gpfs/mmwatchMove.bash @@ -32,6 +32,8 @@ test -e ${cmpdst}/moved || fail=1 if [ $fail -eq 0 ]; then test -d ${workdir} && rm -rv ${workdir} else + date + date +$s ls -l $workdir fi exit $fail -- GitLab From addd307364a66d23cb10284bd4220d38100bf083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Thu, 18 Feb 2021 16:20:41 +0100 Subject: [PATCH 4/6] new test for timing features, fix tz for chrono opts --- src/CMakeLists.txt | 9 ++++++ src/dateTest.cpp | 59 ++++++++++++++++++++++++++++++++++++++++ tests/auto/timeInfo.bash | 21 ++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 src/dateTest.cpp create mode 100755 tests/auto/timeInfo.bash diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5acb80c..7c17d63 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,6 +2,13 @@ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Ofast -g") SET(CMAKE_C_FLAGS_RELWITHDEBINFO "-g") +# Check for timezone files +find_path(TZFILE_PATH "UTC" "/usr/share/zoneinfo" "/usr/lib/zoneinfo") +if (TZFILE_PATH) + message(STATUS "the time zone info files will be saerched in ${TZFILE_PATH}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DTZFILE_PATH='\"${TZFILE_PATH}/\"'") +endif() + # The 'real' git information file SET(GITREV_FILE ${CMAKE_BINARY_DIR}/src/git-rev.h) @@ -167,3 +174,5 @@ install(TARGETS createLargeTestFileTree DESTINATION bin) add_executable(dirCount dirCount.cpp ${OPTION_SRCS}) target_link_libraries(dirCount pthread) install(TARGETS dirCount DESTINATION bin) + +add_executable(dateTest dateTest.cpp ${OPTION_SRCS} ${OPTION_CHRONO}) diff --git a/src/dateTest.cpp b/src/dateTest.cpp new file mode 100644 index 0000000..72235c5 --- /dev/null +++ b/src/dateTest.cpp @@ -0,0 +1,59 @@ +#include +#include + +typedef std::chrono::system_clock clock_type; + +std::ostream& operator<< (std::ostream& stream, + const clock_type::time_point& t) { + stream << std::fixed; + stream << std::chrono::duration_cast>(t.time_since_epoch()).count(); + return stream; +} +std::ostream& operator<< (std::ostream& stream, + const clock_type::duration& dt) { + stream << std::fixed; + stream << std::chrono::duration_cast>(dt).count(); + return stream; +} + +int main(int argc, const char* argv[]) { + options::parser parser("", "", {}); + options::single t('t',"time", + "a time point"); + options::single d('d',"duration", + "a duration"); + options::single readFormat('r',"readFormat", + "timestamp read format", + "%Y-%m-%d_%H:%M:%S%z"); + options::single writeFormat('w',"writeFormat", + "timestamp write format", + "%Y-%m-%d_%H:%M:%S%z"); + options::single quiet('q',"quiet","be quiet, minimal out"); + auto strings = parser.fParse(argc, argv); + + if (t.fIsSet()) { + if (quiet) { + std::cout << std::chrono::duration_cast(t.time_since_epoch()).count() << "\n"; + } else { + std::cout << " t is " << t << "\n"; + } + } + if (d.fIsSet()) { + if (quiet) { + std::cout << std::chrono::duration_cast(d).count() << "\n"; + } else { + std::cout << "dt is " << d << "\n"; + } + } + + for (const auto& string : strings) { + struct tm tmStruct; + strptime(string.c_str(), readFormat.c_str(), &tmStruct); + auto ts = clock_type::time_point(std::chrono::duration_cast(std::chrono::duration(mktime(&tmStruct)))); + if (quiet) { + std::cout << std::chrono::duration_cast(ts.time_since_epoch()).count() << "\n"; + } else { + std::cout << "\"" << string << "\" as time: " << ts << "\n"; + } + } +} diff --git a/tests/auto/timeInfo.bash b/tests/auto/timeInfo.bash new file mode 100755 index 0000000..ac04ff6 --- /dev/null +++ b/tests/auto/timeInfo.bash @@ -0,0 +1,21 @@ +#!/bin/bash + +fail=0 + +ref=$(date +%s) +tst=$(dateTest -t "now" -q) +if [ "$tst" -ne "$ref" ]; then + echo "for now: dateTest ${tst}, date ${ref}, $[ ${ref} - ${tst} ] difference" + fail=1 +fi + +ref=$(date +%s -d "2021/02/17 18:16:07") +tst=$(dateTest -q 2021-02-17_18:16:07+0100) +if [ "$tst" -ne "$ref" ]; then + echo "for now: dateTest ${tst}, date ${ref}, $[ ${ref} - ${tst} ] difference" + fail=1 +fi + +exit $fail + + -- GitLab From c70941a3b8e4331d352ba44ea20de20465692747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Thu, 18 Feb 2021 18:48:28 +0100 Subject: [PATCH 5/6] fix time handling --- src/dateTest.cpp | 11 ++++++++++- src/followJsonRequestProvider.cpp | 10 +++++++++- tests/auto/timeInfo.bash | 13 +++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/dateTest.cpp b/src/dateTest.cpp index 72235c5..7d4749a 100644 --- a/src/dateTest.cpp +++ b/src/dateTest.cpp @@ -47,9 +47,18 @@ int main(int argc, const char* argv[]) { } for (const auto& string : strings) { - struct tm tmStruct; + struct tm tmStruct{}; + strptime(string.c_str(), readFormat.c_str(), &tmStruct); auto ts = clock_type::time_point(std::chrono::duration_cast(std::chrono::duration(mktime(&tmStruct)))); + if (readFormat.back() == 'z') { + auto tzStart=string.find_last_of("+-"); + if (tzStart != std::string::npos) { + auto offset = std::stoi(string.substr(tzStart)); + ts -= std::chrono::hours(offset/100); + ts -= std::chrono::minutes(offset%100); + } + } if (quiet) { std::cout << std::chrono::duration_cast(ts.time_since_epoch()).count() << "\n"; } else { diff --git a/src/followJsonRequestProvider.cpp b/src/followJsonRequestProvider.cpp index bdf2995..6516622 100644 --- a/src/followJsonRequestProvider.cpp +++ b/src/followJsonRequestProvider.cpp @@ -93,9 +93,17 @@ void followJsonRequestProvider::processJson(const Json::Value& json) { if (timestampFormat == "%s") { // use timestamp = copyRequest::clock_type::time_point(std::chrono::duration_cast(std::chrono::duration(jsonTimestamp->asDouble()))); } else { - struct tm tmStruct; + struct tm tmStruct{}; strptime(jsonTimestamp->asCString(), timestampFormat.c_str(), &tmStruct); timestamp = copyRequest::clock_type::time_point(std::chrono::duration_cast(std::chrono::duration(mktime(&tmStruct)))) + timestampOffset; + if (timestampFormat.back() == 'z') { + auto tzStart=jsonTimestamp->asString().find_last_of("+-"); + if (tzStart != std::string::npos) { + auto offset = std::stoi(jsonTimestamp->asString().substr(tzStart)); + timestamp -= std::chrono::hours(offset/100); + timestamp -= std::chrono::minutes(offset%100); + } + } } } else { timestamp = copyRequest::clock_type::now(); diff --git a/tests/auto/timeInfo.bash b/tests/auto/timeInfo.bash index ac04ff6..d98a743 100755 --- a/tests/auto/timeInfo.bash +++ b/tests/auto/timeInfo.bash @@ -10,12 +10,21 @@ if [ "$tst" -ne "$ref" ]; then fi ref=$(date +%s -d "2021/02/17 18:16:07") -tst=$(dateTest -q 2021-02-17_18:16:07+0100) +tst=$(TZ=UTC dateTest -q 2021-02-17_18:16:07+0100) if [ "$tst" -ne "$ref" ]; then - echo "for now: dateTest ${tst}, date ${ref}, $[ ${ref} - ${tst} ] difference" + echo "for 2021/02/17 18:16:0: dateTest ${tst}, date ${ref}, $[ ${ref} - ${tst} ] difference" fail=1 fi +ref=$(TZ=EST date +%s -d "2021/02/17 18:16:07") +tst=$(TZ=UTC dateTest -q $(date +"%Y-%m-%d_%H:%M:%S%z" -d @$ref)) +if [ "$tst" -ne "$ref" ]; then + echo "for 2021/02/17 18:16:07: dateTest ${tst}, date ${ref}, $[ ${ref} - ${tst} ] difference" + fail=1 +fi + + + exit $fail -- GitLab From 3af7f12f0b3747cca6cbf99bf5f6132fc54b70c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Thu, 18 Feb 2021 19:07:24 +0100 Subject: [PATCH 6/6] use UTC TZ for wf time decode --- tests/auto.gpfs/mmwatchMove.bash | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/auto.gpfs/mmwatchMove.bash b/tests/auto.gpfs/mmwatchMove.bash index 897b92b..fb4ca4f 100755 --- a/tests/auto.gpfs/mmwatchMove.bash +++ b/tests/auto.gpfs/mmwatchMove.bash @@ -7,14 +7,14 @@ ewmscplog=${workdir}/ewmscp.log ewmscperr=${workdir}/ewmscp.err cd workdir ulimit -c unlimited -ewmscp --nThreads 2 \ - --requestProvider KafkaStream \ - --prohibitive-attrs user.truncated \ - --preserve timestamps \ - --kafkaIn-pcfg metadata.broker.list:asap3-events-01,asap3-events-02 \ - --kafkaIn-topic test1-watch \ - --pidFile ${pidfile} --pathMap ${src}:${dst} \ - --printErrorLocation -v --allowCoreDumps -- - 2> ${ewmscperr} > ${ewmscplog} & +TZ=UTC ewmscp --nThreads 2 \ + --requestProvider KafkaStream \ + --prohibitive-attrs user.truncated \ + --preserve timestamps \ + --kafkaIn-pcfg metadata.broker.list:asap3-events-01,asap3-events-02 \ + --kafkaIn-topic test1-watch \ + --pidFile ${pidfile} --pathMap ${src}:${dst} \ + --printErrorLocation -v --allowCoreDumps -- - 2> ${ewmscperr} > ${ewmscplog} & sleep 5 dd if=/dev/urandom of=${cmpsrc}/file bs=1k count=10 # we need the long sleep here because the watchfolder tells about the file only after the move otherwise -- GitLab