diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5acb80c56e731d8633981fc2f645e97fff83bb75..7c17d6365e07ca4e0333f2337641822be42785db 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,6 +2,13 @@ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Ofast -g") SET(CMAKE_C_FLAGS_RELWITHDEBINFO "-g") +# Check for timezone files +find_path(TZFILE_PATH "UTC" "/usr/share/zoneinfo" "/usr/lib/zoneinfo") +if (TZFILE_PATH) + message(STATUS "the time zone info files will be saerched in ${TZFILE_PATH}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DTZFILE_PATH='\"${TZFILE_PATH}/\"'") +endif() + # The 'real' git information file SET(GITREV_FILE ${CMAKE_BINARY_DIR}/src/git-rev.h) @@ -167,3 +174,5 @@ install(TARGETS createLargeTestFileTree DESTINATION bin) add_executable(dirCount dirCount.cpp ${OPTION_SRCS}) target_link_libraries(dirCount pthread) install(TARGETS dirCount DESTINATION bin) + +add_executable(dateTest dateTest.cpp ${OPTION_SRCS} ${OPTION_CHRONO}) diff --git a/src/dateTest.cpp b/src/dateTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7d4749a2d5b3615802be28f43635de5a4d5b7ffd --- /dev/null +++ b/src/dateTest.cpp @@ -0,0 +1,68 @@ +#include +#include + +typedef std::chrono::system_clock clock_type; + +std::ostream& operator<< (std::ostream& stream, + const clock_type::time_point& t) { + stream << std::fixed; + stream << std::chrono::duration_cast>(t.time_since_epoch()).count(); + return stream; +} +std::ostream& operator<< (std::ostream& stream, + const clock_type::duration& dt) { + stream << std::fixed; + stream << std::chrono::duration_cast>(dt).count(); + return stream; +} + +int main(int argc, const char* argv[]) { + options::parser parser("", "", {}); + options::single t('t',"time", + "a time point"); + options::single d('d',"duration", + "a duration"); + options::single readFormat('r',"readFormat", + "timestamp read format", + "%Y-%m-%d_%H:%M:%S%z"); + options::single writeFormat('w',"writeFormat", + "timestamp write format", + "%Y-%m-%d_%H:%M:%S%z"); + options::single quiet('q',"quiet","be quiet, minimal out"); + auto strings = parser.fParse(argc, argv); + + if (t.fIsSet()) { + if (quiet) { + std::cout << std::chrono::duration_cast(t.time_since_epoch()).count() << "\n"; + } else { + std::cout << " t is " << t << "\n"; + } + } + if (d.fIsSet()) { + if (quiet) { + std::cout << std::chrono::duration_cast(d).count() << "\n"; + } else { + std::cout << "dt is " << d << "\n"; + } + } + + for (const auto& string : strings) { + struct tm tmStruct{}; + + strptime(string.c_str(), readFormat.c_str(), &tmStruct); + auto ts = clock_type::time_point(std::chrono::duration_cast(std::chrono::duration(mktime(&tmStruct)))); + if (readFormat.back() == 'z') { + auto tzStart=string.find_last_of("+-"); + if (tzStart != std::string::npos) { + auto offset = std::stoi(string.substr(tzStart)); + ts -= std::chrono::hours(offset/100); + ts -= std::chrono::minutes(offset%100); + } + } + if (quiet) { + std::cout << std::chrono::duration_cast(ts.time_since_epoch()).count() << "\n"; + } else { + std::cout << "\"" << string << "\" as time: " << ts << "\n"; + } + } +} diff --git a/src/followJsonRequestProvider.cpp b/src/followJsonRequestProvider.cpp index e907cba0693c4e25883c62a5ade567b4971d487e..65166221b0335c222fd2848c02eafa34e79cd071 100644 --- a/src/followJsonRequestProvider.cpp +++ b/src/followJsonRequestProvider.cpp @@ -93,9 +93,17 @@ void followJsonRequestProvider::processJson(const Json::Value& json) { if (timestampFormat == "%s") { // use timestamp = copyRequest::clock_type::time_point(std::chrono::duration_cast(std::chrono::duration(jsonTimestamp->asDouble()))); } else { - struct tm tmStruct; + struct tm tmStruct{}; strptime(jsonTimestamp->asCString(), timestampFormat.c_str(), &tmStruct); timestamp = copyRequest::clock_type::time_point(std::chrono::duration_cast(std::chrono::duration(mktime(&tmStruct)))) + timestampOffset; + if (timestampFormat.back() == 'z') { + auto tzStart=jsonTimestamp->asString().find_last_of("+-"); + if (tzStart != std::string::npos) { + auto offset = std::stoi(jsonTimestamp->asString().substr(tzStart)); + timestamp -= std::chrono::hours(offset/100); + timestamp -= std::chrono::minutes(offset%100); + } + } } } else { timestamp = copyRequest::clock_type::now(); @@ -142,17 +150,23 @@ void followJsonRequestProvider::processJson(const Json::Value& json) { } +std::ostream& operator<< (std::ostream& stream, + const copyRequest::clock_type::time_point& t); + + void followJsonRequestProvider::cleanupRenameEvents(copyRequest::clock_type::duration minAge) { while (auto event = renameHandler.getNextStaleSingle(minAge)) { if (event->moveType == renameHalfEvent::moveTypeEnum::moveFrom) { errMsg::emit(errMsg::level::debug, errMsg::location(), - event->dstPath, "found stale moveFrom, cookie", event->cookie); + event->dstPath, "found stale moveFrom, cookie", event->cookie, + "from", std::fixed, event->timestamp); if (mayDelete) { handleOther(event->srcPath, event->dstPath, event->mapEntry, event->timestamp, true); } } else { errMsg::emit(errMsg::level::debug, errMsg::location(), - event->dstPath, "found stale moveTo, cookie", event->cookie); + event->dstPath, "found stale moveTo, cookie", event->cookie, + "from", std::fixed, event->timestamp); handleOther(event->srcPath, event->dstPath, event->mapEntry, event->timestamp, false); } renameHandler.forgetRenameEvent(event); diff --git a/tests/auto.gpfs/mmwatchMove.bash b/tests/auto.gpfs/mmwatchMove.bash index 5ac25887bf953f1d34de165906ab852b52d3f4bc..fb4ca4ff6cc07250e3a88318e3b861452928c74f 100755 --- a/tests/auto.gpfs/mmwatchMove.bash +++ b/tests/auto.gpfs/mmwatchMove.bash @@ -5,15 +5,16 @@ workdir=$(mktemp -d --suffix .$(basename $0)) pidfile=${workdir}/ewmscp.pid ewmscplog=${workdir}/ewmscp.log ewmscperr=${workdir}/ewmscp.err - -ewmscp --nThreads 2 \ - --requestProvider KafkaStream \ - --prohibitive-attrs user.truncated \ - --preserve timestamps \ - --kafkaIn-pcfg metadata.broker.list:asap3-events-01,asap3-events-02 \ - --kafkaIn-topic test1-watch \ - --pidFile ${pidfile} --pathMap ${src}:${dst} \ - --printErrorLocation -v --allowCoreDumps -- - 2> ${ewmscperr} > ${ewmscplog} & +cd workdir +ulimit -c unlimited +TZ=UTC ewmscp --nThreads 2 \ + --requestProvider KafkaStream \ + --prohibitive-attrs user.truncated \ + --preserve timestamps \ + --kafkaIn-pcfg metadata.broker.list:asap3-events-01,asap3-events-02 \ + --kafkaIn-topic test1-watch \ + --pidFile ${pidfile} --pathMap ${src}:${dst} \ + --printErrorLocation -v --allowCoreDumps -- - 2> ${ewmscperr} > ${ewmscplog} & sleep 5 dd if=/dev/urandom of=${cmpsrc}/file bs=1k count=10 # we need the long sleep here because the watchfolder tells about the file only after the move otherwise @@ -28,6 +29,11 @@ grep -v -e 'json decode no dir mapping found for' \ ls -l ${cmpsrc} ${cmpdst} test -e ${cmpdst}/file && fail=1 test -e ${cmpdst}/moved || fail=1 - -test -d ${workdir} && rm -rv ${workdir} +if [ $fail -eq 0 ]; then + test -d ${workdir} && rm -rv ${workdir} +else + date + date +$s + ls -l $workdir +fi exit $fail diff --git a/tests/auto/timeInfo.bash b/tests/auto/timeInfo.bash new file mode 100755 index 0000000000000000000000000000000000000000..d98a7435b332dc9258a048edb33782beb8074e6c --- /dev/null +++ b/tests/auto/timeInfo.bash @@ -0,0 +1,30 @@ +#!/bin/bash + +fail=0 + +ref=$(date +%s) +tst=$(dateTest -t "now" -q) +if [ "$tst" -ne "$ref" ]; then + echo "for now: dateTest ${tst}, date ${ref}, $[ ${ref} - ${tst} ] difference" + fail=1 +fi + +ref=$(date +%s -d "2021/02/17 18:16:07") +tst=$(TZ=UTC dateTest -q 2021-02-17_18:16:07+0100) +if [ "$tst" -ne "$ref" ]; then + echo "for 2021/02/17 18:16:0: dateTest ${tst}, date ${ref}, $[ ${ref} - ${tst} ] difference" + fail=1 +fi + +ref=$(TZ=EST date +%s -d "2021/02/17 18:16:07") +tst=$(TZ=UTC dateTest -q $(date +"%Y-%m-%d_%H:%M:%S%z" -d @$ref)) +if [ "$tst" -ne "$ref" ]; then + echo "for 2021/02/17 18:16:07: dateTest ${tst}, date ${ref}, $[ ${ref} - ${tst} ] difference" + fail=1 +fi + + + +exit $fail + +