From 72b30982e1c0cf70f1f147c41534dd9ed380f5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Thu, 11 Feb 2021 14:07:13 +0100 Subject: [PATCH 1/5] fix err handliong and status fore cksums --- src/checksumTestRequestProvider.cpp | 8 +- src/ewmscp.cpp | 138 +++++++++++++++------------- tests/auto/cksum.bash | 26 ++++++ 3 files changed, 105 insertions(+), 67 deletions(-) create mode 100755 tests/auto/cksum.bash diff --git a/src/checksumTestRequestProvider.cpp b/src/checksumTestRequestProvider.cpp index 96fed9b..88a5c6f 100644 --- a/src/checksumTestRequestProvider.cpp +++ b/src/checksumTestRequestProvider.cpp @@ -52,11 +52,6 @@ namespace copyRequest { logStream << name << (sum->checkExpectation() ? ": OK\n" : ": FAILED\n"); } } - if (exitOnMismatch && !sum->checkExpectation()) { - std::string name; - escaper->escape(source, name); - throw std::runtime_error("bad checksum " + sum->getName() + " on " + name); - } } } if (! matchesInAllSums) { @@ -67,6 +62,9 @@ namespace copyRequest { if (numberOfMismatches != 0) { stream << numberOfMismatches << " computed checksum" << (numberOfMismatches > 1 ? "s" : "") << " did NOT match\n"; + if (exitOnMismatch) { + throw std::runtime_error("bad checksums found"); + } } } /// setup statPrinter to print the final line for compatibiity mith eg. md5sum diff --git a/src/ewmscp.cpp b/src/ewmscp.cpp index f46b7c9..de2b11b 100644 --- a/src/ewmscp.cpp +++ b/src/ewmscp.cpp @@ -236,83 +236,97 @@ void printResults(copyRequest::simpleQueue& results, std::ostream& hashStream, std::ostream& logStream, std::ostream& statStream) { - bool timedOut = true; - auto lastQueueSize = requests.size(); - auto lastQueueSizePrint = std::chrono::system_clock::now(); - - while (timedOut == true) { - while (auto request = results.dequeue(std::chrono::seconds(1), timedOut)) { - request->printResults(hashStream, logStream); + try { + bool timedOut = true; + auto lastQueueSize = requests.size(); + auto lastQueueSizePrint = std::chrono::system_clock::now(); + + while (timedOut == true) { + while (auto request = results.dequeue(std::chrono::seconds(1), timedOut)) { + request->printResults(hashStream, logStream); // delete request; - } - - if (timedOut) { - logStream.flush(); - } - - auto now = std::chrono::system_clock::now(); - - if (statPrintRequested) { - statStream << statPrefix << "statistics values at " << std::fixed + } + + if (timedOut) { + logStream.flush(); + } + + auto now = std::chrono::system_clock::now(); + + if (statPrintRequested) { + statStream << statPrefix << "statistics values at " << std::fixed << std::chrono::duration(now.time_since_epoch()).count() << "\n"; - statStream << statPrefix << "outstanding requests: " << requests.size() << "\n"; - copyRequest::base::getStatPrinter()(statStream); - statStream.flush(); - statPrintRequested = false; - } else { - auto queueSize = requests.size(); - - if (lastQueueSize != queueSize && - queueSize > 1 && - now - lastQueueSizePrint > std::chrono::seconds(1)) { statStream << statPrefix << "outstanding requests: " << requests.size() << "\n"; - lastQueueSize = queueSize; - lastQueueSizePrint = now; + copyRequest::base::getStatPrinter()(statStream); + statStream.flush(); + statPrintRequested = false; + } else { + auto queueSize = requests.size(); + + if (lastQueueSize != queueSize && + queueSize > 1 && + now - lastQueueSizePrint > std::chrono::seconds(1)) { + statStream << statPrefix << "outstanding requests: " << requests.size() << "\n"; + lastQueueSize = queueSize; + lastQueueSizePrint = now; + } + + } + if (statResetRequested) { + copyRequest::base::resetStats(); + statResetRequested = false; } - - } - if (statResetRequested) { - copyRequest::base::resetStats(); - statResetRequested = false; } + + copyRequest::base::getStatPrinter()(statStream); + } catch (const std::exception& e) { + errMsg::emit(errMsg::level::warning, errMsg::location(), + "printThread","caught exception ",e.what()); + stopRequest::RequestStop(e.what()); + } catch (...) { + stopRequest::RequestStop("unknown exception"); } - - copyRequest::base::getStatPrinter()(statStream); } static options::single printErrorLocation('\0', "printErrorLocation", "print error location (file,line,v fct) instead of prefix"); void printErrors(std::ostream& errStream, messageQueue::queue* errQueue) { - while (auto msg = errMsg::message::getQueue().dequeue()) { - timerInst(produceLogMsg); - if (printErrorLocation) { - errStream << msg->getLoc().getFile() << ":" << msg->getLoc().getLine() - << ": in " << msg->getLoc().getFunc() << "(): "; - } else { - errStream << msg->getLogLevel() << statPrefix << ": "; - } - errStream << msg->getObject() << " " << msg->getAction() << " " << msg->getMessage() << "\n"; - if (errQueue) { - #ifdef WithJsonCpp - Json::Value root; - if (!prefixJsonName.empty()) { - root[prefixJsonName] = statPrefix; + try { + while (auto msg = errMsg::message::getQueue().dequeue()) { + timerInst(produceLogMsg); + if (printErrorLocation) { + errStream << msg->getLoc().getFile() << ":" << msg->getLoc().getLine() + << ": in " << msg->getLoc().getFunc() << "(): "; + } else { + errStream << msg->getLogLevel() << statPrefix << ": "; } - for (auto& item : jsonExtraFields) { - root[item.first] = item.second; + errStream << msg->getObject() << " " << msg->getAction() << " " << msg->getMessage() << "\n"; + if (errQueue) { +#ifdef WithJsonCpp + Json::Value root; + if (!prefixJsonName.empty()) { + root[prefixJsonName] = statPrefix; + } + for (auto& item : jsonExtraFields) { + root[item.first] = item.second; + } + root["level"] = logstream::namedLevel::getNameByLevel(msg->getLogLevel()); + root["path"] = msg->getObject(); + root["operation"] = msg->getAction(); + root["message"] = msg->getMessage(); + root["time"] = std::chrono::duration_cast>(msg->getTimeStamp().time_since_epoch()).count(); + static Json::FastWriter jsonWriter; + errQueue->send(jsonWriter.write(root)); +#else + errQueue->send(msg->getMessage()); +#endif } - root["level"] = logstream::namedLevel::getNameByLevel(msg->getLogLevel()); - root["path"] = msg->getObject(); - root["operation"] = msg->getAction(); - root["message"] = msg->getMessage(); - root["time"] = std::chrono::duration_cast>(msg->getTimeStamp().time_since_epoch()).count(); - static Json::FastWriter jsonWriter; - errQueue->send(jsonWriter.write(root)); - #else - errQueue->send(msg->getMessage()); - #endif } + } catch (const std::exception& e) { + stopRequest::RequestStop(e.what()); + } catch (...) { + stopRequest::RequestStop("unknown exception"); } } diff --git a/tests/auto/cksum.bash b/tests/auto/cksum.bash new file mode 100755 index 0000000..77c1224 --- /dev/null +++ b/tests/auto/cksum.bash @@ -0,0 +1,26 @@ +#!/bin/bash +mkdir -p ${cmpsrc}/d +date > ${cmpsrc}/d/a +date > ${cmpsrc}/d/b +set -x +stdargs="--printErrorLocation" +list=${cmpsrc}/list +retval=0 +if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 -R ${src}/d --hashList ${list} ; then + if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 --check ${list} --status; then + echo -e "${GREEN}: no changed file detected${NORM}" + date >> ${cmpsrc}/d/a + if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 --check ${list} --status; then + echo -e "${RED}: changed file not detected${NORM}" + retval=1 + else + echo -e "${GREEN}: changed file detected${NORM}" + fi + else + echo -e "${RED}: changed file detected wrongly${NORM}" + retval=$? + fi +else + $retval=$? +fi +exit $retval -- GitLab From b78f2d643113265be758341c2bf586563419750a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Thu, 11 Feb 2021 14:33:18 +0100 Subject: [PATCH 2/5] fix cksum test on dcache --- tests/auto/cksum.bash | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/auto/cksum.bash b/tests/auto/cksum.bash index 77c1224..8f43c61 100755 --- a/tests/auto/cksum.bash +++ b/tests/auto/cksum.bash @@ -9,7 +9,11 @@ retval=0 if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 -R ${src}/d --hashList ${list} ; then if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 --check ${list} --status; then echo -e "${GREEN}: no changed file detected${NORM}" - date >> ${cmpsrc}/d/a + if [ -f "${cmpdst}/.(get)(.)(checksum)" ]; then + echo "mist" > ${cmpsrc}/d/a + else + date >> ${cmpsrc}/d/a + fi if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 --check ${list} --status; then echo -e "${RED}: changed file not detected${NORM}" retval=1 -- GitLab From 3b4feb90f16ad634893602d6d6f32f358e8cb6d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Thu, 11 Feb 2021 14:52:34 +0100 Subject: [PATCH 3/5] fix cksum test on dcache --- tests/auto/cksum.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/auto/cksum.bash b/tests/auto/cksum.bash index 8f43c61..52713a0 100755 --- a/tests/auto/cksum.bash +++ b/tests/auto/cksum.bash @@ -10,6 +10,7 @@ if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 -R ${s if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 --check ${list} --status; then echo -e "${GREEN}: no changed file detected${NORM}" if [ -f "${cmpdst}/.(get)(.)(checksum)" ]; then + rm ${cmpsrc}/d/a echo "mist" > ${cmpsrc}/d/a else date >> ${cmpsrc}/d/a -- GitLab From 12f8770b93c1a31e5ce1264e35c1fc7655d9f078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Thu, 11 Feb 2021 15:23:49 +0100 Subject: [PATCH 4/5] fix cksum test on dcache --- tests/auto/cksum.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/auto/cksum.bash b/tests/auto/cksum.bash index 52713a0..541c83a 100755 --- a/tests/auto/cksum.bash +++ b/tests/auto/cksum.bash @@ -9,7 +9,7 @@ retval=0 if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 -R ${src}/d --hashList ${list} ; then if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 --check ${list} --status; then echo -e "${GREEN}: no changed file detected${NORM}" - if [ -f "${cmpdst}/.(get)(.)(checksum)" ]; then + if [ -f "${cmpsrc/.(get)(.)(checksum)" ]; then rm ${cmpsrc}/d/a echo "mist" > ${cmpsrc}/d/a else -- GitLab From 6d326d74711921b6044d01e5c3f40e5ede8533d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Thu, 11 Feb 2021 15:36:36 +0100 Subject: [PATCH 5/5] fix cksum test on dcache --- tests/auto/cksum.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/auto/cksum.bash b/tests/auto/cksum.bash index 541c83a..6f3e203 100755 --- a/tests/auto/cksum.bash +++ b/tests/auto/cksum.bash @@ -9,7 +9,7 @@ retval=0 if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 -R ${src}/d --hashList ${list} ; then if ewmscp ${stdargs} ${srcargs} ${dstargs} -v --noCopy --checksum adler32 --check ${list} --status; then echo -e "${GREEN}: no changed file detected${NORM}" - if [ -f "${cmpsrc/.(get)(.)(checksum)" ]; then + if [ -f "${cmpsrc}/.(get)(.)(checksum)" ]; then rm ${cmpsrc}/d/a echo "mist" > ${cmpsrc}/d/a else -- GitLab