diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d480a263f5629e7cefeab7af822df5fc1c1085bc..4d01e74c1f794f51552e4dcb7df577e2c78d9350 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,15 @@ build_a: tags: - kubernetes +build_arm: + stage: build + script: gitlab/build.sh + artifacts: + paths: + - install/ + tags: + - arm + build_noxattr: stage: build script: gitlab/build.sh devtoolset-9 -DUSE_XATTR=OFF @@ -111,6 +120,16 @@ test_a: tags: - kubernetes +test_arm: + stage: test + script: gitlab/test.sh tests/auto /data/tmp /data/tmp + needs: + - build_arm + dependencies: + - build_arm + tags: + - arm + test_tmp: stage: test script: gitlab/test.sh tests/auto /tmp /tmp @@ -309,3 +328,13 @@ test_inotify_b: - build_9 tags: - bare-metal + +test_inotify_arm: + stage: test + script: gitlab/test.sh tests/auto.inotify /tmp /tmp + needs: + - build_arm + dependencies: + - build_arm + tags: + - arm diff --git a/CMakeLists.txt b/CMakeLists.txt index 502048f98bfa01e9fe9110f76d427ea3a1ff7eb5..35cf52f4b480e1a9507542ca6826a7b6a683e9e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,8 @@ IF(NOT CMAKE_BUILD_TYPE) ENDIF(NOT CMAKE_BUILD_TYPE) MESSAGE("Selected build type: ${CMAKE_BUILD_TYPE}") +MESSAGE("Building for ${CMAKE_SYSTEM_PROCESSOR} (${CMAKE_SYSTEM},${CMAKE_SYSTEM_NAME}) on ${CMAKE_HOST_SYSTEM_PROCESSOR}") + SET (CPLUSPLUSVERSION "11") CHECK_CXX_COMPILER_FLAG("-std=c++${CPLUSPLUSVERSION}" COMPILER_SUPPORTS_CXXVERSION) if(COMPILER_SUPPORTS_CXXVERSION) @@ -27,7 +29,7 @@ else() endif() SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_STD_FLAG}") -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wunused -Woverloaded-virtual -Wshadow -Wextra -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -Wvla -fstack-protector -fstack-check -Wstack-protector") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wunused -Woverloaded-virtual -Wshadow -Wextra -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -Wvla -fstack-protector -Wstack-protector") IF (${CMAKE_SIZEOF_VOID_P} EQUAL 4) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_FILE_OFFSET_BITS=64") diff --git a/gitlab/test.sh b/gitlab/test.sh index 95e79cb6969dfe0d622d4ffedb5ed6e1edfda78d..f2d4f9d7174621cbfab8509ae8c37364874159d8 100755 --- a/gitlab/test.sh +++ b/gitlab/test.sh @@ -76,14 +76,12 @@ for test in ${tests}; do export dst=$argdstbase/$(basename $cmpdst) result=$(mktemp) if [ -f /bin/time ]; then - timed="/bin/time -a -o ${result} -f" - format=' %E %I %O %M' + timed="/bin/time -a -o ${result} -f %E\t%I\t%O\t%M" elif [ -f /usr/bin/time ]; then - timed="/usr/bin/time -a -o ${result} -f" - format=' %E %M' + timed="/usr/bin/time -a -o ${result} -f %E\t%M" fi - if ${timed} "${format}" ${test} &> ${result}; then + if ${timed} ${test} &> ${result}; then echo -en "${GREEN}success${NORM}" tail -1 ${result} grep -e 'Warning:' -e 'CRIT:' -e 'DEBUG:' $result && warnings=$[ $warnings + $(grep 'Warning:' $result | wc -l) ] diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9dd758dacb42e821c4613aa1a1e807d4e918ce46..df7b09933488a5bdd5a3a2cdfb378e3074116f23 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,10 @@ SET(OPTION_CHRONO ${CMAKE_SOURCE_DIR}/OptionParser/src/OptionsChrono.cpp) SET(EWMSCP_INCLUDES) SET(EWMSCP_OPTIONS) SET(EWMSCP_LIBS "pthread") +if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "armv7l") + list(APPEND EWMSCP_LIBS "atomic") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-psabi") +endif() SET(EWMSCP_SOURCES ewmscp.cpp copyRequest.cpp block.cpp md5sum.cpp md5.c genericStat.cpp outputHandler.cpp) list(APPEND EWMSCP_SOURCES ${OPTION_SRCS} ${OPTION_CHRONO}) @@ -175,7 +179,11 @@ add_executable(createLargeTestFileTree createLargeTestFileTree.cpp) install(TARGETS createLargeTestFileTree DESTINATION bin) add_executable(dirCount dirCount.cpp ${OPTION_SRCS}) -target_link_libraries(dirCount pthread) +if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "armv7l") + target_link_libraries(dirCount pthread atomic) +else() + target_link_libraries(dirCount pthread) +endif() install(TARGETS dirCount DESTINATION bin) add_executable(dateTest dateTest.cpp ${OPTION_SRCS} ${OPTION_CHRONO}) diff --git a/src/copyRequest.cpp b/src/copyRequest.cpp index c6c727c5beb875f315d94e4dc344e807c89710a2..516743004ffe3f379af21609c846244aea15bb4f 100644 --- a/src/copyRequest.cpp +++ b/src/copyRequest.cpp @@ -539,8 +539,8 @@ void copyRequest::base::reader(inputHandler::base::reader& input, && nReadThreads > 1 && nBlocks >= nReadThreads) { blockReadRequestQueue readRequests; - for (off_t o = 0; o < readInitialStat->size; o += memoryBlockSize) { - size_t s = readInitialStat->size - o; + for (size_t o = 0; o < readInitialStat->size; o += memoryBlockSize) { + auto s = readInitialStat->size - o; s = std::min(s, memoryBlockSize); readRequests.emplace(s, o); } diff --git a/src/ewmscp.cpp b/src/ewmscp.cpp index de2b11b7c6676d6e81cd565d525f41ee8cc04592..473212aad4d026d163b09df698e032c06c3845f5 100644 --- a/src/ewmscp.cpp +++ b/src/ewmscp.cpp @@ -133,51 +133,48 @@ void sigUsrHandler(int sigNum) { } } -defineStatic(stopRequest::instance, nullptr); +defineStatic(stopRequest::handlerId, stopRequest::handlerIdType::unhandled); +defineStaticNoArg(stopRequest::reason); +defineStatic(stopRequest::stopRequested, false); void stopRequest::sigHandler(int) { - stopRequest::instance->stopRequested = true; - static_assert(ATOMIC_BOOL_LOCK_FREE == 2, "atomic bool is not always lock free"); // assert that stopRequested is always lock-free + stopRequested = true; + // static_assert(std::atomic_is_lock_free(&stopRequested), "atomic bool is not always lock free"); // assert that stopRequested is always lock-free } -stopRequest::stopRequest(): handlerId(handlerIdType::unhandled), stopRequested(false) {} void stopRequest::instantiate(stopRequest::handlerIdType id) { - if (instance != nullptr) { + if (handlerId != handlerIdType::unhandled) { throw std::logic_error("stop request handler cannot be instatiated twice"); } if (id == handlerIdType::unhandled || id == handlerIdType::undefined) { throw std::logic_error("illegal handler id value"); } - instance = new stopRequest; std::signal(SIGTERM, sigHandler); - instance->handlerId = id; + handlerId = id; } bool stopRequest::Requested() { - if (instance != nullptr) { - return instance->stopRequested; + if (handlerId != handlerIdType::unhandled) { + return stopRequested; } return false; } bool stopRequest::Requested(handlerIdType id) { - if (instance != nullptr) { - return instance->handlerId == id && instance->stopRequested; - } - return false; + return handlerId == id && stopRequested; } void stopRequest::RequestStop(const std::string& aReason) { - if (instance != nullptr) { + if (handlerId != handlerIdType::unhandled) { static std::mutex instanceProtector; std::unique_lock lock(instanceProtector); - stopRequest::instance->stopRequested = true; - stopRequest::instance->reason = aReason; + stopRequested = true; + reason = aReason; } else { errMsg::emit(errMsg::level::crit, errMsg::location(), "stopRequest", "requestStop", "no stop request instance"); } } void stopRequest::ThrowUpReasonably() { - if (instance != nullptr) { - if (instance->stopRequested) { - if (! instance->reason.empty()) { - throw std::runtime_error(instance->reason); + if (handlerId != handlerIdType::unhandled) { + if (stopRequested) { + if (! reason.empty()) { + throw std::runtime_error(reason); } else { throw std::runtime_error("SIGTERM received"); } diff --git a/src/ewmscp.h b/src/ewmscp.h index bd8416a0f856a8373b18ff11460f43ca6d717937..d53dd3a2dd2ab4f62b2aef0fb43786c2ba93fc8c 100644 --- a/src/ewmscp.h +++ b/src/ewmscp.h @@ -123,12 +123,10 @@ class stopRequest { processLoop }; private: - handlerIdType handlerId; - std::string reason; - static stopRequest* instance; + static handlerIdType handlerId; + static std::string reason; static void sigHandler(int); - std::atomic stopRequested; - stopRequest(); + static std::atomic stopRequested; public: static void instantiate(handlerIdType id); static bool Requested(); diff --git a/src/genericStat.cpp b/src/genericStat.cpp index b69806e0c0c8c5a1c2c3fb719440cf5561161894..7113ebc408db438749003fdc12c4148f7d2d259a 100644 --- a/src/genericStat.cpp +++ b/src/genericStat.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include genericStat::genericStat(const struct stat& statBuf, const clock_type::duration& aTimeResolution): device(statBuf.st_dev), @@ -49,8 +51,11 @@ void genericStat::chronoToString(std::string& spec, const clock_type::time_point char buffer[123]; strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S.", &time); spec = buffer; - std::sprintf(buffer, "%06ld", std::chrono::duration_cast(Chrono.time_since_epoch()).count() % 1000000); - spec += buffer; + std::ostringstream buf2; + buf2 << std::setw(6) << std::setfill('0') + << std::chrono::duration_cast(Chrono.time_since_epoch()).count() % 1000000; + // std::sprintf(buffer, "%06ld", std::chrono::duration_cast(Chrono.time_since_epoch()).count() % 1000000); + spec += buf2.str(); } diff --git a/src/genericStat.h b/src/genericStat.h index 74f1bbfc1c75c3f5e001c5daf1f50af744277bfe..c1bb16df6c22017fb3b1843edebb91e89b19e548 100644 --- a/src/genericStat.h +++ b/src/genericStat.h @@ -13,9 +13,9 @@ class genericStat { public: typedef std::chrono::system_clock clock_type; dev_t device; - off_t size; - off_t sizeOnDisk; - blksize_t blksize; + size_t size; + size_t sizeOnDisk; + size_t blksize; clock_type::time_point aTime; clock_type::time_point mTime; clock_type::duration timeResolution; diff --git a/src/inotify_watch.cpp b/src/inotify_watch.cpp index 6617781f0ccec5069465d945b8fe4d691bab1403..9d640e2ab2f2dac90cb7f5a6be77c837826b9164 100644 --- a/src/inotify_watch.cpp +++ b/src/inotify_watch.cpp @@ -59,7 +59,7 @@ #include typedef std::chrono::system_clock clock_type; -std::ostream* errStream = &std::cerr; +std::ostream* errStream(nullptr); options::single errPrefix('\0', "errPrefix", "prefix for error messages"); @@ -297,13 +297,13 @@ class watchedDirType { public: static void dump(std::ostream& out) { - out << "begin list of watched directories (" << getWatchedDirs().size() << "):\n"; + out << errPrefix << "begin list of watched directories (" << getWatchedDirs().size() << "):\n"; for (const auto& item: getWatchedDirs()) { std::string path; item.second->buildPath(path); - out << item.first << ": " << path << "\n"; + out << errPrefix << item.first << ": " << path << "\n"; } - out << "end list of watched directories.\n"; + out << errPrefix << "end list of watched directories.\n"; } @@ -645,7 +645,7 @@ int main(int argc, const char* argv[]) { } parser.fParse(argc, argv); - logstream::provider errStreamBase(errorStream, *errStream << errPrefix); + logstream::provider errStreamBase(errorStream, std::cerr); errStream = &(errStreamBase.getStream()); { diff --git a/src/inputHandler.h b/src/inputHandler.h index eb5c4cae4af537d39a56ba44839dbf1bd218a164..e025bafc5b07d6abe7baa9a366f99eaaa37b3b6c 100644 --- a/src/inputHandler.h +++ b/src/inputHandler.h @@ -109,7 +109,7 @@ namespace inputHandler { /// like the standard seek, to be used only when appending new data to a file /// \throw std::system_error if seek fails /// \throw unimplementedActionError if seek is not implemented (most input handlers) - virtual void seek(std::make_unsigned::type /*pos*/ ///< [in] position from the start of the file + virtual void seek(size_t /*pos*/ ///< [in] position from the start of the file ) { throw unimplementedActionError("seek not implemented"); }; diff --git a/src/inputHandlerLibssh.cpp b/src/inputHandlerLibssh.cpp index 2f6ef6b4ceec398f35f8d6537f9dc19e6e878a32..9b27264c36867c34b2b2e724a59cf4e58a71215a 100644 --- a/src/inputHandlerLibssh.cpp +++ b/src/inputHandlerLibssh.cpp @@ -89,7 +89,7 @@ namespace inputHandler { - void libssh::readerLibssh::seek(std::make_unsigned::type pos) { + void libssh::readerLibssh::seek(size_t pos) { throwcall::sftp::good0(sftp_seek64(file, pos), handler, "can't seek ", path, " to ", pos); } diff --git a/src/inputHandlerLibssh.h b/src/inputHandlerLibssh.h index b5e89bda360a445903be8f3663ed5c667836fc33..24fa90476e4d8e95acb6ebc3aaf4297ce8f1c40a 100644 --- a/src/inputHandlerLibssh.h +++ b/src/inputHandlerLibssh.h @@ -24,7 +24,7 @@ namespace inputHandler { copyRequest::stateType &state, const genericStat& inititalStat); ~readerLibssh() override; - void seek(std::make_unsigned::type pos) override; + void seek(size_t pos) override; bool readBlock(block &b) override; void checkUnchangedness() override; }; diff --git a/src/inputHandlerPosixFile.cpp b/src/inputHandlerPosixFile.cpp index 3863fc4a8ee2784f26737070c869956a81d92dce..e715d4ef657fdd728954cf0e9530ba1e76e4bf98 100644 --- a/src/inputHandlerPosixFile.cpp +++ b/src/inputHandlerPosixFile.cpp @@ -115,13 +115,15 @@ namespace inputHandler { pos < readInitialStat.size; what = what == SEEK_HOLE ? SEEK_DATA : SEEK_HOLE) { timerInst(lseek); - auto next = lseek(fd, pos, what); - if (next == -1 && errno == ENXIO) { // we found a hole at the end of the file + size_t next; + auto result = lseek(fd, pos, what); + if (result == -1 && errno == ENXIO) { // we found a hole at the end of the file next = readInitialStat.size; } else { - throwcall::badval(next, -1, + throwcall::badval(result, -1, "seek ", path, " for ", what == SEEK_HOLE ? "hole" : "data", ", from offset ", pos); + next = result; } if (next != pos) { regions.emplace_back(pos, next, what == SEEK_DATA); @@ -134,7 +136,7 @@ namespace inputHandler { } - void posixFile::readerPosixFile::seek(std::make_unsigned::type pos) { + void posixFile::readerPosixFile::seek(size_t pos) { throwcall::badval(lseek(fd, pos, SEEK_SET), -1, "can't seek ", path, " to ", pos); if (! regions.empty()) { @@ -160,11 +162,13 @@ namespace inputHandler { b.setHoleState(true); b.bump_size(region.size()); timerInst(lseek); - auto nextData = lseek(fd, totalBytesRead, SEEK_DATA); - if (nextData == -1 && errno == ENXIO) { // this is the final hole + size_t nextData; + auto result = lseek(fd, totalBytesRead, SEEK_DATA); + if (result == -1 && errno == ENXIO) { // this is the final hole nextData = region.getEnd(); } else { - throwcall::badval(nextData, -1, "seek in ", path, " to data after ", totalBytesRead); + throwcall::badval(result, -1, "seek in ", path, " to data after ", totalBytesRead); + nextData = result; } totalBytesRead += region.size(); // hole counts as total size... if (nextData != totalBytesRead) { diff --git a/src/inputHandlerPosixFile.h b/src/inputHandlerPosixFile.h index 16b3d74adec34dfceef5f8dc8e60f56c18e76f4a..98130cef89f13edcc2773b5e8ccb5d5f1d815269 100644 --- a/src/inputHandlerPosixFile.h +++ b/src/inputHandlerPosixFile.h @@ -18,8 +18,10 @@ namespace inputHandler { public: /// class for data or hole regions, needed for sparse file handling class regionType { - std::make_unsigned::type start; - std::make_unsigned::type end; + // std::make_unsigned::type start; + //std::make_unsigned::type end; + size_t start; + size_t end; bool hole; public: regionType(decltype(start) aStart, decltype(end) aEnd, decltype(hole) aHole): @@ -50,7 +52,7 @@ namespace inputHandler { ~readerPosixFile() override; bool parallelizable() const override; bool setupSparseRegions(const std::string& sparseHandling) override; - void seek(std::make_unsigned::type pos) override; + void seek(size_t pos) override; bool readBlock(block &b) override; void readBlockP(block& b, size_t bytesToRead, off_t offset) override; void checkUnchangedness() override; diff --git a/src/sparseMap.cpp b/src/sparseMap.cpp index 8ef48f45e60c3c7fd6f4c115dcde515d572fcfae..549c6950f0bea8f12bbd2c810ed8dfe716ab2cf6 100644 --- a/src/sparseMap.cpp +++ b/src/sparseMap.cpp @@ -96,7 +96,7 @@ int main(int argc, const char *argv[]) { printRegion(std::cout, pos, next); std::cout << "\n"; if (findZeroes && what.at(toggle).first == SEEK_HOLE) { - off_t zeroBlocksStart = 0; + size_t zeroBlocksStart = 0; bool lastBlockWasData = true; for (auto offset = pos; offset < next;) { auto bytesRead = pread(fd, blkBuf, blockSize, offset); diff --git a/tests/auto/simpleCopyBackupModeRemove.bash b/tests/auto/simpleCopyBackupModeRemove.bash index 2f3c03c237803d18bf8b20f932cf0c068e0f2b97..8f07b75b8db37d24a70cc774c8f98bcfc333a1fd 100755 --- a/tests/auto/simpleCopyBackupModeRemove.bash +++ b/tests/auto/simpleCopyBackupModeRemove.bash @@ -4,6 +4,7 @@ copy=${cmpdst}/a echo "oldfile" > ${cmpdst}/a copyOldInode=$(stat -c %i $copy) +ln ${copy} ${cmpdst}/inodesaver ewmscp ${stdargs} ${srcargs} ${dstargs} -v --backupMode remove ${src}/a ${dst} fail=$?