From 76eb84c9df26c1ab2ad385e8f791cd663f07234e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Tue, 1 Jun 2021 16:42:15 +0200 Subject: [PATCH 1/2] add input handler for synthetic rand files and dirs --- src/CMakeLists.txt | 1 + src/inputHandler.h | 4 + src/inputHandlerDummy.cpp | 191 ++++++++++++++++++++++++++++++++++++++ src/inputHandlerDummy.h | 59 ++++++++++++ 4 files changed, 255 insertions(+) create mode 100644 src/inputHandlerDummy.cpp create mode 100644 src/inputHandlerDummy.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index df7b099..73bb003 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,6 +58,7 @@ list(APPEND EWMSCP_SOURCES followInotifyWatchRequestProvider.cpp) list(APPEND EWMSCP_SOURCES posixFileCommon.cpp) list(APPEND EWMSCP_SOURCES inputHandlerPosixFile.cpp) list(APPEND EWMSCP_SOURCES outputHandlerPosixFile.cpp) +list(APPEND EWMSCP_SOURCES inputHandlerDummy.cpp) list(APPEND EWMSCP_SOURCES acl.cpp) if (LIBSSH_FOUND) list(APPEND EWMSCP_SOURCES libsshCommon.cpp) diff --git a/src/inputHandler.h b/src/inputHandler.h index e025baf..60203fe 100644 --- a/src/inputHandler.h +++ b/src/inputHandler.h @@ -168,6 +168,10 @@ namespace inputHandler { std::unique_ptr& aStat): name(aName), stat(std::move(aStat)) { }; + Entry(const std::string& aName, + std::unique_ptr& aStat): name(aName), + stat(std::move(aStat)) { + }; ~Entry() = default; const std::string& getName() const { return name; diff --git a/src/inputHandlerDummy.cpp b/src/inputHandlerDummy.cpp new file mode 100644 index 0000000..028173c --- /dev/null +++ b/src/inputHandlerDummy.cpp @@ -0,0 +1,191 @@ +#include "inputHandlerDummy.h" +#include "block.h" +#include "copyRequestTypes.h" +#include "inputHandlerPosixFile.h" +#include "ewmscp.h" +#include "timer.h" + +#include +#include +#include +#include +#include + + +namespace inputHandler { + decltype(dummy::factory) dummy::factory("dummy"); + defineStatic(dummy::nFiles, '\0',"dummyNFiles","number of files in the dummy dirs"); + defineStatic(dummy::nDirs, '\0',"dummyNDirs","number of subdirs in the dummy dirs"); + defineStatic(dummy::minSize, '\0',"dummyMinSize","minimum size of generated files",0); + defineStatic(dummy::maxSize, '\0',"dummyMaxSize","minimum size of generated files",16*1024*1024); + defineStatic(dummy::mTime, genericStat::clock_type::now()); + const block& dummy::randomBlock() { + static const block* data = nullptr; + if (data == nullptr) { + timerInst(createRandomBlock); + auto buffer = new block(16*1024*1024); + std::default_random_engine re; + for (auto p=static_cast(buffer->bufferAt(0)); + p < static_cast(buffer->bufferAt(buffer->max_size()/sizeof(*p))); + ++p) { + *p = re(); + } + buffer->bump_size(buffer->max_size()); + data = buffer; + } + return *data; + } + + dummy::dummy() { + } + + std::unique_ptr dummy::newReader(const std::string& aPath, + copyRequest::stateType &state, + const genericStat& inititalStat) { + return std::unique_ptr(new readerDummy(aPath, + state, + inititalStat)); + } + + + + + + dummy::readerDummy::readerDummy(const std::string& /*aPath*/, + copyRequest::stateType &/*state*/, + const genericStat&inititalStat): + ioHandle(), + reader(inititalStat) { + } + + dummy::readerDummy::~readerDummy() { + } + + + + bool dummy::readerDummy::parallelizable() const { + return true; + } + bool dummy::readerDummy::readBlock(block& b) { + b.clear(totalBytesRead); + auto bytesToRead = readInitialStat.size - totalBytesRead; + if (bytesToRead > b.max_size()) { + bytesToRead = b.max_size(); + } + if (randomBlock().size() < bytesToRead) { + throw std::runtime_error("random data block too small"); + } + memcpy(b.bufferAt(0), randomBlock().bufferAt(0), bytesToRead); + b.bump_size(bytesToRead); + totalBytesRead += bytesToRead; + return totalBytesRead >= readInitialStat.size; + } + + + void dummy::readerDummy::readBlockP(block& b, size_t bytesToRead, off_t offset) { + b.clear(offset); + if (randomBlock().size() < bytesToRead) { + throw std::runtime_error("random data block too small"); + } + memcpy(b.bufferAt(0), randomBlock().bufferAt(0), bytesToRead); + b.bump_size(bytesToRead); + } + + + void dummy::readerDummy::checkUnchangedness() { + } + + std::unique_ptr dummy::readerDummy::getStat() { + return std::unique_ptr(new genericStat(readInitialStat)); + } + + bool dummy::pathExists(const std::string&) { + return true; + } + std::unique_ptr dummy::getStat(const std::string& path, + bool followLink) { + if (path.find("/dummy/") == 0) { + auto basename = path.substr(path.find_last_of('/')+1); + auto stat = new genericStat; + if (basename.find("file_") == 0) { + stat->mode = S_IFREG; + stat->size = std::stoll(basename.substr(5)); + stat->mTime = mTime; + stat->aTime = mTime; + stat->blksize = randomBlock().max_size(); + } else { + stat->mode = S_IFDIR; + } + return std::unique_ptr(stat); + } else { + static posixFileCommon posixHandler; + return posixHandler.getStat(path, followLink); + } + } + + dummy::DummyDirectory::DummyDirectory(const std::string& aPath): Directory(aPath) { + if (path.find("/dummy/") == 0) { + auto basename = path.substr(path.find_last_of('/')+1); + if (basename.empty()) { // tld + level = 0; + } else if (basename.find("dir_") == 0) { + level = std::stoul(basename.substr(4)); + } else { + throw std::logic_error(aPath + " is not a directory"); + } + if (nFiles.size() <= level) { + nFilesLeft = 0; + } else { + nFilesLeft = nFiles.at(level); + } + if (nDirs.size() <= level) { + nDirsLeft = 0; + } else { + nDirsLeft = nDirs.at(level); + } + } else { + throw std::logic_error("dummy directories mist start with /dummy/, not " + aPath); + } + } + dummy::DummyDirectory::~DummyDirectory() noexcept(false) { + } + std::unique_ptr dummy::DummyDirectory::getNextEntry() { + if (nFilesLeft) { + --nFilesLeft; + std::string name("file_"); + auto stat = std::unique_ptr(new genericStat); + std::uniform_int_distribution distribution(minSize, maxSize); + static std::default_random_engine rnd; + stat->size = distribution(rnd); + stat->mode = S_IFREG; + stat->blksize = randomBlock().max_size(); + stat->mTime = mTime; + stat->aTime = mTime; + name += std::to_string(stat->size); + name += "_"; + name += std::to_string(nFilesLeft); + return std::unique_ptr(new Entry(name, stat)); + } else if (nDirsLeft) { + --nDirsLeft; + std::string name("dir_"); + auto stat = std::unique_ptr(new genericStat); + stat->mode = S_IFDIR; + stat->mTime = mTime; + stat->aTime = mTime; + name += std::to_string(level+1); + name += "_"; + name += std::to_string(nDirsLeft); + return std::unique_ptr(new Entry(name, stat)); + } + return nullptr; + } + std::unique_ptr dummy::getDirectory(const std::string& path) { + if (path.find("/dummy/") == 0) { + return std::unique_ptr(new DummyDirectory(path)); + } else { + static inputHandler::posixFile posixHandler; + return posixHandler.getDirectory(path); + } + } + +} //end namespace inputHandler diff --git a/src/inputHandlerDummy.h b/src/inputHandlerDummy.h new file mode 100644 index 0000000..dd42cf6 --- /dev/null +++ b/src/inputHandlerDummy.h @@ -0,0 +1,59 @@ +#ifndef __inputHandlerDummy_h__ +#define __inputHandlerDummy_h__ +#include "inputHandler.h" +#include +#include +#include +#include +#include +#include + + +namespace inputHandler { + + class dummy: public base, virtual public pathHandler { + static factoryTemplate factory; + static const block& randomBlock(); + static options::container nFiles; + static options::container nDirs; + static options::single> minSize; + static options::single> maxSize; + static genericStat::clock_type::time_point mTime; + public: + class readerDummy: public reader, virtual public ioHandle { + size_t size; + public: + readerDummy(const std::string& aPath, + copyRequest::stateType &state, + const genericStat& inititalStat); + ~readerDummy() override; + bool parallelizable() const override; + bool readBlock(block &b) override; + void readBlockP(block& b, size_t bytesToRead, off_t offset) override; + void checkUnchangedness() override; + std::unique_ptr getStat() override; + }; + dummy(); + ~dummy() override = default; + std::unique_ptr newReader(const std::string& aPath, + copyRequest::stateType &state, + const genericStat& inititalStat) override; + bool pathExists(const std::string&) override; + std::unique_ptr getStat(const std::string& path, + bool followLink) override; + + class DummyDirectory: public Directory { + unsigned int nFilesLeft; + unsigned int nDirsLeft; + unsigned int level; + public: + DummyDirectory(const std::string& path); + ~DummyDirectory() noexcept(false) override; + std::unique_ptr getNextEntry() override; + }; + std::unique_ptr getDirectory(const std::string& path) override; + + }; + +} // end namespace inputHandler +#endif -- GitLab From 78bb200120d93425adcfbecea2054e2fc9c0f02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hannappel?= Date: Tue, 1 Jun 2021 17:19:37 +0200 Subject: [PATCH 2/2] get rid of createLargeTestFileTree, ewmscp does that better --- src/CMakeLists.txt | 3 -- src/createLargeTestFileTree.cpp | 40 ------------------------- tests/auto.gpfs/mmwatchDelete.bash | 5 +++- tests/auto.inotify/inotifiedDelete.bash | 5 +++- tests/auto/recursiveCopy.helper | 8 +++-- 5 files changed, 13 insertions(+), 48 deletions(-) delete mode 100644 src/createLargeTestFileTree.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 73bb003..2c5ec2c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -176,9 +176,6 @@ add_dependencies(statvfs gitrev) target_link_libraries(statvfs pthread) install(TARGETS statvfs DESTINATION bin) -add_executable(createLargeTestFileTree createLargeTestFileTree.cpp) -install(TARGETS createLargeTestFileTree DESTINATION bin) - add_executable(dirCount dirCount.cpp ${OPTION_SRCS}) if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "armv7l") target_link_libraries(dirCount pthread atomic) diff --git a/src/createLargeTestFileTree.cpp b/src/createLargeTestFileTree.cpp deleted file mode 100644 index cde34d1..0000000 --- a/src/createLargeTestFileTree.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "throwcall.h" - -typedef std::list fplListType; - -void makestuff(int dirfd, fplListType::iterator from, fplListType::iterator to) { - auto next = from; - ++next; - if (next == to) { - for (auto i = *from; i; --i) { - auto fd = throwcall::badval(openat(dirfd, std::to_string(i).c_str(), O_CREAT | O_WRONLY, 0644), - -1, "open ", i); - throwcall::good0(close(fd), "close"); - } - } else { - for (auto i = *from; i; --i) { - throwcall::good0(mkdirat(dirfd, std::to_string(i).c_str(), 0755), "mkdirat", i); - auto fd = throwcall::badval(openat(dirfd, std::to_string(i).c_str(), O_RDONLY), - -1, "open ", i); - makestuff(fd, next, to); - throwcall::good0(close(fd), "close"); - } - } -} - -int main(int argc, const char *argv[]) { - fplListType filesPerLevel; - for (int i = 2; i < argc; i++) { - filesPerLevel.push_back(std::stoul(argv[i])); - } - throwcall::good0(mkdir(argv[1], 0755), "mkdir ", argv[1]); - auto dirfd = throwcall::badval(open(argv[1], O_RDONLY), -1, "open ", argv[1]); - makestuff(dirfd, filesPerLevel.begin(), filesPerLevel.end()); - throwcall::good0(close(dirfd), "close"); -} diff --git a/tests/auto.gpfs/mmwatchDelete.bash b/tests/auto.gpfs/mmwatchDelete.bash index 01f2aa9..4afeb25 100755 --- a/tests/auto.gpfs/mmwatchDelete.bash +++ b/tests/auto.gpfs/mmwatchDelete.bash @@ -13,7 +13,10 @@ ewmscp --nThreads 2 \ --pidFile ${pidfile} --pathMap ${src}:${dst} \ --printErrorLocation -v --allowCoreDumps --delete -- - 2> ${ewmscperr} > ${ewmscplog} & sleep 5 -createLargeTestFileTree ${cmpsrc}/bla 1 2 3 4 5 6 || exit 1 +ewmscp --inputHandler dummy --dummyMinSize 4k --dummyMaxSize 4k -R \ + --dummyNDirs 2 --dummyNDirs 3 --dummyNDirs 4 --dummyNDirs 5 \ + --dummyNFiles 0 --dummyNFiles 0 --dummyNFiles 0 --dummyNFiles 0 --dummyNFiles 6 \ + /dummy/ ${cmpsrc}/bla || { echo -e "${RED}Failed creating input files${NORM}"; exit 1; } sleep 5 rm -rf ${cmpsrc}/bla sleep 5 diff --git a/tests/auto.inotify/inotifiedDelete.bash b/tests/auto.inotify/inotifiedDelete.bash index 15e8423..3d28902 100755 --- a/tests/auto.inotify/inotifiedDelete.bash +++ b/tests/auto.inotify/inotifiedDelete.bash @@ -9,7 +9,10 @@ watchlog=${workdir}/inotify.log inotify_watch --pidFile ${ipidfile} ${src} --keepalive 3 2> ${watchlog} | \ ewmscp --nThreads 2 --requestProvider InotifyWatch --pidFile ${pidfile} --pathMap ${src}:${dst} --printErrorLocation -v --allowCoreDumps --delete -- - &> ${ewmscplog} & sleep 5 -createLargeTestFileTree ${cmpsrc}/bla 1 2 3 4 5 6 || exit 1 +ewmscp --inputHandler dummy --dummyMinSize 4k --dummyMaxSize 4k -R \ + --dummyNDirs 2 --dummyNDirs 3 --dummyNDirs 4 --dummyNDirs 5 \ + --dummyNFiles 0 --dummyNFiles 0 --dummyNFiles 0 --dummyNFiles 0 --dummyNFiles 6 \ + /dummy/ ${cmpsrc}/bla || { echo -e "${RED}Failed creating input files${NORM}"; exit 1; } sleep 5 rm -rf ${cmpsrc}/bla sleep 5 diff --git a/tests/auto/recursiveCopy.helper b/tests/auto/recursiveCopy.helper index 89c2880..d704a15 100644 --- a/tests/auto/recursiveCopy.helper +++ b/tests/auto/recursiveCopy.helper @@ -1,7 +1,9 @@ -createLargeTestFileTree ${cmpsrc}/ldt 2 3 4 5 -find ${cmpsrc}/ldt -type f -exec dd if=/dev/urandom of={} bs=4k count=1 \; 2> /dev/null +ewmscp --inputHandler dummy --dummyMinSize 4k --dummyMaxSize 4k -R \ + --dummyNDirs 2 --dummyNDirs 3 --dummyNDirs 4 \ + --dummyNFiles 0 --dummyNFiles 0 --dummyNFiles 0 --dummyNFiles 5 \ + /dummy/ ${cmpsrc}/ldt || { echo -e "${RED}Failed creating input files${NORM}"; exit 1; } if echo ${dst} ${src} | grep -q -v dcap:// ; then - # only fo non-dcap stuff, doesn't wor there anyway + # only for non-dcap stuff, doesn't work there anyway for f in $(find ${cmpsrc}/ldt -type f| head -1); do ln -s ${f} ${f}.link done -- GitLab