Commit ff064228 authored by Eric Cano's avatar Eric Cano
Browse files

Ported DataTransferSessionTest unit test DataTransferSessionGooddayMigration...

Ported DataTransferSessionTest unit test  DataTransferSessionGooddayMigration and DataTransferSessionEmptyOnVolReq.
parent be4a9fe2
......@@ -403,7 +403,17 @@ std::unique_ptr<cta::TapeMount> cta::Scheduler::getNextMount(const std::string &
[](decltype(*mountInfo->potentialMounts.cbegin())& m){ return m.type == cta::MountType::ARCHIVE; } )) {
tapeList = m_catalogue.getTapesForWriting(logicalLibraryName);
}
auto fullTapeList=m_catalogue.getTapes(cta::catalogue::TapeSearchCriteria());
for (auto & ftle: fullTapeList) {
ftle.capacityInBytes++;
}
for (auto & t:tapeList) {
auto const & tc=t;
uint64_t cap = tc.capacityInBytes+1;
cap++;
}
// We can now simply iterate on the candidates until we manage to create a
// mount for one of them
for (auto m = mountInfo->potentialMounts.begin(); m!=mountInfo->potentialMounts.end(); m++) {
......
......@@ -224,7 +224,7 @@ TEST_P(SchedulerTest, archive_to_new_file) {
diskFileInfo.owner="cms_user";
diskFileInfo.path="path/to/file";
cta::common::dataStructures::ArchiveRequest request;
request.checksumType="Adler32";
request.checksumType="ADLER32";
request.checksumValue="1111";
request.creationLog=creationLog;
request.diskpoolName="diskpool1";
......@@ -278,7 +278,7 @@ TEST_P(SchedulerTest, delete_archive_request) {
diskFileInfo.owner="cms_user";
diskFileInfo.path="path/to/file";
cta::common::dataStructures::ArchiveRequest request;
request.checksumType="Adler32";
request.checksumType="ADLER32";
request.checksumValue="1111";
request.creationLog=creationLog;
request.diskpoolName="diskpool1";
......@@ -347,7 +347,7 @@ TEST_P(SchedulerTest, archive_and_retrieve_new_file) {
diskFileInfo.owner="cms_user";
diskFileInfo.path="path/to/file";
cta::common::dataStructures::ArchiveRequest request;
request.checksumType="adler32";
request.checksumType="ADLER32";
request.checksumValue="1234abcd";
request.creationLog=creationLog;
request.diskpoolName="diskpool1";
......@@ -412,7 +412,7 @@ TEST_P(SchedulerTest, archive_and_retrieve_new_file) {
ASSERT_NE((cta::ArchiveJob*)NULL, archiveJob.get());
archiveJob->tapeFile.blockId = 1;
archiveJob->tapeFile.fSeq = 1;
archiveJob->tapeFile.checksumType = "adler32";
archiveJob->tapeFile.checksumType = "ADLER32";
archiveJob->tapeFile.checksumValue = "1234abcd";
archiveJob->complete();
archiveJob.reset(archiveMount->getNextJob().release());
......@@ -500,7 +500,7 @@ TEST_P(SchedulerTest, retry_archive_until_max_reached) {
diskFileInfo.owner="cms_user";
diskFileInfo.path="path/to/file";
cta::common::dataStructures::ArchiveRequest request;
request.checksumType="Adler32";
request.checksumType="ADLER32";
request.checksumValue="1111";
request.creationLog=creationLog;
request.diskpoolName="diskpool1";
......
......@@ -55,6 +55,7 @@
#include "scheduler/testingMocks/MockRetrieveMount.hpp"
#include "scheduler/testingMocks/MockArchiveJob.hpp"
#include "scheduler/testingMocks/MockArchiveMount.hpp"
#include "tests/TempFile.hpp"
#include <dirent.h>
#include <fcntl.h>
......@@ -895,168 +896,175 @@ TEST_P(DataTransferSessionTest, DataTransferSessionFailtoMount) {
ASSERT_NE(std::string::npos, logger.getLog().find("Failed to mount the tape"));
}
//TEST_P(DataTransferSessionTest, DataTransferSessionEmptyOnVolReq) {
//
// // 0) Prepare the logger for everyone
// castor::log::StringLogger logger("tapeServerUnitTest");
//
// // 1) prepare the fake scheduler
// std::string vid = "V12345";
// // cta::MountType::Enum mountType = cta::MountType::RETRIEVE;
//
// // 3) Prepare the necessary environment (logger, plus system wrapper),
// castor::tape::System::mockWrapper mockSys;
// mockSys.delegateToFake();
// mockSys.disableGMockCallsCounting();
// mockSys.fake.setupForVirtualDriveSLC6();
//
// // The drive will not even be opened. so no need for one.
// mockSys.fake.m_pathToDrive["/dev/nst0"] = NULL;
//
// // 4) Create the scheduler
// cta::catalogue::SqliteCatalogue sqliteCatalogue;
// cta::MockRemoteNS rns;
// cta::OStoreDBWrapper<cta::objectstore::BackendVFS> db("Unittest");
// cta::Scheduler scheduler(ns, db, rns);
//
// // Always use the same requester
// const cta::SecurityIdentity requester;
//
// DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
// DataTransferConfig castorConf;
// castorConf.bufsz = 1024*1024; // 1 MB memory buffers
// castorConf.nbBufs = 10;
// castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
// castorConf.bulkRequestRecallMaxFiles = 1000;
// castorConf.nbDiskThreads = 3;
// castor::messages::AcsProxyDummy acs;
// castor::mediachanger::MmcProxyDummy mmc;
// castor::legacymsg::RmcProxyDummy rmc;
// castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
// castor::server::ProcessCap capUtils;
// castor::messages::TapeserverProxyDummy initialProcess;
// DataTransferSession sess("tapeHost", logger, mockSys,
// driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
// ASSERT_NO_THROW(sess.execute());
// std::string temp = logger.getLog();
// temp += "";
// ASSERT_EQ("", sess.getVid());
// // We should not have logged any error
// ASSERT_EQ(std::string::npos, logger.getLog().find("LVL=E"));
//}
//
//TEST_P(DataTransferSessionTest, DataTransferSessionGooddayMigration) {
//
// // 0) Prepare the logger for everyone
// castor::log::StringLogger logger("tapeServerUnitTest");
//
// // 1) prepare the fake scheduler
// std::string vid = "V12345";
//
// // 3) Prepare the necessary environment (logger, plus system wrapper),
// castor::tape::System::mockWrapper mockSys;
// mockSys.delegateToFake();
// mockSys.disableGMockCallsCounting();
// mockSys.fake.setupForVirtualDriveSLC6();
//
// // 4) Create the scheduler
// cta::catalogue::SqliteCatalogue sqliteCatalogue;
// cta::MockRemoteFullFS rns;
// cta::OStoreDBWrapper<cta::objectstore::BackendVFS> db("Unittest");
// cta::Scheduler scheduler(ns, db, rns);
//
// // Always use the same requester
// const cta::SecurityIdentity requester;
//
// // Create the bootstrap admin user and host
// ASSERT_NO_THROW(scheduler.createAdminUserWithoutAuthorizingRequester(requester, requester.getUser(), "admin user"));
// ASSERT_NO_THROW(scheduler.createAdminHostWithoutAuthorizingRequester(requester, requester.getHost(), "admin host"));
//
// // create a single copy storage class
// ASSERT_NO_THROW(scheduler.createStorageClass(requester, "SINGLE", 1, 1, "comment"));
//
// // assign it to the root directory
// ASSERT_NO_THROW(scheduler.setDirStorageClass(requester, "/", "SINGLE"));
//
// // create the logical library
// ASSERT_NO_THROW(scheduler.createLogicalLibrary(requester, "T10KD6", "the illogical library"));
//
// // create the tape pool
// ASSERT_NO_THROW(scheduler.createTapePool(requester, "swimmingpool", 2, "the swimming pool"));
//
// cta::MountCriteria immediateMount;
// immediateMount.maxAge = 0;
// immediateMount.maxBytesQueued = 1;
// immediateMount.maxFilesQueued = 1;
// immediateMount.quota = 10;
// ASSERT_NO_THROW(scheduler.setTapePoolMountCriteria("swimmingpool", cta::MountCriteriaByDirection(immediateMount, immediateMount)));
//
// // create the route
// ASSERT_NO_THROW(scheduler.createArchiveRoute(requester, "SINGLE", 1, "swimmingpool", "iArchive"));
//
// // create the tape
// std::string comment = "the magic tape";
// ASSERT_NO_THROW(scheduler.createTape(requester, "V12345", "T10KD6", "swimmingpool", 100000, comment));
//
// // List to remember the path of each remote file so that the existence of the
// // files can be tested for at the end of the test
// std::list<std::string> remoteFilePaths;
//
// //delete is unnecessary
// //pointer with ownership will be passed to the application,
// //which will do the delete
// mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive();
//
// // We can prepare files for writing on the drive
// {
// // Label the tape
// castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], "V12345");
// mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
//
// // schedule the archivals
// for(int fseq=1; fseq <= 10 ; fseq ++) {
// // Create a path to a remote source file
// std::ostringstream fileName;
// fileName << "/test" << fseq;
// remoteFilePaths.push_back(fileName.str());
//
// // Create the file to be migrated in the fake remote NS
// cta::RemotePath rpath(rns.createFullURL(fileName.str()));
// rns.createFile(rpath.getRaw(), 1000);
//
// // Schedule the archival of the file
// std::list<std::string> remoteFilePathList;
// remoteFilePathList.push_back(rns.createFullURL(fileName.str()));
// ASSERT_NO_THROW(scheduler.queueArchiveRequest(requester, remoteFilePathList, fileName.str()));
// }
// }
// DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
// DataTransferConfig castorConf;
// castorConf.bufsz = 1024*1024; // 1 MB memory buffers
// castorConf.nbBufs = 10;
// castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
// castorConf.bulkRequestRecallMaxFiles = 1000;
// castorConf.nbDiskThreads = 1;
// castor::messages::AcsProxyDummy acs;
// castor::mediachanger::MmcProxyDummy mmc;
// castor::legacymsg::RmcProxyDummy rmc;
// castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
// castor::server::ProcessCap capUtils;
// castor::messages::TapeserverProxyDummy initialProcess;
// DataTransferSession sess("tapeHost", logger, mockSys, driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
// ASSERT_NO_THROW(sess.execute());
// std::string temp = logger.getLog();
// temp += "";
// ASSERT_EQ("V12345", sess.getVid());
// for(auto i=remoteFilePaths.begin(); i!=remoteFilePaths.end(); i++) {
// ASSERT_NO_THROW(ns.statFile(requester, *i));
// std::unique_ptr<cta::common::archiveNS::ArchiveFileStatus> stat(ns.statFile(requester, *i));
// ASSERT_NE(NULL, (uint64_t)(stat.get()));
// ASSERT_EQ(0100777, stat->mode);
// ASSERT_EQ(1000, stat->size);
// }
//}
//
TEST_P(DataTransferSessionTest, DataTransferSessionEmptyOnVolReq) {
// 0) Prepare the logger for everyone
castor::log::StringLogger logger("tapeServerUnitTest");
setupDefaultCatalogue();
// 1) prepare the fake scheduler
std::string vid = s_vid;
// cta::MountType::Enum mountType = cta::MountType::RETRIEVE;
// 3) Prepare the necessary environment (logger, plus system wrapper),
castor::tape::System::mockWrapper mockSys;
mockSys.delegateToFake();
mockSys.disableGMockCallsCounting();
mockSys.fake.setupForVirtualDriveSLC6();
//delete is unnecessary
//pointer with ownership will be passed to the application,
//which will do the delete
const bool failOnMount=true;
mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive(failOnMount);
// 4) Create the scheduler
auto & scheduler = getScheduler();
// Always use the same requester
const cta::common::dataStructures::SecurityIdentity requester;
DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
DataTransferConfig castorConf;
castorConf.bufsz = 1024*1024; // 1 MB memory buffers
castorConf.nbBufs = 10;
castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
castorConf.bulkRequestRecallMaxFiles = 1000;
castorConf.nbDiskThreads = 3;
castor::messages::AcsProxyDummy acs;
castor::mediachanger::MmcProxyDummy mmc;
castor::legacymsg::RmcProxyDummy rmc;
castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
castor::server::ProcessCap capUtils;
castor::messages::TapeserverProxyDummy initialProcess;
DataTransferSession sess("tapeHost", logger, mockSys,
driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
ASSERT_NO_THROW(sess.execute());
std::string temp = logger.getLog();
temp += "";
ASSERT_EQ("", sess.getVid());
// We should not have logged any error
ASSERT_EQ(std::string::npos, logger.getLog().find("LVL=E"));
}
TEST_P(DataTransferSessionTest, DataTransferSessionGooddayMigration) {
// 0) Prepare the logger for everyone
castor::log::StringLogger logger("tapeServerUnitTest");
setupDefaultCatalogue();
// 1) prepare the fake scheduler
std::string vid = s_vid;
// cta::MountType::Enum mountType = cta::MountType::RETRIEVE;
// 3) Prepare the necessary environment (logger, plus system wrapper),
castor::tape::System::mockWrapper mockSys;
mockSys.delegateToFake();
mockSys.disableGMockCallsCounting();
mockSys.fake.setupForVirtualDriveSLC6();
//delete is unnecessary
//pointer with ownership will be passed to the application,
//which will do the delete
const bool failOnMount=true;
mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive(failOnMount);
// 4) Create the scheduler
auto & catalogue = getCatalogue();
auto & scheduler = getScheduler();
// Always use the same requester
const cta::common::dataStructures::SecurityIdentity requester("user", "group");
// List to remember the path of each remote file so that the existance of the
// files can be tested for at the end of the test
std::list<std::string> remoteFilePaths;
// 5) Create the environment for the migration to happen (library + tape)
const std::string libraryComment = "Library comment";
catalogue.createLogicalLibrary(s_adminOnAdminHost, s_libraryName,
libraryComment);
{
auto libraries = catalogue.getLogicalLibraries();
ASSERT_EQ(1, libraries.size());
ASSERT_EQ(s_libraryName, libraries.front().name);
ASSERT_EQ(libraryComment, libraries.front().comment);
}
const uint64_t capacityInBytes = 12345678;
const std::string tapeComment = "Tape comment";
bool notDisabled = false;
bool notFull = false;
catalogue.createTape(s_adminOnAdminHost, s_vid, s_libraryName, s_tapePoolName, cta::nullopt, capacityInBytes,
notDisabled, notFull, tapeComment);
// Create the mount criteria
catalogue.createMountPolicy(requester, "immediateMount", 1000, 0, 1000, 0, 1, "Policy comment");
catalogue.createRequesterMountRule(requester, "immediateMount", s_diskInstance, requester.username, "Rule comment");
//delete is unnecessary
//pointer with ownership will be passed to the application,
//which will do the delete
mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive();
// We can prepare files for writing on the drive.
// Tempfiles are in this scope so they are kept alive
std::list<std::unique_ptr<unitTests::TempFile>> sourceFiles;
std::list<uint64_t> archiveFileIds;
{
// Label the tape
castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], s_vid, false);
catalogue.tapeLabelled(s_vid, "T10D6116", true);
mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
// Create the files and schedule the archivals
for(int fseq=1; fseq <= 10 ; fseq ++) {
// Create a source file.
sourceFiles.emplace_back(cta::make_unique<unitTests::TempFile>());
sourceFiles.back()->randomFill(1000);
remoteFilePaths.push_back(sourceFiles.back()->path());
// Schedule the archival of the file
cta::common::dataStructures::ArchiveRequest ar;
ar.checksumType="ADLER32";
ar.checksumValue=sourceFiles.back()->adler32();
ar.storageClass=s_storageClassName;
ar.srcURL=std::string("file://") + sourceFiles.back()->path();
ar.requester.name = requester.username;
ar.requester.group = "group";
ar.fileSize = 1000;
ar.diskFileID = "x";
ar.diskFileInfo.path = "y";
ar.diskFileInfo.owner = "z";
ar.diskFileInfo.group = "g";
ar.diskFileInfo.recoveryBlob = "b";
archiveFileIds.push_back(scheduler.queueArchive(s_diskInstance,ar));
}
}
DriveConfig driveConfig("T10D6116", "TestLogicalLibrary", "/dev/tape_T10D6116", "manual");
DataTransferConfig castorConf;
castorConf.bufsz = 1024*1024; // 1 MB memory buffers
castorConf.nbBufs = 10;
castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
castorConf.bulkRequestRecallMaxFiles = 1000;
castorConf.nbDiskThreads = 1;
castor::messages::AcsProxyDummy acs;
castor::mediachanger::MmcProxyDummy mmc;
castor::legacymsg::RmcProxyDummy rmc;
castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
castor::server::ProcessCap capUtils;
castor::messages::TapeserverProxyDummy initialProcess;
DataTransferSession sess("tapeHost", logger, mockSys, driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
sess.execute();
std::string temp = logger.getLog();
temp += "";
ASSERT_EQ(s_vid, sess.getVid());
auto afiiter = archiveFileIds.begin();
for(auto & sf: sourceFiles) {
auto afi = *(afiiter++);
auto afs = catalogue.getArchiveFileById(afi);
ASSERT_EQ(1, afs.tapeFiles.size());
ASSERT_EQ(sf->adler32(), afs.checksumValue);
ASSERT_EQ(1000, afs.fileSize);
}
}
////
//// This test is the same as the previous one, except that the files are deleted
//// from filesystem immediately. The disk tasks will then fail on open.
......
......@@ -120,7 +120,7 @@ bool DiskWriteTask::execute(RecallReportPacker& reporter,log::LogContext& lc,
} //end of while(1)
logWithStat(LOG_INFO, "File successfully transfered to disk",lc);
m_retrieveJob->transferredSize = m_stats.dataVolume;
m_retrieveJob->transferredChecksumType = "adler32";
m_retrieveJob->transferredChecksumType = "ADLER32";
{
std::stringstream cs;
cs << std::hex << std::nouppercase << std::setfill('0') << std::setw(8) << (uint32_t)checksum;
......
......@@ -159,10 +159,8 @@ void MigrationReportPacker::ReportFlush::execute(MigrationReportPacker& reportPa
reportPacker.m_successfulArchiveJobs.pop();
}
reportPacker.m_lc.log(LOG_INFO,"Reported to the client that a batch of files was written on tape");
}
catch(const castor::exception::Exception& e){
} catch(const cta::exception::Exception& e){
LogContext::ScopedParam sp[]={
LogContext::ScopedParam(reportPacker.m_lc, Param("exceptionCode",e.code())),
LogContext::ScopedParam(reportPacker.m_lc, Param("exceptionMessageValue", e.getMessageValue())),
LogContext::ScopedParam(reportPacker.m_lc, Param("exceptionWhat",e.what()))
};
......@@ -171,6 +169,15 @@ void MigrationReportPacker::ReportFlush::execute(MigrationReportPacker& reportPa
const std::string msg_error="An exception was caught trying to call reportMigrationResults";
reportPacker.m_lc.log(LOG_ERR,msg_error);
throw failedMigrationRecallResult(msg_error);
} catch(const std::exception& e){
LogContext::ScopedParam sp[]={
LogContext::ScopedParam(reportPacker.m_lc, Param("exceptionWhat",e.what()))
};
tape::utils::suppresUnusedVariable(sp);
const std::string msg_error="An std::exception was caught trying to call reportMigrationResults";
reportPacker.m_lc.log(LOG_ERR,msg_error);
throw failedMigrationRecallResult(msg_error);
}
} else {
// This is an abnormal situation: we should never flush after an error!
......
......@@ -202,8 +202,8 @@ private:
/** a shared flag among the all tasks related to migration, set as true
* as soon a single task encounters a failure. That way we go into a degraded mode
* where we only circulate memory without writing anything on tape
*/
castor::server::AtomicFlag m_errorFlag;
*/
castor::server::AtomicFlag m_errorFlag;
/// The maximum number of files we ask per request.
const uint64_t m_maxFiles;
......@@ -226,4 +226,3 @@ private:
} //end namespace tapeserver
} //end namespace tape
} //end namespace castor
......@@ -127,10 +127,10 @@ namespace daemon {
m_taskStats.filesCount ++;
// Record the fSeq in the tape session
session.reportWrittenFSeq(m_archiveJob->tapeFile.fSeq);
m_archiveJob->tapeFile.checksumType = "adler32";
m_archiveJob->tapeFile.checksumType = "ADLER32";
{
std::stringstream cs;
cs << std::hex << std::nouppercase << std::setfill('0') << std::setw(8) << (uint32_t)ckSum;
cs << std::hex << std::showbase << std::uppercase << std::setfill('0') << std::setw(8) << (uint32_t)ckSum;
m_archiveJob->tapeFile.checksumValue = cs.str();
}
m_archiveJob->tapeFile.compressedSize = m_taskStats.dataVolume;
......
......@@ -18,11 +18,13 @@
#include "TempFile.hpp"
#include "common/exception/Errnum.hpp"
#include "common/utils/utils.hpp"
#include <stdlib.h>
#include <unistd.h>
#include <fstream>
#include <memory>
#include <sys/stat.h>
namespace unitTests {
......@@ -55,6 +57,16 @@ void TempFile::randomFill(size_t size) {
out.write(buff.get(), size);
}
std::string TempFile::adler32() {
struct ::stat fileStat;
cta::exception::Errnum::throwOnMinusOne(::stat(m_path.c_str(), &fileStat),
"In TempFile::adler32(): failed to stat file.");
std::unique_ptr<char[] > buff(new char[fileStat.st_size]);
std::ifstream in(m_path, std::ios::in | std::ios::binary);
in.read(buff.get(), fileStat.st_size);
return cta::utils::getAdler32String((uint8_t*)buff.get(), fileStat.st_size);
}
void TempFile::stringFill(const std::string& string) {
std::ofstream out(m_path, std::ios::out | std::ios::binary | std::ios::trunc);
out << string;
......
......@@ -31,6 +31,7 @@ public:
TempFile(const std::string& path);
std::string path();
void randomFill(size_t size);
std::string adler32();
void stringFill(const std::string &string);
void stringAppend(const std::string &string);
~TempFile();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment