Commit 32573337 authored by Eric Cano's avatar Eric Cano
Browse files

Ported last data transfer session unit test and fixed report packer overlooking last reports.

parent a408c2ac
......@@ -1305,124 +1305,132 @@ TEST_P(DataTransferSessionTest, DataTransferSessionTapeFullMigration) {
}
}
//TEST_P(DataTransferSessionTest, DataTransferSessionTapeFullOnFlushMigration) {
//
//
// // 0) Prepare the logger for everyone
// castor::log::StringLogger logger("tapeServerUnitTest");
//
// // 1) prepare the fake scheduler
// std::string vid = "V12345";
//
// // 3) Prepare the necessary environment (logger, plus system wrapper),
// castor::tape::System::mockWrapper mockSys;
// mockSys.delegateToFake();
// mockSys.disableGMockCallsCounting();
// mockSys.fake.setupForVirtualDriveSLC6();
//
// // 4) Create the scheduler
// cta::catalogue::SqliteCatalogue sqliteCatalogue;
// cta::MockRemoteNS rns;
// cta::OStoreDBWrapper<cta::objectstore::BackendVFS> db("Unittest");
// cta::Scheduler scheduler(ns, db, rns);
//
// // Always use the same requester
// const cta::SecurityIdentity requester;
//
// // Create the bootstrap admin user and host
// ASSERT_NO_THROW(scheduler.createAdminUserWithoutAuthorizingRequester(requester, requester.getUser(), "admin user"));
// ASSERT_NO_THROW(scheduler.createAdminHostWithoutAuthorizingRequester(requester, requester.getHost(), "admin host"));
//
// // create a single copy storage class
// ASSERT_NO_THROW(scheduler.createStorageClass(requester, "SINGLE", 1, 1, "comment"));
//
// // assign it to the root directory
// ASSERT_NO_THROW(scheduler.setDirStorageClass(requester, "/", "SINGLE"));
//
// // create the logical library
// ASSERT_NO_THROW(scheduler.createLogicalLibrary(requester, "T10KD6", "the illogical library"));
//
// // create the tape pool
// ASSERT_NO_THROW(scheduler.createTapePool(requester, "swimmingpool", 2, "the swimming pool"));
//
// // Make the pool mount immediately
// cta::MountCriteria immediateMount;
// immediateMount.maxAge = 0;
// immediateMount.maxBytesQueued = 1;
// immediateMount.maxFilesQueued = 1;
// immediateMount.quota = 10;
// ASSERT_NO_THROW(scheduler.setTapePoolMountCriteria("swimmingpool", cta::MountCriteriaByDirection(immediateMount, immediateMount)));
//
// // create the route
// ASSERT_NO_THROW(scheduler.createArchiveRoute(requester, "SINGLE", 1, "swimmingpool", "iArchive"));
//
// // create the tape
// std::string comment = "the magic tape";
// ASSERT_NO_THROW(scheduler.createTape(requester, "V12345", "T10KD6", "swimmingpool", 100000, comment));
//
// // List to remember the path of each remote file so that the existence of the
// // files can be tested for at the end of the test
// std::list<std::string> remoteFilePaths;
//
// //delete is unnecessary
// //pointer with ownership will be passed to the application,
// //which will do the delete
// const uint64_t tapeSize = 5000;
// mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive(tapeSize,
// castor::tape::tapeserver::drive::FakeDrive::OnFlush);
//
// // We can prepare files for writing on the drive
// {
// // Label the tape
// castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], "V12345");
// mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
//
// // schedule the archivals
// for(int fseq=1; fseq <= 10 ; fseq ++) {
// // Create a path to a remote destination file
// std::ostringstream remoteFilePath;
// remoteFilePath << "file://test" << fseq;
// remoteFilePaths.push_back(remoteFilePath.str());
//
// // Create the entry in the remote namespace (same user id of the requester)
// cta::RemotePath rpath(remoteFilePath.str());
// cta::RemoteFileStatus rstatus(requester.getUser(), 0777, 1000);
// rns.createEntry(rpath, rstatus);
//
// // Schedule the archival of the file
// std::list<std::string> remoteFilePathList;
// remoteFilePathList.push_back(remoteFilePath.str());
// ASSERT_NO_THROW(scheduler.queueArchiveRequest(requester, remoteFilePathList, rpath.getAfterScheme()));
// }
// }
// DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
// DataTransferConfig castorConf;
// castorConf.bufsz = 1024*1024; // 1 MB memory buffers
// castorConf.nbBufs = 10;
// castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
// castorConf.bulkRequestRecallMaxFiles = 1000;
// castorConf.nbDiskThreads = 1;
// castor::messages::AcsProxyDummy acs;
// castor::mediachanger::MmcProxyDummy mmc;
// castor::legacymsg::RmcProxyDummy rmc;
// castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
// castor::server::ProcessCap capUtils;
// castor::messages::TapeserverProxyDummy initialProcess;
// DataTransferSession sess("tapeHost", logger, mockSys, driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
// ASSERT_NO_THROW(sess.execute());
// std::string temp = logger.getLog();
// temp += "";
// ASSERT_EQ("V12345", sess.getVid());
//// //TODO: only 4 files out of 10 should have been migrated to this small tape (we need to check this with the "m" bit)
//// for(auto i=remoteFilePaths.begin(); i!=remoteFilePaths.end(); i++) {
//// cta::RemotePath rpath(*i);
//// ASSERT_NO_THROW(ns.statFile(requester, rpath.getAfterScheme()));
//// std::unique_ptr<cta::ArchiveFileStatus> stat(ns.statFile(requester, rpath.getAfterScheme()));
//// ASSERT_NE((uint64_t)(stat.get()), NULL);
//// ASSERT_EQ(stat->mode, 0777);
//// ASSERT_EQ(stat->size, 1000);
//// }
//}
TEST_P(DataTransferSessionTest, DataTransferSessionTapeFullOnFlushMigration) {
// 0) Prepare the logger for everyone
castor::log::StringLogger logger("tapeServerUnitTest");
setupDefaultCatalogue();
// 1) prepare the fake scheduler
std::string vid = s_vid;
// cta::MountType::Enum mountType = cta::MountType::RETRIEVE;
// 3) Prepare the necessary environment (logger, plus system wrapper),
castor::tape::System::mockWrapper mockSys;
mockSys.delegateToFake();
mockSys.disableGMockCallsCounting();
mockSys.fake.setupForVirtualDriveSLC6();
// 4) Create the scheduler
auto & catalogue = getCatalogue();
auto & scheduler = getScheduler();
// Always use the same requester
const cta::common::dataStructures::SecurityIdentity requester("user", "group");
// List to remember the path of each remote file so that the existance of the
// files can be tested for at the end of the test
std::list<std::string> remoteFilePaths;
// 5) Create the environment for the migration to happen (library + tape)
const std::string libraryComment = "Library comment";
catalogue.createLogicalLibrary(s_adminOnAdminHost, s_libraryName,
libraryComment);
{
auto libraries = catalogue.getLogicalLibraries();
ASSERT_EQ(1, libraries.size());
ASSERT_EQ(s_libraryName, libraries.front().name);
ASSERT_EQ(libraryComment, libraries.front().comment);
}
const uint64_t capacityInBytes = 12345678;
const std::string tapeComment = "Tape comment";
bool notDisabled = false;
bool notFull = false;
catalogue.createTape(s_adminOnAdminHost, s_vid, s_libraryName, s_tapePoolName, cta::nullopt, capacityInBytes,
notDisabled, notFull, tapeComment);
// Create the mount criteria
catalogue.createMountPolicy(requester, "immediateMount", 1000, 0, 1000, 0, 1, "Policy comment");
catalogue.createRequesterMountRule(requester, "immediateMount", s_diskInstance, requester.username, "Rule comment");
//delete is unnecessary
//pointer with ownership will be passed to the application,
//which will do the delete
const uint64_t tapeSize = 5000;
mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive(tapeSize,
castor::tape::tapeserver::drive::FakeDrive::OnFlush);
// We can prepare files for writing on the drive.
// Tempfiles are in this scope so they are kept alive
std::list<std::unique_ptr<unitTests::TempFile>> sourceFiles;
std::list<uint64_t> archiveFileIds;
{
// Label the tape
castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], s_vid, false);
catalogue.tapeLabelled(s_vid, "T10D6116", true);
mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
// Create the files and schedule the archivals
for(int fseq=1; fseq <= 10 ; fseq ++) {
// Create a source file.
sourceFiles.emplace_back(cta::make_unique<unitTests::TempFile>());
sourceFiles.back()->randomFill(1000);
remoteFilePaths.push_back(sourceFiles.back()->path());
// Schedule the archival of the file
cta::common::dataStructures::ArchiveRequest ar;
ar.checksumType="ADLER32";
ar.checksumValue=sourceFiles.back()->adler32();
ar.storageClass=s_storageClassName;
ar.srcURL=std::string("file://") + sourceFiles.back()->path();
ar.requester.name = requester.username;
ar.requester.group = "group";
ar.fileSize = 1000;
ar.diskFileID = "x";
ar.diskFileInfo.path = "y";
ar.diskFileInfo.owner = "z";
ar.diskFileInfo.group = "g";
ar.diskFileInfo.recoveryBlob = "b";
archiveFileIds.push_back(scheduler.queueArchive(s_diskInstance,ar));
}
}
DriveConfig driveConfig("T10D6116", "TestLogicalLibrary", "/dev/tape_T10D6116", "manual");
DataTransferConfig castorConf;
castorConf.bufsz = 1024*1024; // 1 MB memory buffers
castorConf.nbBufs = 10;
castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
castorConf.bulkRequestRecallMaxFiles = 1000;
castorConf.nbDiskThreads = 1;
castor::messages::AcsProxyDummy acs;
castor::mediachanger::MmcProxyDummy mmc;
castor::legacymsg::RmcProxyDummy rmc;
castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
castor::server::ProcessCap capUtils;
castor::messages::TapeserverProxyDummy initialProcess;
DataTransferSession sess("tapeHost", logger, mockSys, driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
sess.execute();
std::string temp = logger.getLog();
temp += "";
ASSERT_EQ(s_vid, sess.getVid());
auto afiiter = archiveFileIds.begin();
for(auto & sf: sourceFiles) {
auto afi = *(afiiter++);
// Only the first files made it through.
if (afi <= 3) {
auto afs = catalogue.getArchiveFileById(afi);
ASSERT_EQ(1, afs.tapeFiles.size());
ASSERT_EQ(sf->adler32(), afs.checksumValue);
ASSERT_EQ(1000, afs.fileSize);
} else {
ASSERT_THROW(catalogue.getArchiveFileById(afi), cta::exception::Exception);
}
// The tape should now be marked as full
cta::catalogue::TapeSearchCriteria crit;
crit.vid = s_vid;
auto tapes = catalogue.getTapes(crit);
ASSERT_EQ(1, tapes.size());
ASSERT_EQ(s_vid, tapes.front().vid);
ASSERT_EQ(true, tapes.front().full);
}
}
#undef TEST_MOCK_DB
#ifdef TEST_MOCK_DB
......
......@@ -141,10 +141,6 @@ void MigrationReportPacker::reportDriveStatus(cta::common::DriveStatus status) {
//------------------------------------------------------------------------------
void MigrationReportPacker::ReportDriveStatus::execute(MigrationReportPacker& parent){
parent.m_archiveMount->setDriveStatus(m_status);
if(m_status==cta::common::DriveStatus::Unmounting) {
parent.m_continue=false;
parent.m_archiveMount->complete();
}
}
//------------------------------------------------------------------------------
......@@ -203,6 +199,8 @@ void MigrationReportPacker::ReportTapeFull::execute(MigrationReportPacker& repor
//ReportEndofSession::execute
//------------------------------------------------------------------------------
void MigrationReportPacker::ReportEndofSession::execute(MigrationReportPacker& reportPacker){
reportPacker.m_continue=false;
reportPacker.m_archiveMount->complete();
if(!reportPacker.m_errorHappened){
log::ScopedParamContainer sp(reportPacker.m_lc);
reportPacker.m_lc.log(LOG_INFO,"Reported end of session to client");
......@@ -233,6 +231,8 @@ void MigrationReportPacker::ReportEndofSession::execute(MigrationReportPacker& r
//ReportEndofSessionWithErrors::execute
//------------------------------------------------------------------------------
void MigrationReportPacker::ReportEndofSessionWithErrors::execute(MigrationReportPacker& reportPacker){
reportPacker.m_continue=false;
reportPacker.m_archiveMount->complete();
if(reportPacker.m_errorHappened) {
log::ScopedParamContainer sp(reportPacker.m_lc);
sp.add("errorMessage", m_message)
......
......@@ -302,6 +302,7 @@ void castor::tape::tapeserver::daemon::TapeWriteSingleThread::run() {
}
// This is indeed the end of the tape. Not an error.
m_watchdog.setErrorCount("Info_tapeFilledUp",1);
m_reportPacker.reportTapeFull();
} catch (...) {
// The error is not an ENOSPC, so it is, indeed, an error.
// If we got here with a new error, currentErrorToCount will be non-empty,
......@@ -368,6 +369,7 @@ void castor::tape::tapeserver::daemon::TapeWriteSingleThread::run() {
}
// This is indeed the end of the tape. Not an error.
m_watchdog.setErrorCount("Info_tapeFilledUp",1);
m_reportPacker.reportTapeFull();
} catch (...) {
// The error is not an ENOSPC, so it is, indeed, an error.
// If we got here with a new error, currentErrorToCount will be non-empty,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment