Skip to content
GitLab
Commit 32573337 authored by Eric Cano's avatar Eric Cano
Browse files

Ported last data transfer session unit test and fixed report packer overlooking last reports.

parent a408c2ac
Hide whitespace changes
Inline Side-by-side
tapeserver/castor/tape/tapeserver/daemon/DataTransferSessionTest.cpp
View file @ 32573337
......@@ -1305,124 +1305,132 @@ TEST_P(DataTransferSessionTest, DataTransferSessionTapeFullMigration) {
}
}
//TEST_P(DataTransferSessionTest, DataTransferSessionTapeFullOnFlushMigration) {
//
//
// // 0) Prepare the logger for everyone
// castor::log::StringLogger logger("tapeServerUnitTest");
//
// // 1) prepare the fake scheduler
// std::string vid = "V12345";
//
// // 3) Prepare the necessary environment (logger, plus system wrapper),
// castor::tape::System::mockWrapper mockSys;
// mockSys.delegateToFake();
// mockSys.disableGMockCallsCounting();
// mockSys.fake.setupForVirtualDriveSLC6();
//
// // 4) Create the scheduler
// cta::catalogue::SqliteCatalogue sqliteCatalogue;
// cta::MockRemoteNS rns;
// cta::OStoreDBWrapper<cta::objectstore::BackendVFS> db("Unittest");
// cta::Scheduler scheduler(ns, db, rns);
//
// // Always use the same requester
// const cta::SecurityIdentity requester;
//
// // Create the bootstrap admin user and host
// ASSERT_NO_THROW(scheduler.createAdminUserWithoutAuthorizingRequester(requester, requester.getUser(), "admin user"));
// ASSERT_NO_THROW(scheduler.createAdminHostWithoutAuthorizingRequester(requester, requester.getHost(), "admin host"));
//
// // create a single copy storage class
// ASSERT_NO_THROW(scheduler.createStorageClass(requester, "SINGLE", 1, 1, "comment"));
//
// // assign it to the root directory
// ASSERT_NO_THROW(scheduler.setDirStorageClass(requester, "/", "SINGLE"));
//
// // create the logical library
// ASSERT_NO_THROW(scheduler.createLogicalLibrary(requester, "T10KD6", "the illogical library"));
//
// // create the tape pool
// ASSERT_NO_THROW(scheduler.createTapePool(requester, "swimmingpool", 2, "the swimming pool"));
//
// // Make the pool mount immediately
// cta::MountCriteria immediateMount;
// immediateMount.maxAge = 0;
// immediateMount.maxBytesQueued = 1;
// immediateMount.maxFilesQueued = 1;
// immediateMount.quota = 10;
// ASSERT_NO_THROW(scheduler.setTapePoolMountCriteria("swimmingpool", cta::MountCriteriaByDirection(immediateMount, immediateMount)));
//
// // create the route
// ASSERT_NO_THROW(scheduler.createArchiveRoute(requester, "SINGLE", 1, "swimmingpool", "iArchive"));
//
// // create the tape
// std::string comment = "the magic tape";
// ASSERT_NO_THROW(scheduler.createTape(requester, "V12345", "T10KD6", "swimmingpool", 100000, comment));
//
// // List to remember the path of each remote file so that the existence of the
// // files can be tested for at the end of the test
// std::list<std::string> remoteFilePaths;
//
// //delete is unnecessary
// //pointer with ownership will be passed to the application,
// //which will do the delete
// const uint64_t tapeSize = 5000;
// mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive(tapeSize,
// castor::tape::tapeserver::drive::FakeDrive::OnFlush);
//
// // We can prepare files for writing on the drive
// {
// // Label the tape
// castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], "V12345");
// mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
//
// // schedule the archivals
// for(int fseq=1; fseq <= 10 ; fseq ++) {
// // Create a path to a remote destination file
// std::ostringstream remoteFilePath;
// remoteFilePath << "file://test" << fseq;
// remoteFilePaths.push_back(remoteFilePath.str());
//
// // Create the entry in the remote namespace (same user id of the requester)
// cta::RemotePath rpath(remoteFilePath.str());
// cta::RemoteFileStatus rstatus(requester.getUser(), 0777, 1000);
// rns.createEntry(rpath, rstatus);
//
// // Schedule the archival of the file
// std::list<std::string> remoteFilePathList;
// remoteFilePathList.push_back(remoteFilePath.str());
// ASSERT_NO_THROW(scheduler.queueArchiveRequest(requester, remoteFilePathList, rpath.getAfterScheme()));
// }
// }
// DriveConfig driveConfig("T10D6116", "T10KD6", "/dev/tape_T10D6116", "manual");
// DataTransferConfig castorConf;
// castorConf.bufsz = 1024*1024; // 1 MB memory buffers
// castorConf.nbBufs = 10;
// castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
// castorConf.bulkRequestRecallMaxFiles = 1000;
// castorConf.nbDiskThreads = 1;
// castor::messages::AcsProxyDummy acs;
// castor::mediachanger::MmcProxyDummy mmc;
// castor::legacymsg::RmcProxyDummy rmc;
// castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
// castor::server::ProcessCap capUtils;
// castor::messages::TapeserverProxyDummy initialProcess;
// DataTransferSession sess("tapeHost", logger, mockSys, driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
// ASSERT_NO_THROW(sess.execute());
// std::string temp = logger.getLog();
// temp += "";
// ASSERT_EQ("V12345", sess.getVid());
//// //TODO: only 4 files out of 10 should have been migrated to this small tape (we need to check this with the "m" bit)
//// for(auto i=remoteFilePaths.begin(); i!=remoteFilePaths.end(); i++) {
//// cta::RemotePath rpath(*i);
//// ASSERT_NO_THROW(ns.statFile(requester, rpath.getAfterScheme()));
//// std::unique_ptr<cta::ArchiveFileStatus> stat(ns.statFile(requester, rpath.getAfterScheme()));
//// ASSERT_NE((uint64_t)(stat.get()), NULL);
//// ASSERT_EQ(stat->mode, 0777);
//// ASSERT_EQ(stat->size, 1000);
//// }
//}
TEST_P(DataTransferSessionTest, DataTransferSessionTapeFullOnFlushMigration) {
// 0) Prepare the logger for everyone
castor::log::StringLogger logger("tapeServerUnitTest");
setupDefaultCatalogue();
// 1) prepare the fake scheduler
std::string vid = s_vid;
// cta::MountType::Enum mountType = cta::MountType::RETRIEVE;
// 3) Prepare the necessary environment (logger, plus system wrapper),
castor::tape::System::mockWrapper mockSys;
mockSys.delegateToFake();
mockSys.disableGMockCallsCounting();
mockSys.fake.setupForVirtualDriveSLC6();
// 4) Create the scheduler
auto & catalogue = getCatalogue();
auto & scheduler = getScheduler();
// Always use the same requester
const cta::common::dataStructures::SecurityIdentity requester("user", "group");
// List to remember the path of each remote file so that the existance of the
// files can be tested for at the end of the test
std::list<std::string> remoteFilePaths;
// 5) Create the environment for the migration to happen (library + tape)
const std::string libraryComment = "Library comment";
catalogue.createLogicalLibrary(s_adminOnAdminHost, s_libraryName,
libraryComment);
{
auto libraries = catalogue.getLogicalLibraries();
ASSERT_EQ(1, libraries.size());
ASSERT_EQ(s_libraryName, libraries.front().name);
ASSERT_EQ(libraryComment, libraries.front().comment);
}
const uint64_t capacityInBytes = 12345678;
const std::string tapeComment = "Tape comment";
bool notDisabled = false;
bool notFull = false;
catalogue.createTape(s_adminOnAdminHost, s_vid, s_libraryName, s_tapePoolName, cta::nullopt, capacityInBytes,
notDisabled, notFull, tapeComment);
// Create the mount criteria
catalogue.createMountPolicy(requester, "immediateMount", 1000, 0, 1000, 0, 1, "Policy comment");
catalogue.createRequesterMountRule(requester, "immediateMount", s_diskInstance, requester.username, "Rule comment");
//delete is unnecessary
//pointer with ownership will be passed to the application,
//which will do the delete
const uint64_t tapeSize = 5000;
mockSys.fake.m_pathToDrive["/dev/nst0"] = new castor::tape::tapeserver::drive::FakeDrive(tapeSize,
castor::tape::tapeserver::drive::FakeDrive::OnFlush);
// We can prepare files for writing on the drive.
// Tempfiles are in this scope so they are kept alive
std::list<std::unique_ptr<unitTests::TempFile>> sourceFiles;
std::list<uint64_t> archiveFileIds;
{
// Label the tape
castor::tape::tapeFile::LabelSession ls(*mockSys.fake.m_pathToDrive["/dev/nst0"], s_vid, false);
catalogue.tapeLabelled(s_vid, "T10D6116", true);
mockSys.fake.m_pathToDrive["/dev/nst0"]->rewind();
// Create the files and schedule the archivals
for(int fseq=1; fseq <= 10 ; fseq ++) {
// Create a source file.
sourceFiles.emplace_back(cta::make_unique<unitTests::TempFile>());
sourceFiles.back()->randomFill(1000);
remoteFilePaths.push_back(sourceFiles.back()->path());
// Schedule the archival of the file
cta::common::dataStructures::ArchiveRequest ar;
ar.checksumType="ADLER32";
ar.checksumValue=sourceFiles.back()->adler32();
ar.storageClass=s_storageClassName;
ar.srcURL=std::string("file://") + sourceFiles.back()->path();
ar.requester.name = requester.username;
ar.requester.group = "group";
ar.fileSize = 1000;
ar.diskFileID = "x";
ar.diskFileInfo.path = "y";
ar.diskFileInfo.owner = "z";
ar.diskFileInfo.group = "g";
ar.diskFileInfo.recoveryBlob = "b";
archiveFileIds.push_back(scheduler.queueArchive(s_diskInstance,ar));
}
}
DriveConfig driveConfig("T10D6116", "TestLogicalLibrary", "/dev/tape_T10D6116", "manual");
DataTransferConfig castorConf;
castorConf.bufsz = 1024*1024; // 1 MB memory buffers
castorConf.nbBufs = 10;
castorConf.bulkRequestRecallMaxBytes = UINT64_C(100)*1000*1000*1000;
castorConf.bulkRequestRecallMaxFiles = 1000;
castorConf.nbDiskThreads = 1;
castor::messages::AcsProxyDummy acs;
castor::mediachanger::MmcProxyDummy mmc;
castor::legacymsg::RmcProxyDummy rmc;
castor::mediachanger::MediaChangerFacade mc(acs, mmc, rmc);
castor::server::ProcessCap capUtils;
castor::messages::TapeserverProxyDummy initialProcess;
DataTransferSession sess("tapeHost", logger, mockSys, driveConfig, mc, initialProcess, capUtils, castorConf, scheduler);
sess.execute();
std::string temp = logger.getLog();
temp += "";
ASSERT_EQ(s_vid, sess.getVid());
auto afiiter = archiveFileIds.begin();
for(auto & sf: sourceFiles) {
auto afi = *(afiiter++);
// Only the first files made it through.
if (afi <= 3) {
auto afs = catalogue.getArchiveFileById(afi);
ASSERT_EQ(1, afs.tapeFiles.size());
ASSERT_EQ(sf->adler32(), afs.checksumValue);
ASSERT_EQ(1000, afs.fileSize);
} else {
ASSERT_THROW(catalogue.getArchiveFileById(afi), cta::exception::Exception);
}
// The tape should now be marked as full
cta::catalogue::TapeSearchCriteria crit;
crit.vid = s_vid;
auto tapes = catalogue.getTapes(crit);
ASSERT_EQ(1, tapes.size());
ASSERT_EQ(s_vid, tapes.front().vid);
ASSERT_EQ(true, tapes.front().full);
}
}
#undef TEST_MOCK_DB
#ifdef TEST_MOCK_DB
......
tapeserver/castor/tape/tapeserver/daemon/MigrationReportPacker.cpp
View file @ 32573337
......@@ -141,10 +141,6 @@ void MigrationReportPacker::reportDriveStatus(cta::common::DriveStatus status) {
//------------------------------------------------------------------------------
void MigrationReportPacker::ReportDriveStatus::execute(MigrationReportPacker& parent){
parent.m_archiveMount->setDriveStatus(m_status);
if(m_status==cta::common::DriveStatus::Unmounting) {
parent.m_continue=false;
parent.m_archiveMount->complete();
}
}
//------------------------------------------------------------------------------
......@@ -203,6 +199,8 @@ void MigrationReportPacker::ReportTapeFull::execute(MigrationReportPacker& repor
//ReportEndofSession::execute
//------------------------------------------------------------------------------
void MigrationReportPacker::ReportEndofSession::execute(MigrationReportPacker& reportPacker){
reportPacker.m_continue=false;
reportPacker.m_archiveMount->complete();
if(!reportPacker.m_errorHappened){
log::ScopedParamContainer sp(reportPacker.m_lc);
reportPacker.m_lc.log(LOG_INFO,"Reported end of session to client");
......@@ -233,6 +231,8 @@ void MigrationReportPacker::ReportEndofSession::execute(MigrationReportPacker& r
//ReportEndofSessionWithErrors::execute
//------------------------------------------------------------------------------
void MigrationReportPacker::ReportEndofSessionWithErrors::execute(MigrationReportPacker& reportPacker){
reportPacker.m_continue=false;
reportPacker.m_archiveMount->complete();
if(reportPacker.m_errorHappened) {
log::ScopedParamContainer sp(reportPacker.m_lc);
sp.add("errorMessage", m_message)
......
tapeserver/castor/tape/tapeserver/daemon/TapeWriteSingleThread.cpp
View file @ 32573337
......@@ -302,6 +302,7 @@ void castor::tape::tapeserver::daemon::TapeWriteSingleThread::run() {
}
// This is indeed the end of the tape. Not an error.
m_watchdog.setErrorCount("Info_tapeFilledUp",1);
m_reportPacker.reportTapeFull();
} catch (...) {
// The error is not an ENOSPC, so it is, indeed, an error.
// If we got here with a new error, currentErrorToCount will be non-empty,
......@@ -368,6 +369,7 @@ void castor::tape::tapeserver::daemon::TapeWriteSingleThread::run() {
}
// This is indeed the end of the tape. Not an error.
m_watchdog.setErrorCount("Info_tapeFilledUp",1);
m_reportPacker.reportTapeFull();
} catch (...) {
// The error is not an ENOSPC, so it is, indeed, an error.
// If we got here with a new error, currentErrorToCount will be non-empty,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

imprint | privacy | Conditions of Use