diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 3c14c7a75d2878d3b43fbe72e4f76ae085355fce..bf46e09f63b4cda421ca06f7de0d54ce1dd0b2e2 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,3 +1,10 @@ +# v4.8.4-1 + +## Summary +This release reduces the number of DB queries issued to the CTA catalogue. +### Bug Fixes +- cta/CTA#275 - Avoid DB queries via getTapesByVid in OStoreDB::fetchMountInfo + # v4.8.3-1 ## Summary diff --git a/catalogue/CountGetTapesByVid.hpp b/catalogue/CountGetTapesByVid.hpp index ffdeb4b47452d809b9948f126f223bd400fc3844..4eba3f75f95171aefab7e467a884096abf7101d7 100644 --- a/catalogue/CountGetTapesByVid.hpp +++ b/catalogue/CountGetTapesByVid.hpp @@ -42,6 +42,7 @@ enum Enum { FCTCBR, TTSC, CS, + GT, EMAX }; @@ -64,7 +65,8 @@ static struct Logstr { {FMIUNLCK, "OStoreDB::fetchMountInfo_unlocked" }, {FCTCBR, "Scheduler::checkTapeCanBeRepacked" }, {TTSC, "Scheduler::triggerTapeStateChange" }, - {CS, "castor::tape::tapeserver::daemon::CleanerSession::exceptionThrowingExecute" } + {CS, "castor::tape::tapeserver::daemon::CleanerSession::exceptionThrowingExecute" }, + {GT, "Scheduler::sortAndGetTapesForMountInfo_getTapes()" } }; struct CounterSpace { diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp index 4a6c9a16bd7579f1ab557ceaf7e8248bb13fdb54..864e8db589c86b603e5012f89160b88298685844 100644 --- a/scheduler/OStoreDB/OStoreDB.cpp +++ b/scheduler/OStoreDB/OStoreDB.cpp @@ -244,7 +244,7 @@ std::list<SchedulerDatabase::RetrieveQueueCleanupInfo> OStoreDB::getRetrieveQueu // OStoreDB::fetchMountInfo() //------------------------------------------------------------------------------ void OStoreDB::fetchMountInfo(SchedulerDatabase::TapeMountDecisionInfo& tmdi, RootEntry& re, - SchedulerDatabase::PurposeGetMountInfo purpose, bool locked, log::LogContext & logContext) { + SchedulerDatabase::PurposeGetMountInfo /* not used */, bool /* not used */, log::LogContext & logContext) { utils::Timer t, t2; std::list<common::dataStructures::MountPolicy> mountPolicies = m_catalogue.getCachedMountPolicies(); // Walk the archive queues for USER for statistics @@ -401,19 +401,7 @@ void OStoreDB::fetchMountInfo(SchedulerDatabase::TapeMountDecisionInfo& tmdi, Ro // If there are files queued, we create an entry for this retrieve queue in the // mount candidates list. auto rqSummary = rqueue.getJobsSummary(); - bool isPotentialMount = false; - if (locked) { - m_catalogue.countGetTapesByVid(cta::catalogue::countGetTapesByVid::FMILCK); - } else { - m_catalogue.countGetTapesByVid(cta::catalogue::countGetTapesByVid::FMIUNLCK); - } - auto vidToTapeMap = m_catalogue.getTapesByVid(rqp.vid); - common::dataStructures::Tape::State tapeState = vidToTapeMap.at(rqp.vid).state; - if (tapeState == common::dataStructures::Tape::ACTIVE || - tapeState == common::dataStructures::Tape::REPACKING) { - isPotentialMount = true; - } - if (rqSummary.jobs && (isPotentialMount || purpose == SchedulerDatabase::PurposeGetMountInfo::SHOW_QUEUES)) { + if (rqSummary.jobs) { //Getting the default mountPolicies parameters from the queue summary uint64_t minRetrieveRequestAge = rqSummary.minRetrieveRequestAge; uint64_t priority = rqSummary.priority; @@ -460,7 +448,7 @@ void OStoreDB::fetchMountInfo(SchedulerDatabase::TapeMountDecisionInfo& tmdi, Ro m.mediaType = ""; // The logical library is not known here, and will be determined by the caller. m.vo = ""; // The vo is not known here, and will be determined by the caller. m.capacityInBytes = 0; // The capacity is not known here, and will be determined by the caller. - m.labelFormat = vidToTapeMap.at(rqp.vid).labelFormat; + m.labelFormat = std::nullopt; // The labelFormat is not known here, and may be determined by the caller. m.activity = ac.activity; m.mountPolicyNames = queueMountPolicyNames; // We will display the sleep flag only if it is not expired (15 minutes timeout, hardcoded). @@ -493,7 +481,7 @@ void OStoreDB::fetchMountInfo(SchedulerDatabase::TapeMountDecisionInfo& tmdi, Ro m.mediaType = ""; // The logical library is not known here, and will be determined by the caller. m.vo = ""; // The vo is not known here, and will be determined by the caller. m.capacityInBytes = 0; // The capacity is not known here, and will be determined by the caller. - m.labelFormat = vidToTapeMap.at(rqp.vid).labelFormat; + m.labelFormat = std::nullopt; // The labelFormat is not known here, and may be determined by the caller. m.mountPolicyNames = queueMountPolicyNames; // We will display the sleep flag only if it is not expired (15 minutes timeout, hardcoded). // This allows having a single decision point instead of implementing is at the consumer levels. diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp index 8e5574512f8a5111af965fe1ded20c0e37becd92..f66a7af49d3e65d938557ec552a5be379a9461b6 100644 --- a/scheduler/Scheduler.cpp +++ b/scheduler/Scheduler.cpp @@ -996,14 +996,25 @@ void Scheduler::sortAndGetTapesForMountInfo(std::unique_ptr<SchedulerDatabase::T ExistingMountSummaryPerVo& existingMountsBasicTypeSummaryPerVo, std::set<std::string>& tapesInUse, std::list<catalogue::TapeForWriting>& tapeList, double& getTapeInfoTime, double& candidateSortingTime, double& getTapeForWriteTime, log::LogContext& lc) { - // The library information is not known for tapes involved in retrieves. Get the library information from the DB so + // Neither the library information nor the tape status is known for tapes involved in retrieves. + // Build the eligible set of tapes in the library and with required status so // we can filter the potential mounts to the ones that this tape server can serve. - catalogue::TapeSearchCriteria searchCriteria; - searchCriteria.logicalLibrary = logicalLibraryName; - auto eligibleTapesList = m_catalogue.getTapes(searchCriteria); std::set<std::string> eligibleTapeSet; - for(auto& t : eligibleTapesList) { - eligibleTapeSet.insert(t.vid); + { + catalogue::TapeSearchCriteria searchCriteria; + searchCriteria.logicalLibrary = logicalLibraryName; + searchCriteria.state = common::dataStructures::Tape::ACTIVE; + m_catalogue.countGetTapesByVid(cta::catalogue::countGetTapesByVid::GT); + auto eligibleTapesList = m_catalogue.getTapes(searchCriteria); + for(auto& t : eligibleTapesList) { + eligibleTapeSet.insert(t.vid); + } + searchCriteria.state = common::dataStructures::Tape::REPACKING; + m_catalogue.countGetTapesByVid(cta::catalogue::countGetTapesByVid::GT); + eligibleTapesList = m_catalogue.getTapes(searchCriteria); + for(auto& t : eligibleTapesList) { + eligibleTapeSet.insert(t.vid); + } } // Filter the potential mounts to keep only the ones that match the logical library for retrieves, @@ -1589,7 +1600,7 @@ auto logicalLibrary = getLogicalLibrary(logicalLibraryName,getLogicalLibrariesTi m->vendor, m->capacityInBytes, m->activity, - m->labelFormat).release()); + m->labelFormat.value()).release()); mountCreationTime += timer.secs(utils::Timer::resetCounter); internalRet->m_sessionRunning = true; internalRet->m_diskRunning = true; @@ -1607,7 +1618,7 @@ auto logicalLibrary = getLogicalLibrary(logicalLibraryName,getLogicalLibrariesTi schedulerDbTime = getMountInfoTime + queueTrimingTime + mountCreationTime + driveStatusSetTime; catalogueTime = getTapeInfoTime + getTapeForWriteTime; std::ostringstream ossLabelFormat; - ossLabelFormat << std::showbase << std::internal << std::setfill('0') << std::hex << std::setw(4) << static_cast<unsigned int>(m->labelFormat); + ossLabelFormat << std::showbase << std::internal << std::setfill('0') << std::hex << std::setw(4) << static_cast<unsigned int>(m->labelFormat.value()); params.add("tapePool", m->tapePool) .add("tapeVid", m->vid) .add("vo",m->vo) diff --git a/scheduler/SchedulerDatabase.hpp b/scheduler/SchedulerDatabase.hpp index 6801ae0b4590af927997d9a3f09ed5f9c187bc96..0e9a2f50a617ec72923638d14dc23da92d4cbb0a 100644 --- a/scheduler/SchedulerDatabase.hpp +++ b/scheduler/SchedulerDatabase.hpp @@ -685,7 +685,7 @@ class SchedulerDatabase { std::string mediaType; // Media type of the tape std::string vendor; // Vendor of the tape uint64_t capacityInBytes; // Capacity in bytes of the tape - cta::common::dataStructures::Label::Format labelFormat; // Label format of the tape + std::optional<cta::common::dataStructures::Label::Format> labelFormat; // Label format of the tape uint64_t priority; /**< The priority for the mount, defined as the highest * priority of all queued jobs */