diff --git a/ReleaseNotes.md b/ReleaseNotes.md index b3af0b9160e16a96382012fd511270f7d5531d3c..de46dadf71bfb20fbccbca14c85fde4bc23c5bf3 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -5,7 +5,7 @@ ### Upgrade Instructions This CTA release requires a non backwards compatible database schema upgrade to CTA catalogue schema v5.0. -Please consult the [database upgrade documentation](https://tapeoperations.docs.cern.ch/ctaops/upgrade_production_database). +Please consult the [database upgrade documentation](https://eoscta.docs.cern.ch/catalogue/upgrade/). ### Features - cta/CTA#1123 - Add mount id to disk space reservations, prevent tape servers from releasing disk space from a previous mount @@ -14,6 +14,7 @@ Please consult the [database upgrade documentation](https://tapeoperations.docs. ### Bug fixes - cta/CTA#1117 - Update masterDataInBytes when writing files to tape - cta/CTA#1125 - cta-admin dr ls should show '-' instead of "NO_MOUNT" for Mount Type +- cta/CTA#1138 - sortAndGetTapesForMountInfo only queries tapes in the current logical library ### Catalogue Schema - cta/CTA#1112 - Remove activities_weights table from catalogue diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp index 7fff60e0cef56745ec2949308feb91680b774789..ea66f3c6dc5adcc00f31bf0a9f1d7f1f5279b80f 100644 --- a/catalogue/Catalogue.hpp +++ b/catalogue/Catalogue.hpp @@ -632,6 +632,16 @@ public: virtual std::list<common::dataStructures::Tape> getTapes( const TapeSearchCriteria &searchCriteria = TapeSearchCriteria()) const = 0; + /** + * Returns the tape with the specified volume identifier. + * + * This method will throw an exception if it cannot find the specified tape. + * + * @param vids The tape volume identifier (VID). + * @return Map from tape volume identifier to tape. + */ + virtual common::dataStructures::VidToTapeMap getTapesByVid(const std::string& vid) const = 0; + /** * Returns the tapes with the specified volume identifiers. * diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp index 3a0ac34a1b9db24f9034fd79a230f5b66fd2a599..a202a72d5daaa9cfac99417a0bb97183630cfab9 100644 --- a/catalogue/CatalogueRetryWrapper.hpp +++ b/catalogue/CatalogueRetryWrapper.hpp @@ -344,6 +344,10 @@ public: return retryOnLostConnection(m_log, [&]{return m_catalogue->getTapes(searchCriteria);}, m_maxTriesToConnect); } + common::dataStructures::VidToTapeMap getTapesByVid(const std::string& vid) const override { + return retryOnLostConnection(m_log, [&]{return m_catalogue->getTapesByVid(vid);}, m_maxTriesToConnect); + } + common::dataStructures::VidToTapeMap getTapesByVid(const std::set<std::string> &vids) const override { return retryOnLostConnection(m_log, [&]{return m_catalogue->getTapesByVid(vids);}, m_maxTriesToConnect); } diff --git a/catalogue/CatalogueTest.cpp b/catalogue/CatalogueTest.cpp index 8e6a73015c18b40854d86d24426928e6ca0cea76..a8c05f255b8c3c0fd2ff60f348e9009ce331219e 100644 --- a/catalogue/CatalogueTest.cpp +++ b/catalogue/CatalogueTest.cpp @@ -6061,8 +6061,8 @@ TEST_P(cta_catalogue_CatalogueTest, modifyTapeState) { ASSERT_NO_THROW(m_catalogue->modifyTapeState(m_admin,vid,common::dataStructures::Tape::State::BROKEN,reason)); { - //catalogue getTapesByVid test - auto vidToTapeMap = m_catalogue->getTapesByVid({vid}); + //catalogue getTapesByVid test (single VID) + auto vidToTapeMap = m_catalogue->getTapesByVid(vid); auto tape = vidToTapeMap.at(vid); ASSERT_EQ(vid,tape.vid); ASSERT_EQ(common::dataStructures::Tape::BROKEN,tape.state); @@ -6085,7 +6085,9 @@ TEST_P(cta_catalogue_CatalogueTest, modifyTapeState) { } { - auto vidToTapeMap = m_catalogue->getTapesByVid({vid}); + //catalogue getTapesByVid test (set of VIDs) + std::set<std::string> vids = {vid}; + auto vidToTapeMap = m_catalogue->getTapesByVid(vids); auto tape = vidToTapeMap.at(vid); ASSERT_EQ(vid,tape.vid); ASSERT_EQ(common::dataStructures::Tape::BROKEN,tape.state); @@ -6119,7 +6121,7 @@ TEST_P(cta_catalogue_CatalogueTest, modifyTapeStateResetReasonWhenBackToActiveSt ASSERT_NO_THROW(m_catalogue->modifyTapeState(m_admin,vid,common::dataStructures::Tape::State::ACTIVE,cta::nullopt)); { - auto vidToTapeMap = m_catalogue->getTapesByVid({vid}); + auto vidToTapeMap = m_catalogue->getTapesByVid(vid); auto tape = vidToTapeMap.at(vid); ASSERT_EQ(vid,tape.vid); ASSERT_EQ(common::dataStructures::Tape::ACTIVE,tape.state); diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp index 2e4ca90c0a9155158b1ce401b97f0bc663922e88..385d72b71504845af41b364c4c1421e31204c14d 100644 --- a/catalogue/DummyCatalogue.hpp +++ b/catalogue/DummyCatalogue.hpp @@ -199,6 +199,10 @@ public: threading::MutexLocker lm(m_tapeEnablingMutex); m_tapeEnabling[vid]=common::dataStructures::Tape::DISABLED; } + common::dataStructures::VidToTapeMap getTapesByVid(const std::string& vid) const { + std::set<std::string> vids = {vid}; + return getTapesByVid(vids); + } common::dataStructures::VidToTapeMap getTapesByVid(const std::set<std::string>& vids) const { // Minimal implementation of VidToMap for retrieve request unit tests. We just support // disabled status for the tapes. diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 2a418067e34c7b496621dcd90261502aada04127..7ff070d410629687ba38b20e81f0c7ce48bdf56d 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -4178,7 +4178,79 @@ std::list<common::dataStructures::Tape> RdbmsCatalogue::getTapes(rdbms::Conn &co } //------------------------------------------------------------------------------ -// getTapesByVid +// getTapesByVid (single VID) +//------------------------------------------------------------------------------ +common::dataStructures::VidToTapeMap RdbmsCatalogue::getTapesByVid(const std::string& vid) const { + try { + const char* const sql = + "SELECT " + "TAPE.VID AS VID," + "MEDIA_TYPE.MEDIA_TYPE_NAME AS MEDIA_TYPE," + "TAPE.VENDOR AS VENDOR," + "LOGICAL_LIBRARY.LOGICAL_LIBRARY_NAME AS LOGICAL_LIBRARY_NAME," + "TAPE_POOL.TAPE_POOL_NAME AS TAPE_POOL_NAME," + "VIRTUAL_ORGANIZATION.VIRTUAL_ORGANIZATION_NAME AS VO," + "TAPE.ENCRYPTION_KEY_NAME AS ENCRYPTION_KEY_NAME," + "MEDIA_TYPE.CAPACITY_IN_BYTES AS CAPACITY_IN_BYTES," + "TAPE.DATA_IN_BYTES AS DATA_IN_BYTES," + "TAPE.LAST_FSEQ AS LAST_FSEQ," + "TAPE.IS_FULL AS IS_FULL," + "TAPE.IS_FROM_CASTOR AS IS_FROM_CASTOR," + "TAPE.LABEL_DRIVE AS LABEL_DRIVE," + "TAPE.LABEL_TIME AS LABEL_TIME," + "TAPE.LAST_READ_DRIVE AS LAST_READ_DRIVE," + "TAPE.LAST_READ_TIME AS LAST_READ_TIME," + "TAPE.LAST_WRITE_DRIVE AS LAST_WRITE_DRIVE," + "TAPE.LAST_WRITE_TIME AS LAST_WRITE_TIME," + "TAPE.READ_MOUNT_COUNT AS READ_MOUNT_COUNT," + "TAPE.WRITE_MOUNT_COUNT AS WRITE_MOUNT_COUNT," + "TAPE.USER_COMMENT AS USER_COMMENT," + "TAPE.TAPE_STATE AS TAPE_STATE," + "TAPE.STATE_REASON AS STATE_REASON," + "TAPE.STATE_UPDATE_TIME AS STATE_UPDATE_TIME," + "TAPE.STATE_MODIFIED_BY AS STATE_MODIFIED_BY," + "TAPE.CREATION_LOG_USER_NAME AS CREATION_LOG_USER_NAME," + "TAPE.CREATION_LOG_HOST_NAME AS CREATION_LOG_HOST_NAME," + "TAPE.CREATION_LOG_TIME AS CREATION_LOG_TIME," + "TAPE.LAST_UPDATE_USER_NAME AS LAST_UPDATE_USER_NAME," + "TAPE.LAST_UPDATE_HOST_NAME AS LAST_UPDATE_HOST_NAME," + "TAPE.LAST_UPDATE_TIME AS LAST_UPDATE_TIME " + "FROM " + "TAPE " + "INNER JOIN TAPE_POOL ON " + "TAPE.TAPE_POOL_ID = TAPE_POOL.TAPE_POOL_ID " + "INNER JOIN LOGICAL_LIBRARY ON " + "TAPE.LOGICAL_LIBRARY_ID = LOGICAL_LIBRARY.LOGICAL_LIBRARY_ID " + "INNER JOIN MEDIA_TYPE ON " + "TAPE.MEDIA_TYPE_ID = MEDIA_TYPE.MEDIA_TYPE_ID " + "INNER JOIN VIRTUAL_ORGANIZATION ON " + "TAPE_POOL.VIRTUAL_ORGANIZATION_ID = VIRTUAL_ORGANIZATION.VIRTUAL_ORGANIZATION_ID " + "WHERE " + "VID = :VID"; + + common::dataStructures::VidToTapeMap vidToTapeMap; + + auto conn = m_connPool.getConn(); + auto stmt = conn.createStmt(sql); + stmt.bindString(":VID", vid); + executeGetTapesByVidStmtAndCollectResults(stmt, vidToTapeMap); + + if(vidToTapeMap.size() != 1) { + exception::Exception ex; + ex.getMessage() << "Not all tapes were found: expected=1 actual=" << vidToTapeMap.size(); + throw ex; + } + return vidToTapeMap; + } catch(exception::UserError &) { + throw; + } catch(exception::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); + throw; + } +} + +//------------------------------------------------------------------------------ +// getTapesByVid (set of VIDs) //------------------------------------------------------------------------------ common::dataStructures::VidToTapeMap RdbmsCatalogue::getTapesByVid(const std::set<std::string> &vids) const { try { @@ -4206,7 +4278,7 @@ common::dataStructures::VidToTapeMap RdbmsCatalogue::getTapesByVid(const std::se vidNb = 1; // Execute the query and collect the results - executeGetTapesBy100VidsStmtAndCollectResults(stmt, vidToTapeMap); + executeGetTapesByVidStmtAndCollectResults(stmt, vidToTapeMap); // Create a new statement stmt = conn.createStmt(selectTapesBy100VidsSql); @@ -4226,7 +4298,7 @@ common::dataStructures::VidToTapeMap RdbmsCatalogue::getTapesByVid(const std::se } // Execute the query and collect the results - executeGetTapesBy100VidsStmtAndCollectResults(stmt, vidToTapeMap); + executeGetTapesByVidStmtAndCollectResults(stmt, vidToTapeMap); } if(vids.size() != vidToTapeMap.size()) { @@ -4314,9 +4386,9 @@ std::string RdbmsCatalogue::getSelectTapesBy100VidsSql() const { } //------------------------------------------------------------------------------ -// executeGetTapesBy100VidsStmtAndCollectResults +// executeGetTapesByVidStmtAndCollectResults //------------------------------------------------------------------------------ -void RdbmsCatalogue::executeGetTapesBy100VidsStmtAndCollectResults(rdbms::Stmt &stmt, +void RdbmsCatalogue::executeGetTapesByVidStmtAndCollectResults(rdbms::Stmt &stmt, common::dataStructures::VidToTapeMap &vidToTapeMap) const { auto rset = stmt.executeQuery(); while (rset.next()) { diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index 416b08f98d6b36ed5248549ad3ca6029f9eaeb46..889f7d2ebb77c9f8c1229f7715b3370c73d396d0 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -553,6 +553,16 @@ public: */ std::list<common::dataStructures::Tape> getTapes(const TapeSearchCriteria &searchCriteria) const override; + /** + * Returns the tape with the specified volume identifier. + * + * This method will throw an exception if it cannot find the specified tape. + * + * @param vid The tape volume identifier (VIDs). + * @return Map from tape volume identifier to tape. + */ + common::dataStructures::VidToTapeMap getTapesByVid(const std::string& vid) const override; + /** * Returns the tapes with the specified volume identifiers. * @@ -2296,12 +2306,13 @@ protected: std::string getSelectTapesBy100VidsSql() const; /** - * Executes the specified "getTapesBy100Vids" statement and collects the - * results into the specified vidToTapeMap. - * @param stmt the "getTapesBy100Vids" statement + * Executes the specified "getTapesByVid" statement and collects the results into + * the specified vidToTapeMap. + * + * @param stmt "getTapesByVid" statement * @param vidToTapeMap the map from VID to tape */ - void executeGetTapesBy100VidsStmtAndCollectResults(rdbms::Stmt &stmt, + void executeGetTapesByVidStmtAndCollectResults(rdbms::Stmt &stmt, common::dataStructures::VidToTapeMap &vidToTapeMap) const; /** diff --git a/objectstore/Helpers.cpp b/objectstore/Helpers.cpp index d9a5e2c14aa1d7b377841a385be37ceaec478232..81d0aa9de3a9fd883e81fd963ed7255cf5dbf666 100644 --- a/objectstore/Helpers.cpp +++ b/objectstore/Helpers.cpp @@ -413,7 +413,7 @@ std::string Helpers::selectBestRetrieveQueue(const std::set<std::string>& candid // Give other threads a chance to access the cache for other vids. grqsmLock.unlock(); // Get the informations (stages, so we don't access the global variable without the mutex. - auto tapeStatus=catalogue.getTapesByVid({v}); + auto tapeStatus=catalogue.getTapesByVid(v); // Build a minimal service retrieve file queue criteria to query queues. common::dataStructures::RetrieveFileQueueCriteria rfqc; common::dataStructures::TapeFile tf; diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp index 4816b4aba31bee4ca11b31b491e23df4ea1b529d..a8dd46302db9f9f3eb3c0776402a4c215eacc7e1 100644 --- a/scheduler/OStoreDB/OStoreDB.cpp +++ b/scheduler/OStoreDB/OStoreDB.cpp @@ -347,7 +347,7 @@ void OStoreDB::fetchMountInfo(SchedulerDatabase::TapeMountDecisionInfo& tmdi, Ro // mount candidates list. auto rqSummary = rqueue.getJobsSummary(); bool isPotentialMount = false; - auto vidToTapeMap = m_catalogue.getTapesByVid({rqp.vid}); + auto vidToTapeMap = m_catalogue.getTapesByVid(rqp.vid); common::dataStructures::Tape::State tapeState = vidToTapeMap.at(rqp.vid).state; bool tapeIsDisabled = tapeState == common::dataStructures::Tape::DISABLED; bool tapeIsActive = tapeState == common::dataStructures::Tape::ACTIVE; diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp index c99891595546eb449e65e356f371f76fcf614a46..4a5952e70209cbc7ced053664eeaddca8ca59eab 100644 --- a/scheduler/Scheduler.cpp +++ b/scheduler/Scheduler.cpp @@ -352,9 +352,8 @@ void Scheduler::queueLabel(const common::dataStructures::SecurityIdentity &cliId } void Scheduler::checkTapeCanBeRepacked(const std::string & vid, const SchedulerDatabase::QueueRepackRequest & repackRequest){ - std::set<std::string> vidToRepack({vid}); try{ - auto vidToTapesMap = m_catalogue.getTapesByVid(vidToRepack); //throws an exception if the vid is not found on the database + auto vidToTapesMap = m_catalogue.getTapesByVid(vid); //throws an exception if the vid is not found on the database cta::common::dataStructures::Tape tapeToCheck = vidToTapesMap.at(vid); if(!tapeToCheck.full){ throw exception::UserError("You must set the tape as full before repacking it."); @@ -1011,24 +1010,44 @@ std::list<common::dataStructures::TapeDrive> Scheduler::getDriveStates(const com // sortAndGetTapesForMountInfo //------------------------------------------------------------------------------ void Scheduler::sortAndGetTapesForMountInfo(std::unique_ptr<SchedulerDatabase::TapeMountDecisionInfo>& mountInfo, - const std::string & logicalLibraryName, const std::string & driveName, utils::Timer & timer, - ExistingMountSummaryPerTapepool & existingMountsDistinctTypeSummaryPerTapepool, ExistingMountSummaryPerVo & existingMountsBasicTypeSummaryPerVo, std::set<std::string> & tapesInUse, std::list<catalogue::TapeForWriting> & tapeList, - double & getTapeInfoTime, double & candidateSortingTime, double & getTapeForWriteTime, log::LogContext & lc) { - // The library information is not know for the tapes involved in retrieves. We - // need to query the catalogue now about all those tapes. - // Build the list of tapes. + const std::string& logicalLibraryName, const std::string& driveName, utils::Timer& timer, + ExistingMountSummaryPerTapepool& existingMountsDistinctTypeSummaryPerTapepool, + ExistingMountSummaryPerVo& existingMountsBasicTypeSummaryPerVo, std::set<std::string>& tapesInUse, + std::list<catalogue::TapeForWriting>& tapeList, double& getTapeInfoTime, double& candidateSortingTime, + double& getTapeForWriteTime, log::LogContext& lc) { + // The library information is not known for tapes involved in retrieves. Get the library information from the DB so + // we can filter the potential mounts to the ones that this tape server can serve. + catalogue::TapeSearchCriteria searchCriteria; + searchCriteria.logicalLibrary = logicalLibraryName; + auto eligibleTapesList = m_catalogue.getTapes(searchCriteria); + std::set<std::string> eligibleTapeSet; + for(auto& t : eligibleTapesList) { + eligibleTapeSet.insert(t.vid); + } + + // Filter the potential mounts to keep only the ones that match the logical library for retrieves, + // and build the list of tapes that can potentially be mounted by this tape server std::set<std::string> retrieveTapeSet; - for (auto &m:mountInfo->potentialMounts) { - if (m.type==common::dataStructures::MountType::Retrieve) retrieveTapeSet.insert(m.vid); + for(auto m_it = mountInfo->potentialMounts.begin(); m_it != mountInfo->potentialMounts.end(); ) { + if(m_it->type == common::dataStructures::MountType::Retrieve) { + if(eligibleTapeSet.count(m_it->vid) == 0) { + m_it = mountInfo->potentialMounts.erase(m_it); + continue; + } else { + retrieveTapeSet.insert(m_it->vid); + } + } + m_it++; } + common::dataStructures::VidToTapeMap retrieveTapesInfo; - if (retrieveTapeSet.size()) { - retrieveTapesInfo=m_catalogue.getTapesByVid(retrieveTapeSet); + if(!retrieveTapeSet.empty()) { + retrieveTapesInfo = m_catalogue.getTapesByVid(retrieveTapeSet); getTapeInfoTime = timer.secs(utils::Timer::resetCounter); - for (auto &m:mountInfo->potentialMounts) { - if (m.type==common::dataStructures::MountType::Retrieve) { - m.logicalLibrary=retrieveTapesInfo[m.vid].logicalLibraryName; - m.tapePool=retrieveTapesInfo[m.vid].tapePoolName; + for(auto& m : mountInfo->potentialMounts) { + if(m.type == common::dataStructures::MountType::Retrieve) { + m.logicalLibrary = retrieveTapesInfo[m.vid].logicalLibraryName; + m.tapePool = retrieveTapesInfo[m.vid].tapePoolName; m.vendor = retrieveTapesInfo[m.vid].vendor; m.mediaType = retrieveTapesInfo[m.vid].mediaType; m.vo = retrieveTapesInfo[m.vid].vo; @@ -1037,18 +1056,6 @@ void Scheduler::sortAndGetTapesForMountInfo(std::unique_ptr<SchedulerDatabase::T } } - // We should now filter the potential mounts to keep only the ones we are - // compatible with (match the logical library for retrieves). - // We also only want the potential mounts for which we still have - // We cannot filter the archives yet - for (auto m = mountInfo->potentialMounts.begin(); m!= mountInfo->potentialMounts.end();) { - if (m->type == common::dataStructures::MountType::Retrieve && m->logicalLibrary != logicalLibraryName) { - m = mountInfo->potentialMounts.erase(m); - } else { - m++; - } - } - //Get the tapepools of the potential and existing mounts std::set<std::string> tapepoolsPotentialOrExistingMounts; for (auto & pm: mountInfo->potentialMounts) {