Commit a669fd92 authored by Steven Murray's avatar Steven Murray
Browse files

[Catalogue] cta/operations#179 Slow `cta-admin sq` even when there is very little activity

The catalogue is now only queried for the tapes of interest.
parent ff0f4600
......@@ -554,11 +554,12 @@ public:
virtual common::dataStructures::VidToTapeMap getTapesByVid(const std::set<std::string> &vids) const = 0;
/**
* Returns map from VID to logical library name.
* Returns map from VID to logical library name for specified set of VIDs.
*
* @param vids The tape volume identifiers (VIDs).
* @return map from VID to logical library name.
*/
virtual std::map<std::string, std::string> getVidToLogicalLibrary() const = 0;
virtual std::map<std::string, std::string> getVidToLogicalLibrary(const std::set<std::string> &vids) const = 0;
/**
* Reclaims the specified tape.
......
......@@ -316,8 +316,8 @@ public:
return retryOnLostConnection(m_log, [&]{return m_catalogue->getTapesByVid(vids);}, m_maxTriesToConnect);
}
std::map<std::string, std::string> getVidToLogicalLibrary() const override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->getVidToLogicalLibrary();}, m_maxTriesToConnect);
std::map<std::string, std::string> getVidToLogicalLibrary(const std::set<std::string> &vids) const override {
return retryOnLostConnection(m_log, [&]{return m_catalogue->getVidToLogicalLibrary(vids);}, m_maxTriesToConnect);
}
void reclaimTape(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, cta::log::LogContext & lc) override {
......
......@@ -12960,13 +12960,56 @@ TEST_P(cta_catalogue_CatalogueTest, getTapesByVid_350_tapes) {
}
}
 
TEST_P(cta_catalogue_CatalogueTest, getVidToLogicalLibrary_no_tapes) {
TEST_P(cta_catalogue_CatalogueTest, getVidToLogicalLibrary_no_vids) {
using namespace cta;
 
ASSERT_TRUE(m_catalogue->getVidToLogicalLibrary().empty());
std::set<std::string> vids;
ASSERT_TRUE(m_catalogue->getVidToLogicalLibrary(vids).empty());
}
TEST_P(cta_catalogue_CatalogueTest, getVidToLogicalLibrary_1_tape) {
using namespace cta;
const bool logicalLibraryIsDisabled= false;
const uint64_t nbPartialTapes = 2;
const bool isEncrypted = true;
const cta::optional<std::string> supply("value for the supply pool mechanism");
m_catalogue->createMediaType(m_admin, m_mediaType);
m_catalogue->createLogicalLibrary(m_admin, m_tape1.logicalLibraryName, logicalLibraryIsDisabled, "Create logical library");
m_catalogue->createVirtualOrganization(m_admin, m_vo);
m_catalogue->createTapePool(m_admin, m_tape1.tapePoolName, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
const uint32_t nbTapes = 1;
std::set<std::string> allVids;
for(uint32_t i = 0; i < nbTapes; i++) {
std::ostringstream vid;
vid << "V" << std::setfill('0') << std::setw(5) << i;
const std::string tapeComment = "Create tape " + vid.str();
auto tape = m_tape1;
tape.vid = vid.str();
m_catalogue->createTape(m_admin, tape);
allVids.insert(vid.str());
}
const auto vidToLogicalLibrary = m_catalogue->getVidToLogicalLibrary(allVids);
ASSERT_EQ(nbTapes, vidToLogicalLibrary.size());
for(uint32_t i = 0; i < nbTapes; i++) {
std::ostringstream vid;
vid << "V" << std::setfill('0') << std::setw(5) << i;
const std::string tapeComment = "Create tape " + vid.str();
const auto itor = vidToLogicalLibrary.find(vid.str());
ASSERT_NE(vidToLogicalLibrary.end(), itor);
ASSERT_EQ(m_tape1.logicalLibraryName, itor->second);
}
}
 
TEST_P(cta_catalogue_CatalogueTest, getVidToLogicalLibrary_many_tapes) {
TEST_P(cta_catalogue_CatalogueTest, getVidToLogicalLibrary_310_tapes) {
using namespace cta;
 
const bool logicalLibraryIsDisabled= false;
......@@ -12979,7 +13022,8 @@ TEST_P(cta_catalogue_CatalogueTest, getVidToLogicalLibrary_many_tapes) {
m_catalogue->createVirtualOrganization(m_admin, m_vo);
m_catalogue->createTapePool(m_admin, m_tape1.tapePoolName, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
 
const uint32_t nbTapes = 10;
const uint32_t nbTapes = 310;
std::set<std::string> allVids;
 
for(uint32_t i = 0; i < nbTapes; i++) {
std::ostringstream vid;
......@@ -12989,9 +13033,10 @@ TEST_P(cta_catalogue_CatalogueTest, getVidToLogicalLibrary_many_tapes) {
auto tape = m_tape1;
tape.vid = vid.str();
m_catalogue->createTape(m_admin, tape);
allVids.insert(vid.str());
}
 
const auto vidToLogicalLibrary = m_catalogue->getVidToLogicalLibrary();
const auto vidToLogicalLibrary = m_catalogue->getVidToLogicalLibrary(allVids);
ASSERT_EQ(nbTapes, vidToLogicalLibrary.size());
 
for(uint32_t i = 0; i < nbTapes; i++) {
......@@ -12999,9 +13044,10 @@ TEST_P(cta_catalogue_CatalogueTest, getVidToLogicalLibrary_many_tapes) {
vid << "V" << std::setfill('0') << std::setw(5) << i;
const std::string tapeComment = "Create tape " + vid.str();
 
const auto tapeItor = vidToLogicalLibrary.find(vid.str());
ASSERT_NE(vidToLogicalLibrary.end(), tapeItor);
ASSERT_EQ(m_tape1.logicalLibraryName, tapeItor->second);
const auto itor = vidToLogicalLibrary.find(vid.str());
ASSERT_NE(vidToLogicalLibrary.end(), itor);
ASSERT_EQ(m_tape1.logicalLibraryName, itor->second);
}
}
 
......
......@@ -96,7 +96,7 @@ public:
std::list<TapePool> getTapePools() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
std::list<common::dataStructures::Tape> getTapes(const TapeSearchCriteria& searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
// getTapesByVid is implemented below (and works).
std::map<std::string, std::string> getVidToLogicalLibrary() const override { throw exception::Exception(std::string("In ") + __PRETTY_FUNCTION__ + ": not implemented"); }
std::map<std::string, std::string> getVidToLogicalLibrary(const std::set<std::string> &vids) const override { throw exception::Exception(std::string("In ") + __PRETTY_FUNCTION__ + ": not implemented"); }
std::list<TapeForWriting> getTapesForWriting(const std::string& logicalLibraryName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
bool isAdmin(const common::dataStructures::SecurityIdentity& admin) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
void modifyActivitiesFairShareWeight(const common::dataStructures::SecurityIdentity& admin, const std::string& diskInstanceName, const std::string& acttivity, double weight, const std::string & comment) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
......
......@@ -3609,7 +3609,7 @@ std::string RdbmsCatalogue::getSelectTapesBy100VidsSql() const {
"TAPE.IS_DISABLED AS IS_DISABLED,"
"TAPE.IS_FULL AS IS_FULL,"
"TAPE.IS_READ_ONLY AS IS_READ_ONLY,"
"TAPE.IS_FROM_CASTOR AS IS_FROM_CASTOR,"
"TAPE.IS_FROM_CASTOR AS IS_FROM_CASTOR,"
"TAPE.LABEL_DRIVE AS LABEL_DRIVE,"
"TAPE.LABEL_TIME AS LABEL_TIME,"
......@@ -3619,7 +3619,7 @@ std::string RdbmsCatalogue::getSelectTapesBy100VidsSql() const {
"TAPE.LAST_WRITE_DRIVE AS LAST_WRITE_DRIVE,"
"TAPE.LAST_WRITE_TIME AS LAST_WRITE_TIME,"
"TAPE.READ_MOUNT_COUNT AS READ_MOUNT_COUNT,"
"TAPE.WRITE_MOUNT_COUNT AS WRITE_MOUNT_COUNT,"
......@@ -3698,23 +3698,60 @@ void RdbmsCatalogue::executeGetTapesBy100VidsStmtAndCollectResults(rdbms::Stmt &
//------------------------------------------------------------------------------
// getVidToLogicalLibrary
//------------------------------------------------------------------------------
std::map<std::string, std::string> RdbmsCatalogue::getVidToLogicalLibrary() const {
std::map<std::string, std::string> RdbmsCatalogue::getVidToLogicalLibrary(const std::set<std::string> &vids) const {
try {
std::map<std::string, std::string> vidToLogicalLibrary;
std::string sql =
"SELECT" "\n"
"TAPE.VID AS VID," "\n"
"LOGICAL_LIBRARY.LOGICAL_LIBRARY_NAME AS LOGICAL_LIBRARY_NAME" "\n"
"FROM" "\n"
"TAPE" "\n"
"INNER JOIN LOGICAL_LIBRARY ON" "\n"
"TAPE.LOGICAL_LIBRARY_ID = LOGICAL_LIBRARY.LOGICAL_LIBRARY_ID";
if(vids.empty()) return vidToLogicalLibrary;
static const std::string sql = getSelectVidToLogicalLibraryBy100Sql();
auto conn = m_connPool.getConn();
auto stmt = conn.createStmt(sql);
auto rset = stmt.executeQuery();
while (rset.next()) {
vidToLogicalLibrary[rset.columnString("VID")] = rset.columnString("LOGICAL_LIBRARY_NAME");
uint64_t vidNb = 1;
for(const auto &vid: vids) {
// Bind the current tape VID
std::ostringstream paramName;
paramName << ":V" << vidNb;
stmt.bindString(paramName.str(), vid);
// If the 100th tape VID has not yet been reached
if(100 > vidNb) {
vidNb++;
} else { // The 100th VID has been reached
vidNb = 1;
// Execute the query and collect the results
executeGetVidToLogicalLibraryBy100StmtAndCollectResults(stmt, vidToLogicalLibrary);
// Create a new statement
stmt = conn.createStmt(sql);
}
}
// If there is a statement under construction
if(1 != vidNb) {
// Bind the remaining parameters with last tape VID. This has no effect
// on the search results but makes the statement valid.
const std::string &lastVid = *vids.rbegin();
while(100 >= vidNb) {
std::ostringstream paramName;
paramName << ":V" << vidNb;
stmt.bindString(paramName.str(), lastVid);
vidNb++;
}
// Execute the query and collect the results
executeGetVidToLogicalLibraryBy100StmtAndCollectResults(stmt, vidToLogicalLibrary);
}
if(vids.size() != vidToLogicalLibrary.size()) {
exception::Exception ex;
ex.getMessage() << "Not all tapes were found: expected=" << vids.size() << " actual=" <<
vidToLogicalLibrary.size();
throw ex;
}
return vidToLogicalLibrary;
......@@ -3726,6 +3763,43 @@ std::map<std::string, std::string> RdbmsCatalogue::getVidToLogicalLibrary() cons
}
}
//------------------------------------------------------------------------------
// getSelectVidToLogicalLibraryBy100Sql
//------------------------------------------------------------------------------
std::string RdbmsCatalogue::getSelectVidToLogicalLibraryBy100Sql() const {
std::stringstream sql;
sql <<
"SELECT" "\n"
"TAPE.VID AS VID," "\n"
"LOGICAL_LIBRARY.LOGICAL_LIBRARY_NAME AS LOGICAL_LIBRARY_NAME" "\n"
"FROM" "\n"
"TAPE" "\n"
"INNER JOIN LOGICAL_LIBRARY ON" "\n"
"TAPE.LOGICAL_LIBRARY_ID = LOGICAL_LIBRARY.LOGICAL_LIBRARY_ID" "\n"
"WHERE" "\n"
"VID IN (:V1";
for(uint32_t i=2; i<=100; i++) {
sql << ",:V" << i;
}
sql << ")";
return sql.str();
}
//------------------------------------------------------------------------------
// executeGetVidToLogicalLibraryBy100StmtAndCollectResults
//------------------------------------------------------------------------------
void RdbmsCatalogue::executeGetVidToLogicalLibraryBy100StmtAndCollectResults(rdbms::Stmt &stmt,
std::map<std::string, std::string> &vidToLogicalLibrary) const {
auto rset = stmt.executeQuery();
while (rset.next()) {
vidToLogicalLibrary[rset.columnString("VID")] = rset.columnString("LOGICAL_LIBRARY_NAME");
}
}
//------------------------------------------------------------------------------
//getNbNonSupersededFilesOnTape
//------------------------------------------------------------------------------
......
......@@ -496,11 +496,12 @@ public:
common::dataStructures::VidToTapeMap getTapesByVid(const std::set<std::string> &vids) const override;
/**
* Returns map from VID to logical library name.
* Returns map from VID to logical library name for specified set of VIDs.
*
* @param vids The tape volume identifiers (VIDs).
* @return map from VID to logical library name.
*/
std::map<std::string, std::string> getVidToLogicalLibrary() const override;
std::map<std::string, std::string> getVidToLogicalLibrary(const std::set<std::string> &vids) const override;
/**
* Reclaims the specified tape.
......@@ -1927,6 +1928,23 @@ protected:
void executeGetTapesBy100VidsStmtAndCollectResults(rdbms::Stmt &stmt,
common::dataStructures::VidToTapeMap &vidToTapeMap) const;
/**
* Generates the SELECT statement required to search for VID to logical
* library name mappings using 100 tape VIDs. Each tape VID is represented in
* the SQL by a bind parameter with the naming convention of ":VN" where N is
* an integer from 1 to 100 with no padding.
*/
std::string getSelectVidToLogicalLibraryBy100Sql() const;
/**
* Executes the specified "getVidToLogicalLibraryBy100" statement and collects the
* results into the specified vidToLogicalLibrary map.
* @param stmt the "getVidToLogicalLibraryBy100" statement.
* @param vidLogicalLibrary the map from VID to logical library name.
*/
void executeGetVidToLogicalLibraryBy100StmtAndCollectResults(rdbms::Stmt &stmt,
std::map<std::string, std::string> &vidToLogicalLibrary) const;
/**
* Returns an iterator across the files on the specified tape ordered by
* FSEQ.
......
......@@ -1565,16 +1565,29 @@ std::list<SchedulingInfos> Scheduler::getSchedulingInformations(log::LogContext&
//------------------------------------------------------------------------------
std::list<common::dataStructures::QueueAndMountSummary> Scheduler::getQueuesAndMountSummaries(log::LogContext& lc) {
std::list<common::dataStructures::QueueAndMountSummary> ret;
utils::Timer t;
// Obtain a map of vids to tape info from the catalogue
const auto vid_to_logical_library = m_catalogue.getVidToLogicalLibrary();
const auto catalogueVidToLogicalLibraryTime = t.secs(utils::Timer::resetCounter);
// Extract relevant information from the object store.
utils::Timer t;
auto mountDecisionInfo=m_db.getMountInfoNoLock(SchedulerDatabase::PurposeGetMountInfo::SHOW_QUEUES,lc);
auto schedulerDbTime = t.secs(utils::Timer::resetCounter);
auto & mdi __attribute__((unused)) = *mountDecisionInfo;
std::set<std::string> tapesWithAQueue;
for (const auto & pm: mountDecisionInfo->potentialMounts) {
if(!pm.vid.empty()) {
tapesWithAQueue.emplace(pm.vid);
}
}
for (const auto & em: mountDecisionInfo->existingOrNextMounts) {
if (!em.vid.empty()) {
tapesWithAQueue.emplace(em.vid);
}
}
// Obtain a map of vids to tape info from the catalogue
const auto vid_to_logical_library = m_catalogue.getVidToLogicalLibrary(tapesWithAQueue);
const auto catalogueVidToLogicalLibraryTime = t.secs(utils::Timer::resetCounter);
for (auto & pm: mountDecisionInfo->potentialMounts) {
// Find or create the relevant entry.
auto &summary = common::dataStructures::QueueAndMountSummary::getOrCreateEntry(ret, pm.type, pm.tapePool, pm.vid, vid_to_logical_library);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment