[Catalogue] cta/operations#179 Slow `cta-admin sq` even when there is very little activity

Replaced Catalogue::getAllTape() with getVidToLogicalibrary()
because only the logical library was needed by the caller.

catalogue/Catalogue.hpp

@@ -554,11 +554,11 @@ public:
   virtual common::dataStructures::VidToTapeMap getTapesByVid(const std::set<std::string> &vids) const = 0;
 
   /**
-   * Returns all the tapes within the CTA catalogue.
+   * Returns map from VID to logical library name.
    *
-   * @return Map from tape volume identifier to tape.
+   * @return map from VID to logical library name.
    */
-  virtual common::dataStructures::VidToTapeMap getAllTapes() const = 0;
+  virtual std::map<std::string, std::string> getVidToLogicalLibrary() const = 0;
 
   /**
    * Reclaims the specified tape.

catalogue/CatalogueRetryWrapper.hpp

@@ -316,8 +316,8 @@ public:
     return retryOnLostConnection(m_log, [&]{return m_catalogue->getTapesByVid(vids);}, m_maxTriesToConnect);
   }
 
-  common::dataStructures::VidToTapeMap getAllTapes() const override {
-    return retryOnLostConnection(m_log, [&]{return m_catalogue->getAllTapes();}, m_maxTriesToConnect);
+  std::map<std::string, std::string> getVidToLogicalLibrary() const override {
+    return retryOnLostConnection(m_log, [&]{return m_catalogue->getVidToLogicalLibrary();}, m_maxTriesToConnect);
   }
 
   void reclaimTape(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, cta::log::LogContext & lc) override {

catalogue/CatalogueTest.cpp

@@ -12960,13 +12960,13 @@ TEST_P(cta_catalogue_CatalogueTest, getTapesByVid_350_tapes) {
   }
 }
-TEST_P(cta_catalogue_CatalogueTest, getAllTapes_no_tapes) {
+TEST_P(cta_catalogue_CatalogueTest, getVidToLogicalLibrary_no_tapes) {
   using namespace cta;
-  ASSERT_TRUE(m_catalogue->getAllTapes().empty());
+  ASSERT_TRUE(m_catalogue->getVidToLogicalLibrary().empty());
 }
-TEST_P(cta_catalogue_CatalogueTest, getAllTapes_many_tapes) {
+TEST_P(cta_catalogue_CatalogueTest, getVidToLogicalLibrary_many_tapes) {
   using namespace cta;
   const bool logicalLibraryIsDisabled= false;
@@ -12991,31 +12991,17 @@ TEST_P(cta_catalogue_CatalogueTest, getAllTapes_many_tapes) {
     m_catalogue->createTape(m_admin, tape);
   }
-  const auto vidToTapeMap = m_catalogue->getAllTapes();
-  ASSERT_EQ(nbTapes, vidToTapeMap.size());
+  const auto vidToLogicalLibrary = m_catalogue->getVidToLogicalLibrary();
+  ASSERT_EQ(nbTapes, vidToLogicalLibrary.size());
   for(uint32_t i = 0; i < nbTapes; i++) {
     std::ostringstream vid;
     vid << "V" << std::setfill('0') << std::setw(5) << i;
     const std::string tapeComment = "Create tape " + vid.str();
-    const auto tapeItor = vidToTapeMap.find(vid.str());
-    ASSERT_NE(vidToTapeMap.end(), tapeItor);
-
-    ASSERT_EQ(vid.str(), tapeItor->second.vid);
-    ASSERT_EQ(m_tape1.mediaType, tapeItor->second.mediaType);
-    ASSERT_EQ(m_tape1.vendor, tapeItor->second.vendor);
-    ASSERT_EQ(m_tape1.logicalLibraryName, tapeItor->second.logicalLibraryName);
-    ASSERT_EQ(m_tape1.tapePoolName, tapeItor->second.tapePoolName);
-    ASSERT_EQ(m_vo.name, tapeItor->second.vo);
-    ASSERT_EQ(m_mediaType.capacityInBytes, tapeItor->second.capacityInBytes);
-    ASSERT_EQ(m_tape1.disabled, tapeItor->second.disabled);
-    ASSERT_EQ(m_tape1.full, tapeItor->second.full);
-    ASSERT_EQ(m_tape1.readOnly, tapeItor->second.readOnly);
-    ASSERT_FALSE(tapeItor->second.isFromCastor);
-    ASSERT_EQ(0, tapeItor->second.readMountCount);
-    ASSERT_EQ(0, tapeItor->second.writeMountCount);
-    ASSERT_EQ(m_tape1.comment, tapeItor->second.comment);
+    const auto tapeItor = vidToLogicalLibrary.find(vid.str());
+    ASSERT_NE(vidToLogicalLibrary.end(), tapeItor);
+    ASSERT_EQ(m_tape1.logicalLibraryName, tapeItor->second);
   }
 }

catalogue/DummyCatalogue.hpp

@@ -96,7 +96,7 @@ public:
   std::list<TapePool> getTapePools() const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   std::list<common::dataStructures::Tape> getTapes(const TapeSearchCriteria& searchCriteria) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
  // getTapesByVid is implemented below (and works).
-  common::dataStructures::VidToTapeMap getAllTapes() const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+  std::map<std::string, std::string> getVidToLogicalLibrary() const override { throw exception::Exception(std::string("In ") + __PRETTY_FUNCTION__ + ": not implemented"); }
   std::list<TapeForWriting> getTapesForWriting(const std::string& logicalLibraryName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   bool isAdmin(const common::dataStructures::SecurityIdentity& admin) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   void modifyActivitiesFairShareWeight(const common::dataStructures::SecurityIdentity& admin, const std::string& diskInstanceName, const std::string& acttivity, double weight, const std::string & comment) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }

catalogue/RdbmsCatalogue.cpp

@@ -3696,102 +3696,28 @@ void RdbmsCatalogue::executeGetTapesBy100VidsStmtAndCollectResults(rdbms::Stmt &
 }
 
 //------------------------------------------------------------------------------
-// getAllTapes
+// getVidToLogicalLibrary
 //------------------------------------------------------------------------------
-common::dataStructures::VidToTapeMap RdbmsCatalogue::getAllTapes() const {
+std::map<std::string, std::string> RdbmsCatalogue::getVidToLogicalLibrary() const {
   try {
-    common::dataStructures::VidToTapeMap vidToTapeMap;
+    std::map<std::string, std::string> vidToLogicalLibrary;
     std::string sql =
-      "SELECT "
-        "TAPE.VID AS VID,"
-        "MEDIA_TYPE.MEDIA_TYPE_NAME AS MEDIA_TYPE,"
-        "TAPE.VENDOR AS VENDOR,"
-        "LOGICAL_LIBRARY.LOGICAL_LIBRARY_NAME AS LOGICAL_LIBRARY_NAME,"
-        "TAPE_POOL.TAPE_POOL_NAME AS TAPE_POOL_NAME,"
-        "VIRTUAL_ORGANIZATION.VIRTUAL_ORGANIZATION_NAME AS VO,"
-        "TAPE.ENCRYPTION_KEY_NAME AS ENCRYPTION_KEY_NAME,"
-        "MEDIA_TYPE.CAPACITY_IN_BYTES AS CAPACITY_IN_BYTES,"
-        "TAPE.DATA_IN_BYTES AS DATA_IN_BYTES,"
-        "TAPE.LAST_FSEQ AS LAST_FSEQ,"
-        "TAPE.IS_DISABLED AS IS_DISABLED,"
-        "TAPE.IS_FULL AS IS_FULL,"
-        "TAPE.IS_READ_ONLY AS IS_READ_ONLY,"
-        "TAPE.IS_FROM_CASTOR AS IS_FROM_CASTOR,"    
-
-        "TAPE.LABEL_DRIVE AS LABEL_DRIVE,"
-        "TAPE.LABEL_TIME AS LABEL_TIME,"
-
-        "TAPE.LAST_READ_DRIVE AS LAST_READ_DRIVE,"
-        "TAPE.LAST_READ_TIME AS LAST_READ_TIME,"
-
-        "TAPE.LAST_WRITE_DRIVE AS LAST_WRITE_DRIVE,"
-        "TAPE.LAST_WRITE_TIME AS LAST_WRITE_TIME,"
-                            
-        "TAPE.READ_MOUNT_COUNT AS READ_MOUNT_COUNT,"
-        "TAPE.WRITE_MOUNT_COUNT AS WRITE_MOUNT_COUNT,"
-
-        "TAPE.USER_COMMENT AS USER_COMMENT,"
-
-        "TAPE.CREATION_LOG_USER_NAME AS CREATION_LOG_USER_NAME,"
-        "TAPE.CREATION_LOG_HOST_NAME AS CREATION_LOG_HOST_NAME,"
-        "TAPE.CREATION_LOG_TIME AS CREATION_LOG_TIME,"
-
-        "TAPE.LAST_UPDATE_USER_NAME AS LAST_UPDATE_USER_NAME,"
-        "TAPE.LAST_UPDATE_HOST_NAME AS LAST_UPDATE_HOST_NAME,"
-        "TAPE.LAST_UPDATE_TIME AS LAST_UPDATE_TIME "
-      "FROM "
-        "TAPE "
-      "INNER JOIN TAPE_POOL ON "
-        "TAPE.TAPE_POOL_ID = TAPE_POOL.TAPE_POOL_ID "
-      "INNER JOIN LOGICAL_LIBRARY ON "
-        "TAPE.LOGICAL_LIBRARY_ID = LOGICAL_LIBRARY.LOGICAL_LIBRARY_ID "
-      "INNER JOIN MEDIA_TYPE ON "
-        "TAPE.MEDIA_TYPE_ID = MEDIA_TYPE.MEDIA_TYPE_ID "
-      "INNER JOIN VIRTUAL_ORGANIZATION ON "
-        "TAPE_POOL.VIRTUAL_ORGANIZATION_ID = VIRTUAL_ORGANIZATION.VIRTUAL_ORGANIZATION_ID";
+      "SELECT"                                                         "\n"
+        "TAPE.VID AS VID,"                                             "\n"
+        "LOGICAL_LIBRARY.LOGICAL_LIBRARY_NAME AS LOGICAL_LIBRARY_NAME" "\n"
+      "FROM"                                                           "\n"
+        "TAPE"                                                         "\n"
+      "INNER JOIN LOGICAL_LIBRARY ON"                                  "\n"
+        "TAPE.LOGICAL_LIBRARY_ID = LOGICAL_LIBRARY.LOGICAL_LIBRARY_ID";
 
     auto conn = m_connPool.getConn();
     auto stmt = conn.createStmt(sql);
-
     auto rset = stmt.executeQuery();
     while (rset.next()) {
-      common::dataStructures::Tape tape;
-
-      tape.vid = rset.columnString("VID");
-      tape.mediaType = rset.columnString("MEDIA_TYPE");
-      tape.vendor = rset.columnString("VENDOR");
-      tape.logicalLibraryName = rset.columnString("LOGICAL_LIBRARY_NAME");
-      tape.tapePoolName = rset.columnString("TAPE_POOL_NAME");
-      tape.vo = rset.columnString("VO");
-      tape.encryptionKeyName = rset.columnOptionalString("ENCRYPTION_KEY_NAME");
-      tape.capacityInBytes = rset.columnUint64("CAPACITY_IN_BYTES");
-      tape.dataOnTapeInBytes = rset.columnUint64("DATA_IN_BYTES");
-      tape.lastFSeq = rset.columnUint64("LAST_FSEQ");
-      tape.disabled = rset.columnBool("IS_DISABLED");
-      tape.full = rset.columnBool("IS_FULL");
-      tape.readOnly = rset.columnBool("IS_READ_ONLY");
-      tape.isFromCastor = rset.columnBool("IS_FROM_CASTOR");
-
-      tape.labelLog = getTapeLogFromRset(rset, "LABEL_DRIVE", "LABEL_TIME");
-      tape.lastReadLog = getTapeLogFromRset(rset, "LAST_READ_DRIVE", "LAST_READ_TIME");
-      tape.lastWriteLog = getTapeLogFromRset(rset, "LAST_WRITE_DRIVE", "LAST_WRITE_TIME");
-      
-      tape.readMountCount = rset.columnUint64("READ_MOUNT_COUNT");
-      tape.writeMountCount = rset.columnUint64("WRITE_MOUNT_COUNT");
-
-      auto optionalComment = rset.columnOptionalString("USER_COMMENT");
-      tape.comment = optionalComment ? optionalComment.value() : "";
-      tape.creationLog.username = rset.columnString("CREATION_LOG_USER_NAME");
-      tape.creationLog.host = rset.columnString("CREATION_LOG_HOST_NAME");
-      tape.creationLog.time = rset.columnUint64("CREATION_LOG_TIME");
-      tape.lastModificationLog.username = rset.columnString("LAST_UPDATE_USER_NAME");
-      tape.lastModificationLog.host = rset.columnString("LAST_UPDATE_HOST_NAME");
-      tape.lastModificationLog.time = rset.columnUint64("LAST_UPDATE_TIME");
-
-      vidToTapeMap[tape.vid] = tape;
+      vidToLogicalLibrary[rset.columnString("VID")] = rset.columnString("LOGICAL_LIBRARY_NAME");
     }
 
-    return vidToTapeMap;
+    return vidToLogicalLibrary;
   } catch(exception::UserError &) {
     throw;
   } catch(exception::Exception &ex) {

catalogue/RdbmsCatalogue.hpp

@@ -496,11 +496,11 @@ public:
   common::dataStructures::VidToTapeMap getTapesByVid(const std::set<std::string> &vids) const override;
 
   /**
-   * Returns all the tapes within the CTA catalogue.
+   * Returns map from VID to logical library name.
    *
-   * @return Map from tape volume identifier to tape.
+   * @return map from VID to logical library name.
    */
-  common::dataStructures::VidToTapeMap getAllTapes() const override;
+  std::map<std::string, std::string> getVidToLogicalLibrary() const override;
 
   /**
    * Reclaims the specified tape.

common/dataStructures/QueueAndMountSummary.cpp

@@ -27,7 +27,7 @@ namespace dataStructures {
 
 QueueAndMountSummary &QueueAndMountSummary::getOrCreateEntry(std::list<QueueAndMountSummary> &summaryList,
     MountType mountType, const std::string &tapePool, const std::string &vid,
-    const common::dataStructures::VidToTapeMap &vid_to_tapeinfo)
+    const std::map<std::string, std::string> &vid_to_logical_library)
 {
   for (auto & summary: summaryList) {
     if((summary.tapePool == tapePool && summary.mountType == mountType && (getMountBasicType(mountType) == MountType::ArchiveAllTypes)) || (summary.vid == vid && mountType == MountType::Retrieve)) {
@@ -43,7 +43,7 @@ QueueAndMountSummary &QueueAndMountSummary::getOrCreateEntry(std::list<QueueAndM
       summaryList.back().logicalLibrary="-";
     } else {
       summaryList.back().vid=vid;
-      summaryList.back().logicalLibrary=vid_to_tapeinfo.at(vid).logicalLibraryName;
+      summaryList.back().logicalLibrary=vid_to_logical_library.at(vid);
     }
     return summaryList.back();
   }

common/dataStructures/QueueAndMountSummary.hpp

@@ -61,7 +61,7 @@ struct QueueAndMountSummary {
   
   static QueueAndMountSummary &getOrCreateEntry(std::list<QueueAndMountSummary> &summaryList,
     MountType mountType, const std::string &tapePool, const std::string &vid,
-    const common::dataStructures::VidToTapeMap &vid_to_tapeinfo);
+    const std::map<std::string, std::string> &vid_to_logical_library);
 };
 
 }}} //namespace cta::common::dataStructures

scheduler/Scheduler.cpp

@@ -1566,7 +1566,7 @@ std::list<SchedulingInfos> Scheduler::getSchedulingInformations(log::LogContext&
 std::list<common::dataStructures::QueueAndMountSummary> Scheduler::getQueuesAndMountSummaries(log::LogContext& lc) {
   std::list<common::dataStructures::QueueAndMountSummary> ret;
   // Obtain a map of vids to tape info from the catalogue
-  auto vid_to_tapeinfo = m_catalogue.getAllTapes();
+  auto vid_to_tapeinfo = m_catalogue.getVidToLogicalLibrary();
 
   // Extract relevant information from the object store.
   utils::Timer t;