cta-statistics-update tool is now working

catalogue/Catalogue.hpp

@@ -464,6 +464,9 @@ public:
   virtual void setTapeIsFromCastorInUnitTests(const std::string &vid) = 0;
 
   virtual void setTapeDisabled(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const bool disabledValue) = 0;
+  
+  virtual void setTapeDirty(const std::string & vid) = 0;
+  
   virtual void modifyTapeComment(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &comment) = 0;
 
   virtual void modifyRequesterMountRulePolicy(const common::dataStructures::SecurityIdentity &admin, const std::string &instanceName, const std::string &requesterName, const std::string &mountPolicy) = 0;

catalogue/CatalogueRetryWrapper.hpp

@@ -299,6 +299,10 @@ public:
   void setTapeDisabled(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const bool disabledValue) override {
     return retryOnLostConnection(m_log, [&]{return m_catalogue->setTapeDisabled(admin, vid, disabledValue);}, m_maxTriesToConnect);
   }
+  
+  void setTapeDirty(const std::string & vid) override {
+    return retryOnLostConnection(m_log,[&]{ return m_catalogue->setTapeDirty(vid);}, m_maxTriesToConnect);
+  }
 
   void modifyTapeComment(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &comment) override {
     return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyTapeComment(admin, vid, comment);}, m_maxTriesToConnect);

catalogue/DummyCatalogue.hpp

@@ -130,6 +130,7 @@ public:
   uint64_t getNbFilesOnTape(const std::string& vid) const  override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   void setTapeDisabled(const common::dataStructures::SecurityIdentity& admin, const std::string& vid, const bool disabledValue) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   void setTapeFull(const common::dataStructures::SecurityIdentity& admin, const std::string& vid, const bool fullValue) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
+  void setTapeDirty(const std::string & vid) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   void setTapeReadOnly(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const bool readOnlyValue) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   void setTapeReadOnlyOnError(const std::string &vid) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }
   void setTapeIsFromCastorInUnitTests(const std::string &vid) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); }

catalogue/MysqlCatalogue.cpp

@@ -428,6 +428,7 @@ void MysqlCatalogue::deleteArchiveFile(const std::string &diskInstanceName, cons
     rdbms::Rset selectRset = selectStmt.executeQuery();
     const auto selectFromArchiveFileTime = t.secs();
     std::unique_ptr<common::dataStructures::ArchiveFile> archiveFile;
+    std::set<std::string> vidsToSetDirty;
     while(selectRset.next()) {
       if(nullptr == archiveFile.get()) {
         archiveFile = cta::make_unique<common::dataStructures::ArchiveFile>();
@@ -450,6 +451,7 @@ void MysqlCatalogue::deleteArchiveFile(const std::string &diskInstanceName, cons
         // Add the tape file to the archive file's in-memory structure
         common::dataStructures::TapeFile tapeFile;
         tapeFile.vid = selectRset.columnString("VID");
+        vidsToSetDirty.insert(tapeFile.vid);
         tapeFile.fSeq = selectRset.columnUint64("FSEQ");
         tapeFile.blockId = selectRset.columnUint64("BLOCK_ID");
         tapeFile.fileSize = selectRset.columnUint64("LOGICAL_SIZE_IN_BYTES");
@@ -525,9 +527,9 @@ void MysqlCatalogue::deleteArchiveFile(const std::string &diskInstanceName, cons
       stmt.executeNonQuery();
     }
 
-    for(auto &tapeFile: archiveFile->tapeFiles){
+    for(auto &vidToSetDirty: vidsToSetDirty){
       //We deleted the TAPE_FILE so the tapes containing them should be set as dirty
-      setTapeDirty(conn,tapeFile.vid);
+      setTapeDirty(conn,vidToSetDirty);
     }
     
     const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);

catalogue/OracleCatalogue.cpp

@@ -776,6 +776,7 @@ void OracleCatalogue::deleteArchiveFile(const std::string &diskInstanceName, con
     rdbms::Rset selectRset = selectStmt.executeQuery();
     const auto selectFromArchiveFileTime = t.secs();
     std::unique_ptr<common::dataStructures::ArchiveFile> archiveFile;
+    std::set<std::string> vidsToSetDirty;
     while(selectRset.next()) {
       if(nullptr == archiveFile.get()) {
         archiveFile = cta::make_unique<common::dataStructures::ArchiveFile>();
@@ -798,6 +799,7 @@ void OracleCatalogue::deleteArchiveFile(const std::string &diskInstanceName, con
         // Add the tape file to the archive file's in-memory structure
         common::dataStructures::TapeFile tapeFile;
         tapeFile.vid = selectRset.columnString("VID");
+        vidsToSetDirty.insert(tapeFile.vid);
         tapeFile.fSeq = selectRset.columnUint64("FSEQ");
         tapeFile.blockId = selectRset.columnUint64("BLOCK_ID");
         tapeFile.fileSize = selectRset.columnUint64("LOGICAL_SIZE_IN_BYTES");
@@ -870,9 +872,9 @@ void OracleCatalogue::deleteArchiveFile(const std::string &diskInstanceName, con
       stmt.executeNonQuery();
     }
     
-    for(auto &tapeFile: archiveFile->tapeFiles){
-      //We deleted the TAPE_FILE so the tapes containing them should be set as dirty
-      setTapeDirty(conn,tapeFile.vid);
+    //Set the tapes where the files have been deleted to dirty
+    for(auto &vidToSetDirty: vidsToSetDirty){
+      setTapeDirty(conn,vidToSetDirty);
     }
     
     const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);

catalogue/PostgresCatalogue.cpp

@@ -714,6 +714,7 @@ void PostgresCatalogue::deleteArchiveFile(const std::string &diskInstanceName, c
     rdbms::Rset selectRset = selectStmt.executeQuery();
     const auto selectFromArchiveFileTime = t.secs();
     std::unique_ptr<common::dataStructures::ArchiveFile> archiveFile;
+    std::set<std::string> vidsToSetDirty;
     while(selectRset.next()) {
       if(nullptr == archiveFile.get()) {
         archiveFile = cta::make_unique<common::dataStructures::ArchiveFile>();
@@ -736,6 +737,7 @@ void PostgresCatalogue::deleteArchiveFile(const std::string &diskInstanceName, c
         // Add the tape file to the archive file's in-memory structure
         common::dataStructures::TapeFile tapeFile;
         tapeFile.vid = selectRset.columnString("VID");
+        vidsToSetDirty.insert(tapeFile.vid);
         tapeFile.fSeq = selectRset.columnUint64("FSEQ");
         tapeFile.blockId = selectRset.columnUint64("BLOCK_ID");
         tapeFile.fileSize = selectRset.columnUint64("LOGICAL_SIZE_IN_BYTES");
@@ -809,9 +811,9 @@ void PostgresCatalogue::deleteArchiveFile(const std::string &diskInstanceName, c
       stmt.executeNonQuery();
     }
     
-    for(auto &tapeFile: archiveFile->tapeFiles){
+    for(auto &vidToSetDirty: vidsToSetDirty){
       //We deleted the TAPE_FILE so the tapes containing them should be set as dirty
-      setTapeDirty(conn,tapeFile.vid);
+      setTapeDirty(conn,vidToSetDirty);
     }
     
     const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);

catalogue/RdbmsCatalogue.cpp

@@ -3588,6 +3588,17 @@ void RdbmsCatalogue::setTapeDisabled(const common::dataStructures::SecurityIdent
   }
 }
 
+void RdbmsCatalogue::setTapeDirty(const std::string& vid) {
+  try {
+    auto conn = m_connPool.getConn();
+    setTapeDirty(conn,vid);
+  } catch(exception::Exception &ex) {
+    ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+    throw;
+  }
+}
+
+
 //------------------------------------------------------------------------------
 // modifyTapeComment
 //------------------------------------------------------------------------------

catalogue/RdbmsCatalogue.hpp

@@ -487,6 +487,7 @@ public:
   void setTapeIsFromCastorInUnitTests(const std::string &vid) override;
   
   void setTapeDisabled(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const bool disabledValue) override;
+  void setTapeDirty(const std::string & vid) override;
   void modifyTapeComment(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::string &comment) override;
 
   void modifyRequesterMountRulePolicy(const common::dataStructures::SecurityIdentity &admin, const std::string &instanceName, const std::string &requesterName, const std::string &mountPolicy) override;

catalogue/SqliteCatalogue.cpp

@@ -128,9 +128,15 @@ void SqliteCatalogue::deleteArchiveFile(const std::string &diskInstanceName, con
       stmt.executeNonQuery();
     }
     
+    std::set<std::string> vidsToSetDirty;
+    //We will insert the vids to set dirty in a set so that
+    //we limit the calls to setTapeDirty to the number of tapes that contained the deleted tape files 
     for(auto &tapeFile: archiveFile->tapeFiles){
-      //We deleted the TAPE_FILE so the tapes containing them should be set as dirty
-      setTapeDirty(conn,tapeFile.vid);
+      vidsToSetDirty.insert(tapeFile.vid);
+    }
+    
+    for(auto &vidToSetDirty: vidsToSetDirty){
+       setTapeDirty(conn,vidToSetDirty);
     }
     
     const auto deleteFromTapeFileTime = t.secs(utils::Timer::resetCounter);

scheduler/ArchiveMount.cpp

@@ -208,7 +208,7 @@ void cta::ArchiveMount::reportJobsBatchTransferred(std::queue<std::unique_ptr<ct
       validatedSuccessfulDBArchiveJobs.emplace_back(std::move(schJob->m_dbJob));
     }
     validatedSuccessfulArchiveJobs.clear();
-    
+
     updateCatalogueWithTapeFilesWritten(tapeItemsWritten);
     catalogueTime=t.secs(utils::Timer::resetCounter);
     {

scheduler/OStoreDB/OStoreDB.cpp

@@ -4645,6 +4645,9 @@ objectstore::ArchiveRequest::RepackInfo OStoreDB::ArchiveJob::getRepackInfoAfter
 // OStoreDB::RepackArchiveSuccessesReportBatch::report()
 //------------------------------------------------------------------------------
 void OStoreDB::RepackArchiveSuccessesReportBatch::report(log::LogContext& lc) {
+  objectstore::RepackRequest req(m_repackRequest.getAddressIfSet(),this->m_oStoreDb.m_objectStore);
+  req.fetchNoLock();
+  this->m_oStoreDb.m_catalogue.setTapeDirty(req.getInfo().vid);
   OStoreDB::RepackArchiveReportBatch::report(lc);
 }
 

statistics/CMakeLists.txt

@@ -24,6 +24,7 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow")
 
 set (STATISTICS_LIB_SRC_FILES
   StatisticsSchema.cpp
+  TapeStatisticsUpdater.cpp
 )
 
 add_library (ctastatistics SHARED

statistics/StatisticsUpdateCmd.cpp

@@ -21,6 +21,8 @@
 #include "statistics/StatisticsUpdateCmd.hpp"
 #include "catalogue/SchemaChecker.hpp"
 #include "statistics/StatisticsUpdateCmdLineArgs.hpp"
+#include "common/Timer.hpp"
+#include "TapeStatisticsUpdater.hpp"
 
 namespace cta {
 namespace statistics {
@@ -55,10 +57,6 @@ int StatisticsUpdateCmd::exceptionThrowingMain(const int argc, char *const *cons
   auto loginCatalogue = rdbms::Login::parseFile(cmdLineArgs.catalogueDbConfigPath);
   rdbms::ConnPool catalogueConnPool(loginCatalogue, maxNbConns);
   auto catalogueConn = catalogueConnPool.getConn();
-  
-  /*auto loginStatistics = rdbms::Login::parseFile(cmdLineArgs.statisticsDbConfigPath);
-  rdbms::ConnPool statisticsConnPool(loginStatistics, maxNbConns);
-  auto statisticsConn = statisticsConnPool.getConn();*/
 
   SchemaChecker::Builder catalogueCheckerBuilder("catalogue",loginCatalogue.dbType,catalogueConn);
   std::unique_ptr<cta::catalogue::SchemaChecker> catalogueChecker;
@@ -71,14 +69,12 @@ int StatisticsUpdateCmd::exceptionThrowingMain(const int argc, char *const *cons
   if(tapeTableStatus == SchemaChecker::Status::FAILURE || tapeFileTableStatus == SchemaChecker::Status::FAILURE || archiveFileTableStatus == SchemaChecker::Status::FAILURE){
     return EXIT_FAILURE;
   }
- 
-  /*SchemaChecker::Builder statisticsCheckerBuilder("statistics",loginStatistics.dbType,statisticsConn);
-  cta::statistics::MysqlStatisticsSchema mysqlSchema;
-  std::unique_ptr<SchemaChecker> statisticsChecker = 
-  statisticsCheckerBuilder.useCppSchemaStatementsReader(mysqlSchema)
-                          .useSQLiteSchemaComparer()
-                          .build();
-  statisticsChecker->compareTablesLocatedInSchema();*/
+  
+  std::unique_ptr<TapeStatisticsUpdater> updater = TapeStatisticsUpdaterFactory::create(loginCatalogue.dbType,catalogueConn);
+  std::cout<<"Updating tape statistics in the catalogue..."<<std::endl;
+  cta::utils::Timer t;
+  updater->updateTapeStatistics();
+  std::cout<<"Updated catalogue tape statistics in "<<t.secs()<<", "<<updater->getNbUpdatedTapes()<<" tape(s) have been updated"<<std::endl;  
   
   return EXIT_SUCCESS;
 }

statistics/TapeStatisticsUpdater.cpp

+/**
+ * The CERN Tape Archive (CTA) project
+ * Copyright © 2018 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "TapeStatisticsUpdater.hpp"
+#include "rdbms/Login.hpp"
+
+namespace cta { namespace statistics {
+  
+TapeStatisticsUpdater::TapeStatisticsUpdater(cta::rdbms::Conn &conn):m_conn(conn) {
+}
+
+TapeStatisticsUpdater::~TapeStatisticsUpdater() {
+}
+
+void TapeStatisticsUpdater::updateTapeStatistics() {
+  const char * const sql = 
+  "UPDATE TAPE TAPE_TO_UPDATE SET"
+  "("
+    "DIRTY,"
+    "NB_MASTER_FILES,"
+    "MASTER_DATA_IN_BYTES"
+  ")"
+  " = "
+  "("
+    "SELECT "
+      "'0' AS DIRTY,"
+      "COALESCE(NON_EMPTY_TAPE_STATS.NB_MASTER_FILES, 0) AS NB_MASTER_FILES,"
+      "COALESCE(NON_EMPTY_TAPE_STATS.MASTER_DATA_IN_BYTES, 0) AS MASTER_DATA_IN_BYTES "
+    "FROM "
+      "TAPE T "
+    "LEFT OUTER JOIN "
+    "("
+      "SELECT "
+        "TAPE_FILE.VID,"
+        "'0' AS DIRTY,"
+        "COUNT(TAPE_FILE.FSEQ) AS NB_MASTER_FILES,"
+        "SUM(ARCHIVE_FILE.SIZE_IN_BYTES) AS MASTER_DATA_IN_BYTES "
+      "FROM "
+        "TAPE_FILE "
+      "INNER JOIN ARCHIVE_FILE ON "
+        "TAPE_FILE.ARCHIVE_FILE_ID = ARCHIVE_FILE.ARCHIVE_FILE_ID "
+      "WHERE "
+        "TAPE_FILE.SUPERSEDED_BY_VID IS NULL AND "
+        "TAPE_FILE.SUPERSEDED_BY_FSEQ IS NULL "
+      "GROUP BY TAPE_FILE.VID"
+    ") NON_EMPTY_TAPE_STATS ON "
+      "T.VID = NON_EMPTY_TAPE_STATS.VID "
+    "WHERE T.VID = TAPE_TO_UPDATE.VID"
+  ") "
+  "WHERE TAPE_TO_UPDATE.DIRTY='1'";
+  try {
+    auto stmt = m_conn.createStmt(sql);
+    stmt.executeNonQuery();
+    m_nbUpdatedTapes = stmt.getNbAffectedRows();
+  } catch(cta::exception::Exception &ex) {
+    ex.getMessage().str(std::string(__PRETTY_FUNCTION__) + ": " + ex.getMessage().str());
+    throw;
+  }
+}
+
+uint64_t TapeStatisticsUpdater::getNbUpdatedTapes() {
+  return m_nbUpdatedTapes;
+}
+
+std::unique_ptr<TapeStatisticsUpdater> TapeStatisticsUpdaterFactory::create(cta::rdbms::Login::DbType dbType, cta::rdbms::Conn& conn){
+  typedef cta::rdbms::Login::DbType DbType;
+  std::unique_ptr<TapeStatisticsUpdater> ret;
+  switch(dbType){
+    case DbType::DBTYPE_IN_MEMORY:
+    case DbType::DBTYPE_SQLITE:
+    case DbType::DBTYPE_MYSQL:
+      throw cta::exception::Exception("In TapeStatisticsUpdaterFactory::create(), the "+cta::rdbms::Login::dbTypeToString(dbType)+" database type is not supported.");
+    case DbType::DBTYPE_ORACLE:
+    case DbType::DBTYPE_POSTGRESQL:
+      ret.reset(new TapeStatisticsUpdater(conn));
+      return std::move(ret);
+    default:
+      throw cta::exception::Exception("In TapeStatisticsUpdaterFactory::create(), unknown database type.");
+  }
+}
+
+
+}}
+

statistics/TapeStatisticsUpdater.hpp

+/**
+ * The CERN Tape Archive (CTA) project
+ * Copyright © 2018 CERN
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "rdbms/Conn.hpp"
+#include "rdbms/Login.hpp"
+
+namespace cta { namespace statistics {
+
+class TapeStatisticsUpdater {
+public:
+  TapeStatisticsUpdater(cta::rdbms::Conn &conn);
+  virtual void updateTapeStatistics();
+  virtual uint64_t getNbUpdatedTapes();
+  virtual ~TapeStatisticsUpdater();
+protected:
+  cta::rdbms::Conn &m_conn;
+  uint64_t m_nbUpdatedTapes = 0;
+};
+
+//TODO if necessary
+class MySQLStatisticsUpdater: public TapeStatisticsUpdater {
+public:
+  MySQLStatisticsUpdater(cta::rdbms::Conn &conn);
+  virtual ~MySQLStatisticsUpdater();
+  void updateTapeStatistics() override;
+};
+
+class TapeStatisticsUpdaterFactory {
+public:
+  static std::unique_ptr<TapeStatisticsUpdater> create(cta::rdbms::Login::DbType dbType, cta::rdbms::Conn &conn);
+};
+
+}}
\ No newline at end of file