Modified statistics schema structure + store the statistics in the

statistics database

statistics/DatabaseStatisticsService.cpp

@@ -27,7 +27,7 @@ DatabaseStatisticsService::DatabaseStatisticsService(cta::rdbms::Conn & database
 DatabaseStatisticsService::~DatabaseStatisticsService() {
 }
 
-void DatabaseStatisticsService::update(){
+void DatabaseStatisticsService::updateStatistics(){
   const char * const sql = 
   "UPDATE TAPE TAPE_TO_UPDATE SET"
   "("
@@ -92,11 +92,101 @@ void DatabaseStatisticsService::update(){
   }
 }
 
-void DatabaseStatisticsService::save(const cta::statistics::Statistics& statistics) {
+void DatabaseStatisticsService::saveStatistics(const cta::statistics::Statistics& statistics) {
+  //First we save the general FILE statistics, then we go for the per-vo statistics
+  saveFileStatistics(statistics);
+  saveStatisticsPerVo(statistics);
+}
 
+void DatabaseStatisticsService::saveFileStatistics(const cta::statistics::Statistics& statistics) {
+  try {
+    const time_t now = time(nullptr);
+    const char * const sql = 
+    "INSERT INTO "
+      "FILE_STATISTICS "
+      "("
+        "NB_MASTER_FILES,"
+        "MASTER_DATA_IN_BYTES,"
+        "NB_COPY_NB_1,"
+        "NB_COPY_NB_1_IN_BYTES,"
+        "NB_COPY_NB_GT_1,"
+        "NB_COPY_NB_GT_1_IN_BYTES,"
+        "UPDATE_TIME"
+    ") "
+    "VALUES "
+    "("
+      ":NB_MASTER_FILES,"
+      ":MASTER_DATA_IN_BYTES,"
+      ":NB_COPY_NB_1,"
+      ":NB_COPY_NB_1_IN_BYTES,"
+      ":NB_COPY_NB_GT_1,"
+      ":NB_COPY_NB_GT_1_IN_BYTES,"
+      ":UPDATE_TIME"
+    ")";
+    auto stmt = m_conn.createStmt(sql);
+    stmt.bindUint64(":NB_MASTER_FILES",statistics.getTotalFiles());
+    stmt.bindUint64(":MASTER_DATA_IN_BYTES",statistics.getTotalBytes());
+    stmt.bindUint64(":NB_COPY_NB_1",statistics.getTotalFilesCopyNb1());
+    stmt.bindUint64(":NB_COPY_NB_1_IN_BYTES",statistics.getTotalBytesCopyNb1());
+    stmt.bindUint64(":NB_COPY_NB_GT_1",statistics.getTotalFilesCopyNbGt1());
+    stmt.bindUint64(":NB_COPY_NB_GT_1_IN_BYTES",statistics.getTotalBytesCopyNbGt1());
+    stmt.bindUint64(":UPDATE_TIME",now);
+    stmt.executeNonQuery();
+  } catch(exception::Exception &ex) {
+    ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+    throw;
+  }
 }
 
-std::unique_ptr<cta::statistics::Statistics> DatabaseStatisticsService::get() {
+void DatabaseStatisticsService::saveStatisticsPerVo(const cta::statistics::Statistics& statistics) {
+  try {
+    const time_t now = time(nullptr);
+    const char * const sql = 
+    "INSERT INTO "
+      "VO_STATISTICS "
+      "("
+        "VO,"
+        "NB_MASTER_FILES,"
+        "MASTER_DATA_IN_BYTES,"
+        "NB_COPY_NB_1,"
+        "NB_COPY_NB_1_IN_BYTES,"
+        "NB_COPY_NB_GT_1,"
+        "NB_COPY_NB_GT_1_IN_BYTES,"
+        "UPDATE_TIME"
+      ") "
+      "VALUES "
+      "("
+        ":VO,"
+        ":NB_MASTER_FILES,"
+        ":MASTER_DATA_IN_BYTES,"
+        ":NB_COPY_NB_1,"
+        ":NB_COPY_NB_1_IN_BYTES,"
+        ":NB_COPY_NB_GT_1,"
+        ":NB_COPY_NB_GT_1_IN_BYTES,"
+        ":UPDATE_TIME"
+      ")";
+    for(const auto & stat: statistics.getAllStatistics()){
+      auto stmt = m_conn.createStmt(sql);
+      auto voFileStatistics = stat.second;
+      stmt.bindString(":VO",stat.first);
+      stmt.bindUint64(":NB_MASTER_FILES",voFileStatistics.nbMasterFiles);
+      stmt.bindUint64(":MASTER_DATA_IN_BYTES",voFileStatistics.nbMasterFiles);
+      stmt.bindUint64(":NB_COPY_NB_1",voFileStatistics.nbCopyNb1);
+      stmt.bindUint64(":NB_COPY_NB_1_IN_BYTES",voFileStatistics.copyNb1InBytes);
+      stmt.bindUint64(":NB_COPY_NB_GT_1",voFileStatistics.nbCopyNbGt1);
+      stmt.bindUint64(":NB_COPY_NB_GT_1_IN_BYTES",voFileStatistics.copyNbGt1InBytes);
+      stmt.bindUint64(":UPDATE_TIME",now);
+      stmt.executeNonQuery();
+    }
+  } catch(exception::Exception &ex) {
+    ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str());
+    throw;
+  }
+}
+
+
+
+std::unique_ptr<cta::statistics::Statistics> DatabaseStatisticsService::getStatistics() {
   const char * const sql = 
   "SELECT "
     "VIRTUAL_ORGANIZATION_NAME AS VO,"

statistics/DatabaseStatisticsService.hpp

@@ -32,11 +32,13 @@ public:
   DatabaseStatisticsService & operator=(const DatabaseStatisticsService &) = delete;
   virtual ~DatabaseStatisticsService();
   
-  virtual void update() override;
-  virtual void save(const cta::statistics::Statistics &statistics) override;
-  virtual std::unique_ptr<cta::statistics::Statistics> get() override;
+  virtual void updateStatistics() override;
+  virtual void saveStatistics(const cta::statistics::Statistics &statistics) override;
+  virtual std::unique_ptr<cta::statistics::Statistics> getStatistics() override;
 protected:
   cta::rdbms::Conn & m_conn;
+  virtual void saveFileStatistics(const cta::statistics::Statistics & statistics);
+  virtual void saveStatisticsPerVo(const cta::statistics::Statistics & statistics);
 };
 
 }}

statistics/FileStatistics.cpp

@@ -25,6 +25,15 @@ namespace statistics {
 FileStatistics::FileStatistics() {
 }
 
+FileStatistics::FileStatistics(const FileStatistics &other){
+  nbMasterFiles = other.nbMasterFiles;
+  masterDataInBytes = other.masterDataInBytes;
+  nbCopyNb1 = other.nbCopyNb1;
+  copyNb1InBytes = other.copyNb1InBytes;
+  nbCopyNbGt1 = other.nbCopyNbGt1;
+  copyNbGt1InBytes = other.copyNbGt1InBytes;
+}
+
 FileStatistics FileStatistics::operator +=(const FileStatistics& other) {
   nbMasterFiles += other.nbMasterFiles;
   masterDataInBytes += other.masterDataInBytes;
@@ -35,5 +44,15 @@ FileStatistics FileStatistics::operator +=(const FileStatistics& other) {
   return *this;
 }
 
+FileStatistics& FileStatistics::operator =(const FileStatistics& other){
+  nbMasterFiles = other.nbMasterFiles;
+  masterDataInBytes = other.masterDataInBytes;
+  nbCopyNb1 = other.nbCopyNb1;
+  copyNb1InBytes = other.copyNb1InBytes;
+  nbCopyNbGt1 = other.nbCopyNbGt1;
+  copyNbGt1InBytes = other.copyNbGt1InBytes;
+  return *this;
+}
+
 }}
 

statistics/FileStatistics.hpp

@@ -26,7 +26,7 @@ namespace statistics {
 struct FileStatistics {
 
   FileStatistics();
-
+  FileStatistics(const FileStatistics &other);
   uint64_t nbMasterFiles = 0;
   uint64_t masterDataInBytes = 0;
   uint64_t nbCopyNb1 = 0;
@@ -34,6 +34,8 @@ struct FileStatistics {
   uint64_t nbCopyNbGt1 = 0;
   uint64_t copyNbGt1InBytes = 0;
   
+  FileStatistics &operator=(const FileStatistics &other);
+  
   FileStatistics operator +=(const FileStatistics &other);
   
 };

statistics/MySQLStatisticsService.cpp

@@ -25,7 +25,7 @@ MySQLStatisticsService::MySQLStatisticsService(cta::rdbms::Conn & databaseConnec
 MySQLStatisticsService::~MySQLStatisticsService() {
 }
 
-void MySQLStatisticsService::update() {
+void MySQLStatisticsService::updateStatistics() {
   throw cta::exception::Exception("In MySQLStatisticsService::update() cannot update tape statistics because it is not implemented for MySQL databases");
 }
 

statistics/MySQLStatisticsService.hpp

@@ -29,7 +29,7 @@ public:
   MySQLStatisticsService & operator=(const MySQLStatisticsService &) = delete;
   virtual ~MySQLStatisticsService();
   
-  void update() override;
+  void updateStatistics() override;
 private:
 
 };

statistics/Statistics.cpp

@@ -18,6 +18,8 @@
 
 #include "Statistics.hpp"
 #include "common/make_unique.hpp"
+#include <iostream>
+
 namespace cta { 
 namespace statistics {
   
@@ -29,6 +31,10 @@ Statistics::Statistics(const Statistics& other) {
     m_statisticsPerVo = other.m_statisticsPerVo;
     m_totalFiles = other.m_totalFiles;
     m_totalBytes = other.m_totalBytes;
+    m_totalFilesCopyNb1 = other.m_totalFilesCopyNb1;
+    m_totalBytesCopyNb1 = other.m_totalBytesCopyNb1;
+    m_totalFilesCopyNbGt1 = other.m_totalFilesCopyNbGt1;
+    m_totalBytesCopyNbGt1 = other.m_totalBytesCopyNbGt1;
   }
 }
 
@@ -37,6 +43,10 @@ Statistics& Statistics::operator=(const Statistics& other) {
     m_statisticsPerVo = other.m_statisticsPerVo;
     m_totalFiles = other.m_totalFiles;
     m_totalBytes = other.m_totalBytes;
+    m_totalFilesCopyNb1 = other.m_totalFilesCopyNb1;
+    m_totalBytesCopyNb1 = other.m_totalBytesCopyNb1;
+    m_totalFilesCopyNbGt1 = other.m_totalFilesCopyNbGt1;
+    m_totalBytesCopyNbGt1 = other.m_totalBytesCopyNbGt1;
   }
   return *this;
 }
@@ -45,6 +55,10 @@ void Statistics::insertStatistics(const std::string& vo, const FileStatistics& f
   m_statisticsPerVo[vo] = fileStatistics;
   m_totalFiles += fileStatistics.nbMasterFiles;
   m_totalBytes += fileStatistics.masterDataInBytes;
+  m_totalFilesCopyNb1 += fileStatistics.nbCopyNb1;
+  m_totalBytesCopyNb1 += fileStatistics.copyNb1InBytes;
+  m_totalFilesCopyNbGt1 += fileStatistics.nbCopyNbGt1;
+  m_totalBytesCopyNbGt1 += fileStatistics.copyNbGt1InBytes;
 }
 
 const Statistics::StatisticsPerVo& Statistics::getAllStatistics() const {
@@ -59,6 +73,22 @@ uint64_t Statistics::getTotalBytes() const {
   return m_totalBytes;
 }
 
+uint64_t Statistics::getTotalFilesCopyNb1() const {
+  return m_totalFilesCopyNb1;
+}
+
+uint64_t Statistics::getTotalBytesCopyNb1() const {
+  return m_totalBytesCopyNb1;
+}
+
+uint64_t Statistics::getTotalFilesCopyNbGt1() const {
+  return m_totalFilesCopyNbGt1;
+}
+
+uint64_t Statistics::getTotalBytesCopyNbGt1() const {
+  return m_totalBytesCopyNbGt1;
+}
+
 Statistics::Builder::Builder(cta::rdbms::Rset & rset):m_rset(rset) {}
 
 std::unique_ptr<Statistics> Statistics::Builder::build(){
@@ -91,10 +121,14 @@ std::ostream & operator <<(std::ostream& stream, Statistics stats) {
     stream << "\"copyNbGt1InBytes\": " << stat.second.copyNbGt1InBytes;
     stream << "},";
   }
-  stream << "],";
-  stream << "\"totalFiles\": " << stats.getTotalFiles() << ",";
-  stream << "\"totalBytes\": " << stats.getTotalBytes();
-  stream << "}";
+  stream << "],"
+         << "\"totalFiles\": " << stats.getTotalFiles() << ","
+         << "\"totalBytes\": " << stats.getTotalBytes() << ","
+         << "\"totalFilesCopyNb1\": " << stats.getTotalFilesCopyNb1() << ","
+         << "\"totalBytesCopyNb1\": " << stats.getTotalBytesCopyNb1() << ","
+         << "\"totalFilesCopyNbGt1\": " << stats.getTotalFilesCopyNbGt1() << ","
+         << "\"totalBytesCopyNbGt1\": " << stats.getTotalBytesCopyNbGt1() << ","
+         << "}";
   return stream;
 }
 

statistics/Statistics.hpp

@@ -41,6 +41,10 @@ public:
   const StatisticsPerVo &getAllStatistics() const;
   uint64_t getTotalFiles() const;
   uint64_t getTotalBytes() const;
+  uint64_t getTotalFilesCopyNb1() const;
+  uint64_t getTotalBytesCopyNb1() const;
+  uint64_t getTotalFilesCopyNbGt1() const;
+  uint64_t getTotalBytesCopyNbGt1() const;
   
   /**
    * This builder class allows to build the statistics
@@ -61,6 +65,10 @@ private:
   StatisticsPerVo m_statisticsPerVo;
   uint64_t m_totalFiles = 0;
   uint64_t m_totalBytes = 0;
+  uint64_t m_totalFilesCopyNb1 = 0;
+  uint64_t m_totalBytesCopyNb1 = 0;
+  uint64_t m_totalFilesCopyNbGt1 = 0;
+  uint64_t m_totalBytesCopyNbGt1 = 0;
 };
 
 std::ostream &operator <<(std::ostream &stream, Statistics stats);

statistics/StatisticsSaveCmd.cpp

@@ -23,6 +23,7 @@
 #include "StatisticsSchemaFactory.hpp"
 #include "StatisticsService.hpp"
 #include "StatisticsServiceFactory.hpp"
+#include "common/utils/utils.hpp"
 
 #include <algorithm>
 
@@ -110,13 +111,24 @@ int StatisticsSaveCmd::exceptionThrowingMain(const int argc, char *const *const
   statisticsCheckerBuilder.useCppSchemaStatementsReader(*statisticsSchema)
                           .useSQLiteSchemaComparer()
                           .build();
-  statisticsChecker->compareTablesLocatedInSchema();
+  SchemaChecker::Status compareTablesStatus = statisticsChecker->compareTablesLocatedInSchema();
+  if(compareTablesStatus == SchemaChecker::Status::FAILURE){
+    return EXIT_FAILURE;
+  }
   
   //Compute the statistics
-  std::unique_ptr<StatisticsService> service = StatisticsServiceFactory::create(catalogueConn,loginCatalogue.dbType);
-  std::unique_ptr<Statistics> statistics = service->get();
-  std::cout << *statistics << std::endl;
+  std::unique_ptr<StatisticsService> catalogueStatisticsService = StatisticsServiceFactory::create(catalogueConn,loginCatalogue.dbType);
+  std::unique_ptr<Statistics> statistics = catalogueStatisticsService->getStatistics();
   //Insert them into the statistics database
+  //std::cout<<*statistics<<std::endl;
+  try {
+    std::unique_ptr<StatisticsService> statisticsStatisticsService = StatisticsServiceFactory::create(statisticsConn,loginStatistics.dbType);
+    statisticsStatisticsService->saveStatistics(*statistics);
+  } catch (cta::exception::Exception &ex) {
+    std::cerr << ex.getMessageValue() << std::endl;
+    std::cerr << "StatisticsJson:" << *statistics << std::endl;
+    return EXIT_FAILURE;
+  }
   return EXIT_SUCCESS;
 }
 
@@ -149,7 +161,24 @@ void StatisticsSaveCmd::verifyCmdLineArgs(const StatisticsSaveCmdLineArgs& cmdLi
 }
 
 void StatisticsSaveCmd::buildStatisticsDatabase(cta::rdbms::Conn& statisticsDatabaseConn, const StatisticsSchema& statisticsSchema) {
-  statisticsDatabaseConn.executeNonQuery(statisticsSchema.sql);
+  try {
+    std::string::size_type searchPos = 0;
+    std::string::size_type findResult = std::string::npos;
+    std::string sqlStmts = statisticsSchema.sql;
+
+    while(std::string::npos != (findResult = sqlStmts.find(';', searchPos))) {
+      // Calculate the length of the current statement without the trailing ';'
+      const std::string::size_type stmtLen = findResult - searchPos;
+      const std::string sqlStmt = utils::trimString(sqlStmts.substr(searchPos, stmtLen));
+      searchPos = findResult + 1;
+
+      if(0 < sqlStmt.size()) { // Ignore empty statements
+        statisticsDatabaseConn.executeNonQuery(sqlStmt);
+      }
+    }
+  } catch(exception::Exception &ex) {
+    throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str());
+  }
 }
 
 bool StatisticsSaveCmd::userConfirmDropStatisticsSchemaFromDb(const rdbms::Login &dbLogin) {

statistics/StatisticsService.hpp

@@ -30,9 +30,9 @@ public:
   StatisticsService & operator=(const StatisticsService &) = delete;
   virtual ~StatisticsService();
   
-  virtual void update() = 0;
-  virtual void save(const cta::statistics::Statistics &statistics) = 0;
-  virtual std::unique_ptr<cta::statistics::Statistics> get() = 0;
+  virtual void updateStatistics() = 0;
+  virtual void saveStatistics(const cta::statistics::Statistics &statistics) = 0;
+  virtual std::unique_ptr<cta::statistics::Statistics> getStatistics() = 0;
   uint64_t getNbUpdatedTapes();
   
 protected:

statistics/StatisticsUpdateCmd.cpp

@@ -75,7 +75,7 @@ int StatisticsUpdateCmd::exceptionThrowingMain(const int argc, char *const *cons
   std::unique_ptr<StatisticsService> service = StatisticsServiceFactory::create(catalogueConn,loginCatalogue.dbType);
   std::cout<<"Updating tape statistics in the catalogue..."<<std::endl;
   cta::utils::Timer t;
-  service->update();
+  service->updateStatistics();
   std::cout<<"Updated catalogue tape statistics in "<<t.secs()<<", "<<service->getNbUpdatedTapes()<<" tape(s) have been updated"<<std::endl; 
   
   return EXIT_SUCCESS;

statistics/common_statistics_schema.sql

-CREATE TABLE CTA_STATISTICS(
-  GID UINT32TYPE DEFAULT 0 CONSTRAINT CTA_STATISTICS_GID_NN NOT NULL,
-  TIMESTAMP UINT64TYPE DEFAULT 0 CONSTRAINT CTA_STATISTICS_TS_NN NOT NULL,
-  NB_MASTER_FILES UINT64TYPE NOT NULL,
-  MASTER_DATA_BYTES UINT64TYPE NOT NULL,
-  UPDATE_TIME UINT64TYPE NOT NULL,
-  CONSTRAINT CTA_STATISTICS_GID_TS_PK PRIMARY KEY (GID, TIMESTAMP)
+CREATE TABLE VO_STATISTICS (
+  VO VARCHAR(100) CONSTRAINT VO_STATISTICS_VO_NN NOT NULL,
+  NB_MASTER_FILES UINT64TYPE CONSTRAINT VO_STATISTICS_NMF_NN NOT NULL,
+  MASTER_DATA_IN_BYTES UINT64TYPE CONSTRAINT VO_STATISTICS_MDIB_NN NOT NULL,
+  NB_COPY_NB_1 UINT64TYPE CONSTRAINT VO_STATISTICS_NCN1_NN NOT NULL,
+  NB_COPY_NB_1_IN_BYTES UINT64TYPE CONSTRAINT VO_STATISTICS_CN1IB_NN NOT NULL,
+  NB_COPY_NB_GT_1 UINT64TYPE CONSTRAINT VO_STATISTICS_NCNG1_NN NOT NULL,
+  NB_COPY_NB_GT_1_IN_BYTES UINT64TYPE CONSTRAINT VO_STATISTICS_NCNG1IB_NN NOT NULL,
+  UPDATE_TIME UINT64TYPE CONSTRAINT VO_STATISTICS_UT_NN NOT NULL,
+  CONSTRAINT VO_STATISTICS_VU_PK PRIMARY KEY(VO, UPDATE_TIME)
+);
+CREATE TABLE FILE_STATISTICS (
+  NB_MASTER_FILES UINT64TYPE CONSTRAINT FILE_STATISTICS_NMF_NN NOT NULL,
+  MASTER_DATA_IN_BYTES UINT64TYPE CONSTRAINT FILE_STATISTICS_MDIB_NN NOT NULL,
+  NB_COPY_NB_1 UINT64TYPE CONSTRAINT FILE_STATISTICS_NCN1_NN NOT NULL,
+  NB_COPY_NB_1_IN_BYTES UINT64TYPE CONSTRAINT FILE_STATISTICS_CN1IB_NN NOT NULL,
+  NB_COPY_NB_GT_1 UINT64TYPE CONSTRAINT FILE_STATISTICS_NCNG1_NN NOT NULL,
+  NB_COPY_NB_GT_1_IN_BYTES UINT64TYPE CONSTRAINT FILE_STATISTICS_NCNG1IB_NN NOT NULL,
+  UPDATE_TIME UINT64TYPE CONSTRAINT FILE_STATISTICS_UT_NN NOT NULL,
+  CONSTRAINT FILE_STATISTICS_UT_PK PRIMARY KEY(UPDATE_TIME)
 );
\ No newline at end of file