diff --git a/statistics/DatabaseStatisticsService.cpp b/statistics/DatabaseStatisticsService.cpp index fe548cd03273c7e6f494fabe99cf6a61b417283c..bcdf500f99fe4cd2ba4c03b1c77676ed8b1a25a8 100644 --- a/statistics/DatabaseStatisticsService.cpp +++ b/statistics/DatabaseStatisticsService.cpp @@ -27,7 +27,7 @@ DatabaseStatisticsService::DatabaseStatisticsService(cta::rdbms::Conn & database DatabaseStatisticsService::~DatabaseStatisticsService() { } -void DatabaseStatisticsService::update(){ +void DatabaseStatisticsService::updateStatistics(){ const char * const sql = "UPDATE TAPE TAPE_TO_UPDATE SET" "(" @@ -92,11 +92,101 @@ void DatabaseStatisticsService::update(){ } } -void DatabaseStatisticsService::save(const cta::statistics::Statistics& statistics) { +void DatabaseStatisticsService::saveStatistics(const cta::statistics::Statistics& statistics) { + //First we save the general FILE statistics, then we go for the per-vo statistics + saveFileStatistics(statistics); + saveStatisticsPerVo(statistics); +} +void DatabaseStatisticsService::saveFileStatistics(const cta::statistics::Statistics& statistics) { + try { + const time_t now = time(nullptr); + const char * const sql = + "INSERT INTO " + "FILE_STATISTICS " + "(" + "NB_MASTER_FILES," + "MASTER_DATA_IN_BYTES," + "NB_COPY_NB_1," + "NB_COPY_NB_1_IN_BYTES," + "NB_COPY_NB_GT_1," + "NB_COPY_NB_GT_1_IN_BYTES," + "UPDATE_TIME" + ") " + "VALUES " + "(" + ":NB_MASTER_FILES," + ":MASTER_DATA_IN_BYTES," + ":NB_COPY_NB_1," + ":NB_COPY_NB_1_IN_BYTES," + ":NB_COPY_NB_GT_1," + ":NB_COPY_NB_GT_1_IN_BYTES," + ":UPDATE_TIME" + ")"; + auto stmt = m_conn.createStmt(sql); + stmt.bindUint64(":NB_MASTER_FILES",statistics.getTotalFiles()); + stmt.bindUint64(":MASTER_DATA_IN_BYTES",statistics.getTotalBytes()); + stmt.bindUint64(":NB_COPY_NB_1",statistics.getTotalFilesCopyNb1()); + stmt.bindUint64(":NB_COPY_NB_1_IN_BYTES",statistics.getTotalBytesCopyNb1()); + stmt.bindUint64(":NB_COPY_NB_GT_1",statistics.getTotalFilesCopyNbGt1()); + stmt.bindUint64(":NB_COPY_NB_GT_1_IN_BYTES",statistics.getTotalBytesCopyNbGt1()); + stmt.bindUint64(":UPDATE_TIME",now); + stmt.executeNonQuery(); + } catch(exception::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); + throw; + } } -std::unique_ptr<cta::statistics::Statistics> DatabaseStatisticsService::get() { +void DatabaseStatisticsService::saveStatisticsPerVo(const cta::statistics::Statistics& statistics) { + try { + const time_t now = time(nullptr); + const char * const sql = + "INSERT INTO " + "VO_STATISTICS " + "(" + "VO," + "NB_MASTER_FILES," + "MASTER_DATA_IN_BYTES," + "NB_COPY_NB_1," + "NB_COPY_NB_1_IN_BYTES," + "NB_COPY_NB_GT_1," + "NB_COPY_NB_GT_1_IN_BYTES," + "UPDATE_TIME" + ") " + "VALUES " + "(" + ":VO," + ":NB_MASTER_FILES," + ":MASTER_DATA_IN_BYTES," + ":NB_COPY_NB_1," + ":NB_COPY_NB_1_IN_BYTES," + ":NB_COPY_NB_GT_1," + ":NB_COPY_NB_GT_1_IN_BYTES," + ":UPDATE_TIME" + ")"; + for(const auto & stat: statistics.getAllStatistics()){ + auto stmt = m_conn.createStmt(sql); + auto voFileStatistics = stat.second; + stmt.bindString(":VO",stat.first); + stmt.bindUint64(":NB_MASTER_FILES",voFileStatistics.nbMasterFiles); + stmt.bindUint64(":MASTER_DATA_IN_BYTES",voFileStatistics.nbMasterFiles); + stmt.bindUint64(":NB_COPY_NB_1",voFileStatistics.nbCopyNb1); + stmt.bindUint64(":NB_COPY_NB_1_IN_BYTES",voFileStatistics.copyNb1InBytes); + stmt.bindUint64(":NB_COPY_NB_GT_1",voFileStatistics.nbCopyNbGt1); + stmt.bindUint64(":NB_COPY_NB_GT_1_IN_BYTES",voFileStatistics.copyNbGt1InBytes); + stmt.bindUint64(":UPDATE_TIME",now); + stmt.executeNonQuery(); + } + } catch(exception::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); + throw; + } +} + + + +std::unique_ptr<cta::statistics::Statistics> DatabaseStatisticsService::getStatistics() { const char * const sql = "SELECT " "VIRTUAL_ORGANIZATION_NAME AS VO," diff --git a/statistics/DatabaseStatisticsService.hpp b/statistics/DatabaseStatisticsService.hpp index 5483c2249026894f05453d38df65cd1815c34b3c..91891a6e9b380e3373a7c29cdd9585f66cf992d9 100644 --- a/statistics/DatabaseStatisticsService.hpp +++ b/statistics/DatabaseStatisticsService.hpp @@ -32,11 +32,13 @@ public: DatabaseStatisticsService & operator=(const DatabaseStatisticsService &) = delete; virtual ~DatabaseStatisticsService(); - virtual void update() override; - virtual void save(const cta::statistics::Statistics &statistics) override; - virtual std::unique_ptr<cta::statistics::Statistics> get() override; + virtual void updateStatistics() override; + virtual void saveStatistics(const cta::statistics::Statistics &statistics) override; + virtual std::unique_ptr<cta::statistics::Statistics> getStatistics() override; protected: cta::rdbms::Conn & m_conn; + virtual void saveFileStatistics(const cta::statistics::Statistics & statistics); + virtual void saveStatisticsPerVo(const cta::statistics::Statistics & statistics); }; }} diff --git a/statistics/FileStatistics.cpp b/statistics/FileStatistics.cpp index bae8e40a2b404bde15c62eec11c72e06f1198507..7c3c6a6b486f117085e141876d377d42f2e2b1cc 100644 --- a/statistics/FileStatistics.cpp +++ b/statistics/FileStatistics.cpp @@ -25,6 +25,15 @@ namespace statistics { FileStatistics::FileStatistics() { } +FileStatistics::FileStatistics(const FileStatistics &other){ + nbMasterFiles = other.nbMasterFiles; + masterDataInBytes = other.masterDataInBytes; + nbCopyNb1 = other.nbCopyNb1; + copyNb1InBytes = other.copyNb1InBytes; + nbCopyNbGt1 = other.nbCopyNbGt1; + copyNbGt1InBytes = other.copyNbGt1InBytes; +} + FileStatistics FileStatistics::operator +=(const FileStatistics& other) { nbMasterFiles += other.nbMasterFiles; masterDataInBytes += other.masterDataInBytes; @@ -35,5 +44,15 @@ FileStatistics FileStatistics::operator +=(const FileStatistics& other) { return *this; } +FileStatistics& FileStatistics::operator =(const FileStatistics& other){ + nbMasterFiles = other.nbMasterFiles; + masterDataInBytes = other.masterDataInBytes; + nbCopyNb1 = other.nbCopyNb1; + copyNb1InBytes = other.copyNb1InBytes; + nbCopyNbGt1 = other.nbCopyNbGt1; + copyNbGt1InBytes = other.copyNbGt1InBytes; + return *this; +} + }} diff --git a/statistics/FileStatistics.hpp b/statistics/FileStatistics.hpp index cb5990b7076b1cfd8acb43ce1c612e70c9a81579..8ee55a008d74a69f8330f3b899b363eeb1959478 100644 --- a/statistics/FileStatistics.hpp +++ b/statistics/FileStatistics.hpp @@ -26,7 +26,7 @@ namespace statistics { struct FileStatistics { FileStatistics(); - + FileStatistics(const FileStatistics &other); uint64_t nbMasterFiles = 0; uint64_t masterDataInBytes = 0; uint64_t nbCopyNb1 = 0; @@ -34,6 +34,8 @@ struct FileStatistics { uint64_t nbCopyNbGt1 = 0; uint64_t copyNbGt1InBytes = 0; + FileStatistics &operator=(const FileStatistics &other); + FileStatistics operator +=(const FileStatistics &other); }; diff --git a/statistics/MySQLStatisticsService.cpp b/statistics/MySQLStatisticsService.cpp index 854b44992d14b97b8a45a957b9c02a59d361a15b..285760232b3b0a625d80312a8ce9e6a3a07f73f9 100644 --- a/statistics/MySQLStatisticsService.cpp +++ b/statistics/MySQLStatisticsService.cpp @@ -25,7 +25,7 @@ MySQLStatisticsService::MySQLStatisticsService(cta::rdbms::Conn & databaseConnec MySQLStatisticsService::~MySQLStatisticsService() { } -void MySQLStatisticsService::update() { +void MySQLStatisticsService::updateStatistics() { throw cta::exception::Exception("In MySQLStatisticsService::update() cannot update tape statistics because it is not implemented for MySQL databases"); } diff --git a/statistics/MySQLStatisticsService.hpp b/statistics/MySQLStatisticsService.hpp index b7e621437e522a1ecc0be35a951c80c8f8fc776a..680d04c109c8c08d47bad633e55bbd14fefd8420 100644 --- a/statistics/MySQLStatisticsService.hpp +++ b/statistics/MySQLStatisticsService.hpp @@ -29,7 +29,7 @@ public: MySQLStatisticsService & operator=(const MySQLStatisticsService &) = delete; virtual ~MySQLStatisticsService(); - void update() override; + void updateStatistics() override; private: }; diff --git a/statistics/Statistics.cpp b/statistics/Statistics.cpp index 76c40931401bc1ba80a1e22078e3b455d1c21a87..5427ecc2443f26b690e1509fce60185eab1d04b0 100644 --- a/statistics/Statistics.cpp +++ b/statistics/Statistics.cpp @@ -18,6 +18,8 @@ #include "Statistics.hpp" #include "common/make_unique.hpp" +#include <iostream> + namespace cta { namespace statistics { @@ -29,6 +31,10 @@ Statistics::Statistics(const Statistics& other) { m_statisticsPerVo = other.m_statisticsPerVo; m_totalFiles = other.m_totalFiles; m_totalBytes = other.m_totalBytes; + m_totalFilesCopyNb1 = other.m_totalFilesCopyNb1; + m_totalBytesCopyNb1 = other.m_totalBytesCopyNb1; + m_totalFilesCopyNbGt1 = other.m_totalFilesCopyNbGt1; + m_totalBytesCopyNbGt1 = other.m_totalBytesCopyNbGt1; } } @@ -37,6 +43,10 @@ Statistics& Statistics::operator=(const Statistics& other) { m_statisticsPerVo = other.m_statisticsPerVo; m_totalFiles = other.m_totalFiles; m_totalBytes = other.m_totalBytes; + m_totalFilesCopyNb1 = other.m_totalFilesCopyNb1; + m_totalBytesCopyNb1 = other.m_totalBytesCopyNb1; + m_totalFilesCopyNbGt1 = other.m_totalFilesCopyNbGt1; + m_totalBytesCopyNbGt1 = other.m_totalBytesCopyNbGt1; } return *this; } @@ -45,6 +55,10 @@ void Statistics::insertStatistics(const std::string& vo, const FileStatistics& f m_statisticsPerVo[vo] = fileStatistics; m_totalFiles += fileStatistics.nbMasterFiles; m_totalBytes += fileStatistics.masterDataInBytes; + m_totalFilesCopyNb1 += fileStatistics.nbCopyNb1; + m_totalBytesCopyNb1 += fileStatistics.copyNb1InBytes; + m_totalFilesCopyNbGt1 += fileStatistics.nbCopyNbGt1; + m_totalBytesCopyNbGt1 += fileStatistics.copyNbGt1InBytes; } const Statistics::StatisticsPerVo& Statistics::getAllStatistics() const { @@ -59,6 +73,22 @@ uint64_t Statistics::getTotalBytes() const { return m_totalBytes; } +uint64_t Statistics::getTotalFilesCopyNb1() const { + return m_totalFilesCopyNb1; +} + +uint64_t Statistics::getTotalBytesCopyNb1() const { + return m_totalBytesCopyNb1; +} + +uint64_t Statistics::getTotalFilesCopyNbGt1() const { + return m_totalFilesCopyNbGt1; +} + +uint64_t Statistics::getTotalBytesCopyNbGt1() const { + return m_totalBytesCopyNbGt1; +} + Statistics::Builder::Builder(cta::rdbms::Rset & rset):m_rset(rset) {} std::unique_ptr<Statistics> Statistics::Builder::build(){ @@ -91,10 +121,14 @@ std::ostream & operator <<(std::ostream& stream, Statistics stats) { stream << "\"copyNbGt1InBytes\": " << stat.second.copyNbGt1InBytes; stream << "},"; } - stream << "],"; - stream << "\"totalFiles\": " << stats.getTotalFiles() << ","; - stream << "\"totalBytes\": " << stats.getTotalBytes(); - stream << "}"; + stream << "]," + << "\"totalFiles\": " << stats.getTotalFiles() << "," + << "\"totalBytes\": " << stats.getTotalBytes() << "," + << "\"totalFilesCopyNb1\": " << stats.getTotalFilesCopyNb1() << "," + << "\"totalBytesCopyNb1\": " << stats.getTotalBytesCopyNb1() << "," + << "\"totalFilesCopyNbGt1\": " << stats.getTotalFilesCopyNbGt1() << "," + << "\"totalBytesCopyNbGt1\": " << stats.getTotalBytesCopyNbGt1() << "," + << "}"; return stream; } diff --git a/statistics/Statistics.hpp b/statistics/Statistics.hpp index e3e1732496a1918355e1f639aab4ac853e8fffeb..4600ad16fc7ca2c4b543083464c4b182333913dc 100644 --- a/statistics/Statistics.hpp +++ b/statistics/Statistics.hpp @@ -41,6 +41,10 @@ public: const StatisticsPerVo &getAllStatistics() const; uint64_t getTotalFiles() const; uint64_t getTotalBytes() const; + uint64_t getTotalFilesCopyNb1() const; + uint64_t getTotalBytesCopyNb1() const; + uint64_t getTotalFilesCopyNbGt1() const; + uint64_t getTotalBytesCopyNbGt1() const; /** * This builder class allows to build the statistics @@ -61,6 +65,10 @@ private: StatisticsPerVo m_statisticsPerVo; uint64_t m_totalFiles = 0; uint64_t m_totalBytes = 0; + uint64_t m_totalFilesCopyNb1 = 0; + uint64_t m_totalBytesCopyNb1 = 0; + uint64_t m_totalFilesCopyNbGt1 = 0; + uint64_t m_totalBytesCopyNbGt1 = 0; }; std::ostream &operator <<(std::ostream &stream, Statistics stats); diff --git a/statistics/StatisticsSaveCmd.cpp b/statistics/StatisticsSaveCmd.cpp index 3af7dbef9ff32846ed3f44ecfae1e43ba98c9f16..fa13fade75ccc5d8e2380ff82551bf969dc80385 100644 --- a/statistics/StatisticsSaveCmd.cpp +++ b/statistics/StatisticsSaveCmd.cpp @@ -23,6 +23,7 @@ #include "StatisticsSchemaFactory.hpp" #include "StatisticsService.hpp" #include "StatisticsServiceFactory.hpp" +#include "common/utils/utils.hpp" #include <algorithm> @@ -110,13 +111,24 @@ int StatisticsSaveCmd::exceptionThrowingMain(const int argc, char *const *const statisticsCheckerBuilder.useCppSchemaStatementsReader(*statisticsSchema) .useSQLiteSchemaComparer() .build(); - statisticsChecker->compareTablesLocatedInSchema(); + SchemaChecker::Status compareTablesStatus = statisticsChecker->compareTablesLocatedInSchema(); + if(compareTablesStatus == SchemaChecker::Status::FAILURE){ + return EXIT_FAILURE; + } //Compute the statistics - std::unique_ptr<StatisticsService> service = StatisticsServiceFactory::create(catalogueConn,loginCatalogue.dbType); - std::unique_ptr<Statistics> statistics = service->get(); - std::cout << *statistics << std::endl; + std::unique_ptr<StatisticsService> catalogueStatisticsService = StatisticsServiceFactory::create(catalogueConn,loginCatalogue.dbType); + std::unique_ptr<Statistics> statistics = catalogueStatisticsService->getStatistics(); //Insert them into the statistics database + //std::cout<<*statistics<<std::endl; + try { + std::unique_ptr<StatisticsService> statisticsStatisticsService = StatisticsServiceFactory::create(statisticsConn,loginStatistics.dbType); + statisticsStatisticsService->saveStatistics(*statistics); + } catch (cta::exception::Exception &ex) { + std::cerr << ex.getMessageValue() << std::endl; + std::cerr << "StatisticsJson:" << *statistics << std::endl; + return EXIT_FAILURE; + } return EXIT_SUCCESS; } @@ -149,7 +161,24 @@ void StatisticsSaveCmd::verifyCmdLineArgs(const StatisticsSaveCmdLineArgs& cmdLi } void StatisticsSaveCmd::buildStatisticsDatabase(cta::rdbms::Conn& statisticsDatabaseConn, const StatisticsSchema& statisticsSchema) { - statisticsDatabaseConn.executeNonQuery(statisticsSchema.sql); + try { + std::string::size_type searchPos = 0; + std::string::size_type findResult = std::string::npos; + std::string sqlStmts = statisticsSchema.sql; + + while(std::string::npos != (findResult = sqlStmts.find(';', searchPos))) { + // Calculate the length of the current statement without the trailing ';' + const std::string::size_type stmtLen = findResult - searchPos; + const std::string sqlStmt = utils::trimString(sqlStmts.substr(searchPos, stmtLen)); + searchPos = findResult + 1; + + if(0 < sqlStmt.size()) { // Ignore empty statements + statisticsDatabaseConn.executeNonQuery(sqlStmt); + } + } + } catch(exception::Exception &ex) { + throw exception::Exception(std::string(__FUNCTION__) + " failed: " + ex.getMessage().str()); + } } bool StatisticsSaveCmd::userConfirmDropStatisticsSchemaFromDb(const rdbms::Login &dbLogin) { diff --git a/statistics/StatisticsService.hpp b/statistics/StatisticsService.hpp index 1287936f3305d50a2bd1073b80bd7df1f6bbf9a0..616d0dc372b2d4c8e79c534d19942ef9657d2073 100644 --- a/statistics/StatisticsService.hpp +++ b/statistics/StatisticsService.hpp @@ -30,9 +30,9 @@ public: StatisticsService & operator=(const StatisticsService &) = delete; virtual ~StatisticsService(); - virtual void update() = 0; - virtual void save(const cta::statistics::Statistics &statistics) = 0; - virtual std::unique_ptr<cta::statistics::Statistics> get() = 0; + virtual void updateStatistics() = 0; + virtual void saveStatistics(const cta::statistics::Statistics &statistics) = 0; + virtual std::unique_ptr<cta::statistics::Statistics> getStatistics() = 0; uint64_t getNbUpdatedTapes(); protected: diff --git a/statistics/StatisticsUpdateCmd.cpp b/statistics/StatisticsUpdateCmd.cpp index 39293bf55538ad605e89980452a9e97931deb8f6..9ee095c167f7dd4220b5ff5eb76e0eade96e0b25 100644 --- a/statistics/StatisticsUpdateCmd.cpp +++ b/statistics/StatisticsUpdateCmd.cpp @@ -75,7 +75,7 @@ int StatisticsUpdateCmd::exceptionThrowingMain(const int argc, char *const *cons std::unique_ptr<StatisticsService> service = StatisticsServiceFactory::create(catalogueConn,loginCatalogue.dbType); std::cout<<"Updating tape statistics in the catalogue..."<<std::endl; cta::utils::Timer t; - service->update(); + service->updateStatistics(); std::cout<<"Updated catalogue tape statistics in "<<t.secs()<<", "<<service->getNbUpdatedTapes()<<" tape(s) have been updated"<<std::endl; return EXIT_SUCCESS; diff --git a/statistics/common_statistics_schema.sql b/statistics/common_statistics_schema.sql index deecea0b3a1973921816c3125905ec638b880fa8..465a681d18e78064ea4f8cb4021762d51ce9f392 100644 --- a/statistics/common_statistics_schema.sql +++ b/statistics/common_statistics_schema.sql @@ -1,8 +1,21 @@ -CREATE TABLE CTA_STATISTICS( - GID UINT32TYPE DEFAULT 0 CONSTRAINT CTA_STATISTICS_GID_NN NOT NULL, - TIMESTAMP UINT64TYPE DEFAULT 0 CONSTRAINT CTA_STATISTICS_TS_NN NOT NULL, - NB_MASTER_FILES UINT64TYPE NOT NULL, - MASTER_DATA_BYTES UINT64TYPE NOT NULL, - UPDATE_TIME UINT64TYPE NOT NULL, - CONSTRAINT CTA_STATISTICS_GID_TS_PK PRIMARY KEY (GID, TIMESTAMP) +CREATE TABLE VO_STATISTICS ( + VO VARCHAR(100) CONSTRAINT VO_STATISTICS_VO_NN NOT NULL, + NB_MASTER_FILES UINT64TYPE CONSTRAINT VO_STATISTICS_NMF_NN NOT NULL, + MASTER_DATA_IN_BYTES UINT64TYPE CONSTRAINT VO_STATISTICS_MDIB_NN NOT NULL, + NB_COPY_NB_1 UINT64TYPE CONSTRAINT VO_STATISTICS_NCN1_NN NOT NULL, + NB_COPY_NB_1_IN_BYTES UINT64TYPE CONSTRAINT VO_STATISTICS_CN1IB_NN NOT NULL, + NB_COPY_NB_GT_1 UINT64TYPE CONSTRAINT VO_STATISTICS_NCNG1_NN NOT NULL, + NB_COPY_NB_GT_1_IN_BYTES UINT64TYPE CONSTRAINT VO_STATISTICS_NCNG1IB_NN NOT NULL, + UPDATE_TIME UINT64TYPE CONSTRAINT VO_STATISTICS_UT_NN NOT NULL, + CONSTRAINT VO_STATISTICS_VU_PK PRIMARY KEY(VO, UPDATE_TIME) +); +CREATE TABLE FILE_STATISTICS ( + NB_MASTER_FILES UINT64TYPE CONSTRAINT FILE_STATISTICS_NMF_NN NOT NULL, + MASTER_DATA_IN_BYTES UINT64TYPE CONSTRAINT FILE_STATISTICS_MDIB_NN NOT NULL, + NB_COPY_NB_1 UINT64TYPE CONSTRAINT FILE_STATISTICS_NCN1_NN NOT NULL, + NB_COPY_NB_1_IN_BYTES UINT64TYPE CONSTRAINT FILE_STATISTICS_CN1IB_NN NOT NULL, + NB_COPY_NB_GT_1 UINT64TYPE CONSTRAINT FILE_STATISTICS_NCNG1_NN NOT NULL, + NB_COPY_NB_GT_1_IN_BYTES UINT64TYPE CONSTRAINT FILE_STATISTICS_NCNG1IB_NN NOT NULL, + UPDATE_TIME UINT64TYPE CONSTRAINT FILE_STATISTICS_UT_NN NOT NULL, + CONSTRAINT FILE_STATISTICS_UT_PK PRIMARY KEY(UPDATE_TIME) ); \ No newline at end of file