From 368e1451139fe71397412f9c22a86db4ee0d4071 Mon Sep 17 00:00:00 2001
From: Cedric Caffy <cedric.caffy@cern.ch>
Date: Wed, 25 Mar 2020 11:03:52 +0100
Subject: [PATCH] cta-statistics-save now display the json output after
inserting them in the CTA statistics database
---
catalogue/SchemaChecker.cpp | 4 +-
catalogue/SchemaComparerResult.cpp | 2 +-
statistics/StatisticsSaveCmd.cpp | 105 +++++++++++++++---------
statistics/StatisticsSaveCmd.hpp | 14 ++++
statistics/StatisticsServiceFactory.cpp | 11 +++
5 files changed, 92 insertions(+), 44 deletions(-)
diff --git a/catalogue/SchemaChecker.cpp b/catalogue/SchemaChecker.cpp
index 8c95687963..d671da69d9 100644
--- a/catalogue/SchemaChecker.cpp
+++ b/catalogue/SchemaChecker.cpp
@@ -89,14 +89,14 @@ SchemaChecker::Status SchemaChecker::checkTableContainsColumns(const std::string
std::map<std::string, std::string> mapColumnsTypes = m_databaseMetadataGetter->getColumns(tableName);
SchemaChecker::Status status = SchemaChecker::Status::OK;
if(mapColumnsTypes.empty()){
- std::cout << "TABLE " << tableName << " does not exist." << std::endl;
+ std::cerr << "TABLE " << tableName << " does not exist." << std::endl;
return SchemaChecker::Status::FAILURE;
}
for(auto &columnName: columnNames){
try{
mapColumnsTypes.at(columnName);
} catch(std::out_of_range &){
- std::cout << "TABLE " << tableName << " does not contain the column " << columnName << "."<< std::endl;
+ std::cerr << "TABLE " << tableName << " does not contain the column " << columnName << "."<< std::endl;
status = SchemaChecker::Status::FAILURE;
}
}
diff --git a/catalogue/SchemaComparerResult.cpp b/catalogue/SchemaComparerResult.cpp
index 1db07c4251..d90165ccb5 100644
--- a/catalogue/SchemaComparerResult.cpp
+++ b/catalogue/SchemaComparerResult.cpp
@@ -62,7 +62,7 @@ SchemaComparerResult::Status SchemaComparerResult::getStatus() const {
void SchemaComparerResult::printDiffs() const {
for(auto &diff: m_diffs){
- std::cout << " ERROR: " << diff << std::endl;
+ std::cerr << " ERROR: " << diff << std::endl;
}
}
diff --git a/statistics/StatisticsSaveCmd.cpp b/statistics/StatisticsSaveCmd.cpp
index bb939a17b1..7a7e0c25a9 100644
--- a/statistics/StatisticsSaveCmd.cpp
+++ b/statistics/StatisticsSaveCmd.cpp
@@ -61,12 +61,13 @@ int StatisticsSaveCmd::exceptionThrowingMain(const int argc, char *const *const
const uint64_t maxNbConns = 1;
- std::unique_ptr<StatisticsService> statisticsStatisticsService;
+ std::unique_ptr<StatisticsService> statisticsDbService;
+ std::unique_ptr<StatisticsService> statisticsStdoutJsonService = StatisticsServiceFactory::create(std::cout);
std::unique_ptr<rdbms::ConnPool> statisticsConnPool;
std::unique_ptr<rdbms::Conn> statisticsConn;
- std::unique_ptr<Statistics> statistics;
+ std::unique_ptr<Statistics> computedStatistics = nullptr;
if(!cmdLineArgs.buildDatabase && !cmdLineArgs.dropDatabase){
//Compute the statistics from the Catalogue
@@ -75,34 +76,14 @@ int StatisticsSaveCmd::exceptionThrowingMain(const int argc, char *const *const
rdbms::ConnPool catalogueConnPool(loginCatalogue, maxNbConns);
auto catalogueConn = catalogueConnPool.getConn();
- //Check that the catalogue contains the table TAPE with the columns we need
- SchemaChecker::Builder catalogueCheckerBuilder("catalogue",loginCatalogue.dbType,catalogueConn);
- std::unique_ptr<cta::catalogue::SchemaChecker> catalogueChecker = catalogueCheckerBuilder.build();
-
- SchemaChecker::Status tapeTableStatus =
- catalogueChecker->checkTableContainsColumns("TAPE",{
- "NB_MASTER_FILES",
- "MASTER_DATA_IN_BYTES",
- "NB_COPY_NB_1",
- "COPY_NB_1_IN_BYTES",
- "NB_COPY_NB_GT_1",
- "COPY_NB_GT_1_IN_BYTES",
- "DIRTY"
- }
- );
- SchemaChecker::Status voTableStatus =
- catalogueChecker->checkTableContainsColumns("TAPE_POOL",{"VIRTUAL_ORGANIZATION_ID"});
-
- if(tapeTableStatus == SchemaChecker::Status::FAILURE || voTableStatus == SchemaChecker::Status::FAILURE ){
- return EXIT_FAILURE;
- }
-
+ checkCatalogueSchema(catalogueConn,loginCatalogue.dbType);
+
//Compute the statistics
std::unique_ptr<StatisticsService> catalogueStatisticsService = StatisticsServiceFactory::create(catalogueConn,loginCatalogue.dbType);
- statistics = catalogueStatisticsService->getStatistics();
+ computedStatistics = catalogueStatisticsService->getStatistics();
}
try {
- if(!cmdLineArgs.statisticsDbConfigPath.empty()){
+ if(!isJsonOutput){
auto loginStatistics = rdbms::Login::parseFile(cmdLineArgs.statisticsDbConfigPath);
statisticsConnPool = cta::make_unique<rdbms::ConnPool>(loginStatistics, maxNbConns);
@@ -126,33 +107,32 @@ int StatisticsSaveCmd::exceptionThrowingMain(const int argc, char *const *const
}
//Here, we want to store the statistics in the Statistics database
+ checkStatisticsSchema(*statisticsConn,loginStatistics.dbType);
- //Check that the statistics schema tables are in the statistics database
- SchemaChecker::Builder statisticsCheckerBuilder("statistics",loginStatistics.dbType,*statisticsConn);
- std::unique_ptr<SchemaChecker> statisticsChecker =
- statisticsCheckerBuilder.useCppSchemaStatementsReader(*statisticsSchema)
- .useSQLiteSchemaComparer()
- .build();
- SchemaChecker::Status compareTablesStatus = statisticsChecker->compareTablesLocatedInSchema();
- if(compareTablesStatus == SchemaChecker::Status::FAILURE){
- throw cta::exception::Exception("Schema checking failure.");
- }
//Create the statistics service with the statistics database connection
- statisticsStatisticsService = StatisticsServiceFactory::create(*statisticsConn,loginStatistics.dbType);
+ statisticsDbService = StatisticsServiceFactory::create(*statisticsConn,loginStatistics.dbType);
} else if (isJsonOutput){
//Create the statistics service with the JSON output connection
- statisticsStatisticsService = StatisticsServiceFactory::create(std::cout);
+ statisticsDbService = StatisticsServiceFactory::create(std::cout);
} else {
//We should normally not be there
throw cta::exception::Exception("No --json option provided and no statistics database configuration file provided.");
}
//Insert the statistics to the statistics database
- statisticsStatisticsService->saveStatistics(*statistics);
+ if(!isJsonOutput){
+ statisticsDbService->saveStatistics(*computedStatistics);
+ }
+ //In any case, output the statistics in json format
+ statisticsStdoutJsonService->saveStatistics(*computedStatistics);
} catch (cta::exception::Exception &ex) {
std::cerr << ex.getMessageValue() << std::endl;
- if(statistics != nullptr)
- std::cerr << "StatisticsJson:" << *statistics << std::endl;
+ if(computedStatistics != nullptr){
+ std::unique_ptr<StatisticsService> statisticsStderrJsonService = StatisticsServiceFactory::create(std::cerr);
+ std::cerr << "StatisticsJson:" << std::endl;
+ statisticsStderrJsonService->saveStatistics(*computedStatistics);
+ std::cerr << std::endl;
+ }
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
@@ -243,5 +223,48 @@ void StatisticsSaveCmd::dropStatisticsDatabase(cta::rdbms::Conn& statisticsDatab
}
}
+void StatisticsSaveCmd::checkCatalogueSchema(cta::rdbms::Conn& catalogueConn, cta::rdbms::Login::DbType dbType) {
+ using namespace cta::catalogue;
+ //Check that the catalogue contains the table TAPE with the columns we need
+ SchemaChecker::Builder catalogueCheckerBuilder("catalogue",dbType,catalogueConn);
+ std::unique_ptr<cta::catalogue::SchemaChecker> catalogueChecker = catalogueCheckerBuilder.build();
+
+ SchemaChecker::Status tapeTableStatus =
+ catalogueChecker->checkTableContainsColumns("TAPE",{
+ "NB_MASTER_FILES",
+ "MASTER_DATA_IN_BYTES",
+ "NB_COPY_NB_1",
+ "COPY_NB_1_IN_BYTES",
+ "NB_COPY_NB_GT_1",
+ "COPY_NB_GT_1_IN_BYTES",
+ "DIRTY"
+ }
+ );
+ SchemaChecker::Status voTableStatus =
+ catalogueChecker->checkTableContainsColumns("TAPE_POOL",{"VIRTUAL_ORGANIZATION_ID"});
+
+ if(tapeTableStatus == SchemaChecker::Status::FAILURE || voTableStatus == SchemaChecker::Status::FAILURE ){
+ throw cta::exception::Exception("Catalogue schema checking failed.");
+ }
+}
+
+
+void StatisticsSaveCmd::checkStatisticsSchema(cta::rdbms::Conn& statisticsDatabaseConn, cta::rdbms::Login::DbType dbType) {
+ using namespace cta::catalogue;
+ auto statisticsSchema = StatisticsSchemaFactory::create(dbType);
+ //Check that the statistics schema tables are in the statistics database
+ SchemaChecker::Builder statisticsCheckerBuilder("statistics",dbType,statisticsDatabaseConn);
+ std::unique_ptr<SchemaChecker> statisticsChecker =
+ statisticsCheckerBuilder.useCppSchemaStatementsReader(*statisticsSchema)
+ .useSQLiteSchemaComparer()
+ .build();
+ SchemaChecker::Status compareTablesStatus = statisticsChecker->compareTablesLocatedInSchema();
+ if(compareTablesStatus == SchemaChecker::Status::FAILURE){
+ throw cta::exception::Exception("Statistics schema checking failed.");
+ }
+}
+
+
+
} // namespace statistics
} // namespace cta
diff --git a/statistics/StatisticsSaveCmd.hpp b/statistics/StatisticsSaveCmd.hpp
index 6d6d8d2459..acc92a8573 100644
--- a/statistics/StatisticsSaveCmd.hpp
+++ b/statistics/StatisticsSaveCmd.hpp
@@ -94,6 +94,20 @@ private:
*/
void dropStatisticsDatabase(cta::rdbms::Conn &statisticsDatabaseConn, const StatisticsSchema & statisticsSchema);
+ /**
+ * Check the content of the Catalogue database regarding what is needed to compute the statistics
+ * @param catalogueDatabaseConn the connection to the Catalogue database
+ * @param dbType the dbType of the Catalogue database
+ */
+ void checkCatalogueSchema(cta::rdbms::Conn &catalogueDatabaseConn, cta::rdbms::Login::DbType dbType);
+
+ /**
+ * Checks that the tables needed for statistics are present in the statistics database
+ * @param statisticsDatabaseConn the connection to the statistics database
+ * @param dbType the dbType of the statistics database
+ */
+ void checkStatisticsSchema(cta::rdbms::Conn &statisticsDatabaseConn,cta::rdbms::Login::DbType dbType);
+
}; // class VerifySchemaCmd
} // namespace statistics
diff --git a/statistics/StatisticsServiceFactory.cpp b/statistics/StatisticsServiceFactory.cpp
index 2c6bb4ccd8..d7706f716a 100644
--- a/statistics/StatisticsServiceFactory.cpp
+++ b/statistics/StatisticsServiceFactory.cpp
@@ -23,10 +23,21 @@
namespace cta { namespace statistics {
+/**
+ * Creates a database StatisticsService corresponding to the dbType passed in parameter
+ * @param connection the connection to the database
+ * @param dbType the databaseType of the database
+ * @return the DatabaseStatisticsService corresponding to the dbType passed in parameter
+ */
std::unique_ptr<StatisticsService> StatisticsServiceFactory::create(cta::rdbms::Conn& connection, cta::rdbms::Login::DbType dbType) {
return DatabaseStatisticsServiceFactory::create(connection,dbType);
}
+/**
+ * Returns a JsonStatisticsService that will output in the ostream passed in parameter
+ * @param ostream the stream where the json output will inserted
+ * @return the JsonStatisticsService
+ */
std::unique_ptr<StatisticsService> StatisticsServiceFactory::create(std::ostream& ostream) {
return JsonStatisticsServiceFactory::create(&ostream);
}
--
GitLab